diff --git a/.github/workflows/mobile-eas-preview-fork.yml b/.github/workflows/mobile-eas-preview-fork.yml new file mode 100644 index 0000000000000..e7615d46011bd --- /dev/null +++ b/.github/workflows/mobile-eas-preview-fork.yml @@ -0,0 +1,145 @@ +name: Mobile EAS Preview (Fork) + +# EAS previews for fork PRs. This runs the PR's code with an Expo token: the +# app config and plugins run on this runner, and native builds run on EAS +# with preview signing credentials. The label is the maintainer's sign-off +# that they reviewed that code, so it covers exactly the commit it was applied +# to: this pins that commit and consumes the label before running anything. A +# later push deploys nothing until the label is applied again. +# +# Same-repo PRs deploy on every push while labeled, via mobile-eas-preview.yml. + +on: + pull_request_target: + types: [labeled] + +permissions: + contents: read + +jobs: + preview: + name: EAS Preview + if: >- + github.event.pull_request.head.repo.full_name != github.repository && + github.event.label.name == '🚀 Mobile Continuous Deployment' + runs-on: blacksmith-8vcpu-ubuntu-2404 + concurrency: + group: mobile-eas-preview-fork-${{ github.event.pull_request.number }} + cancel-in-progress: true + permissions: + contents: read + pull-requests: write + env: + APP_VARIANT: preview + NODE_OPTIONS: --max-old-space-size=8192 + MOBILE_VERSION_POLICY: fingerprint + steps: + # If a maintainer removed the label first, that removal wins. + - id: consume + name: Consume the deployment label + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + name: context.payload.label.name, + }); + core.setOutput("consumed", "true"); + } catch (error) { + if (error.status !== 404) throw error; + core.info("The label was removed before this run could consume it. Skipping."); + core.setOutput("consumed", "false"); + } + + - id: expo-token + name: Check for EXPO_PREVIEW_TOKEN + if: steps.consume.outputs.consumed == 'true' + env: + EXPO_TOKEN: ${{ secrets.EXPO_PREVIEW_TOKEN }} + run: | + if [ -n "$EXPO_TOKEN" ]; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + echo "EXPO_PREVIEW_TOKEN is not set; skipping EAS preview." + fi + + # The head SHA from the label event, not the branch, so a push after the + # label cannot swap in unreviewed code. No credentials stay in .git for + # the PR's scripts to read. + - name: Checkout reviewed commit + if: steps.expo-token.outputs.present == 'true' + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + # No sparse-checkout: see mobile-eas-preview.yml. + + # No caches: pull_request_target saves them under main's scope, where + # trusted workflows with other secrets would restore what PR code wrote. + - name: Setup Vite+ + if: steps.expo-token.outputs.present == 'true' + uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: false + run-install: | + args: + - --filter=@t3tools/mobile... + + - name: Expose pnpm + if: steps.expo-token.outputs.present == 'true' + run: | + pnpm_version="$(node --print "require('./package.json').packageManager.split('@').pop()")" + vp_pnpm_bin="$HOME/.vite-plus/package_manager/pnpm/$pnpm_version/pnpm/bin" + echo "$vp_pnpm_bin" >> "$GITHUB_PATH" + "$vp_pnpm_bin/pnpm" --version + + - name: Setup EAS + if: steps.expo-token.outputs.present == 'true' + uses: expo/expo-github-action@v8 + with: + # Pinned and installed with npm for the same reasons as mobile-eas-preview.yml. + eas-version: 21.8.0 + eas-cache: false + token: ${{ secrets.EXPO_PREVIEW_TOKEN }} + packager: npm + + - name: Pull preview environment variables + if: steps.expo-token.outputs.present == 'true' + working-directory: apps/mobile + env: + EXPO_TOKEN: ${{ secrets.EXPO_PREVIEW_TOKEN }} + run: | + eas env:pull preview --non-interactive + # See mobile-eas-preview.yml: keeps Android build and OTA fingerprints matching. + node --input-type=module <<'NODE' + import { appendFileSync, readFileSync } from "node:fs"; + import { parseEnv } from "node:util"; + const env = parseEnv(readFileSync(".env.local", "utf8")); + const googleServicesFile = env.T3CODE_ANDROID_GOOGLE_SERVICES_FILE; + if (googleServicesFile) { + appendFileSync(process.env.GITHUB_ENV, `T3CODE_ANDROID_GOOGLE_SERVICES_FILE=${googleServicesFile}\n`); + } + NODE + + - name: Deploy with fingerprint check + if: steps.expo-token.outputs.present == 'true' + uses: expo/expo-github-action/continuous-deploy-fingerprint@main + env: + EXPO_TOKEN: ${{ secrets.EXPO_PREVIEW_TOKEN }} + RUNNER_DEBUG: "1" + # The action only comments for pull_request events; this hook gives + # it the PR under pull_request_target. Without it, it skips the comment. + EXPO_TEST_GITHUB_PULL: ${{ github.event.pull_request.number }} + with: + profile: preview:dev + branch: pr-${{ github.event.pull_request.number }} + platform: all + environment: preview + working-directory: apps/mobile + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/mobile-eas-preview.yml b/.github/workflows/mobile-eas-preview.yml index f1a376332c2cd..d752166045fd9 100644 --- a/.github/workflows/mobile-eas-preview.yml +++ b/.github/workflows/mobile-eas-preview.yml @@ -7,7 +7,9 @@ on: jobs: preview: name: EAS Preview + # Fork PRs deploy one reviewed commit per label through mobile-eas-preview-fork.yml. if: | + github.event.pull_request.head.repo.full_name == github.repository && contains(github.event.pull_request.labels.*.name, '🚀 Mobile Continuous Deployment') && (github.event.action != 'labeled' || github.event.label.name == '🚀 Mobile Continuous Deployment') runs-on: ubuntu-24.04 @@ -23,15 +25,15 @@ jobs: MOBILE_VERSION_POLICY: fingerprint steps: - id: expo-token - name: Check for EXPO_TOKEN + name: Check for EXPO_PREVIEW_TOKEN env: - EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} + EXPO_TOKEN: ${{ secrets.EXPO_PREVIEW_TOKEN }} run: | if [ -n "$EXPO_TOKEN" ]; then echo "present=true" >> "$GITHUB_OUTPUT" else echo "present=false" >> "$GITHUB_OUTPUT" - echo "EXPO_TOKEN is not available; skipping EAS preview." + echo "EXPO_PREVIEW_TOKEN is not available; skipping EAS preview." fi - name: Checkout @@ -70,7 +72,7 @@ jobs: # continuous-deploy-fingerprint still reads `build.project.ownerAccount`. # Unpin once https://github.com/expo/expo-github-action/issues/358 is fixed. eas-version: 21.8.0 - token: ${{ secrets.EXPO_TOKEN }} + token: ${{ secrets.EXPO_PREVIEW_TOKEN }} # npm, not pnpm: this only installs eas-cli into the action's own # tool dir, and pnpm 11 hard-fails that install on dtrace-provider's # ignored build script (no allowBuilds config outside the repo). @@ -80,7 +82,7 @@ jobs: if: steps.expo-token.outputs.present == 'true' working-directory: apps/mobile env: - EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} + EXPO_TOKEN: ${{ secrets.EXPO_PREVIEW_TOKEN }} run: | eas env:pull preview --non-interactive # EAS Update disables dotenv loading. Keep the downloaded native config @@ -99,7 +101,7 @@ jobs: if: steps.expo-token.outputs.present == 'true' uses: expo/expo-github-action/continuous-deploy-fingerprint@eab7a230208c952974db8c3245cfd78402c7b385 # 9.0.0 env: - EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} + EXPO_TOKEN: ${{ secrets.EXPO_PREVIEW_TOKEN }} # The Expo action otherwise suppresses the EAS command's failure output. RUNNER_DEBUG: "1" with: diff --git a/.github/workflows/mobile-eas-production.yml b/.github/workflows/mobile-eas-production.yml index 88956ba2b3183..4cc2030b9b5ad 100644 --- a/.github/workflows/mobile-eas-production.yml +++ b/.github/workflows/mobile-eas-production.yml @@ -97,13 +97,13 @@ jobs: - id: expo-token name: Check for EXPO_TOKEN env: - EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} + EXPO_TOKEN: ${{ secrets.EXPO_PRODUCTION_TOKEN }} run: | if [ -n "$EXPO_TOKEN" ]; then echo "present=true" >> "$GITHUB_OUTPUT" else echo "present=false" >> "$GITHUB_OUTPUT" - echo "EXPO_TOKEN is not available; skipping EAS production job." + echo "EXPO_PRODUCTION_TOKEN is not available; skipping EAS production job." fi - id: version_app_token @@ -149,7 +149,7 @@ jobs: uses: expo/expo-github-action@c7b66a9c327a43a8fa7c0158e7f30d6040d2481e # 8.2.1 with: eas-version: latest - token: ${{ secrets.EXPO_TOKEN }} + token: ${{ secrets.EXPO_PRODUCTION_TOKEN }} # npm, not pnpm: this only installs eas-cli into the action's own # tool dir, and pnpm 11 hard-fails that install on dtrace-provider's # ignored build script (no allowBuilds config outside the repo). @@ -159,7 +159,7 @@ jobs: if: steps.expo-token.outputs.present == 'true' working-directory: apps/mobile env: - EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} + EXPO_TOKEN: ${{ secrets.EXPO_PRODUCTION_TOKEN }} run: | eas env:pull production --non-interactive # EAS commands skip dotenv, and the Android fingerprint hashes this @@ -244,7 +244,7 @@ jobs: if: steps.expo-token.outputs.present == 'true' && github.event_name == 'workflow_dispatch' && inputs.mode == 'build' working-directory: apps/mobile env: - EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} + EXPO_TOKEN: ${{ secrets.EXPO_PRODUCTION_TOKEN }} MOBILE_BUILD_PROFILE: ${{ inputs.profile || 'production' }} run: eas build --platform ${{ inputs.platform }} --profile "$MOBILE_BUILD_PROFILE" --auto-submit --non-interactive --no-wait @@ -255,7 +255,7 @@ jobs: if: steps.expo-token.outputs.present == 'true' && (github.event_name == 'push' || inputs.mode == 'update') working-directory: apps/mobile env: - EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} + EXPO_TOKEN: ${{ secrets.EXPO_PRODUCTION_TOKEN }} PLATFORM: ${{ inputs.platform || 'all' }} run: | if [ "$PLATFORM" = "all" ]; then platforms="ios android"; else platforms="$PLATFORM"; fi @@ -275,7 +275,7 @@ jobs: if: steps.expo-token.outputs.present == 'true' && github.event_name == 'workflow_dispatch' && inputs.mode == 'update' working-directory: apps/mobile env: - EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} + EXPO_TOKEN: ${{ secrets.EXPO_PRODUCTION_TOKEN }} MOBILE_BUILD_PROFILE: ${{ inputs.profile || 'production' }} OTA_MESSAGE: ${{ inputs.message || format('Production OTA ({0})', github.sha) }} run: | @@ -299,7 +299,7 @@ jobs: if: steps.expo-token.outputs.present == 'true' && github.event_name == 'push' working-directory: apps/mobile env: - EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} + EXPO_TOKEN: ${{ secrets.EXPO_PRODUCTION_TOKEN }} run: | failed=0 message="$(git log -1 --pretty=%s | head -c 120) ($(git rev-parse --short=9 HEAD))" diff --git a/apps/desktop/src/app/DesktopCliCommand.test.ts b/apps/desktop/src/app/DesktopCliCommand.test.ts index 7141ebde70537..8566613dd6069 100644 --- a/apps/desktop/src/app/DesktopCliCommand.test.ts +++ b/apps/desktop/src/app/DesktopCliCommand.test.ts @@ -136,6 +136,14 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { yield* command.uninstall; expect(yield* fs.readFileString(theirs)).toBe("npm's t3\n"); expect(yield* fs.readLink(path.join(home, "bin", "t3"))).toBe(path.join(home, "gone")); + + const launcher = path.join(home, ".t3", "bin", "t3"); + yield* fs.writeFileString(launcher, "npm's t3\n"); + process.env.PATH = path.dirname(launcher); + expect(yield* command.state).toMatchObject({ onPath: false, shadowedBy: launcher }); + expect((yield* Effect.flip(command.install)).message).toContain("Could not set up"); + yield* command.uninstall; + expect(yield* fs.readFileString(launcher)).toBe("npm's t3\n"); }).pipe(Effect.scoped), ); @@ -217,6 +225,34 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { }).pipe(Effect.scoped), ); + it.effect("does not treat its own launcher as another t3 when its folder is first on PATH", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped(); + const launcher = path.join(home, ".t3", "bin", "t3"); + process.env.PATH = [path.dirname(launcher), path.join(home, ".local", "bin")].join(":"); + + const command = yield* commandIn({ home }); + const installed = yield* command.install; + + expect(yield* fs.exists(launcher)).toBe(true); + expect(installed).toMatchObject({ + onPath: true, + installedPath: path.join(home, ".local", "bin", "t3"), + }); + expect(installed.shadowedBy).toBeUndefined(); + expect(yield* command.state).toEqual(installed); + + const removed = yield* command.uninstall; + expect(removed.installedPath).toBeNull(); + expect(removed.onPath).toBe(true); + expect(yield* command.state).toEqual(removed); + expect(yield* fs.exists(installed.installedPath!)).toBe(false); + expect(yield* fs.exists(launcher)).toBe(true); + }).pipe(Effect.scoped), + ); + it.effect("leaves the Windows PATH alone when it cannot be read", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/desktop/src/app/DesktopCliCommand.ts b/apps/desktop/src/app/DesktopCliCommand.ts index 8a89a366814ed..f53895a60a570 100644 --- a/apps/desktop/src/app/DesktopCliCommand.ts +++ b/apps/desktop/src/app/DesktopCliCommand.ts @@ -92,6 +92,19 @@ export const make = Effect.gen(function* () { Effect.orElseSucceed(() => false), ); + // PATH can reach the regular launcher directly; removal still requires isOurLink. + const runsOurLauncher = Effect.fn("DesktopCliCommand.runsOurLauncher")( + function* (candidate: string) { + if (yield* isOurLink(candidate)) return true; + const [resolved, ours] = yield* Effect.all([fs.realPath(candidate), fs.realPath(launcher)]); + if (resolved !== ours) return false; + const info = yield* fs.stat(candidate); + if (info.type !== "File" || Number(info.size) > 16_384) return false; + return (yield* fs.readFileString(candidate)).includes(DesktopCliShim.MARKER); + }, + Effect.orElseSucceed(() => false), + ); + const powershell = (env: Record) => Effect.scoped( Effect.gen(function* () { @@ -141,7 +154,7 @@ export const make = Effect.gen(function* () { const foreignFirstOnPath = Effect.gen(function* () { if (windows) return Option.none(); const first = yield* firstOnPath; - if (Option.isNone(first) || (yield* isOurLink(first.value))) return Option.none(); + if (Option.isNone(first) || (yield* runsOurLauncher(first.value))) return Option.none(); return first; }); @@ -168,12 +181,17 @@ export const make = Effect.gen(function* () { const installed = yield* installedAt; const shadowedBy = yield* foreignFirstOnPath; const shadow = Option.isSome(shadowedBy) ? { shadowedBy: shadowedBy.value } : {}; - if (Option.isNone(installed)) { - return { supported: true, installedPath: null, onPath: false, launcherPath, ...shadow }; - } const first = yield* firstOnPath; - const onPath = windows || (Option.isSome(first) && (yield* isOurLink(first.value))); - return { supported: true, installedPath: installed.value, onPath, launcherPath, ...shadow }; + const onPath = windows + ? Option.isSome(installed) + : Option.isSome(first) && (yield* runsOurLauncher(first.value)); + return { + supported: true, + installedPath: Option.getOrNull(installed), + onPath, + launcherPath, + ...shadow, + }; }).pipe( Effect.orElseSucceed(() => ({ supported: false, diff --git a/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts b/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts index feedbbd580074..9404b0ac3087c 100644 --- a/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts +++ b/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts @@ -1,7 +1,11 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -9,6 +13,8 @@ import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; +import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as DesktopEnvironment from "./DesktopEnvironment.ts"; @@ -56,6 +62,7 @@ const layerHandler = ( recorded: RecordedRegistration, input: { readonly environment?: Record; + readonly processSpawner?: ChildProcessSpawner.ChildProcessSpawner["Service"]; readonly updateDesktopDatabaseExitCode?: number; readonly updateDesktopDatabaseStalled?: boolean; readonly updateDesktopDatabaseStarted?: Deferred.Deferred; @@ -106,6 +113,7 @@ const layerHandler = ( Layer.succeed( ChildProcessSpawner.ChildProcessSpawner, ChildProcessSpawner.make((command) => { + if (input.processSpawner) return input.processSpawner.spawn(command); const childProcess = command as unknown as { readonly command: string; readonly args: ReadonlyArray; @@ -358,6 +366,69 @@ describe("DesktopLinuxUrlHandler", () => { assert.deepEqual(writeFailed.commands, []); }); }); + + it.effect.skipIf(HostProcess.Platform.defaultValue() !== "linux")( + "continues registration after a timed-out cache helper ignores SIGTERM", + () => + Effect.gen(function* () { + const realSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const nodeExecutable = yield* HostProcess.ExecutablePath; + const recorded = emptyRecording(); + const spawned = yield* Deferred.make(); + const ready = yield* Deferred.make(); + const processSpawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (!ChildProcess.isStandardCommand(command)) { + return yield* Effect.die("expected a single registration command"); + } + recorded.commands.push({ command: command.command, args: command.args }); + if (command.command !== "update-desktop-database") return mockProcess(0); + const child = yield* realSpawner.spawn( + ChildProcess.make( + nodeExecutable, + [ + "-e", + // The watchdog makes the unfixed case exit with a distinguishable code. + "process.on('SIGTERM', () => setTimeout(() => process.exit(99), 5000));" + + "process.stdout.write('ready\\n'); setInterval(() => {}, 2147483647);", + ], + { ...command.options, stdout: "pipe" }, + ), + ); + yield* Deferred.succeed(spawned, child); + yield* Stream.runHead(child.stdout); + yield* Deferred.succeed(ready, undefined); + return child; + }), + ); + const registration = yield* runRegister(recorded, { processSpawner }).pipe( + Effect.forkScoped, + ); + const child = yield* Deferred.await(spawned); + yield* Effect.addFinalizer(() => + child.isRunning.pipe( + Effect.flatMap((running) => + running ? child.kill({ killSignal: "SIGKILL" }) : Effect.void, + ), + Effect.ignore, + ), + ); + yield* Deferred.await(ready); + yield* TestClock.adjust("5 seconds"); + yield* Fiber.join(registration); + + const exit = yield* Effect.exit(child.exitCode); + if (Exit.isFailure(exit)) { + assert.include(Cause.pretty(exit.cause), "SIGKILL"); + } else { + assert.fail(`cache helper exited with ${exit.value} instead of being force-killed`); + } + assert.deepEqual( + recorded.commands.map(({ command }) => command), + ["update-desktop-database", "xdg-mime"], + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); }); interface ThemeIconRun { diff --git a/apps/desktop/src/app/DesktopLinuxUrlHandler.ts b/apps/desktop/src/app/DesktopLinuxUrlHandler.ts index dd38acbc3894e..ebd7d71b25726 100644 --- a/apps/desktop/src/app/DesktopLinuxUrlHandler.ts +++ b/apps/desktop/src/app/DesktopLinuxUrlHandler.ts @@ -174,6 +174,7 @@ export const make = Effect.gen(function* () { stdin: "ignore", stdout: "ignore", stderr: "ignore", + forceKillAfter: "1 second", }, ); const handle = yield* spawner.spawn(command); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index bba4f6449b8c1..538dd06b0b42a 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -14,6 +14,7 @@ export const SNAP_SHOT_EVENT_CHANNEL = "desktop:snap-shot-event"; export const WEB_LINK_OPEN_CHANNEL = "desktop:web-link-open"; export const WEB_LINK_READY_CHANNEL = "desktop:web-link-ready"; export const QUIT_SHORTCUT_CHANNEL = "desktop:quit-shortcut"; +export const TRACKPAD_SCROLL_BEGIN_CHANNEL = "desktop:trackpad-scroll-begin"; export const TRACKPAD_SCROLL_END_CHANNEL = "desktop:trackpad-scroll-end"; export const GET_WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:get-window-fullscreen-state"; export const WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:window-fullscreen-state"; diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 965b7db5d6d40..950ed972c6453 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -91,6 +91,11 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.on(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, handler); return () => ipcRenderer.removeListener(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, handler); }, + onTrackpadScrollBegin: (listener) => { + const handler = () => listener(); + ipcRenderer.on(IpcChannels.TRACKPAD_SCROLL_BEGIN_CHANNEL, handler); + return () => ipcRenderer.removeListener(IpcChannels.TRACKPAD_SCROLL_BEGIN_CHANNEL, handler); + }, onTrackpadScrollEnd: (listener) => { const handler = () => listener(); ipcRenderer.on(IpcChannels.TRACKPAD_SCROLL_END_CHANNEL, handler); diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index ab5fc8be26a92..c1c0cf5446cd2 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -23,6 +23,7 @@ import { MENU_ACTION_CHANNEL, QUIT_SHORTCUT_CHANNEL, SNAP_SHOT_EVENT_CHANNEL, + TRACKPAD_SCROLL_BEGIN_CHANNEL, TRACKPAD_SCROLL_END_CHANNEL, WINDOW_FULLSCREEN_STATE_CHANNEL, } from "../ipc/channels.ts"; @@ -652,7 +653,11 @@ export const make = Effect.gen(function* () { } }); window.webContents.on("input-event", (_event, input) => { - if (input.type === "gestureScrollEnd") window.webContents.send(TRACKPAD_SCROLL_END_CHANNEL); + if (input.type === "gestureScrollBegin") { + window.webContents.send(TRACKPAD_SCROLL_BEGIN_CHANNEL); + } else if (input.type === "gestureScrollEnd") { + window.webContents.send(TRACKPAD_SCROLL_END_CHANNEL); + } }); // Ctrl+wheel bypasses menus; guest contents emit their own zoom-changed event. window.webContents.on("zoom-changed", (_event, direction) => { diff --git a/apps/mobile/src/features/files/fileTree.test.ts b/apps/mobile/src/features/files/fileTree.test.ts new file mode 100644 index 0000000000000..d978333d2732f --- /dev/null +++ b/apps/mobile/src/features/files/fileTree.test.ts @@ -0,0 +1,61 @@ +import { expect, it } from "vite-plus/test"; + +import { buildFileTree, flattenFileTree } from "./fileTree"; + +it("flattens a wide expanded directory without exceeding argument limits", () => { + const children = Array.from({ length: 200_000 }, (_, index) => ({ + path: `root/file-${index}`, + name: `file-${index}`, + kind: "file" as const, + children: [], + })); + const nodes = [{ path: "root", name: "root", kind: "directory" as const, children }]; + const rows = flattenFileTree({ nodes, expanded: new Set(["root"]) }); + expect(rows).toHaveLength(children.length + 1); + expect(rows[0]).toEqual({ node: nodes[0], depth: 0 }); + const middleChildIndex = Math.floor(children.length / 2); + expect(rows[middleChildIndex + 1]).toEqual({ + node: children[middleChildIndex], + depth: 1, + }); + expect(rows.at(-1)).toEqual({ node: children.at(-1), depth: 1 }); +}); + +it("prunes unmatched subtrees without losing adjacent rows or their depths", () => { + const nodes = buildFileTree([ + { kind: "file", path: "root/a/hidden.txt" }, + { kind: "file", path: "root/b/match.txt" }, + { kind: "file", path: "root/b/other.txt" }, + { kind: "file", path: "root/c/hidden.txt" }, + { kind: "file", path: "tail.txt" }, + ]); + const expanded = new Set(["root", "root/a", "root/b", "root/c"]); + const rows = (searchResultPaths: ReadonlySet | null) => + flattenFileTree({ nodes, expanded, searchResultPaths }).map( + ({ node, depth }) => `${depth}:${node.path}`, + ); + + expect(rows(new Set(["root/b/match.txt", "tail.txt"]))).toEqual([ + "0:root", + "1:root/b", + "2:root/b/match.txt", + "0:tail.txt", + ]); + expect(rows(new Set(["root/b"]))).toEqual(["0:root", "1:root/b"]); + expect(rows(new Set())).toEqual([]); + expect(rows(null)).toEqual([ + "0:root", + "1:root/a", + "2:root/a/hidden.txt", + "1:root/b", + "2:root/b/match.txt", + "2:root/b/other.txt", + "1:root/c", + "2:root/c/hidden.txt", + "0:tail.txt", + ]); + expect(flattenFileTree({ nodes, expanded: new Set() }).map(({ node }) => node.path)).toEqual([ + "root", + "tail.txt", + ]); +}); diff --git a/apps/mobile/src/features/files/fileTree.ts b/apps/mobile/src/features/files/fileTree.ts index 4e4b4b77171dd..22e8e9bd5bd8d 100644 --- a/apps/mobile/src/features/files/fileTree.ts +++ b/apps/mobile/src/features/files/fileTree.ts @@ -100,11 +100,12 @@ function flattenNode( const isSearching = searchResultPaths !== null; const matches = searchResultPaths?.has(node.path) ?? false; let descendantMatches = false; - const childOutput: VisibleFileTreeNode[] = []; + const start = output.length; + output.push({ node, depth }); if (node.kind === "directory" && (expanded.has(node.path) || isSearching)) { for (const child of node.children) { - if (flattenNode(childOutput, child, depth + 1, expanded, searchResultPaths)) { + if (flattenNode(output, child, depth + 1, expanded, searchResultPaths)) { descendantMatches = true; } } @@ -112,11 +113,10 @@ function flattenNode( const visible = !isSearching || matches || descendantMatches; if (!visible) { + output.length = start; return false; } - output.push({ node, depth }); - output.push(...childOutput); return matches || descendantMatches; } diff --git a/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx b/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx index 446794800eadb..bd3f04d9d6b4d 100644 --- a/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx +++ b/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx @@ -794,7 +794,12 @@ export function PullRequestRouteScreen({ route }: StaticScreenProps ) : null} - {tab === "guide" ? : null} + {tab === "guide" ? ( + + ) : null} {tab === "activity" ? ( - !project.isScratch && (environmentId === null || project.environmentId === environmentId), - ); + const projects = pullRequestProjectChoices(allProjects, { + environmentId, + controls, + environmentLabels: new Map( + environments.map((item) => [item.environmentId, item.environmentLabel]), + ), + servesPullRequests: (id) => configs.get(id)?.environment.capabilities.pullRequests === true, + }); const filters = usePullRequestListFilters({ environments: environments.map((item) => ({ id: String(item.environmentId), diff --git a/apps/mobile/src/features/pull-requests/guide/PullRequestGuideScreen.tsx b/apps/mobile/src/features/pull-requests/guide/PullRequestGuideScreen.tsx index d198ea5b09e07..caa1e72938c87 100644 --- a/apps/mobile/src/features/pull-requests/guide/PullRequestGuideScreen.tsx +++ b/apps/mobile/src/features/pull-requests/guide/PullRequestGuideScreen.tsx @@ -48,9 +48,15 @@ const attentionLabels = { accessibility: "An accessibility call", }; -export function PullRequestGuideContent({ params }: { readonly params: PullRequestParams }) { +export function PullRequestGuideContent({ + params, + visible = true, +}: { + readonly params: PullRequestParams; + readonly visible?: boolean; +}) { const navigation = useNavigation(); - const state = useReviewGuide(params); + const state = useReviewGuide(params, visible); const { guide, stale } = reviewGuideState(state.data); return ( diff --git a/apps/mobile/src/features/pull-requests/guide/useReviewGuide.test.tsx b/apps/mobile/src/features/pull-requests/guide/useReviewGuide.test.tsx index 764cd5f881ac9..e797a12a57217 100644 --- a/apps/mobile/src/features/pull-requests/guide/useReviewGuide.test.tsx +++ b/apps/mobile/src/features/pull-requests/guide/useReviewGuide.test.tsx @@ -3,18 +3,21 @@ import { type NavigationProp, type ParamListBase, } from "@react-navigation/native"; -import { act } from "react"; +import { AsyncResult } from "effect/reactivity"; +import { act, useLayoutEffect } from "react"; import { createRoot } from "react-dom/client"; import { expect, it, vi } from "vite-plus/test"; const mocks = vi.hoisted(() => ({ + enabled: true, + generate: vi.fn(), query: { - data: null as { run: { _tag: "Running" | "Completed" } } | null, + data: null as { run: { _tag: "Running" | "Completed" | "Failed" } | null } | null, isPending: false, refresh: vi.fn<() => void>(), }, otherQuery: { - data: null as { run: { _tag: "Running" | "Completed" } } | null, + data: null as { run: { _tag: "Running" | "Completed" | "Failed" } | null } | null, isPending: false, refresh: vi.fn<() => void>(), }, @@ -33,7 +36,8 @@ vi.mock("@react-navigation/native", async () => { return { NavigationContext, useFocusEffect }; }); vi.mock("@effect/atom-react", () => ({ - useAtomValue: (atom: unknown) => (atom === "settings" ? { reviewGuidesEnabled: true } : true), + useAtomValue: (atom: unknown) => + atom === "settings" ? { reviewGuidesEnabled: mocks.enabled } : true, })); vi.mock("../../../state/server", () => ({ serverEnvironment: { settingsValueAtom: () => "settings" }, @@ -48,7 +52,7 @@ vi.mock("../../../state/pull-requests", () => ({ vi.mock("../../../state/query", () => ({ useEnvironmentQuery: (number: number) => ({ ...(number === 1 ? mocks.query : mocks.otherQuery) }), })); -vi.mock("../../../state/use-atom-command", () => ({ useAtomCommand: () => vi.fn() })); +vi.mock("../../../state/use-atom-command", () => ({ useAtomCommand: () => mocks.generate })); vi.mock("../usePullRequestConnected", () => ({ usePullRequestConnected: () => true })); vi.mock("../usePullRequestDiff", () => ({ pullRequestTarget: (params: unknown) => params, @@ -78,6 +82,8 @@ function createNavigation() { function setup() { (globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + mocks.enabled = true; + mocks.generate = vi.fn().mockResolvedValue(AsyncResult.success(undefined)); mocks.query = { data: null, isPending: false, refresh: vi.fn() }; mocks.otherQuery = { data: null, isPending: false, refresh: vi.fn() }; const root = createRoot(document.createElement("div")); @@ -88,20 +94,35 @@ function setup() { repository: "repo", number: 1, }; - function Probe({ number, environmentId }: { number: number; environmentId: string }) { - useReviewGuide({ ...params, number, environmentId }); + let current: ReturnType; + function Probe({ + number, + environmentId, + visible, + }: { + number: number; + environmentId: string; + visible?: boolean; + }) { + const state = useReviewGuide({ ...params, number, environmentId }, visible); + useLayoutEffect(() => { + current = state; + }); return null; } return { - render(number = 1, environmentId = params.environmentId) { + render(number = 1, environmentId = params.environmentId, visible?: boolean) { act(() => root.render( - + , ), ); }, + get current() { + return current; + }, focus: navigation.focus, [Symbol.dispose]() { act(() => root.unmount()); @@ -188,3 +209,94 @@ it("retires the old environment's polling on target change and clears polling on expect(mocks.otherQuery.refresh).toHaveBeenCalledTimes(2); expect(vi.getTimerCount()).toBe(0); }); + +it("suspends only manual refresh while hidden, including focus and target changes", () => { + vi.useFakeTimers(); + using _timers = { [Symbol.dispose]: () => vi.useRealTimers() }; + { + using screen = setup(); + mocks.query.data = { run: { _tag: "Running" } }; + mocks.otherQuery.data = { run: { _tag: "Running" } }; + screen.render(); + const cached = screen.current.data; + screen.render(1, "environment", false); + act(() => vi.advanceTimersByTime(60_000)); + expect(mocks.query.refresh).toHaveBeenCalledTimes(1); + expect(screen.current.data).toBe(cached); + screen.render(); + expect(mocks.query.refresh).toHaveBeenCalledTimes(2); + act(() => vi.advanceTimersByTime(5_000)); + expect(mocks.query.refresh).toHaveBeenCalledTimes(3); + screen.render(1, "environment", false); + screen.focus(false); + screen.focus(true); + expect(mocks.query.refresh).toHaveBeenCalledTimes(3); + + screen.render(2, "other-environment", false); + act(() => vi.advanceTimersByTime(5_000)); + expect(mocks.otherQuery.refresh).not.toHaveBeenCalled(); + screen.focus(false); + screen.render(2, "other-environment", true); + expect(mocks.otherQuery.refresh).not.toHaveBeenCalled(); + screen.focus(true); + expect(mocks.otherQuery.refresh).toHaveBeenCalledTimes(1); + act(() => vi.advanceTimersByTime(5_000)); + expect(mocks.otherQuery.refresh).toHaveBeenCalledTimes(2); + expect(mocks.query.refresh).toHaveBeenCalledTimes(3); + } + expect(vi.getTimerCount()).toBe(0); +}); + +it("keeps disabled, completed and failed guides idle and refreshes once on reveal", () => { + vi.useFakeTimers(); + using _timers = { [Symbol.dispose]: () => vi.useRealTimers() }; + using screen = setup(); + mocks.enabled = false; + screen.render(); + act(() => vi.advanceTimersByTime(5_000)); + expect(mocks.query.refresh).not.toHaveBeenCalled(); + mocks.enabled = true; + mocks.query.data = { run: null }; + screen.render(); + expect(mocks.query.refresh).toHaveBeenCalledTimes(1); + act(() => vi.advanceTimersByTime(10_000)); + expect(mocks.query.refresh).toHaveBeenCalledTimes(1); + screen.render(1, "environment", false); + mocks.query.data = { run: { _tag: "Failed" } }; + screen.render(1, "environment", false); + expect(mocks.query.refresh).toHaveBeenCalledTimes(1); + screen.render(); + expect(mocks.query.refresh).toHaveBeenCalledTimes(2); + act(() => vi.advanceTimersByTime(10_000)); + expect(mocks.query.refresh).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); +}); + +it("leaves generation running when hidden and accepts its completion without a refresh", async () => { + vi.useFakeTimers(); + using _timers = { [Symbol.dispose]: () => vi.useRealTimers() }; + using screen = setup(); + const generated = Promise.withResolvers>>(); + mocks.generate.mockReturnValue(generated.promise); + screen.render(); + let writing: Promise; + act(() => { + writing = screen.current.write(); + }); + expect(screen.current.running).toBe(true); + screen.render(1, "environment", false); + const beforeHidden = mocks.query.refresh.mock.calls.length; + act(() => vi.advanceTimersByTime(60_000)); + expect(mocks.query.refresh).toHaveBeenCalledTimes(beforeHidden); + expect(mocks.generate).toHaveBeenCalledTimes(1); + expect(screen.current.running).toBe(true); + await act(async () => { + generated.resolve(AsyncResult.success(undefined)); + await writing; + }); + expect(screen.current.running).toBe(false); + expect(mocks.query.refresh).toHaveBeenCalledTimes(beforeHidden); + screen.render(); + expect(mocks.query.refresh).toHaveBeenCalledTimes(beforeHidden + 1); + expect(vi.getTimerCount()).toBe(0); +}); diff --git a/apps/mobile/src/features/pull-requests/guide/useReviewGuide.ts b/apps/mobile/src/features/pull-requests/guide/useReviewGuide.ts index 5dad083e76108..bbdb6caee00c3 100644 --- a/apps/mobile/src/features/pull-requests/guide/useReviewGuide.ts +++ b/apps/mobile/src/features/pull-requests/guide/useReviewGuide.ts @@ -17,7 +17,7 @@ export function useReviewGuidesEnabled(params: PullRequestParams) { return settings?.reviewGuidesEnabled === true; } -export function useReviewGuide(params: PullRequestParams) { +export function useReviewGuide(params: PullRequestParams, visible = true) { const target = pullRequestTarget(params); const enabled = useReviewGuidesEnabled(params); const query = useEnvironmentQuery(enabled ? composerPullRequests.reviewGuide(target) : null); @@ -35,7 +35,7 @@ export function useReviewGuide(params: PullRequestParams) { const running = starting || query.data?.run?._tag === "Running"; const refresh = query.refresh; useFocusEffect(() => { - if (!enabled) return; + if (!enabled || !visible) return; refresh(); if (!running) return; const timer = setInterval(refresh, 5_000); diff --git a/apps/mobile/src/features/pull-requests/pullRequestList.test.ts b/apps/mobile/src/features/pull-requests/pullRequestList.test.ts index f7b1a7a1bf832..b6f47b6533ee2 100644 --- a/apps/mobile/src/features/pull-requests/pullRequestList.test.ts +++ b/apps/mobile/src/features/pull-requests/pullRequestList.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vite-plus/test"; import { EnvironmentId, + ProjectId, SourceControlProviderKind, type PullRequestListEntry, type PullRequestListResult, @@ -26,6 +27,7 @@ import { withListedPullRequests, withShownPullRequestStats, withoutScratchProjectScope, + pullRequestProjectChoices, } from "./pullRequestList"; function entry(number: number, patch: Partial = {}): PullRequestListEntry { @@ -402,6 +404,53 @@ describe("pullRequestFilterChoices", () => { }); }); +describe("pullRequestProjectChoices", () => { + const laptop = EnvironmentId.make("env-laptop"); + const desktop = EnvironmentId.make("env-desktop"); + const project = (id: string, environmentId: EnvironmentId) => ({ + id: ProjectId.make(id), + environmentId, + title: "t3code", + workspaceRoot: `/work/${id}`, + repositoryIdentity: { canonicalKey: "github.com/pingdotgg/t3code" }, + }); + const projects = [project("main", laptop), project("worktree", laptop), project("main", desktop)]; + const choices = ( + servesPullRequests: (environmentId: EnvironmentId) => boolean, + scope: { projectId?: string; projectEnvironmentId?: string } = {}, + ) => + pullRequestProjectChoices(projects, { + environmentId: null, + controls: { ...DEFAULT_PR_LIST_CONTROLS, ...scope }, + environmentLabels: new Map([ + [laptop, "laptop"], + [desktop, "desktop"], + ]), + servesPullRequests, + }).map(({ id, environmentId, title }) => ({ id, environmentId, title })); + + it("lists each repository once per environment that serves pull requests", () => { + expect(choices(() => true)).toEqual([ + { id: "main", environmentId: desktop, title: "t3code · desktop" }, + { id: "main", environmentId: laptop, title: "t3code · laptop" }, + ]); + expect(choices((environmentId) => environmentId === desktop)).toEqual([ + { id: "main", environmentId: desktop, title: "t3code" }, + ]); + }); + + it("keeps the saved scope when its environment stops serving or has not loaded", () => { + const scope = { projectId: "worktree", projectEnvironmentId: laptop }; + expect(choices((environmentId) => environmentId === desktop, scope)).toEqual([ + { id: "main", environmentId: desktop, title: "t3code · desktop" }, + { id: "worktree", environmentId: laptop, title: "t3code · laptop" }, + ]); + expect(choices(() => false, scope)).toEqual([ + { id: "worktree", environmentId: laptop, title: "t3code" }, + ]); + }); +}); + describe("shown PR diff stats", () => { const cups = EnvironmentId.make("cups"); const stat = (number: number, additions: number, deletions: number) => ({ diff --git a/apps/mobile/src/features/pull-requests/pullRequestList.ts b/apps/mobile/src/features/pull-requests/pullRequestList.ts index fcb41ac8ec861..674d3b0e24fe3 100644 --- a/apps/mobile/src/features/pull-requests/pullRequestList.ts +++ b/apps/mobile/src/features/pull-requests/pullRequestList.ts @@ -1,3 +1,4 @@ +import { pullRequestFilterProjects } from "@t3tools/client-runtime/pull-request-project-filter"; import { parsePullRequestQuery } from "@t3tools/client-runtime/pull-request-query"; import { ProjectId, @@ -34,6 +35,34 @@ export function withoutScratchProjectScope( : controls; } +type FilterProject = Parameters[0][number]; + +export function pullRequestProjectChoices< + Project extends FilterProject & { readonly isScratch?: true }, +>( + projects: readonly Project[], + input: { + readonly environmentId: string | null; + readonly controls: PullRequestListControls; + readonly environmentLabels: ReadonlyMap; + readonly servesPullRequests: (environmentId: EnvironmentId) => boolean; + }, +) { + const isSelected = (project: Project) => + project.id === input.controls.projectId && + project.environmentId === input.controls.projectEnvironmentId; + return pullRequestFilterProjects( + projects.filter( + (project) => + !project.isScratch && + (input.environmentId === null || project.environmentId === input.environmentId) && + (input.servesPullRequests(project.environmentId) || isSelected(project)), + ), + input.environmentLabels, + projects.find(isSelected), + ); +} + export function pullRequestListKey( item: Pick, ) { diff --git a/apps/mobile/src/features/review/reviewModel.ts b/apps/mobile/src/features/review/reviewModel.ts index bc0e23b336582..21b57322da1cd 100644 --- a/apps/mobile/src/features/review/reviewModel.ts +++ b/apps/mobile/src/features/review/reviewModel.ts @@ -1,6 +1,9 @@ import { parsePatchFiles } from "@pierre/diffs/utils/parsePatchFiles"; import type { ChangeTypes, FileDiffMetadata } from "@pierre/diffs/types"; -import type { ThreadCheckpointSummary } from "@t3tools/client-runtime/state/thread-checkpoints"; +import { + checkpointFileTotals, + type ThreadCheckpointSummary, +} from "@t3tools/client-runtime/state/thread-checkpoints"; import type { ReviewDiffPreviewSource } from "@t3tools/contracts"; import { quoteGitPatchPath } from "@t3tools/shared/gitPatchPath"; import * as Arr from "effect/Array"; @@ -107,7 +110,7 @@ function checkpointTitle(checkpoint: ThreadCheckpointSummary): string { } function checkpointSubtitle(checkpoint: ThreadCheckpointSummary): string { - const fileCount = checkpoint.files.length; + const fileCount = checkpointFileTotals(checkpoint).count; if (checkpoint.status !== "ready") { return `Diff ${checkpoint.status}`; } @@ -531,17 +534,22 @@ export function buildReviewSectionItems(input: { }, ); - const gitItems = input.gitSections.map((section) => ({ - id: `git:${section.kind}`, - kind: section.kind, - title: section.title, - subtitle: gitSubtitle(section), - diff: section.diff, - source: section, - ...(section.files ? { files: section.files } : {}), - truncated: section.truncated, - isLoading: false, - })); + const gitItems = input.gitSections + .filter( + (section): section is typeof section & { kind: ReviewSectionKind } => + section.kind !== "commit", + ) + .map((section) => ({ + id: `git:${section.kind}`, + kind: section.kind, + title: section.title, + subtitle: gitSubtitle(section), + diff: section.diff, + source: section, + ...(section.files ? { files: section.files } : {}), + truncated: section.truncated, + isLoading: false, + })); const hasChangesItem = gitItems.some((item) => item.id === CHANGES_SECTION_ID); const visibleGitItems = input.loadingGitSections && !hasChangesItem diff --git a/apps/mobile/src/features/settings/EnvironmentRoutesSection.tsx b/apps/mobile/src/features/settings/EnvironmentRoutesSection.tsx index 012d00737d1df..842965dbf1575 100644 --- a/apps/mobile/src/features/settings/EnvironmentRoutesSection.tsx +++ b/apps/mobile/src/features/settings/EnvironmentRoutesSection.tsx @@ -12,7 +12,10 @@ import { isLearned, } from "@t3tools/client-runtime/connection"; import { type EnvironmentId, sessionGrantsScope } from "@t3tools/contracts"; -import { AUTH_SCOPE_OPTIONS } from "@t3tools/shared/authScopeOptions"; +import { + AUTH_SCOPE_OPTIONS, + sessionPermissionsUnavailableMessage, +} from "@t3tools/shared/authScopeOptions"; import { AsyncResult } from "effect/reactivity"; import * as Option from "effect/Option"; import { useState } from "react"; @@ -163,8 +166,14 @@ export function EnvironmentRoutesSection({ ) : ( - Permissions not checked. Connect to this environment to view this session’s - permissions. + {sessionPermissionsUnavailableMessage({ + connected: activeRouteId !== null, + failed: sessionResult._tag === "Failure", + rejected: + sessionResult._tag !== "Failure" && + !sessionResult.waiting && + session?.authenticated === false, + })} )} diff --git a/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx index cd9d8e03c9c1f..5f83979197bbc 100644 --- a/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx @@ -8,6 +8,7 @@ import { type ServerProvider, } from "@t3tools/contracts"; import { settlePromise, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; +import { sessionPermissionsUnavailableMessage } from "@t3tools/shared/authScopeOptions"; import * as Cause from "effect/Cause"; import * as Exit from "effect/Exit"; import { AsyncResult } from "effect/reactivity"; @@ -208,11 +209,14 @@ function EnvironmentDetail({ environmentId }: { readonly environmentId: Environm ) : !allowed ? ( - {AsyncResult.isFailure(sessionResult) - ? "Could not verify your permissions. Reconnect to try again." - : session === null - ? "Checking permissions…" - : "This connection does not have permission to manage the environment."} + {AsyncResult.isFailure(sessionResult) || session === null || !session.authenticated + ? sessionPermissionsUnavailableMessage({ + connected, + failed: AsyncResult.isFailure(sessionResult), + rejected: + !AsyncResult.isFailure(sessionResult) && session?.authenticated === false, + }) + : "This connection does not have permission to manage the environment."} ) : null} {error ? ( @@ -290,7 +294,10 @@ function EnvironmentDetail({ environmentId }: { readonly environmentId: Environm void run("refresh", async () => { if (!readEnvironmentScope(environmentId, AuthOrchestrationReadScope)) return; - const result = await refreshProviders({ environmentId, input: {} }); + const result = await refreshProviders({ + environmentId, + input: { reloadSecrets: true }, + }); if (AsyncResult.isFailure(result)) throw squashAtomCommandFailure(result); setNotice("Provider status refreshed."); }); diff --git a/apps/mobile/src/features/threads/DraftThreadHeader.tsx b/apps/mobile/src/features/threads/DraftThreadHeader.tsx index 509545b58c8bb..5c3d758f45513 100644 --- a/apps/mobile/src/features/threads/DraftThreadHeader.tsx +++ b/apps/mobile/src/features/threads/DraftThreadHeader.tsx @@ -87,7 +87,11 @@ export function DraftThreadHeader(props: { }, onRunAction: async (input) => { if (!canWriteGit || !props.cwd) return null; - const result = await runAction({ ...input, actionId: uuidv4() }); + const result = await runAction({ + ...input, + projectId: props.project.id, + actionId: uuidv4(), + }); if (AsyncResult.isFailure(result)) { Dialog.alert("Git action failed", String(Cause.squash(result.cause))); return null; diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index 27655f503bd65..6695e7963eff0 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -46,6 +46,7 @@ import type { ScopedThreadRef, ServerConfig as T3ServerConfig, UsageLimitsReport, + RunId, } from "@t3tools/contracts"; import { appendCodexArtifactTemplateUsePrompt, @@ -156,8 +157,10 @@ import { import { appAtomRegistry } from "../../state/atom-registry"; import { threadStreamHaptics } from "./threadStreamHaptics"; import { useAtomCommand } from "../../state/use-atom-command"; +import { orchestrationEnvironment } from "../../state/orchestration"; import { useDelayedStatus } from "../../lib/useDelayedStatus"; import type { PendingUserInputDraftAnswer } from "@t3tools/client-runtime/state/pending-user-input"; +import type { RunStopPresentation } from "@t3tools/client-runtime/state/thread-execution"; import type { PendingApproval, PendingUserInput, @@ -339,6 +342,9 @@ const threadVisitTracker = createThreadVisitTracker({ // Past the composer's 180 ms height animation. const COMPOSER_HEIGHT_SETTLE_MS = 220; +const NO_RUN_STOP_ATOM = Atom.make(null).pipe( + Atom.withLabel("mobile-thread-run-stop:empty"), +); // Anchors, the submitted message, follow state and pending scrolls belong to // one thread, so the body remounts when the thread changes. @@ -528,6 +534,50 @@ function ThreadDetailBody(props: ThreadDetailScreenProps) { const unsettleThread = useAtomCommand(threadEnvironment.unsettle, { reportFailure: false }); const updateMetadata = useAtomCommand(threadEnvironment.updateMetadata, { reportFailure: false }); const visitThread = useAtomCommand(threadEnvironment.visit, { reportFailure: false }); + const anyRunStop = useAtomValue( + props.selectedThreadDetailRef === null + ? NO_RUN_STOP_ATOM + : environmentThreadDetails.runStopAtom(props.selectedThreadDetailRef), + ); + // A background Stop matters only while that work is still pending. + const runStop = + anyRunStop?.background && props.selectedThread.pendingBackgroundTasks.length === 0 + ? null + : anyRunStop; + const forceStopThread = useAtomCommand(orchestrationEnvironment.forceStopThread, { + reportFailure: false, + }); + const canForceStop = + useAtomValue(orchestrationEnvironment.forceStopThread.permissionAtom(props.environmentId)) && + props.serverConfig?.environment.capabilities.threadForceStop === true; + const [forceStoppingKey, setForceStoppingKey] = useState(null); + // The run that ignored Stop, so a confirmation left open cannot hit a newer run. + const forceStop = async (runId: RunId | undefined) => { + const threadKey = selectedThreadKey; + setForceStoppingKey(threadKey); + const result = await forceStopThread({ + environmentId: props.environmentId, + input: + runId === undefined + ? { threadId: props.selectedThread.id } + : { threadId: props.selectedThread.id, runId }, + }); + setForceStoppingKey((current) => (current === threadKey ? null : current)); + if (result._tag === "Failure") { + if (!isAtomCommandInterrupted(result)) Dialog.alert("Could not force stop the agent"); + return; + } + const others = result.value.sharedWithThreadIds.length; + if (result.value.outcome === "releasing") { + Dialog.alert("Turn stopped; the agent is still shutting down"); + } + if (result.value.outcome === "shared") { + Dialog.alert( + "Turn stopped; the agent process kept running", + `It is shared with ${others === 0 ? "other" : others} working ${others === 1 ? "thread" : "threads"}. Stop those to free it.`, + ); + } + }; const [statusRequest, setStatusRequest] = useState<{ threadKey: string } | null>(null); const statusPending = statusRequest?.threadKey === selectedThreadKey; const [lifecycleNow, setLifecycleNow] = useState(() => new Date().toISOString()); @@ -619,6 +669,32 @@ function ThreadDetailBody(props: ThreadDetailScreenProps) { if (connectionStatus !== null && !props.draftMode) { return connectionStatus; } + // Before the request check: an agent stuck on an approval may also ignore Stop. + if (runStop !== null && contentPresentationKind === "ready") { + const unresponsive = runStop.state === "unresponsive"; + return { + kind: "stop", + unresponsive, + label: unresponsive ? "Not responding to Stop" : "Stopping…", + action: + unresponsive && canForceStop + ? { + label: forceStoppingKey === selectedThreadKey ? "Force stopping..." : "Force stop", + accessibilityLabel: "Force stop the agent", + disabled: forceStoppingKey === selectedThreadKey, + onPress: () => + Dialog.confirm({ + title: "Force stop the agent?", + message: + "It has not answered Stop. Force stop ends the turn and kills the agent's processes, unless other working threads share them.", + confirmText: "Force stop", + destructive: true, + onConfirm: () => void forceStop(runStop.runId), + }), + } + : null, + }; + } if (props.activePendingApproval !== null || props.activePendingUserInput !== null) { return null; } diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index da127a29ef9bf..33a57859ff8b3 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -1596,6 +1596,16 @@ function renderFeedEntry( const visibleAttachments = attachments.filter( (attachment) => isImageAttachment(attachment) || !inlineAttachmentIds.has(attachment.id), ); + // Images sent without words show as images, not as a bubble around them. + const mediaOnly = + foldRowId === null && + presentation.attribution !== "guest" && + presentation.text.trim() === "" && + visibleAttachments.length + (entry.pendingMessage?.attachments.length ?? 0) > 0 && + visibleAttachments.every(isImageAttachment) && + (entry.pendingMessage?.attachments ?? []).every( + (attachment) => attachment.type === "image", + ); return ( 0) ? { width: props.duoColumns ? "85%" : props.userBubbleMaxWidth } : null), }} @@ -1640,7 +1650,13 @@ function renderFeedEntry( /> ) : null} {bubbleBodyVisible ? ( - + {entry.pendingMessage?.attachments.map((attachment) => attachment.type === "image" && attachment.uploadedAttachmentId ? ( ) : attachment.type === "image" ? ( @@ -1668,7 +1688,7 @@ function renderFeedEntry( > ) : ( @@ -1689,7 +1709,10 @@ function renderFeedEntry( className={ inlineAttachmentIds.size ? "h-24 w-24 rounded-[14px] bg-user-bubble-foreground/15" - : "aspect-[1.3] w-full rounded-[14px] bg-user-bubble-foreground/15" + : mediaOnly + ? // Matches the bubble's corners; native borders sit inside, so the edge adds no size. + "aspect-[1.3] w-full rounded-[20px] border border-border bg-user-bubble-foreground/15" + : "aspect-[1.3] w-full rounded-[14px] bg-user-bubble-foreground/15" } onPressPreview={props.onPressPreview} /> @@ -1707,7 +1730,7 @@ function renderFeedEntry( })} ) : null} - {message.text.trim().length > 0 ? ( + {presentation.text.trim().length > 0 ? ( diff --git a/apps/mobile/src/features/threads/floating-working-control.tsx b/apps/mobile/src/features/threads/floating-working-control.tsx index 3709cf2affa86..c1440535e9cad 100644 --- a/apps/mobile/src/features/threads/floating-working-control.tsx +++ b/apps/mobile/src/features/threads/floating-working-control.tsx @@ -83,7 +83,9 @@ export function FloatingWorkingControl(props: { const [queueWidth, setQueueWidth] = useState(0); const [actionWidth, setActionWidth] = useState(0); const action = - props.status?.kind === "background" || props.status?.kind === "lifecycle" + props.status?.kind === "background" || + props.status?.kind === "lifecycle" || + props.status?.kind === "stop" ? props.status.action : null; const [agentsWidth, setAgentsWidth] = useState(0); @@ -471,6 +473,30 @@ function FloatingStatusLabel(props: { ); } + if (props.status.kind === "stop") { + return ( + + {props.status.unresponsive ? ( + + ) : ( + + )} + + {props.status.label} + + + ); + } if (props.status.kind === "goal") { return ( ({ + projectRoot: "", + worktreePath: null as string | null, + branches: [] as Array<{ name: string; worktreePath: string | null }>, +})); + +vi.mock("react-native", () => ({ + Platform: { OS: "android" }, + View: ({ children }: { children: ReactNode }) =>
{children}
, + ScrollView: ({ children }: { children: ReactNode }) =>
{children}
, + Pressable: ({ children, disabled }: { children: ReactNode; disabled: boolean }) => ( + + ), + useWindowDimensions: () => ({ height: 800 }), +})); +vi.mock("@react-navigation/native", () => ({ useNavigation: () => ({}) })); +vi.mock("react-native-safe-area-context", () => ({ useSafeAreaInsets: () => ({ bottom: 0 }) })); +vi.mock("../../../components/ui/AndroidScreenHeader", () => ({ AndroidSheetHeader: () => null })); +vi.mock("../../../components/ui/SheetHeader", () => ({ SheetHeader: () => null })); +vi.mock("../../../components/ui/ScreenContent", () => ({ + ScreenContent: ({ children }: { children: ReactNode }) =>
{children}
, +})); +vi.mock("../../../native/StackHeader", () => ({ NativeStackScreenOptions: () => null })); +vi.mock("../../../components/ui/AppText", () => ({ + AppText: ({ children }: { children: ReactNode }) => {children}, + AppTextInput: () => null, +})); +vi.mock("./gitSheetComponents", () => ({ SheetActionButton: () => null })); +vi.mock("../../../state/entities", () => ({})); +vi.mock("../../../state/threads", () => ({})); +vi.mock("../../../state/use-thread-selection", () => ({ + useThreadSelection: () => ({ + selectedThread: { environmentId: "environment-1", worktreePath: checkout.worktreePath }, + selectedThreadProject: { workspaceRoot: checkout.projectRoot }, + }), +})); +vi.mock("../../../state/use-thread-detail", () => ({ + useSelectedThreadWorktreePath: () => checkout.worktreePath, +})); +vi.mock("../../../state/use-selected-thread-git-state", () => ({ + useSelectedThreadGitState: () => ({ + selectedThreadBranches: checkout.branches, + selectedThreadBranchesLoading: false, + gitOperationLabel: null, + }), +})); +vi.mock("../../../state/use-selected-thread-git-actions", () => ({ + useSelectedThreadGitActions: () => ({ canChangeThreadBranch: true }), +})); +vi.mock("../../../state/query", () => ({ useEnvironmentQuery: () => ({ data: null }) })); +vi.mock("../../../state/vcs", () => ({ vcsEnvironment: { status: () => null } })); + +import { GitBranchesSheet } from "./GitBranchesSheet"; + +it("recognizes both the main checkout and a linked worktree as this thread's checkout", () => { + const directory = "/repo"; + const linked = "/repo-linked"; + checkout.projectRoot = directory; + checkout.branches = [ + { name: "feature/available", worktreePath: null }, + { name: "feature/linked", worktreePath: linked }, + { name: "main", worktreePath: directory }, + ]; + (globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + const container = document.createElement("div"); + const root = createRoot(container); + const rows = () => { + act(() => { + root.render( + , + ); + }); + return [...container.querySelectorAll("button")].map((button) => ({ + name: button.querySelectorAll("span")[0]?.textContent, + subtitle: button.querySelectorAll("span")[1]?.textContent, + disabled: button.disabled, + })); + }; + try { + checkout.worktreePath = null; + expect(rows()).toEqual([ + { name: "feature/available", subtitle: "Local branch", disabled: false }, + { name: "feature/linked", subtitle: "Checked out in another worktree", disabled: true }, + { name: "main", subtitle: "Checked out in this thread", disabled: false }, + ]); + + checkout.worktreePath = linked; + expect(rows()).toEqual([ + { name: "feature/available", subtitle: "Local branch", disabled: false }, + { name: "feature/linked", subtitle: "Checked out in this thread", disabled: false }, + { name: "main", subtitle: "Checked out in another worktree", disabled: true }, + ]); + } finally { + act(() => root.unmount()); + } +}); diff --git a/apps/mobile/src/features/threads/git/GitBranchesSheet.tsx b/apps/mobile/src/features/threads/git/GitBranchesSheet.tsx index 4075a49e9774a..dcaf96818bfe0 100644 --- a/apps/mobile/src/features/threads/git/GitBranchesSheet.tsx +++ b/apps/mobile/src/features/threads/git/GitBranchesSheet.tsx @@ -28,7 +28,7 @@ export function GitBranchesSheet(_props: GitBranchesSheetProps) { const insets = useSafeAreaInsets(); const { height: windowHeight } = useWindowDimensions(); const { selectedThread } = useThreadSelection(); - const { selectedThreadCwd, selectedThreadWorktreePath } = useSelectedThreadWorktree(); + const { selectedThreadCwd } = useSelectedThreadWorktree(); const gitState = useSelectedThreadGitState(); const gitActions = useSelectedThreadGitActions(); const { canChangeThreadBranch } = gitActions; @@ -43,7 +43,7 @@ export function GitBranchesSheet(_props: GitBranchesSheetProps) { ); const currentBranchLabel = gitStatus.data?.refName ?? selectedThread?.branch ?? "Detached HEAD"; - const currentWorktreePath = selectedThreadWorktreePath; + const currentWorktreePath = selectedThreadCwd; const availableBranches = gitState.selectedThreadBranches; const branchesLoading = gitState.selectedThreadBranchesLoading; const busy = gitState.gitOperationLabel !== null; diff --git a/apps/mobile/src/features/threads/threadQueueControlPresentation.ts b/apps/mobile/src/features/threads/threadQueueControlPresentation.ts index 444d969176ba2..a62e29e14795d 100644 --- a/apps/mobile/src/features/threads/threadQueueControlPresentation.ts +++ b/apps/mobile/src/features/threads/threadQueueControlPresentation.ts @@ -1,5 +1,6 @@ import type { EnvironmentId, RunId, ThreadId } from "@t3tools/contracts"; import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations"; +import { isAttachmentOnlyBootstrapPrompt } from "@t3tools/shared/composerContextReferences"; export const REMOVE_QUEUED_MESSAGE_ACCESSIBILITY_LABEL = "Remove queued message"; @@ -35,7 +36,9 @@ export function resolveThreadQueueRowControls(input: { canSend: mutationEnabled && (input.canPromoteToSteer || input.canSendNow) && !isEditing, dismissAccessibilityLabel: REMOVE_QUEUED_MESSAGE_ACCESSIBILITY_LABEL, // Editing and sending keep the original text so quoted citations stay linked. - displayText: assistantCitationsToPlainText(input.text), + displayText: isAttachmentOnlyBootstrapPrompt(input.text) + ? "" + : assistantCitationsToPlainText(input.text), isEditing, }; } diff --git a/apps/mobile/src/features/workspace/WorkspaceNativeTabs.ios.tsx b/apps/mobile/src/features/workspace/WorkspaceNativeTabs.ios.tsx index 929660587d731..e93175677fc02 100644 --- a/apps/mobile/src/features/workspace/WorkspaceNativeTabs.ios.tsx +++ b/apps/mobile/src/features/workspace/WorkspaceNativeTabs.ios.tsx @@ -18,7 +18,11 @@ import { NativeWorkspaceTabActivityContext, WorkspaceSlotOptionsContext, } from "../../native/v5-workspace-context"; -import { useCompactWorkspaceTabs, useSystemWorkspaceTabBar } from "./useSystemWorkspaceTabBar"; +import { + useCompactWorkspaceTabs, + useFloatingWorkspaceTabs, + useSystemWorkspaceTabBar, +} from "./useSystemWorkspaceTabBar"; import { useControlTheme } from "../../components/ui/useControlTheme"; import { useDeviceType } from "../../native/native-layout-metrics"; import { WorkspaceSidebarTabs } from "./WorkspaceSidebarTabs"; @@ -80,7 +84,7 @@ export function WorkspaceNativeTabs(props: { const deviceType = useDeviceType(); const compactTabs = useCompactWorkspaceTabs(); const { layout, startNewTask } = useAdaptiveWorkspaceLayout(); - const compactSidebarTabs = deviceType === "phone" || !layout.usesSplitView; + const floatingTabs = useFloatingWorkspaceTabs(); const { width } = useWindowDimensions(); const { colors } = useControlTheme(); const selectedTab = workspace.state.selectedTab; @@ -158,17 +162,17 @@ export function WorkspaceNativeTabs(props: { bottom: 0, // UIKit can shrink the sidebar to 280pt independently of its preferred width. width: - deviceType === "pad" + deviceType === "pad" && (floatingTabs || layout.usesSplitView) ? Math.min(280, layout.listPaneWidth ?? width, width) : (layout.listPaneWidth ?? width), height: 128, }} > (null); // The Settings list takes the selected tab's place, so search follows it. const selectedTab = workspace.settingsOpen ? "settings" : workspace.state.selectedTab; @@ -189,9 +189,9 @@ export function WorkspaceToolbar(props: { {composeButton} {selectedTab !== "threads" || isDuoDevice(deviceType) || - deviceType === "pad" || + (deviceType === "pad" && layout.usesSplitView) || sidebarTabs === "picker" || - compactTabs ? null : ( + floatingTabs ? null : ( { + const mcpApp = { + attachmentId: "deferred-app-html", + server: "github", + tool: "create_pull_request", + resourceUri: "ui://github-mcp-server/pr-create", + }; + const item: OrchestrationV2TurnItem = { + ...base("deferred-app", "2026-08-29T00:00:00.000Z", 1), + type: "dynamic_tool", + toolName: "github.create_pull_request", + status: "failed", + input: {}, + output: { t3McpApp: mcpApp, result: { content: [], isError: true } }, + }; + expect(buildThreadFeed([projected(item, 0)])).toMatchObject([{ type: "mcp-app", app: mcpApp }]); +}); + it("keeps historical plan detail accessible from its paged turn item", () => { const item = { ...base("historical-plan", "2026-08-29T00:00:00.000Z", 1), @@ -828,7 +846,7 @@ describe("buildThreadFeed", () => { expect(collapsed[1]).toMatchObject({ type: "run-fold", createdAt: "2026-06-20T00:00:01.500Z", - label: "Worked for 2.0s", + label: "Worked for 2s", }); const expanded = deriveThreadFeedPresentation(feed, latestRun, new Set([runId])); @@ -1082,10 +1100,10 @@ describe("buildThreadFeed", () => { const settled = deriveThreadFeedPresentation(feed(false), null, new Set()); expect(shape(settled)).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:launch-answer", "user:resume", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:resume-answer", ]); const launchFold = settled.find((entry) => entry.type === "run-fold"); @@ -1094,11 +1112,11 @@ describe("buildThreadFeed", () => { shape(deriveThreadFeedPresentation(feed(false), null, new Set([launchFold.runId]))), ).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "activity-group", "assistant:launch-answer", "user:resume", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:resume-answer", ]); @@ -1115,7 +1133,7 @@ describe("buildThreadFeed", () => { ), ).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:launch-answer", "user:resume", "activity-group", diff --git a/apps/mobile/src/lib/threadActivity.ts b/apps/mobile/src/lib/threadActivity.ts index 4cc5a842c21f9..bd7af44b2fa07 100644 --- a/apps/mobile/src/lib/threadActivity.ts +++ b/apps/mobile/src/lib/threadActivity.ts @@ -2102,10 +2102,7 @@ export function buildThreadFeed( entryRows.push(index); continue; } - const app = - item.type === "dynamic_tool" && item.status === "completed" - ? mcpAppFromToolItem(item) - : undefined; + const app = item.type === "dynamic_tool" ? mcpAppFromToolItem(item) : undefined; if (app) { const entry: RawThreadFeedEntry = { type: "mcp-app", diff --git a/apps/mobile/src/state/queued-run-edit.test.ts b/apps/mobile/src/state/queued-run-edit.test.ts index 361a743ac71b5..6670505ad3988 100644 --- a/apps/mobile/src/state/queued-run-edit.test.ts +++ b/apps/mobile/src/state/queued-run-edit.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vite-plus/test"; +import { describe, expect, it, vi } from "vite-plus/test"; import { Atom } from "effect/reactivity"; import { ChatAttachmentId, @@ -20,7 +20,25 @@ import { type QueuedRunEdit, } from "./queued-run-edit"; import { threadComposerErrorsAtom } from "./thread-composer-error"; -import { getComposerDraftSnapshot, setComposerDraftText } from "./use-composer-drafts"; +import { + flushComposerDrafts, + getComposerDraftSnapshot, + setComposerDraftText, + waitForComposerDraftsLoaded, +} from "./use-composer-drafts"; + +vi.mock("expo-file-system", () => ({ + Directory: class { + create() {} + }, + File: class { + exists = false; + create() {} + write() {} + moveSync() {} + }, + Paths: { document: { uri: "file:///documents" } }, +})); const image = (id: string): ChatAttachment => ({ type: "image", @@ -94,6 +112,64 @@ describe("resolveQueuedEditPayload", () => { }); describe("beginQueuedRunEdit", () => { + it("silently drops an untouched edit when its run starts", () => { + const threadKey = "env-exit:thread-untouched"; + const runId = RunId.make("run-1"); + const runs = Atom.make | null>([ + { id: runId, status: "queued" }, + ]); + beginQueuedRunEdit(threadKey, edit([]), runs); + + appAtomRegistry.set(runs, [{ id: runId, status: "running" }]); + + expect(getQueuedRunEdit(threadKey)).toBeNull(); + expect(getComposerDraftSnapshot(threadKey).text).toBe(""); + expect(getComposerDraftSnapshot(queuedEditDraftKey(threadKey, runId)).text).toBe(""); + expect(appAtomRegistry.get(threadComposerErrorsAtom)[threadKey]?.message).toBeUndefined(); + }); + + it("recovers text after a server attachment was removed", async () => { + await waitForComposerDraftsLoaded(); + const threadKey = "env-exit:thread-removed-recover"; + const runId = RunId.make("run-1"); + const runs = Atom.make | null>([ + { id: runId, status: "queued" }, + ]); + beginQueuedRunEdit(threadKey, edit([image("removed")]), runs); + removeQueuedRunEditAttachment(threadKey, "removed"); + + appAtomRegistry.set(runs, [{ id: runId, status: "running" }]); + await flushComposerDrafts(); + + expect(getQueuedRunEdit(threadKey)).toBeNull(); + expect(getComposerDraftSnapshot(threadKey).text).toBe("original"); + expect(getComposerDraftSnapshot(queuedEditDraftKey(threadKey, runId)).text).toBe(""); + expect(appAtomRegistry.get(threadComposerErrorsAtom)[threadKey]?.message).toBe( + "That message already started. Your edit is back in the composer.", + ); + }); + + it("recovers changed text into an empty thread draft", async () => { + await waitForComposerDraftsLoaded(); + const threadKey = "env-exit:thread-changed-recover"; + const runId = RunId.make("run-1"); + const runs = Atom.make | null>([ + { id: runId, status: "queued" }, + ]); + beginQueuedRunEdit(threadKey, edit([]), runs); + setComposerDraftText(queuedEditDraftKey(threadKey, runId), "revised"); + + appAtomRegistry.set(runs, [{ id: runId, status: "running" }]); + await flushComposerDrafts(); + + expect(getQueuedRunEdit(threadKey)).toBeNull(); + expect(getComposerDraftSnapshot(threadKey).text).toBe("revised"); + expect(getComposerDraftSnapshot(queuedEditDraftKey(threadKey, runId)).text).toBe(""); + expect(appAtomRegistry.get(threadComposerErrorsAtom)[threadKey]?.message).toBe( + "That message already started. Your edit is back in the composer.", + ); + }); + it("ends a changed edit when its run starts, without the thread's composer mounted", () => { const threadKey = "env-exit:thread-exit"; const runId = RunId.make("run-1"); diff --git a/apps/mobile/vite.config.ts b/apps/mobile/vite.config.ts index 6a92a025778d9..9ddb84efbc832 100644 --- a/apps/mobile/vite.config.ts +++ b/apps/mobile/vite.config.ts @@ -4,6 +4,7 @@ import { defineConfig, normalizePath } from "vite-plus"; import "vite-plus/test/config"; const reactTests = [ + "src/features/threads/git/GitBranchesSheet.occupancy.test.tsx", "src/features/files/useFileTreeEntries.test.tsx", "src/features/projects/AddProjectScreen.test.ts", "src/lib/useMediaVideoPlayback.test.tsx", diff --git a/apps/server/integration/transferBudgetV2.integration.test.ts b/apps/server/integration/transferBudgetV2.integration.test.ts index 22db0e643db9b..68a622c541fe9 100644 --- a/apps/server/integration/transferBudgetV2.integration.test.ts +++ b/apps/server/integration/transferBudgetV2.integration.test.ts @@ -117,6 +117,7 @@ const layerManagement = Layer.unwrap( .pipe(Effect.orDie), getThreadShell: (id) => projections.getThreadShell(id).pipe(Effect.orDie), getShellSnapshot: (options) => projections.getShellSnapshot(options).pipe(Effect.orDie), + listActiveShellIndex: () => projections.listActiveShellIndex().pipe(Effect.orDie), listActiveSubagentThreadIds: () => projections.listActiveSubagentThreadIds().pipe(Effect.orDie), readShellSnapshot: (options) => diff --git a/apps/server/package.json b/apps/server/package.json index b48725ff9d170..42893866df867 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -78,6 +78,11 @@ "@opencode-ai/sdk": "^1.3.15", "@t3tools/contracts": "workspace:*", "@t3tools/provider-testing": "workspace:*", + "@t3tools/secret-provider-aws": "workspace:*", + "@t3tools/secret-provider-core": "workspace:*", + "@t3tools/secret-provider-definitions": "workspace:*", + "@t3tools/secret-provider-onepassword": "workspace:*", + "@t3tools/secret-provider-openbao": "workspace:*", "@t3tools/shared": "workspace:*", "@t3tools/source-control-testing": "workspace:*", "@t3tools/ssh": "workspace:*", diff --git a/apps/server/src/auth/EnvironmentAuth.test.ts b/apps/server/src/auth/EnvironmentAuth.test.ts index 8558a6664c8a6..bd5d3b569958f 100644 --- a/apps/server/src/auth/EnvironmentAuth.test.ts +++ b/apps/server/src/auth/EnvironmentAuth.test.ts @@ -377,6 +377,35 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { }).pipe(Effect.provide(layerEnvironmentAuth())), ); + it.effect( + "preserves a rejected MCP pairing code for exactly one concurrent lower-access retry", + () => + Effect.gen(function* () { + const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const pairing = yield* serverAuth.issuePairingCredential({ + scopes: ["orchestration:read"], + }); + + const rejected = yield* Effect.flip( + serverAuth.consumeMcpApprovalCode(pairing.credential, "full-access"), + ); + expect(rejected).toMatchObject({ + _tag: "ServerAuthMcpApprovalCodeError", + reason: "insufficient_scope", + }); + + const retry = Effect.result( + serverAuth.consumeMcpApprovalCode(pairing.credential, "read-only"), + ); + const results = yield* Effect.all([retry, retry], { concurrency: "unbounded" }); + + expect(results.filter((result) => result._tag === "Success")).toHaveLength(1); + expect(results.filter((result) => result._tag === "Failure")).toMatchObject([ + { failure: { _tag: "ServerAuthMcpApprovalCodeError", reason: "unknown_or_used" } }, + ]); + }).pipe(Effect.provide(layerEnvironmentAuth())), + ); + it.effect("narrows seeded desktop grants to the requested scopes", () => Effect.gen(function* () { const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; diff --git a/apps/server/src/auth/EnvironmentAuth.ts b/apps/server/src/auth/EnvironmentAuth.ts index 3fbe5a0061c0e..7d73e65925228 100644 --- a/apps/server/src/auth/EnvironmentAuth.ts +++ b/apps/server/src/auth/EnvironmentAuth.ts @@ -429,7 +429,7 @@ export class ServerAuthMcpApprovalCodeError extends Schema.TaggedError; + readonly requiredScopes?: ReadonlyArray; }, ): Effect.Effect< ResolvedBootstrapGrant, @@ -920,6 +921,12 @@ export const make = Effect.gen(function* () { } return sessions.verify(credential).pipe( mapSessionVerificationErrors, + Effect.filterOrFail( + (session) => + input?.requiredScopes === undefined || + input.requiredScopes.every((scope) => session.scopes.includes(scope)), + () => new ServerAuthScopeNotGrantedError({}), + ), Effect.map( (session) => ({ @@ -1340,7 +1347,7 @@ export const make = Effect.gen(function* () { code, access, ) => - resolveBootstrapGrant(code.trim()).pipe( + resolveBootstrapGrant(code.trim(), { requiredScopes: mcpClientScopes(access) }).pipe( Effect.catchTags({ ServerAuthInvalidCredentialError: () => Effect.fail(new ServerAuthMcpApprovalCodeError({ reason: "unknown_or_used" })), @@ -1350,9 +1357,7 @@ export const make = Effect.gen(function* () { Effect.flatMap((grant) => grant.method !== "one-time-token" && grant.method !== "reusable-dev-token" ? Effect.fail(new ServerAuthMcpApprovalCodeError({ reason: "not_a_pairing_code" })) - : mcpClientScopes(access).every((scope) => grant.scopes.includes(scope)) - ? Effect.void - : Effect.fail(new ServerAuthMcpApprovalCodeError({ reason: "insufficient_scope" })), + : Effect.void, ), Effect.withSpan("EnvironmentAuth.consumeMcpApprovalCode"), ); diff --git a/apps/server/src/auth/McpOAuth.test.ts b/apps/server/src/auth/McpOAuth.test.ts index 3dc35aca981b9..29588040217ae 100644 --- a/apps/server/src/auth/McpOAuth.test.ts +++ b/apps/server/src/auth/McpOAuth.test.ts @@ -457,11 +457,10 @@ it.live( }); expect(operatorFull.status).toBe(400); - const readOnlyCode = yield* auth.issuePairingCredential({ scopes: ["orchestration:read"] }); const readOnlyApproval = yield* decide(handler, params, { _tag: "pairing-code", access: "read-only", - code: readOnlyCode.credential, + code: readOnly.credential, }); expect(readOnlyApproval.status).toBe(200); const readOnlyToken = yield* handler( diff --git a/apps/server/src/auth/PairingGrantStore.ts b/apps/server/src/auth/PairingGrantStore.ts index 440dbe3788a45..0709f371684fb 100644 --- a/apps/server/src/auth/PairingGrantStore.ts +++ b/apps/server/src/auth/PairingGrantStore.ts @@ -226,6 +226,8 @@ export class PairingGrantStore extends Context.Service< input?: { readonly proofKeyThumbprint?: string; readonly requestedScopes?: ReadonlyArray; + /** All scopes must be granted; requestedScopes only requires an overlap for OAuth. */ + readonly requiredScopes?: ReadonlyArray; }, ) => Effect.Effect; } @@ -486,8 +488,10 @@ const make = Effect.gen(function* () { } if ( - input?.requestedScopes !== undefined && - !input.requestedScopes.some((scope) => grant.scopes.includes(scope)) + (input?.requestedScopes !== undefined && + !input.requestedScopes.some((scope) => grant.scopes.includes(scope))) || + (input?.requiredScopes !== undefined && + !input.requiredScopes.every((scope) => grant.scopes.includes(scope))) ) { return [ { @@ -550,6 +554,7 @@ const make = Effect.gen(function* () { ...(input?.requestedScopes !== undefined ? { requestedScopes: input.requestedScopes } : {}), + ...(input?.requiredScopes !== undefined ? { requiredScopes: input.requiredScopes } : {}), consumedAt: now, now, }) @@ -596,8 +601,10 @@ const make = Effect.gen(function* () { } if ( - input?.requestedScopes !== undefined && - !input.requestedScopes.some((scope) => matching.value.scopes.includes(scope)) + (input?.requestedScopes !== undefined && + !input.requestedScopes.some((scope) => matching.value.scopes.includes(scope))) || + (input?.requiredScopes !== undefined && + !input.requiredScopes.every((scope) => matching.value.scopes.includes(scope))) ) { return yield* new BootstrapCredentialScopeNotGrantedError({}); } diff --git a/apps/server/src/auth/RpcAuthorization.test.ts b/apps/server/src/auth/RpcAuthorization.test.ts index 24c22280175cc..8044740a7e2ee 100644 --- a/apps/server/src/auth/RpcAuthorization.test.ts +++ b/apps/server/src/auth/RpcAuthorization.test.ts @@ -5,6 +5,7 @@ import { AuthProvidersManageScope, AuthSettingsWriteScope, DEFAULT_SERVER_SETTINGS, + ProviderDriverKind, ProviderInstanceId, SecretProviderId, ThreadId, @@ -164,13 +165,34 @@ describe("settings mutation authorization", () => { ); yield* client[WS_METHODS.serverUpdateSettings]({ patch: {}, providerInstanceMutation }); expect(handled).toBe(1); + yield* client[WS_METHODS.serverUpdateSettings]({ + patch: {}, + providerInstanceMutation: { + operation: "upsert", + instanceId: ProviderInstanceId.make("codex_work"), + instance: { + driver: ProviderDriverKind.make("codex"), + environment: [ + { + name: "OPENAI_API_KEY", + sensitive: true, + value: { + provider: SecretProviderId.make("personal"), + reference: "op://Private/OpenAI/credential", + }, + }, + ], + }, + }, + }); + expect(handled).toBe(2); expect( yield* client[WS_METHODS.serverUpdateSettings]({ patch: { defaultRuntimeMode: "full-access" }, providerInstanceMutation, }).pipe(Effect.flip), ).toMatchObject({ requiredPermission: AuthSettingsWriteScope }); - expect(handled).toBe(1); + expect(handled).toBe(2); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index bcd4b16b74c2c..9e20a50e60b7c 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -42,6 +42,7 @@ export const RPC_REQUIRED_SCOPES = { [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: AuthOrchestrationOperateScope, [ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: AuthOrchestrationOperateScope, + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: AuthOrchestrationOperateScope, [ORCHESTRATION_V2_WS_METHODS.getTurnDiff]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.getFullThreadDiff]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.searchThreads]: AuthOrchestrationReadScope, @@ -170,6 +171,7 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.vcsRefreshStatus]: AuthOrchestrationReadScope, [WS_METHODS.gitResolvePullRequest]: AuthOrchestrationReadScope, [WS_METHODS.vcsListRefs]: AuthOrchestrationReadScope, + [WS_METHODS.reviewGetBranchCommits]: AuthFilesystemReadScope, [WS_METHODS.reviewGetDiffPreview]: AuthFilesystemReadScope, [WS_METHODS.reviewGetDiffFileContents]: AuthFilesystemReadScope, [WS_METHODS.terminalOpen]: AuthTerminalOperateScope, @@ -228,7 +230,8 @@ const SettingsUpdate = Schema.Struct({ patch: ServerSettingsPatch, providerInstanceMutation: Schema.optionalKey(ProviderInstanceMutation), }); -const decodeSettingsUpdate = Schema.decodeUnknownSync(SettingsUpdate); +// Middleware receives the payload already decoded, so it is checked against the decoded shape. +const decodeSettingsUpdate = Schema.decodeUnknownSync(Schema.toType(SettingsUpdate)); const requiredScopesForSettingsUpdate = (payload: unknown) => { const input = decodeSettingsUpdate(payload); diff --git a/apps/server/src/cli/auth.ts b/apps/server/src/cli/auth.ts index 0aeb7d6a3b5ab..06b949e3013ab 100644 --- a/apps/server/src/cli/auth.ts +++ b/apps/server/src/cli/auth.ts @@ -4,6 +4,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as References from "effect/References"; +import * as Schema from "effect/Schema"; import { Argument, Command, Flag, GlobalFlag } from "effect/cli"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; @@ -68,6 +69,15 @@ const subjectFlag = Flag.String("subject").pipe( ); const baseUrlFlag = Flag.String("base-url").pipe( + Flag.withSchema( + Schema.URLFromString.pipe( + Schema.check( + Schema.makeFilter((url) => url.protocol === "http:" || url.protocol === "https:", { + expected: "an http:// or https:// URL", + }), + ), + ), + ), Flag.withDescription("Optional public base URL used to print a ready `/pair#token=...` link."), Flag.optional, ); @@ -99,7 +109,7 @@ const pairingCreateCommand = Command.make("create", { }); const output = formatIssuedPairingCredential(issued, { json: flags.json, - ...(Option.isSome(flags.baseUrl) ? { baseUrl: flags.baseUrl.value } : {}), + ...(Option.isSome(flags.baseUrl) ? { baseUrl: flags.baseUrl.value.href } : {}), }); yield* Console.log(output); }), diff --git a/apps/server/src/cli/pair.test.ts b/apps/server/src/cli/pair.test.ts index 5ad9bde0acc28..88fc9cec74c13 100644 --- a/apps/server/src/cli/pair.test.ts +++ b/apps/server/src/cli/pair.test.ts @@ -488,3 +488,63 @@ describe("auth scope options", () => { }), ); }); + +describe("auth pairing create --base-url", () => { + it.effect.each(["192.168.1.5:3773", "localhost:3773", "ftp://localhost:3773"])( + "rejects %s before issuing a token", + (baseUrl) => + Effect.gen(function* () { + const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-cli-base-url-test-")); + const error = yield* runCli([ + "auth", + "pairing", + "create", + "--base-dir", + baseDir, + "--base-url", + baseUrl, + ]).pipe(Effect.provide(layerCliRuntime), Effect.flip); + + if (!CliError.isCliError(error) || error._tag !== "ShowHelp") { + assert.fail(`Expected ShowHelp, got ${String(error)}`); + } + const flagError = error.errors[0]; + if (flagError?._tag !== "InvalidValue") { + assert.fail(`Expected InvalidValue, got ${String(flagError?._tag)}`); + } + assert.equal(flagError.option, "base-url"); + assert.equal(flagError.value, baseUrl); + + const listOutput = yield* captureStdout( + runCli(["auth", "pairing", "list", "--base-dir", baseDir, "--json"]), + ); + assert.deepEqual(JSON.parse(listOutput), []); + }), + ); + + it.effect.each(["http://192.168.1.5:3773", "https://example.com", "http:localhost:3773"])( + "prints a pair URL for %s", + (baseUrl) => + Effect.gen(function* () { + const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-cli-base-url-test-")); + const output = yield* captureStdout( + runCli([ + "auth", + "pairing", + "create", + "--base-dir", + baseDir, + "--base-url", + baseUrl, + "--json", + ]), + ); + const issued = JSON.parse(output) as { + readonly credential: string; + readonly pairUrl: string; + }; + + assert.equal(issued.pairUrl, `${new URL(baseUrl).origin}/pair#token=${issued.credential}`); + }), + ); +}); diff --git a/apps/server/src/cli/project.test.ts b/apps/server/src/cli/project.test.ts index 40a7ab7113839..723f5254e7a2c 100644 --- a/apps/server/src/cli/project.test.ts +++ b/apps/server/src/cli/project.test.ts @@ -18,11 +18,14 @@ import { import * as NetService from "@t3tools/shared/Net"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as References from "effect/References"; import * as Stream from "effect/Stream"; import { Command } from "effect/cli"; +import { FetchHttpClient } from "effect/http"; +import * as TestClock from "effect/testing/TestClock"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { cli } from "../binCli.ts"; @@ -37,6 +40,10 @@ import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; import * as T3ProjectFileLoader from "../project/T3ProjectFileLoader.ts"; +import { + makePersistedServerRuntimeState, + persistServerRuntimeState, +} from "../serverRuntimeState.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; @@ -121,6 +128,66 @@ it.effect("adds, renames, and removes projects through the V2 project CLI domain }).pipe(Effect.provide(NodeServices.layer)), ); +it.effect.each(["snapshot", "mutation"] as const)( + "waits for a busy live server's %s response", + (slowRequest) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-live-project-cli-" }); + const baseDir = NodePath.join(root, "state"); + const workspaceRoot = NodePath.join(root, "workspace"); + yield* fs.makeDirectory(workspaceRoot); + yield* persistServerRuntimeState({ + path: NodePath.join(baseDir, "userdata", "server-runtime.json"), + state: yield* makePersistedServerRuntimeState({ + config: { host: "127.0.0.1", devUrl: undefined }, + port: 3_773, + }), + }); + const now = "2026-01-01T00:00:00.000Z"; + const requestReceived = Promise.withResolvers(); + const releaseResponse = Promise.withResolvers(); + let snapshotRequests = 0; + let mutations = 0; + const fetch: typeof globalThis.fetch = async (_input, init) => { + const isMutation = init?.method === "POST"; + if (isMutation) mutations++; + else snapshotRequests++; + if (slowRequest === "mutation" ? isMutation : !isMutation && snapshotRequests === 2) { + requestReceived.resolve(); + await releaseResponse.promise; + } + return Response.json( + isMutation + ? { + id: "live-project", + title: NodePath.basename(workspaceRoot), + workspaceRoot, + defaultModelSelection: null, + scripts: [], + createdAt: now, + updatedAt: now, + deletedAt: null, + } + : { projects: [], updatedAt: now }, + ); + }; + + const add = yield* runCli(["project", "add", workspaceRoot, "--base-dir", baseDir]).pipe( + Effect.provideService(FetchHttpClient.Fetch, fetch), + Effect.forkChild, + ); + yield* Effect.promise(() => requestReceived.promise); + yield* TestClock.adjust("2 seconds"); + releaseResponse.resolve(); + yield* Fiber.join(add); + + assert.equal(snapshotRequests, 2); + assert.equal(mutations, 1); + assert.deepEqual((yield* readProjects(baseDir)).projects, []); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), +); + const makeProjectLookupFixture = Effect.fn("ProjectCliTest.makeProjectLookupFixture")(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-v2-project-lookup-" }); diff --git a/apps/server/src/cli/project.ts b/apps/server/src/cli/project.ts index a5c8aa532ff3e..3d02f9c921fce 100644 --- a/apps/server/src/cli/project.ts +++ b/apps/server/src/cli/project.ts @@ -214,7 +214,8 @@ const layerProjectCliRuntime = RuntimeLayer.layerProjectService.pipe( Layer.provideMerge(SqlitePersistence.layerConfig), ); -const PROJECT_CLI_LIVE_SERVER_TIMEOUT = Duration.seconds(1); +const PROJECT_CLI_LIVE_SERVER_PROBE_TIMEOUT = Duration.seconds(1); +const PROJECT_CLI_LIVE_SERVER_REQUEST_TIMEOUT = Duration.seconds(10); const withProjectCliSessionToken = ( environmentAuth: EnvironmentAuth.EnvironmentAuth["Service"], run: (token: string) => Effect.Effect, @@ -228,9 +229,6 @@ const withProjectCliSessionToken = ( (issued) => environmentAuth.revokeSession(issued.sessionId).pipe(Effect.ignore({ log: true })), ); -const withProjectCliLiveServerTimeout = (effect: Effect.Effect) => - effect.pipe(Effect.timeout(PROJECT_CLI_LIVE_SERVER_TIMEOUT)); - const makeLiveServerClient = (origin: string) => HttpApiClient.make(EnvironmentHttpApi, { baseUrl: origin, @@ -317,17 +315,14 @@ const findActiveProjectTarget = Effect.fn("findActiveProjectTarget")(function* ( }); const fetchLiveOrchestrationSnapshot = Effect.fn("project.fetchLiveOrchestrationSnapshot")( - function* (origin: string, bearerToken: string) { + function* (origin: string, bearerToken: string, _timeout?: Duration.Duration) { const client = yield* makeLiveServerClient(origin); return yield* client.projects.snapshot({ headers: { authorization: `Bearer ${bearerToken}` }, }); }, - (effect, _origin, _bearerToken) => - effect.pipe( - withProjectCliLiveServerTimeout, - Effect.mapError(projectCommandErrorFromLiveServerRequest), - ), + (effect, _origin, _bearerToken, timeout = PROJECT_CLI_LIVE_SERVER_REQUEST_TIMEOUT) => + effect.pipe(Effect.timeout(timeout), Effect.mapError(projectCommandErrorFromLiveServerRequest)), ); const dispatchLiveOrchestrationCommand = Effect.fn("project.dispatchLiveOrchestrationCommand")( @@ -340,7 +335,7 @@ const dispatchLiveOrchestrationCommand = Effect.fn("project.dispatchLiveOrchestr }, (effect, _origin, _bearerToken, _command) => effect.pipe( - withProjectCliLiveServerTimeout, + Effect.timeout(PROJECT_CLI_LIVE_SERVER_REQUEST_TIMEOUT), Effect.mapError(projectCommandErrorFromLiveServerRequest), ), ); @@ -361,7 +356,11 @@ const tryResolveLiveProjectExecutionMode = Effect.fn("tryResolveLiveProjectExecu } const attempt = withProjectCliSessionToken(environmentAuth, (token) => - fetchLiveOrchestrationSnapshot(runtimeState.value.origin, token).pipe( + fetchLiveOrchestrationSnapshot( + runtimeState.value.origin, + token, + PROJECT_CLI_LIVE_SERVER_PROBE_TIMEOUT, + ).pipe( Effect.as({ origin: runtimeState.value.origin, }), diff --git a/apps/server/src/cli/service.test.ts b/apps/server/src/cli/service.test.ts index 033a96693ada9..2ea799d9f09e4 100644 --- a/apps/server/src/cli/service.test.ts +++ b/apps/server/src/cli/service.test.ts @@ -90,6 +90,37 @@ it.layer(Layer.mergeAll(NodeServices.layer, NetService.layer))("service commands }), ); + it.effect("protect refuses root before creating any T3 directories", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const parent = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-cli-test-" }); + const baseDir = `${parent}/t3`; + vi.spyOn(BootService, "layer").mockReturnValue( + Layer.succeed(BootService.BootService, makeTestService(status).service), + ); + + const output = yield* Command.runWith(serviceCommand, { version: packageJson.version })([ + "protect", + "--base-dir", + baseDir, + ]).pipe( + Effect.andThen(TestConsole.logLines), + Effect.map((lines) => lines.join("\n")), + Effect.provideService(HostProcess.Environment, {}), + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HostProcess.UserId, 0), + Effect.provide( + Layer.mergeAll( + ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })), + Layer.fresh(TestConsole.layer), + ), + ), + ); + expect(output).toContain("without sudo"); + expect(yield* fs.exists(baseDir)).toBe(false); + }), + ); + // regression: #13239 a stale nvm-installed CLI reported "already using" its own version it.effect.each(["install", "update"] as const)( "%s names the CLI that found nothing to change", diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index 97ceafd941f1e..884cd28c9df87 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -2,12 +2,14 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Console from "effect/Console"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; import * as Terminal from "effect/Terminal"; import { Command, Flag, GlobalFlag, Prompt } from "effect/cli"; import { FetchHttpClient } from "effect/http"; import packageJson from "../../package.json" with { type: "json" }; import * as BootService from "../cloud/bootService.ts"; +import * as ServiceProtection from "../cloud/serviceProtection.ts"; import { compareExactServiceVersions } from "../cloud/serviceProtocol.ts"; import type * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; @@ -68,7 +70,23 @@ function formatServiceChange( return `${result.previouslyInstalled ? "Updated" : "Installed"} T3 Code service with t3@${packageJson.version}.\nLogs: ${result.plan.logPath}`; } -function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion: string): string { +function formatProtection( + protection: ServiceProtection.ServiceProtectionState | undefined, +): ReadonlyArray { + if (protection === undefined) return []; + return [ + protection.unprotected === undefined + ? ` Memory protection: on${protection.setByProtect ? " (t3 service protect)" : ""}` + : ` Memory protection: off, ${protection.unprotected} grants none. Run \`t3 service protect\`.`, + ...(protection.oomScoreAdj === undefined ? [] : [` OOM score: ${protection.oomScoreAdj}`]), + ]; +} + +function formatServiceStatus( + status: BootService.BootServiceStatus, + cliVersion: string, + protection?: ServiceProtection.ServiceProtectionState, +): string { if (!status.supported) { return "T3 Code service\n Status: unavailable on this machine\n Supported on: Linux with systemd, macOS with launchd"; } @@ -89,6 +107,7 @@ function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion: ` Status: installed · t3@${installedVersion} (newer than this t3@${cliVersion} CLI)`, ` Unit: ${status.unitPath}`, ` Logs: ${status.logPath}`, + ...formatProtection(protection), ...problems, ` Next: Run \`t3 update ${installedVersion}\` to match it, or pass \`--allow-downgrade\` to \`t3 service install\` explicitly.`, ].join("\n"); @@ -98,6 +117,7 @@ function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion: ` Status: ${status.current ? `installed · t3@${installedVersion}` : "needs an update or repair"}`, ` Unit: ${status.unitPath}`, ` Logs: ${status.logPath}`, + ...formatProtection(protection), ...problems, ...(status.current ? [] : [" Next: Run `t3 service install` to repair it."]), ].join("\n"); @@ -210,12 +230,129 @@ const serviceStatusCommand = Command.make("status", projectLocationFlags).pipe( flags, Effect.gen(function* () { const service = yield* BootService.BootService; - yield* Console.log(formatServiceStatus(yield* service.status, packageJson.version)); + const path = yield* Path.Path; + const status = yield* service.status; + const protection = + status.installed && (yield* HostProcess.Platform) === "linux" + ? yield* ServiceProtection.readProtection(path.dirname(status.unitPath)).pipe( + Effect.provide(ProcessRunner.layer), + ) + : undefined; + yield* Console.log(formatServiceStatus(status, packageJson.version, protection)); }), ), ), ); +const formatDropIns = (dropIns: ReadonlyArray) => + dropIns.map( + ({ path, contents }) => ` ${path}: ${contents.trimEnd().split("\n").slice(1).join(" ")}`, + ); + +const confirm = (yes: boolean, message: string) => + yes ? Effect.succeed(true) : Prompt.run(Prompt.Confirm({ message, initial: false })); + +const serviceProtectCommand = Command.make("protect", { + ...projectLocationFlags, + undo: Flag.Boolean("undo").pipe( + Flag.withDescription("Remove the protection this command set up."), + Flag.withDefault(false), + ), + yes: Flag.Boolean("yes").pipe( + Flag.withDescription("Answer yes to every question, including SSH and Tailscale."), + Flag.withDefault(false), + ), +}).pipe( + Command.withDescription( + "Protect the T3 Code server from agents under memory pressure (Linux with systemd). Asks for sudo.", + ), + Command.withHandler((flags) => + Effect.gen(function* () { + const unsupported = "Service protection needs Linux with systemd."; + if ((yield* HostProcess.Platform) !== "linux") return yield* Console.log(unsupported); + // Before resolving config, which creates T3 directories that root would then own. + if ((yield* HostProcess.UserId) === 0) { + return yield* Console.log( + "Run `t3 service protect` as the user that runs T3 Code, without sudo. It asks for sudo when it needs root.", + ); + } + return yield* runServiceCommand( + flags, + Effect.gen(function* () { + const uid = yield* HostProcess.UserId; + const remote = yield* ServiceProtection.detectRemoteAccessUnits; + const { unitPath } = yield* (yield* BootService.BootService).status; + if (uid === undefined || remote === undefined || unitPath === "") { + return yield* Console.log(unsupported); + } + const userUnitDir = (yield* Path.Path).dirname(unitPath); + + if (flags.undo) { + const installed = yield* ServiceProtection.installedProtection( + ServiceProtection.protectionPlan({ + uid, + userUnitDir, + remoteAccessUnits: ServiceProtection.REMOTE_ACCESS_UNITS, + }), + ); + if (installed === undefined) { + return yield* Console.log("T3 Code service protection is not set up."); + } + yield* Console.log( + [ + "Removes:", + ...[...installed.system, ...installed.user, installed.managerDefaults].map( + ({ path, shared }) => + ` ${path}${shared === true ? " (kept while another user is protected)" : ""}`, + ), + ].join("\n"), + ); + if (!(yield* confirm(flags.yes, "Remove T3 Code service protection?"))) return; + yield* ServiceProtection.removeProtection(installed); + return yield* Console.log( + "Removed T3 Code service protection. The server's OOM score returns to normal after `t3 service restart`.", + ); + } + + yield* Console.log( + [ + "Keeps T3 Code running when agents fill this machine's memory:", + " - reserves memory for the T3 Code server, so the kernel takes memory from agents first", + " - makes the out-of-memory killer pick agents before the server", + "It adds systemd drop-in files and changes nothing else. Undo with `t3 service protect --undo`.", + ].join("\n"), + ); + const names = remote.map((unit) => unit.replace(/\.service$/, "")).join(" and "); + const protectRemote = + remote.length > 0 && + (yield* confirm( + flags.yes, + `Also protect ${names} so you can still reach this machine under memory pressure?`, + )); + const plan = ServiceProtection.protectionPlan({ + uid, + userUnitDir, + remoteAccessUnits: protectRemote ? remote : [], + }); + yield* Console.log( + [ + "Drop-ins it writes, as root through sudo:", + ...formatDropIns(plan.system), + "As you:", + ...formatDropIns([plan.managerDefaults, ...plan.user]), + ].join("\n"), + ); + if (!(yield* confirm(flags.yes, "Apply these settings?"))) return; + yield* ServiceProtection.applyProtection(plan); + yield* Console.log( + "Protected T3 Code. Memory protection applies now; the server takes its OOM score after `t3 service restart`. Undo with `t3 service protect --undo`.", + ); + }).pipe(Effect.provide(ProcessRunner.layer)), + ); + }), + ), +); + export type ServiceOnboardingOutcome = "ready" | "pending" | "skipped"; export const offerServiceDuringOnboarding = Effect.gen(function* () { @@ -305,6 +442,7 @@ export const serviceCommand = Command.make("service").pipe( serviceRestartCommand, serviceUninstallCommand, serviceStatusCommand, + serviceProtectCommand, serviceUpdateCommand, ]), ); diff --git a/apps/server/src/cli/trace.ts b/apps/server/src/cli/trace.ts index 244636c514061..3e94d0d9957f7 100644 --- a/apps/server/src/cli/trace.ts +++ b/apps/server/src/cli/trace.ts @@ -100,7 +100,7 @@ export function makeTraceSpanSummary(sinceMs = -Infinity) { } const formatMs = (ms: number) => - ms < 1_000 ? `${Math.round(ms)}ms` : `${(ms / 1_000).toFixed(1)}s`; + ms < 1_000 ? `${Math.round(ms)}ms` : `${Math.round(ms / 100) / 10}s`; function formatTraceSummary( summary: ReturnType["finish"]>, @@ -127,7 +127,7 @@ function formatTraceSummary( const widths = header.map((_, column) => Math.max(...table.map((row) => row[column]!.length))); const formatIso = (ms: number) => DateTime.formatIso(DateTime.makeUnsafe(ms)); return [ - `${summary.spanCount} spans ended from ${formatIso(summary.firstEndMs)} to ${formatIso(summary.lastEndMs)} (${summary.minutes.toFixed(1)} min).`, + `${summary.spanCount} spans ended from ${formatIso(summary.firstEndMs)} to ${formatIso(summary.lastEndMs)} (${summary.minutes.toFixed(1).replace(/\.0+$/, "")} min).`, ...(summary.skippedLineCount > 0 ? [`Skipped ${summary.skippedLineCount} lines that are not spans.`] : []), diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index fca51dcac1e09..2edd110648013 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -48,6 +48,25 @@ it("reads the served T3 home back out of a rendered unit or plist", () => { expect(BootService.bootServiceBaseDirOf("[Service]\nExecStart=/x\n")).toBeUndefined(); }); +it("protects the server unit from agents under CPU and memory pressure", () => { + const unit = BootService.renderBootServiceUnit({ + program: ["/home/theo/.t3/runtime/versions/1.2.3/t3", "__service-launcher"], + baseDir: "/home/theo/.t3", + logPath: "/home/theo/.t3/userdata/logs/boot-service.log", + unitPath: "/home/theo/.config/systemd/user/t3code.service", + }); + const service = unit.slice(unit.indexOf("[Service]"), unit.indexOf("[Install]")); + for (const line of [ + "CPUWeight=1000", + "IOWeight=1000", + "MemoryLow=1536M", + "OOMScoreAdjust=100", + "OOMPolicy=continue", + ]) { + expect(service.split("\n")).toContain(line); + } +}); + it.each([ ["a unit", '/home/theo/T3 "Data"/100%\\x', BootService.renderBootServiceUnit], [ diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 15b3b16a7bc0f..948d780c867cc 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -110,9 +110,15 @@ export function renderBootServiceUnit(plan: BootServicePlan): string { "OOMPolicy=continue", "Restart=always", "RestartSec=5", - // Weights favor the server over the sibling agents slice during CPU or disk saturation. + // Weights favor the server over the sibling agents slice; IOWeight needs a delegated io controller. "CPUWeight=1000", "IOWeight=1000", + // Reclaim takes the agents' unprotected pages first. The kernel caps this at the + // protection of user.slice and user@.service, which only root can grant. + "MemoryLow=1536M", + // A user manager's own score, the lowest it may set: below its other services (200) and agents (800). + // `t3 service protect` lowers it further through a drop-in. + "OOMScoreAdjust=100", // systemd-oomd ignores this preference when the watched cgroup is root-owned, as user-1000.slice usually is. "ManagedOOMPreference=avoid", `StandardOutput=append:${escapeSystemdSpecifiers(plan.logPath)}`, diff --git a/apps/server/src/cloud/serviceProtection.test.ts b/apps/server/src/cloud/serviceProtection.test.ts new file mode 100644 index 0000000000000..eef78cd3f82e7 --- /dev/null +++ b/apps/server/src/cloud/serviceProtection.test.ts @@ -0,0 +1,270 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import type * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import * as ProcessRunner from "../processRunner.ts"; +import * as ServiceProtection from "./serviceProtection.ts"; + +const temporaryDirectory = () => + NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-service-protection-")); + +describe.skipIf(HostProcess.Platform.defaultValue() !== "linux")("renderPrivilegedScript", () => { + it("writes and removes only its drop-ins and gives running main processes their score", () => { + const root = temporaryDirectory(); + const target = NodeChildProcess.spawn("sleep", ["60"], { stdio: "ignore" }); + try { + const systemUnitDir = NodePath.join(root, "system"); + const plan = (uid: number, remoteAccessUnits: ReadonlyArray) => + ServiceProtection.protectionPlan({ + uid, + userUnitDir: "/home/theo/.config/systemd/user", + remoteAccessUnits, + systemUnitDir, + }); + const theo = plan(1000, ["tailscaled.service", "ssh.service"]); + expect(theo.system.find((dropIn) => dropIn.unit === "user@1000.service")?.contents).toBe( + "[Service]\nMemoryLow=2G\nOOMScoreAdjust=-900\n", + ); + expect(theo.managerDefaults.path).toBe( + "/home/theo/.config/systemd/user.conf.d/90-t3code-protect.conf", + ); + expect(theo.oomUnits).toEqual(["user@1000.service", "tailscaled.service", "ssh.service"]); + + const unrelated = NodePath.join(systemUnitDir, "ssh.service.d", "50-admin.conf"); + NodeFS.mkdirSync(NodePath.dirname(unrelated), { recursive: true }); + NodeFS.writeFileSync(unrelated, "[Service]\nNice=1\n"); + const bin = NodePath.join(root, "bin"); + NodeFS.mkdirSync(bin); + // Raising a score needs no privilege, so the test's own child stands in for a main process. + NodeFS.writeFileSync( + NodePath.join(bin, "systemctl"), + [ + "#!/bin/sh", + 'echo "$*" >>"$CALLS"', + 'case "$*" in', + ' *MainPID*) echo "$TARGET_PID" ;;', + " *OOMScoreAdjust*) echo 900 ;;", + "esac", + "", + ].join("\n"), + { mode: 0o755 }, + ); + const calls = NodePath.join(root, "calls"); + const runScript = (protection: ServiceProtection.ProtectionPlan, mode: "apply" | "undo") => { + const result = NodeChildProcess.spawnSync( + "/bin/sh", + ["-c", ServiceProtection.renderPrivilegedScript(protection, mode)], + { + encoding: "utf8", + env: { + PATH: `${bin}:/usr/bin:/bin`, + CALLS: calls, + TARGET_PID: String(target.pid), + }, + }, + ); + expect(result.stderr).toBe(""); + expect(result.status).toBe(0); + }; + const exists = (unit: string) => + NodeFS.existsSync(NodePath.join(systemUnitDir, `${unit}.d`, "90-t3code-protect.conf")); + + runScript(theo, "apply"); + runScript(plan(1001, []), "apply"); + for (const { path, contents } of theo.system) { + expect(NodeFS.readFileSync(path, "utf8")).toBe(contents); + } + expect(NodeFS.readFileSync(`/proc/${target.pid}/oom_score_adj`, "utf8").trim()).toBe("900"); + expect(NodeFS.readFileSync(calls, "utf8").split("\n")[0]).toBe("daemon-reload"); + + // Undo removes every drop-in a user could own, and keeps shared ones while another user is protected. + const undo = (uid: number) => plan(uid, ServiceProtection.REMOTE_ACCESS_UNITS); + runScript(undo(1000), "undo"); + expect(["user-1000.slice", "user@1000.service"].some(exists)).toBe(false); + expect( + ["user.slice", "system.slice", "tailscaled.service", "ssh.service"].every(exists), + ).toBe(true); + + runScript(undo(1001), "undo"); + for (const { path } of undo(1001).system) expect(NodeFS.existsSync(path)).toBe(false); + expect(NodeFS.existsSync(NodePath.join(systemUnitDir, "user.slice.d"))).toBe(false); + expect(NodeFS.readFileSync(unrelated, "utf8")).toBe("[Service]\nNice=1\n"); + } finally { + target.kill(); + NodeFS.rmSync(root, { recursive: true, force: true }); + } + }); +}); + +it.each([ + [ + "Debian, where sshd.service aliases ssh.service", + "Id=tailscaled.service\nLoadState=loaded\n\nId=ssh.service\nLoadState=loaded\n\nId=ssh.service\nLoadState=loaded\n", + ["tailscaled.service", "ssh.service"], + ], + [ + "Fedora without Tailscale", + "Id=tailscaled.service\nLoadState=not-found\n\nId=ssh.service\nLoadState=not-found\n\nId=sshd.service\nLoadState=loaded\n", + ["sshd.service"], + ], +])("finds remote-access services on %s", (_, shown, expected) => { + expect(ServiceProtection.loadedRemoteAccessUnits(shown)).toEqual(expected); +}); + +const fakeSudo = (exitCode: number, onRun: (command: ChildProcess.StandardCommand) => void) => + ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (command._tag !== "StandardCommand" || command.command !== "sudo") { + return yield* Effect.die("Unexpected command"); + } + onRun(command); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(123), + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + }), + ); + +const runner = (stdout: string) => + ProcessRunner.ProcessRunner.of({ + run: () => + Effect.succeed({ + stdout, + stderr: "", + code: ChildProcessSpawner.ExitCode(0), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + }), + }); + +it.effect("lowers no score before the manager default is in place, and undoes in reverse", () => { + const home = temporaryDirectory(); + return Effect.gen(function* () { + const plan = ServiceProtection.protectionPlan({ + uid: 1000, + userUnitDir: NodePath.join(home, ".config", "systemd", "user"), + remoteAccessUnits: [], + systemUnitDir: NodePath.join(home, "system"), + }); + const managerDropIn = plan.system.find(({ unit }) => unit === "user@1000.service")!.path; + const present = () => ({ + managerDefaults: NodeFS.existsSync(plan.managerDefaults.path), + user: plan.user.filter(({ path }) => NodeFS.existsSync(path)).length, + }); + const withSudo = ( + effect: Effect.Effect, + exitCode: number, + writesManagerDropIn = false, + ) => { + let atSudo: ReturnType | undefined; + let sudoOptions: ChildProcess.CommandOptions | undefined; + return effect.pipe( + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + fakeSudo(exitCode, (command) => { + atSudo = present(); + sudoOptions = command.options; + if (writesManagerDropIn) { + NodeFS.mkdirSync(NodePath.dirname(managerDropIn), { recursive: true }); + NodeFS.writeFileSync(managerDropIn, "[Service]\nOOMScoreAdjust=-900\n"); + } + }), + ), + Effect.exit, + Effect.map((exit) => ({ exit, atSudo, sudoOptions })), + ); + }; + const refused = yield* withSudo(ServiceProtection.applyProtection(plan), 1); + expect(refused.exit._tag).toBe("Failure"); + expect(refused.atSudo).toEqual({ managerDefaults: true, user: 0 }); + expect(present()).toEqual({ managerDefaults: false, user: 0 }); + + // Failing after it lowered the manager, sudo must leave the manager default in place. + const partial = yield* withSudo(ServiceProtection.applyProtection(plan), 1, true); + expect(partial.exit._tag).toBe("Failure"); + expect(present()).toEqual({ managerDefaults: true, user: 0 }); + NodeFS.rmSync(plan.managerDefaults.path); + NodeFS.rmSync(managerDropIn); + + const applied = yield* withSudo(ServiceProtection.applyProtection(plan), 0); + expect(applied.atSudo).toEqual({ managerDefaults: true, user: 0 }); + // sudo needs the controlling terminal to ask for a password. + expect(applied.sudoOptions).toMatchObject({ detached: false, stdin: "inherit" }); + expect(present()).toEqual({ managerDefaults: true, user: plan.user.length }); + + const removed = yield* withSudo(ServiceProtection.removeProtection(plan), 0); + expect(removed.atSudo).toEqual({ managerDefaults: true, user: 0 }); + expect(present()).toEqual({ managerDefaults: false, user: 0 }); + }).pipe( + Effect.ensuring(Effect.sync(() => NodeFS.rmSync(home, { recursive: true, force: true }))), + Effect.provideService(ProcessRunner.ProcessRunner, runner("")), + Effect.provide(NodeServices.layer), + ); +}); + +it.effect("counts memory.min as protection and names the first cgroup that grants none", () => { + const root = temporaryDirectory(); + const controlGroup = [ + "user.slice", + "user-1000.slice", + "user@1000.service", + "app.slice", + "t3code.service", + ]; + const write = (depth: number, file: string, value: string) => { + const directory = NodePath.join(root, ...controlGroup.slice(0, depth)); + NodeFS.mkdirSync(directory, { recursive: true }); + NodeFS.writeFileSync(NodePath.join(directory, file), `${value}\n`); + }; + write(1, "memory.low", "0"); + write(1, "memory.min", "805306368"); + write(2, "memory.low", "2147483648"); + write(3, "memory.low", "2147483648"); + write(4, "memory.low", "0"); + write(4, "memory.min", "0"); + write(5, "memory.low", "1610612736"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const read = ServiceProtection.readProtection("/home/theo/.config/systemd/user").pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + readFileString: (path, encoding) => + fs.readFileString(path.replace("/sys/fs/cgroup", root), encoding), + }), + ); + expect((yield* read)?.unprotected).toBe("app.slice"); + write(4, "memory.low", "1610612736"); + expect((yield* read)?.unprotected).toBeUndefined(); + }).pipe( + Effect.ensuring(Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true }))), + Effect.provideService( + ProcessRunner.ProcessRunner, + runner(`ControlGroup=/${controlGroup.join("/")}\nMainPID=0\n`), + ), + Effect.provide(NodeServices.layer), + ); +}); diff --git a/apps/server/src/cloud/serviceProtection.ts b/apps/server/src/cloud/serviceProtection.ts new file mode 100644 index 0000000000000..780aee83b4c48 --- /dev/null +++ b/apps/server/src/cloud/serviceProtection.ts @@ -0,0 +1,345 @@ +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import * as ProcessRunner from "../processRunner.ts"; +import { BOOT_SERVICE_UNIT_FILE } from "./bootService.ts"; + +const PROTECT_DROP_IN = "90-t3code-protect.conf"; +const SYSTEM_UNIT_DIR = "/etc/systemd/system"; +export const REMOTE_ACCESS_UNITS = ["tailscaled.service", "ssh.service", "sshd.service"]; + +// Covers app.slice (the server's 1536M) and init.scope (the user manager's 256M). +const USER_CHAIN_LOW = "MemoryLow=2G"; +// Near the floor, but unlike -1000 the kernel can still kill it as a last resort. +const PROTECTED_OOM_SCORE = "OOMScoreAdjust=-900"; + +export interface DropIn { + readonly unit: string; + readonly path: string; + readonly contents: string; + /** Host-wide, so undo keeps it while another user's protection remains. */ + readonly shared?: boolean; +} + +export interface ProtectionPlan { + readonly systemUnitDir: string; + /** Written through sudo. */ + readonly system: ReadonlyArray; + /** + * Written before the manager's own score drops. Unset, a user manager hands + * its services its own score plus 100: -800 once it runs at -900. + */ + readonly managerDefaults: DropIn; + readonly user: ReadonlyArray; + /** System units whose running main process takes its configured OOM score at once. */ + readonly oomUnits: ReadonlyArray; +} + +const dropIn = ( + directory: string, + unit: string, + section: string, + properties: ReadonlyArray, + shared = false, +): DropIn => ({ + unit, + path: `${directory}/${unit}.d/${PROTECT_DROP_IN}`, + contents: [`[${section}]`, ...properties, ""].join("\n"), + ...(shared ? { shared } : {}), +}); + +/** + * Memory protection only reaches the server when every ancestor cgroup grants + * it, and only root can grant it on user.slice and the user manager. + */ +export function protectionPlan(input: { + readonly uid: number; + readonly userUnitDir: string; + readonly remoteAccessUnits: ReadonlyArray; + readonly systemUnitDir?: string; +}): ProtectionPlan { + const manager = `user@${input.uid}.service`; + const remote = input.remoteAccessUnits; + const systemUnitDir = input.systemUnitDir ?? SYSTEM_UNIT_DIR; + const system = (unit: string, section: string, properties: ReadonlyArray) => + dropIn(systemUnitDir, unit, section, properties); + const shared = (unit: string, section: string, properties: ReadonlyArray) => + dropIn(systemUnitDir, unit, section, properties, true); + const user = (unit: string, section: string, properties: ReadonlyArray) => + dropIn(input.userUnitDir, unit, section, properties); + return { + systemUnitDir, + system: [ + shared("user.slice", "Slice", [USER_CHAIN_LOW]), + system(`user-${input.uid}.slice`, "Slice", [USER_CHAIN_LOW]), + system(manager, "Service", [USER_CHAIN_LOW, PROTECTED_OOM_SCORE]), + ...(remote.length > 0 ? [shared("system.slice", "Slice", ["MemoryLow=512M"])] : []), + ...remote.map((unit) => shared(unit, "Service", ["MemoryLow=256M", PROTECTED_OOM_SCORE])), + ], + managerDefaults: { + unit: "user.conf", + path: `${input.userUnitDir.replace(/\/user$/, "")}/user.conf.d/${PROTECT_DROP_IN}`, + contents: "[Manager]\nDefaultOOMScoreAdjust=200\n", + }, + user: [ + user("app.slice", "Slice", ["MemoryLow=1536M"]), + user("init.scope", "Scope", ["MemoryLow=256M"]), + user(BOOT_SERVICE_UNIT_FILE, "Service", [PROTECTED_OOM_SCORE]), + ], + oomUnits: [manager, ...remote], + }; +} + +const quoteShell = (value: string) => `'${value.replaceAll("'", `'\\''`)}'`; +const directoryOf = (file: string) => file.slice(0, file.lastIndexOf("/")); + +export function renderPrivilegedScript(plan: ProtectionPlan, mode: "apply" | "undo"): string { + const lines = ["set -e"]; + const remove = (dropIns: ReadonlyArray, indent = "") => { + for (const { path } of dropIns) { + lines.push( + `${indent}rm -f ${quoteShell(path)}`, + `${indent}rmdir ${quoteShell(directoryOf(path))} 2>/dev/null || true`, + ); + } + }; + if (mode === "apply") { + for (const { path, contents } of plan.system) { + lines.push( + `mkdir -p ${quoteShell(directoryOf(path))}`, + `cat >${quoteShell(path)} <<'T3CODE_PROTECT'`, + contents.trimEnd(), + "T3CODE_PROTECT", + ); + } + } else { + remove(plan.system.filter((dropIn) => dropIn.shared !== true)); + const otherUsers = `${quoteShell(plan.systemUnitDir)}/user@*.service.d/${PROTECT_DROP_IN}`; + lines.push(`set -- ${otherUsers}`, 'if [ ! -e "$1" ]; then'); + remove( + plan.system.filter((dropIn) => dropIn.shared === true), + " ", + ); + lines.push("fi"); + } + lines.push( + "systemctl daemon-reload", + // OOMScoreAdjust applies at exec; this gives running main processes the configured score. + `for unit in ${plan.oomUnits.map(quoteShell).join(" ")}; do`, + ' pid=$(systemctl show --property=MainPID --value "$unit")', + ' if [ "${pid:-0}" -gt 0 ]; then', + ' systemctl show --property=OOMScoreAdjust --value "$unit" >"/proc/$pid/oom_score_adj" || true', + " fi", + "done", + "", + ); + return lines.join("\n"); +} + +export class ServiceProtectionError extends Schema.TaggedError()( + "ServiceProtectionError", + { + step: Schema.String, + exitCode: Schema.optional(Schema.Number), + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return this.exitCode === undefined + ? `Could not ${this.step}.` + : `Could not ${this.step} (exit code ${this.exitCode}).`; + } +} + +function parseUnits(text: string): Array> { + return text + .split(/\n\s*\n/) + .map( + (block) => + new Map( + block + .split("\n") + .filter((line) => line.includes("=")) + .map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)]), + ), + ) + .filter((values) => values.size > 0); +} + +/** Loaded remote-access units by canonical name, so an alias such as sshd.service counts once. */ +export function loadedRemoteAccessUnits(systemctlShow: string): ReadonlyArray { + const units = parseUnits(systemctlShow) + .filter((values) => values.get("LoadState") === "loaded") + .map((values) => values.get("Id") ?? "") + .filter((id) => REMOTE_ACCESS_UNITS.includes(id)); + return [...new Set(units)]; +} + +const run = Effect.fn("ServiceProtection.run")(function* ( + step: string, + command: string, + args: ReadonlyArray, +) { + const runner = yield* ProcessRunner.ProcessRunner; + const result = yield* runner + .run({ command, args, timeout: "30 seconds" }) + .pipe(Effect.mapError((cause) => new ServiceProtectionError({ step, cause }))); + if (result.code !== 0) { + return yield* new ServiceProtectionError({ step, exitCode: Number(result.code) }); + } + return result.stdout; +}); + +const userSettingsError = (cause: unknown) => + new ServiceProtectionError({ step: "change systemd user settings", cause }); + +/** Undefined when there is no systemd system manager to ask. */ +export const detectRemoteAccessUnits = run("list remote-access services", "systemctl", [ + "show", + "--property=Id,LoadState", + ...REMOTE_ACCESS_UNITS, +]).pipe( + Effect.map(loadedRemoteAccessUnits), + Effect.option, + Effect.map(Option.getOrUndefined), + Effect.withSpan("ServiceProtection.detectRemoteAccessUnits"), +); + +const reloadUserManager = run("reload systemd user units", "systemctl", [ + "--user", + "daemon-reload", +]); + +const writeDropIns = Effect.fn("ServiceProtection.writeDropIns")(function* ( + dropIns: ReadonlyArray, +) { + const fs = yield* FileSystem.FileSystem; + for (const { path, contents } of dropIns) { + yield* fs.makeDirectory(directoryOf(path), { recursive: true }); + yield* fs.writeFileString(path, contents); + } +}, Effect.mapError(userSettingsError)); + +const removeDropIns = Effect.fn("ServiceProtection.removeDropIns")(function* ( + dropIns: ReadonlyArray, +) { + const fs = yield* FileSystem.FileSystem; + for (const { path } of dropIns) { + yield* fs.remove(path, { force: true }); + yield* fs.remove(directoryOf(path)).pipe(Effect.ignore); + } +}, Effect.mapError(userSettingsError)); + +const runPrivileged = Effect.fn("ServiceProtection.runPrivileged")(function* (script: string) { + const step = "change the system's service settings"; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const exitCode = yield* spawner + .exitCode( + // Effect detaches children on POSIX by default; sudo then has no controlling + // terminal and cannot ask for a password. + ChildProcess.make("sudo", ["/bin/sh", "-c", script], { + detached: false, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + }), + ) + .pipe(Effect.mapError((cause) => new ServiceProtectionError({ step, cause }))); + if (exitCode !== 0) { + return yield* new ServiceProtectionError({ step, exitCode: Number(exitCode) }); + } +}); + +export const applyProtection = Effect.fn("ServiceProtection.apply")(function* ( + plan: ProtectionPlan, +) { + const fs = yield* FileSystem.FileSystem; + const hadManagerDefaults = yield* fs + .exists(plan.managerDefaults.path) + .pipe(Effect.orElseSucceed(() => false)); + yield* writeDropIns([plan.managerDefaults]); + // A partial run may already have lowered the manager; its default must then stay. + const manager = plan.system.find(({ unit }) => unit.startsWith("user@")); + const rollBack = Effect.gen(function* () { + if (hadManagerDefaults) return; + if (manager !== undefined && (yield* fs.exists(manager.path))) return; + yield* removeDropIns([plan.managerDefaults]); + }).pipe(Effect.ignore); + yield* runPrivileged(renderPrivilegedScript(plan, "apply")).pipe(Effect.tapError(() => rollBack)); + yield* writeDropIns(plan.user); + yield* reloadUserManager; +}); + +export const removeProtection = Effect.fn("ServiceProtection.remove")(function* ( + plan: ProtectionPlan, +) { + yield* removeDropIns(plan.user); + yield* reloadUserManager; + if (plan.system.length > 0) yield* runPrivileged(renderPrivilegedScript(plan, "undo")); + yield* removeDropIns([plan.managerDefaults]); + yield* reloadUserManager; +}); + +/** The plan narrowed to drop-ins that exist, for undo. */ +export const installedProtection = Effect.fn("ServiceProtection.installed")(function* ( + plan: ProtectionPlan, +) { + const fs = yield* FileSystem.FileSystem; + const exists = (dropIn: DropIn) => fs.exists(dropIn.path).pipe(Effect.orElseSucceed(() => false)); + const system = yield* Effect.filter(plan.system, exists); + const user = yield* Effect.filter(plan.user, exists); + const managerDefaults = yield* exists(plan.managerDefaults); + return system.length + user.length === 0 && !managerDefaults + ? undefined + : { ...plan, system, user }; +}); + +export interface ServiceProtectionState { + /** The first cgroup from the top that grants the server no memory protection. */ + readonly unprotected: string | undefined; + readonly oomScoreAdj: number | undefined; + readonly setByProtect: boolean; +} + +/** Reads what the kernel applies to the running service, whoever configured it. */ +export const readProtection = Effect.fn("ServiceProtection.read")(function* (userUnitDir: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const shown = yield* run("read the service's cgroup", "systemctl", [ + "--user", + "show", + BOOT_SERVICE_UNIT_FILE, + "--property=ControlGroup,MainPID", + ]).pipe(Effect.option); + if (Option.isNone(shown)) return undefined; + const values = parseUnits(shown.value)[0] ?? new Map(); + const controlGroup = (values.get("ControlGroup") ?? "").split("/").filter(Boolean); + if (controlGroup.length === 0) return undefined; + const read = (file: string) => + fs.readFileString(file).pipe( + Effect.map((text) => text.trim()), + Effect.orElseSucceed(() => ""), + ); + let unprotected: string | undefined; + const grants = (value: string) => value !== "" && value !== "0"; + for (let depth = 1; depth <= controlGroup.length && unprotected === undefined; depth++) { + const directory = path.join("/sys/fs/cgroup", ...controlGroup.slice(0, depth)); + const low = yield* read(path.join(directory, "memory.low")); + const min = yield* read(path.join(directory, "memory.min")); + if (!grants(low) && !grants(min)) unprotected = controlGroup[depth - 1]; + } + const pid = Number(values.get("MainPID") ?? 0); + const score = pid > 0 ? yield* read(`/proc/${pid}/oom_score_adj`) : ""; + return { + unprotected, + oomScoreAdj: score === "" ? undefined : Number(score), + setByProtect: yield* fs + .exists(path.join(userUnitDir, `${BOOT_SERVICE_UNIT_FILE}.d`, PROTECT_DROP_IN)) + .pipe(Effect.orElseSucceed(() => false)), + } satisfies ServiceProtectionState; +}); diff --git a/apps/server/src/device/DeviceHubProxy.test.ts b/apps/server/src/device/DeviceHubProxy.test.ts index 07b45737f6730..72c8f9e0e5d56 100644 --- a/apps/server/src/device/DeviceHubProxy.test.ts +++ b/apps/server/src/device/DeviceHubProxy.test.ts @@ -169,6 +169,32 @@ describe("device hub proxy", () => { expect(headers[0]?.["content-type"]).toBe("application/json"); }); + it("lets a viewer read Android stream capabilities and signal WebRTC with read scope", async () => { + const { handler, requests } = fixture([AuthOrchestrationReadScope]); + const hub = "http://t3.test/api/device-hub/vendor/serve-emu"; + const api = await handler(new Request(`${hub}/api?device=session-1`)); + expect(api.status).toBe(200); + await api.text(); + for (const action of ["offer", "close"]) { + const signal = await handler( + new Request(`${hub}/webrtc/${action}?device=session-1`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"sessionId":"viewer"}', + }), + ); + expect(signal.status).toBe(200); + await signal.text(); + } + expect(requests).toEqual([ + "http://hub.test/vendor/serve-emu/api?device=session-1", + "http://hub.test/vendor/serve-emu/webrtc/offer?device=session-1", + "http://hub.test/vendor/serve-emu/webrtc/close?device=session-1", + ]); + expect((await handler(new Request(`${hub}/api`, { method: "POST" }))).status).toBe(405); + expect((await handler(new Request(`${hub}/webrtc/offer`))).status).toBe(405); + }); + it("sends requests a host routes itself to its upstream with its credentials", async () => { const { handler, requests, headers } = fixture( [AuthOrchestrationReadScope], diff --git a/apps/server/src/device/DeviceHubProxy.ts b/apps/server/src/device/DeviceHubProxy.ts index 89cad0d7206e2..c1aaab3c61219 100644 --- a/apps/server/src/device/DeviceHubProxy.ts +++ b/apps/server/src/device/DeviceHubProxy.ts @@ -27,14 +27,17 @@ const ALLOWED_PATHS: ReadonlyArray = [ /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/, /^\/vendor\/serve-sim\/helper\/[^/]+\/webrtc\/(offer|close)$/, /^\/vendor\/serve-sim\/appstate$/, + /^\/vendor\/serve-emu\/api$/, /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility|fold)$/, /^\/vendor\/serve-emu\/health$/, ]; +// Paths that take methods besides GET; one missing from ALLOWED_PATHS takes no GET. const MUTABLE_PATHS: ReadonlyArray = [ /^\/vendor\/serve-sim\/api\/screenshot$/, // WebRTC signalling: opening and closing a view of the stream, so read scope. /^\/vendor\/serve-sim\/helper\/[^/]+\/webrtc\/(offer|close)$/, + /^\/vendor\/serve-emu\/webrtc\/(offer|close)$/, /^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/, /^\/vendor\/serve-emu\/api\/fold$/, ]; @@ -178,14 +181,15 @@ const handler = Effect.gen(function* () { } const hubPath = url.value.pathname.slice(DeviceService.DEVICE_HUB_ROUTE_PREFIX.length) || "/"; const upgrade = isWebSocketUpgrade(request); - const allowed = (upgrade ? ALLOWED_WS_PATHS : ALLOWED_PATHS).some((pattern) => - pattern.test(hubPath), - ); + const matches = (paths: ReadonlyArray) => paths.some((pattern) => pattern.test(hubPath)); + const allowed = upgrade + ? matches(ALLOWED_WS_PATHS) + : matches(ALLOWED_PATHS) || matches(MUTABLE_PATHS); if (!allowed) { return HttpServerResponse.text("Not Found", { status: 404 }); } const readOnly = request.method === "GET" || request.method === "HEAD"; - if (!upgrade && !readOnly && !MUTABLE_PATHS.some((pattern) => pattern.test(hubPath))) { + if (!upgrade && !matches(readOnly ? ALLOWED_PATHS : MUTABLE_PATHS)) { return HttpServerResponse.text("Method Not Allowed", { status: 405 }); } const controlsDevice = diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index a9ff957b91997..b458142424a43 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -18,6 +18,7 @@ import * as Exit from "effect/Exit"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; @@ -445,6 +446,11 @@ describe("device setup consent", () => { ); }); +const layerSecretResolver = Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => Effect.succeed(references.map(() => undefined)), + invalidate: () => Effect.void, +}); + it.effect("publishes boot progress and does not restore sessions after support is disabled", () => Effect.gen(function* () { const started = yield* Deferred.make(); @@ -1296,7 +1302,9 @@ it.effect("reconciles SSH host settings without publishing no-op changes", () => expect(removedFiles).toHaveLength(1); }).pipe( Effect.scoped, - Effect.provide(Layer.mergeAll(ServerSettings.layerTest(), NodeServices.layer)), + Effect.provide( + Layer.mergeAll(ServerSettings.layerTest(), layerSecretResolver, NodeServices.layer), + ), ), ); @@ -1354,6 +1362,7 @@ it.effect("a failed tool update reports npm's exit code but not its output", () Effect.provide( Layer.mergeAll( ServerSettings.layerTest({ enableDeviceSupport: true }), + layerSecretResolver, ServerConfig.layerTest(process.cwd(), { prefix: "t3-device-update-" }), NodeCrypto.layer, NetService.layer, diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index f452763fae0c2..2be98ccae0c65 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -68,7 +68,7 @@ import { isLocalSshDeviceHost, remoteSshDeviceHosts } from "./localSshDeviceHost import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; import * as ProcessRunner from "../processRunner.ts"; -import { ProviderSecretResolver } from "../provider/ProviderSecretResolver.ts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as DeviceHost from "./DeviceHost.ts"; import * as SshDeviceHost from "./SshDeviceHost.ts"; import * as EasDeviceHost from "./EasDeviceHost.ts"; @@ -1163,7 +1163,7 @@ export const make = Effect.gen(function* () { const path = yield* Path.Path; const runner = yield* ProcessRunner.ProcessRunner; const settings = yield* ServerSettings.ServerSettingsService; - const secrets = yield* ProviderSecretResolver; + const secrets = yield* SecretResolver.SecretResolver; const scope = yield* Scope.Scope; const hosts = new Map([ [localHost.id, localHost], diff --git a/apps/server/src/device/EasDeviceHost.test.ts b/apps/server/src/device/EasDeviceHost.test.ts index 01b3285785409..9152c094554eb 100644 --- a/apps/server/src/device/EasDeviceHost.test.ts +++ b/apps/server/src/device/EasDeviceHost.test.ts @@ -1,10 +1,7 @@ import { describe, expect, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { - type EasDeviceHostConfig, - SecretProviderId, - SecretProviderReference, -} from "@t3tools/contracts"; +import { type EasDeviceHostConfig, SecretProviderId } from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -19,7 +16,6 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; -import { ProviderSecretResolver } from "../provider/ProviderSecretResolver.ts"; import * as EasDeviceHost from "./EasDeviceHost.ts"; const APP_ID = "d763fcb8-d37c-41ea-a773-b54a0ab4a454"; @@ -67,6 +63,7 @@ const fakeExpo = ( const calls: Array<{ operation: string; variables: Record }> = []; let created = 0; let createdName = ""; + const previews: string[] = []; let previewFails = options.failPreview ?? false; let createFails = options.failCreate ?? false; let sessionGated = false; @@ -74,6 +71,7 @@ const fakeExpo = ( Effect.gen(function* () { const url = new URL(request.url); if (url.origin === "https://preview.test") { + previews.push(`${url.pathname}${url.search}`); expect(request.headers.authorization).toBe("Bearer preview-token"); if (options.previewGate) { yield* Deferred.succeed(options.previewGate.requested, undefined); @@ -82,7 +80,17 @@ const fakeExpo = ( if (previewFails) { return HttpClientResponse.fromWeb(request, new Response("bad gateway", { status: 502 })); } - return HttpClientResponse.fromWeb(request, Response.json({ device: "UDID-1" })); + if (url.pathname.endsWith("/api/screenshot")) { + return HttpClientResponse.fromWeb(request, new Response(Uint8Array.of(137, 80, 78, 71))); + } + return HttpClientResponse.fromWeb( + request, + Response.json( + url.pathname === "/vendor/serve-emu/api" + ? { serial: "emulator-5554" } + : { device: "UDID-1" }, + ), + ); } const body = request.body._tag === "Uint8Array" @@ -200,12 +208,27 @@ const fakeExpo = ( const allowCreate = () => { createFails = false; }; - return { http, named, stopped, allowPreview, allowCreate }; + return { http, named, stopped, allowPreview, allowCreate, previews }; }; +const makeSecretResolver = (values: Record = {}) => + Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => + Effect.succeed(references.map((reference) => values[reference.reference])), + invalidate: () => Effect.void, + }); + +const layerWith = (secrets: Layer.Layer) => + Layer.mergeAll( + ProcessRunner.layer, + ServerConfig.layerTest(process.cwd(), { prefix: "t3-eas-host-" }), + secrets, + ).pipe(Layer.provideMerge(NodeServices.layer)); + const layer = Layer.mergeAll( ProcessRunner.layer, ServerConfig.layerTest(process.cwd(), { prefix: "t3-eas-host-" }), + makeSecretResolver(), ).pipe(Layer.provideMerge(NodeServices.layer)); const makeHost = ( @@ -226,7 +249,10 @@ describe("EAS Simulator host", () => { const host = yield* makeHost(expo, config(), scope); const ready = yield* host.ensureReady(() => Effect.void); const catalog = (yield* ready.devices.list).devices; - expect(catalog.map((device) => [device.id, device.booted])).toEqual([["eas-ios", false]]); + expect(catalog.map((device) => [device.id, device.booted])).toEqual([ + ["eas-ios", false], + ["eas-android", false], + ]); expect(yield* ready.devices.boot(catalog[0]!)).toBe("session-1"); expect(expo.named("T3CreateEasSimulatorSession")[0]?.variables.input).toMatchObject({ @@ -274,46 +300,21 @@ describe("EAS Simulator host", () => { it.effect("reads its token from the secret provider the setting names", () => Effect.gen(function* () { const expo = fakeExpo(); - const source = { - kind: "secret-provider" as const, - provider: SecretProviderId.make("homelab"), - reference: SecretProviderReference.make("expo#token"), - }; - const requested: Array = []; - const resolver = yield* ProviderSecretResolver; - const host = yield* makeHost(expo, config({ accessToken: source }), yield* Scope.make()).pipe( - Effect.provideService(ProviderSecretResolver, { - ...resolver, - resolve: (environment) => - Effect.sync(() => { - requested.push(...(environment ?? []).map((variable) => variable.value)); - return { - variables: (environment ?? []).map((variable) => ({ - name: variable.name, - value: "expo-token", - sensitive: true, - })), - unresolved: [], - }; - }), - }), - ); + const source = { provider: SecretProviderId.make("homelab"), reference: "expo#token" }; + const host = yield* makeHost(expo, config({ accessToken: source }), yield* Scope.make()); const ready = yield* host.ensureReady(() => Effect.void); const [catalogDevice] = (yield* ready.devices.list).devices; expect(yield* ready.devices.boot(catalogDevice!)).toBe("session-1"); - expect(requested).not.toHaveLength(0); - expect(new Set(requested)).toEqual(new Set([source])); - }).pipe(Effect.scoped, Effect.provide(layer)), + }).pipe( + Effect.scoped, + Effect.provide(layerWith(makeSecretResolver({ "expo#token": "expo-token" }))), + ), ); it.effect("fails instead of using another Expo login when its secret cannot be read", () => Effect.gen(function* () { const expo = fakeExpo(); - const source = { - kind: "secret-provider" as const, - provider: SecretProviderId.make("homelab"), - reference: SecretProviderReference.make("expo#token"), - }; + const source = { provider: SecretProviderId.make("homelab"), reference: "expo#token" }; const host = yield* makeHost(expo, config({ accessToken: source }), yield* Scope.make()).pipe( Effect.provideService(HostProcess.Environment, { EXPO_TOKEN: "expo-token" }), ); @@ -325,15 +326,52 @@ describe("EAS Simulator host", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); - it.effect("falls back to EXPO_TOKEN when no token is saved", () => + it.effect("runs an Android session through serve-emu beside an iOS one", () => Effect.gen(function* () { const expo = fakeExpo(); - const host = yield* makeHost(expo, config({ accessToken: "" }), yield* Scope.make()).pipe( - Effect.provideService(HostProcess.Environment, { EXPO_TOKEN: "expo-token" }), - ); + const scope = yield* Scope.make(); + const host = yield* makeHost(expo, config(), scope); + expect(yield* host.platformAvailability("android")).toEqual({ + platform: "android", + available: true, + }); const ready = yield* host.ensureReady(() => Effect.void); - const [catalogDevice] = (yield* ready.devices.list).devices; - expect(yield* ready.devices.boot(catalogDevice!)).toBe("session-1"); + const [ios, android] = (yield* ready.devices.list).devices; + expect(yield* ready.devices.boot(ios!)).toBe("session-1"); + expect(yield* ready.devices.boot(android!)).toBe("session-2"); + expect(expo.named("T3CreateEasSimulatorSession")[1]?.variables.input).toMatchObject({ + platform: "ANDROID", + }); + const running = (yield* ready.devices.list).devices.find( + (device) => device.id === "session-2", + ); + expect(running).toMatchObject({ platform: "android", booted: true }); + + expect(yield* ready.devices.agentEndpoint!("session-2")).toMatchObject({ + agentDeviceId: "emulator-5554", + }); + expect(Array.from(yield* ready.devices.screenshot(running!))).toEqual([137, 80, 78, 71]); + expect(expo.previews.slice(-2)).toEqual([ + "/vendor/serve-emu/api", + "/vendor/serve-emu/api/screenshot?device=emulator-5554", + ]); + expect( + yield* ready.upstream!( + "/vendor/serve-emu/webrtc/offer", + new URLSearchParams("device=session-2&hostId=eas"), + ), + ).toEqual({ + url: "https://preview.test/vendor/serve-emu/webrtc/offer?device=emulator-5554&hostId=eas", + headers: { authorization: "Bearer preview-token" }, + protocols: ["serve-emu.token.preview-token"], + }); + // Each session answers only on its own platform's stream. + expect( + yield* ready.upstream!("/vendor/serve-emu/api", new URLSearchParams("device=session-1")), + ).toBeNull(); + expect( + yield* ready.upstream!("/vendor/serve-sim/api", new URLSearchParams("device=session-2")), + ).toBeNull(); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -363,6 +401,7 @@ describe("EAS Simulator host", () => { expect(yield* ready.devices.boot(catalogDevice!)).toBe("session-2"); expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual([ "eas-ios", + "eas-android", "session-1", "session-2", ]); @@ -377,6 +416,7 @@ describe("EAS Simulator host", () => { expect(expo.stopped()).toEqual(["session-1"]); expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual([ "eas-ios", + "eas-android", "session-2", ]); yield* Scope.close(scope, Exit.void); @@ -492,7 +532,10 @@ describe("EAS Simulator host", () => { expect(refused.message).toContain("limited to 1 running session"); stopFails = false; yield* ready.devices.list; - expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual(["eas-ios"]); + expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual([ + "eas-ios", + "eas-android", + ]); expect(expo.stopped().filter((id) => id === "session-1").length).toBeGreaterThanOrEqual(2); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -509,7 +552,10 @@ describe("EAS Simulator host", () => { expect((yield* ready.devices.list).devices.map((device) => device.id)).toContain("session-1"); // Expo ended it by itself: the stop keeps failing, but the status says so. status = "STOPPED"; - expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual(["eas-ios"]); + expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual([ + "eas-ios", + "eas-android", + ]); status = "IN_PROGRESS"; expect(yield* ready.devices.boot(catalogDevice!)).toBe("session-2"); }).pipe(Effect.scoped, Effect.provide(layer)), @@ -559,7 +605,10 @@ describe("EAS Simulator host", () => { expect(expo.stopped()).toEqual(["session-1"]); const failure = yield* Fiber.join(booting).pipe(Effect.flip); expect(failure.message).toContain("was stopped while the session started"); - expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual(["eas-ios"]); + expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual([ + "eas-ios", + "eas-android", + ]); }).pipe(Effect.scoped, Effect.provide(layer)), ); }); diff --git a/apps/server/src/device/EasDeviceHost.ts b/apps/server/src/device/EasDeviceHost.ts index f2f31e10398d0..6f3a9984ce50e 100644 --- a/apps/server/src/device/EasDeviceHost.ts +++ b/apps/server/src/device/EasDeviceHost.ts @@ -33,11 +33,8 @@ import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as ServerConfig from "../config.ts"; -import { - ProviderSecretResolver, - type ProviderSecretResolverShape, - requireSettingsSecret, -} from "../provider/ProviderSecretResolver.ts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts"; import * as DeviceHost from "./DeviceHost.ts"; import { findEasProjectId } from "./easProjectId.ts"; import { type ExpoApi, makeExpoApi, resolveExpoCredential } from "./ExpoApi.ts"; @@ -48,14 +45,48 @@ const STATUS_TTL_MS = 15_000; // Expo's own CLI waits this long once a worker picks the session up. const START_TIMEOUT = Duration.minutes(15); const PREVIEW_TIMEOUT = Duration.seconds(20); -const SERVE_SIM_PREFIX = "/vendor/serve-sim"; +const decodeServeSimApi = Schema.decodeUnknownEffect( + Schema.Struct({ device: Schema.NonEmptyString }), +); +const decodeServeEmuApi = Schema.decodeUnknownEffect( + Schema.Struct({ serial: Schema.NonEmptyString }), +); +/** + * Each session's preview host serves serve-sim at its root and serve-emu under + * its vendor prefix, and names its device by an id of its own that agent-device + * also selects it by. + */ +const PLATFORMS = { + ios: { + name: "iPhone (EAS Simulator)", + hubPrefix: "/vendor/serve-sim", + previewPrefix: "", + protocol: "serve-sim", + remoteDeviceId: (api: unknown) => decodeServeSimApi(api).pipe(Effect.map((api) => api.device)), + }, + android: { + name: "Android (EAS Simulator)", + hubPrefix: "/vendor/serve-emu", + previewPrefix: "/vendor/serve-emu", + protocol: "serve-emu", + remoteDeviceId: (api: unknown) => decodeServeEmuApi(api).pipe(Effect.map((api) => api.serial)), + }, +} as const; /** EAS picks the simulator model; one entry starts a session. */ const CATALOG: ReadonlyArray = [ { id: "eas-ios", platform: "ios", - name: "iPhone (EAS Simulator)", + name: PLATFORMS.ios.name, + version: "Cloud", + booted: false, + physical: false, + }, + { + id: "eas-android", + platform: "android", + name: PLATFORMS.android.name, version: "Cloud", booted: false, physical: false, @@ -65,7 +96,8 @@ const CATALOG: ReadonlyArray = [ interface RunningSession { readonly sessionId: string; readonly name: string; - readonly udid: string; + readonly platform: DevicePlatform; + readonly remoteDeviceId: string; readonly preview: { readonly origin: string; readonly token: Redacted.Redacted }; readonly agent: DeviceHost.AgentDaemonEndpoint; /** Set once a stop has been asked for but Expo has not confirmed it. */ @@ -100,7 +132,6 @@ const RECORD_MAX_AGE_MS = 24 * 60 * 60_000; const UNCERTAIN_CREATE_MS = 5 * 60_000; // One failed request to Expo during a minutes-long start does not end the start. const TRANSIENT_RETRY = { times: 3, schedule: Schedule.spaced("2 seconds") } as const; -const decodePreviewApi = Schema.decodeUnknownEffect(Schema.Struct({ device: Schema.String })); /** A malformed escape names no device, so the proxy answers 404 instead of failing. */ const decodePathSegment = (segment: string): string | null => { @@ -116,7 +147,7 @@ const decodePathSegment = (segment: string): string | null => { * it, so a named secret that cannot be read fails rather than falling back to another login. */ export const resolveSavedExpoCredential = ( - secrets: ProviderSecretResolverShape, + secrets: SecretResolver.SecretResolver["Service"], saved: SettingsSecretValue, ) => requireSettingsSecret(secrets, saved).pipe(Effect.flatMap(resolveExpoCredential)); @@ -125,14 +156,14 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const server = yield* ServerConfig.ServerConfig; - const secrets = yield* ProviderSecretResolver; + const secrets = yield* SecretResolver.SecretResolver; const services = yield* Effect.context< FileSystem.FileSystem | Path.Path | Effect.Services> >(); const hostId = config.id; const ledgerPath = path.join(server.stateDir, "device", "eas", `${hostId}.json`); // Keyed by Expo's session id, which is also the device id: every EAS - // simulator reports the same UDID, so the UDID cannot tell sessions apart. + // simulator reports the same remote device id, so that id cannot tell sessions apart. const sessions = new Map(); const checkedAt = new Map(); const startLock = yield* Semaphore.make(1); @@ -365,18 +396,21 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice ), ); - const previewDevice = (origin: string, token: Redacted.Redacted) => + const previewDevice = ( + origin: string, + token: Redacted.Redacted, + platform: DevicePlatform, + ) => http .execute( - HttpClientRequest.get(`${origin}/api`).pipe( + HttpClientRequest.get(`${origin}${PLATFORMS[platform].previewPrefix}/api`).pipe( HttpClientRequest.bearerToken(Redacted.value(token)), ), ) .pipe( Effect.flatMap(HttpClientResponse.filterStatusOk), Effect.flatMap((response) => response.json), - Effect.flatMap(decodePreviewApi), - Effect.map((api) => api.device), + Effect.flatMap(PLATFORMS[platform].remoteDeviceId), Effect.scoped, Effect.timeout(PREVIEW_TIMEOUT), Effect.retry(TRANSIENT_RETRY), @@ -517,7 +551,9 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice ), Effect.mapError((error) => unavailable(error.message, error)), ); - return yield* restore(trackSession(expo, sessionId, name, startedIn)).pipe( + return yield* restore( + trackSession(expo, sessionId, name, device.platform, startedIn), + ).pipe( Effect.onError(() => stopSession(expo, sessionId, name).pipe( // Still running and billed: owned, counted, and retried by listing. @@ -537,6 +573,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice expo: ExpoApi, sessionId: string, name: string, + platform: DevicePlatform, startedIn: number, ) { const endpoints = yield* awaitRunning(expo, sessionId); @@ -549,7 +586,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice }); } const origin = new URL(endpoints.preview.url).origin; - const udid = yield* previewDevice(origin, endpoints.preview.token).pipe( + const remoteDeviceId = yield* previewDevice(origin, endpoints.preview.token, platform).pipe( Effect.mapError( (cause) => new DeviceBootError({ @@ -566,7 +603,8 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice sessions.set(sessionId, { sessionId, name, - udid, + platform, + remoteDeviceId, preview: { origin, token: endpoints.preview.token }, agent: { baseUrl: endpoints.agentDevice.url, @@ -667,8 +705,8 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice ...CATALOG, ...[...sessions.values()].map((session): DeviceHost.HostDevice => ({ id: session.sessionId, - platform: "ios", - name: "iPhone (EAS Simulator)", + platform: session.platform, + name: PLATFORMS[session.platform].name, version: "Cloud", booted: true, physical: false, @@ -725,7 +763,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice http .execute( HttpClientRequest.post( - `${session.preview.origin}/api/screenshot?device=${encodeURIComponent(session.udid)}`, + `${session.preview.origin}${PLATFORMS[session.platform].previewPrefix}/api/screenshot?device=${encodeURIComponent(session.remoteDeviceId)}`, ).pipe(HttpClientRequest.bearerToken(Redacted.value(session.preview.token))), ) .pipe( @@ -747,42 +785,45 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice agentEndpoint: (deviceId) => Option.match(sessionFor(deviceId), { onNone: () => Effect.fail(unavailable("This EAS Simulator device is no longer running.")), - // agent-device selects the simulator by its UDID, not the session id T3 shows. - onSome: (session) => Effect.succeed({ ...session.agent, agentDeviceId: session.udid }), + onSome: (session) => + Effect.succeed({ ...session.agent, agentDeviceId: session.remoteDeviceId }), }), endsDevices: true, }; /** - * serve-sim runs at the root of each session's preview host and wants its - * token on every request. Clients name the device by its session id; serve-sim - * knows it by its UDID, so the id is swapped for the UDID in the path and in - * the `device` parameter. + * Clients name the device by its session id; the preview host knows it by its + * own id, so the session id is swapped for that in the path and in the + * `device` parameter. A session answers only on its own platform's server. */ const upstream: NonNullable = (hubPath, search) => Effect.sync(() => { - if (!hubPath.startsWith(`${SERVE_SIM_PREFIX}/`)) return null; + const platform = (["ios", "android"] as const).find((candidate) => + hubPath.startsWith(`${PLATFORMS[candidate].hubPrefix}/`), + ); + if (!platform) return null; const helper = /^\/vendor\/serve-sim\/helper\/([^/]+)\//.exec(hubPath); const deviceId = helper ? decodePathSegment(helper[1]!) : search.get("device"); if (helper && deviceId === null) return null; const session = - deviceId !== null && deviceId !== undefined + deviceId !== null ? sessions.get(deviceId) : sessions.size === 1 ? [...sessions.values()][0] : undefined; - if (!session) return null; + if (session?.platform !== platform) return null; + const { hubPrefix, previewPrefix, protocol } = PLATFORMS[platform]; const path = helper - ? `/helper/${encodeURIComponent(session.udid)}${hubPath.slice(helper[0].length - 1)}` - : hubPath.slice(SERVE_SIM_PREFIX.length); + ? `/helper/${encodeURIComponent(session.remoteDeviceId)}${hubPath.slice(helper[0].length - 1)}` + : `${previewPrefix}${hubPath.slice(hubPrefix.length)}`; const params = new URLSearchParams(search); - if (params.has("device")) params.set("device", session.udid); + if (params.has("device")) params.set("device", session.remoteDeviceId); const query = params.size > 0 ? `?${params.toString()}` : ""; const token = Redacted.value(session.preview.token); return { url: `${session.preview.origin}${path}${query}`, headers: { authorization: `Bearer ${token}` }, - protocols: [`serve-sim.token.${token}`], + protocols: [`${protocol}.token.${token}`], }; }); @@ -850,19 +891,13 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice // Each server signs in on its own, so one with no Expo credential answers for itself. Reading // the credential is local; whether Expo accepts it is only known once a session starts. const platformAvailability = (platform: DevicePlatform) => - platform === "ios" - ? resolveSavedExpoCredential(secrets, config.accessToken).pipe( - Effect.provide(services), - Effect.as({ platform, available: true }), - Effect.catch((error) => - Effect.succeed({ platform, available: false, reason: error.message }), - ), - ) - : Effect.succeed({ - platform, - available: false, - reason: "EAS Simulator Android emulators are not supported yet.", - }); + resolveSavedExpoCredential(secrets, config.accessToken).pipe( + Effect.provide(services), + Effect.as({ platform, available: true }), + Effect.catch((error) => + Effect.succeed({ platform, available: false, reason: error.message }), + ), + ); const summary: Effect.Effect = Effect.all([ platformAvailability("ios"), diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index cef18fbe325c3..5e0bcfc742e08 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -240,6 +240,7 @@ export const make = Effect.gen(function* () { projectSettingsOverrides: true, projectProviderInstanceOverrides: true, threadSnooze: true, + threadForceStop: true, environmentThemes: true, usageLimitSources: true, providerWorkspaceSnapshotsOnDemand: true, diff --git a/apps/server/src/git/GitManager.test.ts b/apps/server/src/git/GitManager.test.ts index 5d363826458df..3c13991426f92 100644 --- a/apps/server/src/git/GitManager.test.ts +++ b/apps/server/src/git/GitManager.test.ts @@ -2599,7 +2599,45 @@ it.layer(layerGitManagerTest)("GitManager", (it) => { }), ); - it.effect("applies a V2-native thread's project settings to generated commit text", () => + it.effect("uses an explicit project's provider settings for generated commit text", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + NodeFS.writeFileSync(NodePath.join(repoDir, "README.md"), "hello\nproject\n"); + const projectId = ProjectId.make("project:git-action-settings"); + const disabledInstanceId = DEFAULT_SERVER_SETTINGS.textGenerationModelSelection.instanceId; + let generatedModelSelection: + | TextGeneration.CommitMessageGenerationInput["modelSelection"] + | undefined; + + const { manager } = yield* makeManager({ + serverSettings: { + projectSettingsOverrides: { + [projectId]: { + providerInstances: { [disabledInstanceId]: { enabled: false } }, + }, + }, + }, + textGeneration: { + generateCommitMessage: (input) => { + generatedModelSelection = input.modelSelection; + return Effect.succeed({ subject: "Use the project writer", body: "" }); + }, + }, + }); + + yield* manager.runStackedAction({ + actionId: "test-project-action-settings", + cwd: repoDir, + action: "commit", + projectId, + }); + + expect(generatedModelSelection?.instanceId).toBe(ProviderInstanceId.make("claudeAgent")); + }), + ); + + it.effect("prefers a V2-native thread's project settings over an explicit project", () => Effect.gen(function* () { const repoDir = yield* makeTempDir("t3code-git-manager-"); yield* initRepo(repoDir); @@ -2686,6 +2724,7 @@ it.layer(layerGitManagerTest)("GitManager", (it) => { cwd: repoDir, action: "commit", threadId, + projectId: ProjectId.make("project:other"), }); expect(generatedModelSelection).toEqual(projectModel); diff --git a/apps/server/src/git/GitManager.ts b/apps/server/src/git/GitManager.ts index 53d0b4f5853ff..71e5edb3b697b 100644 --- a/apps/server/src/git/GitManager.ts +++ b/apps/server/src/git/GitManager.ts @@ -672,19 +672,25 @@ export const make = Effect.gen(function* () { const projectSettingsFor = Effect.fnUntraced(function* (input: { readonly cwd: string; readonly threadId?: ThreadId | undefined; + readonly projectId?: ProjectId | undefined; }) { const settings = yield* serverSettingsService.getSettings; if (!hasProjectSettingsOverrides(settings)) return { settings, projectId: null as ProjectId | null }; - const projectId: ProjectId | null = yield* input.threadId !== undefined - ? threads.getThreadShell(input.threadId).pipe( - Effect.map((thread) => thread?.projectId ?? null), - Effect.orElseSucceed(() => null), - ) - : projects.findActiveByWorkspaceRoot(input.cwd).pipe( - Effect.map((project) => Option.getOrNull(project)?.projectId ?? null), - Effect.orElseSucceed(() => null), - ); + const threadProjectId = + input.threadId === undefined + ? null + : yield* threads.getThreadShell(input.threadId).pipe( + Effect.map((thread) => thread?.projectId ?? null), + Effect.orElseSucceed(() => null), + ); + const projectId = + threadProjectId ?? + input.projectId ?? + (yield* projects.findActiveByWorkspaceRoot(input.cwd).pipe( + Effect.map((project) => Option.getOrNull(project)?.projectId ?? null), + Effect.orElseSucceed(() => null), + )); return { settings: resolveProjectSettings(settings, projectId).settings, projectId }; }); const readThreadContext = Effect.fnUntraced( diff --git a/apps/server/src/guest/GuestShareService.test.ts b/apps/server/src/guest/GuestShareService.test.ts index 1ef25744c8c55..ac5ffc76aae77 100644 --- a/apps/server/src/guest/GuestShareService.test.ts +++ b/apps/server/src/guest/GuestShareService.test.ts @@ -7,6 +7,7 @@ import { GuestShareId, type GuestStreamEvent, MessageId, + type OrchestrationV2ThreadHistoryPage, type OrchestrationV2TurnItem, ProjectId, ProviderDriverKind, @@ -529,6 +530,90 @@ it.effect("gives the guest a command's detail and output, but no file diffs", () }).pipe(Effect.provide(layerTest)), ); +it.effect("pages the guest back through turns older than the stream's snapshot", () => + Effect.gen(function* () { + const { guestShares, credential } = yield* shareThread; + const { token: session } = yield* guestShares.redeem({ credential, name: "Sam" }); + const runId = yield* ownerRun; + const fileSystem = yield* FileSystem.FileSystem; + const directory = yield* fileSystem.makeTempDirectoryScoped(); + const shot = `${directory}/old.png`; + yield* fileSystem.writeFile(shot, PNG); + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const base = { threadId, runId, nodeId: null, providerThreadId: null, providerTurnId: null }; + const turnItem = (id: string, ordinal: number, payload: object) => ({ + id: EventId.make(`event:${id}`), + type: "turn-item.updated" as const, + threadId, + driver: ProviderDriverKind.make("codex"), + providerInstanceId: instanceId, + occurredAt: now, + payload: { + ...base, + id: TurnItemId.make(id), + nativeItemRef: null, + parentItemId: null, + ordinal, + title: null, + status: "completed" as const, + startedAt: now, + completedAt: now, + updatedAt: now, + messageId: MessageId.make(id), + ...payload, + } as OrchestrationV2TurnItem, + }); + yield* eventSink.write({ + events: [ + turnItem("item:old-image", 1, { + type: "assistant_message", + text: `![Old](${shot})`, + streaming: false, + }), + ...Array.from({ length: 30 }, (_, index) => + turnItem(`item:ask-${index}`, index + 2, { + type: "user_message", + text: `Ask ${index}`, + createdBy: "user", + creationSource: "web", + inputIntent: "turn_start", + attachments: [], + }), + ), + ], + }); + + const [, opening] = Array.from( + yield* Stream.runCollect(Stream.take(yield* guestShares.stream(session), 2)), + ); + if (opening?.type !== "thread" || opening.item.kind !== "snapshot") { + return assert.fail("Expected the thread snapshot second."); + } + const ids = (rows: ReadonlyArray<{ readonly sourceItemId: string }>) => + rows.map((row) => row.sourceItemId); + assert.notInclude(ids(opening.item.projection.visibleTurnItems), "item:old-image"); + let cursor = opening.item.historyCursor ?? null; + assert.isNotNull(cursor); + // The page has not shown the old reply yet, so its image does not load. + assert.equal((yield* Effect.flip(guestShares.media(session, shot))).reason, "unavailable"); + // Only a session pages. + assert.isTrue(Exit.isFailure(yield* Effect.exit(guestShares.history(credential, cursor!)))); + + const older: Array = []; + while (cursor !== null) { + const page: OrchestrationV2ThreadHistoryPage = yield* guestShares.history(session, cursor); + older.push(...ids(page.items)); + cursor = page.hasMoreHistory ? page.nextCursor : null; + } + assert.include(older, "item:old-image"); + assert.deepStrictEqual(yield* guestShares.media(session, shot), { + path: shot, + mimeType: "image/png", + }); + }).pipe(Effect.scoped, Effect.provide(layerTest)), +); + it.effect("ends a running command's output once the owner stops sharing", () => Effect.gen(function* () { const { orchestrator, guestShares, credential } = yield* shareThread; diff --git a/apps/server/src/guest/GuestShareService.ts b/apps/server/src/guest/GuestShareService.ts index 347bdc11ece5b..2a1bf853507ba 100644 --- a/apps/server/src/guest/GuestShareService.ts +++ b/apps/server/src/guest/GuestShareService.ts @@ -15,6 +15,7 @@ import { type OrchestrationV2CommandOutputFrame, type OrchestrationV2DomainEvent, type OrchestrationV2GetTurnItemResult, + type OrchestrationV2ThreadHistoryPage, type OrchestrationV2TurnItem, ThreadId, TurnItemId, @@ -185,6 +186,11 @@ export class GuestShareService extends Context.Service< session: string, itemId: string, ) => Effect.Effect; + /** Older turns of the shared thread, as the owner's client pages them. */ + readonly history: ( + session: string, + cursor: string, + ) => Effect.Effect; /** A command's output frames, as the owner's expanded row gets them, until the link ends. */ readonly commandOutput: ( session: string, @@ -665,6 +671,21 @@ const make = Effect.gen(function* () { }; }); + const history = Effect.fn("GuestShareService.history")(function* ( + session: string, + cursor: string, + ) { + const { thread, share } = yield* authorize(session, "session"); + const page = yield* orchestrator + .getThreadHistoryPage(thread.id, cursor) + .pipe(Effect.mapError(() => unavailable)); + rememberSent( + share.id, + page.items.map((row) => row.item), + ); + return page; + }); + const commandOutput = Effect.fn("GuestShareService.commandOutput")(function* ( session: string, itemId: string, @@ -739,6 +760,7 @@ const make = Effect.gen(function* () { stream, render, item, + history, commandOutput, attachment, media, diff --git a/apps/server/src/guest/http.ts b/apps/server/src/guest/http.ts index bf368ac884d92..b3e856200c845 100644 --- a/apps/server/src/guest/http.ts +++ b/apps/server/src/guest/http.ts @@ -1,6 +1,7 @@ import { GUEST_ATTACHMENT_ROUTE, GUEST_COMMAND_OUTPUT_ROUTE, + GUEST_HISTORY_ROUTE, GUEST_ITEM_ROUTE, GUEST_MEDIA_ROUTE, GUEST_RENDER_ROUTE, @@ -16,6 +17,7 @@ import { isProviderSendTurnSupportedImageMimeType, OrchestrationV2CommandOutputFrame, OrchestrationV2GetTurnItemResult, + OrchestrationV2ThreadHistoryPage, } from "@t3tools/contracts"; import * as ByteSize from "effect/ByteSize"; import * as Duration from "effect/Duration"; @@ -52,6 +54,7 @@ const encodeOutputFrame = Schema.encodeEffect( Schema.fromJsonString(OrchestrationV2CommandOutputFrame), ); const encodeItem = Schema.encodeEffect(Schema.toCodecJson(OrchestrationV2GetTurnItemResult)); +const encodeHistoryPage = Schema.encodeEffect(Schema.toCodecJson(OrchestrationV2ThreadHistoryPage)); const errorResponse = (error: GuestShareAccessError) => HttpServerResponse.json( @@ -292,6 +295,26 @@ const itemRoute = HttpRouter.add( }), ); +const historyRoute = HttpRouter.add( + "GET", + GUEST_HISTORY_ROUTE, + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const cursor = new URLSearchParams(request.url.split("?", 2)[1] ?? "").get("cursor"); + if (!cursor) return yield* badRequest; + const guestShares = yield* GuestShareService; + return yield* guestShares.history(yield* sessionToken, cursor).pipe( + Effect.flatMap((page) => + encodeHistoryPage(page).pipe( + Effect.mapError(() => new GuestShareAccessError({ reason: "unavailable" })), + ), + ), + Effect.map((body) => HttpServerResponse.jsonUnsafe(body, { headers: NO_STORE })), + Effect.catchTags({ GuestShareAccessError: errorResponse }), + ); + }), +); + // Ends when the command settles; the page closes its EventSource on that last frame, so the // browser does not reconnect. const commandOutputRoute = HttpRouter.add( @@ -365,6 +388,7 @@ export const layer = Layer.mergeAll( streamRoute, renderRoute, itemRoute, + historyRoute, commandOutputRoute, attachmentRoute, mediaRoute, diff --git a/apps/server/src/htmlRender/HtmlRender.test.ts b/apps/server/src/htmlRender/HtmlRender.test.ts index 80d367b2f8ac9..1916327672516 100644 --- a/apps/server/src/htmlRender/HtmlRender.test.ts +++ b/apps/server/src/htmlRender/HtmlRender.test.ts @@ -316,6 +316,58 @@ describe("HtmlRender", () => { }, ); + // The fake browser is a shebang script, which Windows cannot execute. + describe.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "when the browser cannot be signalled", + () => { + // A browser T3 cannot signal, as under AppArmor in an LXC container, must still exit. + it.live("asks the browser to close when the preview finishes instead of signalling it", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-fake-browser-", + }); + const executable = path.join(directory, "chrome-headless-shell"); + const exitRecord = path.join(directory, "exited"); + yield* fileSystem.writeFileString( + executable, + `#!${process.execPath} +const fs = require("node:fs"); +process.on("SIGTERM", () => {}); +const quit = () => { fs.writeFileSync(${JSON.stringify(exitRecord)}, "asked to close"); process.exit(0); }; +const reply = (message) => fs.writeSync(4, JSON.stringify(message) + "\\0"); +const results = { + "Target.createTarget": { targetId: "page" }, + "Target.attachToTarget": { sessionId: "session" }, + "Runtime.evaluate": { result: { value: 100 } }, + "Page.captureScreenshot": { data: "iVBORw0KGgo=" }, +}; +let buffered = ""; +fs.createReadStream(null, { fd: 3 }).on("end", quit).on("data", (chunk) => { + const messages = (buffered + chunk).split("\\0"); + buffered = messages.pop(); + for (const { id, method, sessionId } of messages.map((raw) => JSON.parse(raw))) { + if (method === "Browser.close") quit(); + reply({ id, sessionId, result: results[method] ?? {} }); + if (method === "Page.navigate") reply({ sessionId, method: "Page.loadEventFired", params: {} }); + } +}); +`, + { mode: 0o755 }, + ); + + yield* HtmlRender.HtmlRender.use((htmlRender) => + htmlRender.preview({ html: "

x

" }), + ).pipe(Effect.provide(layerHtmlRender(executable))); + + const exitedBy = yield* fileSystem.readFileString(exitRecord).pipe(Effect.option); + expect(Option.getOrElse(exitedBy, () => "killed by a signal")).toBe("asked to close"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + }, + ); + it.live("matches preview media queries and text colors to the requested appearance", (ctx) => Effect.gen(function* () { const executable = (yield* HostProcess.Environment)[TEST_BROWSER_ENV]; diff --git a/apps/server/src/htmlRender/headlessChrome.ts b/apps/server/src/htmlRender/headlessChrome.ts index aaf64bdcecc36..13ce0c6845740 100644 --- a/apps/server/src/htmlRender/headlessChrome.ts +++ b/apps/server/src/htmlRender/headlessChrome.ts @@ -247,7 +247,7 @@ const launchBrowser = Effect.fnUntraced(function* (input: { let disconnected: BrowserFailure | undefined; let nextId = 0; - const post = (method: string, params: Record, sessionId: string) => + const post = (method: string, params: Record, sessionId?: string) => Queue.offer( outgoing, new TextEncoder().encode( @@ -374,6 +374,18 @@ const launchBrowser = Effect.fnUntraced(function* (input: { Effect.forkScoped, ); + // Signals can miss a confined browser (AppArmor in LXC), so ask it to quit first. + // Runs before the spawn's release, which still signals a browser that stays up. + yield* Effect.addFinalizer(() => + disconnected + ? Effect.void + : post("Browser.close", {}).pipe( + Effect.andThen(child.exitCode), + Effect.timeout("2 seconds"), + Effect.ignore, + ), + ); + const send = Effect.fn("headlessChrome.send")(function* ( method: string, params: Record, diff --git a/apps/server/src/http.test.ts b/apps/server/src/http.test.ts index 0319bbb55c03d..5f7df20c738c8 100644 --- a/apps/server/src/http.test.ts +++ b/apps/server/src/http.test.ts @@ -13,7 +13,10 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Queue from "effect/Queue"; +import * as Tracer from "effect/Tracer"; import { HttpClient, HttpClientRequest, HttpRouter, HttpServerResponse } from "effect/http"; +import { ASSET_ROUTE_PREFIX } from "./assets/AssetAccess.ts"; +import { ATTACHMENT_UPLOAD_ROUTE_PREFIX } from "./assets/AttachmentUpload.ts"; import { openMediaFile } from "./assets/MediaFile.ts"; import { @@ -139,6 +142,66 @@ describe("browser API CORS", () => { }); }); +describe("untraced requests", () => { + it.effect("keeps signed asset and upload URLs out of HTTP server spans", () => + Effect.gen(function* () { + const serverSpans: Array = []; + const layerTracer = Layer.succeed( + Tracer.Tracer, + Tracer.make({ + span: (options) => { + if (options.kind === "server") serverSpans.push(options.name); + return new Tracer.NativeSpan(options); + }, + }), + ); + const layerRoutes = Layer.effectDiscard( + Effect.gen(function* () { + const router = yield* HttpRouter.HttpRouter; + yield* router.add("GET", `${ASSET_ROUTE_PREFIX}/*`, HttpServerResponse.empty()); + yield* router.add( + "POST", + `${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/*`, + HttpServerResponse.empty(), + ); + yield* router.add("GET", "/api/environment", HttpServerResponse.empty()); + }), + ); + const services = yield* Layer.build( + HttpRouter.serve(layerRoutes, { disableListenLog: true, disableLogger: true }).pipe( + ServerHttp.withUntracedRequests, + Layer.provideMerge(NodeHttpServer.layerTest), + Layer.provide(layerTracer), + ), + ); + const client = Context.get(services, HttpClient.HttpClient); + + // Each URL is a bearer credential: url.full and url.path would carry it. + for (const request of [ + HttpClientRequest.get(`${ASSET_ROUTE_PREFIX}/payload.signature/image.png`), + HttpClientRequest.get(`${ASSET_ROUTE_PREFIX}/payload.signature/image.png?v=1`), + HttpClientRequest.post(`${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/payload.signature`), + HttpClientRequest.post(`${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/payload.signature?v=1`), + HttpClientRequest.head(`${ASSET_ROUTE_PREFIX}/payload.signature/image.png`), + ]) { + expect((yield* client.execute(request)).status).toBe(204); + } + expect(serverSpans).toEqual([]); + + expect((yield* client.execute(HttpClientRequest.get("/api/environment"))).status).toBe(204); + expect(serverSpans).toEqual(["GET"]); + + for (const path of [ + `${ASSET_ROUTE_PREFIX}-other/file`, + `${ATTACHMENT_UPLOAD_ROUTE_PREFIX}-other/token`, + ]) { + expect((yield* client.execute(HttpClientRequest.get(path))).status).toBe(404); + } + expect(serverSpans).toEqual(["GET", "GET", "GET"]); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); + const layerFileResponse = Layer.mergeAll(NodeHttpPlatform.layer, NodeServices.layer); const makeStaticRequest = Effect.fn("HttpTest.makeStaticRequest")(function* (staticDir: string) { diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index 8f9737a3efc18..e792a233c6f06 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -391,8 +391,13 @@ const UNTRACED_REQUEST_PATHS: ReadonlySet = new Set([OTLP_TRACES_PROXY_P const layerUntracedRequests = Layer.succeed(HttpMiddleware.TracerDisabledWhen)((request) => { const queryIndex = request.url.indexOf("?"); const path = queryIndex === -1 ? request.url : request.url.slice(0, queryIndex); - // Webhook URLs carry their secret token in the path, so they never reach a trace. - return UNTRACED_REQUEST_PATHS.has(path) || path.startsWith(`${WEBHOOK_ROUTE_PREFIX}/`); + // Signed URLs carry bearer tokens in the path, so they never reach a trace. + return ( + UNTRACED_REQUEST_PATHS.has(path) || + path.startsWith(`${WEBHOOK_ROUTE_PREFIX}/`) || + path.startsWith(`${ASSET_ROUTE_PREFIX}/`) || + path.startsWith(`${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/`) + ); }); export const withUntracedRequests = Layer.provide(layerUntracedRequests); diff --git a/apps/server/src/mcp/toolkits/core.test.ts b/apps/server/src/mcp/toolkits/core.test.ts index a978dbc4d1ff9..93aee4d52f688 100644 --- a/apps/server/src/mcp/toolkits/core.test.ts +++ b/apps/server/src/mcp/toolkits/core.test.ts @@ -350,6 +350,48 @@ it.effect("records only failed calls of tools that hand off no work", () => { ); }); +it.effect("rejects an unknown organize target instead of archiving the calling thread", () => { + const dispatched: Array = []; + return Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const outcome = yield* server + .callTool({ + name: "t3_thread_organize", + arguments: { action: "archive", threadIds: ["other-thread"] }, + }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, scope), + Effect.provideService(McpSchema.McpServerClient, client), + Effect.result, + ); + expect(dispatched).toEqual([]); + expect(outcome).toMatchObject({ + _tag: "Failure", + failure: { message: expect.stringContaining("Expected no excess property") }, + }); + }).pipe( + Effect.provide( + layerThreadToolkit.pipe( + Layer.provideMerge(McpServer.McpServer.layer), + Layer.provide(NodeCrypto.layer), + Layer.provide( + Layer.mock(ThreadManagement.ThreadManagementService)({ + getThreadShell: (id) => Effect.succeed(McpToolAccessTestkit.liveThreadShell(id)), + getProjectThreadRecords: ({ threadId: id }) => + Effect.succeed( + McpToolAccessTestkit.idleThreadProjection(McpToolAccessTestkit.liveThreadShell(id)), + ), + dispatch: (command) => { + dispatched.push(command); + return Effect.succeed({ sequence: 1 } as never); + }, + }), + ), + ), + ), + ); +}); + it.effect("keeps unexpected handler defects private through the production registration", () => Effect.gen(function* () { const server = yield* McpServer.McpServer; diff --git a/apps/server/src/mcp/toolkits/thread/tools.ts b/apps/server/src/mcp/toolkits/thread/tools.ts index 4b566c44a69d1..018acac5c715f 100644 --- a/apps/server/src/mcp/toolkits/thread/tools.ts +++ b/apps/server/src/mcp/toolkits/thread/tools.ts @@ -61,6 +61,7 @@ const ThreadOrganizeTool = Tool.make("t3_thread_organize", { ], }) .annotate(Tool.Title, "Organize a thread") + .annotate(Tool.Strict, true) .annotate(Tool.Destructive, true); const commandTool = { diff --git a/apps/server/src/observability/HeapAttribution.test.ts b/apps/server/src/observability/HeapAttribution.test.ts new file mode 100644 index 0000000000000..b355ba5c35e1f --- /dev/null +++ b/apps/server/src/observability/HeapAttribution.test.ts @@ -0,0 +1,69 @@ +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Context from "effect/Context"; +import * as TestClock from "effect/testing/TestClock"; +import * as Tracer from "effect/Tracer"; + +import * as HeapAttribution from "./HeapAttribution.ts"; + +const MB = 1024 * 1024; + +it.effect("names the span that grew the heap in a spike, not an older open or ended one", () => + Effect.gen(function* () { + const ended: Array = []; + const inner = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + const end = span.end.bind(span); + span.end = (endTime, exit) => { + end(endTime, exit); + ended.push(span); + }; + return span; + }, + }); + let heap = 300 * MB; + const attribution = yield* HeapAttribution.make(inner, () => heap); + yield* attribution.run; + // The sampler's baseline: heap before the spans allocate. + yield* TestClock.adjust("1 second"); + const open = (name: string) => + attribution.tracer.span({ + name, + parent: Option.none(), + annotations: Context.empty(), + links: [], + startTime: 0n, + kind: "internal", + root: false, + sampled: true, + }); + + const subscription = open("ws.rpc.orchestration.subscribeShell"); + const old = open("ProjectionStore.getSnapshot"); + // Slow drift never spikes, but leaves the long-lived span far above its start. + for (let step = 0; step < 5; step++) { + heap += 80 * MB; + yield* TestClock.adjust("11 seconds"); + if (step === 1) old.end(1n, undefined as never); + } + const small = open("ProjectionStore.getThreadShell"); + const big = open("ProjectionStore.getThreadProjection"); + heap += 2 * MB; + small.end(1n, undefined as never); + heap += 198 * MB; + yield* TestClock.adjust("2 seconds"); + + const spike = ended.find((span) => span.name === "server.heap.spike"); + assert.isDefined(spike); + assert.deepEqual(spike!.attributes.get("suspects"), [ + "ProjectionStore.getThreadProjection +200 MB", + ]); + big.end(2n, undefined as never); + subscription.end(3n, undefined as never); + assert.equal(big.attributes.get("memory.heap_growth_bytes"), 200 * MB); + assert.isUndefined(small.attributes.get("memory.heap_growth_bytes")); + assert.equal(old.attributes.get("memory.heap_growth_bytes"), 160 * MB); + }), +); diff --git a/apps/server/src/observability/HeapAttribution.ts b/apps/server/src/observability/HeapAttribution.ts new file mode 100644 index 0000000000000..4b18efd0fb911 --- /dev/null +++ b/apps/server/src/observability/HeapAttribution.ts @@ -0,0 +1,108 @@ +// @effect-diagnostics nodeBuiltinImport:off - only node:v8 exposes a heap reading cheap enough to take per span. +import * as NodeV8 from "node:v8"; + +import * as Effect from "effect/Effect"; +import type * as Scope from "effect/Scope"; +import * as Tracer from "effect/Tracer"; + +const MB = 1024 * 1024; +// Smaller growth is noise from concurrent fibers; an attribute on every span would bloat exports. +const SPAN_GROWTH_ATTRIBUTE_BYTES = 8 * MB; +const SPIKE_GROWTH_BYTES = 128 * MB; +const SPIKE_WINDOW_SAMPLES = 10; +const SPIKE_COOLDOWN_SAMPLES = 60; +const RECENT_SPANS = 32; +const REPORTED_SPANS = 8; + +const readHeapUsed = () => NodeV8.getHeapStatistics().used_heap_size; + +interface Growth { + readonly name: string; + readonly growthBytes: number; +} + +/** + * Heap growth is approximate: it includes other fibers' allocations and GC, so it names + * suspects, not culprits. A spike lists what was open or just grew. + */ +export const make = (tracer: Tracer.Tracer, heapUsed: () => number = readHeapUsed) => + Effect.sync(() => { + const open = new Map< + Tracer.Span, + { readonly name: string; readonly startHeap: number; readonly startTick: number } + >(); + const recent: Array = []; + const samples: Array = []; + let cooldown = 0; + let tick = 0; + + const wrapped = Tracer.make({ + ...(tracer.context === undefined ? {} : { context: tracer.context }), + span(options) { + const span = tracer.span(options); + const startHeap = heapUsed(); + open.set(span, { name: options.name, startHeap, startTick: tick }); + const end = span.end.bind(span); + span.end = (endTime, exit) => { + open.delete(span); + const growthBytes = heapUsed() - startHeap; + if (growthBytes >= SPAN_GROWTH_ATTRIBUTE_BYTES) { + span.attribute("memory.heap_growth_bytes", growthBytes); + recent.push({ name: options.name, growthBytes, endTick: tick }); + if (recent.length > RECENT_SPANS) recent.shift(); + } + end(endTime, exit); + }; + return span; + }, + }); + + // Untraced: it runs every second and only emits a span when the heap spikes. + const sample = Effect.fnUntraced(function* () { + const heap = heapUsed(); + tick++; + samples.push(heap); + if (samples.length > SPIKE_WINDOW_SAMPLES) samples.shift(); + if (cooldown > 0) { + cooldown--; + return; + } + const growthBytes = heap - Math.min(...samples); + if (growthBytes < SPIKE_GROWTH_BYTES) return; + cooldown = SPIKE_COOLDOWN_SAMPLES; + const suspects = [ + // Spans open since before the window, such as subscriptions, would claim all of its growth. + ...[...open.values()] + .filter(({ startTick }) => tick - startTick <= samples.length) + .map(({ name, startHeap }) => ({ name, growthBytes: heap - startHeap })), + ...recent.filter(({ endTick }) => tick - endTick <= samples.length), + ] + .filter((suspect) => suspect.growthBytes >= SPAN_GROWTH_ATTRIBUTE_BYTES) + .toSorted((left, right) => right.growthBytes - left.growthBytes) + .slice(0, REPORTED_SPANS) + .map(({ name, growthBytes }) => `${name} +${Math.round(growthBytes / MB)} MB`); + recent.length = 0; + // A span, not a log: this fiber starts before the server logger is provided. + yield* Effect.void.pipe( + Effect.withSpan("server.heap.spike", { + root: true, + level: "Warn", + attributes: { + heapUsedMb: Math.round(heap / MB), + growthMb: Math.round(growthBytes / MB), + suspects, + }, + }), + ); + }); + + const run: Effect.Effect = sample().pipe( + Effect.andThen(Effect.sleep("1 second")), + Effect.forever, + Effect.withTracer(wrapped), + Effect.forkScoped, + Effect.asVoid, + ); + + return { tracer: wrapped, run }; + }); diff --git a/apps/server/src/observability/Observability.ts b/apps/server/src/observability/Observability.ts index 343851e3d1d11..166d8ac194a8c 100644 --- a/apps/server/src/observability/Observability.ts +++ b/apps/server/src/observability/Observability.ts @@ -16,6 +16,7 @@ import * as ServerConfig from "../config.ts"; import * as ResourceAttribution from "../resourceTelemetry/ResourceAttribution.ts"; import * as ServerLogger from "../serverLogger.ts"; import * as BrowserTraceCollector from "./BrowserTraceCollector.ts"; +import * as HeapAttribution from "./HeapAttribution.ts"; export const layer = Layer.unwrap( Effect.gen(function* () { @@ -71,8 +72,11 @@ export const layer = Layer.unwrap( ...(delegate ? { delegate } : {}), }); + const heapAttribution = yield* HeapAttribution.make(tracer); + yield* heapAttribution.run; + return Layer.mergeAll( - Layer.succeed(Tracer.Tracer, tracer), + Layer.succeed(Tracer.Tracer, heapAttribution.tracer), BrowserTraceCollector.layer(sink), ); }), diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index da564fea1b933..ce73c5dcea6ca 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -14,6 +14,7 @@ const RPC_AGGREGATES = { [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: "orchestrationV2", [ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: "orchestration", + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.getTurnItem]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.getTurnDiff]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.getFullThreadDiff]: "orchestration", @@ -182,6 +183,7 @@ const RPC_AGGREGATES = { [WS_METHODS.vcsCreateRef]: "vcs", [WS_METHODS.vcsSwitchRef]: "vcs", [WS_METHODS.vcsInit]: "vcs", + [WS_METHODS.reviewGetBranchCommits]: "review", [WS_METHODS.reviewGetDiffPreview]: "review", [WS_METHODS.reviewGetDiffFileContents]: "review", [WS_METHODS.terminalOpen]: "terminal", diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 8a68e92b89160..b924f6efb3e70 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -7498,6 +7498,96 @@ describe("AcpAdapterV2", () => { }).pipe(Effect.provide(layerTest), Effect.scoped), ); + it.live("lets a caller abandon Stop while hard teardown stalls, and finishes the teardown", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const path = yield* Path.Path; + const selfInvocation = yield* resolveSelfInvocation(); + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const teardownStarted = yield* Deferred.make(); + const releaseTeardown = yield* Deferred.make(); + let runtimeOrdinalSeen = 0; + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = yield* makeAcpAdapterV2({ + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + interruptPromptOnCancel: true, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + ownDetachedProcessGroup: true, + processGroupPlatform: "win32", + windowsProcessTreeTerminator: () => + Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseTeardown)), + ), + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = runtimeOrdinal; + return runtimeOrdinal === 1 ? { T3_ACP_HANG_PROMPT_FOREVER: "1" } : {}; + }, + protocolEvents, + }), + }, + selfInvocation, + }); + const threadId = ThreadId.make("thread-acp-abandoned-hard-teardown"); + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-abandoned-hard-teardown"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: acpScopedNativeId(instanceId, "mock-session-1:turn:1"), + }); + yield* pollProtocolMethods(protocolEvents); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId, requestRuntimeRestart: true }) + .pipe(Effect.forkScoped); + yield* Deferred.await(teardownStarted); + + // The session manager's Stop timeout interrupts this call; it must not wait for teardown. + yield* Fiber.interrupt(interruptFiber); + + const startFiber = yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), + ) + .pipe(Effect.forkScoped); + yield* Deferred.succeed(releaseTeardown, undefined); + yield* Fiber.join(startFiber); + assert.equal(runtimeOrdinalSeen, 2); + const replacementMethods = yield* pollProtocolMethods(protocolEvents); + assert.equal(replacementMethods.filter((method) => method === "session/resume").length, 1); + }).pipe(Effect.provide(layerTest), Effect.scoped), + ); + it.effect( "direct Stop on a deferred subagent hold terminalizes the subagent and does not carry it forward", () => diff --git a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts index 90226cc68bd2a..0f5670f1d2de0 100644 --- a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts @@ -24,6 +24,7 @@ import * as Path from "effect/Path"; import * as Stream from "effect/Stream"; import { ChildProcessSpawner } from "effect/process"; import type * as EffectAcpSchema from "effect-acp/compat"; +import * as EffectAcpErrors from "effect-acp/errors"; import * as ProviderHost from "@t3tools/provider-core/server/ProviderHost"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; @@ -32,6 +33,7 @@ import type * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSession import { makeAntigravityAcpRuntime } from "../../provider/acp/AntigravityAcpSupport.ts"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { normalizeAntigravitySessionUpdate } from "../../provider/acp/AntigravityProtocol.ts"; +import { ANTIGRAVITY_SIGN_IN_REQUIRED_MESSAGE } from "../../provider/antigravityAuthSupport.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import { @@ -126,6 +128,63 @@ const layerSession = Layer.mergeAll( TestProviderHost.layer().pipe(Layer.provide(NodeServices.layer)), ); +describe("AntigravityAdapterV2 session open", () => { + it.effect.each([ + { + name: "browser sign-in", + cause: new EffectAcpErrors.AcpTransportError({ + detail: ANTIGRAVITY_SIGN_IN_REQUIRED_MESSAGE, + cause: undefined, + }), + requiresSignIn: true, + }, + { + name: "ACP authentication required", + cause: EffectAcpErrors.AcpRequestError.authRequired(), + requiresSignIn: true, + }, + { + name: "transport failure", + cause: new EffectAcpErrors.AcpTransportError({ + detail: "Connection closed", + cause: undefined, + }), + requiresSignIn: undefined, + }, + ] as const)("classifies $name during session open", ({ cause, requiresSignIn }) => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("antigravity-sign-in-test"); + const adapter = yield* makeAntigravityAdapterV2({ + instanceId, + selfInvocation: yield* resolveSelfInvocation(), + makeRuntime: () => Effect.fail(cause), + withProcess: (_stop, task) => task, + // @effect-diagnostics-next-line effectSucceedWithVoid:off - This optional result requires undefined, not the wider void type. + defaultModel: Effect.succeed(undefined), + }); + const threadId = ThreadId.make("thread-antigravity-sign-in"); + const error = yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-antigravity-sign-in"), + modelSelection: { instanceId, model: "gemini-test-low" }, + runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }), + }) + .pipe(Effect.flip); + + assert.equal(error._tag, "ProviderAdapterOpenSessionError"); + assert.equal( + error._tag === "ProviderAdapterOpenSessionError" && error.requiresSignIn, + requiresSignIn, + ); + }).pipe(Effect.provide(layerSession), Effect.scoped), + ); +}); + describe("AntigravityAdapterV2 client file system", () => { it.effect("serves confined reads and leaves edits to the agent's diff tools", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts index 4d06da900f4a8..9ccaca83b542f 100644 --- a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts @@ -34,6 +34,7 @@ import { makeAntigravityUserInputResponse, normalizeAntigravityToolCall, } from "../../provider/acp/AntigravityProtocol.ts"; +import { isAntigravitySignInRequiredError } from "../../provider/antigravityAuthSupport.ts"; import { AcpProviderCapabilitiesV2, makeAcpAdapterV2, @@ -238,6 +239,7 @@ export function makeAntigravityAcpAdapterFlavor( extractSubagentUpdate: extractAntigravitySubagentUpdate, supportsImagePrompts: true, supportsCompaction: true, + requiresSignIn: isAntigravitySignInRequiredError, }; } diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index a98ac8474276e..62e8065501b0f 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -40,6 +40,7 @@ import { OrchestrationV2ProviderThread, ProjectId, ProviderInstanceId, + type ResolvedProviderInstanceEnvironment, ProviderDriverKind, type ProviderReplayTranscript, type ProviderApprovalDecision, @@ -94,7 +95,6 @@ import type * as ProviderContinuationRequests from "@t3tools/provider-core/serve import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import * as ClaudeAdapterV2 from "./ClaudeAdapterV2.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; -import type { ResolvedProviderEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import { ClaudeOrchestratorReplayHarness } from "./ClaudeAdapterV2.testkit.ts"; import { provideDeterministicTestRuntime } from "../testkit/DeterministicRuntime.ts"; import { runOrchestratorV2Scenario } from "../testkit/OrchestratorScenario.ts"; @@ -297,6 +297,28 @@ describe("ClaudeAdapterV2 runtime query policy", () => { assert.isUndefined(options.extraArgs?.["dangerously-skip-permissions"]); }); + it("passes every --add-dir launch directory to Claude", () => { + const options = ClaudeAdapterV2.makeClaudeQueryOptions({ + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + nativeThreadId: "native-add-dir", + resume: false, + cwd: "/workspace", + attachmentsDir: "/attachments", + settings: { + ...AUTO_COMPACT_CLAUDE_SETTINGS, + launchArgs: "--add-dir ../apps ../lib --verbose --add-dir=/shared", + }, + }); + assert.deepEqual(options.additionalDirectories, [ + "/workspace", + "/attachments", + "../apps", + "../lib", + "/shared", + ]); + assert.isUndefined(options.extraArgs?.["add-dir"]); + }); + it("gives delegated children Claude's subagent prompt cache TTL", () => { const lineage = (relationshipToParent: "fork" | "subagent" | null) => ({ lineage: { @@ -1196,7 +1218,7 @@ describe("ClaudeAdapterV2 approval cancellation", () => { const captureSdkExecutablePaths = Effect.fn("captureSdkExecutablePaths")(function* ( binaryPath: string, - environment: ResolvedProviderEnvironment = [], + environment: ResolvedProviderInstanceEnvironment = [], ) { const executablePaths: Array = []; const taskToolEnvironment: Array = []; @@ -9166,6 +9188,172 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + // regression: #17531 a mod that holds a tool behind a focused pane must fail the turn, not hang it + it.effect.each([ + { case: "fails a turn whose mod requests focus for a pane T3 cannot show", focus: true }, + { case: "keeps a turn running beside a passive mod pane", focus: false }, + ] as const)("$case", ({ focus }) => + Effect.scoped( + Effect.gen(function* () { + const closed = yield* Deferred.make(); + const harness = yield* makeWakeHarnessWithOptions({ + close: () => Deferred.succeed(closed, undefined), + }); + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-mod-pane-focus"), + text: "Run rm -r build", + attachments: [], + }), + ); + // Captured from Claude 2.1.303 with the blast-radius mod holding a Bash + // call: the mod waits for keyboard input in its pane and nothing else + // arrives until it gives up. + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "system", + subtype: "ui_panes", + panes: [ + { id: "blast-radius", title: "Blast Radius", plugin: "blast-radius", rows: 10 }, + ], + shown_id: "blast-radius", + focused_id: null, + focus_requested_id: focus ? "blast-radius" : null, + uuid: "00000000-0000-4000-8000-000000000795", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "system", + subtype: "ui_invalidate", + event: "ui.render", + uuid: "00000000-0000-4000-8000-000000000796", + session_id: WAKE_NATIVE_SESSION, + }), + ); + // Frames are handled in order, so a later notice proves the pane frame + // was handled without the turn ending. + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "system", + subtype: "informational", + content: "Processing barrier.", + level: "warning", + uuid: "00000000-0000-4000-8000-000000000797", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* harness.until( + () => + harness.terminalEvents().length > 0 || + harness.events.some( + (event) => + event.type === "turn_item.updated" && event.turnItem.type === "system_notice", + ), + ); + const [terminal] = harness.terminalEvents(); + if (!focus) { + assert.equal(terminal, undefined); + return; + } + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.include(terminal.failure.message, "blast-radius"); + // The held tool call is abandoned by closing the query, never answered. + yield* Deferred.await(closed); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + // regression: #17531 a follow-up sent while the failed mod turn's process closes gets a new process + it.effect("runs a follow-up in a fresh query while the mod-blocked query is closing", () => + Effect.scoped( + Effect.gen(function* () { + const releaseClose = yield* Deferred.make(); + const replacementClosedOld = yield* Deferred.make(); + let oldCloses = 0; + const harness = yield* makeWakeHarnessWithOptions({ + // The held CLI never answers an interrupt; closure must not need one. + interrupt: Effect.never, + close: (sdkMessages) => + Effect.suspend(() => + ++oldCloses === 2 ? Deferred.succeed(replacementClosedOld, undefined) : Effect.void, + ).pipe( + Effect.andThen(Deferred.await(releaseClose)), + Effect.andThen(Queue.shutdown(sdkMessages)), + ), + freshQueueOnReopen: true, + }); + // Added after the harness so it runs first: a failed assertion must not + // leave teardown waiting on the held close. + yield* Effect.addFinalizer(() => Deferred.succeed(releaseClose, undefined)); + const now = yield* DateTime.now; + const turnInput = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-mod-pane-blocked"), + text: "Run rm -r build", + attachments: [], + }); + yield* harness.runtime.startTurn(turnInput); + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "system", + subtype: "ui_panes", + panes: [ + { id: "blast-radius", title: "Blast Radius", plugin: "blast-radius", rows: 10 }, + ], + shown_id: "blast-radius", + focused_id: null, + focus_requested_id: "blast-radius", + uuid: "00000000-0000-4000-8000-000000000798", + session_id: WAKE_NATIVE_SESSION, + }), + ); + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "failed"); + + const followUp = yield* harness.runtime + .startTurn({ + ...turnInput, + attemptId: RunAttemptId.make("attempt-claude-after-mod-pane"), + providerTurnOrdinal: 2, + }) + .pipe(Effect.forkScoped); + // Reusing the closing process would offer the prompt to it instead. + yield* Effect.raceFirst( + Deferred.await(replacementClosedOld), + harness.until(() => harness.offeredMessages.length >= 2), + ); + assert.isTrue(yield* Deferred.isDone(replacementClosedOld)); + yield* Deferred.succeed(releaseClose, undefined); + yield* Fiber.join(followUp); + assert.equal(harness.processQueues.length, 2); + yield* harness.offerAndWait( + makeResultFrame({ uuid: "after-mod-pane-result", result: "Done." }), + ); + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "completed"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + // regression: #16897 a prompt whose uuid the session already holds must not hang or end early it.effect("keeps a started prompt running until its result", () => Effect.scoped( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 32383332acf49..38a9c3022e2b6 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -135,7 +135,10 @@ import { planClaudeSkillDispatch, type ClaudeSkillTarget, } from "../../provider/Drivers/ClaudeSkillDispatch.ts"; -import { discoverClaudeSkills } from "../../provider/Drivers/ClaudeSkills.ts"; +import { + claudeLaunchArgDirectories, + discoverClaudeSkills, +} from "../../provider/Drivers/ClaudeSkills.ts"; import { compileClaudeModelSelection, type CompiledClaudeModelSelection, @@ -320,6 +323,8 @@ const WORKFLOW_MEMBER_TRANSCRIPT_FLUSH_MS = 500; // A steer waits at most this long, from its request, for its foreground // subagents to move to the background before it is sent. const BACKGROUND_MOVE_TIMEOUT_MS = 5_000; +// How long a Stop waits for the CLI to exit before the user is offered Force stop. +const CLAUDE_STOP_CONFIRM_TIMEOUT_MS = 10_000; const CLAUDE_GROUP_WATCH_SCRIPT = [ "{ trap '' TERM; read -r _ <&3; kill -TERM 0; sleep 5; kill -KILL 0; } /dev/null 2>&1 &", @@ -1475,6 +1480,8 @@ export function makeClaudeQueryOptions(input: { }): ClaudeAgentSdkQueryOptions { const compiledSelection = compileClaudeModelSelection(input.modelSelection); const { + // claudeLaunchArgDirectories reads every --add-dir into additionalDirectories. + "add-dir": _launchArgAddDir, "permission-mode": launchArgPermissionMode, "dangerously-skip-permissions": launchArgSkipPermissions, "append-system-prompt": launchArgAppendSystemPrompt, @@ -1601,6 +1608,7 @@ export function makeClaudeQueryOptions(input: { const additionalDirectories = [ ...(input.cwd === null ? [] : [input.cwd]), ...(input.attachmentsDir === undefined ? [] : [input.attachmentsDir]), + ...claudeLaunchArgDirectories(input.settings?.launchArgs), ]; const withDirectories = additionalDirectories.length === 0 ? options : { ...options, additionalDirectories }; @@ -4220,6 +4228,20 @@ function claudeTaskString(value: unknown, key: string): string | undefined { return typeof field === "string" && field.trim().length > 0 ? field.trim() : undefined; } +// A Claude Code mod can hold a tool call until the user acts in its terminal +// pane. T3 cannot provide the requested pane interaction. +// Passive panes never request focus and keep working. +function claudeFocusRequestedPanePlugin(message: SDKMessage): string | null { + if (message.type !== "system" || claudeTaskField(message, "subtype") !== "ui_panes") return null; + const paneId = claudeTaskString(message, "focus_requested_id"); + if (paneId === undefined) return null; + const panes = claudeTaskField(message, "panes"); + const pane = Array.isArray(panes) + ? panes.find((entry) => claudeTaskField(entry, "id") === paneId) + : undefined; + return claudeTaskString(pane, "plugin") ?? paneId; +} + function claudeTaskStatus(value: unknown): OrchestrationV2PlanStep["status"] | undefined { switch (value) { case "pending": @@ -8273,6 +8295,28 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } return; } + const focusedPlugin = claudeFocusRequestedPanePlugin(message); + if (focusedPlugin !== null && context === (yield* Ref.get(activeTurn))) { + yield* Effect.logWarning("orchestration-v2.claude-mod-pane-focus-unsupported", { + providerTurnId: context.providerTurnId, + plugin: focusedPlugin, + }); + liveQuery.stopping = true; + yield* finalizeActiveTurn({ + context, + status: "failed", + completedAt: yield* DateTime.now, + failure: makeProviderFailure({ + class: "provider_error", + code: "claude_mod_ui_unsupported", + message: `The Claude Code mod "${focusedPlugin}" is waiting for input in its terminal pane, which T3 Code cannot show. Disable the mod for sessions run from T3 Code, or run the task in the Claude Code terminal.`, + }), + }); + // Closing ends the CLI process and abandons the held tool call + // without answering it. Not interrupt: the held CLI may never ack it. + yield* liveQuery.query.close.pipe(Effect.ignore); + return; + } yield* trackClaudeGoal({ nativeThreadId: liveQuery.nativeThreadId, context, message }); if (message.type === "stream_event" && !message.parent_tool_use_id) { @@ -9908,6 +9952,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if ( existing !== null && existing.nativeThreadId === nativeThreadId && + !existing.stopping && isClaudeProviderContinuationTurn(turnInput) ) { return existing; @@ -9939,6 +9984,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( existing !== null && existing.nativeThreadId === nativeThreadId && existing.queryPolicyKey === queryPolicyKey && + !existing.stopping && existing.selection.queryIdentity === compiledSelection.queryIdentity ) { // A denied ExitPlanMode leaves Claude in plan mode. Restore only from plan: @@ -10533,44 +10579,36 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( yield* existing.query.interrupt; yield* existing.query.close.pipe(Effect.ignore); const closed = yield* Deferred.await(existing.closed).pipe( - Effect.timeoutOption("10 seconds"), + Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS), ); if (Option.isSome(closed)) { return; } - - const completedAt = yield* DateTime.now; + // The turn stays open: the CLI may still be working, and the user decides + // whether to Force stop it. yield* Effect.logWarning("orchestration-v2.claude-query-interrupt-timeout", { providerSessionId: input.providerSessionId, providerThreadId: turnInput.providerThread.id, providerTurnId: turnInput.providerTurnId, }); - const ownedLiveQuery = yield* Ref.modify(queryContext, (current) => - current?.query === existing.query ? [true, null] : [false, current], - ); - if (ownedLiveQuery) { - existing.stopping = true; - yield* finalizeActiveTurnAfterQueryExit(existing); - yield* endSubagentCallsOfClosedQuery(existing.nativeThreadId, "interrupted"); - } else { - yield* finalizeActiveTurn({ - context: currentTurn, - status: "interrupted", - completedAt, - }); - } - yield* Deferred.succeed(existing.closed, undefined); + return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + timeoutMs: CLAUDE_STOP_CONFIRM_TIMEOUT_MS, + }); }, (effect, turnInput) => effect.pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterInterruptError({ - driver: CLAUDE_PROVIDER, - providerThreadId: turnInput.providerThread.id, - providerTurnId: turnInput.providerTurnId, - cause, - }), + Effect.mapError((cause) => + cause._tag === "ProviderAdapterInterruptTimeoutError" + ? cause + : new ProviderAdapter.ProviderAdapterInterruptError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + cause, + }), ), ), ); diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 69fca8c0825b8..c20994423f3e9 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -16,6 +16,7 @@ import { type OrchestrationV2ProviderThread, type OrchestrationV2ProviderTurn, type OrchestrationV2RuntimeRequest, + type OrchestrationV2Subagent, type OrchestrationV2TurnItem, ProjectId, type ProviderApprovalDecision, @@ -566,6 +567,7 @@ function codexReplayPreamble(input: { readonly nativeTurnId: string; readonly prompt: string; readonly sentPrompt?: string; + readonly effort?: string; }): Array { return [ { @@ -664,7 +666,14 @@ function codexReplayPreamble(input: { approvalsReviewer: "user", sandboxPolicy: { type: "dangerFullAccess" }, summary: "detailed", - collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } }, + ...(input.effort === undefined ? {} : { effort: input.effort }), + collaborationMode: { + mode: "default", + settings: { + model: "gpt-5.4", + ...(input.effort === undefined ? {} : { reasoning_effort: input.effort }), + }, + }, }, }, }, @@ -5129,6 +5138,73 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + const MCP_APP_DEFERRED_RESULT = { + content: [{ type: "text", text: "An interactive form is being shown to the user." }], + structuredContent: { status: "awaiting_user_submission" }, + }; + const mcpAppDeferredTranscript = makeCodexReplayTranscript({ + scenario: MCP_APP_SCENARIO, + entries: mcpAppTranscript.entries.map((entry) => + entry.type === "emit_inbound" && entry.label === "item/completed/app-tool" + ? { + ...entry, + frame: { + method: "item/completed", + params: { + item: { + ...mcpAppToolItem("inProgress"), + status: "failed", + result: MCP_APP_DEFERRED_RESULT, + }, + threadId: MCP_APP_NATIVE_THREAD, + turnId: MCP_APP_NATIVE_TURN, + completedAtMs: 1782622441500, + }, + }, + } + : entry, + ), + }); + + it.effect("captures the app of a tool call that defers to its form", () => + Effect.scoped( + Effect.gen(function* () { + const appCaptured = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness(mcpAppDeferredTranscript, (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "dynamic_tool" && + event.turnItem.status === "failed" + ? Deferred.succeed(appCaptured, event.turnItem) + : Effect.void, + ); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-mcp-app-deferred"), + text: MCP_APP_PROMPT, + }), + ); + const item = yield* Deferred.await(appCaptured); + + const output = item.type === "dynamic_tool" ? (item.output as Record) : {}; + assert.equal( + readMcpAppReference(output[MCP_APP_OUTPUT_KEY])?.resourceUri, + MCP_APP_RESOURCE, + ); + assert.deepEqual(output.result, { ...MCP_APP_DEFERRED_RESULT, isError: true }); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + const ORPHAN_WAIT_SCENARIO = "codex-orphaned-dynamic-tool"; const ORPHAN_WAIT_NATIVE_THREAD = "native-codex-orphan-wait-thread"; const ORPHAN_WAIT_NATIVE_TURN = "native-codex-orphan-wait-turn"; @@ -6211,6 +6287,101 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect.each(["activity", "spawn", "activity then spawn"] as const)( + "creates a child from %s without the parent's effort", + (source) => + Effect.scoped( + Effect.gen(function* () { + const created = yield* Deferred.make(); + const completed = yield* Deferred.make(); + const activity = { + type: "subAgentActivity", + id: "call-codex-resume-spawn", + kind: "started", + agentThreadId: RESUME_CHILD_THREAD, + agentPath: "/root/resume_agent", + }; + const spawn = { + type: "collabAgentToolCall", + id: "call-codex-resume-spawn", + tool: "spawnAgent", + status: "completed", + senderThreadId: RESUME_NATIVE_THREAD, + receiverThreadIds: [RESUME_CHILD_THREAD], + model: "gpt-5.4", + reasoningEffort: "high", + agentsStates: { [RESUME_CHILD_THREAD]: { status: "completed", message: "finished" } }, + }; + const items = + source === "activity" ? [activity] : source === "spawn" ? [spawn] : [activity, spawn]; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: RESUME_SCENARIO, + entries: [ + ...codexReplayPreamble({ + nativeThreadId: RESUME_NATIVE_THREAD, + nativeTurnId: RESUME_NATIVE_TURN, + prompt: RESUME_PROMPT, + effort: "low", + }), + ...items.map((item) => ({ + type: "emit_inbound" as const, + frame: { + method: "item/completed", + params: { + item, + threadId: RESUME_NATIVE_THREAD, + turnId: RESUME_NATIVE_TURN, + completedAtMs: 1782622441000, + }, + }, + })), + ], + }), + (event) => + event.type === "app_thread.created" + ? Deferred.succeed(created, event.appThread) + : event.type === "subagent.updated" && event.subagent.status === "completed" + ? Deferred.succeed(completed, event.subagent) + : Effect.void, + ); + yield* harness.runtime.startTurn({ + ...makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-child-effort"), + text: RESUME_PROMPT, + }), + modelSelection: { + ...CODEX_TEST_MODEL_SELECTION, + options: [{ id: "reasoningEffort", value: "low" }], + }, + }); + const child = yield* Deferred.await(created); + assert.equal( + child.modelSelection.options?.find(({ id }) => id === "reasoningEffort")?.value, + source === "spawn" ? "high" : undefined, + ); + if (source !== "activity") { + const task = yield* Deferred.await(completed); + assert.equal( + task.modelSelection?.options?.find(({ id }) => id === "reasoningEffort")?.value, + "high", + ); + assert.lengthOf( + harness.events.filter((event) => event.type === "app_thread.created"), + 1, + ); + } + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect("preserves a subagent result across a trailing empty final and resume", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 58cd4a0c0c17b..51a686e270389 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -2717,9 +2717,6 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( readonly emitInitialPrompt: boolean; }) { const registeredSubagents = yield* Ref.get(subagentThreads); - if (registeredSubagents.has(input.nativeThreadId)) { - return; - } if (input.model && !subagentSelections.has(input.nativeThreadId)) { yield* updateSubagentSelection( input.nativeThreadId, @@ -2727,6 +2724,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( input.reasoningEffort, ); } + if (registeredSubagents.has(input.nativeThreadId)) { + return; + } const now = yield* DateTime.now; const subagentNodeId = idAllocator.derive.nodeFromProviderItem({ @@ -2803,9 +2803,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( parentNodeId: subagentNodeId, activeProviderThreadId: providerThread.id, providerInstanceId: input.context.input.modelSelection.instanceId, - modelSelection: { + // A child's effort can differ from the parent's. Until Codex reports it, show none. + modelSelection: task.modelSelection ?? { ...input.context.input.modelSelection, model: task.model ?? input.context.input.modelSelection.model, + options: (input.context.input.modelSelection.options ?? []).filter( + ({ id }) => id !== "reasoningEffort", + ), }, title: subagentThreadTitle({ parentTitle: input.context.projectionAppThread.title, @@ -3740,7 +3744,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( Effect.uninterruptibleMask((restore) => Effect.gen(function* () { const reference = - item.status === "completed" && item.result != null && item.error == null + item.result != null && item.error == null ? yield* restore( getNativeThreadId(context.providerThread).pipe( Effect.flatMap((threadId) => @@ -3794,7 +3798,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ? artifacts.turnItem : { ...artifacts.turnItem, - output: { [MCP_APP_OUTPUT_KEY]: reference, result: item.result }, + output: { + [MCP_APP_OUTPUT_KEY]: reference, + result: + item.status === "failed" + ? { ...item.result, isError: true } + : item.result, + }, }; yield* emitProviderEvent({ type: "node.updated", diff --git a/apps/server/src/orchestration-v2/Adapters/PiAdapterV2.testkit.ts b/apps/server/src/orchestration-v2/Adapters/PiAdapterV2.testkit.ts index b256ee9671375..8001da2290baf 100644 --- a/apps/server/src/orchestration-v2/Adapters/PiAdapterV2.testkit.ts +++ b/apps/server/src/orchestration-v2/Adapters/PiAdapterV2.testkit.ts @@ -2,7 +2,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { ProviderReplayEntry, ProviderInstanceId, - type ProviderInstanceEnvironment, + type ResolvedProviderInstanceEnvironment, type ProviderReplayTranscript, type ProviderReplayEntry as ProviderReplayEntryType, } from "@t3tools/contracts"; @@ -364,7 +364,7 @@ export function layer(input: { readonly spawner: Layer.Layer; readonly binaryPath: string; readonly launchArgs: string; - readonly environment?: ProviderInstanceEnvironment; + readonly environment?: ResolvedProviderInstanceEnvironment; }) { return ProviderAdapterRegistry.layerFromAdaptersEffect( PiAdapterV2Driver.create({ diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 42a8f410db348..ad2bc395d083f 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -103,8 +103,6 @@ const layerTestProviderInstanceRegistry = Layer.succeed( Effect.succeed(instanceId === providerInstance.instanceId ? providerInstance : undefined), listInstances: Effect.succeed([providerInstance]), listUnavailable: Effect.succeed([]), - listEnvironments: Effect.succeed(new Map()), - rebuildInstanceWhen: () => Effect.succeed(false), streamChanges: Stream.empty, subscribeChanges: Effect.never, }, @@ -2450,6 +2448,65 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { const task = settled.subagents.find((row) => row.id === child.taskId); assert.equal(task?.status, "cancelled"); assert.equal(task?.completionDelivery?.state, "claimed"); + + // The parent's Stop committed, but the restart cut the effect that stops its children. + const stoppedChild = yield* seedRestartCancelledChild({ + parentThreadId: threadId, + projectId, + parentRunId: runId, + rootNodeId, + name: "restart-parent-stopped-child", + completionWake: "always", + continuationPending: true, + now, + }); + yield* (yield* EventSink.EventSinkV2).write({ + commandId: CommandId.make("command:restart-declined-parent:stop"), + events: [ + { + id: EventId.make("event:restart-declined-parent:interrupt-request"), + type: "turn-item.updated", + threadId, + runId, + nodeId: rootNodeId, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { + id: TurnItemId.make("turn-item:restart-declined-parent:interrupt-request"), + type: "run_interrupt_request", + threadId, + runId, + nodeId: rootNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 90, + status: "completed", + title: "Interrupt requested", + startedAt: now, + completedAt: now, + updatedAt: now, + message: "Stopped by user", + }, + }, + ], + }); + yield* continueRestartedRun({ + threadId: stoppedChild.childThreadId, + sourceRunId: stoppedChild.childRunId, + }).pipe(Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }))); + const afterStop = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + afterStop.subagents.find((row) => row.id === stoppedChild.taskId)?.status, + "cancelled", + ); + assert.deepEqual( + (yield* orchestrator.getThreadProjection(stoppedChild.childThreadId)).runs.map( + (run) => run.id, + ), + [stoppedChild.childRunId], + ); }), ); diff --git a/apps/server/src/orchestration-v2/EffectWorker.test.ts b/apps/server/src/orchestration-v2/EffectWorker.test.ts index 45477c1d92522..7e564363694a2 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.test.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.test.ts @@ -112,6 +112,8 @@ function layerExecutorFor(input: { closeInstance: () => Effect.void, release: () => record("release"), detach: () => record("detach"), + otherWorkingThreads: () => Effect.succeed([]), + releaseUnlessBusy: () => Effect.succeed(true), }), ), Layer.succeed( @@ -168,7 +170,18 @@ function layerExecutorFor(input: { ); } -it.effect("settles a stopped run when its adapter has already lost the native turn", () => +it.effect.each([ + { + name: "settles a stopped run when its adapter has already lost the native turn", + unresponsive: false, + dispatched: ["thread.background-work.settle"], + }, + { + name: "leaves the run running and flags the Stop when the provider never confirms it", + unresponsive: true, + dispatched: ["thread.interrupt.unresponsive"], + }, +])("$name", ({ unresponsive, dispatched }) => Effect.gen(function* () { const now = yield* DateTime.now; const events = yield* Ref.make>([]); @@ -179,7 +192,8 @@ it.effect("settles a stopped run when its adapter has already lost the native tu threadId, operation: "interrupt", providerTurnId, - cause: "Provider turn is not active.", + ...(unresponsive ? { unresponsive } : {}), + cause: unresponsive ? "Timed out." : "Provider turn is not active.", }), threads: { dispatch: (command) => @@ -200,7 +214,7 @@ it.effect("settles a stopped run when its adapter has already lost the native tu }, }); }).pipe(Effect.provide(layer)); - assert.deepEqual(yield* Ref.get(events), ["thread.background-work.settle"]); + assert.deepEqual(yield* Ref.get(events), dispatched); }), ); diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index c21d548c76709..461a6801e3a26 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -178,6 +178,13 @@ export const layerExecutor: Layer.Layer< cause, }), ), + Effect.map((outcome) => ({ unresponsive: false, outcome }) as const), + // Settling here would end the run while the agent may still be running; the + // user decides with Force stop instead. + Effect.catchIf( + (cause) => request.subagent === undefined && cause.unresponsive === true, + () => Effect.succeed({ unresponsive: true, outcome: undefined } as const), + ), Effect.catchIf( (cause) => request.subagent === undefined && @@ -185,27 +192,35 @@ export const layerExecutor: Layer.Layer< request.type, Cause.pretty(Cause.fail(cause)), ), - () => Effect.void, + () => Effect.succeed({ unresponsive: false, outcome: undefined } as const), ), - Effect.andThen((outcome) => - request.subagent !== undefined && outcome === undefined - ? Effect.void - : threads.dispatch({ - type: "thread.background-work.settle", - commandId: CommandId.make(`${effect.id}:background-work-settled`), - ...(request.subagent === undefined || outcome === undefined - ? {} - : { - subagent: { - id: request.subagent.id, - stoppedAt: DateTime.makeUnsafe(effect.createdAt), - outcome, - }, - }), + Effect.andThen(({ unresponsive, outcome }) => + unresponsive + ? threads.dispatch({ + type: "thread.interrupt.unresponsive", + commandId: CommandId.make(`${effect.id}:unresponsive`), threadId: effect.threadId, providerThreadId: request.providerThreadId, providerTurnId: request.providerTurnId, - }), + }) + : request.subagent !== undefined && outcome === undefined + ? Effect.void + : threads.dispatch({ + type: "thread.background-work.settle", + commandId: CommandId.make(`${effect.id}:background-work-settled`), + ...(request.subagent === undefined || outcome === undefined + ? {} + : { + subagent: { + id: request.subagent.id, + stoppedAt: DateTime.makeUnsafe(effect.createdAt), + outcome, + }, + }), + threadId: effect.threadId, + providerThreadId: request.providerThreadId, + providerTurnId: request.providerTurnId, + }), ), Effect.mapError( (cause) => diff --git a/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts b/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts index 1d703950fd102..19ec63c520814 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts @@ -260,6 +260,120 @@ it.effect("marks a never-visited thread's first completion unread", () => }).pipe(Effect.provide(layerTest)), ); +const completeAndSnoozeThread = Effect.fn("test.completeAndSnoozeThread")(function* ( + name: string, + snoozeMinutes: number, +) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make(`thread:${name}`); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`create-${name}`), + threadId, + projectId: ProjectId.make(`project:${name}`), + title: "Snoozed completion", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`dispatch-${name}`), + threadId, + messageId: MessageId.make(`dispatch-${name}`), + text: "Run in the background", + attachments: [], + dispatchMode: { type: "defer_start" }, + createdBy: "agent", + creationSource: "mcp", + }); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make(`interrupt-${name}`), + threadId, + runId: (yield* projections.getThreadProjection(threadId)).runs[0]!.id, + }); + const completedAt = (yield* projections.getThreadProjection(threadId)).runs.at(-1)?.completedAt; + assert.ok(completedAt); + yield* TestClock.adjust("1 second"); + const snoozedUntil = DateTime.add(yield* DateTime.now, { minutes: snoozeMinutes }); + yield* orchestrator.dispatch({ + type: "thread.snooze", + commandId: CommandId.make(`snooze-${name}`), + threadId, + snoozedUntil: DateTime.formatIso(snoozedUntil), + }); + return { threadId, completedAt, snoozedUntil }; +}); + +it.effect("mark unread retires an expired snooze so a dismissed wake stays dismissed", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const { threadId, completedAt, snoozedUntil } = yield* completeAndSnoozeThread( + "mark-unread-expired-snooze", + 1, + ); + yield* TestClock.adjust("2 minutes"); + yield* orchestrator.dispatch({ + type: "thread.visit", + commandId: CommandId.make("dismiss-woke"), + threadId, + visitedAt: DateTime.formatIso(snoozedUntil), + }); + const dismissed = yield* projections.getThread(threadId); + assert.deepEqual(dismissed.lastVisitedAt, snoozedUntil); + + yield* orchestrator.dispatch({ + type: "thread.mark-unread", + commandId: CommandId.make("mark-unread-expired-snooze"), + threadId, + }); + assert.deepEqual(yield* projections.getThread(threadId), { + ...dismissed, + lastVisitedAt: DateTime.subtract(completedAt, { milliseconds: 1 }), + snoozedUntil: null, + snoozedAt: null, + lastSnoozeWakeAt: snoozedUntil, + }); + }).pipe(Effect.provide(layerTest)), +); + +it.effect("mark unread leaves a future snooze untouched", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const { threadId, completedAt } = yield* completeAndSnoozeThread( + "mark-unread-future-snooze", + 60, + ); + yield* TestClock.adjust("1 minute"); + yield* orchestrator.dispatch({ + type: "thread.visit", + commandId: CommandId.make("visit-snoozed"), + threadId, + visitedAt: DateTime.formatIso(yield* DateTime.now), + }); + const visited = yield* projections.getThread(threadId); + assert.isNotNull(visited.snoozedUntil); + + yield* orchestrator.dispatch({ + type: "thread.mark-unread", + commandId: CommandId.make("mark-unread-future-snooze"), + threadId, + }); + assert.deepEqual(yield* projections.getThread(threadId), { + ...visited, + lastVisitedAt: DateTime.subtract(completedAt, { milliseconds: 1 }), + }); + }).pipe(Effect.provide(layerTest)), +); + it.effect( "dispatches metadata, queue resume and request controls without hydrating unrelated history", () => diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 26db4f13c1ed8..bdcf1a9a4ca18 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -110,6 +110,8 @@ import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts"; import { notificationTurnItem } from "@t3tools/provider-core/server/notification"; import { cancelledBeforeStart, + continuationFailedBeforePrompt, + continuationStartRetryDelayMs, isRestartContinuationSource, runOutcome, } from "./RestartBackgroundNote.ts"; @@ -148,6 +150,7 @@ import { type ProjectionRecords, type ProjectionCheckpointContext, type OlderSettledThreadsListing, + type ActiveShellIndexEntry, type OlderSettledThreadsQuery, type ShellSnapshotOptions, } from "./ProjectionStore.ts"; @@ -400,6 +403,11 @@ export interface OrchestratorV2Shape { ReadonlyArray, OrchestratorV2Error >; + readonly countThreads: () => Effect.Effect; + readonly listActiveShellIndex: () => Effect.Effect< + ReadonlyArray, + OrchestratorV2Error + >; readonly listOlderSettledThreads: ( query: OlderSettledThreadsQuery, ) => Effect.Effect; @@ -536,6 +544,7 @@ function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId { case "checkpoint.rollback": case "checkpoint.rollback.fail": case "thread.background-work.settle": + case "thread.interrupt.unresponsive": case "thread.stop": case "provider.switch": return command.threadId; @@ -762,6 +771,20 @@ function delegatedTaskTerminalStatus( } } +/** Failed worktree setup keeps the queue held so queued messages cannot start in the project checkout. */ +function worktreeSetupPending( + projection: Pick, +): boolean { + return ( + projection.thread.worktreePath === null && + projection.runs.some( + (run) => + (run.status === "failed" || run.status === "interrupted") && + run.workspacePreparation?.type === "worktree", + ) + ); +} + function nextQueuedRun( projection: Pick, ): OrchestrationV2Run | undefined { @@ -2216,6 +2239,81 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ); + // Under load a resumed provider can time out before taking the prompt; the restart still owes the turn. + const retryContinuationStart = Effect.fn("Orchestrator.retryContinuationStart")(function* ( + threadId: ThreadId, + runId: RunId, + ) { + const projection = yield* projectionStore.getThreadRecords( + threadId, + ["runs", "providerTurns", "turnItems"], + { turnItemTypes: ["error"], turnItemRunIds: [runId] }, + ); + const run = projection.runs.find((candidate) => candidate.id === runId); + if (run === undefined) return; + const delayMs = continuationStartRetryDelayMs(run, projection); + if (delayMs === undefined) return; + const commandId = CommandId.make(`command:restart-continuation-retry:${run.id}`); + yield* writeSystemEvents( + [], + [ + { + id: `effect:restart-continuation:${run.id}`, + commandId, + threadId, + request: { type: "provider-runtime.continue", sourceRunId: run.id }, + availableAt: DateTime.add(yield* DateTime.now, { milliseconds: delayMs }), + }, + ], + ); + }); + + // A hold waits for a person to review what they queued. Nobody reviews messages from agents, + // the server or scheduled tasks, so a later completed run releases those unless it was stopped. + const releaseUnattendedHolds = Effect.fn("Orchestrator.releaseUnattendedHolds")(function* ( + threadId: ThreadId, + completedRunId: RunId, + ) { + const held = yield* projectionStore.getHeldQueuedRuns(threadId); + if (held.length === 0) return; + const projection = yield* projectionStore.getThreadRecords( + threadId, + ["runs", "messages", "turnItems"], + { + messageIds: held.map((run) => run.userMessageId), + turnItemTypes: ["run_interrupt_request"], + turnItemRunIds: [completedRunId], + }, + ); + // Only the newest run completing shows the thread moved past the hold; a late or + // bookkeeping update to an older run, overtaken by a failure or Stop, does not. + if ( + latestExecutedRun(projection.runs)?.id !== completedRunId || + projection.turnItems.length > 0 || + worktreeSetupPending(projection) + ) + return; + const { messages } = projection; + const unattended = new Set( + messages + .filter((message) => message.createdBy !== "user" || message.scheduledTaskId !== undefined) + .map((message) => message.id), + ); + const now = yield* DateTime.now; + const released = held.filter((run) => unattended.has(run.userMessageId)); + if (released.length === 0) return; + yield* writeSystemEvents( + released.map((run) => ({ + type: "run.updated" as const, + threadId, + runId: run.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...run, queueHeld: false }, + })), + ); + }); + const resumeQueuedRuns = Effect.gen(function* () { const threadIds = yield* projectionStore.getRecoveryThreadIds("queued-runs"); let resumed = 0; @@ -3264,8 +3362,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } let markUnreadVisitedAt: DateTime.Utc | null = null; if (command.type === "thread.mark-unread") { - const projection = yield* loadProjectionForCommand(command, ["runs"]); - const latestRunCompletedAt = projection.runs.at(-1)?.completedAt ?? null; + const shell = yield* projectionStore + .getThreadShell(command.threadId) + .pipe( + Effect.mapError( + (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }), + ), + ); + const latestRunCompletedAt = shell?.latestRunCompletedAt ?? null; if (latestRunCompletedAt === null) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, @@ -3389,8 +3493,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio updatedAt: thread.updatedAt, }; } - case "thread.mark-unread": - return { ...thread, lastVisitedAt: markUnreadVisitedAt }; + case "thread.mark-unread": { + // Retire the wake before rewinding the visit, preserving its settlement timer. + const snoozeExpired = + thread.snoozedUntil != null && + DateTime.toEpochMillis(thread.snoozedUntil) <= DateTime.toEpochMillis(now); + return { + ...thread, + lastVisitedAt: markUnreadVisitedAt, + ...(snoozeExpired + ? { snoozedUntil: null, snoozedAt: null, lastSnoozeWakeAt: thread.snoozedUntil } + : {}), + }; + } case "thread.metadata.update": { const previousRecovery = thread.limitRecovery?.runId === command.limitRecovery?.runId && @@ -5153,7 +5268,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const source = projection.runs.find((run) => run.id === command.restartContinuationOfRunId); if ( !source || - !isRestartContinuationSource(source, projection.providerTurns) || + !( + isRestartContinuationSource(source, projection.providerTurns) || + continuationFailedBeforePrompt(source, projection.providerTurns) + ) || projection.thread.archivedAt !== null || projection.thread.deletedAt !== null || projection.thread.providerInstanceId !== source.providerInstanceId || @@ -5564,15 +5682,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio dispatchMode.type === "queue_after_active"; const shouldQueue = queueBehindRun !== undefined && startsOrQueuesRun; // Client messages resume held queues; server continuations and scheduled tasks do not. - // Provider failure or usage limits re-hold the queue. Failed worktree setup keeps it held - // until a worktree exists, so queued messages cannot start in the project checkout. - const worktreeSetupUnresolved = - projection.thread.worktreePath === null && - projection.runs.some( - (run) => - (run.status === "failed" || run.status === "interrupted") && - run.workspacePreparation?.type === "worktree", - ); + // Provider failure or usage limits re-hold the queue. + const worktreeSetupUnresolved = worktreeSetupPending(projection); const sentByUserFromClient = command.createdBy === "user" && (command.creationSource === "web" || command.creationSource === "mobile") && @@ -10216,7 +10327,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio run, rootNode, providerThread, - reason: yield* runInterruptReason(input.command, input.events, run.id), + reason: + (input.command.type === "thread.background-work.settle" + ? input.command.reason + : undefined) ?? (yield* runInterruptReason(input.command, input.events, run.id)), completedAt: input.now, }), }); @@ -10391,7 +10505,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // A resumed queue can have a lower ordinal than a pending continuation source. // Stop must reach those sources even when interrupting the resumed run succeeds. if ( - run.status === "cancelled" && + (run.status === "cancelled" || + (run.status === "failed" && run.restartContinuationOfRunId !== undefined)) && (yield* awaitsRestartContinuation(run).pipe(mapDispatchError(input.command))) ) { yield* markStoppedRun({ ...input, thread, run }); @@ -10462,6 +10577,55 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio yield* closePendingLinkRequests({ ...input, threadId: thread.id }); }); + const dispatchInterruptUnresponsive = Effect.fn("Orchestrator.dispatchInterruptUnresponsive")( + function* ( + command: Extract< + OrchestrationV2InternalCommand, + { readonly type: "thread.interrupt.unresponsive" } + >, + events: Ref.Ref>, + ) { + const { providerTurn } = yield* projectionStore + .getProviderControlContext(command.threadId, { + providerThreadId: command.providerThreadId, + providerTurnId: command.providerTurnId, + }) + .pipe(mapDispatchError(command)); + const { runs } = yield* projectionStore + .getThreadRecords(command.threadId, ["runs"]) + .pipe(mapDispatchError(command)); + const run = runs.find( + (candidate) => + providerTurn !== undefined && candidate.activeAttemptId === providerTurn.runAttemptId, + ); + // A settled run still counts: Stop may have targeted its background work. + if (run === undefined) return; + const { turnItems } = yield* projectionStore + .getThreadRecords(command.threadId, ["turnItems"], { + turnItemTypes: ["run_interrupt_request", "run_interrupt_result"], + turnItemRunIds: [run.id], + }) + .pipe(mapDispatchError(command)); + // A provider that stopped late already wrote the result; it needs no Force stop. + if (turnItems.some((item) => item.type === "run_interrupt_result")) return; + const request = turnItems.find((item) => item.type === "run_interrupt_request"); + if (request?.type !== "run_interrupt_request" || request.unresponsiveAt !== undefined) return; + const now = yield* DateTime.now; + yield* emit( + events, + command, + )({ + type: "turn-item.updated", + threadId: command.threadId, + runId: run.id, + ...(request.nodeId === null ? {} : { nodeId: request.nodeId }), + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...request, unresponsiveAt: now, updatedAt: now }, + }); + }, + ); + const dispatchBackgroundWorkSettle = Effect.fn("Orchestrator.dispatchBackgroundWorkSettle")( function* ( command: Extract< @@ -10993,6 +11157,45 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio occurredAt: now, payload: { ...run, status: "interrupted", completedAt: now }, }); + // Stop cancels a pending restart, so it must end the turn that restart could not stop. + const supersededTurn = projection.providerTurns.findLast( + (candidate) => + candidate.status === "running" && + projection.attempts.some( + (attempt) => attempt.id === candidate.runAttemptId && attempt.runId === run.id, + ), + ); + const supersededOwner = + supersededTurn === undefined + ? undefined + : projection.providerThreads.find( + (candidate) => candidate.id === supersededTurn.providerThreadId, + ); + if ( + supersededTurn !== undefined && + supersededOwner !== undefined && + supersededOwner.providerSessionId !== null && + Option.isSome( + yield* providerSessions + .get(supersededOwner.providerSessionId) + .pipe(Effect.orElseSucceed(() => Option.none())), + ) + ) { + const id = `effect:${command.commandId}:provider-turn.interrupt:${supersededTurn.id}`; + if (!otherProviderInterrupts.some((effect) => effect.id === id)) { + otherProviderInterrupts.push({ + id, + commandId: command.commandId, + threadId: command.threadId, + request: { + type: "provider-turn.interrupt", + providerSessionId: supersededOwner.providerSessionId, + providerThreadId: supersededOwner.id, + providerTurnId: supersededTurn.id, + }, + }); + } + } yield* stopRemainingWork(); yield* Ref.update(effects, (existing) => [...existing, ...otherProviderInterrupts]); return { @@ -12789,6 +12992,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio case "thread.background-work.settle": yield* dispatchBackgroundWorkSettle(command, events, effects); break; + case "thread.interrupt.unresponsive": + yield* dispatchInterruptUnresponsive(command, events); + break; case "thread.fork": yield* dispatchThreadFork(command, events); break; @@ -12920,6 +13126,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (planned) => planned.events.length > 0 || command.type === "thread.background-work.settle" || + command.type === "thread.interrupt.unresponsive" || command.type === "thread.stop" || command.type === "subagent.stop" || // Closing a share again, or after it ended, is not an error. @@ -13099,6 +13306,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // while writing child events, so nesting parent inside child would invert the order. // The keyed semaphores are neither reentrant nor deadlock-aware. if (delegatedTaskTerminalStatus(run.status) !== null) { + // Before the parent hears of the failure: a pending continuation keeps the task open. + if (run.status === "failed" && run.restartContinuationOfRunId !== undefined) { + yield* threadDispatch.withLock(threadId, retryContinuationStart(threadId, run.id)).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Failed to schedule a restart continuation retry", { + threadId, + runId: run.id, + cause, + }), + ), + ); + } yield* threadDispatch.withLock( threadId, finalizeDelegatedCompletionDelivery(threadId, run.id), @@ -13106,10 +13325,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio yield* threadDispatch .withLock( threadId, - startNextQueuedRun(threadId, { - endedRunId: run.id, - ...(run.status === "failed" ? { failedRunId: run.id } : {}), - }), + (run.status === "completed" + ? releaseUnattendedHolds(threadId, run.id) + : Effect.void + ).pipe( + Effect.andThen( + startNextQueuedRun(threadId, { + endedRunId: run.id, + ...(run.status === "failed" ? { failedRunId: run.id } : {}), + }), + ), + ), ) .pipe( Effect.catchCause((cause) => @@ -13466,6 +13692,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe(Effect.mapError(shellProjectionError)); }, ), + countThreads: Effect.fn("OrchestratorV2.countThreads")(function () { + return projectionStore.countThreads().pipe(Effect.mapError(shellProjectionError)); + }), + listActiveShellIndex: Effect.fn("OrchestratorV2.listActiveShellIndex")(function () { + return projectionStore.listActiveShellIndex().pipe(Effect.mapError(shellProjectionError)); + }), listOlderSettledThreads: Effect.fn("OrchestratorV2.listOlderSettledThreads")(function (query) { return projectionStore .listOlderSettledThreads(query) diff --git a/apps/server/src/orchestration-v2/ProjectSettingsUpgrade.integration.test.ts b/apps/server/src/orchestration-v2/ProjectSettingsUpgrade.integration.test.ts index 9376dedfe90a8..9772a559f32c1 100644 --- a/apps/server/src/orchestration-v2/ProjectSettingsUpgrade.integration.test.ts +++ b/apps/server/src/orchestration-v2/ProjectSettingsUpgrade.integration.test.ts @@ -161,8 +161,6 @@ const layerRuntime = (dbPath: string) => { getInstance: () => Effect.succeed(undefined), listInstances: Effect.succeed([]), listUnavailable: Effect.succeed([]), - listEnvironments: Effect.succeed(new Map()), - rebuildInstanceWhen: () => Effect.succeed(false), streamChanges: Stream.empty, subscribeChanges: Effect.never, }), diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 17b8bb818864a..1dc7c912e7765 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -26,6 +26,7 @@ import { ThreadId, TurnItemId, type ThreadPullRequestLink, + isLiveSubagentShell, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -76,6 +77,33 @@ const traceSqlStatements = () => { return { statements, tracer }; }; +/** The shell index flags every live shell and matches its shells on the fields subscriber trims read. */ +const assertShellIndexCoversShells = Effect.fnUntraced(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + const shells = (yield* store.getShellSnapshot({ location: "active" })).threads; + const index = yield* store.listActiveShellIndex(); + const iso = (value: DateTime.Utc | null | undefined) => + value == null ? null : DateTime.formatIso(value); + const trimmed = ( + row: Pick< + ProjectionStore.ActiveShellIndexEntry, + "id" | "projectId" | "lineage" | "settledOverride" | "settledAt" | "pinnedAt" | "snoozedUntil" + >, + ) => ({ + id: row.id, + projectId: row.projectId, + lineage: row.lineage, + settledOverride: row.settledOverride, + settledAt: iso(row.settledAt), + pinnedAt: iso(row.pinnedAt), + snoozedUntil: iso(row.snoozedUntil), + }); + assert.deepEqual(index.map(trimmed), shells.map(trimmed)); + shells.forEach((shell, position) => { + if (isLiveSubagentShell(shell)) assert.isTrue(index[position]?.mayBeLive, shell.id); + }); +}); + const addRolledBackRecoveryCandidate = Effect.fn("addRolledBackRecoveryCandidate")(function* ( suffix: string, ) { @@ -272,6 +300,7 @@ it.effect("SQL shells flag only an active turn blocked solely on T3 thread waits assert.equal(shell?.status, "running"); assert.equal(shell?.blockedOnThreadWait ?? false, expected); } + yield* assertShellIndexCoversShells(); }); const wait = { ...itemBase, @@ -531,6 +560,7 @@ it.effect("SQL shells use the latest subagent owner for duplicate provider-threa const listed = (yield* store.getShellSnapshot()).threads.find( (thread) => thread.id === targetThreadId, ); + yield* assertShellIndexCoversShells(); for (const shell of [yield* store.getThreadShell(targetThreadId), listed]) { assert.equal(shell?.providerInstanceId, otherInstanceId); assert.equal(shell?.activeRunId, null); @@ -1336,6 +1366,7 @@ it.effect("SQL list older settled threads below a cursor", () => const next = yield* list({ settledAt: DateTime.formatIso(at(4)), threadId: id("b") }, 10); assert.deepEqual(next.threadIds, [id("c"), id("d"), id("snoozed")]); assert.deepEqual(yield* list(before, 25, []), { threadIds: [], counts: [] }); + yield* assertShellIndexCoversShells(); const scoped = yield* list(before, 0, [ProjectId.make("project:b")]); assert.deepEqual(scoped, { threadIds: [], @@ -3798,6 +3829,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { const sqlShell = (yield* projectionStore.getShellSnapshot()).threads.find( (row) => row.id === threadId, )!; + yield* assertShellIndexCoversShells(); for (const questionShell of [ sqlShell, ProjectionStore.threadShellFromProjection(projection), @@ -4860,6 +4892,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { SET status = 'running', payload_json = json_set(payload_json, '$.status', 'running') WHERE run_id = ${runId} `; + yield* assertShellIndexCoversShells(); const secretOnlyShell = (yield* projectionStore.getShellSnapshot()).threads.find( (thread) => thread.id === threadId, ); @@ -6466,6 +6499,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { (candidate) => candidate.id === threadId, ); assert.deepEqual(listed?.pendingBackgroundTasks, shell?.pendingBackgroundTasks); + yield* assertShellIndexCoversShells(); return shell && projectThreadAwarenessV2({ environmentId, project, thread: shell })?.phase; }); @@ -6640,10 +6674,23 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { completedAt: releasedAt, }, }); + const { statements, tracer } = traceSqlStatements(); assert.deepEqual( - (yield* store.getRuntimeRecoveryProjection(threadId)).runs.map((run) => run.id), + (yield* store + .getRuntimeRecoveryProjection(threadId) + .pipe(Effect.withTracer(tracer))).runs.map((run) => run.id), [runId, newerRunId], ); + // Live-item filters must hit the active partial indexes, not walk the thread's history. + const sql = yield* SqlClient.SqlClient; + for (const statement of statements.filter((query) => query.includes("turn_items"))) { + const plan = (yield* sql.unsafe<{ readonly detail: string }>( + `EXPLAIN QUERY PLAN ${statement}`, + )).map((row) => row.detail); + for (const detail of plan) { + assert.notMatch(detail, /turn_items_(thread_ordinal|thread_run)_idx \(thread_id=\?\)$/); + } + } }), ); }); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 287a30a9d904e..88a9073117dd6 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -393,6 +393,15 @@ export interface ShellSnapshotOptions { readonly threadIds?: ReadonlyArray; } +/** What a shell subscriber's trims read from an active thread, without building its shell. */ +export interface ActiveShellIndexEntry extends Pick< + OrchestrationV2ThreadShell, + "id" | "projectId" | "lineage" | "settledOverride" | "settledAt" | "pinnedAt" | "snoozedUntil" +> { + /** False only when no run, request, work item, or watch can make the shell live. */ + readonly mayBeLive: boolean; +} + export interface OlderSettledThreadsQuery { readonly before: OrchestrationV2OlderSettledCursor; readonly projectIds?: ReadonlyArray | undefined; @@ -450,6 +459,13 @@ export interface ProjectionStoreV2Shape { ReadonlyArray, ProjectionStoreV2Error >; + /** Threads that are not deleted, archived ones included. */ + readonly countThreads: () => Effect.Effect; + /** Every active thread, in shell order. */ + readonly listActiveShellIndex: () => Effect.Effect< + ReadonlyArray, + ProjectionStoreV2Error + >; /** Older settled thread ids below a cursor, newest settle first, and per-project counts. */ readonly listOlderSettledThreads: ( query: OlderSettledThreadsQuery, @@ -596,6 +612,9 @@ export interface ProjectionStoreV2Shape { readonly canStartQueuedRun: ( threadId: ThreadId, ) => Effect.Effect; + readonly getHeldQueuedRuns: ( + threadId: ThreadId, + ) => Effect.Effect, ProjectionStoreV2Error>; readonly getRecoveryThreadIds: ( kind: ProjectionRecoveryKind, ) => Effect.Effect, ProjectionStoreV2Error>; @@ -5167,6 +5186,17 @@ export const layer: Layer.Layer = ) .pipe(Effect.mapError(controlReadError(threadId))); + const getHeldQueuedRuns: ProjectionStoreV2Shape["getHeldQueuedRuns"] = (threadId) => + sql` + SELECT payload_json FROM orchestration_v2_projection_runs + WHERE thread_id = ${threadId} AND status = 'queued' + AND json_extract(payload_json, '$.queueHeld') IS 1 + ORDER BY ordinal ASC + `.pipe( + Effect.flatMap(decodeRows(decodeRunPayload, threadId)), + Effect.mapError(controlReadError(threadId)), + ); + const getMessageCount: ProjectionStoreV2Shape["getMessageCount"] = (threadId) => sql<{ count: number; @@ -6614,6 +6644,98 @@ export const layer: Layer.Layer = Effect.mapError(shellSnapshotReadError), ); + const countThreads = Effect.fn("ProjectionStore.countThreads")(function* () { + const rows = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM orchestration_v2_projection_threads WHERE deleted_at IS NULL + `; + return rows[0]?.count ?? 0; + }, Effect.mapError(shellSnapshotReadError)); + + const listActiveShellIndex = Effect.fn("ProjectionStore.listActiveShellIndex")(function* () { + const rows = yield* sql<{ + readonly thread_id: string; + readonly project_id: string; + readonly relationship_to_parent: string | null; + readonly parent_thread_id: string | null; + readonly root_thread_id: string; + readonly settled_override: string | null; + readonly settled_at: string | null; + readonly pinned_at: string | null; + readonly snoozed_until: string | null; + readonly may_be_live: number; + }>` + SELECT + t.thread_id, + t.project_id, + json_extract(t.payload_json, '$.lineage.relationshipToParent') AS relationship_to_parent, + json_extract(t.payload_json, '$.lineage.parentThreadId') AS parent_thread_id, + json_extract(t.payload_json, '$.lineage.rootThreadId') AS root_thread_id, + json_extract(t.payload_json, '$.settledOverride') AS settled_override, + json_extract(t.payload_json, '$.settledAt') AS settled_at, + json_extract(t.payload_json, '$.pinnedAt') AS pinned_at, + json_extract(t.payload_json, '$.snoozedUntil') AS snoozed_until, + -- A superset of isLiveSubagentShell: the sources of its run status, pending request, + -- and pending background work. + ( + EXISTS ( + SELECT 1 FROM orchestration_v2_projection_runs r + WHERE r.thread_id = t.thread_id + AND r.status IN ('queued', 'preparing', 'starting', 'running', 'waiting') + ) + OR EXISTS ( + SELECT 1 FROM orchestration_v2_projection_runtime_requests request + WHERE request.thread_id = t.thread_id AND request.status = 'pending' + ) + -- Same predicates as the shell's pending item and link reads, for their partial indexes. + OR EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items item + WHERE item.thread_id = t.thread_id + AND item.type IN ('command_execution', 'dynamic_tool', 'subagent') + AND item.status NOT IN ('completed', 'interrupted', 'failed', 'cancelled') + ) + OR EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items link_request + WHERE link_request.thread_id = t.thread_id + AND link_request.type = 'link_request' AND link_request.status = 'waiting' + ) + OR EXISTS ( + SELECT 1 FROM orchestration_v2_projection_provider_threads provider_thread + WHERE provider_thread.thread_id = t.thread_id + AND json_array_length(provider_thread.payload_json, '$.pendingBackgroundTasks') > 0 + ) + OR EXISTS ( + SELECT 1 FROM json_each(t.payload_json, '$.pullRequests') link + WHERE json_type(link.value, '$.watch') IS NOT NULL + ) + ) AS may_be_live + FROM orchestration_v2_projection_threads t + WHERE t.deleted_at IS NULL + AND json_extract(t.payload_json, '$.archivedAt') IS NULL + ORDER BY t.updated_at ASC, t.thread_id ASC + `; + return rows.map((row): ActiveShellIndexEntry => ({ + id: ThreadId.make(row.thread_id), + projectId: ProjectId.make(row.project_id), + lineage: { + parentThreadId: + row.parent_thread_id === null ? null : ThreadId.make(row.parent_thread_id), + relationshipToParent: + row.relationship_to_parent === "fork" || row.relationship_to_parent === "subagent" + ? row.relationship_to_parent + : null, + rootThreadId: ThreadId.make(row.root_thread_id), + }, + settledOverride: + row.settled_override === "settled" || row.settled_override === "active" + ? row.settled_override + : null, + settledAt: row.settled_at === null ? null : DateTime.makeUnsafe(row.settled_at), + pinnedAt: row.pinned_at === null ? null : DateTime.makeUnsafe(row.pinned_at), + snoozedUntil: row.snoozed_until === null ? null : DateTime.makeUnsafe(row.snoozed_until), + mayBeLive: row.may_be_live === 1, + })); + }, Effect.mapError(shellSnapshotReadError)); + const findGuestShareThreadId = Effect.fn("ProjectionStore.findGuestShareThreadId")(function* ( shareId: GuestShareId, ) { @@ -6768,6 +6890,8 @@ export const layer: Layer.Layer = apply, getShellSnapshot: readOnReader(getShellSnapshot), listActiveSubagentThreadIds: readOnReader(listActiveSubagentThreadIds), + countThreads: readOnReader(countThreads), + listActiveShellIndex: readOnReader(listActiveShellIndex), readShellSnapshot: readOnReader(readShellSnapshot), listOlderSettledThreads: readOnReader(listOlderSettledThreads), getThreadShell: readOnReader(getThreadShell), @@ -6789,6 +6913,7 @@ export const layer: Layer.Layer = getCheckpointRollbackContext: readOnReader(getCheckpointRollbackContext), getRunMessage: readOnReader(getRunMessage), canStartQueuedRun: readOnReader(canStartQueuedRun), + getHeldQueuedRuns: readOnReader(getHeldQueuedRuns), getPendingNativeUserInputs: readOnReader(getPendingNativeUserInputs), hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest), isNodeStored: readOnReader(isNodeStored), diff --git a/apps/server/src/orchestration-v2/ProviderAdapterRegistry.test.ts b/apps/server/src/orchestration-v2/ProviderAdapterRegistry.test.ts index 54b1b62d4a404..97739c0b3d01e 100644 --- a/apps/server/src/orchestration-v2/ProviderAdapterRegistry.test.ts +++ b/apps/server/src/orchestration-v2/ProviderAdapterRegistry.test.ts @@ -2,29 +2,22 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { ProviderDriverKind, - ProviderInstanceEnvironment, ProviderInstanceId, ProviderSessionId, ThreadId, ProviderSetupError, } from "@t3tools/contracts"; -import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; -import * as Ref from "effect/Ref"; -import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import * as ProviderAuthFlow from "@t3tools/provider-core/server/providerAuthFlow"; import type { ProviderAuthController } from "../provider/ProviderAuthService.ts"; -import type { ProviderDriver, ProviderInstance } from "@t3tools/provider-core/server/driver"; -import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; +import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; -import { hasProviderSecretReference } from "../provider/ProviderSecretReference.ts"; -import * as ProviderSecretResolver from "../provider/ProviderSecretResolver.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; @@ -32,13 +25,6 @@ const driver = ProviderDriverKind.make("codex"); const personalId = ProviderInstanceId.make("codex_personal"); const workId = ProviderInstanceId.make("codex_work"); -const HOME_ACCOUNT = "my.1password.com"; -const decodeEnvironment = Schema.decodeSync(ProviderInstanceEnvironment); -const onePasswordVariable = (name: string, reference: string) => ({ - name, - value: { kind: "1password" as const, reference, account: HOME_ACCOUNT }, -}); - const makeAdapter = ( instanceId: ProviderInstanceId, ): ProviderAdapter.ProviderAdapterV2["Service"] => @@ -78,10 +64,8 @@ const layerInstanceRegistry = Layer.succeed(ProviderInstanceRegistry.ProviderIns Effect.succeed(instances.find((instance) => instance.instanceId === instanceId)), listInstances: Effect.succeed(instances), listUnavailable: Effect.succeed([]), - listEnvironments: Effect.succeed(new Map()), streamChanges: Stream.empty, subscribeChanges: Effect.never, - rebuildInstanceWhen: () => Effect.succeed(false), }); const layerTest = ProviderAdapterRegistry.layerFromProviderInstanceRegistry.pipe( Layer.provide(layerInstanceRegistry), @@ -211,121 +195,3 @@ it.effect("interrupts admitted session startup when a shared peer signs out", () assert.isTrue(Exit.isFailure(yield* Fiber.await(startup))); }).pipe(Effect.provide(NodeServices.layer)), ); - -it.effect("opens v2 sessions with resolved secrets and rebuilds them when a secret rotates", () => - Effect.gen(function* () { - const secretInstanceId = ProviderInstanceId.make("codex_secret"); - const apiKeyReference = "op://Vault/Codex/api-key"; - const apiKey = yield* Ref.make("sk-first"); - const sessionEnvironments = yield* Ref.make>([]); - const resolver: ProviderSecretResolver.ProviderSecretResolverShape = { - resolve: (environment) => - Effect.gen(function* () { - const variables = []; - const unresolved = []; - for (const { name, value, sensitive } of environment ?? []) { - if (typeof value !== "string" && value.reference === apiKeyReference) { - variables.push({ name, value: yield* Ref.get(apiKey), sensitive: true }); - } else if (typeof value !== "string") { - unresolved.push(name); - } else { - variables.push({ name, value, sensitive }); - } - } - return { variables, unresolved }; - }), - prime: () => Effect.void, - invalidate: () => Effect.void, - check: () => Effect.succeed({ _tag: "Resolved" }), - listOnePasswordAccounts: Effect.succeed([]), - }; - const secretDriver: ProviderDriver> = { - driverKind: driver, - metadata: { displayName: "Codex" }, - configSchema: Schema.Struct({}), - defaultConfig: () => ({}), - create: (input) => - Effect.gen(function* () { - const environment = yield* mergeProviderInstanceEnvironment( - input.environment, - yield* HostProcess.Environment, - ); - return { - ...makeInstance(input.instanceId, { - ...makeAdapter(input.instanceId), - openSession: () => - Ref.update(sessionEnvironments, (previous) => [...previous, environment]).pipe( - Effect.andThen(Effect.die("session environment recorded")), - ), - }), - snapshot: { - getSnapshot: Effect.succeed({}), - } as unknown as ProviderInstance["snapshot"], - }; - }), - }; - const openSession = (registry: ProviderAdapterRegistry.ProviderAdapterRegistryV2["Service"]) => - registry.get(secretInstanceId).pipe( - Effect.flatMap((adapter) => - adapter.openSession({ - threadId: ThreadId.make("secret-thread"), - providerSessionId: ProviderSessionId.make("secret-session"), - modelSelection: { instanceId: secretInstanceId, model: "test-model" }, - runtimePolicy: { - runtimeMode: "full-access", - interactionMode: "default", - cwd: "/workspace", - }, - }), - ), - Effect.exit, - ); - - const { registry: instanceRegistry } = - yield* ProviderInstanceRegistry.makeProviderInstanceRegistry({ - drivers: [secretDriver], - configMap: { - [secretInstanceId]: { - driver, - environment: decodeEnvironment([ - onePasswordVariable("OPENAI_API_KEY", apiKeyReference), - { name: "CODEX_PROFILE", value: "work", sensitive: false }, - ]), - config: {}, - }, - }, - }).pipe( - Effect.provideService(ProviderSecretResolver.ProviderSecretResolver, resolver), - Effect.provideService(HostProcess.Environment, { - PATH: "/bin", - }), - ); - const registry = yield* Effect.service(ProviderAdapterRegistry.ProviderAdapterRegistryV2).pipe( - Effect.provide( - ProviderAdapterRegistry.layerFromProviderInstanceRegistry.pipe( - Layer.provide( - Layer.succeed(ProviderInstanceRegistry.ProviderInstanceRegistry, instanceRegistry), - ), - ), - ), - ); - - yield* openSession(registry); - yield* Ref.set(apiKey, "sk-rotated"); - assert.isTrue( - yield* instanceRegistry.rebuildInstanceWhen(secretInstanceId, (entry) => - hasProviderSecretReference(entry.environment), - ), - ); - yield* openSession(registry); - - const [first, rotated] = yield* Ref.get(sessionEnvironments); - assert.deepEqual(first, { - PATH: "/bin", - OPENAI_API_KEY: "sk-first", - CODEX_PROFILE: "work", - OTEL_RESOURCE_ATTRIBUTES: "t3code.launched=true", - }); - assert.strictEqual(rotated?.OPENAI_API_KEY, "sk-rotated"); - }).pipe(Effect.scoped), -); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index d085b79d26c43..3428f37fed2d7 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -342,6 +342,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( readonly startTurn?: | Effect.Effect | ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"]; + readonly interruptTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; readonly spawnBeforeOpen?: boolean; readonly scopeCloseReached?: Deferred.Deferred; @@ -444,7 +445,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( Ref.update(state, (current) => ({ ...current, interruptCount: current.interruptCount + 1, - })), + })).pipe(Effect.andThen(options.interruptTurn ?? Effect.void)), unloadThread: ({ providerThread }) => (options.beforeUnload ?? Effect.void).pipe( Effect.andThen( @@ -494,6 +495,7 @@ function layerTest(input: { readonly startTurn?: | Effect.Effect | ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"]; + readonly interruptTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; readonly spawnBeforeOpen?: boolean; readonly scopeCloseReached?: Deferred.Deferred; @@ -529,6 +531,7 @@ function layerTest(input: { ? {} : { hangSessionScopeClose: input.hangSessionScopeClose }), ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), + ...(input.interruptTurn === undefined ? {} : { interruptTurn: input.interruptTurn }), ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), ...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }), ...(input.scopeCloseReached === undefined @@ -1428,6 +1431,159 @@ it.effect( }), ); +it.effect("ProviderSessionManagerV2 gives up on a provider that never confirms a Stop", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-provider-session-manager-unresponsive-stop"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const stopping = yield* runtime + .interruptTurn({ + providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }), + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-unresponsive", + }), + }) + .pipe(Effect.flip, Effect.forkChild); + yield* TestClock.adjust(ProviderSessionManager.PROVIDER_INTERRUPT_TIMEOUT_MS); + assert.equal((yield* Fiber.join(stopping))._tag, "ProviderAdapterInterruptTimeoutError"); + }).pipe( + Effect.provide(layerTest({ state, idleTimeoutMs: 60_000, interruptTurn: Effect.never })), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps a shared session a sibling is busy on when releasing", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const stopping = ThreadId.make("thread-provider-session-manager-release-stopping"); + const sibling = ThreadId.make("thread-provider-session-manager-release-sibling"); + const providerSessionId = idAllocator.derive.providerSession({ + providerInstanceId: modelSelection.instanceId, + }); + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator, threadId: stopping, now }), + yield* makeThreadCreatedEvent({ idAllocator, threadId: sibling, now }), + ], + }); + const runtime = yield* manager.open({ + threadId: stopping, + providerSessionId, + modelSelection, + runtimePolicy, + }); + yield* manager.open({ + threadId: sibling, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThreadOf = (threadId: ThreadId) => + makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + nativeThreadId: `native-${threadId}`, + }); + const runId = idAllocator.derive.run({ threadId: sibling, ordinal: 1 }); + yield* runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(sibling)).thread, + threadId: sibling, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread: providerThreadOf(sibling), + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId: sibling, ordinal: 1 }), + text: "turn", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + const release = (own: ThreadId) => + manager.releaseUnlessBusy({ + providerSessionId, + ownProviderThreadIds: [providerThreadOf(own).id], + reason: "manual_shutdown", + }); + + assert.isFalse(yield* release(stopping)); + assert.isTrue(yield* manager.isLive(providerSessionId)); + assert.isTrue(yield* release(sibling)); + assert.isFalse(yield* manager.isLive(providerSessionId)); + + // A turn that starts on a runtime released meanwhile fails instead of running on it. + const startOnReleasedRuntime = Effect.gen(function* () { + const lateRunId = idAllocator.derive.run({ threadId: stopping, ordinal: 1 }); + return yield* Effect.flip( + runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(stopping)).thread, + threadId: stopping, + runId: lateRunId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId: lateRunId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId: lateRunId }), + providerThread: providerThreadOf(stopping), + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId: stopping, ordinal: 1 }), + text: "turn", + attachments: [], + }, + modelSelection, + runtimePolicy, + }), + ); + }); + assert.equal((yield* startOnReleasedRuntime)._tag, "ProviderAdapterTurnStartError"); + + // Nor may it leave its turn busy on a replacement opened under the same id. + yield* manager.open({ + threadId: sibling, + providerSessionId, + modelSelection, + runtimePolicy, + }); + assert.equal((yield* startOnReleasedRuntime)._tag, "ProviderAdapterTurnStartError"); + assert.isTrue(yield* release(sibling)); + }).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60 * 60_000 }))); + }), +); + it.effect("ProviderSessionManagerV2 stops a session still opening when its layer shuts down", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); @@ -2612,6 +2768,10 @@ it.effect("ProviderSessionManagerV2 releases sessions when provider event stream assert.isTrue(Option.isNone(liveSession)); assert.equal(runtimeState.closeCount, 1); assert.equal(projection.providerSessions.at(-1)?.status, "error"); + assert.equal( + projection.providerSessions.at(-1)?.lastError, + "The provider event stream closed unexpectedly. Retry the turn; if it keeps failing, check the provider and server logs.", + ); }); yield* effect.pipe( diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 0b06d401ba1f9..94c38045a5d13 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -51,17 +51,26 @@ import * as CommandOutputHub from "./CommandOutputHub.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000; const RELEASE_SCOPE_CLOSE_TIMEOUT_MS = 30 * 1000; +/** Past adapters' own 10 s stop fallbacks, so they settle first when the provider answers at all. */ +export const PROVIDER_INTERRUPT_TIMEOUT_MS = 15 * 1000; const busyTurnPrefix = (providerThreadId: ProviderThreadId) => `${providerThreadId}#`; const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) => `${busyTurnPrefix(providerThreadId)}${runOrdinal}`; const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000; +const WORKING_RUN_STATUSES: ReadonlySet = new Set([ + "preparing", + "starting", + "running", + "waiting", +]); export const ProviderSessionReleaseReason = Schema.Literals([ "idle_timeout", @@ -181,6 +190,25 @@ export interface ProviderSessionManagerV2Shape { readonly detail?: string; readonly revokeMcpCredential?: boolean; }) => Effect.Effect; + /** + * Releases the session unless, at the moment of removal, a turn of a provider thread outside + * `ownProviderThreadIds` is busy on it. Checked atomically with turn starts. Succeeds with + * whether it was released. + */ + readonly releaseUnlessBusy: (input: { + readonly providerSessionId: ProviderSessionId; + readonly ownProviderThreadIds: ReadonlyArray; + readonly reason: ProviderSessionReleaseReason; + readonly detail?: string; + }) => Effect.Effect; + /** + * Other threads with a run or background work on this session's shared process; empty when + * it serves one thread. + */ + readonly otherWorkingThreads: (input: { + readonly providerSessionId: ProviderSessionId; + readonly threadId: ThreadId; + }) => Effect.Effect>; } export class ProviderSessionManagerV2 extends Context.Service< @@ -895,6 +923,7 @@ export const layerWithOptions = ( function* (input: { readonly providerSessionId: ProviderSessionId; readonly onlyIfIdleGeneration?: number; + readonly keepIfBusyOtherThan?: ReadonlyArray; }): Effect.fn.Return, DateTime.Utc]> { const key = sessionKey(input.providerSessionId); const candidate = (yield* Ref.get(sessions)).get(key); @@ -914,6 +943,15 @@ export const layerWithOptions = ( ) { return ["kept", current] as const; } + const own = input.keepIfBusyOtherThan; + if ( + own !== undefined && + [...existing.busyTurns].some( + (turnKey) => !own.some((id) => turnKey.startsWith(busyTurnPrefix(id))), + ) + ) { + return ["kept", current] as const; + } const updated = new Map(current); updated.delete(key); return ["removed", updated] as const; @@ -972,6 +1010,7 @@ export const layerWithOptions = ( readonly detachedThreadId?: ThreadId; readonly cancelIdleFiber?: boolean; readonly onlyIfIdleGeneration?: number; + readonly keepIfBusyOtherThan?: ReadonlyArray; readonly gracefulSubscribers?: boolean; }) => Effect.acquireUseRelease( @@ -1380,15 +1419,21 @@ export const layerWithOptions = ( .pipe(Effect.ensuring(Effect.sync(dropReservation))); }); - const markBusy = (providerSessionId: ProviderSessionId, turnKey: string) => + // Marks only the entry this runtime belongs to: a released or replaced session must not + // gain a busy turn that never ends. Succeeds with whether the turn was marked. + const markBusy = ( + providerSessionId: ProviderSessionId, + turnKey: string, + runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime, + ) => withActivityError( providerSessionId, Effect.gen(function* () { const key = sessionKey(providerSessionId); - const idleFiber = yield* Ref.modify(sessions, (current) => { + const marked = yield* Ref.modify(sessions, (current) => { const entry = current.get(key); - if (entry === undefined) { - return [null, current] as const; + if (entry === undefined || entry.runtime !== runtime) { + return [undefined, current] as const; } const updated = new Map(current); updated.set(key, { @@ -1397,9 +1442,11 @@ export const layerWithOptions = ( idleFiber: null, pinnedSinceMs: null, }); - return [entry.idleFiber, updated] as const; + return [{ idleFiber: entry.idleFiber }, updated] as const; }); - yield* cancelIdleFiber(idleFiber); + if (marked === undefined) return false; + yield* cancelIdleFiber(marked.idleFiber); + return true; }), ); @@ -1785,11 +1832,31 @@ export const layerWithOptions = ( ), Effect.andThen( Effect.acquireUseRelease( - observeActivity( + markBusy( providerSessionId, - markBusy( - providerSessionId, - busyTurnKey(input.providerThread.id, input.runOrdinal), + busyTurnKey(input.providerThread.id, input.runOrdinal), + runtime, + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.activity-failed", { + providerSessionId, + cause, + }).pipe(Effect.as(true)), + ), + // A release between attach and markBusy leaves nothing to mark: fail the + // start instead of running it on a runtime being torn down. + Effect.flatMap((marked) => + marked + ? Effect.void + : Effect.fail( + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: runtime.driver, + threadId: input.threadId, + providerThreadId: input.providerThread.id, + runId: input.runId, + cause: "The provider session was released while the turn started.", + }), + ), ), ), () => @@ -1808,9 +1875,26 @@ export const layerWithOptions = ( observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( Effect.andThen(runtime.steerTurn(input).pipe(turnMetrics("steer"))), ), + // A provider starved of memory can stop answering, so no caller may wait on it unbounded. interruptTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( - Effect.andThen(runtime.interruptTurn(input).pipe(turnMetrics("interrupt"))), + Effect.andThen( + runtime.interruptTurn(input).pipe( + Effect.timeoutOrElse({ + duration: PROVIDER_INTERRUPT_TIMEOUT_MS, + orElse: () => + Effect.fail( + new ProviderAdapter.ProviderAdapterInterruptTimeoutError({ + driver: runtime.driver, + providerThreadId: input.providerThread.id, + providerTurnId: input.providerTurnId, + timeoutMs: PROVIDER_INTERRUPT_TIMEOUT_MS, + }), + ), + }), + turnMetrics("interrupt"), + ), + ), ), respondToRuntimeRequest: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( @@ -1988,10 +2072,16 @@ export const layerWithOptions = ( cause, }); yield* Ref.set(entry.eventSubscribers, new Map()); + // lastError is the thread banner, so it gets the same bounded + // message as a failed turn; the raw cause survives only in the log. + yield* Effect.logWarning("orchestration-v2.provider-session-runtime-failed", { + providerSessionId: entry.runtime.providerSessionId, + cause, + }); yield* releaseEntry({ providerSessionId: entry.runtime.providerSessionId, reason: "runtime_error", - detail: Cause.pretty(cause), + detail: makeProviderFailure({ cause }).message, }).pipe(logReleaseFailure(entry.runtime.providerSessionId)); }), ), @@ -2293,6 +2383,40 @@ export const layerWithOptions = ( ), ), release: releaseEntry, + releaseUnlessBusy: Effect.fn("ProviderSessionManager.releaseUnlessBusy")(function* (input) { + const key = sessionKey(input.providerSessionId); + const before = (yield* Ref.get(sessions)).get(key); + yield* releaseEntry({ + providerSessionId: input.providerSessionId, + reason: input.reason, + ...(input.detail === undefined ? {} : { detail: input.detail }), + keepIfBusyOtherThan: input.ownProviderThreadIds, + }); + // Identity, not presence: open can install a replacement under the same id meanwhile. + return before !== undefined && (yield* Ref.get(sessions)).get(key) !== before; + }), + otherWorkingThreads: Effect.fn("ProviderSessionManager.otherWorkingThreads")( + function* (input) { + const entry = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId)); + if (entry === undefined || !entry.supportsMultipleProviderThreads) return []; + return yield* Effect.filter( + [...entry.attachedThreadIds].filter((threadId) => threadId !== input.threadId), + (threadId) => + // Runs, not the shell's activeRunId: the shell omits silent runs such as idle + // compaction. Background work outlives its run and still needs the process. + Effect.zipWith( + projectionStore.getThreadRecords(threadId, ["runs"]), + projectionStore.getThreadShell(threadId), + ({ runs }, shell) => + runs.some((run) => WORKING_RUN_STATUSES.has(run.status)) || + (shell?.pendingBackgroundTasks?.length ?? 0) > 0, + ).pipe( + // An unreadable thread may still be working; never kill it unseen. + Effect.orElseSucceed(() => true), + ), + ); + }, + ), detach: Effect.fn("ProviderSessionManager.detach")( function* (input) { const key = sessionKey(input.providerSessionId); diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index a39f6b7f036a6..65da8b23676b0 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -213,6 +213,8 @@ it.effect( getOpenRemoteDelegatedTasks: Effect.die("unused getOpenRemoteDelegatedTasks"), getShellSnapshot: () => Effect.die("unused getShellSnapshot"), listActiveSubagentThreadIds: () => Effect.die("unused listActiveSubagentThreadIds"), + countThreads: () => Effect.die("unused countThreads"), + listActiveShellIndex: () => Effect.die("unused listActiveShellIndex"), listOlderSettledThreads: () => Effect.die("unused listOlderSettledThreads"), readShellSnapshot: () => Effect.die("unused readShellSnapshot"), getThreadShell: () => Effect.die("unused getThreadShell"), @@ -261,6 +263,7 @@ it.effect( getRunMessage: () => Effect.die("not used"), isNodeStored: () => Effect.die("not used"), canStartQueuedRun: () => Effect.die("not used"), + getHeldQueuedRuns: () => Effect.die("not used"), getRecoveryThreadIds: () => Effect.die("unused getRecoveryThreadIds"), getUnreadableThreadIds: () => Effect.die("unused getUnreadableThreadIds"), getThreadSnapshot: () => Effect.die("unused getThreadSnapshot"), @@ -281,6 +284,8 @@ it.effect( closeInstance: () => Effect.void, release: () => Effect.void, detach: () => Effect.void, + otherWorkingThreads: () => Effect.succeed([]), + releaseUnlessBusy: () => Effect.succeed(true), }), ); const layerControl = ProviderTurnControlService.layer.pipe( diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts index e1dbac21bc52e..bd225bb41773b 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts @@ -38,6 +38,8 @@ export class ProviderTurnControlError extends Schema.TaggedError + Effect.fail( + new ProviderTurnControlError({ + threadId: input.threadId, + operation: "interrupt", + providerTurnId: input.providerTurnId, + unresponsive: true, + cause, + }), + ), + }), + ); // Use real time; ingestion fibers do not advance TestClock and would leave Stop blocked. yield* Effect.gen(function* () { const deadline = (yield* Clock.currentTimeMillis) + 2_000; diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 7a922d79c7317..beaf7961e3782 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -1304,6 +1304,34 @@ effectIt.effect("leaves the run starting when a session-open failure will be ret }), ); +effectIt.effect("fails the run without a retry when the provider needs sign-in", () => + Effect.gen(function* () { + const harness = makeLocalCommandHarness({ + text: "Continue", + openFailure: new ProviderAdapter.ProviderAdapterOpenSessionError({ + driver: ProviderDriverKind.make("antigravity"), + providerSessionId: ProviderSessionId.make("new-provider-session"), + requiresSignIn: true, + }), + }); + + yield* harness.startWithRetry; + + expect(harness.open).toHaveBeenCalledOnce(); + expect(harness.projection().runs.at(-1)?.status).toBe("failed"); + expect(harness.projection().turnItems).toMatchObject([ + { + type: "error", + status: "failed", + failure: { + class: "provider_error", + message: "Failed to open antigravity provider session new-provider-session.", + }, + }, + ]); + }), +); + effectIt.effect("does not terminalize a provider-session open interruption", () => Effect.gen(function* () { const harness = makeLocalCommandHarness({ text: "Continue", interruptOpen: true }); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 94ba406b106f5..fad06732fb48e 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -92,6 +92,9 @@ class WorktreeRecreateNoBaseError extends Schema.TaggedError; @@ -181,6 +181,50 @@ export function cancelledBeforeStart( return run.status === "cancelled" && neverStarted(run); } +/** The provider never accepted this continuation's prompt, so starting it again repeats nothing. */ +export function continuationFailedBeforePrompt( + run: Pick, + providerTurns: ReadonlyArray>, +): boolean { + return ( + run.status === "failed" && + run.restartContinuationOfRunId !== undefined && + !providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId) + ); +} + +const CONTINUATION_START_RETRY_DELAYS_MS = [60_000, 180_000]; + +/** Backoff before restarting a continuation that failed to start, or undefined to let it fail. */ +export function continuationStartRetryDelayMs( + run: OrchestrationV2Run, + projection: { + readonly runs: ReadonlyArray; + readonly providerTurns: ReadonlyArray>; + readonly turnItems: Parameters[1]; + }, +): number | undefined { + if (!continuationFailedBeforePrompt(run, projection.providerTurns)) return; + // These fail the same way however often they start. + const failureClass = latestRootProviderFailure(run, projection.turnItems)?.class; + if ( + failureClass === "usage_limit" || + failureClass === "permission_error" || + failureClass === "validation_error" + ) + return; + let failedStarts = 0; + for ( + let current: OrchestrationV2Run | undefined = run; + current !== undefined && continuationFailedBeforePrompt(current, projection.providerTurns); + current = projection.runs.find( + (candidate) => candidate.id === current?.restartContinuationOfRunId, + ) + ) + failedStarts += 1; + return CONTINUATION_START_RETRY_DELAYS_MS[failedStarts - 1]; +} + export function isRestartContinuationSource( source: Pick, providerTurns: ReadonlyArray>, diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index 061fc54a7f112..d2224720b89f1 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -20,6 +20,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as ServerSettings from "../serverSettings.ts"; import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts"; +import { continuationStartRetryDelayMs } from "./RestartBackgroundNote.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; @@ -46,6 +47,8 @@ function makeProjection() { providerInstanceId: instanceId, archivedAt: null, deletedAt: null, + lineage: { parentThreadId: null, relationshipToParent: null }, + forkedFrom: null, }, runs: [ { @@ -839,10 +842,13 @@ it.effect("does not cancel or resume a run that completes while shutdown intent }), ); -const continuationDispatches = (projection: OrchestrationV2ThreadProjection) => +const continuationDispatches = ( + projection: OrchestrationV2ThreadProjection, + sourceRunId: RunId = runId, +) => Effect.gen(function* () { const dispatches: Array> = []; - yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe( + yield* continueRestartedRun({ threadId, sourceRunId }).pipe( Effect.provide( Layer.merge( Layer.mock(ThreadManagementService.ThreadManagementService)({ @@ -943,6 +949,118 @@ it.effect("continues a resumed queued run that ran after an earlier continuation }), ); +it("backs off twice from a continuation the provider never took, then lets it fail", () => { + const base = makeProjection(); + const failedStart = (id: string, of: RunId) => ({ + ...base.runs[0]!, + id: RunId.make(id), + activeAttemptId: RunAttemptId.make(`attempt:${id}`), + status: "failed" as const, + restartContinuationOfRunId: of, + }); + const first = failedStart("run:retry-1", runId); + const second = failedStart("run:retry-2", first.id); + const third = failedStart("run:retry-3", second.id); + const projection = (failureClass = "provider_error", providerTurns = base.providerTurns) => + ({ + runs: [{ ...base.runs[0]!, status: "cancelled" }, first, second, third], + providerTurns, + turnItems: [first, second, third].map((run) => ({ + id: `error:${run.id}`, + type: "error", + status: "failed", + runId: run.id, + nodeId: run.rootNodeId, + ordinal: 1, + updatedAt: now, + failure: { class: failureClass, message: "Failed.", code: null, retryable: null }, + })), + }) as unknown as Parameters[1]; + assert.deepEqual( + [first, second, third].map((run) => continuationStartRetryDelayMs(run, projection())), + [60_000, 180_000, undefined], + ); + for (const failureClass of ["usage_limit", "permission_error", "validation_error"]) + assert.isUndefined(continuationStartRetryDelayMs(first, projection(failureClass))); + // Once the provider took the prompt, starting again would repeat the turn. + assert.isUndefined( + continuationStartRetryDelayMs( + first, + projection("provider_error", [ + { ...base.providerTurns[0]!, runAttemptId: first.activeAttemptId, status: "failed" }, + ] as never), + ), + ); +}); + +it.effect("continues again after a continuation fails before its prompt", () => + Effect.gen(function* () { + const cut = cutMidTurn(); + const failedStart = { + ...cut.runs[0]!, + id: RunId.make("run:failed-start"), + ordinal: 2, + activeAttemptId: RunAttemptId.make("attempt:failed-start"), + status: "failed", + restartContinuationOfRunId: runId, + }; + const dispatches = yield* continuationDispatches( + { ...cut, runs: [...cut.runs, failedStart] } as unknown as OrchestrationV2ThreadProjection, + failedStart.id, + ); + assert.deepEqual( + dispatches.map((command) => command.restartContinuationOfRunId), + [failedStart.id], + ); + }), +); + +it.effect.each([ + ["running", null, 0], + ["completed", "Done.", 1], +] as const)( + "a parent's Stop on the task's run blocks the child only while the task is %s", + ([status, result, expected]) => + Effect.gen(function* () { + const parentThreadId = ThreadId.make("thread:parent"); + const parentRunId = RunId.make("run:parent"); + const taskId = NodeId.make("node:task"); + const cut = cutMidTurn(); + const child = { + ...cut, + thread: { + ...cut.thread, + lineage: { parentThreadId, relationshipToParent: "subagent" }, + forkedFrom: { type: "node", nodeId: taskId }, + }, + } as unknown as OrchestrationV2ThreadProjection; + const parent = { + thread: { id: parentThreadId }, + subagents: [{ id: taskId, runId: parentRunId, origin: "app_owned", status, result }], + turnItems: [{ id: "turn-item:stop", runId: parentRunId, type: "run_interrupt_request" }], + }; + let dispatched = 0; + yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe( + Effect.provide( + Layer.merge( + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: (id) => + Effect.succeed((id === parentThreadId ? parent : child) as never), + recoverDelegatedTask: () => Effect.void, + dispatch: () => + Effect.sync(() => { + dispatched += 1; + return {} as never; + }), + }), + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }), + ), + ), + ); + assert.equal(dispatched, expected); + }), +); + it.effect("does not continue a run the user asked to stop before the restart", () => Effect.gen(function* () { const texts = yield* continuationTexts({ diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index c81faa6cf47cb..e4d7ac2b7f900 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -2,6 +2,7 @@ import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { CommandId, + isOrchestrationV2WorkActive, latestProviderTurnForAttempt, MessageId, type OrchestrationV2Run, @@ -19,6 +20,7 @@ import { awaitingProviderThreadId, cancelledRosterTaskWork, cancelledTurnItemWork, + continuationFailedBeforePrompt, isAwaitedWork, isRestartContinuationSource, cancelledBeforeStart, @@ -200,7 +202,10 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta const source = projection.runs.find((run) => run.id === input.sourceRunId); if ( !source || - !isRestartContinuationSource(source, projection.providerTurns) || + !( + isRestartContinuationSource(source, projection.providerTurns) || + continuationFailedBeforePrompt(source, projection.providerTurns) + ) || restartContinuationBlocked( projection, source, @@ -255,6 +260,32 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta ) ) return; + // A parent's Stop reaches its open delegated tasks through an effect a restart can cut short. + const { lineage, forkedFrom } = projection.thread; + const taskId = + source.delegatedTaskId ?? (forkedFrom?.type === "node" ? forkedFrom.nodeId : undefined); + if ( + lineage.relationshipToParent === "subagent" && + lineage.parentThreadId !== null && + taskId !== undefined + ) { + const task = (yield* threads.getThreadRecords(lineage.parentThreadId, ["subagents"], { + subagentIds: [taskId], + })).subagents.find((candidate) => candidate.id === taskId); + // A settled task no longer owns later work in its thread. + const parentRunId = + task !== undefined && task.result === null && isOrchestrationV2WorkActive(task.status) + ? task.runId + : null; + if ( + parentRunId !== null && + (yield* threads.getThreadRecords(lineage.parentThreadId, ["turnItems"], { + turnItemTypes: ["run_interrupt_request"], + turnItemRunIds: [parentRunId], + })).turnItems.length > 0 + ) + return; + } const sourceMessage = sourceRecords.messages.find( (message) => message.id === source.userMessageId, ); diff --git a/apps/server/src/orchestration-v2/RuntimePolicy.test.ts b/apps/server/src/orchestration-v2/RuntimePolicy.test.ts index 0c31dbe6d0e8d..9ab00130f8d36 100644 --- a/apps/server/src/orchestration-v2/RuntimePolicy.test.ts +++ b/apps/server/src/orchestration-v2/RuntimePolicy.test.ts @@ -79,8 +79,6 @@ const layerTest = RuntimePolicy.layerFromProjectStore.pipe( getInstance: (instanceId) => Effect.succeed(providerInstanceFor(instanceId)), listInstances: Effect.succeed([]), listUnavailable: Effect.succeed([]), - listEnvironments: Effect.succeed(new Map()), - rebuildInstanceWhen: () => Effect.succeed(false), streamChanges: Stream.empty, subscribeChanges: Effect.never, }), diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index 03cdf5e85a642..d059c4c20e9d7 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -438,6 +438,118 @@ it.live("retries a failed selection restart session open before completing", () ), ); +it.live("Stop during a restart interrupts the turn the restart could not stop", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("selection-restart-stop"); + const threadId = ThreadId.make("thread:selection-restart-stop"); + const state = yield* Ref.make({ + activeTurn: null, + opened: [], + started: [], + closedSessionCount: 0, + failedReplacementOpen: false, + }); + const base = makeRestartAdapter(state); + // A wedged provider ignores the restart's soft cancel; only Stop ends its turn. + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + ...base, + openSession: (input) => + base.openSession(input).pipe( + Effect.map((session) => ({ + ...session, + interruptTurn: (interrupt: ProviderAdapter.ProviderAdapterV2InterruptInput) => + interrupt.requestRuntimeRestart === true + ? session.interruptTurn(interrupt) + : Effect.never, + })), + ), + }; + + const result = yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const running = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => event.type === "provider-turn.updated" && event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped({ startImmediately: true }), + ); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:selection-restart-stop:create"), + threadId, + projectId: ProjectId.make("project:selection-restart-stop"), + title: "Stop during restart", + modelSelection: initialSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:selection-restart-stop:first"), + threadId, + messageId: MessageId.make("message:selection-restart-stop:first"), + text: "first", + attachments: [], + modelSelection: initialSelection, + dispatchMode: { type: "start_immediately" }, + }); + yield* worker.drain(); + yield* Fiber.join(running); + const runId = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!.id; + + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:selection-restart-stop:steer"), + threadId, + messageId: MessageId.make("message:selection-restart-stop:steer"), + text: "Are you still going?", + attachments: [], + modelSelection: initialSelection, + dispatchMode: { type: "restart_active", targetRunId: runId }, + }); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("command:selection-restart-stop:stop"), + threadId, + runId, + }); + yield* worker.drain(); + return { + projection: yield* orchestrator.getThreadProjection(threadId), + captured: yield* Ref.get(state), + }; + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "selection-restart-stop" }, + ProviderAdapterRegistry.layerSingle(adapter), + ), + ), + ); + + assert.isNull(result.captured.activeTurn); + assert.lengthOf(result.captured.started, 1); + assert.equal(result.projection.runs[0]?.status, "interrupted"); + assert.deepEqual( + result.projection.providerTurns.map((turn) => turn.status), + ["interrupted"], + ); + }), + ), +); + it.live( "restarts the live session on a model change when a newer stopped session record exists", () => diff --git a/apps/server/src/orchestration-v2/ShellStream.ts b/apps/server/src/orchestration-v2/ShellStream.ts index 8a075e13c528b..67d07d6684c08 100644 --- a/apps/server/src/orchestration-v2/ShellStream.ts +++ b/apps/server/src/orchestration-v2/ShellStream.ts @@ -31,6 +31,7 @@ import type * as SqlClient from "effect/sql/SqlClient"; import * as Stream from "effect/Stream"; import { bufferLatestLiveStream } from "./LiveStreamBudget.ts"; +import type { ActiveShellIndexEntry } from "./ProjectionStore.ts"; type ShellDelta = Extract; @@ -56,7 +57,7 @@ export const bufferShellLiveStream = ( limits, ); -export function buildActiveShellSnapshot(input: { +function buildActiveShellSnapshot(input: { readonly projects: ReadonlyArray; readonly threads: OrchestrationV2ThreadShellSnapshot; readonly snapshotSequence: number; @@ -70,26 +71,119 @@ export function buildActiveShellSnapshot(input: { }; } +type ShellTrimRow = Pick< + OrchestrationV2ThreadShell, + "id" | "projectId" | "lineage" | "settledOverride" | "settledAt" +>; + /** - * Idle subagent children have no sidebar row, so clients that opt in read them from their thread - * family instead. A child stays while it is live or the parent of another child, so the request - * and working walks over shell rows never meet a gap in a nested chain. + * What a subscriber's trims read across every active row. Computed from full shells or from the + * store's index, so a snapshot read only for the rows it sends trims the same way. */ -export function omitIdleSubagentShells( - threads: ReadonlyArray, -): ReadonlyArray { - const parentIds = new Set(); - for (const thread of threads) { - if (thread.lineage.relationshipToParent === "subagent" && thread.lineage.parentThreadId) { - parentIds.add(thread.lineage.parentThreadId); +interface ShellTrimFacts { + readonly subagentParentIds: ReadonlySet; + readonly subagentThreadIds: Set; + /** Undefined when no thread is settled, so none is older. */ + readonly olderSettledBeforeMs: number | undefined; + readonly olderSettledCounts: ReadonlyArray; +} + +const NEWEST_SETTLED_KEPT = 25; + +function shellTrimFacts(rows: ReadonlyArray): ShellTrimFacts { + const subagentParentIds = new Set(); + const subagentThreadIds = new Set(); + const settledTimes: Array = []; + for (const row of rows) { + if (row.lineage.relationshipToParent === "subagent") { + subagentThreadIds.add(row.id); + if (row.lineage.parentThreadId) subagentParentIds.add(row.lineage.parentThreadId); + } else if (row.settledOverride === "settled" && row.settledAt !== null) { + settledTimes.push(DateTime.toEpochMillis(row.settledAt)); + } + } + settledTimes.sort((left, right) => right - left); + // With fewer than 25, the oldest one is the cut, so nothing is older. + const olderSettledBeforeMs = settledTimes[NEWEST_SETTLED_KEPT - 1] ?? settledTimes.at(-1); + const counts = new Map(); + if (olderSettledBeforeMs !== undefined) { + for (const row of rows) { + if (isOlderSettledThreadShell(row, olderSettledBeforeMs)) { + counts.set(row.projectId, (counts.get(row.projectId) ?? 0) + 1); + } } } + return { + subagentParentIds, + subagentThreadIds, + olderSettledBeforeMs, + olderSettledCounts: Array.from(counts, ([projectId, count]) => ({ projectId, count })), + }; +} + +function subagentAncestorIds(threads: ReadonlyArray>) { + const ancestorIds = new Set(); + for (const thread of threads) { + if (thread.lineage.relationshipToParent !== "subagent") continue; + if (thread.lineage.parentThreadId !== null) ancestorIds.add(thread.lineage.parentThreadId); + ancestorIds.add(thread.lineage.rootThreadId); + } + return ancestorIds; +} + +function omitIdleSubagentShellsWith( + threads: ReadonlyArray, + facts: Pick, +): ReadonlyArray { return threads.filter( (thread) => thread.lineage.relationshipToParent !== "subagent" || isLiveSubagentShell(thread) || - parentIds.has(thread.id), + facts.subagentParentIds.has(thread.id), + ); +} + +function omitOlderSettledShellsWith( + input: { + readonly threads: ReadonlyArray; + readonly snapshotSequence: number; + readonly now: DateTime.Utc; + }, + facts: Pick, +): { + readonly threads: ReadonlyArray; + readonly olderSettled: OrchestrationV2OlderSettledThreads; +} { + const nowMs = DateTime.toEpochMillis(input.now); + const beforeMs = facts.olderSettledBeforeMs ?? nowMs; + const ancestorIds = subagentAncestorIds(input.threads); + const threads = input.threads.filter( + (thread) => + !isOlderSettledThreadShell(thread, beforeMs) || + thread.pinnedAt != null || + (thread.snoozedUntil != null && DateTime.toEpochMillis(thread.snoozedUntil) > nowMs) || + isLiveSubagentShell(thread) || + ancestorIds.has(thread.id), ); + return { + threads, + olderSettled: { + before: DateTime.formatIso(DateTime.makeUnsafe(beforeMs)), + asOfSequence: input.snapshotSequence, + counts: facts.olderSettledCounts, + }, + }; +} + +/** + * Idle subagent children have no sidebar row, so clients that opt in read them from their thread + * family instead. A child stays while it is live or the parent of another child, so the request + * and working walks over shell rows never meet a gap in a nested chain. + */ +export function omitIdleSubagentShells( + threads: ReadonlyArray, +): ReadonlyArray { + return omitIdleSubagentShellsWith(threads, shellTrimFacts(threads)); } /** @@ -103,18 +197,13 @@ export function activeShellRowsForSubscriber( readonly threads: ReadonlyArray; readonly subagentThreadIds: Set; } { + const facts = shellTrimFacts(threads); return { - threads: omitIdleSubagents ? omitIdleSubagentShells(threads) : threads, - subagentThreadIds: new Set( - threads - .filter((thread) => thread.lineage.relationshipToParent === "subagent") - .map((thread) => thread.id), - ), + threads: omitIdleSubagents ? omitIdleSubagentShellsWith(threads, facts) : threads, + subagentThreadIds: facts.subagentThreadIds, }; } -const NEWEST_SETTLED_KEPT = 25; - /** * Opted-in snapshots keep the 25 newest settled top-level threads (and ties) and count the rest per * project in `olderSettled`; clients page them with `getOlderSettledThreads`. Older rows stay while @@ -128,77 +217,107 @@ export function omitOlderSettledShells(input: { readonly threads: ReadonlyArray; readonly olderSettled: OrchestrationV2OlderSettledThreads; } { - const nowMs = DateTime.toEpochMillis(input.now); - const settledTimes = input.threads - .flatMap((thread) => - thread.lineage.relationshipToParent !== "subagent" && - thread.settledOverride === "settled" && - thread.settledAt !== null - ? [DateTime.toEpochMillis(thread.settledAt)] + return omitOlderSettledShellsWith(input, shellTrimFacts(input.threads)); +} + +interface ShellSubscriberOptions { + readonly omitIdleSubagents: boolean; + readonly omitOlderSettled: boolean; +} + +function trimShellSnapshot( + base: OrchestrationV2ShellSnapshot, + facts: ShellTrimFacts, + options: ShellSubscriberOptions & { readonly now: DateTime.Utc }, +): OrchestrationV2ShellSnapshot { + const threads = options.omitIdleSubagents + ? omitIdleSubagentShellsWith(base.threads, facts) + : base.threads; + if (!options.omitOlderSettled) return { ...base, threads }; + const older = omitOlderSettledShellsWith( + { threads, snapshotSequence: base.snapshotSequence, now: options.now }, + facts, + ); + return { ...base, threads: older.threads, olderSettled: older.olderSettled }; +} + +/** + * The threads whose shells a trimmed snapshot can contain: a superset of what the trims keep, so + * trimming the shells read for these ids gives the snapshot trimming every shell would. + */ +function shellSubscriberThreadIds( + index: ReadonlyArray, + options: ShellSubscriberOptions, +): ReadonlyArray { + const facts = shellTrimFacts(index); + const subagents = index.filter( + (entry) => + entry.lineage.relationshipToParent === "subagent" && + (!options.omitIdleSubagents || entry.mayBeLive || facts.subagentParentIds.has(entry.id)), + ); + const ancestorIds = subagentAncestorIds(subagents); + const keptSubagentIds = new Set(subagents.map((entry) => entry.id)); + const beforeMs = facts.olderSettledBeforeMs; + return index.flatMap((entry) => + entry.lineage.relationshipToParent === "subagent" + ? keptSubagentIds.has(entry.id) + ? [entry.id] + : [] + : !options.omitOlderSettled || + beforeMs === undefined || + !isOlderSettledThreadShell(entry, beforeMs) || + entry.pinnedAt != null || + entry.snoozedUntil != null || + entry.mayBeLive || + ancestorIds.has(entry.id) + ? [entry.id] : [], - ) - .sort((left, right) => right - left); - // With fewer than 25, the oldest one is the cut, so nothing is older. - const beforeMs = settledTimes[NEWEST_SETTLED_KEPT - 1] ?? settledTimes.at(-1) ?? nowMs; - const ancestorIds = new Set(); - for (const thread of input.threads) { - if (thread.lineage.relationshipToParent !== "subagent") continue; - if (thread.lineage.parentThreadId !== null) ancestorIds.add(thread.lineage.parentThreadId); - ancestorIds.add(thread.lineage.rootThreadId); - } - const counts = new Map(); - const threads = input.threads.filter((thread) => { - if (!isOlderSettledThreadShell(thread, beforeMs)) return true; - counts.set(thread.projectId, (counts.get(thread.projectId) ?? 0) + 1); - return ( - thread.pinnedAt != null || - (thread.snoozedUntil != null && DateTime.toEpochMillis(thread.snoozedUntil) > nowMs) || - isLiveSubagentShell(thread) || - ancestorIds.has(thread.id) - ); - }); - return { - threads, - olderSettled: { - before: DateTime.formatIso(DateTime.makeUnsafe(beforeMs)), - asOfSequence: input.snapshotSequence, - counts: Array.from(counts, ([projectId, count]) => ({ projectId, count })), - }, - }; + ); } /** - * The snapshot a shell subscriber gets for its opt-ins, and the subagent ids whose transcript - * events its stream drops (from every row, so an omitted child that wakes keeps that filter). + * A subscriber's active shell snapshot, read like `loadShellSnapshotParts`. With an opt-in, the + * store's index decides which shells to read, so threads the trims drop are never built. */ -export function shellSnapshotForSubscriber( - base: OrchestrationV2ShellSnapshot, - options: { - readonly omitIdleSubagents: boolean; - readonly omitOlderSettled: boolean; - readonly now: DateTime.Utc; - }, -): { - readonly snapshot: OrchestrationV2ShellSnapshot; - readonly subagentThreadIds: Set; -} { - const rows = activeShellRowsForSubscriber(base.threads, options.omitIdleSubagents); - if (!options.omitOlderSettled) { +export const loadSubscriberShellSnapshot = Effect.fn("ShellStream.loadSubscriberShellSnapshot")( + function* (input: { + readonly sql: SqlClient.SqlClient; + readonly listShellIndex: Effect.Effect, E1>; + readonly readThreads: ( + threadIds: ReadonlyArray | undefined, + ) => Effect.Effect, E2>; + readonly listProjects: Effect.Effect, E4>; + readonly latestSequence: Effect.Effect; + readonly options: ShellSubscriberOptions; + }) { + const trims = input.options.omitIdleSubagents || input.options.omitOlderSettled; + const read = yield* input.sql + .withTransaction( + Effect.gen(function* () { + const index = trims ? yield* input.listShellIndex : null; + return { + index, + decodeThreads: yield* input.readThreads( + index === null ? undefined : shellSubscriberThreadIds(index, input.options), + ), + projects: yield* input.listProjects, + snapshotSequence: yield* input.latestSequence, + }; + }), + ) + .pipe(NodeSqliteClient.readOnly); + const base = buildActiveShellSnapshot({ + projects: read.projects, + threads: yield* read.decodeThreads, + snapshotSequence: read.snapshotSequence, + }); + const facts = shellTrimFacts(read.index ?? base.threads); return { - snapshot: { ...base, threads: rows.threads }, - subagentThreadIds: rows.subagentThreadIds, + snapshot: trimShellSnapshot(base, facts, { ...input.options, now: yield* DateTime.now }), + subagentThreadIds: facts.subagentThreadIds, }; - } - const older = omitOlderSettledShells({ - threads: rows.threads, - snapshotSequence: base.snapshotSequence, - now: options.now, - }); - return { - snapshot: { ...base, threads: older.threads, olderSettled: older.olderSettled }, - subagentThreadIds: rows.subagentThreadIds, - }; -} + }, +); /** * One transaction covers the reads so they agree; the threads are decoded after it commits, diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index 7a45a54a312f4..b68eef4b2a40c 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -514,12 +514,9 @@ const make = Effect.gen(function* () { // Update parents first so children inherit names; reread each pass for newly updated descendants. const visited = new Set([threadId]); while (true) { - const snapshot = yield* threads.getShellSnapshot(); + const snapshot = yield* threads.getShellSnapshot({ worktreePath: worktreeCwd }); const sharing = [...snapshot.threads, ...snapshot.archivedThreads].filter( - (thread) => - !visited.has(thread.id) && - thread.worktreePath === worktreeCwd && - thread.branch === oldBranch, + (thread) => !visited.has(thread.id) && thread.branch === oldBranch, ); if (sharing.length === 0) break; for (const thread of sharing) { @@ -1066,12 +1063,14 @@ const make = Effect.gen(function* () { "ThreadLaunchService.prepareRun", )(function* (input) { const scheduled = yield* Effect.gen(function* () { - const projection = yield* threads.getThreadProjection(input.threadId); - const run = projection.runs.find((candidate) => candidate.id === input.runId); + const { thread, runs } = yield* threads.getThreadRecords(input.threadId, ["runs"], { + runIds: [input.runId], + }); + const run = runs.find((candidate) => candidate.id === input.runId); const workspacePreparation = run?.workspacePreparation; if (run?.status !== "preparing" || workspacePreparation === undefined) return; if (!(yield* reservePreparation(input.commandId))) return; - yield* scheduleRunPreparation(input, projection, run, workspacePreparation).pipe( + yield* scheduleRunPreparation(input, thread, run, workspacePreparation).pipe( Effect.onError(() => releasePreparation(input.commandId)), ); }).pipe(Effect.exit); @@ -1093,30 +1092,33 @@ const make = Effect.gen(function* () { return dispatched; }); - const scheduleRunPreparation = ( + const scheduleRunPreparation = Effect.fn("ThreadLaunchService.scheduleRunPreparation")(function* ( input: ThreadLaunchPreparationInput, - projection: OrchestrationV2ThreadProjection, + thread: OrchestrationV2ThreadProjection["thread"], run: OrchestrationV2ThreadProjection["runs"][number], workspacePreparation: ThreadLaunchWorkspaceStrategy, - ) => { - const message = projection.messages.find((candidate) => candidate.id === run.userMessageId); + ) { + const { messages } = yield* threads.getThreadRecords(input.threadId, ["messages"], { + messageIds: [run.userMessageId], + }); + const message = messages.find((candidate) => candidate.id === run.userMessageId); const reuse = workspacePreparation.type === "worktree" && - projection.thread.worktreePath !== null && - projection.thread.branch !== null + thread.worktreePath !== null && + thread.branch !== null ? { strategy: { type: "existing_worktree" as const, - worktreePath: projection.thread.worktreePath, - branch: projection.thread.branch, + worktreePath: thread.worktreePath, + branch: thread.branch, }, reusedWorktree: { baseRef: workspacePreparation.baseRef }, } : null; - return schedulePreparation( + return yield* schedulePreparation( { commandId: input.commandId, - projectId: projection.thread.projectId, + projectId: thread.projectId, workspaceStrategy: reuse?.strategy ?? workspacePreparation, ...(reuse === null ? {} : { reusedWorktree: reuse.reusedWorktree }), ...(message === undefined @@ -1132,7 +1134,7 @@ const make = Effect.gen(function* () { input.threadId, run.id, ); - }; + }); return ThreadLaunchService.of({ launch, checkWorktreeBase, retryPreparation, prepareRun }); }); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts index b2daad62681e9..9285f72d46f56 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts @@ -27,6 +27,8 @@ import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; +import { AgentScope } from "@t3tools/shared/AgentScope"; + import * as ServerSettings from "../serverSettings.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; @@ -596,3 +598,174 @@ it.effect.each([ expect(dispatched).toEqual(settles ? ["thread.settle"] : []); }), ); + +it.effect.each([ + { + name: "kills the thread's own agent and ends its turn", + others: [] as ReadonlyArray, + killed: true, + expected: { outcome: "killed", sharedWithThreadIds: [] }, + teardown: ["release"], + }, + { + name: "keeps a process another working thread shares and only ends this turn", + others: ["thread:thread-management:force-stop:other"], + killed: false, + expected: { + outcome: "shared", + sharedWithThreadIds: ["thread:thread-management:force-stop:other"], + }, + teardown: ["detach"], + }, + { + name: "still settles a finished run's background work in a shared process", + others: ["thread:thread-management:force-stop:other"], + killed: false, + settledRun: true, + expected: { + outcome: "shared", + sharedWithThreadIds: ["thread:thread-management:force-stop:other"], + }, + teardown: ["detach"], + }, + { + name: "detaches when a sibling starts a turn just before the session is released", + others: [] as ReadonlyArray, + killed: false, + siblingBusy: true, + expected: { outcome: "shared", sharedWithThreadIds: [] }, + teardown: ["release", "detach"], + }, + { + name: "reports a teardown that has not finished in time as still releasing", + others: [] as ReadonlyArray, + killed: false, + releaseHangs: true, + expected: { outcome: "releasing", sharedWithThreadIds: [] }, + teardown: ["release"], + }, +])("Force stop $name", ({ others, killed, expected, teardown, ...row }) => + Effect.gen(function* () { + const threadId = ThreadId.make("thread:thread-management:force-stop"); + const providerSessionId = ProviderSessionId.make("provider-session:force-stop"); + const providerThreadId = ProviderThreadId.make("provider-thread:force-stop"); + const runId = RunId.make("run:thread-management:force-stop"); + const dispatched: Array = []; + const teardowns: Array = []; + const tornDown = yield* Deferred.make(); + const layerTest = ThreadManagementService.layer.pipe( + Layer.provide( + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + isLive: () => Effect.succeed(true), + otherWorkingThreads: () => Effect.succeed(others.map((id) => ThreadId.make(id))), + releaseUnlessBusy: () => + Effect.sync(() => teardowns.push("release")).pipe( + Effect.andThen( + "releaseHangs" in row ? Effect.never : Effect.succeed(!("siblingBusy" in row)), + ), + ), + detach: () => + Effect.sync(() => teardowns.push("detach")).pipe( + Effect.andThen(Deferred.succeed(tornDown, undefined)), + Effect.asVoid, + ), + }), + ), + Layer.provide( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadRecords: () => + Effect.succeed({ + thread: { id: threadId, deletedAt: null }, + runs: [ + { + id: runId, + ordinal: 1, + status: "settledRun" in row ? "completed" : "running", + activeAttemptId: "attempt", + }, + // A queued follow-up neither supersedes the request nor becomes the target. + { id: RunId.make("run:queued-follow-up"), ordinal: 2, status: "queued" }, + ], + providerThreads: [{ id: providerThreadId, appThreadId: threadId, providerSessionId }], + providerTurns: [ + { + id: "provider-turn:force-stop", + providerThreadId, + runAttemptId: "attempt", + status: "settledRun" in row ? "completed" : "running", + }, + ], + } as unknown as OrchestrationV2ThreadProjection), + dispatch: (command) => + Effect.sync(() => { + dispatched.push(command); + return { sequence: 1, storedEvents: [] } as never; + }), + }), + ), + Layer.provide( + Layer.succeed(AgentScope, { + ...AgentScope.defaultValue(), + kill: () => Effect.succeed(killed), + }), + ), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(layerReceiptsTest), + ); + yield* Effect.gen(function* () { + const service = yield* ThreadManagementService.ThreadManagementService; + const stopping = yield* service.forceStopThread({ threadId, runId }).pipe(Effect.forkChild); + yield* TestClock.adjust("10 seconds"); + expect(yield* Fiber.join(stopping)).toEqual(expected); + expect(dispatched).toMatchObject([ + { + type: "thread.background-work.settle", + providerThreadId, + reason: "Force stopped by user", + }, + ]); + if (teardown.at(-1) === "detach") yield* Deferred.await(tornDown); + expect(teardowns).toEqual(teardown); + }).pipe(Effect.provide(layerTest)); + }), +); + +it.effect("Force stop refuses once a newer run has started, before killing anything", () => + Effect.gen(function* () { + const threadId = ThreadId.make("thread:thread-management:force-stop-superseded"); + let killed = false; + const layerTest = ThreadManagementService.layer.pipe( + Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), + Layer.provide( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadRecords: () => + Effect.succeed({ + thread: { id: threadId, deletedAt: null }, + runs: [ + { id: RunId.make("run:stopped"), ordinal: 1, status: "running" }, + { id: RunId.make("run:newer"), ordinal: 2, status: "running" }, + ], + providerThreads: [], + providerTurns: [], + } as unknown as OrchestrationV2ThreadProjection), + }), + ), + Layer.provide( + Layer.succeed(AgentScope, { + ...AgentScope.defaultValue(), + kill: () => Effect.sync(() => (killed = true)), + }), + ), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(layerReceiptsTest), + ); + const error = yield* Effect.gen(function* () { + const service = yield* ThreadManagementService.ThreadManagementService; + return yield* Effect.flip( + service.forceStopThread({ threadId, runId: RunId.make("run:stopped") }), + ); + }).pipe(Effect.provide(layerTest)); + expect(error.reason).toBe("superseded"); + expect(killed).toBe(false); + }), +); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index cfbbd4a68a51d..192ed4007fc17 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -14,6 +14,9 @@ import { MessageId, OrchestrationV2StopWorkflowError, type OrchestrationV2StopWorkflowInput, + OrchestrationV2ForceStopThreadError, + type OrchestrationV2ForceStopThreadInput, + type OrchestrationV2ForceStopThreadResult, type ModelSelection, type OrchestrationV2Actor, type OrchestrationV2Command, @@ -30,6 +33,7 @@ import { ProjectId, PROVIDER_SEND_TURN_MAX_FILE_BYTES, ProviderInstanceId, + type ProviderSessionId, resolveProjectProviderInstanceEnabled, RunId, type ScheduledTaskId, @@ -38,10 +42,13 @@ import { } from "@t3tools/contracts"; import { delegatedTaskProgress } from "@t3tools/provider-core/server/subagentProjection"; import { threadTranscriptHeader } from "@t3tools/shared/threadTranscript"; +import { AgentScope } from "@t3tools/shared/AgentScope"; +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; @@ -394,10 +401,15 @@ export interface ThreadManagementServiceShape { ProjectionRecords, ThreadManagementProjectionLoadError | ThreadManagementThreadNotFoundError >; - readonly getShellSnapshot: (options?: { - readonly location?: "active" | "archive"; - }) => Effect.Effect; + readonly getShellSnapshot: ( + options?: Pick< + ShellSnapshotOptions, + "location" | "projectId" | "unsettledOnly" | "worktreePath" | "threadIds" + >, + ) => Effect.Effect; readonly listActiveSubagentThreadIds: Orchestrator.OrchestratorV2["Service"]["listActiveSubagentThreadIds"]; + readonly countThreads: Orchestrator.OrchestratorV2["Service"]["countThreads"]; + readonly listActiveShellIndex: Orchestrator.OrchestratorV2["Service"]["listActiveShellIndex"]; readonly listOlderSettledThreads: Orchestrator.OrchestratorV2["Service"]["listOlderSettledThreads"]; readonly readShellSnapshot: ( options?: Pick, @@ -449,6 +461,13 @@ export interface ThreadManagementServiceShape { readonly stopWorkflow: ( input: OrchestrationV2StopWorkflowInput, ) => Effect.Effect; + /** + * Kills the thread's agent for a provider that no longer responds, then ends + * its turn as interrupted. Never kills a process other working threads share. + */ + readonly forceStopThread: ( + input: OrchestrationV2ForceStopThreadInput, + ) => Effect.Effect; readonly awaitBackgroundTasks: (input: { readonly threadId: ThreadId; readonly taskIds: ReadonlyArray; @@ -542,12 +561,16 @@ function latestSteerableRun( } const SETTLE_AFTER_RUN_WAIT_MS = 24 * 60 * 60 * 1_000; +const FORCE_STOP_REASON = "Force stopped by user"; +const FORCE_STOP_RELEASE_WAIT = "10 seconds"; +const isForceStopThreadError = Schema.is(OrchestrationV2ForceStopThreadError); const make = Effect.gen(function* () { const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor; const orchestrator = yield* Orchestrator.OrchestratorV2; const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; const layerScope = yield* Effect.scope; + const agentScope = yield* AgentScope; const legacyImporter = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; const serverSettings = yield* ServerSettings.ServerSettingsService; const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2; @@ -1067,6 +1090,153 @@ const make = Effect.gen(function* () { return { type: "interrupt_requested", run: interruptibleRun, dispatch } as const; }); + const forceStopThread: ThreadManagementServiceShape["forceStopThread"] = Effect.fn( + "ThreadManagementService.forceStopThread", + )( + function* (input) { + // Under the thread lock no new run can be created between the check and the kill. The + // settlement below dispatches, which takes that lock itself, so it runs after release. + const { records, target, killed } = yield* threadDispatch.withLock( + input.threadId, + Effect.gen(function* () { + const records = yield* orchestrator.getThreadRecords(input.threadId, [ + "runs", + "providerThreads", + "providerTurns", + ]); + // The run Force stop acts on: the active one, else the latest that started. Queued + // follow-ups never ran, so they neither count as targets nor supersede the request. + const target = + latestActiveRun(records) ?? + records.runs + .filter((run) => run.status !== "queued") + .reduce( + (current, run) => + current === undefined || run.ordinal > current.ordinal ? run : current, + undefined, + ); + if (input.runId !== undefined && input.runId !== target?.id) { + return yield* new OrchestrationV2ForceStopThreadError({ + threadId: input.threadId, + reason: "superseded", + }); + } + // Shared scopes are never tracked per thread, so this only kills the thread's own agent. + const killed = yield* agentScope.kill(input.threadId); + return { records, target, killed }; + }), + ); + const sessionIds = new Set( + records.providerThreads.flatMap((providerThread) => + providerThread.appThreadId === input.threadId && providerThread.providerSessionId !== null + ? [providerThread.providerSessionId] + : [], + ), + ); + // A run that already ended may still hold background work; settling it is a no-op otherwise. + const run = target; + if (run !== undefined) { + const active = isActiveRun(run); + const providerTurn = records.providerTurns.findLast( + (turn) => + turn.runAttemptId === run.activeAttemptId && (!active || turn.status === "running"), + ); + const commandId = CommandId.make( + `server:force-stop:${run.id}:${yield* Clock.currentTimeMillis}`, + ); + yield* providerTurn === undefined && !active + ? Effect.void + : providerTurn === undefined + ? orchestrator.dispatch({ + type: "run.interrupt", + commandId, + threadId: input.threadId, + runId: run.id, + reason: FORCE_STOP_REASON, + }) + : orchestrator.dispatch({ + type: "thread.background-work.settle", + commandId, + threadId: input.threadId, + providerThreadId: providerTurn.providerThreadId, + providerTurnId: providerTurn.id, + reason: FORCE_STOP_REASON, + }); + } + const ownProviderThreadIds = records.providerThreads.flatMap((providerThread) => + providerThread.appThreadId === input.threadId ? [providerThread.id] : [], + ); + const detach = (providerSessionId: ProviderSessionId) => + sessions + .detach({ providerSessionId, threadId: input.threadId, detail: FORCE_STOP_REASON }) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.force-stop-detach-failed", { + threadId: input.threadId, + cause, + }), + ), + Effect.forkIn(layerScope), + ); + let shared = false; + const sharedWith = new Set(); + const releases: Array<{ + readonly providerSessionId: ProviderSessionId; + readonly fiber: Fiber.Fiber; + }> = []; + for (const providerSessionId of sessionIds) { + if (!(yield* sessions.isLive(providerSessionId))) continue; + // Decided just before teardown, so the reply matches what happens to the process. + const others = yield* sessions.otherWorkingThreads({ + providerSessionId, + threadId: input.threadId, + }); + for (const threadId of others) sharedWith.add(threadId); + if (others.length > 0) { + shared = true; + yield* detach(providerSessionId); + continue; + } + // The session manager re-checks atomically with turn starts and keeps a session a + // sibling began using in the meantime. + const fiber = yield* sessions + .releaseUnlessBusy({ + providerSessionId, + ownProviderThreadIds, + reason: "manual_shutdown", + detail: FORCE_STOP_REASON, + }) + .pipe(Effect.forkIn(layerScope)); + releases.push({ providerSessionId, fiber }); + } + // Teardown waits on the provider's own exit paths; past this bound it keeps going on its own. + let releasing = false; + for (const release of releases) { + const released = yield* Fiber.join(release.fiber).pipe( + Effect.timeoutOption(FORCE_STOP_RELEASE_WAIT), + ); + if (Option.isNone(released)) { + releasing = true; + } else if (!released.value) { + shared = true; + yield* detach(release.providerSessionId); + } + } + return { + outcome: killed ? "killed" : shared ? "shared" : releasing ? "releasing" : "released", + sharedWithThreadIds: [...sharedWith], + } as const; + }, + (effect, input) => + effect.pipe( + Effect.mapError((cause) => + isForceStopThreadError(cause) + ? cause + : new OrchestrationV2ForceStopThreadError({ threadId: input.threadId, cause }), + ), + ), + ); + const stopWorkflow: ThreadManagementServiceShape["stopWorkflow"] = Effect.fn( "ThreadManagementService.stopWorkflow", )( @@ -1333,6 +1503,8 @@ const make = Effect.gen(function* () { getProjectThreadRecords, getShellSnapshot: orchestrator.getShellSnapshot, listActiveSubagentThreadIds: orchestrator.listActiveSubagentThreadIds, + countThreads: orchestrator.countThreads, + listActiveShellIndex: orchestrator.listActiveShellIndex, listOlderSettledThreads: orchestrator.listOlderSettledThreads, readShellSnapshot: orchestrator.readShellSnapshot, getThreadShell: orchestrator.getThreadShell, @@ -1347,6 +1519,7 @@ const make = Effect.gen(function* () { interruptThread, stopDelegatedTasks, stopWorkflow, + forceStopThread, awaitBackgroundTasks, getThreadEventSequence: orchestrator.getThreadEventSequence, recoverDelegatedTask: orchestrator.recoverDelegatedTask, diff --git a/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts b/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts index 012e3d2f234ca..0a0996c0ede0c 100644 --- a/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts @@ -842,6 +842,29 @@ describe("ThreadSettlementServiceV2 worker", () => { ), ); + it.effect("stops settling the rest of a sweep once auto-settlement is turned off", () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + // Sweeps settle 8 threads at a time, so the ninth starts after a dispatch turned it off. + const threads = Array.from({ length: 9 }, (_, index) => makeThread(`idle-${index}`)); + let disable: Effect.Effect = Effect.void; + const fixture = yield* makeHarness({ + snapshot: makeSnapshot(threads), + settings: { ...DEFAULT_SERVER_SETTINGS, sidebarAutoSettleAfterDays: 3 }, + onDispatch: () => disable, + }); + disable = fixture.updateSettings({ sidebarAutoSettleAfterDays: null }); + yield* Effect.gen(function* () { + const service = yield* ThreadSettlementService.ThreadSettlementServiceV2; + yield* startHarness(service, fixture.activation, fixture.snapshotReads); + const settled = (yield* Ref.get(fixture.commands)).map((command) => command.threadId); + expect(settled).not.toContain(threads[8]!.id); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + it.effect("reevaluates immediately after a pull request merge", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/ThreadSettlementService.ts b/apps/server/src/orchestration-v2/ThreadSettlementService.ts index 509dcd0307dc2..6a3d4fffb0399 100644 --- a/apps/server/src/orchestration-v2/ThreadSettlementService.ts +++ b/apps/server/src/orchestration-v2/ThreadSettlementService.ts @@ -321,8 +321,10 @@ export const make = Effect.gen(function* () { // Merge events resweep all candidates because unlinked threads settle through branch lookup. const candidates = threads.filter((thread) => isAutoSettlementCandidate(thread, nowMs)); - const settleThread = Effect.fn("ThreadSettlementServiceV2.settleThread")( + // Runs per candidate thread, thousands per sweep on busy servers; the sweep owns the span. + const settleThread = Effect.fnUntraced( function* (thread: (typeof candidates)[number], pullRequest: SettlementPullRequest | null) { + // thread.auto-settle does not recheck settings, so read them per decision; reads are cached. const currentSettings = resolveProjectSettings( yield* settingsService.getSettings, thread.projectId, @@ -414,7 +416,8 @@ export const make = Effect.gen(function* () { }; const groups = Map.groupBy(lookupCandidates, lookupKey); - const wouldSettle = Effect.fn("ThreadSettlementServiceV2.wouldSettle")(function* ( + // wouldSettle and pullRequestFor run per lookup group; the git and pull request reads they make carry the spans. + const wouldSettle = Effect.fnUntraced(function* ( group: ReadonlyArray<(typeof lookupCandidates)[number]>, pullRequest: SettlementPullRequest, ) { @@ -435,7 +438,7 @@ export const make = Effect.gen(function* () { }); }); - const pullRequestFor = Effect.fn("ThreadSettlementServiceV2.pullRequestFor")(function* ( + const pullRequestFor = Effect.fnUntraced(function* ( group: ReadonlyArray<(typeof lookupCandidates)[number]>, ) { const thread = group[0]!; diff --git a/apps/server/src/orchestration-v2/ThreadStop.test.ts b/apps/server/src/orchestration-v2/ThreadStop.test.ts index 23a52f39d33d3..16dcedaf8ac9c 100644 --- a/apps/server/src/orchestration-v2/ThreadStop.test.ts +++ b/apps/server/src/orchestration-v2/ThreadStop.test.ts @@ -1552,3 +1552,115 @@ it.effect.each(["missing session", "completed", "failed", "cancelled"] as const) ); }).pipe(Effect.provide(layerTest)), ); + +it.effect("an unanswered Stop is flagged until Force stop ends the run with its own reason", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("thread:force-stop"); + yield* createWatchingThread(threadId, 6); + yield* send(threadId, "work", "start_immediately"); + const now = yield* DateTime.now; + const started = yield* orchestrator.getThreadProjection(threadId); + const run = started.runs[0]!; + const providerThreadId = run.providerThreadId!; + const providerTurnId = ProviderTurnId.make("provider-turn:force-stop"); + const apply = (event: Omit) => + projections.apply({ + ...event, + id: EventId.make(`event:force-stop:${event.type}`), + threadId, + } as OrchestrationV2DomainEvent); + const attempt = started.attempts[0]!; + const attemptId = attempt.id; + yield* apply({ + type: "run.updated", + runId: run.id, + occurredAt: now, + payload: { ...run, status: "running", startedAt: now, activeAttemptId: attemptId }, + }); + yield* apply({ + type: "run-attempt.updated", + occurredAt: now, + payload: { ...attempt, status: "running", startedAt: now }, + }); + yield* apply({ + type: "provider-turn.updated", + occurredAt: now, + payload: { + id: providerTurnId, + providerThreadId, + nodeId: run.rootNodeId!, + runAttemptId: attemptId, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + yield* apply({ + type: "turn-item.updated", + runId: run.id, + occurredAt: now, + payload: { + id: TurnItemId.make("turn-item:force-stop:interrupt-request"), + threadId, + runId: run.id, + nodeId: run.rootNodeId!, + providerThreadId, + providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: 1_000, + status: "completed", + title: "Interrupt requested", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "run_interrupt_request", + message: "Stopped by user", + }, + }); + const target = { threadId, providerThreadId, providerTurnId }; + const request = Effect.map(orchestrator.getThreadProjection(threadId), (projection) => + projection.turnItems.find((item) => item.type === "run_interrupt_request"), + ); + + yield* orchestrator.dispatch({ + ...target, + type: "thread.interrupt.unresponsive", + commandId: CommandId.make("force-stop:unresponsive"), + }); + const flagged = yield* request; + assert.isTrue( + flagged?.type === "run_interrupt_request" && flagged.unresponsiveAt !== undefined, + ); + assert.deepEqual((yield* threadState(threadId)).runs, ["running"]); + + yield* orchestrator.dispatch({ + ...target, + type: "thread.background-work.settle", + commandId: CommandId.make("force-stop:settle"), + reason: "Force stopped by user", + }); + const settled = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual( + settled.runs.map((candidate) => candidate.status), + ["interrupted"], + ); + const result = settled.turnItems.find((item) => item.type === "run_interrupt_result"); + assert.equal( + result?.type === "run_interrupt_result" ? result.message : undefined, + "Force stopped by user", + ); + + // A provider confirming late changes nothing once the run has ended. + const late = yield* orchestrator.dispatch({ + ...target, + type: "thread.interrupt.unresponsive", + commandId: CommandId.make("force-stop:late-unresponsive"), + }); + assert.lengthOf(late.storedEvents, 0); + }).pipe(Effect.provide(layerTest)), +); diff --git a/apps/server/src/orchestration-v2/ThreadStream.ts b/apps/server/src/orchestration-v2/ThreadStream.ts index 3c4baab80ef07..80ee5c32d6152 100644 --- a/apps/server/src/orchestration-v2/ThreadStream.ts +++ b/apps/server/src/orchestration-v2/ThreadStream.ts @@ -10,7 +10,10 @@ import { buildBoundedThreadProjection, type BoundedProjectionResult, } from "./threadHistoryPaging.ts"; -import { projectThreadProjectionForWire } from "./WireProjection.ts"; +import { + capProjectionCheckpointFilesForWire, + projectThreadProjectionForWire, +} from "./WireProjection.ts"; export const THREAD_RESUME_MAX_REPLAY_EVENTS = 128; @@ -44,10 +47,9 @@ export function boundedSnapshotResponseFields(input: { readonly compactIds: boolean; }) { const { bounded } = input; + const capped = capProjectionCheckpointFilesForWire(bounded.projection); // Dedupe keeps shared items one object, which the compaction's identity check needs. - const projection = input.dedupeBodies - ? dedupeSnapshotBodies(bounded.projection) - : bounded.projection; + const projection = input.dedupeBodies ? dedupeSnapshotBodies(capped) : capped; const compactProjection = input.compactTurnItems ? omitLocalVisibleTurnItems(projection) : null; return { projection: compactProjection ?? projection, diff --git a/apps/server/src/orchestration-v2/WireProjection.test.ts b/apps/server/src/orchestration-v2/WireProjection.test.ts index d449aeb933dbf..0d63d832f0ca9 100644 --- a/apps/server/src/orchestration-v2/WireProjection.test.ts +++ b/apps/server/src/orchestration-v2/WireProjection.test.ts @@ -1,4 +1,7 @@ import { + CheckpointId, + CheckpointRef, + CheckpointScopeId, MessageId, ContextHandoffId, ProviderThreadId, @@ -381,6 +384,46 @@ describe("live body deduper", () => { ]); }); + it("caps a large checkpoint's files for the wire and still dedupes its item", () => { + const files = Array.from({ length: 250 }, (_, index) => ({ + path: `vendor/file-${index}.ts`, + kind: "modified", + additions: 2, + deletions: 1, + })); + const checkpointId = CheckpointId.make("checkpoint-large"); + const scopeId = CheckpointScopeId.make("scope-large"); + const captured = event(1, { + type: "checkpoint.captured", + payload: { + id: checkpointId, + threadId: base.threadId, + scopeId, + runId: null, + nodeId: NodeId.make("node-large"), + parentCheckpointId: null, + ordinalWithinScope: 1, + appRunOrdinal: null, + ref: CheckpointRef.make("refs/t3/checkpoints/large"), + status: "ready", + files, + capturedAt: at, + }, + }); + const item = event(2, { + type: "turn-item.updated", + payload: { ...base, type: "checkpoint", checkpointId, scopeId, files }, + }); + const dedupe = makeLiveBodyDeduper(); + const [record, card] = [captured, item].map((streamed) => + dedupe({ ...streamed, event: projectDomainEventForWire(streamed.event) }), + ); + + const filesTotal = { count: 250, additions: 500, deletions: 250 }; + expect(payloadOf(record!)).toMatchObject({ files: files.slice(0, 200), filesTotal }); + expect(payloadOf(card!)).toMatchObject({ files: [], filesOmitted: true, filesTotal }); + }); + it("ships a reply whole when its message arrives after it", () => { const first = reply(1, "a"); expect([first, message(2, "a")].map(makeLiveBodyDeduper())[0]).toBe(first); diff --git a/apps/server/src/orchestration-v2/WireProjection.ts b/apps/server/src/orchestration-v2/WireProjection.ts index 6b66f1b91f204..2181b7feec5fa 100644 --- a/apps/server/src/orchestration-v2/WireProjection.ts +++ b/apps/server/src/orchestration-v2/WireProjection.ts @@ -21,6 +21,8 @@ import { toolOutputIndicatesFailure, } from "@t3tools/shared/toolOutput"; +// A checkpoint over vendored or generated trees can list 15k+ files; the card needs a prefix. +const MAX_WIRE_CHECKPOINT_FILES = 200; const MAX_DETAIL_STRING_BYTES = 32_768; const MAX_DYNAMIC_VALUE_BYTES = 16_384; const MAX_ON_DEMAND_BYTES = 256 * 1024; @@ -96,8 +98,59 @@ function summarizeDynamicValue(value: unknown): unknown { }; } +type CheckpointFiles = Pick; + +// Same input, same output object: snapshot compaction relies on shared item identity. +const cappedCheckpointFiles = new WeakMap(); + +function capCheckpointFiles(value: T): T { + if (value.files.length <= MAX_WIRE_CHECKPOINT_FILES || value.filesTotal !== undefined) { + return value; + } + const cached = cappedCheckpointFiles.get(value); + if (cached !== undefined) return cached as T; + let additions = 0; + let deletions = 0; + for (const file of value.files) { + additions += file.additions; + deletions += file.deletions; + } + const capped = { + ...value, + files: value.files.slice(0, MAX_WIRE_CHECKPOINT_FILES), + filesTotal: { count: value.files.length, additions, deletions }, + }; + cappedCheckpointFiles.set(value, capped); + return capped; +} + +/** Idempotent; bounded snapshots that skip `projectThreadProjectionForWire` still need it. */ +export function capProjectionCheckpointFilesForWire( + projection: OrchestrationV2ThreadProjection, +): OrchestrationV2ThreadProjection { + const checkpoints = projection.checkpoints.map(capCheckpointFiles); + const turnItems = projection.turnItems.map(capCheckpointItemFiles); + const visibleTurnItems = projection.visibleTurnItems.map((row) => { + const item = capCheckpointItemFiles(row.item); + return item === row.item ? row : { ...row, item }; + }); + const unchanged = (next: ReadonlyArray, previous: ReadonlyArray) => + next.every((value, index) => value === previous[index]); + return unchanged(checkpoints, projection.checkpoints) && + unchanged(turnItems, projection.turnItems) && + unchanged(visibleTurnItems, projection.visibleTurnItems) + ? projection + : { ...projection, checkpoints, turnItems, visibleTurnItems }; +} + +function capCheckpointItemFiles(item: OrchestrationV2TurnItem): OrchestrationV2TurnItem { + return item.type === "checkpoint" ? capCheckpointFiles(item) : item; +} + export function projectTurnItemForWire(item: OrchestrationV2TurnItem): OrchestrationV2TurnItem { switch (item.type) { + case "checkpoint": + return capCheckpointFiles(item); case "handoff": { const { summary: _summary, ...projected } = item; return projected; @@ -241,6 +294,7 @@ export function projectThreadProjectionForWire( }; return { ...projection, + checkpoints: projection.checkpoints.map(capCheckpointFiles), contextHandoffs: projection.contextHandoffs.map(projectContextHandoffForWire), turnItems: projection.turnItems.map(project), visibleTurnItems: projection.visibleTurnItems.map((row) => ({ @@ -257,7 +311,9 @@ export function projectDomainEventForWire( ? { ...event, payload: projectTurnItemForWire(event.payload) } : event.type === "context-handoff.updated" ? { ...event, payload: projectContextHandoffForWire(event.payload) } - : event; + : event.type === "checkpoint.captured" + ? { ...event, payload: capCheckpointFiles(event.payload) } + : event; } const LIVE_DEDUPE_MEMORY = 32; diff --git a/apps/server/src/orchestration-v2/http.ts b/apps/server/src/orchestration-v2/http.ts index 61ff54d0c794f..c2c67eb04d9dc 100644 --- a/apps/server/src/orchestration-v2/http.ts +++ b/apps/server/src/orchestration-v2/http.ts @@ -3,7 +3,6 @@ import { EnvironmentHttpApi, type OrchestrationProjectShell, } from "@t3tools/contracts"; -import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Result from "effect/Result"; import * as Predicate from "effect/Predicate"; @@ -32,11 +31,7 @@ import { } from "./threadHistoryPaging.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; import * as ProjectStore from "./ProjectStore.ts"; -import { - buildActiveShellSnapshot, - loadShellSnapshotParts, - shellSnapshotForSubscriber, -} from "./ShellStream.ts"; +import { loadSubscriberShellSnapshot } from "./ShellStream.ts"; import { boundedSnapshotResponseFields } from "./ThreadStream.ts"; import { projectThreadProjectionForWire } from "./WireProjection.ts"; @@ -84,19 +79,19 @@ export const layer = HttpApiBuilder.group( readonly omitIdleSubagents: boolean; readonly omitOlderSettled: boolean; }) { - const base = buildActiveShellSnapshot( - yield* loadShellSnapshotParts({ - sql, - readThreads: threadManagement.readShellSnapshot({ location: "active" }), - listProjects: projectStore.listShells(), - latestSequence: applicationEvents.latestApplicationSequence, - }), - ); - const { snapshot } = shellSnapshotForSubscriber(base, { - ...options, - now: yield* DateTime.now, + const { snapshot } = yield* loadSubscriberShellSnapshot({ + sql, + listShellIndex: threadManagement.listActiveShellIndex(), + readThreads: (threadIds) => + threadManagement.readShellSnapshot({ + location: "active", + ...(threadIds === undefined ? {} : { threadIds }), + }), + listProjects: projectStore.listShells(), + latestSequence: applicationEvents.latestApplicationSequence, + options, }); - const projects = yield* enrichProjectShells(base.projects); + const projects = yield* enrichProjectShells(snapshot.projects); return { ...snapshot, projects }; }, ); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 351a74128d3e8..543c00a28a739 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -169,8 +169,6 @@ const layerTestProviderInstanceRegistry = Layer.succeed( ), listInstances: Effect.succeed([providerInstance, alternateProviderInstance]), listUnavailable: Effect.succeed([]), - listEnvironments: Effect.succeed(new Map()), - rebuildInstanceWhen: () => Effect.succeed(false), streamChanges: Stream.empty, subscribeChanges: Effect.never, }, @@ -5013,6 +5011,265 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }), ); + // A thread fed by agent messages, with helpers to end its runs as a provider would. + const queueTestThread = (threadId: ThreadId) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${threadId}:create`), + threadId, + projectId: ProjectId.make(`${threadId}:project`), + title: "Dispatcher", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + }); + const send = (key: string, createdBy: "user" | "agent", start: boolean) => + orchestrator.dispatch({ + type: "message.dispatch", + createdBy, + creationSource: createdBy === "user" ? "web" : "server", + commandId: CommandId.make(`${threadId}:message:${key}`), + threadId, + messageId: MessageId.make(`${threadId}:message:${key}`), + text: key, + attachments: [], + modelSelection, + dispatchMode: { type: start ? "start_immediately" : "queue_after_active" }, + }); + const runFor = (key: string) => + Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) => { + const run = runs.find( + (candidate) => candidate.userMessageId === MessageId.make(`${threadId}:message:${key}`), + ); + assert.isDefined(run); + return run; + }); + const startedRunIds = yield* Queue.unbounded(); + // Run updates in commit order; the run-update listener handles them in this order too. + const runUpdates: Array = []; + yield* eventSink + .stream({ threadId, afterSequence: yield* orchestrator.getThreadEventSequence(threadId) }) + .pipe( + Stream.runForEach((stored) => { + if (stored.event.type !== "run.updated") return Effect.void; + runUpdates.push(stored.event.payload); + return stored.event.payload.status === "starting" + ? Queue.offer(startedRunIds, stored.event.payload.id) + : Effect.void; + }), + Effect.forkScoped, + ); + yield* Effect.yieldNow; + const releasedAfter = (key: string, completedKey: string) => + Effect.gen(function* () { + const [run, completed] = [yield* runFor(key), yield* runFor(completedKey)]; + const released = runUpdates.findIndex( + (update) => update.id === run.id && update.queueHeld === false, + ); + const completion = runUpdates.findIndex( + (update) => update.id === completed.id && update.status === "completed", + ); + assert.isAbove(released, completion, `${key} was released before ${completedKey} ended`); + }); + const end = ( + key: string, + status: "completed" | "failed", + revision = "", + patch: Partial = {}, + ) => + Effect.gen(function* () { + const run = yield* runFor(key); + assert.isNotNull(run.rootNodeId); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + ...(status === "failed" + ? [ + { + id: EventId.make(`${threadId}:${key}:error`), + type: "turn-item.updated" as const, + threadId, + runId: run.id, + nodeId: run.rootNodeId, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { + id: TurnItemId.make(`${threadId}:${key}:error`), + type: "error" as const, + threadId, + runId: run.id, + nodeId: run.rootNodeId, + providerThreadId: run.providerThreadId, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 2, + status: "failed" as const, + title: "Provider failure", + startedAt: now, + completedAt: now, + updatedAt: now, + failure: { + class: "provider_error" as const, + message: "Provider failed.", + code: "provider_failed", + retryable: null, + }, + }, + }, + ] + : []), + { + id: EventId.make(`${threadId}:${key}:${status}${revision}`), + type: "run.updated", + threadId, + runId: run.id, + nodeId: run.rootNodeId, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...run, ...patch, status, startedAt: now, completedAt: now }, + }, + ], + }); + }); + + // The run-update listener places failure holds after the terminal event commits. + const awaitHeld = (key: string) => + Effect.gen(function* () { + const run = yield* runFor(key); + yield* eventSink.stream({ threadId, afterSequence: 0, eventType: "run.updated" }).pipe( + Stream.filter( + (stored) => + stored.event.type === "run.updated" && + stored.event.payload.id === run.id && + stored.event.payload.queueHeld === true, + ), + Stream.runHead, + ); + }); + return { send, runFor, end, startedRunIds, releasedAfter, awaitHeld }; + }); + + it.effect("releases held agent messages once a later run completes, not the user's", () => + Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const threadId = ThreadId.make("runtime-layer-failed-queue-agent-messages"); + const { send, runFor, end, startedRunIds, releasedAfter, awaitHeld } = + yield* queueTestThread(threadId); + yield* send("earlier", "agent", true); + yield* end("earlier", "completed"); + yield* send("active", "agent", true); + yield* send("issue", "agent", false); + yield* send("user", "user", false); + yield* end("active", "failed"); + yield* awaitHeld("issue"); + assert.isTrue((yield* runFor("issue")).queueHeld); + // Delivery bookkeeping rewrites an older completed run; the thread has not moved on. + yield* end("earlier", "completed", ":bookkeeping"); + // A scheduled tick finds the thread idle and starts past the held queue. + yield* send("tick", "agent", true); + assert.equal((yield* runFor("tick")).status, "starting"); + yield* end("tick", "completed"); + assert.equal(yield* Queue.take(startedRunIds), (yield* runFor("issue")).id); + yield* releasedAfter("issue", "tick"); + const user = yield* runFor("user"); + assert.equal(user.status, "queued"); + assert.isTrue(user.queueHeld); + + // A run that finishes despite a stop request does not release what the stop held. + yield* send("after-stop", "agent", false); + assert.isTrue((yield* runFor("after-stop")).queueHeld); + const issue = yield* runFor("issue"); + assert.isNotNull(issue.rootNodeId); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + { + id: EventId.make(`${threadId}:issue:interrupt-request`), + type: "turn-item.updated", + threadId, + runId: issue.id, + nodeId: issue.rootNodeId, + providerInstanceId: issue.providerInstanceId, + occurredAt: now, + payload: { + id: TurnItemId.make(`${threadId}:issue:interrupt-request`), + type: "run_interrupt_request", + threadId, + runId: issue.id, + nodeId: issue.rootNodeId, + providerThreadId: issue.providerThreadId, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 2, + status: "completed", + title: "Interrupt requested", + startedAt: now, + completedAt: now, + updatedAt: now, + message: "Stopped by user", + }, + }, + ], + }); + yield* end("issue", "completed"); + // The next completed run releases it, which also shows the listener handled the stopped one. + yield* send("probe", "agent", true); + assert.equal((yield* runFor("probe")).status, "starting"); + yield* end("probe", "completed"); + assert.equal(yield* Queue.take(startedRunIds), (yield* runFor("after-stop")).id); + yield* releasedAfter("after-stop", "probe"); + }), + ); + + it.effect("keeps held agent messages while the thread's worktree setup is unresolved", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const dispatcherId = ThreadId.make("runtime-layer-held-queue-worktree"); + const dispatcher = yield* queueTestThread(dispatcherId); + const witness = yield* queueTestThread(ThreadId.make("runtime-layer-held-queue-witness")); + yield* dispatcher.send("setup", "agent", true); + yield* dispatcher.send("issue", "agent", false); + yield* dispatcher.end("setup", "failed", "", { + workspacePreparation: { type: "worktree", baseRef: "main" }, + }); + yield* dispatcher.awaitHeld("issue"); + assert.isTrue((yield* dispatcher.runFor("issue")).queueHeld); + yield* dispatcher.send("tick", "agent", true); + assert.equal((yield* dispatcher.runFor("tick")).status, "starting"); + yield* witness.send("active", "agent", true); + yield* witness.send("queued", "agent", false); + // The failed setup never produced the thread's worktree. + const { thread } = yield* orchestrator.getThreadProjection(dispatcherId); + yield* eventSink.write({ + events: [ + { + id: EventId.make(`${dispatcherId}:worktree-missing`), + type: "thread.metadata-updated", + threadId: dispatcherId, + occurredAt: yield* DateTime.now, + payload: { ...thread, worktreePath: null }, + }, + ], + }); + yield* dispatcher.end("tick", "completed"); + // One listener handles every thread's completions in order, so the witness starting + // means the tick's completion was handled too. + yield* witness.end("active", "completed"); + assert.equal(yield* Queue.take(witness.startedRunIds), (yield* witness.runFor("queued")).id); + assert.isTrue((yield* dispatcher.runFor("issue")).queueHeld); + }), + ); + it.effect("keeps queued user selections and pauses only the limited provider", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -5822,6 +6079,165 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }), ); + it.effect.each([false, true])( + "restarts a continuation that failed before its prompt, unless stopped (%s)", + (stopped) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = ThreadId.make(`runtime-layer-continuation-start-retry-${stopped}`); + const messageId = (key: string) => MessageId.make(`${threadId}:${key}`); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${threadId}:create`), + threadId, + projectId: ProjectId.make(`${threadId}:project`), + title: "Continuation start retry", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + }); + const runFor = (key: string) => + Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) => + runs.find((candidate) => candidate.userMessageId === messageId(key)), + ); + const write = ( + key: string, + payload: OrchestrationV2Run, + extra: Parameters[0]["events"] = [], + ) => + Effect.gen(function* () { + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + ...extra, + { + id: EventId.make(`${threadId}:${key}`), + type: "run.updated", + threadId, + runId: payload.id, + providerInstanceId: payload.providerInstanceId, + occurredAt: now, + payload: { ...payload, startedAt: now, completedAt: now }, + }, + ], + }); + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${threadId}:message:cut`), + threadId, + messageId: messageId("cut"), + text: "Work that the restart cut", + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + }); + const cut = (yield* runFor("cut"))!; + yield* write("cut:cancelled", { ...cut, status: "cancelled" }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "server", + commandId: CommandId.make(`${threadId}:message:continuation`), + threadId, + messageId: messageId("continuation"), + text: "Continue where you left off.", + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + restartContinuationOfRunId: cut.id, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${threadId}:message:queued`), + threadId, + messageId: messageId("queued"), + text: "Queued behind the continuation", + attachments: [], + modelSelection, + dispatchMode: { type: "queue_after_active" }, + }); + const continuation = (yield* runFor("continuation"))!; + assert.isNotNull(continuation.rootNodeId); + const queued = (yield* runFor("queued"))!; + // The failure holds the queue after the reactor scheduled any retry. + const held = yield* eventSink + .stream({ threadId, afterSequence: yield* orchestrator.getThreadEventSequence(threadId) }) + .pipe( + Stream.filter( + (stored) => + stored.event.type === "run.updated" && + stored.event.payload.id === queued.id && + stored.event.payload.queueHeld === true, + ), + Stream.runHead, + Effect.forkScoped, + ); + yield* Effect.yieldNow; + const now = yield* DateTime.now; + yield* write("continuation:failed", { ...continuation, status: "failed" }, [ + { + id: EventId.make(`${threadId}:continuation:error`), + type: "turn-item.updated", + threadId, + runId: continuation.id, + nodeId: continuation.rootNodeId, + providerInstanceId: continuation.providerInstanceId, + occurredAt: now, + payload: { + id: TurnItemId.make(`${threadId}:continuation:error`), + type: "error", + threadId, + runId: continuation.id, + nodeId: continuation.rootNodeId, + providerThreadId: continuation.providerThreadId, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 2, + status: "failed", + title: "Provider failure", + startedAt: now, + completedAt: now, + updatedAt: now, + failure: { + class: "provider_error", + message: "Claude timed out while registering its MCP tools.", + code: null, + retryable: null, + }, + }, + }, + ]); + yield* Fiber.join(held); + const retriesOf = Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) => + runs.filter((run) => run.restartContinuationOfRunId === continuation.id), + ); + yield* worker.drain(); + assert.deepEqual(yield* retriesOf, [], "the retry waits out its backoff"); + if (stopped) { + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make(`${threadId}:stop`), + threadId, + }); + } + yield* TestClock.adjust("61 seconds"); + yield* worker.drain(); + assert.lengthOf(yield* retriesOf, stopped ? 0 : 1); + }), + ); + it.effect.each(["startup", "shutdown"] as const)( "preserves and holds queued messages across %s until explicitly resumed", (trigger) => @@ -5988,9 +6404,10 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { after.runs.find((run) => run.id === held.id), held, ); + // Nobody will resume a server message, so the completed run released it behind the wake. assert.deepEqual( after.runs.find((run) => run.id === textContinuation.id), - textContinuation, + { ...textContinuation, queueHeld: false }, ); }), ); @@ -6330,7 +6747,7 @@ it.layer(layerTest)("snooze queued runs", (it) => { payload: run.status === "queued" ? { ...run, queueHeld: true } - : { ...run, status: "completed" as const, completedAt: now }, + : { ...run, status: "cancelled" as const, completedAt: now }, })), effects: [], }); @@ -6894,6 +7311,114 @@ it.layer(layerSharedApplicationDataPlaneTest)("visited projection", (it) => { assert.instanceOf(invalidVisit, Orchestrator.OrchestratorDispatchError); }), ); + + it.effect("marks unread before the user's last turn when an automatic compaction followed", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadId = ThreadId.make("runtime-layer-mark-unread-compacted"); + const turnCompletedAt = DateTime.makeUnsafe("2026-07-24T10:00:00.000Z"); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${threadId}:create`), + threadId, + projectId: ProjectId.make(`${threadId}:project`), + title: "Compacted thread", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + const runs = [ + { text: "Continue", at: turnCompletedAt }, + { + text: "/compact", + idleCompaction: "beforeCacheExpires" as const, + at: DateTime.makeUnsafe("2026-07-24T11:00:00.000Z"), + }, + ]; + yield* (yield* EventSink.EventSinkV2).write({ + commandId: CommandId.make(`${threadId}:seed`), + events: runs.flatMap(({ at, ...message }, index) => { + const runId = RunId.make(`${threadId}:run-${index}`); + const messageId = MessageId.make(`${threadId}:message-${index}`); + return [ + { + id: EventId.make(`${messageId}:event`), + type: "message.updated" as const, + threadId, + occurredAt: at, + payload: { + ...message, + id: messageId, + threadId, + runId, + nodeId: null, + role: "user" as const, + attachments: [], + streaming: false, + createdBy: "user" as const, + creationSource: "web" as const, + createdAt: at, + updatedAt: at, + }, + }, + { + id: EventId.make(`${runId}:event`), + type: "run.created" as const, + threadId, + runId, + providerInstanceId: modelSelection.instanceId, + occurredAt: at, + payload: { + id: runId, + threadId, + ordinal: index + 1, + providerInstanceId: modelSelection.instanceId, + modelSelection, + providerThreadId: null, + userMessageId: messageId, + rootNodeId: null, + activeAttemptId: null, + status: "completed" as const, + queuePosition: null, + requestedAt: at, + startedAt: at, + completedAt: at, + checkpointId: null, + contextHandoffId: null, + }, + }, + ]; + }), + }); + + yield* orchestrator.dispatch({ + type: "thread.mark-unread", + commandId: CommandId.make(`${threadId}:mark-unread`), + threadId, + }); + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual( + projection.thread.lastVisitedAt, + DateTime.subtract(turnCompletedAt, { milliseconds: 1 }), + ); + const shell = yield* orchestrator.getThreadShell(threadId); + assert.deepEqual(shell!.latestRunCompletedAt, turnCompletedAt); + assert.deepEqual(shell!.lastVisitedAt, projection.thread.lastVisitedAt); + + yield* orchestrator.dispatch({ + type: "thread.visit", + commandId: CommandId.make(`${threadId}:visit`), + threadId, + visitedAt: DateTime.formatIso(turnCompletedAt), + }); + const revisited = yield* orchestrator.getThreadShell(threadId); + assert.deepEqual(revisited!.lastVisitedAt, turnCompletedAt); + }), + ); }); it.layer(Layer.fresh(layerSharedApplicationDataPlaneTest))("thread opens", (it) => { diff --git a/apps/server/src/persistence/AuthPairingLinks.ts b/apps/server/src/persistence/AuthPairingLinks.ts index 4e2d572bd96b4..8754bdbd2edcd 100644 --- a/apps/server/src/persistence/AuthPairingLinks.ts +++ b/apps/server/src/persistence/AuthPairingLinks.ts @@ -48,6 +48,7 @@ export const ConsumeAuthPairingLinkInput = Schema.Struct({ credential: Schema.String, proofKeyThumbprint: Schema.NullOr(Schema.String), requestedScopes: Schema.optionalKey(AuthEnvironmentScopes), + requiredScopes: Schema.optionalKey(AuthEnvironmentScopes), consumedAt: Schema.DateTimeUtcFromString, now: Schema.DateTimeUtcFromString, }); @@ -161,7 +162,14 @@ export const make = Effect.gen(function* () { const consumeAvailablePairingLinkRow = SqlSchema.findOneOption({ Request: ConsumeAuthPairingLinkInput, Result: AuthPairingLinkRawDbRow, - execute: ({ credential, proofKeyThumbprint, requestedScopes, consumedAt, now }) => + execute: ({ + credential, + proofKeyThumbprint, + requestedScopes, + requiredScopes, + consumedAt, + now, + }) => sql` UPDATE auth_pairing_links SET consumed_at = ${consumedAt} @@ -180,6 +188,13 @@ export const make = Effect.gen(function* () { SELECT value FROM json_each(auth_pairing_links.scopes) ) )) + AND NOT EXISTS ( + SELECT 1 + FROM json_each(${JSON.stringify(requiredScopes ?? [])}) AS required + WHERE required.value NOT IN ( + SELECT value FROM json_each(auth_pairing_links.scopes) + ) + ) RETURNING id AS "id", credential AS "credential", diff --git a/apps/server/src/persistence/DatabaseStorage.ts b/apps/server/src/persistence/DatabaseStorage.ts index 4907bc20f15b0..bbb4dd1399efd 100644 --- a/apps/server/src/persistence/DatabaseStorage.ts +++ b/apps/server/src/persistence/DatabaseStorage.ts @@ -49,17 +49,30 @@ const make = Effect.gen(function* () { const doctorTargetsThisDatabase = path.resolve(config.dbPath) === path.resolve(config.baseDir, "userdata", path.basename(config.dbPath)); + // Only VACUUM changes these, and doctor refuses to VACUUM while a server has the database open. + let layout: { readonly pageSize: number; readonly incrementalVacuum: boolean } | undefined; + const readLayout = Effect.suspend(() => + layout !== undefined + ? Effect.succeed(layout) + : Effect.gen(function* () { + const size = yield* sql<{ readonly page_size: number }>`PRAGMA page_size`; + const mode = yield* sql<{ readonly auto_vacuum: number }>`PRAGMA auto_vacuum`; + layout = { + pageSize: size[0]?.page_size ?? 0, + incrementalVacuum: mode[0]?.auto_vacuum === 2, + }; + return layout; + }), + ); const publishStatus = Effect.gen(function* () { if (!doctorTargetsThisDatabase) return; + const { pageSize, incrementalVacuum } = yield* readLayout; const pages = yield* sql<{ readonly page_count: number }>`PRAGMA page_count`; const free = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`; - const size = yield* sql<{ readonly page_size: number }>`PRAGMA page_size`; - const mode = yield* sql<{ readonly auto_vacuum: number }>`PRAGMA auto_vacuum`; - const pageSize = size[0]?.page_size ?? 0; const next: DatabaseStorageStatus = { databaseBytes: (pages[0]?.page_count ?? 0) * pageSize, reclaimableBytes: (free[0]?.freelist_count ?? 0) * pageSize, - incrementalVacuumEnabled: mode[0]?.auto_vacuum === 2, + incrementalVacuumEnabled: incrementalVacuum, compactCommand, }; // Every set reaches each open Storage page; most passes change nothing. @@ -71,8 +84,7 @@ const make = Effect.gen(function* () { ); const reclaim = Effect.gen(function* () { - const mode = yield* sql<{ readonly auto_vacuum: number }>`PRAGMA auto_vacuum`; - if (mode[0]?.auto_vacuum !== 2) return; + if (!(yield* readLayout).incrementalVacuum) return; // The driver steps once per statement; incremental_vacuum frees one page per step. Yield between pages. for (let page = 0; page < RECLAIM_PAGES_PER_PASS; page++) { const free = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`; diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index cc395406eb502..8143a86b15034 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -75,6 +75,7 @@ import Migration0069 from "./Migrations/069_ThreadMessageSearch.ts"; import Migration0070 from "./Migrations/070_PeerLinks.ts"; import Migration0071 from "./Migrations/071_PeerLinkFence.ts"; import Migration0072 from "./Migrations/072_WaitingLinkRequestIndex.ts"; +import Migration0073 from "./Migrations/073_ActiveTurnItemIndexes.ts"; const migrationEntries = [ [1, "OrchestrationEvents", Migration0001], @@ -150,6 +151,7 @@ const migrationEntries = [ [70, "PeerLinks", Migration0070], [71, "PeerLinkFence", Migration0071], [72, "WaitingLinkRequestIndex", Migration0072], + [73, "ActiveTurnItemIndexes", Migration0073], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/073_ActiveTurnItemIndexes.ts b/apps/server/src/persistence/Migrations/073_ActiveTurnItemIndexes.ts new file mode 100644 index 0000000000000..d9759bca6d82a --- /dev/null +++ b/apps/server/src/persistence/Migrations/073_ActiveTurnItemIndexes.ts @@ -0,0 +1,19 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + // Recovery and provider-thread reads filter live items by this exact predicate; without + // these, each read walked every historical item in the thread. + yield* sql` + CREATE INDEX orchestration_v2_projection_turn_items_active_ordinal_idx + ON orchestration_v2_projection_turn_items(thread_id, ordinal, turn_item_id) + WHERE status IN ('pending', 'running', 'waiting') + `; + yield* sql` + CREATE INDEX orchestration_v2_projection_turn_items_active_run_idx + ON orchestration_v2_projection_turn_items(thread_id, run_id) + WHERE status IN ('pending', 'running', 'waiting') + `; +}); diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index 91e40578091a7..c08062c09d1ad 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -52,6 +52,7 @@ describe("V2 preview upgrade", () => { [70, "PeerLinks"], [71, "PeerLinkFence"], [72, "WaitingLinkRequestIndex"], + [73, "ActiveTurnItemIndexes"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -147,6 +148,7 @@ describe("V2 preview upgrade", () => { [70, "PeerLinks"], [71, "PeerLinkFence"], [72, "WaitingLinkRequestIndex"], + [73, "ActiveTurnItemIndexes"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 78097b761cb74..6f3e86a7501ea 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -43,6 +43,7 @@ import { SERVER_BROWSER_AUTOMATION_CLIENT_ID, type PreviewAppearancePreference, } from "@t3tools/contracts"; +import { AgentScope, agentScopeLauncherScript } from "@t3tools/shared/AgentScope"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { normalizePreviewUrl } from "@t3tools/shared/preview"; import { resolvePreviewViewport } from "@t3tools/shared/previewViewport"; @@ -460,6 +461,8 @@ const make = Effect.gen(function* () { const runPromise = Effect.runPromiseWith(yield* Effect.context()); const launchServices = yield* Effect.context(); const setupCommand = yield* resolveRootCliCommand(PreviewBrowserHost.SETUP_SUBCOMMAND); + const agentScope = yield* AgentScope; + const launchersDir = NodePath.join(config.stateDir, "server-browser", "launchers"); const tabs = new Map(); const pendingTabs = new Map>(); @@ -473,6 +476,17 @@ const make = Effect.gen(function* () { const contexts = new ServerBrowserContexts({ profilesDir: NodePath.join(config.stateDir, "server-browser", "profiles"), executable: () => runPromise(previewBrowser.executable), + // An agent scope keeps Chrome out of the server's memory protection and OOM score. + launcher: async (executable) => { + const launch = await runPromise( + agentScope.wrap({ command: executable, args: [], name: "preview-browser" }), + ); + if (launch.command === executable) return undefined; + await NodeFSP.mkdir(launchersDir, { recursive: true }); + const launcher = NodePath.join(launchersDir, NodeCrypto.randomUUID()); + await NodeFSP.writeFile(launcher, agentScopeLauncherScript(launch), { mode: 0o700 }); + return { path: launcher, release: () => NodeFSP.rm(launcher, { force: true }) }; + }, // Playwright's launch error drops Chrome's own output; its sandbox note survives. diagnose: (executable, cause) => Effect.runPromiseWith(launchServices)( diff --git a/apps/server/src/preview/ServerBrowserContexts.test.ts b/apps/server/src/preview/ServerBrowserContexts.test.ts index 7a3babd0dcc0a..3cab864de54f2 100644 --- a/apps/server/src/preview/ServerBrowserContexts.test.ts +++ b/apps/server/src/preview/ServerBrowserContexts.test.ts @@ -55,6 +55,17 @@ const options = () => ({ env: {}, }); +const withLauncher = () => { + const release = vi.fn(async () => {}); + return { + release, + configuration: { + ...options(), + launcher: vi.fn(async (executable: string) => ({ path: `${executable}-launcher`, release })), + }, + }; +}; + beforeEach(() => { launches.launch.mockReset(); launches.persistent.mockReset(); @@ -66,19 +77,29 @@ describe("ServerBrowserContexts", () => { it("lazily shares one browser across concurrent isolated sessions and reuses each context", async () => { const browser = makeBrowser(); const launch = Promise.withResolvers(); - launches.launch.mockReturnValue(launch.promise); - const configuration = options(); + const started = Promise.withResolvers(); + launches.launch.mockImplementation(() => { + started.resolve(); + return launch.promise; + }); + const { configuration, release } = withLauncher(); const pool = new ServerBrowserContexts(configuration); expect(configuration.executable).not.toHaveBeenCalled(); const first = pool.contextFor("default", "agent-a"); const again = pool.contextFor("default", "agent-a"); const second = pool.contextFor("default", "agent-b"); const incognito = pool.contextFor(INCOGNITO_BROWSER_PROFILE_ID); + await started.promise; + // Playwright may retry the same path until the launch settles. + expect(release).not.toHaveBeenCalled(); launch.resolve(browser as unknown as Browser); const [a, reused, b, privateContext] = await Promise.all([first, again, second, incognito]); + expect(release).toHaveBeenCalledTimes(1); expect(a).toBe(reused); expect(new Set([a, b, privateContext]).size).toBe(3); - expect(launches.launch).toHaveBeenCalledTimes(1); + expect(launches.launch).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ executablePath: "/test/chromium-launcher" }), + ); expect(launches.persistent).not.toHaveBeenCalled(); expect(browser.newContext).toHaveBeenCalledWith({ viewport: { width: 1280, height: 800 }, @@ -94,19 +115,44 @@ describe("ServerBrowserContexts", () => { const browser = makeBrowser(); launches.persistent.mockResolvedValue(human as unknown as BrowserContext); launches.launch.mockResolvedValue(browser as unknown as Browser); - const pool = new ServerBrowserContexts(options()); + const { configuration, release } = withLauncher(); + const pool = new ServerBrowserContexts(configuration); const persistent = await pool.contextFor("work/team"); expect(await pool.contextFor("work/team")).toBe(persistent); expect(await pool.contextFor("work/team", "agent")).not.toBe(persistent); expect(launches.persistent).toHaveBeenCalledExactlyOnceWith( "/test/profiles/work%2Fteam", - expect.objectContaining({ chromiumSandbox: true, executablePath: "/test/chromium" }), + expect.objectContaining({ chromiumSandbox: true, executablePath: "/test/chromium-launcher" }), ); + expect(release).toHaveBeenCalledTimes(2); expect(launches.mkdir).toHaveBeenCalledWith("/test/profiles/work%2Fteam", { recursive: true }); await pool.close(); expect(human.close).toHaveBeenCalledTimes(1); }); + it("diagnoses a launcher failure against the real executable", async () => { + const diagnosed = new Error("diagnosed"); + const diagnose = vi.fn(async () => diagnosed); + const pool = new ServerBrowserContexts({ + ...options(), + diagnose, + launcher: vi.fn(async () => { + throw new Error("EACCES"); + }), + }); + await expect(pool.contextFor("default", "agent")).rejects.toBe(diagnosed); + expect(diagnose).toHaveBeenCalledExactlyOnceWith("/test/chromium", expect.any(Error)); + expect(launches.launch).not.toHaveBeenCalled(); + }); + + it("creates no launcher for a profile whose storage cannot be created", async () => { + launches.mkdir.mockRejectedValueOnce(new Error("EACCES")); + const { configuration } = withLauncher(); + const pool = new ServerBrowserContexts(configuration); + await expect(pool.contextFor("work/team")).rejects.toThrow("EACCES"); + expect(configuration.launcher).not.toHaveBeenCalled(); + }); + it("clears a profile by closing its open context before deleting its storage", async () => { const human = makeContext(); const browser = makeBrowser(); diff --git a/apps/server/src/preview/ServerBrowserContexts.ts b/apps/server/src/preview/ServerBrowserContexts.ts index 0e0f71d4e53ca..9f3fd48430752 100644 --- a/apps/server/src/preview/ServerBrowserContexts.ts +++ b/apps/server/src/preview/ServerBrowserContexts.ts @@ -17,6 +17,13 @@ const loadPlaywright = () => interface Options { readonly profilesDir: string; readonly executable: () => Promise; + /** + * A path that starts `executable`, such as one that moves Chrome out of the server's cgroup. + * Released after the whole launch call, since Playwright retries the same path after a glibc race. + */ + readonly launcher?: ( + executable: string, + ) => Promise<{ readonly path: string; readonly release: () => Promise } | undefined>; readonly diagnose?: (executable: string, cause: unknown) => Promise; readonly env?: NodeJS.ProcessEnv; readonly onContextClose?: (context: BrowserContext) => void; @@ -79,10 +86,10 @@ export class ServerBrowserContexts { } private async launchOptions() { - const executablePath = await this.options.executable(); + const executable = await this.options.executable(); const env = this.options.env ?? process.env; return { - executablePath, + executable, env, args: [ "--disable-gpu", @@ -96,19 +103,26 @@ export class ServerBrowserContexts { }; } - private async launch(options: { readonly executablePath: string }, start: () => Promise) { + private async launch(executable: string, start: (executablePath: string) => Promise) { + let release: (() => Promise) | undefined; try { - return await start(); + const launcher = await this.options.launcher?.(executable); + release = launcher?.release; + return await start(launcher?.path ?? executable); } catch (cause) { - throw (await this.options.diagnose?.(options.executablePath, cause)) ?? cause; + throw (await this.options.diagnose?.(executable, cause)) ?? cause; + } finally { + await release?.().catch(constVoid); } } private sharedBrowser() { if (!this.browser) { - const launched = this.launchOptions().then(async (options) => { + const launched = this.launchOptions().then(async ({ executable, ...options }) => { const { chromium } = loadPlaywright(); - const browser = await this.launch(options, () => chromium.launch(options)); + const browser = await this.launch(executable, (executablePath) => + chromium.launch({ ...options, executablePath }), + ); browser.on("disconnected", () => { if (this.browser === launched) this.browser = undefined; }); @@ -156,11 +170,15 @@ export class ServerBrowserContexts { return browser.newContext(contextOptions); } const directory = this.profileDirectory(profileId); - const options = await this.launchOptions(); - const { chromium } = loadPlaywright(); await NodeFSP.mkdir(directory, { recursive: true }); - return this.launch(options, () => - chromium.launchPersistentContext(directory, { ...options, ...contextOptions }), + const { executable, ...options } = await this.launchOptions(); + const { chromium } = loadPlaywright(); + return this.launch(executable, (executablePath) => + chromium.launchPersistentContext(directory, { + ...options, + ...contextOptions, + executablePath, + }), ); } diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 94ca314ac1f33..2a9998f4cfbf6 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -110,6 +110,8 @@ const withFakeSystemd = ( ...fileSystem, readFileString: (path, encoding) => fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding), + writeFileString: (path, data, options) => + fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options), }), ); const { stateDir } = yield* ServerConfig.ServerConfig; @@ -213,6 +215,36 @@ describe("AgentScope service", () => { ), ); + it.effect.each([{ oomPolicy: true }, { oomPolicy: false }])( + "Force stop kills a thread's own scope but never a shared Codex scope (OOM policy $oomPolicy)", + ({ oomPolicy }) => + withFakeSystemd( + ({ scope, units, calls, cgroupRoot }) => + Effect.gen(function* () { + const agent = yield* scope.wrap({ + command: "agent", + args: [], + name: "t", + threadId: "a", + }); + exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: 0 }); + const codex = yield* launchCodex(scope, "b"); + exitedAgent(units, cgroupRoot, unitOf(codex), { populated: true, oomKills: 0 }); + + expect(yield* scope.kill("a")).toBe(true); + expect( + NodeFS.readFileSync(NodePath.join(cgroupRoot, unitOf(agent), "cgroup.kill"), "utf8"), + ).toBe("1"); + expect(yield* scope.kill("b")).toBe(false); + expect(NodeFS.existsSync(NodePath.join(cgroupRoot, unitOf(codex), "cgroup.kill"))).toBe( + false, + ); + expect(calls.filter(([, verb]) => verb === "kill")).toEqual([]); + }), + { oomPolicy }, + ), + ); + it.effect("keeps a running scope while hundreds of other threads launch", () => withFakeSystemd(({ scope, units }) => Effect.gen(function* () { @@ -245,6 +277,34 @@ describe("AgentScope service", () => { ), ); + it.effect("caps agent memory and yields CPU, disk, and tasks to the server", () => + withFakeSystemd(({ scope, calls }) => + Effect.gen(function* () { + yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + const setProperties = calls.filter(([, verb]) => verb === "set-property"); + expect(setProperties).toEqual([ + [ + "--user", + "set-property", + "--runtime", + "app-t3code-agents.slice", + expect.stringMatching(/^MemoryHigh=\d+$/), + expect.stringMatching(/^MemoryMax=\d+$/), + ], + [ + "--user", + "set-property", + "--runtime", + "app-t3code.slice", + "CPUWeight=10", + "IOWeight=10", + "TasksMax=25%", + ], + ]); + }), + ), + ); + it.effect("stops agents a killed server left before the first launch", () => withFakeSystemd(({ scope, units, calls, stateDir }) => Effect.gen(function* () { diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index cc6d3a04121f6..b9b6b56acb140 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -19,6 +19,12 @@ import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; const AGENT_SLICE = "app-t3code-agents.slice"; +// Owns every agent slice and competes with the server's unit or scope under app.slice. +const AGENTS_PARENT_SLICE = "app-t3code.slice"; +// Low weights yield CPU and disk to the server under contention, where the user manager +// has those controllers: current systemd delegates cpu, rarely io. TasksMax keeps agents +// below logind's 33% per-user task limit, leaving forks for the server. +const AGENTS_PARENT_PROPERTIES = ["CPUWeight=10", "IOWeight=10", "TasksMax=25%"]; export function ownedAgentSlice(stateDir: string): string { const hash = NodeCrypto.createHash("sha256").update(stateDir).digest("hex").slice(0, 12); @@ -173,20 +179,28 @@ export const make = Effect.gen(function* () { yield* Effect.logInfo("Agent scopes are off: no systemd user manager"); return undefined; } + const setProperties = Effect.fn("agentScope.setProperties")(function* ( + slice: string, + properties: ReadonlyArray, + ) { + const result = yield* run(systemctl, [ + "--user", + "set-property", + "--runtime", + slice, + ...properties, + ]); + if (result._tag === "None" || result.value.code !== 0) { + yield* Effect.logWarning("Could not set resource limits on an agents slice", { + slice, + properties, + stderr: result._tag === "Some" ? result.value.stderr.trim() : undefined, + }); + } + }); const limits = agentSliceMemoryLimits(NodeOS.totalmem()); - const limited = yield* run(systemctl, [ - "--user", - "set-property", - "--runtime", - AGENT_SLICE, - `MemoryHigh=${limits.high}`, - `MemoryMax=${limits.max}`, - ]); - if (limited._tag === "None" || limited.value.code !== 0) { - yield* Effect.logWarning("Could not set memory limits on the agents slice", { - stderr: limited._tag === "Some" ? limited.value.stderr.trim() : undefined, - }); - } + yield* setProperties(AGENT_SLICE, [`MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`]); + yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES); yield* Effect.logInfo("Agent scopes are on", { slice: AGENT_SLICE, ...limits, @@ -201,7 +215,8 @@ export const make = Effect.gen(function* () { }), ); - const scopeByThread = new Map(); + // `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd. + const scopeByThread = new Map(); const readScope = Effect.fn("agentScope.readScope")(function* (systemctl: string, unit: string) { const shown = yield* run(systemctl, [ @@ -263,10 +278,11 @@ export const make = Effect.gen(function* () { systemctl: string, threadId: string, unit: string | undefined, + shared = false, ) { const previous = scopeByThread.get(threadId); scopeByThread.delete(threadId); - if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false }); + if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false, shared }); if (previous !== undefined) yield* forget(systemctl, previous.unit); if (scopeByThread.size > MAX_TRACKED_THREADS) yield* evictOldest(systemctl); }); @@ -326,6 +342,40 @@ export const make = Effect.gen(function* () { } }); + const kill: AgentScopeShape["kill"] = Effect.fn("AgentScope.kill")(function* (threadId) { + const tracked = scopeByThread.get(threadId); + if (tracked === undefined || tracked.shared) return false; + const scope = yield* systemd; + if (scope === undefined) return false; + const shown = yield* run(scope.systemctl, [ + "--user", + "show", + tracked.unit, + "--property=ActiveState,ControlGroup", + ]); + if (shown._tag === "None" || shown.value.code !== 0) return false; + const values = parseKeyValues(shown.value.stdout); + if (values.get("ActiveState") !== "active") return false; + const controlGroup = values.get("ControlGroup") ?? ""; + // cgroup.kill (Linux 5.14+) kills the whole tree at once; systemd's kill is the fallback. + const killedByCgroup = + controlGroup.length > 0 && + (yield* fileSystem + .writeFileString(path.join("/sys/fs/cgroup", controlGroup, "cgroup.kill"), "1") + .pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + )); + if (killedByCgroup) return true; + const killed = yield* run(scope.systemctl, [ + "--user", + "kill", + "--signal=SIGKILL", + tracked.unit, + ]); + return killed._tag === "Some" && killed.value.code === 0; + }); + const clear: AgentScopeShape["clear"] = Effect.fn("agentScope.clear")(function* (threadId) { if (!scopeByThread.has(threadId)) return; const scope = yield* systemd; @@ -351,7 +401,8 @@ export const make = Effect.gen(function* () { // Clear the launch thread's previous scope instead. const continuing = shared === true && scope.supportsOomPolicy; const owned = threadId !== undefined && !continuing; - if (threadId !== undefined) yield* track(scope.systemctl, threadId, owned ? unit : undefined); + if (threadId !== undefined) + yield* track(scope.systemctl, threadId, owned ? unit : undefined, shared === true); const systemdScope: AgentSystemdScope = { systemdRun: scope.systemdRun, runtimeDir: scope.runtimeDir, @@ -371,6 +422,7 @@ export const make = Effect.gen(function* () { }), oomKilled, stop, + kill, clear, } satisfies AgentScopeShape; }); diff --git a/apps/server/src/process/externalLauncher.test.ts b/apps/server/src/process/externalLauncher.test.ts index 1e816f31b77d5..3262c555bcf8c 100644 --- a/apps/server/src/process/externalLauncher.test.ts +++ b/apps/server/src/process/externalLauncher.test.ts @@ -10,11 +10,14 @@ import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { EDITORS } from "@t3tools/contracts"; +import { resolveEditorCommand } from "@t3tools/shared/editor"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { SpawnExecutableResolution } from "@t3tools/shared/shell"; import * as ExternalLauncher from "./externalLauncher.ts"; @@ -137,6 +140,79 @@ it.effect.skipIf(windowsHost)( }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +// Windows drive letters cannot resolve the POSIX editor stub. +it.effect.skipIf(windowsHost)( + "a Linux editor launch without a display takes the desktop session's display from the systemd user manager", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const binDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-editors-" }); + const codiumPath = path.join(binDir, "codium"); + yield* fileSystem.writeFileString(codiumPath, "#!/bin/sh\n"); + yield* fileSystem.chmod(codiumPath, 0o755); + + const launchEnv = (sessionManager: MockSpawnResult) => + Effect.gen(function* () { + const spawned: ChildProcess.StandardCommand[] = []; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + yield* launcher.launchEditor({ editor: "vscodium", cwd: "/workspace" }); + yield* launcher.launchBrowser("https://example.com"); + }).pipe( + Effect.provide( + layerTest({ + platform: "linux", + env: { PATH: binDir }, + onSpawn: (command) => spawned.push(command), + spawnResult: (command) => + command.command === "busctl" ? sessionManager : undefined, + }), + ), + Effect.provideService(HostProcess.Environment, { PATH: binDir }), + ); + // Only editors borrow the session display; a browser must not open on that screen. + assert.deepEqual(spawned.find((command) => command.command === "xdg-open")?.options.env, { + PATH: binDir, + }); + return spawned.find((command) => command.command === "codium")?.options.env; + }); + + assert.deepEqual( + yield* launchEnv({ + stdout: JSON.stringify({ + type: "as", + data: [ + "DISPLAY=:0", + "HOME=/home/user", + "WAYLAND_DISPLAY=wayland-1", + "XAUTHORITY=/home/user/auth files/cookie", + ], + }), + }), + { + PATH: binDir, + DISPLAY: ":0", + WAYLAND_DISPLAY: "wayland-1", + // Raw, not the $'...' quoting `systemctl show-environment` would print. + XAUTHORITY: "/home/user/auth files/cookie", + }, + ); + assert.deepEqual(yield* launchEnv({ stdout: "invalid JSON" }), { PATH: binDir }); + assert.deepEqual(yield* launchEnv({ stdout: '{"data":[42]}' }), { PATH: binDir }); + assert.deepEqual( + yield* launchEnv({ + spawnError: PlatformError.systemError({ + _tag: "NotFound", + module: "ChildProcess", + method: "spawn", + }), + }), + { PATH: binDir }, + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + it.effect("launches an installed editor with platform-safe arguments", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -335,6 +411,106 @@ for (const cwd of ["/workspace with spaces/", "/workspace with spaces/file.ts:12 ); } +for (const { layout, pathNames, fallbackNames, expected } of [ + { layout: "PATH", pathNames: ["zed", "zeditor"], fallbackNames: [], expected: "zeditor" }, + { + layout: "mixed locations", + pathNames: ["zed"], + fallbackNames: ["zeditor"], + expected: "fallback", + }, + { + layout: "fallback directory", + pathNames: [], + fallbackNames: ["zed", "zeditor"], + expected: "fallback", + }, +]) { + it.effect.skipIf(windowsHost)(`prefers the Zed editor on Linux with ${layout}`, () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-zed-linux-" }); + const binDir = path.join(home, "bin"); + const fallbackDir = path.join(home, ".local/bin"); + for (const [dir, names] of [ + [binDir, pathNames], + [fallbackDir, fallbackNames], + ] as const) { + yield* fs.makeDirectory(dir, { recursive: true }); + for (const name of names) { + yield* fs.writeFileString(path.join(dir, name), "#!/bin/sh\n"); + yield* fs.chmod(path.join(dir, name), 0o755); + } + } + + let spawned: ChildProcess.StandardCommand | undefined; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + assert.include(yield* launcher.resolveAvailableEditors(), "zed"); + yield* launcher.launchEditor({ editor: "zed", cwd: "/workspace" }); + }).pipe( + Effect.provide( + layerTest({ + platform: "linux", + env: { HOME: home, PATH: binDir }, + onSpawn: (command) => { + spawned = command; + }, + }), + ), + ); + assert.ok(spawned); + assert.equal( + spawned.command, + expected === "fallback" ? path.join(fallbackDir, "zeditor") : expected, + ); + assert.deepEqual(spawned.args, ["/workspace"]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +} + +it.effect.skipIf(windowsHost)("prefers zeditor in a later Linux fallback directory", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-zed-fallback-" }); + const localBin = path.join(home, ".local/bin"); + const systemBin = path.join(home, "system-bin"); + for (const [dir, name] of [ + [localBin, "zed"], + [systemBin, "zeditor"], + ] as const) { + yield* fs.makeDirectory(dir, { recursive: true }); + const executable = path.join(dir, name); + yield* fs.writeFileString(executable, "#!/bin/sh\n"); + yield* fs.chmod(executable, 0o755); + } + const editor = EDITORS.find((editor) => editor.id === "zed")!; + const resolve = resolveEditorCommand(editor, { HOME: home, PATH: "" }).pipe( + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(Path.Path, { + ...path, + join: (...parts) => + path.join( + ...parts.map((part) => + ["/usr/local/bin", "/usr/bin", "/snap/bin"].includes(part) ? systemBin : part, + ), + ), + }), + ); + assert.deepEqual( + yield* resolve, + Option.some({ command: path.join(systemBin, "zeditor"), baseArgs: [] }), + ); + yield* fs.remove(path.join(systemBin, "zeditor")); + assert.deepEqual( + yield* resolve, + Option.some({ command: path.join(localBin, "zed"), baseArgs: [] }), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + it.effect.each([ { editor: "webstorm", installPath: "Applications/WebStorm 2026.2.app/Contents/MacOS/webstorm" }, { editor: "idea", installPath: "Applications/IntelliJ IDEA 2026.1.4.app/Contents/MacOS/idea" }, diff --git a/apps/server/src/process/externalLauncher.ts b/apps/server/src/process/externalLauncher.ts index b8c0911a520d2..6a2eb145162e2 100644 --- a/apps/server/src/process/externalLauncher.ts +++ b/apps/server/src/process/externalLauncher.ts @@ -31,6 +31,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as ChildProcess from "effect/process/ChildProcess"; @@ -672,6 +673,56 @@ const launchEditor = Effect.fn("externalLauncher.launchEditor")(function* ( yield* launchEditorProcess(launch); }); +const GRAPHICAL_SESSION_ENV_KEYS: ReadonlySet = new Set([ + "DISPLAY", + "WAYLAND_DISPLAY", + "XAUTHORITY", +]); + +// The user manager's environment over D-Bus. `systemctl show-environment` would +// shell-quote values with spaces as $'...', while this JSON carries them raw. +const decodeManagerEnvironment = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Struct({ data: Schema.Array(Schema.String) })), +); + +// Boot services miss the desktop environment imported later into the systemd user manager. +const readGraphicalSessionEnv = Effect.fn("externalLauncher.readGraphicalSessionEnv")( + function* () { + const hostEnv = yield* HostProcess.Environment; + if ((yield* HostProcess.Platform) !== "linux" || hasGraphicalLinuxSession(hostEnv)) { + return {}; + } + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* spawner.string( + ChildProcess.make( + "busctl", + [ + "--user", + "--json=short", + "get-property", + "org.freedesktop.systemd1", + "/org/freedesktop/systemd1", + "org.freedesktop.systemd1.Manager", + "Environment", + ], + { stdin: "ignore", stderr: "ignore" }, + ), + ); + const { data } = yield* decodeManagerEnvironment(output); + return Object.fromEntries( + data.flatMap((line) => { + const separator = line.indexOf("="); + const key = line.slice(0, separator); + return separator > 0 && GRAPHICAL_SESSION_ENV_KEYS.has(key) + ? [[key, line.slice(separator + 1)]] + : []; + }), + ); + }, + Effect.timeout("2 seconds"), + Effect.orElseSucceed((): NodeJS.ProcessEnv => ({})), +); + const launchAndUnref = Effect.fn("externalLauncher.launchAndUnref")(function* ( launch: ProcessLaunch, onError: (cause: unknown) => ExternalLauncherError, @@ -681,7 +732,7 @@ const launchAndUnref = Effect.fn("externalLauncher.launchAndUnref")(function* ( const { ELECTRON_RUN_AS_NODE: _runAsNode, ...env } = yield* HostProcess.Environment; const command = ChildProcess.make(launch.command, launch.args, { ...launch.options, - env, + env: { ...env, ...launch.options.env }, extendEnv: false, }); @@ -735,6 +786,7 @@ const launchEditorProcess = Effect.fn("externalLauncher.launchEditorProcess")(fu options: { detached: launch.detached ?? true, shell: spawnCommand.shell, + env: yield* readGraphicalSessionEnv(), stdin: "ignore", stdout: "ignore", stderr: "ignore", diff --git a/apps/server/src/project/AgentSessionScanner.ts b/apps/server/src/project/AgentSessionScanner.ts index eb6e531c01b50..7ec79c38a38a9 100644 --- a/apps/server/src/project/AgentSessionScanner.ts +++ b/apps/server/src/project/AgentSessionScanner.ts @@ -32,7 +32,6 @@ import { repositoryPathFromRemoteUrl } from "@t3tools/source-control-core/server import * as ServerConfig from "../config.ts"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts"; -import { literalProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; import * as ServerSettings from "../serverSettings.ts"; import { sourceControlDefinitions } from "../sourceControl/sourceControlDefinitions.ts"; @@ -1113,10 +1112,11 @@ export const make = Effect.gen(function* () { const seenHomes = new Set(); for (const { instanceId, config: instance } of instances) { const homeVariable = source === "claudeAgent" ? "CLAUDE_CONFIG_DIR" : "CODEX_HOME"; + const configuredHome = instance.environment?.findLast( + (variable) => variable.name === homeVariable, + )?.value; const environmentHome = - literalProviderInstanceEnvironment(instance.environment).findLast( - (variable) => variable.name === homeVariable, - )?.value ?? hostEnvironment[homeVariable]; + typeof configuredHome === "string" ? configuredHome : hostEnvironment[homeVariable]; let homePath: string; if (source === "claudeAgent") { diff --git a/apps/server/src/project/ProjectService.ts b/apps/server/src/project/ProjectService.ts index 18004510baa45..3ba57db6d7e5a 100644 --- a/apps/server/src/project/ProjectService.ts +++ b/apps/server/src/project/ProjectService.ts @@ -457,15 +457,13 @@ export const make = Effect.gen(function* () { ) { const { projectId } = input; const snapshot = yield* threadProjections - .getShellSnapshot() + .getShellSnapshot({ projectId }) .pipe( Effect.mapError( (cause) => new ProjectOperationError({ operation: "list-threads", projectId, cause }), ), ); - const projectThreads = [...snapshot.threads, ...snapshot.archivedThreads].filter( - (thread) => thread.projectId === projectId, - ); + const projectThreads = [...snapshot.threads, ...snapshot.archivedThreads]; if (projectThreads.length > 0 && input.force !== true) { return yield* new ProjectNotEmptyError({ projectId }); } diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts index 2897cc0b84282..860280aa91e89 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts @@ -358,6 +358,8 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { '--add-dir "extra skills"', '--add-dir="extra skills"', `--add-dir missing --add-dir "${extraDir}"`, + `--add-dir "${extraDir}" --add-dir missing`, + `--add-dir missing "${extraDir}"`, ]) { const settings = { homePath: configDir, launchArgs }; const skills = yield* discoverClaudeSkills(settings, workspace); diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.ts b/apps/server/src/provider/Drivers/ClaudeSkills.ts index 56c2ff19bebdf..bc68e42b76a2a 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.ts @@ -5,7 +5,7 @@ import * as Path from "effect/Path"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as HostProcess from "@t3tools/shared/HostProcess"; -import { parseCliArgs } from "@t3tools/shared/cliArgs"; +import { tokenizeCliArgs } from "@t3tools/shared/cliArgs"; import { fromLenientJson } from "@t3tools/shared/schemaJson"; import { SKILL_MENTION_PATTERN } from "@t3tools/shared/composerInlineTokens"; import { parse as parseYamlDocument } from "yaml"; @@ -427,6 +427,26 @@ const resolveClaudeConfigDirPath = Effect.fn("resolveClaudeConfigDirPath")(funct return path.join(home, ".claude"); }); +// Claude's --add-dir is repeatable and variadic; parseCliArgs keeps only one value. +export function claudeLaunchArgDirectories(launchArgs?: string): ReadonlyArray { + const directories: string[] = []; + let collecting = false; + for (const token of tokenizeCliArgs(launchArgs)) { + if (token === "--") break; + if (token === "--add-dir") { + collecting = true; + } else if (token.startsWith("--add-dir=")) { + directories.push(token.slice("--add-dir=".length)); + collecting = false; + } else if (token.startsWith("-")) { + collecting = false; + } else if (collecting) { + directories.push(token); + } + } + return directories; +} + /** First name wins, matching Claude Code's precedence of user skills over project skills. */ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* ( config: Pick & Partial>, @@ -442,22 +462,15 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* environment ?? process.env, ); - const additionalDirectory = parseCliArgs(config.launchArgs ?? "").flags["add-dir"]; const roots: ReadonlyArray = [ { directory: path.join(configDirPath, "skills"), scope: "user" }, ...(cwd ? [{ directory: path.join(cwd, ".claude", "skills"), scope: "project" as const }] : []), - ...(additionalDirectory && (cwd || path.isAbsolute(additionalDirectory)) - ? [ - { - directory: path.join( - path.resolve(cwd ?? ".", additionalDirectory), - ".claude", - "skills", - ), - scope: "project" as const, - }, - ] - : []), + ...claudeLaunchArgDirectories(config.launchArgs) + .filter((directory) => directory && (cwd || path.isAbsolute(directory))) + .map((directory) => ({ + directory: path.join(path.resolve(cwd ?? ".", directory), ".claude", "skills"), + scope: "project" as const, + })), ...(yield* discoverPluginSkillRoots( configDirPath, cwd, diff --git a/apps/server/src/provider/Drivers/CodexDriver.test.ts b/apps/server/src/provider/Drivers/CodexDriver.test.ts index 90e37cc66712b..0aacca303e6a6 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.test.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.test.ts @@ -377,6 +377,107 @@ it.layer(layerTest)("CodexDriver", (it) => { ), ); + it.effect.skipIf(windowsHost)( + "runs the standalone updater against the home that holds a shared install", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const tempDir = yield* fs + .makeTempDirectoryScoped({ prefix: "t3-codex-driver-" }) + .pipe(Effect.flatMap((directory) => fs.realPath(directory))); + // One standalone install, linked onto PATH and used by two account homes. + // `İ` gets longer when lowercased, so the home must not be sliced from a + // lowercased copy of the path. + const installHome = NodePath.join(tempDir, "codex-İnstall"); + const realBinaryPath = NodePath.join( + installHome, + "packages", + "standalone", + "releases", + "0.162.1-x86_64-unknown-linux-musl", + "bin", + "codex", + ); + const binaryPath = NodePath.join(tempDir, "bin", "codex"); + yield* fs.makeDirectory(NodePath.dirname(realBinaryPath), { recursive: true }); + yield* fs.makeDirectory(NodePath.dirname(binaryPath), { recursive: true }); + yield* fs.writeFileString(realBinaryPath, "#!/bin/sh\n"); + yield* fs.chmod(realBinaryPath, 0o755); + yield* fs.symlink(realBinaryPath, binaryPath); + + const updates = yield* Effect.forEach(["codex-p", "codex-vm"], (account) => + Effect.gen(function* () { + const homePath = NodePath.join(tempDir, `home-${account}`); + yield* fs.makeDirectory(homePath, { recursive: true }); + const instance = yield* CodexDriver.create({ + instanceId: ProviderInstanceId.make(account), + displayName: account, + enabled: false, + environment: [], + config: { ...CodexDriver.defaultConfig(), binaryPath, homePath }, + }); + return (yield* instance.snapshot.resolveMaintenance()).update; + }), + ); + + for (const update of updates) { + expect(update).toMatchObject({ + command: `${binaryPath} update`, + args: ["update"], + lockKey: "codex-native", + env: { CODEX_HOME: installHome }, + }); + } + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), + Effect.scoped, + ), + ); + + it.effect.skipIf(windowsHost)("derives the standalone update home from either command path", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const tempDir = yield* fs + .makeTempDirectoryScoped({ prefix: "t3-codex-update-home-" }) + .pipe(Effect.flatMap((directory) => fs.realPath(directory))); + const linkedInstallHome = NodePath.join(tempDir, "install"); + const fixtures = [ + { binary: "codex", realPath: "/packages/standalone/bin/codex", home: "/" }, + { binary: "codex", realPath: "C:\\Packages\\Standalone\\bin\\codex.exe", home: "C:\\" }, + // The standalone tree itself is a symlink, so only the resolved path names its home. + { + binary: NodePath.join("install", "packages", "standalone", "bin", "codex"), + realPath: "/disk/codex/releases/0.163.0/bin/codex", + home: linkedInstallHome, + }, + ]; + + for (const fixture of fixtures) { + const binaryPath = NodePath.join(tempDir, fixture.binary); + yield* fs.makeDirectory(NodePath.dirname(binaryPath), { recursive: true }); + yield* fs.writeFileString(binaryPath, "#!/bin/sh\n"); + yield* fs.chmod(binaryPath, 0o755); + const instance = yield* CodexDriver.create({ + instanceId: ProviderInstanceId.make("codex-update-home"), + displayName: "Codex update home", + enabled: false, + environment: [], + config: { ...CodexDriver.defaultConfig(), binaryPath, homePath: tempDir }, + }).pipe( + // Virtualize only the executable target: these trees do not belong to this test. + Effect.provideService(FileSystem.FileSystem, { + ...fs, + realPath: (path) => + path === binaryPath ? Effect.succeed(fixture.realPath) : fs.realPath(path), + }), + ); + expect((yield* instance.snapshot.resolveMaintenance()).update?.env).toEqual({ + CODEX_HOME: fixture.home, + }); + } + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), Effect.scoped), + ); + for (const fixture of [ { name: "leaves mise npm-backend installations manual-only", diff --git a/apps/server/src/provider/Drivers/CodexDriver.ts b/apps/server/src/provider/Drivers/CodexDriver.ts index 2d078cf32594f..bbd910cf5d4a9 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.ts @@ -37,7 +37,6 @@ import { enrichProviderSnapshotWithVersionAdvisory, makeCachedProviderMaintenanceResolution, makePackageManagedProviderMaintenanceResolver, - normalizeCommandPath, resolveProviderMaintenanceCapabilitiesEffect, } from "@t3tools/provider-core/server/maintenanceResolver"; import { @@ -60,20 +59,36 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; const decodeCodexSettings = Schema.decodeSync(CodexSettings); const DRIVER_KIND = ProviderDriverKind.make("codex"); -function isCodexStandaloneCommandPath(commandPath: string): boolean { - return normalizeCommandPath(commandPath).includes("/packages/standalone/"); +const CODEX_STANDALONE_HOME_PATTERN = /^(.*)\/packages\/standalone\//is; + +function codexStandaloneHome(commandPath: string): string | null { + // Slash replacement preserves indices; Unicode lowercasing can change the path length. + const match = CODEX_STANDALONE_HOME_PATTERN.exec(commandPath.replaceAll("\\", "/")); + if (!match) { + return null; + } + const home = match[1]!; + // Keep root separators: `C:` is drive-relative, and an empty home is not `/`. + const isRoot = home === "" || /^[a-z]:$/i.test(home); + return commandPath.slice(0, home.length + (isRoot ? 1 : 0)); } -/** `codex update` owns the shared home's standalone tree; auth-overlay homes do not carry it. */ +// Standalone updates locate their install through CODEX_HOME. Account and auth-overlay +// homes can share an executable, so only the updater uses the executable's owning home. function makeCodexMaintenanceResolver(sharedHomePath: string) { return makePackageManagedProviderMaintenanceResolver({ provider: DRIVER_KIND, npmPackageName: "@openai/codex", nativeUpdate: { args: ["update"], - isCommandPath: isCodexStandaloneCommandPath, + isCommandPath: (commandPath) => codexStandaloneHome(commandPath) !== null, requiresCommandPath: true, - env: { CODEX_HOME: sharedHomePath }, + env: (context) => ({ + CODEX_HOME: + codexStandaloneHome(context.realCommandPath) ?? + codexStandaloneHome(context.resolvedCommandPath) ?? + sharedHomePath, + }), }, }); } diff --git a/apps/server/src/provider/Drivers/claudeUsage.test.ts b/apps/server/src/provider/Drivers/claudeUsage.test.ts index a9ebb05a2e1e7..6f457f4f4cf2f 100644 --- a/apps/server/src/provider/Drivers/claudeUsage.test.ts +++ b/apps/server/src/provider/Drivers/claudeUsage.test.ts @@ -32,9 +32,9 @@ function claudeLine(overrides: { describe("parseClaudeLine", () => { it("extracts token totals and a dedupe key", () => { - const record = parseClaudeLine(claudeLine({ messageId: "msg_1", contentType: "text" })); + const [record] = parseClaudeLine(claudeLine({ messageId: "msg_1", contentType: "text" })); - expect(record).not.toBeNull(); + expect(record).toBeDefined(); expect(record?.provider).toBe("claude"); expect(record?.model).toBe("claude-fable-5"); expect(record?.totals).toEqual({ @@ -50,8 +50,8 @@ describe("parseClaudeLine", () => { it("gives every content block of one message the same dedupe key", () => { // Content-block records repeat full parent usage; summing them overcounts. - const text = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "text" })); - const toolUse = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "tool_use" })); + const [text] = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "text" })); + const [toolUse] = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "tool_use" })); expect(text?.dedupeKey).toBe(toolUse?.dedupeKey); expect(text?.totals).toEqual(toolUse?.totals); @@ -86,3 +86,133 @@ describe("parseClaudeLine", () => { expect(claudeUsageFormat.parseProjected(synthetic, undefined)).toEqual([]); }); }); + +describe("Claude advisor iterations", () => { + const advisorTurn = { + type: "assistant", + timestamp: "2026-10-22T01:31:11.984Z", + sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + requestId: "req_1", + message: { + id: "msg_1", + model: "claude-opus-5-5", + usage: { + input_tokens: 4, + cache_read_input_tokens: 118320, + cache_creation_input_tokens: 2676, + output_tokens: 4506, + speed: "standard", + iterations: [ + { + type: "message", + input_tokens: 2, + cache_read_input_tokens: 58352, + cache_creation_input_tokens: 1616, + output_tokens: 141, + }, + { + type: "advisor_message", + model: "claude-fable-5-1", + input_tokens: 54204, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + output_tokens: 4229, + }, + { + type: "message", + input_tokens: 2, + cache_read_input_tokens: 59968, + cache_creation_input_tokens: 1060, + output_tokens: 4365, + }, + ], + }, + }, + }; + const withIterations = (iterations: readonly unknown[]) => ({ + ...advisorTurn, + message: { ...advisorTurn.message, usage: { ...advisorTurn.message.usage, iterations } }, + }); + const parse = (turn: unknown) => claudeUsageFormat.parseLine(JSON.stringify(turn), undefined); + + it("reports the advisor model's tokens as their own record beside the unchanged executor record", () => { + const executor = { + provider: "claude", + timestampMs: Date.parse("2026-10-22T01:31:11.984Z"), + model: "claude-opus-5-5", + sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + totals: { + uncachedInputTokens: 4, + cachedInputTokens: 118320, + cacheCreationTokens: 2676, + outputTokens: 4506, + reasoningTokens: 0, + }, + reportedCostUsd: null, + speed: "standard", + dedupeKey: "msg_1:req_1", + }; + const expected = [ + executor, + { + ...executor, + model: "claude-fable-5-1", + totals: { + uncachedInputTokens: 54204, + cachedInputTokens: 0, + cacheCreationTokens: 0, + outputTokens: 4229, + reasoningTokens: 0, + }, + dedupeKey: expect.any(String), + }, + ]; + + expect(parse(advisorTurn)).toEqual(expected); + expect(claudeUsageFormat.parseProjected(advisorTurn, undefined)).toEqual(expected); + }); + + it("keys the advisor record apart from the executor and the same across a message's content blocks", () => { + const block = (type: string) => + parse({ ...advisorTurn, message: { ...advisorTurn.message, content: [{ type }] } }); + const [executorKey, advisorKey] = block("text").map((record) => record.dedupeKey); + + expect(advisorKey).toEqual(expect.any(String)); + expect(advisorKey).not.toBe(executorKey); + expect(block("tool_use").map((record) => record.dedupeKey)).toEqual([executorKey, advisorKey]); + + const unkeyed = parse({ + ...advisorTurn, + requestId: undefined, + message: { ...advisorTurn.message, id: undefined }, + }); + expect(unkeyed.map((record) => record.dedupeKey)).toEqual([null, null]); + }); + + it("emits only the executor record without an advisor iteration", () => { + const executorIterations = advisorTurn.message.usage.iterations.filter( + (iteration) => iteration.type === "message", + ); + + expect(parse(withIterations(executorIterations))).toHaveLength(1); + expect(claudeUsageFormat.parseProjected(withIterations([]), undefined)).toHaveLength(1); + expect( + claudeUsageFormat.parseLine( + claudeLine({ messageId: "msg_3", contentType: "text" }), + undefined, + ), + ).toHaveLength(1); + }); + + it("ignores advisor iterations without a usable model and malformed entries", () => { + const unusable = withIterations([ + null, + "advisor_message", + { type: "advisor_message", input_tokens: 5 }, + { type: "advisor_message", model: "", input_tokens: 5 }, + { type: "advisor_message", model: "", input_tokens: 5 }, + ]); + + expect(parse(unusable).map((record) => record.model)).toEqual(["claude-opus-5-5"]); + }); +}); diff --git a/apps/server/src/provider/Drivers/claudeUsage.ts b/apps/server/src/provider/Drivers/claudeUsage.ts index c5ff71a5c839e..aca7696d31df5 100644 --- a/apps/server/src/provider/Drivers/claudeUsage.ts +++ b/apps/server/src/provider/Drivers/claudeUsage.ts @@ -20,35 +20,35 @@ export const CLAUDE_SYNTHETIC_MODEL = ""; * the same complete `usage` object for the parent message. Summing them overcounts by roughly * 2.4x on a real workload, so the caller must drop repeats by `dedupeKey` and keep the first. */ -export function parseClaudeLine(line: string): UsageRecord | null { +export function parseClaudeLine(line: string): readonly UsageRecord[] { let parsed: unknown; try { parsed = JSON.parse(line); } catch { - return null; + return []; } return parseClaudeRecord(parsed); } -function parseClaudeRecord(parsed: unknown): UsageRecord | null { - if (typeof parsed !== "object" || parsed === null) return null; +function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] { + if (typeof parsed !== "object" || parsed === null) return []; const record = parsed as Record; - if (record["type"] !== "assistant") return null; + if (record["type"] !== "assistant") return []; const message = record["message"]; - if (typeof message !== "object" || message === null) return null; + if (typeof message !== "object" || message === null) return []; const messageRecord = message as Record; const usage = messageRecord["usage"]; - if (typeof usage !== "object" || usage === null) return null; + if (typeof usage !== "object" || usage === null) return []; const usageRecord = usage as Record; const timestampMs = parseTimestampMs(record["timestamp"]); - if (timestampMs === null) return null; + if (timestampMs === null) return []; const model = typeof messageRecord["model"] === "string" ? messageRecord["model"] : ""; - if (model.length === 0 || model === CLAUDE_SYNTHETIC_MODEL) return null; + if (model.length === 0 || model === CLAUDE_SYNTHETIC_MODEL) return []; const messageId = typeof messageRecord["id"] === "string" ? messageRecord["id"] : null; const requestId = typeof record["requestId"] === "string" ? record["requestId"] : null; @@ -57,7 +57,7 @@ function parseClaudeRecord(parsed: unknown): UsageRecord | null { const cost = record["costUSD"]; - return { + const executor: UsageRecord = { provider: "claude", timestampMs, model, @@ -74,10 +74,40 @@ function parseClaudeRecord(parsed: unknown): UsageRecord | null { speed: usageRecord["speed"] === "fast" ? "fast" : "standard", dedupeKey, }; -} -const orEmpty = (record: UsageRecord | null): readonly UsageRecord[] => - record === null ? [] : [record]; + // Top-level `usage` is the executor's alone. Advisor calls run on another model at its own rates + // and appear only in `iterations`, so each one is its own record under that model. + const records = [executor]; + const iterations: readonly unknown[] = Array.isArray(usageRecord["iterations"]) + ? usageRecord["iterations"] + : []; + for (const [index, iteration] of iterations.entries()) { + if (typeof iteration !== "object" || iteration === null) continue; + const advisor = iteration as Record; + if (advisor["type"] !== "advisor_message") continue; + + const advisorModel = typeof advisor["model"] === "string" ? advisor["model"] : ""; + if (advisorModel.length === 0 || advisorModel === CLAUDE_SYNTHETIC_MODEL) continue; + + records.push({ + provider: "claude", + timestampMs, + model: advisorModel, + sessionId: executor.sessionId, + totals: { + uncachedInputTokens: tokenCount(advisor["input_tokens"]), + cachedInputTokens: tokenCount(advisor["cache_read_input_tokens"]), + cacheCreationTokens: tokenCount(advisor["cache_creation_input_tokens"]), + outputTokens: tokenCount(advisor["output_tokens"]), + reasoningTokens: 0, + }, + reportedCostUsd: null, + speed: executor.speed, + dedupeKey: dedupeKey === null ? null : `${dedupeKey}:advisor:${index}`, + }); + } + return records; +} export const claudeUsageFormat: TranscriptUsageFormat = { selectFields: { @@ -89,8 +119,8 @@ export const claudeUsageFormat: TranscriptUsageFormat = { message: { id: true, model: true, usage: true }, }, usageMarkers: ['"usage"'], - parseLine: (line) => orEmpty(parseClaudeLine(line)), - parseProjected: (projected) => orEmpty(parseClaudeRecord(projected)), + parseLine: (line) => parseClaudeLine(line), + parseProjected: (projected) => parseClaudeRecord(projected), }; export const claudeUsageReader: ProviderUsageReader = { diff --git a/apps/server/src/provider/ProviderInstanceRegistry.test.ts b/apps/server/src/provider/ProviderInstanceRegistry.test.ts index 3cd6c62336b97..dc9c94e854f68 100644 --- a/apps/server/src/provider/ProviderInstanceRegistry.test.ts +++ b/apps/server/src/provider/ProviderInstanceRegistry.test.ts @@ -12,18 +12,21 @@ import { type CodexSettings, ProviderDriverKind, type ProviderInstanceConfigMap, - ProviderInstanceEnvironment, + type ResolvedProviderInstanceEnvironment, ProviderInstanceId, + SecretProviderId, + type SecretReference, } from "@t3tools/contracts"; import * as ProviderLatestVersions from "@t3tools/provider-core/server/ProviderLatestVersions"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; -import * as Ref from "effect/Ref"; +import * as PubSub from "effect/PubSub"; import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; import { HttpClient, HttpClientResponse } from "effect/http"; @@ -40,7 +43,8 @@ import { makeProviderInstanceRegistry } from "./ProviderInstanceRegistry.ts"; import * as ProviderOrchestrationAdapterInfrastructure from "./ProviderOrchestrationAdapterInfrastructure.ts"; import * as PortScanner from "../preview/PortScanner.ts"; import * as ProviderHostLive from "./ProviderHostLive.ts"; -import { ProviderSecretResolver } from "./ProviderSecretResolver.ts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import type { AnyProviderDriver, ProviderInstance } from "@t3tools/provider-core/server/driver"; vi.mock("@anthropic-ai/claude-agent-sdk", { spy: true }); @@ -54,8 +58,6 @@ const layerTestHttpClient = Layer.succeed( // The mock CLIs from makeTildeProviderFixtures are scripts Windows cannot execute. const windowsHost = HostProcess.Platform.defaultValue() === "win32"; -const decodeProviderInstanceEnvironment = Schema.decodeSync(ProviderInstanceEnvironment); - const TEST_EPOCH = DateTime.makeUnsafe("1970-01-01T00:00:00.000Z"); const layerBackgroundPolicyAlwaysRun = Layer.mock(BackgroundPolicy.BackgroundPolicy)({ @@ -170,6 +172,7 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { }), ), Layer.provideMerge(Layer.mock(ServerSecretStore.ServerSecretStore)({})), + Layer.provideMerge(Layer.mock(SecretResolver.SecretResolver)({})), Layer.provideMerge( Layer.succeed(ServerEnvironment.ServerEnvironmentIdentity, { getEnvironmentId: Effect.succeed( @@ -648,87 +651,170 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { expect(ghost.unavailableReason).toMatch(/ghostDriver/); }).pipe(Effect.provide(layerTest)), ); +}); + +const DisabledByDefaultConfig = Schema.Struct({ + enabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), +}); + +describe("ProviderInstanceRegistry secret environment", () => { + const driverKind = ProviderDriverKind.make("fake"); + const instanceId = ProviderInstanceId.make("fake_main"); + const secret: SecretReference = { + provider: SecretProviderId.make("personal"), + reference: "op://Private/OpenAI/credential", + }; - describe("when a secret cannot be read", () => { - const instanceId = ProviderInstanceId.make("codex_locked"); + function setup( + initial: Record, + configSchema: AnyProviderDriver["configSchema"] = Schema.Unknown, + ) { + const vault = new Map(Object.entries(initial)); + const reads: Array> = []; + const invalidated: Array> = []; + const created: Array = []; + const driver: AnyProviderDriver = { + driverKind, + metadata: { displayName: "Fake" }, + configSchema, + defaultConfig: () => ({}), + create: (input) => + Effect.sync(() => { + created.push(input.environment); + return { instanceId: input.instanceId, enabled: input.enabled } as ProviderInstance; + }), + }; + const gate = Deferred.makeUnsafe(); + const resolver = Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => + Deferred.await(gate).pipe( + Effect.andThen( + Effect.sync(() => { + reads.push(references.map((reference) => reference.reference)); + return references.map((reference) => vault.get(reference.reference)); + }), + ), + ), + invalidate: (references = []) => + Effect.sync(() => { + invalidated.push(references.map((reference) => reference.reference)); + }), + }); const configMap: ProviderInstanceConfigMap = { [instanceId]: { - driver: ProviderDriverKind.make("codex"), - enabled: true, - environment: decodeProviderInstanceEnvironment([ - { - name: "CODEX_TOKEN", - value: { kind: "1password", reference: "op://Private/codex/token", account: "a.b.com" }, - }, - ]), - config: makeCodexConfig({ enabled: true }), + driver: driverKind, + config: {}, + environment: [ + { name: "OPENAI_API_KEY", value: secret, sensitive: true }, + { name: "BASE_URL", value: "https://example.test", sensitive: false }, + ], }, }; - const lockedStore = (unlocked: Ref.Ref, reads?: Ref.Ref) => ({ - prime: () => Effect.void, - invalidate: () => Effect.void, - check: () => Effect.succeed({ _tag: "Resolved" } as const), - listOnePasswordAccounts: Effect.succeed([]), - resolve: () => - (reads === undefined ? Effect.void : Ref.update(reads, (n) => n + 1)).pipe( - Effect.andThen(Ref.get(unlocked)), - Effect.map((open) => - open - ? { - variables: [{ name: "CODEX_TOKEN", value: "sk", sensitive: true }], - unresolved: [], - } - : { variables: [], unresolved: ["CODEX_TOKEN"] }, - ), - ), + const makeGated = makeProviderInstanceRegistry({ drivers: [driver], configMap }).pipe( + Effect.provide(resolver), + ); + // Secrets are read after startup, so wait for the change that applies them. + const make = Effect.gen(function* () { + const built = yield* makeGated; + const changes = yield* built.registry.subscribeChanges; + yield* Deferred.succeed(gate, undefined); + yield* PubSub.take(changes); + return built; }); + return { vault, reads, invalidated, created, make, makeGated, gate }; + } - it.live("is unavailable with a reason that names the variable", () => - Effect.gen(function* () { - const unlocked = yield* Ref.make(false); - const { registry } = yield* makeProviderInstanceRegistry({ - drivers: [CodexDriver], - configMap, - }).pipe(Effect.provideService(ProviderSecretResolver, lockedStore(unlocked))); + it.effect("starts an instance with the secret its environment points at", () => { + const fixture = setup({ [secret.reference]: "sk-live" }); + return Effect.gen(function* () { + const { registry } = yield* fixture.make; + + expect(yield* registry.getInstance(instanceId)).toBeDefined(); + expect(fixture.created).toEqual([ + [ + { name: "OPENAI_API_KEY", value: "sk-live", sensitive: true }, + { name: "BASE_URL", value: "https://example.test", sensitive: false }, + ], + ]); + }).pipe(Effect.scoped); + }); - expect(yield* registry.getInstance(instanceId)).toBeUndefined(); - const [snapshot] = yield* registry.listUnavailable; - expect(snapshot?.unavailableReason).toContain("CODEX_TOKEN"); - }).pipe(Effect.provide(layerTest)), - ); + it.effect("marks an instance unavailable and names the variable it could not read", () => { + const fixture = setup({}); + return Effect.gen(function* () { + const { registry } = yield* fixture.make; + + expect(yield* registry.getInstance(instanceId)).toBeUndefined(); + expect(fixture.created).toEqual([]); + const [unavailable] = yield* registry.listUnavailable; + expect(unavailable?.instanceId).toBe(instanceId); + expect(unavailable?.unavailableReason).toContain("OPENAI_API_KEY"); + }).pipe(Effect.scoped); + }); - it.live("is rebuilt once the store can be read", () => - Effect.gen(function* () { - const unlocked = yield* Ref.make(false); - const { registry } = yield* makeProviderInstanceRegistry({ - drivers: [CodexDriver], - configMap, - }).pipe(Effect.provideService(ProviderSecretResolver, lockedStore(unlocked))); + it.effect("does not read the secrets of an instance its config decodes as disabled", () => { + const fixture = setup({ [secret.reference]: "sk-live" }, DisabledByDefaultConfig); + return Effect.gen(function* () { + const { mutator } = yield* fixture.makeGated; + yield* mutator.reloadSecrets(); - expect(yield* registry.listUnavailable).toHaveLength(1); - yield* Ref.set(unlocked, true); - const rebuilt = yield* registry.rebuildInstanceWhen(instanceId, () => true); + expect(fixture.reads).toEqual([]); + expect(fixture.invalidated).toEqual([]); + expect(fixture.created).toHaveLength(1); + }).pipe(Effect.scoped); + }); - expect(rebuilt).toBe(true); - expect(yield* registry.getInstance(instanceId)).toBeDefined(); - expect(yield* registry.listUnavailable).toHaveLength(0); - }).pipe(Effect.provide(layerTest)), - ); + it.effect("re-reads secrets on reload and rebuilds only when they changed", () => { + const fixture = setup({}); + return Effect.gen(function* () { + const { registry, mutator } = yield* fixture.make; + + fixture.vault.set(secret.reference, "sk-live"); + yield* mutator.reloadSecrets(); + expect(fixture.invalidated).toEqual([[secret.reference]]); + expect(yield* registry.getInstance(instanceId)).toBeDefined(); + expect(yield* registry.listUnavailable).toEqual([]); + + yield* mutator.reloadSecrets(instanceId); + expect(fixture.created).toHaveLength(1); + + fixture.vault.set(secret.reference, "sk-rotated"); + yield* mutator.reloadSecrets(instanceId); + expect(fixture.created.map((environment) => environment[0]?.value)).toEqual([ + "sk-live", + "sk-rotated", + ]); + }).pipe(Effect.scoped); + }); - it.live("is not read again by a settings change that leaves it alone", () => - Effect.gen(function* () { - const unlocked = yield* Ref.make(false); - const reads = yield* Ref.make(0); - const { registry, mutator } = yield* makeProviderInstanceRegistry({ - drivers: [CodexDriver], - configMap, - }).pipe(Effect.provideService(ProviderSecretResolver, lockedStore(unlocked, reads))); + it.effect("keeps a working instance live when a reload cannot read its secret", () => { + const fixture = setup({ [secret.reference]: "sk-live" }); + return Effect.gen(function* () { + const { registry, mutator } = yield* fixture.make; - yield* mutator.reconcile({ ...configMap }); + fixture.vault.delete(secret.reference); + yield* mutator.reloadSecrets(); - expect(yield* Ref.get(reads)).toBe(1); - expect(yield* registry.listUnavailable).toHaveLength(1); - }).pipe(Effect.provide(layerTest)), - ); + expect(yield* registry.getInstance(instanceId)).toBeDefined(); + expect(yield* registry.listUnavailable).toEqual([]); + expect(fixture.created).toHaveLength(1); + }).pipe(Effect.scoped); + }); + + it.effect("starts without waiting for secrets to be read", () => { + const fixture = setup({ [secret.reference]: "sk-live" }); + return Effect.gen(function* () { + const { registry } = yield* fixture.makeGated; + const changes = yield* registry.subscribeChanges; + + expect(yield* registry.getInstance(instanceId)).toBeUndefined(); + const [pending] = yield* registry.listUnavailable; + expect(pending?.unavailableReason).toBe("Reading secrets…"); + + yield* Deferred.succeed(fixture.gate, undefined); + yield* PubSub.take(changes); + expect(yield* registry.getInstance(instanceId)).toBeDefined(); + expect(yield* registry.listUnavailable).toEqual([]); + }).pipe(Effect.scoped); }); }); diff --git a/apps/server/src/provider/ProviderInstanceRegistry.ts b/apps/server/src/provider/ProviderInstanceRegistry.ts index 31e301913baaf..70352c5ef8593 100644 --- a/apps/server/src/provider/ProviderInstanceRegistry.ts +++ b/apps/server/src/provider/ProviderInstanceRegistry.ts @@ -5,14 +5,16 @@ import { type ProviderInstanceConfigMap, type ProviderInstanceEnvironment, type ProviderDriverKind, + type ResolvedProviderInstanceEnvironment, + type SecretReference, type ServerProvider, } from "@t3tools/contracts"; -import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Equal from "effect/Equal"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; @@ -20,9 +22,10 @@ import * as Scope from "effect/Scope"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; + import { buildUnavailableProviderSnapshot } from "./unavailableProviderSnapshot.ts"; import * as ProviderInstanceRegistryMutator from "./ProviderInstanceRegistryMutator.ts"; -import * as ProviderSecretResolver from "./ProviderSecretResolver.ts"; import type { AnyProviderDriver, ProviderInstance } from "@t3tools/provider-core/server/driver"; export class ProviderInstanceRegistry extends Context.Service< @@ -33,19 +36,6 @@ export class ProviderInstanceRegistry extends Context.Service< ) => Effect.Effect; readonly listInstances: Effect.Effect>; readonly listUnavailable: Effect.Effect>; - /** Every instance `rebuildInstanceWhen` can act on, so secrets resolve once per batch of rebuilds. */ - readonly listEnvironments: Effect.Effect< - ReadonlyMap - >; - /** - * Rebuilds an instance from its current config when `shouldRebuild` accepts it, for create-time - * inputs `reconcile` cannot see (an external secret). Unknown ids are a no-op; running sessions - * keep the old instance. Resolves to whether a rebuild happened. - */ - readonly rebuildInstanceWhen: ( - instanceId: ProviderInstanceId, - shouldRebuild: (entry: ProviderInstanceConfig) => boolean, - ) => Effect.Effect; /** streamChanges subscribes on consumption; use subscribeChanges before forking to avoid missing the next publish. */ readonly streamChanges: Stream.Stream; readonly subscribeChanges: Effect.Effect, never, Scope.Scope>; @@ -56,50 +46,51 @@ interface LiveEntry { readonly instance: ProviderInstance; readonly scope: Scope.Closeable; readonly entry: ProviderInstanceConfig; -} - -function withoutKey( - map: ReadonlyMap, - instanceId: ProviderInstanceId, -): ReadonlyMap { - if (!map.has(instanceId)) { - return map; - } - const next = new Map(map); - next.delete(instanceId); - return next; -} - -/** - * Reinstate a rebuilt instance at its old position: map order is settings-author order, so - * appending would shuffle the UI on every refresh. An instance absent from `previousEntries` - * goes last until the next settings change. - */ -function withInstanceAt( - previousEntries: ReadonlyMap, - instanceId: ProviderInstanceId, - rebuilt: LiveEntry, -): ReadonlyMap { - const next = new Map(); - for (const [id, existing] of previousEntries) { - next.set(id, id === instanceId ? rebuilt : existing); - } - next.set(instanceId, rebuilt); - return next; + readonly environment: ResolvedProviderInstanceEnvironment; } interface RegistryState { readonly entries: Ref.Ref>; readonly unavailable: Ref.Ref>; - // Configs parked on an unreadable secret. Settings saves leave them parked - // so a locked vault is not read, or prompted for, on every unrelated save. - readonly secretsUnreadable: Ref.Ref>; readonly changes: PubSub.PubSub; } const entryEqual = (a: ProviderInstanceConfig, b: ProviderInstanceConfig): boolean => Equal.equals(a, b); +const secretReferencesIn = (environment: ProviderInstanceEnvironment): Array => + environment.flatMap((variable) => (typeof variable.value === "string" ? [] : [variable.value])); + +const secretReferencesOf = (entry: ProviderInstanceConfig): ReadonlyArray => + secretReferencesIn(entry.environment ?? []); + +const withoutSecretReferences = ( + environment: ProviderInstanceEnvironment, +): ResolvedProviderInstanceEnvironment => + environment.flatMap((variable) => + typeof variable.value === "string" ? [{ ...variable, value: variable.value }] : [], + ); + +const readSecretEnvironment = Effect.fn("ProviderInstanceRegistry.readSecretEnvironment")( + function* (environment: ProviderInstanceEnvironment) { + const references = secretReferencesIn(environment); + const values = + references.length === 0 ? [] : yield* (yield* SecretResolver.SecretResolver).read(references); + const unreadable: Array = []; + let index = 0; + const resolved: ResolvedProviderInstanceEnvironment = environment.flatMap((variable) => { + if (typeof variable.value === "string") return [{ ...variable, value: variable.value }]; + const secret = values[index++]; + if (secret === undefined) { + unreadable.push(variable.name); + return []; + } + return [{ ...variable, value: secret, sensitive: true }]; + }); + return { environment: resolved, unreadable }; + }, +); + /** Conflicting saved flags must preserve any explicit disable. */ const resolveEntryEnabled = (entry: ProviderInstanceConfig, typedConfig: unknown): boolean => { const rawConfigEnabled = providerInstanceConfigEnabledFlag(entry.config); @@ -109,33 +100,32 @@ const resolveEntryEnabled = (entry: ProviderInstanceConfig, typedConfig: unknown return entry.enabled ?? providerInstanceConfigEnabledFlag(typedConfig) ?? true; }; +// The enabled state `buildEntry` decodes, so a disabled instance never reads its secrets. +const isEntryEnabled = Effect.fn("ProviderInstanceRegistry.isEntryEnabled")(function* ( + driversById: ReadonlyMap>, + entry: ProviderInstanceConfig, +) { + const driver = driversById.get(entry.driver); + if (!driver) return false; + const decoder = Schema.decodeUnknownEffect(driver.configSchema); + const decoded = yield* decoder(entry.config ?? driver.defaultConfig()).pipe(Effect.option); + return Option.isSome(decoded) && resolveEntryEnabled(entry, decoded.value); +}); + const buildEntry = Effect.fn("ProviderInstanceRegistry.buildEntry")(function* (input: { readonly driversById: ReadonlyMap>; readonly parentScope: Scope.Scope; - readonly secretResolver: ProviderSecretResolver.ProviderSecretResolverShape; - readonly hostEnvironment: NodeJS.ProcessEnv; readonly instanceId: ProviderInstanceId; readonly rawInstanceId: string; readonly entry: ProviderInstanceConfig; + readonly deferSecrets: boolean; }): Effect.fn.Return< | { readonly kind: "live"; readonly live: LiveEntry } - | { - readonly kind: "unavailable"; - readonly snapshot: ServerProvider; - readonly secretsUnreadable?: true; - }, + | { readonly kind: "unavailable"; readonly snapshot: ServerProvider }, never, - R + R | SecretResolver.SecretResolver > { - const { - driversById, - parentScope, - secretResolver, - hostEnvironment, - instanceId, - rawInstanceId, - entry, - } = input; + const { driversById, parentScope, instanceId, rawInstanceId, entry, deferSecrets } = input; const driver = driversById.get(entry.driver); if (!driver) { return { @@ -173,20 +163,37 @@ const buildEntry = Effect.fn("ProviderInstanceRegistry.buildEntry")(function* 0) { + const enabled = resolveEntryEnabled(entry, typedConfig); + if (deferSecrets && enabled && secretReferencesOf(entry).length > 0) { + return { + kind: "unavailable" as const, + snapshot: yield* buildUnavailableProviderSnapshot({ + driverKind: entry.driver, + instanceId, + displayName: entry.displayName, + accentColor: entry.accentColor, + reason: "Reading secrets…", + }), + }; + } + const { environment, unreadable } = enabled + ? yield* readSecretEnvironment(entry.environment ?? []) + : { environment: withoutSecretReferences(entry.environment ?? []), unreadable: [] }; + // Starting with a blank credential would fail later with a less useful error, or fall back to + // whatever the server inherited. + if (unreadable.length > 0) { return { kind: "unavailable" as const, - secretsUnreadable: true as const, snapshot: yield* buildUnavailableProviderSnapshot({ driverKind: entry.driver, instanceId, displayName: entry.displayName, accentColor: entry.accentColor, - reason: `Could not read ${resolvedEnvironment.unresolved.join(", ")} from its secret store. Unlock it, then refresh.`, + reason: `Could not read ${unreadable.join(", ")} from its secret provider. Unlock it, then refresh.`, }), }; } + const childScope = yield* Scope.make(); yield* Scope.addFinalizer(parentScope, Scope.close(childScope, Exit.void).pipe(Effect.ignore)); @@ -195,15 +202,11 @@ const buildEntry = Effect.fn("ProviderInstanceRegistry.buildEntry")(function* (input: { readonly state: RegistryState; readonly driversById: ReadonlyMap>; readonly parentScope: Scope.Scope; - readonly secretResolver: ProviderSecretResolver.ProviderSecretResolverShape; - readonly hostEnvironment: NodeJS.ProcessEnv; -}): ((configMap: ProviderInstanceConfigMap) => Effect.Effect) => { - const { state, driversById, parentScope, secretResolver, hostEnvironment } = input; - return (configMap: ProviderInstanceConfigMap) => +}): (( + configMap: ProviderInstanceConfigMap, + rebuild?: ReadonlySet, + deferSecrets?: boolean, +) => Effect.Effect) => { + const { state, driversById, parentScope } = input; + return (configMap, rebuild = new Set(), deferSecrets = false) => Effect.gen(function* () { const previousEntries = yield* Ref.get(state.entries); const previousUnavailable = yield* Ref.get(state.unavailable); - const previousSecretsUnreadable = yield* Ref.get(state.secretsUnreadable); const nextRaw = Object.entries(configMap); const nextKeys = new Set( nextRaw.map(([raw]) => ProviderInstanceId.make(raw)), @@ -260,7 +265,10 @@ const makeReconcile = (input: { continue; } const nextEntry = configMap[instanceId]; - if (nextEntry !== undefined && !entryEqual(live.entry, nextEntry)) { + if ( + rebuild.has(instanceId) || + (nextEntry !== undefined && !entryEqual(live.entry, nextEntry)) + ) { replacedIds.add(instanceId); } } @@ -271,9 +279,22 @@ const makeReconcile = (input: { } } + // Read every secret the builds below need in one go, so a vault unlocks once. + const references = yield* Effect.forEach(nextRaw, ([rawInstanceId, entry]) => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make(rawInstanceId); + const kept = previousEntries.has(instanceId) && !replacedIds.has(instanceId); + return kept || !(yield* isEntryEnabled(driversById, entry)) + ? [] + : secretReferencesOf(entry); + }), + ).pipe(Effect.map((perEntry) => perEntry.flat())); + if (references.length > 0 && !deferSecrets) { + yield* (yield* SecretResolver.SecretResolver).read(references); + } + const builtEntries = new Map(); const builtUnavailable = new Map(); - const builtSecretsUnreadable = new Map(); let orderChanged = false; const previousOrder = [...previousEntries.keys()]; const nextOrder: Array = []; @@ -287,34 +308,19 @@ const makeReconcile = (input: { builtEntries.set(instanceId, existing); continue; } - const parkedEntry = previousSecretsUnreadable.get(instanceId); - const parkedSnapshot = previousUnavailable.get(instanceId); - if ( - parkedEntry !== undefined && - parkedSnapshot !== undefined && - entryEqual(parkedEntry, entry) - ) { - builtUnavailable.set(instanceId, parkedSnapshot); - builtSecretsUnreadable.set(instanceId, entry); - continue; - } const result = yield* buildEntry({ driversById, parentScope, - secretResolver, - hostEnvironment, instanceId, rawInstanceId, entry, + deferSecrets, }); if (result.kind === "live") { builtEntries.set(instanceId, result.live); } else { builtUnavailable.set(instanceId, result.snapshot); - if (result.secretsUnreadable) { - builtSecretsUnreadable.set(instanceId, entry); - } } } @@ -344,7 +350,6 @@ const makeReconcile = (input: { yield* Ref.set(state.entries, builtEntries); yield* Ref.set(state.unavailable, builtUnavailable); - yield* Ref.set(state.secretsUnreadable, builtSecretsUnreadable); if (entriesChanged || unavailableChanged) { yield* PubSub.publish(state.changes, undefined); @@ -363,7 +368,7 @@ export const makeProviderInstanceRegistry = Effect.fn( readonly mutator: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape; }, never, - R | Scope.Scope + R | SecretResolver.SecretResolver | Scope.Scope > { const driversById = new Map>( input.drivers.map((driver) => [driver.driverKind, driver]), @@ -375,137 +380,86 @@ export const makeProviderInstanceRegistry = Effect.fn( // scopes to the *caller's* scope instead of the registry's. const parentScope = yield* Scope.Scope; - const driverContext = yield* Effect.context(); - const secretResolver = yield* ProviderSecretResolver.ProviderSecretResolver; - const hostEnvironment = yield* HostProcess.Environment; + const driverContext = yield* Effect.context(); const entries = yield* Ref.make>(new Map()); const unavailable = yield* Ref.make>(new Map()); - const secretsUnreadable = yield* Ref.make< - ReadonlyMap - >(new Map()); const changes = yield* PubSub.unbounded(); yield* Effect.addFinalizer(() => PubSub.shutdown(changes)); - // Config envelopes for instances a rebuild could not bring back. Without - // this a failed rebuild is a one-way door: the instance leaves `entries`, - // so the next refresh finds nothing live to rebuild and only a settings - // change can restore it. - const rebuildable = yield* Ref.make>( - new Map(), - ); - // Both mutators read the map, do slow work, then write it back. A secret - // read can park on a person at a biometric prompt, which is more than long - // enough for a settings change to land in the middle, so they take turns. - // Reads stay outside: `getInstance` and `listInstances` never wait. - const mutations = yield* Semaphore.make(1); - - const state: RegistryState = { entries, unavailable, secretsUnreadable, changes }; - const reconcileWithR = makeReconcile({ - state, - driversById, - parentScope, - secretResolver, - hostEnvironment, - }); + const state: RegistryState = { entries, unavailable, changes }; + const reconcileWithR = makeReconcile({ state, driversById, parentScope }); + const configMapRef = yield* Ref.make(input.configMap); + const semaphore = yield* Semaphore.make(1); const reconcile: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape["reconcile"] = (configMap) => - mutations.withPermits(1)( - reconcileWithR(configMap).pipe( - Effect.tap(() => - Ref.get(unavailable).pipe( - Effect.flatMap((pending) => - Ref.set( - rebuildable, - new Map( - Object.entries(configMap) - .map(([id, entry]) => [ProviderInstanceId.make(id), entry] as const) - .filter(([id]) => pending.has(id)), - ), - ), - ), - ), - ), + semaphore.withPermits(1)( + Ref.set(configMapRef, configMap).pipe( + Effect.andThen(reconcileWithR(configMap)), Effect.provideContext(driverContext), ), ); - yield* reconcile(input.configMap); - - // Rebuild one instance from the config it already has. Unlike - // `reconcile`, which is driven by a settings diff, this is for inputs the - // envelope cannot show, see the shape docs. Order is preserved by - // rewriting the map in place rather than appending the replacement. - const rebuildInstanceWhen: ProviderInstanceRegistry["Service"]["rebuildInstanceWhen"] = ( - instanceId, - shouldRebuild, - ) => - mutations.withPermits(1)( - Effect.gen(function* () { - const previousEntries = yield* Ref.get(entries); - const live = previousEntries.get(instanceId); - // An instance a previous rebuild could not restore has no live entry - // to read the config from, so its envelope comes from `rebuildable`. - const entry = live?.entry ?? (yield* Ref.get(rebuildable)).get(instanceId); - if (entry === undefined || !shouldRebuild(entry)) { - return false; - } - - // Only the build is interruptible (it can wait minutes on a secret store); the bookkeeping is - // uninterruptible so the instance is never half-moved. Remembering the envelope comes first so - // an interrupt leaves it retryable on the next refresh. - yield* Effect.uninterruptible( - Effect.gen(function* () { - yield* Ref.update(rebuildable, (previous) => new Map(previous).set(instanceId, entry)); - if (live === undefined) { - return; - } - // Drop the instance before closing it so callers never get a bundle with a dead scope. Its last - // snapshot stands in meanwhile: an id in neither list is pruned, blanking the provider card. - const parked = yield* live.instance.snapshot.getSnapshot; - yield* Ref.set(entries, withoutKey(previousEntries, instanceId)); - yield* Ref.update(unavailable, (previous) => new Map(previous).set(instanceId, parked)); - yield* Scope.close(live.scope, Exit.void).pipe(Effect.ignore); - }), + // Rebuilds an instance only when it was unavailable or its secrets changed, so a refresh does + // not restart providers whose credentials are the same. + const reloadSecrets: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape["reloadSecrets"] = + Effect.fn("ProviderInstanceRegistry.reloadSecrets")( + function* (instanceId) { + const configMap = yield* Ref.get(configMapRef); + const targets = yield* Effect.filter(Object.entries(configMap), ([rawInstanceId, entry]) => + (instanceId === undefined || rawInstanceId === instanceId) && + secretReferencesOf(entry).length > 0 + ? isEntryEnabled(driversById, entry) + : Effect.succeed(false), ); - - const result = yield* buildEntry({ - driversById, - parentScope, - secretResolver, - hostEnvironment, - instanceId, - rawInstanceId: instanceId, - entry, - }); - - // Uninterruptible as well: the replacement is already running by - // now, and an interrupt that dropped it on the floor would leave a - // provider process nothing can reach or stop. - yield* Effect.uninterruptible( - Effect.gen(function* () { - if (result.kind === "live") { - yield* Ref.set(entries, withInstanceAt(previousEntries, instanceId, result.live)); - yield* Ref.update(unavailable, (previous) => withoutKey(previous, instanceId)); - yield* Ref.update(rebuildable, (previous) => withoutKey(previous, instanceId)); - yield* Ref.update(secretsUnreadable, (previous) => withoutKey(previous, instanceId)); - } else { - yield* Ref.update(unavailable, (previous) => - new Map(previous).set(instanceId, result.snapshot), - ); - yield* Ref.update(secretsUnreadable, (previous) => - result.secretsUnreadable - ? new Map(previous).set(instanceId, entry) - : withoutKey(previous, instanceId), - ); - } - yield* PubSub.publish(changes, undefined); - }), - ); - return true; - }).pipe(Effect.provideContext(driverContext)), + if (targets.length === 0) return; + const resolver = yield* SecretResolver.SecretResolver; + const references = targets.flatMap(([, entry]) => secretReferencesOf(entry)); + yield* resolver.invalidate(references); + yield* resolver.read(references); + const live = yield* Ref.get(entries); + const rebuild = new Set(); + for (const [rawInstanceId, entry] of targets) { + const current = live.get(ProviderInstanceId.make(rawInstanceId)); + if (current === undefined) continue; + const { environment, unreadable } = yield* readSecretEnvironment(entry.environment ?? []); + // A failed re-read keeps the working instance and the value it already holds. + if (unreadable.length === 0 && !Equal.equals(environment, current.environment)) { + rebuild.add(current.instance.instanceId); + } + } + yield* reconcileWithR(configMap, rebuild); + }, + (effect) => semaphore.withPermits(1)(effect), + Effect.provideContext(driverContext), ); + // Reading secrets can wait on a vault unlock, so the server does not wait for it to start. + yield* semaphore.withPermits(1)( + Ref.set(configMapRef, input.configMap).pipe( + Effect.andThen(reconcileWithR(input.configMap, new Set(), true)), + Effect.provideContext(driverContext), + ), + ); + const startupReferences = yield* Effect.filter(Object.entries(input.configMap), ([, entry]) => + secretReferencesOf(entry).length > 0 + ? isEntryEnabled(driversById, entry) + : Effect.succeed(false), + ).pipe(Effect.map((targets) => targets.flatMap(([, entry]) => secretReferencesOf(entry)))); + if (startupReferences.length > 0) { + // The read waits outside the lock so other mutations are not held for a vault prompt; a + // configuration saved meanwhile has already reconciled itself. + yield* Effect.gen(function* () { + yield* (yield* SecretResolver.SecretResolver).read(startupReferences); + yield* semaphore.withPermits(1)( + Effect.gen(function* () { + const current = yield* Ref.get(configMapRef); + if (current === input.configMap) yield* reconcileWithR(current); + }), + ); + }).pipe(Effect.provideContext(driverContext), Effect.forkScoped); + } + const registry: ProviderInstanceRegistry["Service"] = { getInstance: (id) => Ref.get(entries).pipe(Effect.map((map) => map.get(id)?.instance)), listInstances: Ref.get(entries).pipe( @@ -517,19 +471,6 @@ export const makeProviderInstanceRegistry = Effect.fn( listUnavailable: Ref.get(unavailable).pipe( Effect.map((map) => Array.from(map.values()) as ReadonlyArray), ), - listEnvironments: Effect.gen(function* () { - const environments = new Map(); - for (const [instanceId, entry] of yield* Ref.get(rebuildable)) { - environments.set(instanceId, entry.environment); - } - // Live entries are written second so an instance that is both live and - // pending a retry reports the configuration it is actually running. - for (const [instanceId, live] of yield* Ref.get(entries)) { - environments.set(instanceId, live.entry.environment); - } - return environments; - }), - rebuildInstanceWhen, get streamChanges() { return Stream.fromPubSub(changes); }, @@ -540,6 +481,7 @@ export const makeProviderInstanceRegistry = Effect.fn( const mutator: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape = { reconcile, + reloadSecrets, }; return { registry, mutator }; @@ -551,7 +493,7 @@ export const layer = (input: { }): Layer.Layer< ProviderInstanceRegistry | ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator, never, - R + R | SecretResolver.SecretResolver > => Layer.effectContext( makeProviderInstanceRegistry(input).pipe( @@ -564,5 +506,5 @@ export const layer = (input: { ) as Layer.Layer< ProviderInstanceRegistry | ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator, never, - R + R | SecretResolver.SecretResolver >; diff --git a/apps/server/src/provider/ProviderInstanceRegistryHydration.test.ts b/apps/server/src/provider/ProviderInstanceRegistryHydration.test.ts index e0de79163998d..c497e00ac17e9 100644 --- a/apps/server/src/provider/ProviderInstanceRegistryHydration.test.ts +++ b/apps/server/src/provider/ProviderInstanceRegistryHydration.test.ts @@ -3,8 +3,6 @@ import { ProjectId, ProviderDriverKind, ProviderInstanceId, - OnePasswordAccount, - SecretProviderId, type ServerSettings, } from "@t3tools/contracts"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -17,7 +15,6 @@ import * as ServerConfig from "../config.ts"; import { ServerSettingsService } from "../serverSettings.ts"; import { DEV_REPLAY_INSTANCE_ID, - changedSecretProviderIds, deriveProviderInstanceConfigMap, seedDevReplayInstance, } from "./ProviderInstanceRegistryHydration.ts"; @@ -96,44 +93,6 @@ describe("deriveProviderInstanceConfigMap", () => { }); }); -describe("changedSecretProviderIds", () => { - const home = SecretProviderId.make("home"); - const vault = SecretProviderId.make("vault"); - const homeConfig = { - kind: "1password" as const, - label: "Home", - account: OnePasswordAccount.make("my.1password.com"), - }; - const vaultConfig = { - kind: "openbao" as const, - label: "Vault", - address: "https://bao.example.com", - mount: "secret", - token: "s.old", - auth: "token" as const, - }; - - it("is empty when no provider changed", () => { - const providers = { [home]: homeConfig, [vault]: vaultConfig }; - expect(changedSecretProviderIds(providers, { ...providers })).toEqual(new Set()); - }); - - it("names providers that were added, edited, or removed", () => { - expect( - changedSecretProviderIds( - { [home]: homeConfig }, - { [vault]: { ...vaultConfig, token: "s.new" } }, - ), - ).toEqual(new Set([home, vault])); - expect( - changedSecretProviderIds( - { [vault]: vaultConfig }, - { [vault]: { ...vaultConfig, token: "s.new" } }, - ), - ).toEqual(new Set([vault])); - }); -}); - describe("seedDevReplayInstance", () => { const seed = (devUrl: URL | undefined) => Effect.gen(function* () { diff --git a/apps/server/src/provider/ProviderInstanceRegistryHydration.ts b/apps/server/src/provider/ProviderInstanceRegistryHydration.ts index 82dd04ac96237..62a82a2f1269a 100644 --- a/apps/server/src/provider/ProviderInstanceRegistryHydration.ts +++ b/apps/server/src/provider/ProviderInstanceRegistryHydration.ts @@ -6,26 +6,22 @@ import { resolveProviderInstanceEnabled, type ProviderInstanceConfig, type ProviderInstanceConfigMap, - type SecretProviderId, ServerSettings, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import type * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; -import * as Ref from "effect/Ref"; import * as Path from "effect/Path"; import * as Stream from "effect/Stream"; import * as Settings from "../serverSettings.ts"; +import * as SecretResolverCore from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as SecretResolver from "../secrets/SecretResolver.ts"; import { BUILT_IN_DRIVERS, type BuiltInDriversEnv } from "./builtInDrivers.ts"; -import { - collectProviderSecretReferences, - providerSecretReference, -} from "./ProviderSecretReference.ts"; import * as ProviderInstanceRegistry from "./ProviderInstanceRegistry.ts"; import * as ProviderInstanceRegistryMutator from "./ProviderInstanceRegistryMutator.ts"; -import * as ProviderSecretResolver from "./ProviderSecretResolver.ts"; import * as ProviderOrchestrationAdapterInfrastructure from "./ProviderOrchestrationAdapterInfrastructure.ts"; import * as AcpRegistrySupport from "@t3tools/provider-acp-registry/server/AcpRegistrySupport"; import * as ProviderHostLive from "./ProviderHostLive.ts"; @@ -46,6 +42,8 @@ type ProviderInstanceRegistryHydrationEnv = | ServerConfig.ServerConfig | Path.Path | FileSystem.FileSystem + // Secret provider reads. + | Crypto.Crypto // The Claude runner counts its CLIs as running agents for port discovery. | PortScanner.PortDiscovery; @@ -124,85 +122,18 @@ export const seedDevReplayInstance = Effect.fn( }); }); -/** - * Reads every secret the config map's instances need before any is built, since the secret store - * charges an unlock per read rather than per secret. - */ -const primeConfigMapSecrets = ( - secretResolver: ProviderSecretResolver.ProviderSecretResolverShape, - configMap: ProviderInstanceConfigMap, -) => - secretResolver.prime( - collectProviderSecretReferences(Object.values(configMap).map((entry) => entry.environment)), - ); - -/** Ids of the secret providers added, removed, or edited between two settings snapshots. */ -export const changedSecretProviderIds = ( - previous: ServerSettings["secretProviders"], - next: ServerSettings["secretProviders"], -): ReadonlySet => { - const changed = new Set(); - for (const id of new Set([...Object.keys(previous), ...Object.keys(next)])) { - const providerId = id as SecretProviderId; - if (JSON.stringify(previous[providerId]) !== JSON.stringify(next[providerId])) { - changed.add(providerId); - } - } - return changed; -}; - -const readsThroughSecretProvider = ( - entry: ProviderInstanceConfig, - providers: ReadonlySet, -) => - entry.environment?.some((variable) => { - const reference = providerSecretReference(variable.value); - return reference?._tag === "secret-provider" && providers.has(reference.provider); - }) ?? false; - -const layerSettingsWatcher = ( - settingsChanges: Stream.Stream, - initialSecretProviders: ServerSettings["secretProviders"], -) => +const layerSettingsWatcher = (settingsChanges: Stream.Stream) => Layer.effectDiscard( Effect.gen(function* () { const mutator = yield* ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator; - const registry = yield* ProviderInstanceRegistry.ProviderInstanceRegistry; const serverSettings = yield* Settings.ServerSettingsService; - const secretResolver = yield* ProviderSecretResolver.ProviderSecretResolver; - const secretProvidersRef = yield* Ref.make(initialSecretProviders); yield* settingsChanges.pipe( Stream.runForEach(() => // The subscription predates the seed read, so a queued change can be // older than the seed. A failed current read must leave the registry intact. serverSettings.getSettings.pipe( - Effect.flatMap((next) => - Effect.gen(function* () { - const configMap = deriveProviderInstanceConfigMap(next); - const changedProviders = changedSecretProviderIds( - yield* Ref.get(secretProvidersRef), - next.secretProviders, - ); - if (changedProviders.size > 0) { - yield* secretResolver.invalidate(changedProviders); - } - yield* primeConfigMapSecrets(secretResolver, configMap); - yield* mutator.reconcile(configMap); - if (changedProviders.size > 0) { - const environments = yield* registry.listEnvironments; - yield* Effect.forEach( - environments.keys(), - (instanceId) => - registry.rebuildInstanceWhen(instanceId, (entry) => - readsThroughSecretProvider(entry, changedProviders), - ), - { discard: true }, - ); - } - // Committed last so a failed reconcile is retried on the next emission - // instead of being treated as already applied. - yield* Ref.set(secretProvidersRef, next.secretProviders); - }), + Effect.flatMap((current) => + mutator.reconcile(deriveProviderInstanceConfigMap(current)), ), Effect.catchCause((cause) => Effect.logError("ProviderInstanceRegistry reconcile failed", cause), @@ -215,13 +146,14 @@ const layerSettingsWatcher = ( ); export const layer: Layer.Layer< - ProviderInstanceRegistry.ProviderInstanceRegistry, + | ProviderInstanceRegistry.ProviderInstanceRegistry + | ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator + | SecretResolverCore.SecretResolver, never, ProviderInstanceRegistryHydrationEnv > = Layer.unwrap( Effect.gen(function* () { const serverSettings = yield* Settings.ServerSettingsService; - const secretResolver = yield* ProviderSecretResolver.ProviderSecretResolver; yield* seedDevReplayInstance().pipe( Effect.catchCause((cause) => Effect.logWarning("Dev replay agent unavailable", cause)), ); @@ -235,10 +167,6 @@ export const layer: Layer.Layer< ? ({} as ProviderInstanceConfigMap) : deriveProviderInstanceConfigMap(initialSettings); - // The watcher only sees later writes, so this snapshot is the whole boot - // fleet and the one place where nothing is cached yet. - yield* primeConfigMapSecrets(secretResolver, initialConfigMap); - const layerMutable = ProviderInstanceRegistry.layer({ drivers: BUILT_IN_DRIVERS, configMap: initialConfigMap, @@ -246,14 +174,15 @@ export const layer: Layer.Layer< Layer.provide(ProviderOrchestrationAdapterInfrastructure.layer), Layer.provide(AcpRegistrySupport.layerFromHost), Layer.provide(ProviderHostLive.layer), + Layer.provideMerge(SecretResolver.layer), ); - return layerSettingsWatcher(settingsChanges, initialSettings?.secretProviders ?? {}).pipe( - Layer.provideMerge(layerMutable), - ); + return layerSettingsWatcher(settingsChanges).pipe(Layer.provideMerge(layerMutable)); }), ) as Layer.Layer< - ProviderInstanceRegistry.ProviderInstanceRegistry, + | ProviderInstanceRegistry.ProviderInstanceRegistry + | ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator + | SecretResolverCore.SecretResolver, never, ProviderInstanceRegistryHydrationEnv >; diff --git a/apps/server/src/provider/ProviderInstanceRegistryMutator.ts b/apps/server/src/provider/ProviderInstanceRegistryMutator.ts index 51322826230d8..35eb52d3f2906 100644 --- a/apps/server/src/provider/ProviderInstanceRegistryMutator.ts +++ b/apps/server/src/provider/ProviderInstanceRegistryMutator.ts @@ -1,10 +1,12 @@ /** Reconcile closes removed/replaced scopes before creating replacements: at most one live instance per id. */ -import type { ProviderInstanceConfigMap } from "@t3tools/contracts"; +import type { ProviderInstanceConfigMap, ProviderInstanceId } from "@t3tools/contracts"; import * as Context from "effect/Context"; import type * as Effect from "effect/Effect"; export interface ProviderInstanceRegistryMutatorShape { readonly reconcile: (configMap: ProviderInstanceConfigMap) => Effect.Effect; + /** Re-reads the secrets of every instance, or one, and rebuilds those whose secrets changed. */ + readonly reloadSecrets: (instanceId?: ProviderInstanceId) => Effect.Effect; } export class ProviderInstanceRegistryMutator extends Context.Service< diff --git a/apps/server/src/provider/ProviderRegistry.test.ts b/apps/server/src/provider/ProviderRegistry.test.ts index c5a8ac9105413..5e356c84516a8 100644 --- a/apps/server/src/provider/ProviderRegistry.test.ts +++ b/apps/server/src/provider/ProviderRegistry.test.ts @@ -25,8 +25,6 @@ import { DEFAULT_SERVER_SETTINGS, PROVIDER_WORKSPACE_SNAPSHOT_TTL_MS, ProviderDriverKind, - ProviderInstanceEnvironment, - ProviderSecretSource, ProviderInstanceId, ServerSettings, ServerSettingsError, @@ -67,9 +65,8 @@ import type { ProviderWorkspaceSnapshot, } from "@t3tools/provider-core/server/driver"; import * as ProviderInstanceRegistry from "./ProviderInstanceRegistry.ts"; -import { ProviderSecretResolver } from "./ProviderSecretResolver.ts"; +import * as ProviderInstanceRegistryMutator from "./ProviderInstanceRegistryMutator.ts"; import * as ProviderRegistry from "./ProviderRegistry.ts"; -import { literalProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import { makeManualOnlyProviderMaintenanceCapabilities } from "@t3tools/provider-core/server/maintenanceResolver"; vi.mock("@anthropic-ai/claude-agent-sdk", { spy: true }); @@ -79,14 +76,6 @@ const encodeServerProvider = Schema.encodeSync(ServerProvider); const decodeServerSettings = Schema.decodeSync(ServerSettings); const encodeServerSettings = Schema.encodeSync(ServerSettings); const encodedDefaultServerSettings = encodeServerSettings(DEFAULT_SERVER_SETTINGS); -const decodeEnvironment = Schema.decodeSync(ProviderInstanceEnvironment); -const decodeSecretSource = Schema.decodeSync(ProviderSecretSource); -const HOME_ACCOUNT = "my.1password.com"; -const onePasswordVariable = (name: string, reference: string, sensitive = true) => ({ - name, - value: { kind: "1password" as const, reference, account: HOME_ACCOUNT }, - sensitive, -}); const disabledDefaultSlots = ( ...drivers: ReadonlyArray @@ -112,6 +101,10 @@ const withBundledCompatibility = (snapshot: ServerProvider) => ModelManifest.BUNDLED_MODEL_MANIFEST.compatibility, ); +const layerUnusedMutator = Layer.mock( + ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator, +)({}); + const layerTestHttpClient = Layer.succeed( HttpClient.HttpClient, HttpClient.make((request) => @@ -119,13 +112,6 @@ const layerTestHttpClient = Layer.succeed( ), ).pipe(Layer.provideMerge(ModelManifest.layerTest)); -function layerClaudeCredentialHttp(respond: () => Response) { - return Layer.succeed( - HttpClient.HttpClient, - HttpClient.make((request) => Effect.succeed(HttpClientResponse.fromWeb(request, respond()))), - ); -} - const layerBackgroundPolicyAlwaysRun = Layer.mock(BackgroundPolicy.BackgroundPolicy)({ reportClientActivity: () => Effect.void, removeRpcClient: () => Effect.void, @@ -935,8 +921,6 @@ it.layer( listUnavailable: Effect.succeed([]), streamChanges: Stream.empty, subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), - listEnvironments: Effect.succeed(new Map()), - rebuildInstanceWhen: () => Effect.succeed(false), }, ); const retainedModels = [defaultModel, customModel, currentModel]; @@ -965,7 +949,12 @@ it.layer( retainedModels, ); }).pipe( - Effect.provide(ProviderRegistry.layer.pipe(Layer.provide(layerInstanceRegistry))), + Effect.provide( + ProviderRegistry.layer.pipe( + Layer.provide(layerInstanceRegistry), + Layer.provide(layerUnusedMutator), + ), + ), Effect.scoped, ); } @@ -1065,8 +1054,6 @@ it.layer( Effect.succeed(id === instance.instanceId ? instance : undefined), listInstances: Effect.succeed([instance]), listUnavailable: Effect.succeed([]), - listEnvironments: Effect.succeed(new Map()), - rebuildInstanceWhen: () => Effect.succeed(false), streamChanges: Stream.empty, subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), }, @@ -1098,7 +1085,12 @@ it.layer( retainedModels, ); }).pipe( - Effect.provide(ProviderRegistry.layer.pipe(Layer.provide(layerInstanceRegistry))), + Effect.provide( + ProviderRegistry.layer.pipe( + Layer.provide(layerInstanceRegistry), + Layer.provide(layerUnusedMutator), + ), + ), Effect.scoped, ); } @@ -1305,394 +1297,6 @@ it.layer( ); }); - // Refresh must drop cached secrets AND rebuild instances that read one: the environment is - // resolved once at driver build, so a re-probe would report the old credential. - it.effect("drops cached secrets and rebuilds the instances that read them", () => - Effect.gen(function* () { - const codexDriver = ProviderDriverKind.make("codex"); - const codexInstanceId = ProviderInstanceId.make("codex"); - const initialProvider = { - instanceId: codexInstanceId, - driver: codexDriver, - status: "warning", - enabled: true, - installed: false, - auth: { status: "unknown" }, - checkedAt: "2026-06-10T00:00:00.000Z", - version: null, - models: [], - slashCommands: [], - skills: [], - } as const satisfies ServerProvider; - const instance = { - instanceId: codexInstanceId, - driverKind: codexDriver, - continuationIdentity: { - driverKind: codexDriver, - continuationKey: "codex:instance:codex", - }, - displayName: undefined, - enabled: true, - snapshot: { - resolveMaintenance: () => - Effect.succeed( - makeManualOnlyProviderMaintenanceCapabilities({ - provider: codexDriver, - packageName: null, - }), - ), - getSnapshot: Effect.succeed(initialProvider), - refresh: Effect.succeed(initialProvider), - streamChanges: Stream.empty, - applyUsageLimits: () => Effect.void, - }, - orchestrationAdapter: {} as ProviderInstance["orchestrationAdapter"], - textGeneration: {} as ProviderInstance["textGeneration"], - } satisfies ProviderInstance; - - const invalidations = yield* Ref.make(0); - const rebuiltIds = yield* Ref.make>([]); - const secretResolverLayer = Layer.succeed(ProviderSecretResolver, { - resolve: (environment) => - Effect.succeed({ - variables: literalProviderInstanceEnvironment(environment), - unresolved: [], - }), - prime: () => Effect.void, - invalidate: () => Ref.update(invalidations, (count) => count + 1), - check: () => Effect.succeed({ _tag: "Resolved" }), - listOnePasswordAccounts: Effect.succeed([]), - }); - const instanceRegistryLayer = Layer.succeed( - ProviderInstanceRegistry.ProviderInstanceRegistry, - { - getInstance: (instanceId) => - Effect.succeed(instanceId === codexInstanceId ? instance : undefined), - listInstances: Effect.succeed([instance]), - listUnavailable: Effect.succeed([]), - rebuildInstanceWhen: (instanceId, shouldRebuild) => - shouldRebuild({ - driver: codexDriver, - environment: decodeEnvironment([ - onePasswordVariable("CODEX_TOKEN", "op://Vault/Item/token"), - ]), - }) - ? Ref.update(rebuiltIds, (previous) => [...previous, instanceId]).pipe( - Effect.as(true), - ) - : Effect.succeed(false), - streamChanges: Stream.empty, - listEnvironments: Effect.succeed(new Map()), - subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), - }, - ); - - const scope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); - const runtimeServices = yield* Layer.build( - ProviderRegistry.layer.pipe( - Layer.provideMerge(instanceRegistryLayer), - Layer.provideMerge( - ServerConfig.layerTest(process.cwd(), { - prefix: "t3-provider-registry-secret-refresh-", - }), - ), - Layer.provideMerge(NodeServices.layer), - Layer.provideMerge(secretResolverLayer), - ), - ).pipe(Scope.provide(scope)); - - yield* Effect.gen(function* () { - const registry = yield* ProviderRegistry.ProviderRegistry; - yield* registry.refreshInstance(codexInstanceId); - - assert.strictEqual(yield* Ref.get(invalidations), 1); - assert.deepStrictEqual(yield* Ref.get(rebuiltIds), [codexInstanceId]); - }).pipe(Effect.provide(runtimeServices)); - }), - ); - - // A rebuild that could not finish - a locked vault, a secret store that - // was not running - leaves the instance out of the live set. Refresh is - // the user saying "try again", so it has to reach the instances that - // need trying again, not just the ones that are already working. - it.effect("retries instances that are not live when nothing is targeted", () => - Effect.gen(function* () { - const codexDriver = ProviderDriverKind.make("codex"); - const codexInstanceId = ProviderInstanceId.make("codex"); - const unavailableProvider = { - instanceId: codexInstanceId, - driver: codexDriver, - status: "error", - enabled: true, - installed: false, - auth: { status: "unknown" }, - checkedAt: "2026-06-10T00:00:00.000Z", - version: null, - models: [], - slashCommands: [], - skills: [], - message: "Driver 'codex' failed to create instance: secret store is locked", - } as const satisfies ServerProvider; - - const rebuiltIds = yield* Ref.make>([]); - const secretResolverLayer = Layer.succeed(ProviderSecretResolver, { - resolve: (environment) => - Effect.succeed({ - variables: literalProviderInstanceEnvironment(environment), - unresolved: [], - }), - prime: () => Effect.void, - invalidate: () => Effect.void, - check: () => Effect.succeed({ _tag: "Resolved" }), - listOnePasswordAccounts: Effect.succeed([]), - }); - const instanceRegistryLayer = Layer.succeed( - ProviderInstanceRegistry.ProviderInstanceRegistry, - { - // @effect-diagnostics-next-line effectSucceedWithVoid:off - This optional result requires undefined, not the wider void type. - getInstance: () => Effect.succeed(undefined), - listInstances: Effect.succeed([]), - listUnavailable: Effect.succeed([unavailableProvider]), - rebuildInstanceWhen: (instanceId, shouldRebuild) => - shouldRebuild({ - driver: codexDriver, - environment: decodeEnvironment([ - onePasswordVariable("CODEX_TOKEN", "op://Vault/Item/token"), - ]), - }) - ? Ref.update(rebuiltIds, (previous) => [...previous, instanceId]).pipe( - Effect.as(true), - ) - : Effect.succeed(false), - streamChanges: Stream.empty, - listEnvironments: Effect.succeed(new Map()), - subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), - }, - ); - - const scope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); - const runtimeServices = yield* Layer.build( - ProviderRegistry.layer.pipe( - Layer.provideMerge(instanceRegistryLayer), - Layer.provideMerge( - ServerConfig.layerTest(process.cwd(), { - prefix: "t3-provider-registry-secret-retry-", - }), - ), - Layer.provideMerge(NodeServices.layer), - Layer.provideMerge(secretResolverLayer), - ), - ).pipe(Scope.provide(scope)); - - yield* Effect.gen(function* () { - const registry = yield* ProviderRegistry.ProviderRegistry; - yield* registry.refresh(); - - assert.deepStrictEqual(yield* Ref.get(rebuiltIds), [codexInstanceId]); - }).pipe(Effect.provide(runtimeServices)); - }), - ); - - it.effect("rebuilds every instance that reads a secret a check succeeds on", () => - Effect.gen(function* () { - const codexDriver = ProviderDriverKind.make("codex"); - const readableId = ProviderInstanceId.make("codex"); - const otherId = ProviderInstanceId.make("codex_other"); - const liveId = ProviderInstanceId.make("codex_live"); - const reference = "op://Vault/Item/token"; - const unavailableProvider = (instanceId: ProviderInstanceId) => - ({ - instanceId, - driver: codexDriver, - status: "error", - enabled: true, - installed: false, - auth: { status: "unknown" }, - checkedAt: "2026-06-10T00:00:00.000Z", - version: null, - models: [], - slashCommands: [], - skills: [], - message: "Could not read CODEX_TOKEN from its secret store. Unlock it, then refresh.", - }) as const satisfies ServerProvider; - - const invalidations = yield* Ref.make(0); - const rebuiltIds = yield* Ref.make>([]); - const secretResolverLayer = Layer.succeed(ProviderSecretResolver, { - resolve: (environment) => - Effect.succeed({ - variables: literalProviderInstanceEnvironment(environment), - unresolved: [], - }), - prime: () => Effect.void, - invalidate: () => Ref.update(invalidations, (count) => count + 1), - check: () => Effect.succeed({ _tag: "Resolved" }), - listOnePasswordAccounts: Effect.succeed([]), - }); - const instanceRegistryLayer = Layer.succeed( - ProviderInstanceRegistry.ProviderInstanceRegistry, - { - // @effect-diagnostics-next-line effectSucceedWithVoid:off - This optional result requires undefined, not the wider void type. - getInstance: () => Effect.succeed(undefined), - listInstances: Effect.succeed([]), - listUnavailable: Effect.succeed([ - unavailableProvider(readableId), - unavailableProvider(otherId), - ]), - rebuildInstanceWhen: (instanceId) => - Ref.update(rebuiltIds, (previous) => [...previous, instanceId]).pipe(Effect.as(true)), - streamChanges: Stream.empty, - listEnvironments: Effect.succeed( - new Map([ - [readableId, decodeEnvironment([onePasswordVariable("CODEX_TOKEN", reference)])], - [ - otherId, - decodeEnvironment([onePasswordVariable("CODEX_TOKEN", "op://Vault/Other/token")]), - ], - [liveId, decodeEnvironment([onePasswordVariable("CODEX_TOKEN", reference)])], - ]), - ), - subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), - }, - ); - - const scope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); - const runtimeServices = yield* Layer.build( - ProviderRegistry.layer.pipe( - Layer.provideMerge(instanceRegistryLayer), - Layer.provideMerge( - ServerConfig.layerTest(process.cwd(), { - prefix: "t3-provider-registry-secret-check-", - }), - ), - Layer.provideMerge(NodeServices.layer), - Layer.provideMerge(secretResolverLayer), - ), - ).pipe(Scope.provide(scope)); - - yield* Effect.gen(function* () { - const registry = yield* ProviderRegistry.ProviderRegistry; - const source = decodeSecretSource({ - kind: "1password", - reference, - account: HOME_ACCOUNT, - }); - const result = yield* registry.checkSecretSource(source); - - assert.deepStrictEqual(result, { _tag: "Resolved" }); - assert.deepStrictEqual(yield* Ref.get(rebuiltIds), [readableId, liveId]); - assert.strictEqual(yield* Ref.get(invalidations), 0); - }).pipe(Effect.provide(runtimeServices)); - }), - ); - - it.effect("reads every instance's secret in one go before rebuilding any of them", () => - Effect.gen(function* () { - const codexDriver = ProviderDriverKind.make("codex"); - const claudeInstanceId = ProviderInstanceId.make("claude"); - const codexInstanceId = ProviderInstanceId.make("codex"); - const claudeReference = "op://Vault/claude/token"; - const codexReference = "op://Vault/codex/token"; - const unavailableProvider = (instanceId: ProviderInstanceId) => - ({ - instanceId, - driver: codexDriver, - status: "error", - enabled: true, - installed: false, - auth: { status: "unknown" }, - checkedAt: "2026-06-10T00:00:00.000Z", - version: null, - models: [], - slashCommands: [], - skills: [], - message: "Driver 'codex' failed to create instance: secret store is locked", - }) as const satisfies ServerProvider; - - const primed = yield* Ref.make>>([]); - const rebuiltIds = yield* Ref.make>([]); - const secretResolverLayer = Layer.succeed(ProviderSecretResolver, { - resolve: (environment) => - Effect.succeed({ - variables: literalProviderInstanceEnvironment(environment), - unresolved: [], - }), - prime: (references) => - Ref.update(primed, (previous) => [ - ...previous, - references.map((secret) => secret.reference), - ]).pipe(Effect.asVoid), - invalidate: () => Effect.void, - check: () => Effect.succeed({ _tag: "Resolved" }), - listOnePasswordAccounts: Effect.succeed([]), - }); - const environmentFor = (reference: string) => - decodeEnvironment([onePasswordVariable("TOKEN", reference)]); - const instanceRegistryLayer = Layer.succeed( - ProviderInstanceRegistry.ProviderInstanceRegistry, - { - // @effect-diagnostics-next-line effectSucceedWithVoid:off - This optional result requires undefined, not the wider void type. - getInstance: () => Effect.succeed(undefined), - listInstances: Effect.succeed([]), - listUnavailable: Effect.succeed([ - unavailableProvider(claudeInstanceId), - unavailableProvider(codexInstanceId), - ]), - listEnvironments: Effect.succeed( - new Map([ - [claudeInstanceId, environmentFor(claudeReference)], - [codexInstanceId, environmentFor(codexReference)], - ]), - ), - rebuildInstanceWhen: (instanceId, shouldRebuild) => - shouldRebuild({ - driver: codexDriver, - environment: environmentFor( - instanceId === claudeInstanceId ? claudeReference : codexReference, - ), - }) - ? Ref.update(rebuiltIds, (previous) => [...previous, instanceId]).pipe( - Effect.as(true), - ) - : Effect.succeed(false), - streamChanges: Stream.empty, - subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), - }, - ); - - const scope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); - const runtimeServices = yield* Layer.build( - ProviderRegistry.layer.pipe( - Layer.provideMerge(instanceRegistryLayer), - Layer.provideMerge( - ServerConfig.layerTest(process.cwd(), { - prefix: "t3-provider-registry-secret-prime-", - }), - ), - Layer.provideMerge(NodeServices.layer), - Layer.provideMerge(secretResolverLayer), - ), - ).pipe(Scope.provide(scope)); - - yield* Effect.gen(function* () { - const registry = yield* ProviderRegistry.ProviderRegistry; - yield* registry.refresh(); - - // One call carrying both references. Priming per instance would be - // one unlock prompt per provider, which is the thing this exists to - // avoid, so the count matters as much as the contents. - const calls = yield* Ref.get(primed); - assert.strictEqual(calls.length, 1); - assert.deepStrictEqual(Array.from(calls[0] ?? []), [claudeReference, codexReference]); - assert.deepStrictEqual(yield* Ref.get(rebuiltIds), [claudeInstanceId, codexInstanceId]); - }).pipe(Effect.provide(runtimeServices)); - }), - ); - it.effect( "a second request for a cwd being scanned gets its result, even if the first leaves", () => @@ -1752,14 +1356,13 @@ it.layer( listUnavailable: Effect.succeed([]), streamChanges: Stream.empty, subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), - rebuildInstanceWhen: () => Effect.succeed(false), - listEnvironments: Effect.succeed(new Map()), }, ); const scope = yield* Scope.make(); yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(layerInstanceRegistry), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -1849,14 +1452,13 @@ it.layer( listUnavailable: Effect.succeed([]), streamChanges: Stream.empty, subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), - listEnvironments: Effect.succeed(new Map()), - rebuildInstanceWhen: () => Effect.succeed(false), }, ); const scope = yield* Scope.make(); yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(layerInstanceRegistry), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -1969,8 +1571,6 @@ it.layer( Effect.succeed(instances.find((instance) => instance.instanceId === requestedId)), listInstances: Effect.succeed(instances), listUnavailable: Effect.succeed([]), - listEnvironments: Effect.succeed(new Map()), - rebuildInstanceWhen: () => Effect.succeed(false), streamChanges: Stream.empty, subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), }, @@ -1979,6 +1579,7 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(instanceRegistryLayer), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -2075,8 +1676,6 @@ it.layer( Effect.succeed(instanceId === cursorInstanceId ? instance : undefined), listInstances: Effect.succeed([instance]), listUnavailable: Effect.succeed([]), - listEnvironments: Effect.succeed(new Map()), - rebuildInstanceWhen: () => Effect.succeed(false), streamChanges: Stream.empty, subscribeChanges: Effect.flatMap(PubSub.unbounded(), (pubsub) => PubSub.subscribe(pubsub), @@ -2087,6 +1686,7 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(layerInstanceRegistry), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -2205,8 +1805,6 @@ it.layer( Effect.succeed(instanceId === openCodeInstanceId ? instance : undefined), listInstances: Effect.succeed([instance]), listUnavailable: Effect.succeed([]), - listEnvironments: Effect.succeed(new Map()), - rebuildInstanceWhen: () => Effect.succeed(false), streamChanges: Stream.empty, subscribeChanges: Effect.flatMap(PubSub.unbounded(), (pubsub) => PubSub.subscribe(pubsub), @@ -2217,6 +1815,7 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(layerInstanceRegistry), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -2310,8 +1909,6 @@ it.layer( Effect.succeed(instanceId === codexInstanceId ? instance : undefined), listInstances: Effect.succeed([instance]), listUnavailable: Effect.succeed([]), - listEnvironments: Effect.succeed(new Map()), - rebuildInstanceWhen: () => Effect.succeed(false), streamChanges: Stream.empty, subscribeChanges: Effect.flatMap(PubSub.unbounded(), (pubsub) => PubSub.subscribe(pubsub), @@ -2322,6 +1919,7 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const runtimeServices = yield* Layer.build( ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), Layer.provideMerge(layerInstanceRegistry), Layer.provideMerge( ServerConfig.layerTest(process.cwd(), { @@ -2349,118 +1947,6 @@ it.layer( }), ); - it.effect("reads every boot instance's secret before building any of them", () => - Effect.gen(function* () { - const claudeReference = "op://Vault/claude/token"; - const codexReference = "op://Vault/codex/token"; - // `streamChanges` carries later writes only, so the watcher never - // sees this snapshot. Boot is the run where nothing is cached yet - // and therefore the one that pays the most prompts without priming. - const serverSettings = yield* makeMutableServerSettingsService( - decodeServerSettings( - deepMerge(encodedDefaultServerSettings, { - providerInstances: { - ...disabledDefaultSlots("codex", "claudeAgent", "cursor", "grok", "opencode"), - claude_secret: { - driver: "claudeAgent", - displayName: "Claude Secret", - enabled: false, - environment: [ - { - name: "CLAUDE_CODE_OAUTH_TOKEN", - value: { - kind: "1password", - reference: claudeReference, - account: HOME_ACCOUNT, - }, - }, - ], - }, - codex_secret: { - driver: "codex", - displayName: "Codex Secret", - enabled: false, - environment: [ - { - name: "TOKEN", - value: { - kind: "1password", - reference: codexReference, - account: HOME_ACCOUNT, - }, - }, - ], - }, - } as unknown as ContractServerSettings["providerInstances"], - }), - ), - ); - - const calls = yield* Ref.make>([]); - const recordingSecretResolverLayer = Layer.succeed(ProviderSecretResolver, { - resolve: (environment) => - Ref.update(calls, (previous) => [...previous, "resolve"]).pipe( - Effect.as({ - variables: literalProviderInstanceEnvironment(environment), - unresolved: [], - }), - ), - prime: (references) => - Ref.update(calls, (previous) => [ - ...previous, - `prime:${references.map((secret) => secret.reference).join(",")}`, - ]).pipe(Effect.asVoid), - invalidate: () => Effect.void, - check: () => Effect.succeed({ _tag: "Resolved" }), - listOnePasswordAccounts: Effect.succeed([]), - }); - - const scope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); - yield* Layer.build( - ProviderInstanceRegistryHydration.layer.pipe( - Layer.provide(PortScanner.layerTest), - Layer.provideMerge(AntigravityInstallation.AntigravityInstallation.layer), - Layer.provideMerge( - Layer.succeed(ServerSettingsModule.ServerSettingsService, serverSettings), - ), - Layer.provideMerge( - ServerConfig.layerTest(process.cwd(), { - prefix: "t3-provider-registry-boot-prime-", - }), - ), - Layer.provideMerge(layerTestHttpClient), - Layer.provideMerge( - Layer.succeed( - ProviderEventLoggers.ProviderEventLoggers, - ProviderEventLoggers.NoOpProviderEventLoggers, - ), - ), - Layer.provideMerge(ProviderLatestVersions.layer), - Layer.provideMerge(McpProviderSessions.layer), - Layer.provideMerge(ModelManifest.layerTest), - Layer.provideMerge(ResetCreditCoordinator.layerTest), - Layer.provideMerge( - OpenCodeRuntime.layer.pipe(Layer.provide(OpenCodeServerLedger.layerTest)), - ), - Layer.provideMerge(NodeServices.layer), - Layer.provideMerge(layerBackgroundPolicyAlwaysRun), - Layer.provideMerge(recordingSecretResolverLayer), - ), - ).pipe(Scope.provide(scope)); - - // Both references in one call, and that call ahead of the first - // instance that would have read one on its own. - const recorded = yield* Ref.get(calls); - assert.strictEqual( - recorded[0], - `prime:${claudeReference},${codexReference}`, - `Expected boot to prime both references first; instead saw: ${recorded.join(" | ")}`, - ); - assert.strictEqual(recorded.filter((entry) => entry.startsWith("prime")).length, 1); - }), - ); - it.effect("re-probes when settings change the codex binaryPath", () => Effect.gen(function* () { const firstMissing = `t3code_codex_first_`; @@ -2605,7 +2091,7 @@ it.layer( enabled: false, config: { arbitrary: "payload" }, }, - } as unknown as ContractServerSettings["providerInstances"], + } as unknown as (typeof ServerSettings.Encoded)["providerInstances"], }), ), ); @@ -2829,144 +2315,6 @@ it.layer( ), ); - it.effect("signs out an instance whose OAuth token Anthropic rejects", () => - Effect.gen(function* () { - // The probe only proves the CLI found a token; Anthropic is the one - // that knows whether it still works. - const status = yield* checkClaudeProviderStatus( - defaultClaudeSettings, - claudeCapabilities({ - tokenSource: "CLAUDE_CODE_OAUTH_TOKEN", - apiProvider: "firstParty", - }), - { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: "sk-ant-oat01-stale" }, - ).pipe( - Effect.provide(layerClaudeCredentialHttp(() => new Response("no", { status: 401 }))), - ); - assert.strictEqual(status.status, "error"); - assert.strictEqual(status.auth.status, "unauthenticated"); - assert.include(status.message ?? "", "CLAUDE_CODE_OAUTH_TOKEN"); - }).pipe( - Effect.provide( - layerMockSpawner((args) => { - const joined = args.join(" "); - if (joined === "--version") return { stdout: "1.0.0\n", stderr: "", code: 0 }; - throw new Error(`Unexpected args: ${joined}`); - }), - ), - ), - ); - - it.effect("leaves the instance alone when the token is an unresolved reference", () => - Effect.gen(function* () { - // Nothing resolved the reference into a credential, so Anthropic has - // no opinion worth asking for and the token is not what is wrong. - let requests = 0; - const status = yield* checkClaudeProviderStatus( - defaultClaudeSettings, - claudeCapabilities({ - tokenSource: "CLAUDE_CODE_OAUTH_TOKEN", - apiProvider: "firstParty", - }), - { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: "op://Vault/item/credential" }, - ).pipe( - Effect.provide( - layerClaudeCredentialHttp(() => { - requests += 1; - return new Response("no", { status: 401 }); - }), - ), - ); - assert.strictEqual(requests, 0); - assert.strictEqual(status.auth.status, "authenticated"); - }).pipe( - Effect.provide( - layerMockSpawner((args) => { - const joined = args.join(" "); - if (joined === "--version") return { stdout: "1.0.0\n", stderr: "", code: 0 }; - throw new Error(`Unexpected args: ${joined}`); - }), - ), - ), - ); - - it.effect("stays authenticated while Anthropic still accepts the OAuth token", () => - Effect.gen(function* () { - const status = yield* checkClaudeProviderStatus( - defaultClaudeSettings, - claudeCapabilities({ - tokenSource: "CLAUDE_CODE_OAUTH_TOKEN", - apiProvider: "firstParty", - }), - { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: "sk-ant-oat01-good" }, - ).pipe(Effect.provide(layerClaudeCredentialHttp(() => Response.json({ data: [] })))); - assert.strictEqual(status.status, "ready"); - assert.strictEqual(status.auth.status, "authenticated"); - }).pipe( - Effect.provide( - layerMockSpawner((args) => { - const joined = args.join(" "); - if (joined === "--version") return { stdout: "1.0.0\n", stderr: "", code: 0 }; - throw new Error(`Unexpected args: ${joined}`); - }), - ), - ), - ); - - it.effect("keeps a working instance authenticated when the check cannot reach Anthropic", () => - Effect.gen(function* () { - // An outage or a captive network must never read as a revoked token. - const status = yield* checkClaudeProviderStatus( - defaultClaudeSettings, - claudeCapabilities({ - tokenSource: "CLAUDE_CODE_OAUTH_TOKEN", - apiProvider: "firstParty", - }), - { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: "sk-ant-oat01-good" }, - ).pipe( - Effect.provide(layerClaudeCredentialHttp(() => new Response("down", { status: 503 }))), - ); - assert.strictEqual(status.status, "ready"); - assert.strictEqual(status.auth.status, "authenticated"); - }).pipe( - Effect.provide( - layerMockSpawner((args) => { - const joined = args.join(" "); - if (joined === "--version") return { stdout: "1.0.0\n", stderr: "", code: 0 }; - throw new Error(`Unexpected args: ${joined}`); - }), - ), - ), - ); - - it.effect("asks Anthropic nothing when the CLI authenticates some other way", () => - Effect.gen(function* () { - let requests = 0; - const status = yield* checkClaudeProviderStatus( - defaultClaudeSettings, - claudeCapabilities({ tokenSource: "claude.ai", apiProvider: "firstParty" }), - { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: "sk-ant-oat01-ignored-by-the-cli" }, - ).pipe( - Effect.provide( - layerClaudeCredentialHttp(() => { - requests += 1; - return new Response("no", { status: 401 }); - }), - ), - ); - assert.strictEqual(requests, 0); - assert.strictEqual(status.auth.status, "authenticated"); - }).pipe( - Effect.provide( - layerMockSpawner((args) => { - const joined = args.join(" "); - if (joined === "--version") return { stdout: "1.0.0\n", stderr: "", code: 0 }; - throw new Error(`Unexpected args: ${joined}`); - }), - ), - ), - ); - it.effect.each([ { source: "inherited HTTPS endpoint", baseUrl: "https://proxy.example/api", overrides: [] }, { diff --git a/apps/server/src/provider/ProviderRegistry.ts b/apps/server/src/provider/ProviderRegistry.ts index 8975a43aa2e8b..efa3385d1927e 100644 --- a/apps/server/src/provider/ProviderRegistry.ts +++ b/apps/server/src/provider/ProviderRegistry.ts @@ -3,8 +3,6 @@ import { isProviderWorkspaceSnapshotCurrent, ProviderDriverKind, type ProviderInstanceId, - type ProviderSecretSource, - type SecretSourceCheckResult, type ServerProvider, type ServerProviderUpdateState, } from "@t3tools/contracts"; @@ -30,12 +28,7 @@ import * as ModelManifest from "./ModelManifest.ts"; import { applyProviderCompatibility } from "./providerCompatibility.ts"; import * as ServerConfig from "../config.ts"; import * as ProviderInstanceRegistry from "./ProviderInstanceRegistry.ts"; -import * as ProviderSecretResolver from "./ProviderSecretResolver.ts"; -import { - collectProviderSecretReferences, - hasProviderSecretReference, - providerSecretReference, -} from "./ProviderSecretReference.ts"; +import * as ProviderInstanceRegistryMutator from "./ProviderInstanceRegistryMutator.ts"; import { hydrateCachedProvider, isCachedProviderCorrelated, @@ -70,6 +63,11 @@ export class ProviderRegistry extends Context.Service< instanceId: ProviderInstanceId, ) => Effect.Effect>; + /** Re-reads secret-backed environments first, so a fixed secret applies without a restart. */ + readonly reloadSecretsAndRefresh: ( + instanceId?: ProviderInstanceId, + ) => Effect.Effect>; + /** Only refreshed instances can verify an update; the published list still includes every row. */ readonly refreshAfterUpdate: ( provider: ProviderDriverKind, @@ -94,14 +92,6 @@ export class ProviderRegistry extends Context.Service< readonly state: ServerProviderUpdateState | null; }) => Effect.Effect>; - /** - * Settings "check" for one secret source. A readable source also rebuilds every instance - * that reads it, from the value the check just cached. - */ - readonly checkSecretSource: ( - source: ProviderSecretSource, - ) => Effect.Effect; - readonly streamChanges: Stream.Stream>; } >()("t3/provider/ProviderRegistry") {} @@ -347,12 +337,10 @@ export const layer = Layer.effect( ProviderRegistry, Effect.gen(function* () { const instanceRegistry = yield* ProviderInstanceRegistry.ProviderInstanceRegistry; - const secretResolver = yield* ProviderSecretResolver.ProviderSecretResolver; + const instanceRegistryMutator = + yield* ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator; const manifestService = yield* ModelManifest.ModelManifest; const latestVersions = yield* ProviderLatestVersions.ProviderLatestVersions; - // The layer's own scope. `syncLiveSources` forks per-instance - // subscription fibres into it; when a refresh drives that sync, the - // fibres have to land here rather than in the caller's request scope. const serviceScope = yield* Effect.scope; const config = yield* ServerConfig.ServerConfig; const fileSystem = yield* FileSystem.FileSystem; @@ -635,7 +623,8 @@ export const layer = Layer.effect( ); if (claimed.owner) { // Detach shared passes so the first caller's interruption cannot cancel other callers' work. - yield* reloadSecretBackedInstances().pipe( + // Syncing first keeps a refresh right after a rebuild from probing the replaced instance. + yield* syncLiveSourcesAndContinue.pipe( Effect.andThen(getLiveSources), Effect.flatMap((sources) => Effect.forEach(sources, (source) => refreshOneSource(source), { @@ -660,7 +649,6 @@ export const layer = Layer.effect( return yield* refreshAll(); } const defaultInstanceId = defaultInstanceIdForDriver(provider); - yield* reloadSecretBackedInstances([defaultInstanceId]); const sources = yield* getLiveSources; const providerSource = sources.find( (candidate) => candidate.instanceId === defaultInstanceId, @@ -674,7 +662,7 @@ export const layer = Layer.effect( const refreshInstance = Effect.fn("refreshInstance")(function* ( instanceId: ProviderInstanceId, ) { - yield* reloadSecretBackedInstances([instanceId]); + yield* syncLiveSourcesAndContinue; const sources = yield* getLiveSources; const providerSource = sources.find((candidate) => candidate.instanceId === instanceId); if (!providerSource) { @@ -683,6 +671,13 @@ export const layer = Layer.effect( return yield* refreshOneSource(providerSource); }); + const reloadSecretsAndRefresh = Effect.fn("reloadSecretsAndRefresh")(function* ( + instanceId?: ProviderInstanceId, + ) { + yield* instanceRegistryMutator.reloadSecrets(instanceId); + return yield* instanceId === undefined ? refresh() : refreshInstance(instanceId); + }); + const refreshAfterUpdate = Effect.fn("refreshAfterUpdate")(function* ( provider: ProviderDriverKind, selectedInstanceId: ProviderInstanceId, @@ -870,77 +865,6 @@ export const layer = Layer.effect( }); }), ); - const rebuildSecretBackedInstances = Effect.fn("rebuildSecretBackedInstances")(function* ( - targets: ReadonlyArray, - ) { - const rebuilt = yield* Effect.forEach(targets, (instanceId) => - instanceRegistry.rebuildInstanceWhen(instanceId, (entry) => - hasProviderSecretReference(entry.environment), - ), - ); - if (rebuilt.some(Boolean)) { - // Adopt the replacement instances now rather than waiting on the - // registry's change tick, so the refresh that triggered the rebuild - // probes the new process instead of the one it just closed. - yield* syncLiveSources.pipe(Effect.provideService(Scope.Scope, serviceScope)); - } - }); - - const checkSecretSource = Effect.fn("checkSecretSource")(function* ( - source: ProviderSecretSource, - ) { - const result = yield* secretResolver.check(source); - const reference = providerSecretReference(source); - if (result._tag !== "Resolved" || reference === undefined) { - return result; - } - // Live instances are targets too: they read the secret at create time, so a - // rotated value only reaches them through a rebuild. The check left the - // value in the resolver's cache, so these rebuilds never reach the store again. - const environments = yield* instanceRegistry.listEnvironments; - const consumers = Array.from(environments) - .filter(([, environment]) => - collectProviderSecretReferences([environment]).some((candidate) => - Equal.equals(candidate, reference), - ), - ) - .map(([instanceId]) => instanceId); - yield* rebuildSecretBackedInstances(consumers); - return result; - }); - - /** - * Re-read every external-store credential and rebuild the instances that consume one: drivers - * resolve their environment once at create time, so dropping the cache alone changes nothing. - * Literal-only instances and running threads are untouched. - */ - const reloadSecretBackedInstances = Effect.fn("reloadSecretBackedInstances")(function* ( - instanceIds?: ReadonlyArray, - ) { - yield* secretResolver.invalidate(); - // Unavailable instances are targets too. An instance whose last rebuild - // failed because the vault was locked is not live, so live subscriptions - // alone would never reach it again, and a refresh is exactly the moment - // the user is saying the vault is open now. - const targets = - instanceIds ?? - Array.from( - new Set([ - ...(yield* Ref.get(liveSubsRef)).keys(), - ...(yield* instanceRegistry.listUnavailable).map(snapshotInstanceKey), - ]), - ); - // Resolve every reference the rebuilds are about to need, in one read. - // Each rebuild resolves its own environment, and the secret store - // charges an unlock per read rather than per secret, so without this a - // fleet of five instances is five authorizations for one refresh. - const environments = yield* instanceRegistry.listEnvironments; - yield* secretResolver.prime( - collectProviderSecretReferences(targets.map((instanceId) => environments.get(instanceId))), - ); - yield* rebuildSecretBackedInstances(targets); - }); - const syncLiveSourcesAndContinue = syncLiveSources.pipe( Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) { @@ -953,6 +877,7 @@ export const layer = Layer.effect( }, ); }), + Scope.provide(serviceScope), ); yield* upsertProviders(fallbackProviders, { publish: false }); @@ -1103,12 +1028,13 @@ export const layer = Layer.effect( refresh(provider).pipe(Effect.catchCause(recoverRefreshFailure)), refreshInstance: (instanceId: ProviderInstanceId) => refreshInstance(instanceId).pipe(Effect.catchCause(recoverRefreshFailure)), + reloadSecretsAndRefresh: (instanceId?: ProviderInstanceId) => + reloadSecretsAndRefresh(instanceId).pipe(Effect.catchCause(recoverRefreshFailure)), refreshWorkspaceSnapshot: (input) => refreshWorkspaceSnapshot(input).pipe(Effect.catchCause(recoverRefreshFailure)), refreshAfterUpdate, getProviderMaintenanceCapabilitiesForInstance, setProviderMaintenanceActionState, - checkSecretSource, get streamChanges() { return Stream.fromPubSub(changesPubSub); }, diff --git a/apps/server/src/provider/ProviderSecretReference.test.ts b/apps/server/src/provider/ProviderSecretReference.test.ts deleted file mode 100644 index 02e5a08ed7b64..0000000000000 --- a/apps/server/src/provider/ProviderSecretReference.test.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { ProviderInstanceEnvironment, SecretProviderId } from "@t3tools/contracts"; -import * as Schema from "effect/Schema"; - -import { - collectProviderSecretReferences, - hasProviderSecretReference, - NamedProviderSecretReference, - OnePasswordSecretReference, - providerSecretReference, -} from "./ProviderSecretReference.ts"; - -const decodeEnvironment = Schema.decodeSync(ProviderInstanceEnvironment); - -const ACCOUNT = "my.1password.com"; - -const onePasswordVariable = (name: string, reference: string, account = ACCOUNT) => ({ - name, - value: { kind: "1password" as const, reference, account }, -}); - -const namedVariable = (name: string, provider: string, reference: string) => ({ - name, - value: { kind: "secret-provider" as const, provider, reference }, -}); - -const secret = (reference: string) => - new OnePasswordSecretReference({ - reference, - account: ACCOUNT as OnePasswordSecretReference["account"], - }); - -describe("providerSecretReference", () => { - it("reads a 1Password source with its account", () => { - const [variable] = decodeEnvironment([ - onePasswordVariable("TOKEN", "op://Private/claude-code/credential", "my.1password.com"), - ]); - expect(providerSecretReference(variable!.value)).toEqual( - new OnePasswordSecretReference({ - reference: "op://Private/claude-code/credential", - account: "my.1password.com" as OnePasswordSecretReference["account"], - }), - ); - }); - - it("reads a named provider source with its provider id", () => { - const [variable] = decodeEnvironment([ - namedVariable("TOKEN", "team-vault", "agents/codex#api_key"), - ]); - expect(providerSecretReference(variable!.value)).toEqual( - new NamedProviderSecretReference({ - provider: SecretProviderId.make("team-vault"), - reference: "agents/codex#api_key", - }), - ); - }); - - it("treats every string as a literal, including one that looks like a reference", () => { - expect(providerSecretReference("sk-live-token")).toBeUndefined(); - expect(providerSecretReference("/home/u/.claude/work")).toBeUndefined(); - expect(providerSecretReference("op://Private/claude-code/credential")).toBeUndefined(); - }); -}); - -describe("hasProviderSecretReference", () => { - it("is false for an absent or empty environment", () => { - expect(hasProviderSecretReference(undefined)).toBe(false); - expect(hasProviderSecretReference([])).toBe(false); - }); - - it("is true when any single variable reads from the store", () => { - expect( - hasProviderSecretReference( - decodeEnvironment([ - { name: "CLAUDE_SECURESTORAGE_CONFIG_DIR", value: "/home/u/.claude/work" }, - onePasswordVariable("CLAUDE_CODE_OAUTH_TOKEN", "op://Private/claude-code/credential"), - ]), - ), - ).toBe(true); - }); - - it("is false when every variable is a literal", () => { - expect( - hasProviderSecretReference( - decodeEnvironment([ - { name: "CLAUDE_SECURESTORAGE_CONFIG_DIR", value: "/home/u/.claude/work" }, - ]), - ), - ).toBe(false); - }); -}); - -describe("hasProviderSecretReference with secret sources", () => { - it("is true for a 1Password source", () => { - expect( - hasProviderSecretReference( - decodeEnvironment([ - onePasswordVariable("TOKEN", "op://Private/item/field", "my.1password.com"), - ]), - ), - ).toBe(true); - }); -}); - -describe("hasProviderSecretReference with named providers", () => { - it("is true for a named provider source", () => { - expect( - hasProviderSecretReference( - decodeEnvironment([namedVariable("TOKEN", "team-vault", "agents/codex#api_key")]), - ), - ).toBe(true); - }); -}); - -describe("collectProviderSecretReferences", () => { - it("returns each distinct reference once, in first-seen order", () => { - const shared = "op://Private/shared/credential"; - const environments = [ - decodeEnvironment([ - onePasswordVariable("CLAUDE_CODE_OAUTH_TOKEN", shared), - { name: "HOME", value: "/home/u" }, - ]), - undefined, - decodeEnvironment([ - onePasswordVariable("CODEX_TOKEN", "op://Private/codex/credential"), - // The same item behind two providers is one read, not two. - onePasswordVariable("OTHER_TOKEN", shared), - ]), - ]; - - expect(Array.from(collectProviderSecretReferences(environments))).toEqual([ - secret(shared), - secret("op://Private/codex/credential"), - ]); - }); - - it("keeps one reference read from different accounts apart", () => { - const shared = "op://Private/shared/credential"; - const references = collectProviderSecretReferences([ - decodeEnvironment([ - onePasswordVariable("HOME_TOKEN", shared, "my.1password.com"), - onePasswordVariable("WORK_TOKEN", shared, "acme.1password.com"), - onePasswordVariable("HOME_AGAIN", shared, "my.1password.com"), - ]), - ]); - - expect(references).toEqual([ - new OnePasswordSecretReference({ - reference: shared, - account: "my.1password.com" as OnePasswordSecretReference["account"], - }), - new OnePasswordSecretReference({ - reference: shared, - account: "acme.1password.com" as OnePasswordSecretReference["account"], - }), - ]); - }); - - it("is empty when nothing reads from a secret store", () => { - expect( - Array.from( - collectProviderSecretReferences([ - decodeEnvironment([{ name: "HOME", value: "/home/u" }]), - undefined, - ]), - ), - ).toEqual([]); - }); - - it("keeps one reference behind different providers apart, and apart from 1Password", () => { - const shared = "op://Private/shared/credential"; - const references = collectProviderSecretReferences([ - decodeEnvironment([ - namedVariable("HOME_TOKEN", "home", shared), - namedVariable("WORK_TOKEN", "work", shared), - onePasswordVariable("LEGACY_TOKEN", shared), - namedVariable("HOME_AGAIN", "home", shared), - ]), - ]); - - expect(references).toEqual([ - new NamedProviderSecretReference({ - provider: SecretProviderId.make("home"), - reference: shared, - }), - new NamedProviderSecretReference({ - provider: SecretProviderId.make("work"), - reference: shared, - }), - secret(shared), - ]); - }); -}); diff --git a/apps/server/src/provider/ProviderSecretReference.ts b/apps/server/src/provider/ProviderSecretReference.ts deleted file mode 100644 index 3b9fcce3a626d..0000000000000 --- a/apps/server/src/provider/ProviderSecretReference.ts +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Provider environment values that name a secret (`secret-provider` or inline `1password`) - * instead of carrying one. Pure helpers, so the registry need not depend on the resolver. - * @module provider/ProviderSecretReference - */ -import type { - OnePasswordAccount, - ProviderInstanceEnvironment, - ProviderInstanceEnvironmentVariable, - SecretProviderId, -} from "@t3tools/contracts"; -import * as Data from "effect/Data"; -import * as Equal from "effect/Equal"; - -/** - * One secret to read from 1Password. Structurally comparable, so it doubles as - * the resolver's cache key: the same reference in two accounts is two secrets. - */ -export class OnePasswordSecretReference extends Data.TaggedClass("1password")<{ - readonly reference: string; - readonly account: OnePasswordAccount; -}> {} - -/** - * One secret to read through a named provider. The provider id is part of the - * cache key, so the same reference behind two providers is two secrets. What - * the reference means depends on the provider's kind, which is looked up when - * the secret is read. - */ -export class NamedProviderSecretReference extends Data.TaggedClass("secret-provider")<{ - readonly provider: SecretProviderId; - readonly reference: string; -}> {} - -/** Every secret a provider environment value can name. */ -export type ProviderSecretReference = OnePasswordSecretReference | NamedProviderSecretReference; - -/** - * The secret an environment value names, or `undefined` when the value is a - * literal. - */ -export function providerSecretReference( - value: ProviderInstanceEnvironmentVariable["value"], -): ProviderSecretReference | undefined { - if (typeof value === "string") { - return undefined; - } - switch (value.kind) { - case "1password": - return new OnePasswordSecretReference({ - reference: value.reference, - account: value.account, - }); - case "secret-provider": - return new NamedProviderSecretReference({ - provider: value.provider, - reference: value.reference, - }); - } -} - -/** - * Whether any variable in the environment reads from a secret store. Drives - * the "rebuild this instance on refresh" decision, so instances configured - * entirely with literals keep the process they already have. - */ -export function hasProviderSecretReference( - environment: ProviderInstanceEnvironment | undefined, -): boolean { - return ( - environment?.some((variable) => providerSecretReference(variable.value) !== undefined) ?? false - ); -} - -/** - * Every distinct secret across a set of environments, in first-seen order. Unlocking is per `op` - * invocation, so callers building many instances want the whole list up front. - */ -export function collectProviderSecretReferences( - environments: Iterable, -): ReadonlyArray { - const references: Array = []; - for (const environment of environments) { - for (const variable of environment ?? []) { - const reference = providerSecretReference(variable.value); - if (reference !== undefined && !references.some((seen) => Equal.equals(seen, reference))) { - references.push(reference); - } - } - } - return references; -} diff --git a/apps/server/src/provider/ProviderSecretResolver.ts b/apps/server/src/provider/ProviderSecretResolver.ts deleted file mode 100644 index 6007c35fbb8c0..0000000000000 --- a/apps/server/src/provider/ProviderSecretResolver.ts +++ /dev/null @@ -1,141 +0,0 @@ -/** - * ProviderSecretResolver: resolves secret-naming environment values once and caches them per - * secret (reference plus account) for the process lifetime, so rebuilds do not re-run `op` or - * re-prompt. `invalidate` is wired to the Settings refresh button. - * @module provider/ProviderSecretResolver - */ -import { - type OnePasswordAccountSummary, - type ProviderInstanceEnvironment, - type ProviderSecretSource, - SecretProviderId, - type SecretSourceCheckResult, - type SettingsSecretValue, -} from "@t3tools/contracts"; -import * as Context from "effect/Context"; -import * as Effect from "effect/Effect"; -import * as Schema from "effect/Schema"; - -import type { ResolvedProviderEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; -import type { ProviderSecretReference } from "./ProviderSecretReference.ts"; - -/** - * An instance environment after its secret references were read. `unresolved` is tracked apart - * from `variables` because the child environment starts from the server's own, so an unset name - * would keep an inherited credential. - */ -export interface ResolvedProviderInstanceEnvironment { - readonly variables: ResolvedProviderEnvironment; - readonly unresolved: ReadonlyArray; -} - -export interface ProviderSecretResolverShape { - /** - * Replace every named secret with its value; literals pass through. Never fails: an unreadable - * reference is reported unresolved, never as an empty string, and the registry unsets its name - * so an inherited same-named credential is not used. - */ - readonly resolve: ( - environment: ProviderInstanceEnvironment | undefined, - ) => Effect.Effect; - /** - * Resolve `references` ahead of callers in as few store invocations as possible, since unlocking - * is charged per invocation. Best effort and never fails: a failed batch leaves the cache cold. - */ - readonly prime: (references: ReadonlyArray) => Effect.Effect; - /** - * Drop cached secrets so the next `resolve` re-reads them: all by default, or only those from - * the named providers. Running providers keep their spawn-time environment until rebuilt. - */ - readonly invalidate: (providers?: ReadonlySet) => Effect.Effect; - /** - * Read one secret source fresh for the Settings "check" button. A success is cached; never - * fails and never returns the value. - */ - readonly check: (source: ProviderSecretSource) => Effect.Effect; - /** - * The 1Password accounts signed in on this machine, so settings can offer - * them instead of asking the user to type one. Reads local `op` config only, - * so it never prompts. Empty when `op` is missing or fails. - */ - readonly listOnePasswordAccounts: Effect.Effect>; -} - -/** - * Defaults to handing every literal back untouched, which is what a build - * without secret-store integration behaves like, and what tests want unless - * they are testing resolution itself. A secret source has no literal form, so - * it is reported unresolved. - */ -const resolveWithoutSecretStore = ( - environment: ProviderInstanceEnvironment | undefined, -): ResolvedProviderInstanceEnvironment => { - const variables: Array = []; - const unresolved: Array = []; - for (const { name, value, sensitive } of environment ?? []) { - if (typeof value === "string") { - variables.push({ name, value, sensitive }); - } else { - unresolved.push(name); - } - } - return { variables, unresolved }; -}; - -export class ProviderSecretResolver extends Context.Reference( - "t3/provider/ProviderSecretResolver", - { - defaultValue: () => ({ - resolve: (environment) => Effect.sync(() => resolveWithoutSecretStore(environment)), - prime: () => Effect.void, - invalidate: () => Effect.void, - check: () => - Effect.succeed({ - _tag: "Failed", - detail: "Secret stores are not available on this server", - } as const), - listOnePasswordAccounts: Effect.succeed([]), - }), - }, -) {} - -export class SettingsSecretUnresolvedError extends Schema.TaggedError()( - "SettingsSecretUnresolvedError", - { provider: SecretProviderId }, -) { - override get message(): string { - return `Could not read the credential from the secret provider "${this.provider}". Unlock or fix it in Settings, then try again.`; - } -} - -/** - * The secret a credential setting holds or names, through the resolver's cache. A named secret - * that cannot be read, or reads empty, fails instead of passing for a missing token, so callers - * that would fall back to another credential cannot silently switch accounts. - */ -export const requireSettingsSecret = ( - resolver: ProviderSecretResolverShape, - value: SettingsSecretValue, -): Effect.Effect => - typeof value === "string" - ? Effect.succeed(value) - : resolver.resolve([{ name: "T3_SETTINGS_SECRET", value, sensitive: true }]).pipe( - Effect.flatMap(({ variables }) => { - const secret = variables[0]?.value ?? ""; - return secret.trim() === "" - ? Effect.fail(new SettingsSecretUnresolvedError({ provider: value.provider })) - : Effect.succeed(secret); - }), - ); - -/** - * Like `requireSettingsSecret`, but an unreadable secret comes back as `""`, so callers treat it - * exactly like a missing token. Only for credentials whose documented fallback is acceptable. - */ -export const resolveSettingsSecret = ( - resolver: ProviderSecretResolverShape, - value: SettingsSecretValue, -): Effect.Effect => - requireSettingsSecret(resolver, value).pipe( - Effect.catchTags({ SettingsSecretUnresolvedError: () => Effect.succeed("") }), - ); diff --git a/apps/server/src/provider/ProviderSecretResolverLive.test.ts b/apps/server/src/provider/ProviderSecretResolverLive.test.ts deleted file mode 100644 index e3a26f9f7725c..0000000000000 --- a/apps/server/src/provider/ProviderSecretResolverLive.test.ts +++ /dev/null @@ -1,1408 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off - asserts against the fake op CLI script this test writes to disk. -import { describe, it, assert } from "@effect/vitest"; -import { - OnePasswordAccount, - ProviderInstanceEnvironment, - SecretProviderId, - SecretProviderKind, - type ServerSettings, -} from "@t3tools/contracts"; -import * as Effect from "effect/Effect"; -import * as Fiber from "effect/Fiber"; -import * as HostProcess from "@t3tools/shared/HostProcess"; -import { HttpClient, HttpClientResponse } from "effect/http"; -import * as NodePath from "@effect/platform-node/NodePath"; -import * as Layer from "effect/Layer"; -import * as Logger from "effect/Logger"; -import * as Schema from "effect/Schema"; -import * as Sink from "effect/Sink"; -import * as Stream from "effect/Stream"; -import * as TestClock from "effect/testing/TestClock"; -import { ChildProcessSpawner } from "effect/process"; -import * as NodeFileSystem from "@effect/platform-node/NodeFileSystem"; -import * as NodeFS from "node:fs"; - -import * as Settings from "../serverSettings.ts"; -import { - NamedProviderSecretReference, - OnePasswordSecretReference, -} from "./ProviderSecretReference.ts"; -import { ProviderSecretResolverLive } from "./ProviderSecretResolverLive.ts"; -import { ProviderSecretResolver } from "./ProviderSecretResolver.ts"; - -const encoder = new TextEncoder(); -const decodeEnvironment = Schema.decodeSync(ProviderInstanceEnvironment); - -const TOKEN_REFERENCE = "op://Private/claude-code/credential"; - -const HOME_ACCOUNT = "my.1password.com"; -const WORK_ACCOUNT = "acme.1password.com"; - -const onePasswordVariable = (name: string, reference: string, account: string) => ({ - name, - value: { kind: "1password" as const, reference, account }, -}); - -const onePassword = (reference: string, account: string) => { - const [variable] = decodeEnvironment([onePasswordVariable("TOKEN", reference, account)]); - const source = variable?.value; - if (source === undefined || typeof source === "string" || source.kind !== "1password") { - throw new Error("expected a 1Password source"); - } - return new OnePasswordSecretReference({ reference: source.reference, account: source.account }); -}; - -/** - * The live resolver over a fake spawner. `secretProviders` seeds the settings - * the resolver looks named providers up in. - */ -const noHttpClient = HttpClient.make(() => Effect.die("unexpected HTTP request")); - -const resolverLayer = ( - spawner: Layer.Layer, - secretProviders: ServerSettings["secretProviders"] = {}, - httpClient: HttpClient.HttpClient = noHttpClient, -) => - ProviderSecretResolverLive.pipe( - Layer.provide( - Layer.mergeAll( - spawner, - NodeFileSystem.layer, - NodePath.layer, - Layer.succeed(HttpClient.HttpClient, httpClient), - Settings.layerTest({ secretProviders }), - ), - ), - ); - -/** - * Spawner that answers every `op read` with `result` and records the argv it - * was handed, so tests can assert both the substituted value and how many - * times 1Password was actually consulted. - */ -function recordingOpSpawner(result: { stdout: string; stderr: string; code: number }) { - const invocations: Array> = []; - const layer = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make((command) => { - const cmd = command as unknown as { args: ReadonlyArray }; - invocations.push(cmd.args); - return Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(1), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result.code)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - unref: Effect.succeed(Effect.void), - stdin: Sink.drain, - stdout: Stream.make(encoder.encode(result.stdout)), - stderr: Stream.make(encoder.encode(result.stderr)), - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - }), - ); - }), - ); - return { layer, invocations }; -} - -describe("ProviderSecretResolverLive", () => { - it.effect("leaves an environment of literal values alone", () => { - const spawner = recordingOpSpawner({ stdout: "", stderr: "", code: 0 }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - const environment = decodeEnvironment([ - { name: "CLAUDE_SECURESTORAGE_CONFIG_DIR", value: "/home/u/.claude/work" }, - ]); - - const resolved = yield* resolver.resolve(environment); - - assert.deepStrictEqual(resolved, { - variables: [ - { - name: "CLAUDE_SECURESTORAGE_CONFIG_DIR", - value: "/home/u/.claude/work", - sensitive: false, - }, - ], - unresolved: [], - }); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("swaps a secret reference for the value 1Password returns", () => { - const spawner = recordingOpSpawner({ stdout: "sk-live-token\n", stderr: "", code: 0 }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve( - decodeEnvironment([ - { name: "CLAUDE_SECURESTORAGE_CONFIG_DIR", value: "/home/u/.claude/work" }, - onePasswordVariable("CLAUDE_CODE_OAUTH_TOKEN", TOKEN_REFERENCE, HOME_ACCOUNT), - ]), - ); - - assert.deepStrictEqual( - resolved.variables?.map((variable) => [variable.name, variable.value]), - [ - ["CLAUDE_SECURESTORAGE_CONFIG_DIR", "/home/u/.claude/work"], - ["CLAUDE_CODE_OAUTH_TOKEN", "sk-live-token"], - ], - ); - assert.deepStrictEqual(spawner.invocations, [ - ["--cache=false", "read", "--account", HOME_ACCOUNT, "--no-newline", TOKEN_REFERENCE], - ]); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("reads a reference once and holds it until the caller invalidates", () => { - const spawner = recordingOpSpawner({ stdout: "sk-live-token", stderr: "", code: 0 }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - const environment = decodeEnvironment([ - onePasswordVariable("CLAUDE_CODE_OAUTH_TOKEN", TOKEN_REFERENCE, HOME_ACCOUNT), - ]); - - // Every thread start and every instance rebuild resolves again; none of - // them should reach 1Password while the value is already in memory. - yield* resolver.resolve(environment); - yield* resolver.resolve(environment); - assert.strictEqual(spawner.invocations.length, 1); - - yield* resolver.invalidate(); - yield* resolver.resolve(environment); - assert.strictEqual(spawner.invocations.length, 2); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("drops a variable whose reference cannot be read", () => { - const spawner = recordingOpSpawner({ - stdout: "", - stderr: "[ERROR] could not read secret: not signed in", - code: 1, - }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - // An empty string here would look like a present credential and send - // the provider off to fail mid-turn. Absence is the honest answer, and - // the provider reports itself unauthenticated instead. - const resolved = yield* resolver.resolve( - decodeEnvironment([ - { name: "CLAUDE_SECURESTORAGE_CONFIG_DIR", value: "/home/u/.claude/work" }, - onePasswordVariable("CLAUDE_CODE_OAUTH_TOKEN", TOKEN_REFERENCE, HOME_ACCOUNT), - ]), - ); - - assert.deepStrictEqual( - resolved.variables?.map((variable) => variable.name), - ["CLAUDE_SECURESTORAGE_CONFIG_DIR"], - ); - // Naming it is what gets it unset in the child environment. Merely - // leaving it out would hand the provider whatever the server itself was - // started with under that name. - assert.deepStrictEqual(resolved.unresolved, ["CLAUDE_CODE_OAUTH_TOKEN"]); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("reads a failed reference again on the next resolve", () => { - const spawner = recordingOpSpawner({ stdout: "", stderr: "not signed in", code: 1 }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - const environment = decodeEnvironment([ - onePasswordVariable("CLAUDE_CODE_OAUTH_TOKEN", TOKEN_REFERENCE, HOME_ACCOUNT), - ]); - - yield* resolver.resolve(environment); - yield* resolver.resolve(environment); - - assert.strictEqual(spawner.invocations.length, 2); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); -}); - -const SECOND_REFERENCE = "op://Private/codex/credential"; - -/** - * Spawner that answers each `op` invocation from `handler`. The `op inject` template is read - * back from the `-i` path in argv, and its random separator proves batch and split agree. - */ -function scriptedOpSpawner( - handler: ( - args: ReadonlyArray, - template: string, - ) => { stdout: string; stderr: string; code: number }, -) { - const invocations: Array> = []; - const commands: Array = []; - const environments: Array> | undefined> = []; - const stdinUses: Array = []; - const layer = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make((command) => - Effect.sync(() => { - const cmd = command as unknown as { - command: string; - args: ReadonlyArray; - options?: { - stdin?: Stream.Stream; - env?: Readonly>; - }; - }; - invocations.push(cmd.args); - commands.push(cmd.command); - environments.push(cmd.options?.env); - stdinUses.push(cmd.options?.stdin !== undefined); - const inputPath = cmd.args[cmd.args.indexOf("-i") + 1]; - const template = - cmd.args.includes("-i") && inputPath !== undefined - ? NodeFS.readFileSync(inputPath, "utf8") - : ""; - const result = handler(cmd.args, template); - return ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(1), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result.code)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - unref: Effect.succeed(Effect.void), - stdin: Sink.drain, - stdout: Stream.make(encoder.encode(result.stdout)), - stderr: Stream.make(encoder.encode(result.stderr)), - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - }); - }), - ), - ); - return { layer, invocations, commands, environments, stdinUses }; -} - -/** The separator `prime` chose, read back out of the template it built. */ -function separatorOf(template: string): string { - const parts = template.split(/\{\{[^}]*\}\}/); - return parts[1] ?? ""; -} - -describe("ProviderSecretResolverLive.prime", () => { - it.effect("reads every reference in a single 1Password call", () => { - const spawner = scriptedOpSpawner((args, template) => { - if (args.includes("inject")) { - return { - stdout: ["sk-claude-token", "sk-codex-token"].join(separatorOf(template)), - stderr: "", - code: 0, - }; - } - return { stdout: "should-not-be-read-one-at-a-time", stderr: "", code: 0 }; - }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - yield* resolver.prime([ - onePassword(TOKEN_REFERENCE, HOME_ACCOUNT), - onePassword(SECOND_REFERENCE, HOME_ACCOUNT), - ]); - - assert.strictEqual(spawner.invocations.length, 1); - assert.deepStrictEqual(Array.from(spawner.invocations[0] ?? []).slice(0, 5), [ - "--cache=false", - "inject", - "--account", - HOME_ACCOUNT, - "-i", - ]); - - // Both instances resolve out of the primed cache, so the fleet costs the - // one authorization the batch already paid for. - const claude = yield* resolver.resolve( - decodeEnvironment([ - onePasswordVariable("CLAUDE_CODE_OAUTH_TOKEN", TOKEN_REFERENCE, HOME_ACCOUNT), - ]), - ); - const codex = yield* resolver.resolve( - decodeEnvironment([onePasswordVariable("CODEX_TOKEN", SECOND_REFERENCE, HOME_ACCOUNT)]), - ); - - assert.strictEqual(claude.variables?.[0]?.value, "sk-claude-token"); - assert.strictEqual(codex.variables?.[0]?.value, "sk-codex-token"); - assert.strictEqual(spawner.invocations.length, 1); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("falls back to one read at a time when the batch fails", () => { - const spawner = scriptedOpSpawner((args) => { - if (args.includes("inject")) { - return { stdout: "", stderr: 'could not resolve item "codex"', code: 1 }; - } - return args.includes(SECOND_REFERENCE) - ? { stdout: "", stderr: 'could not resolve item "codex"', code: 1 } - : { stdout: "sk-claude-token", stderr: "", code: 0 }; - }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - yield* resolver.prime([ - onePassword(TOKEN_REFERENCE, HOME_ACCOUNT), - onePassword(SECOND_REFERENCE, HOME_ACCOUNT), - ]); - - // The batch is still attempted; it is the recovery that is per reference. - assert.strictEqual(spawner.invocations[0]?.[1], "inject"); - - // A batch that cannot be trusted leaves the cache cold rather than - // caching a failure for every reference in it, so the good reference - // still resolves and only the bad one is reported unresolved. - const claude = yield* resolver.resolve( - decodeEnvironment([ - onePasswordVariable("CLAUDE_CODE_OAUTH_TOKEN", TOKEN_REFERENCE, HOME_ACCOUNT), - ]), - ); - const codex = yield* resolver.resolve( - decodeEnvironment([onePasswordVariable("CODEX_TOKEN", SECOND_REFERENCE, HOME_ACCOUNT)]), - ); - - assert.strictEqual(claude.variables?.[0]?.value, "sk-claude-token"); - assert.deepStrictEqual(Array.from(claude.unresolved), []); - assert.deepStrictEqual(Array.from(codex.unresolved), ["CODEX_TOKEN"]); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("hands the template to a file `op` will actually read", () => { - let seenTemplate = ""; - const spawner = scriptedOpSpawner((args, template) => { - if (args.includes("inject")) { - seenTemplate = template; - return { - stdout: ["sk-claude-token", "sk-codex-token"].join(separatorOf(template)), - stderr: "", - code: 0, - }; - } - return { stdout: "should-not-be-read-one-at-a-time", stderr: "", code: 0 }; - }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - yield* resolver.prime([ - onePassword(TOKEN_REFERENCE, HOME_ACCOUNT), - onePassword(SECOND_REFERENCE, HOME_ACCOUNT), - ]); - - // `op` only reads piped input from a named pipe, and Node hands a child - // a socket pair, so a template offered on stdin is never seen and the - // batch fails every time. The `-i` path is the delivery that works. - const args = Array.from(spawner.invocations[0] ?? []); - assert.deepStrictEqual(args.slice(0, 5), [ - "--cache=false", - "inject", - "--account", - HOME_ACCOUNT, - "-i", - ]); - assert.isTrue((args[5] ?? "").length > 0); - assert.deepStrictEqual(spawner.stdinUses, [false]); - - // The file `op` was pointed at held both references and nothing else, - // so a secret never reaches the disk. - assert.isTrue(seenTemplate.includes(TOKEN_REFERENCE)); - assert.isTrue(seenTemplate.includes(SECOND_REFERENCE)); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("leaves no template behind once the batch is done", () => { - let templatePath = ""; - const spawner = scriptedOpSpawner((args, template) => { - if (args.includes("inject")) { - templatePath = args[args.indexOf("-i") + 1] ?? ""; - return { - stdout: ["sk-claude-token", "sk-codex-token"].join(separatorOf(template)), - stderr: "", - code: 0, - }; - } - return { stdout: "", stderr: "", code: 0 }; - }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - yield* resolver.prime([ - onePassword(TOKEN_REFERENCE, HOME_ACCOUNT), - onePassword(SECOND_REFERENCE, HOME_ACCOUNT), - ]); - - assert.isTrue(templatePath.length > 0); - assert.isFalse(NodeFS.existsSync(templatePath)); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("does not spawn a batch for a single reference", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "sk-claude-token", stderr: "", code: 0 })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - yield* resolver.prime([onePassword(TOKEN_REFERENCE, HOME_ACCOUNT)]); - - // One reference is one prompt either way, and `op read` names the - // reference it could not resolve. - assert.strictEqual(spawner.invocations.length, 0); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); -}); - -describe("ProviderSecretResolverLive with 1Password accounts", () => { - it.effect("reads a 1Password source from the account it names", () => { - const spawner = recordingOpSpawner({ stdout: "sk-work-token", stderr: "", code: 0 }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve( - decodeEnvironment([onePasswordVariable("CODEX_TOKEN", TOKEN_REFERENCE, WORK_ACCOUNT)]), - ); - - assert.deepStrictEqual(resolved, { - variables: [{ name: "CODEX_TOKEN", value: "sk-work-token", sensitive: true }], - unresolved: [], - }); - assert.deepStrictEqual(spawner.invocations, [ - ["--cache=false", "read", "--account", WORK_ACCOUNT, "--no-newline", TOKEN_REFERENCE], - ]); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("keeps the same reference in two accounts apart", () => { - const spawner = scriptedOpSpawner((args) => ({ - stdout: args.includes(WORK_ACCOUNT) ? "sk-work-token" : "sk-home-token", - stderr: "", - code: 0, - })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve( - decodeEnvironment([ - onePasswordVariable("HOME_TOKEN", TOKEN_REFERENCE, HOME_ACCOUNT), - onePasswordVariable("WORK_TOKEN", TOKEN_REFERENCE, WORK_ACCOUNT), - ]), - ); - - assert.deepStrictEqual(resolved.variables, [ - { name: "HOME_TOKEN", value: "sk-home-token", sensitive: true }, - { name: "WORK_TOKEN", value: "sk-work-token", sensitive: true }, - ]); - assert.strictEqual(spawner.invocations.length, 2); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("primes one batch per account", () => { - const spawner = scriptedOpSpawner((args, template) => { - if (!args.includes("inject")) { - return { stdout: "should-not-be-read-one-at-a-time", stderr: "", code: 0 }; - } - const prefix = args.includes(WORK_ACCOUNT) ? "work" : "home"; - return { - stdout: [`${prefix}-claude`, `${prefix}-codex`].join(separatorOf(template)), - stderr: "", - code: 0, - }; - }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - yield* resolver.prime([ - onePassword(TOKEN_REFERENCE, HOME_ACCOUNT), - onePassword(TOKEN_REFERENCE, WORK_ACCOUNT), - onePassword(SECOND_REFERENCE, HOME_ACCOUNT), - onePassword(SECOND_REFERENCE, WORK_ACCOUNT), - ]); - - assert.deepStrictEqual( - spawner.invocations.map((args) => Array.from(args).slice(0, 5)), - [ - ["--cache=false", "inject", "--account", HOME_ACCOUNT, "-i"], - ["--cache=false", "inject", "--account", WORK_ACCOUNT, "-i"], - ], - ); - - const resolved = yield* resolver.resolve( - decodeEnvironment([ - onePasswordVariable("HOME_CODEX", SECOND_REFERENCE, HOME_ACCOUNT), - onePasswordVariable("WORK_CLAUDE", TOKEN_REFERENCE, WORK_ACCOUNT), - ]), - ); - assert.deepStrictEqual(resolved.variables, [ - { name: "HOME_CODEX", value: "home-codex", sensitive: true }, - { name: "WORK_CLAUDE", value: "work-claude", sensitive: true }, - ]); - assert.strictEqual(spawner.invocations.length, 2); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); -}); - -describe("ProviderSecretResolverLive.listOnePasswordAccounts", () => { - const provideResolver = (spawner: ReturnType) => - Effect.provide(resolverLayer(spawner.layer)); - - it.effect("offers each signed-in account by the address `op --account` accepts", () => { - const spawner = recordingOpSpawner({ - stdout: JSON.stringify([ - { url: HOME_ACCOUNT, email: "me@example.com", user_uuid: "U1", account_uuid: "A1" }, - { url: WORK_ACCOUNT, email: "me@acme.example", user_uuid: "U2", account_uuid: "A2" }, - ]), - stderr: "", - code: 0, - }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const accounts = yield* resolver.listOnePasswordAccounts; - - assert.deepStrictEqual(accounts, [ - { account: OnePasswordAccount.make(HOME_ACCOUNT), email: "me@example.com" }, - { account: OnePasswordAccount.make(WORK_ACCOUNT), email: "me@acme.example" }, - ]); - assert.deepStrictEqual(spawner.invocations, [ - ["--cache=false", "account", "list", "--format", "json"], - ]); - }).pipe(provideResolver(spawner)); - }); - - it.effect("offers no accounts when `op` fails", () => { - const spawner = recordingOpSpawner({ stdout: "", stderr: "not configured", code: 1 }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - assert.deepStrictEqual(yield* resolver.listOnePasswordAccounts, []); - }).pipe(provideResolver(spawner)); - }); - - it.effect("offers no accounts when `op` prints something unexpected", () => { - const spawner = recordingOpSpawner({ stdout: "not json", stderr: "", code: 0 }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - assert.deepStrictEqual(yield* resolver.listOnePasswordAccounts, []); - }).pipe(provideResolver(spawner)); - }); -}); - -const HOME_PROVIDER = SecretProviderId.make("home-1password"); -const VAULT_PROVIDER = SecretProviderId.make("team-vault"); -const BAO_ADDRESS = "https://bao.example.com:8200"; - -const namedVariable = (name: string, provider: SecretProviderId, reference: string) => ({ - name, - value: { kind: "secret-provider" as const, provider, reference }, -}); - -const named = (provider: SecretProviderId, reference: string) => - new NamedProviderSecretReference({ provider, reference }); - -const onePasswordProvider = { - kind: "1password" as const, - label: "Home", - account: OnePasswordAccount.make(HOME_ACCOUNT), -}; - -const openBaoProvider = ( - overrides: { - namespace?: string; - token?: string; - auth?: "token" | "env" | "cli"; - address?: string; - } = {}, -) => ({ - kind: "openbao" as const, - label: "Team vault", - address: BAO_ADDRESS, - mount: "kv", - auth: "token" as const, - ...overrides, -}); - -interface RecordedRequest { - readonly url: string; - readonly headers: Readonly>; -} - -/** HTTP client that answers every request with `status` and `body`, recording what it was sent. */ -const recordingHttpClient = (status: number, body: unknown) => { - const requests: Array = []; - const client = HttpClient.make((request) => { - requests.push({ url: request.url, headers: { ...request.headers } }); - return Effect.succeed( - HttpClientResponse.fromWeb( - request, - new Response(JSON.stringify(body), { - status, - headers: { "content-type": "application/json" }, - }), - ), - ); - }); - return { client, requests }; -}; - -const hostLayer = (env: NodeJS.ProcessEnv) => Layer.succeed(HostProcess.Environment, env); - -const apiRead = (reference = "agents/codex#api_key") => - decodeEnvironment([namedVariable("CODEX_TOKEN", VAULT_PROVIDER, reference)]); - -describe("ProviderSecretResolverLive with named secret providers", () => { - it.effect("reads a named 1Password provider from the account it is configured with", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "sk-home-token", stderr: "", code: 0 })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve( - decodeEnvironment([namedVariable("CODEX_TOKEN", HOME_PROVIDER, TOKEN_REFERENCE)]), - ); - - assert.deepStrictEqual(resolved, { - variables: [{ name: "CODEX_TOKEN", value: "sk-home-token", sensitive: true }], - unresolved: [], - }); - assert.deepStrictEqual(spawner.invocations, [ - ["--cache=false", "read", "--account", HOME_ACCOUNT, "--no-newline", TOKEN_REFERENCE], - ]); - }).pipe(Effect.provide(resolverLayer(spawner.layer, { [HOME_PROVIDER]: onePasswordProvider }))); - }); - - it.effect("reads an upstream one_password envelope provider like the flat 1password kind", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "sk-home-token", stderr: "", code: 0 })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve( - decodeEnvironment([ - namedVariable("CODEX_TOKEN", HOME_PROVIDER, TOKEN_REFERENCE), - namedVariable("OTHER_TOKEN", VAULT_PROVIDER, "agents/codex#api_key"), - ]), - ); - - assert.deepStrictEqual(resolved, { - variables: [{ name: "CODEX_TOKEN", value: "sk-home-token", sensitive: true }], - unresolved: ["OTHER_TOKEN"], - }); - assert.deepStrictEqual(spawner.invocations, [ - ["--cache=false", "read", "--account", HOME_ACCOUNT, "--no-newline", TOKEN_REFERENCE], - ]); - }).pipe( - Effect.provide( - resolverLayer(spawner.layer, { - [HOME_PROVIDER]: { - kind: SecretProviderKind.make("one_password"), - label: "Home", - config: { account: HOME_ACCOUNT }, - }, - [VAULT_PROVIDER]: { - kind: SecretProviderKind.make("vault_x"), - label: "Newer kind", - config: { anything: true }, - }, - }), - ), - ); - }); - - it.effect("reports an OpenBao reference without a field unresolved without a request", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "whole-secret", stderr: "", code: 0 })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve( - decodeEnvironment([ - namedVariable("NO_FIELD", VAULT_PROVIDER, "agents/codex"), - namedVariable("NO_PATH", VAULT_PROVIDER, "#api_key"), - namedVariable("FLAG_PATH", VAULT_PROVIDER, "-address=evil#api_key"), - ]), - ); - - assert.deepStrictEqual(Array.from(resolved.unresolved), ["NO_FIELD", "NO_PATH", "FLAG_PATH"]); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe( - Effect.provide( - resolverLayer( - spawner.layer, - { [VAULT_PROVIDER]: openBaoProvider({ token: "s.t" }) }, - noHttpClient, - ), - ), - ); - }); - - it.effect("reports a reference to a provider that is not configured unresolved", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "sk-token", stderr: "", code: 0 })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve( - decodeEnvironment([namedVariable("CODEX_TOKEN", VAULT_PROVIDER, "agents/codex#api_key")]), - ); - - assert.deepStrictEqual(resolved, { variables: [], unresolved: ["CODEX_TOKEN"] }); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("batches named 1Password references with legacy ones from the same account", () => { - const spawner = scriptedOpSpawner((args, template) => { - if (args.includes("inject")) { - return { - stdout: ["sk-shared", "sk-codex"].join(separatorOf(template)), - stderr: "", - code: 0, - }; - } - return { stdout: "should-not-be-read-one-at-a-time", stderr: "", code: 0 }; - }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - yield* resolver.prime([ - onePassword(TOKEN_REFERENCE, HOME_ACCOUNT), - named(HOME_PROVIDER, TOKEN_REFERENCE), - named(HOME_PROVIDER, SECOND_REFERENCE), - named(VAULT_PROVIDER, "agents/codex#api_key"), - ]); - - // OpenBao has no unlock prompt to save, so priming leaves it alone and - // the shared item is one slot in the template. - assert.strictEqual(spawner.invocations.length, 1); - assert.deepStrictEqual(Array.from(spawner.invocations[0] ?? []).slice(0, 5), [ - "--cache=false", - "inject", - "--account", - HOME_ACCOUNT, - "-i", - ]); - - const resolved = yield* resolver.resolve( - decodeEnvironment([ - onePasswordVariable("LEGACY", TOKEN_REFERENCE, HOME_ACCOUNT), - namedVariable("NAMED", HOME_PROVIDER, TOKEN_REFERENCE), - namedVariable("CODEX", HOME_PROVIDER, SECOND_REFERENCE), - ]), - ); - assert.deepStrictEqual(resolved.variables, [ - { name: "LEGACY", value: "sk-shared", sensitive: true }, - { name: "NAMED", value: "sk-shared", sensitive: true }, - { name: "CODEX", value: "sk-codex", sensitive: true }, - ]); - assert.strictEqual(spawner.invocations.length, 1); - }).pipe( - Effect.provide( - resolverLayer(spawner.layer, { - [HOME_PROVIDER]: onePasswordProvider, - [VAULT_PROVIDER]: openBaoProvider(), - }), - ), - ); - }); -}); - -describe("ProviderSecretResolverLive with OpenBao over the HTTP API", () => { - const apiProvider = openBaoProvider; - - it.effect("reads the KV v2 field with the stored token and namespace", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "", stderr: "", code: 0 })); - const http = recordingHttpClient(200, { data: { data: { api_key: "sk-api", port: 8200 } } }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve(apiRead()); - - assert.deepStrictEqual(resolved, { - variables: [{ name: "CODEX_TOKEN", value: "sk-api", sensitive: true }], - unresolved: [], - }); - assert.deepStrictEqual(http.requests, [ - { - url: `${BAO_ADDRESS}/v1/kv/data/agents/codex`, - headers: { "x-vault-token": "s.configured", "x-vault-namespace": "team" }, - }, - ]); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe( - Effect.provide( - resolverLayer( - spawner.layer, - { [VAULT_PROVIDER]: apiProvider({ namespace: "team", token: "s.configured" }) }, - http.client, - ), - ), - ); - }); - - it.effect("joins a trailing slash address and encodes unsafe path characters", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "", stderr: "", code: 0 })); - const http = recordingHttpClient(200, { data: { data: { api_key: "x" } } }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - yield* resolver.resolve(apiRead("my team/co dex?#api_key")); - - assert.deepStrictEqual( - http.requests.map((request) => [request.url, "x-vault-namespace" in request.headers]), - [[`${BAO_ADDRESS}/v1/kv/data/my%20team/co%20dex%3F`, false]], - ); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe( - Effect.provide( - resolverLayer( - spawner.layer, - { [VAULT_PROVIDER]: apiProvider({ token: "s.t", address: `${BAO_ADDRESS}/` }) }, - http.client, - ), - ), - ); - }); - - it.effect("reports a stored-token provider with no token unresolved without a request", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "", stderr: "", code: 0 })); - const http = recordingHttpClient(200, { data: { data: { api_key: "sk-api" } } }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve(apiRead()); - - assert.deepStrictEqual(resolved, { variables: [], unresolved: ["CODEX_TOKEN"] }); - assert.strictEqual(http.requests.length, 0); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe( - Effect.provide( - resolverLayer(spawner.layer, { [VAULT_PROVIDER]: apiProvider({ token: "" }) }, http.client), - ), - ); - }); - - it.effect("uses BAO_TOKEN from the server environment when auth is env", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "", stderr: "", code: 0 })); - const http = recordingHttpClient(200, { data: { data: { api_key: "sk-api" } } }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - yield* resolver.resolve(apiRead()); - - assert.deepStrictEqual( - http.requests.map((request) => request.headers["x-vault-token"]), - ["s.from-env"], - ); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe( - Effect.provide( - resolverLayer( - spawner.layer, - { [VAULT_PROVIDER]: apiProvider({ auth: "env", token: "s.ignored" }) }, - http.client, - ).pipe(Layer.provide(hostLayer({ BAO_TOKEN: "s.from-env", VAULT_TOKEN: "s.other" }))), - ), - ); - }); - - it.effect("reports auth env with BAO_TOKEN unset unresolved without a request", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "", stderr: "", code: 0 })); - const http = recordingHttpClient(200, { data: { data: { api_key: "sk-api" } } }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve(apiRead()); - - assert.deepStrictEqual(resolved, { variables: [], unresolved: ["CODEX_TOKEN"] }); - assert.strictEqual(http.requests.length, 0); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe( - Effect.provide( - resolverLayer( - spawner.layer, - { [VAULT_PROVIDER]: apiProvider({ auth: "env" }) }, - http.client, - ).pipe(Layer.provide(hostLayer({ VAULT_TOKEN: "s.other" }))), - ), - ); - }); - - it.effect("uses the output of `bao print token` when auth is cli", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "s.from-cli\n", stderr: "", code: 0 })); - const http = recordingHttpClient(200, { data: { data: { api_key: "sk-api" } } }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve(apiRead()); - - assert.deepStrictEqual(resolved, { - variables: [{ name: "CODEX_TOKEN", value: "sk-api", sensitive: true }], - unresolved: [], - }); - assert.deepStrictEqual(spawner.commands, ["bao"]); - assert.deepStrictEqual(spawner.invocations, [["print", "token"]]); - assert.deepStrictEqual(spawner.environments, [ - { - BAO_ADDR: BAO_ADDRESS, - BAO_NAMESPACE: "team", - BAO_TOKEN: undefined, - VAULT_TOKEN: undefined, - }, - ]); - assert.deepStrictEqual( - http.requests.map((request) => request.headers["x-vault-token"]), - ["s.from-cli"], - ); - }).pipe( - Effect.provide( - resolverLayer( - spawner.layer, - { [VAULT_PROVIDER]: apiProvider({ auth: "cli", namespace: "team", token: "s.ignored" }) }, - http.client, - ).pipe(Layer.provide(hostLayer({ BAO_TOKEN: "s.from-env", VAULT_TOKEN: "s.other" }))), - ), - ); - }); - - it.effect("reports a failing `bao print token` unresolved without a request", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "", stderr: "no token found", code: 2 })); - const http = recordingHttpClient(200, { data: { data: { api_key: "sk-api" } } }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve(apiRead()); - - assert.deepStrictEqual(resolved, { variables: [], unresolved: ["CODEX_TOKEN"] }); - assert.strictEqual(http.requests.length, 0); - assert.deepStrictEqual(spawner.invocations, [["print", "token"]]); - }).pipe( - Effect.provide( - resolverLayer( - spawner.layer, - { [VAULT_PROVIDER]: apiProvider({ auth: "cli" }) }, - http.client, - ), - ), - ); - }); - - it.effect("reports a 403 unresolved", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "", stderr: "", code: 0 })); - const http = recordingHttpClient(403, { errors: ["permission denied"] }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve(apiRead()); - - assert.deepStrictEqual(resolved, { variables: [], unresolved: ["CODEX_TOKEN"] }); - assert.strictEqual(http.requests.length, 1); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe( - Effect.provide( - resolverLayer( - spawner.layer, - { [VAULT_PROVIDER]: apiProvider({ token: "s.t" }) }, - http.client, - ), - ), - ); - }); - - it.effect("reports a missing or non-scalar field unresolved", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "", stderr: "", code: 0 })); - const http = recordingHttpClient(200, { data: { data: { nested: { a: 1 } } } }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve( - decodeEnvironment([ - namedVariable("MISSING", VAULT_PROVIDER, "agents/codex#api_key"), - namedVariable("OBJECT", VAULT_PROVIDER, "agents/codex#nested"), - ]), - ); - - assert.deepStrictEqual(Array.from(resolved.unresolved), ["MISSING", "OBJECT"]); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe( - Effect.provide( - resolverLayer( - spawner.layer, - { [VAULT_PROVIDER]: apiProvider({ token: "s.t" }) }, - http.client, - ), - ), - ); - }); - - it.effect("invalidating one provider keeps the other providers' secrets cached", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "sk-home-token", stderr: "", code: 0 })); - const http = recordingHttpClient(200, { data: { data: { api_key: "sk-api" } } }); - const environment = decodeEnvironment([ - onePasswordVariable("LEGACY", TOKEN_REFERENCE, HOME_ACCOUNT), - namedVariable("HOME", HOME_PROVIDER, SECOND_REFERENCE), - namedVariable("VAULT", VAULT_PROVIDER, "agents/codex#api_key"), - ]); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - yield* resolver.resolve(environment); - assert.strictEqual(spawner.invocations.length, 2); - assert.strictEqual(http.requests.length, 1); - - // An OpenBao edit must not cost a 1Password prompt, legacy or named. - yield* resolver.invalidate(new Set([VAULT_PROVIDER])); - yield* resolver.resolve(environment); - assert.strictEqual(spawner.invocations.length, 2); - assert.strictEqual(http.requests.length, 2); - - yield* resolver.invalidate(new Set([HOME_PROVIDER])); - yield* resolver.resolve(environment); - assert.strictEqual(spawner.invocations.length, 3); - assert.strictEqual(http.requests.length, 2); - }).pipe( - Effect.provide( - resolverLayer( - spawner.layer, - { [HOME_PROVIDER]: onePasswordProvider, [VAULT_PROVIDER]: apiProvider({ token: "s.t" }) }, - http.client, - ), - ), - ); - }); - - it.effect("reports an unresponsive address unresolved instead of holding the start", () => { - const spawner = scriptedOpSpawner(() => ({ stdout: "", stderr: "", code: 0 })); - const http = HttpClient.make(() => Effect.never); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const pending = yield* resolver.resolve(apiRead()).pipe(Effect.forkScoped); - yield* Effect.yieldNow; - yield* TestClock.adjust("10 seconds"); - const resolved = yield* Fiber.join(pending); - - assert.deepStrictEqual(resolved, { variables: [], unresolved: ["CODEX_TOKEN"] }); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe( - Effect.scoped, - Effect.provide( - resolverLayer(spawner.layer, { [VAULT_PROVIDER]: apiProvider({ token: "s.t" }) }, http), - ), - ); - }); -}); - -const AWS_PROVIDER = SecretProviderId.make("prod-aws"); - -const awsProvider = (overrides: { region?: string; profile?: string } = {}) => ({ - kind: "aws-secrets-manager" as const, - label: "Production", - region: "us-east-1", - ...overrides, -}); - -const awsRead = (reference: string) => - decodeEnvironment([namedVariable("OPENAI_API_KEY", AWS_PROVIDER, reference)]); - -const awsBody = (body: { SecretString?: string; SecretBinary?: string }) => - JSON.stringify({ ARN: "arn:aws:secretsmanager:us-east-1:000000000000:secret:prod", ...body }); - -describe("ProviderSecretResolverLive with AWS Secrets Manager", () => { - it.effect("reads the whole secret string in the provider's region", () => { - const spawner = scriptedOpSpawner(() => ({ - stdout: awsBody({ SecretString: "sk-aws-token" }), - stderr: "", - code: 0, - })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve(awsRead("prod/openai")); - - assert.deepStrictEqual(resolved, { - variables: [{ name: "OPENAI_API_KEY", value: "sk-aws-token", sensitive: true }], - unresolved: [], - }); - assert.deepStrictEqual(spawner.commands, ["aws"]); - assert.deepStrictEqual(spawner.invocations, [ - [ - "secretsmanager", - "get-secret-value", - "--secret-id", - "prod/openai", - "--region", - "us-east-1", - "--output", - "json", - "--no-cli-pager", - "--no-cli-auto-prompt", - ], - ]); - }).pipe(Effect.provide(resolverLayer(spawner.layer, { [AWS_PROVIDER]: awsProvider() }))); - }); - - it.effect("passes the named profile when one is set", () => { - const spawner = scriptedOpSpawner(() => ({ - stdout: awsBody({ SecretString: "sk-aws-token" }), - stderr: "", - code: 0, - })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - yield* resolver.resolve(awsRead("prod/openai")); - - assert.deepStrictEqual(spawner.invocations, [ - [ - "secretsmanager", - "get-secret-value", - "--secret-id", - "prod/openai", - "--region", - "eu-west-2", - "--profile", - "production", - "--output", - "json", - "--no-cli-pager", - "--no-cli-auto-prompt", - ], - ]); - }).pipe( - Effect.provide( - resolverLayer(spawner.layer, { - [AWS_PROVIDER]: awsProvider({ region: "eu-west-2", profile: "production" }), - }), - ), - ); - }); - - it.effect("reads one key of a JSON secret", () => { - const spawner = scriptedOpSpawner(() => ({ - stdout: awsBody({ - SecretString: JSON.stringify({ api_key: "sk-aws-key", retries: 3, strict: true }), - }), - stderr: "", - code: 0, - })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve( - decodeEnvironment([ - namedVariable("KEY", AWS_PROVIDER, "prod/openai#api_key"), - namedVariable("RETRIES", AWS_PROVIDER, "prod/openai#retries"), - namedVariable("STRICT", AWS_PROVIDER, "prod/openai#strict"), - ]), - ); - - assert.deepStrictEqual(resolved, { - variables: [ - { name: "KEY", value: "sk-aws-key", sensitive: true }, - { name: "RETRIES", value: "3", sensitive: true }, - { name: "STRICT", value: "true", sensitive: true }, - ], - unresolved: [], - }); - assert.strictEqual(spawner.invocations[0]?.[3], "prod/openai"); - }).pipe(Effect.provide(resolverLayer(spawner.layer, { [AWS_PROVIDER]: awsProvider() }))); - }); - - it.effect("reports a missing, non-scalar, or non-object key unresolved", () => { - const spawner = scriptedOpSpawner((args) => ({ - stdout: awsBody({ - SecretString: args.includes("prod/plain") - ? "not json" - : JSON.stringify({ nested: { a: 1 }, list: [1] }), - }), - stderr: "", - code: 0, - })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve( - decodeEnvironment([ - namedVariable("MISSING", AWS_PROVIDER, "prod/openai#absent"), - namedVariable("NESTED", AWS_PROVIDER, "prod/openai#nested"), - namedVariable("LIST", AWS_PROVIDER, "prod/openai#list"), - namedVariable("PLAIN", AWS_PROVIDER, "prod/plain#api_key"), - ]), - ); - - assert.deepStrictEqual(resolved, { - variables: [], - unresolved: ["MISSING", "NESTED", "LIST", "PLAIN"], - }); - }).pipe(Effect.provide(resolverLayer(spawner.layer, { [AWS_PROVIDER]: awsProvider() }))); - }); - - it.effect("reports a binary-only secret unresolved", () => { - const spawner = scriptedOpSpawner(() => ({ - stdout: awsBody({ SecretBinary: "aGVsbG8=" }), - stderr: "", - code: 0, - })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve(awsRead("prod/blob")); - - assert.deepStrictEqual(resolved, { variables: [], unresolved: ["OPENAI_API_KEY"] }); - }).pipe(Effect.provide(resolverLayer(spawner.layer, { [AWS_PROVIDER]: awsProvider() }))); - }); - - it.effect("reports a failing `aws` unresolved and logs stderr but never stdout", () => { - const logged: Array = []; - const logger = Logger.make(({ message }) => { - logged.push(JSON.stringify(message)); - }); - const spawner = scriptedOpSpawner(() => ({ - stdout: awsBody({ SecretString: "sk-must-not-be-logged" }), - stderr: "Error when retrieving token from sso: Token has expired", - code: 254, - })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve(awsRead("prod/openai")); - - assert.deepStrictEqual(resolved, { variables: [], unresolved: ["OPENAI_API_KEY"] }); - const output = logged.join("\n"); - assert.include(output, "Could not read provider secret from AWS Secrets Manager"); - assert.include(output, "Token has expired"); - assert.notInclude(output, "sk-must-not-be-logged"); - }).pipe( - Effect.provide( - Layer.mergeAll( - resolverLayer(spawner.layer, { [AWS_PROVIDER]: awsProvider() }), - Logger.layer([logger]), - ), - ), - ); - }); - - it.effect("refuses a flag-shaped secret id and an empty key without spawning", () => { - const spawner = scriptedOpSpawner(() => ({ - stdout: awsBody({ SecretString: "sk-aws-token" }), - stderr: "", - code: 0, - })); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const resolved = yield* resolver.resolve( - decodeEnvironment([ - namedVariable("FLAG", AWS_PROVIDER, "--profile=evil"), - namedVariable("FLAG_KEY", AWS_PROVIDER, "-x#api_key"), - namedVariable("EMPTY_KEY", AWS_PROVIDER, "prod/openai#"), - namedVariable("NO_ID", AWS_PROVIDER, "#api_key"), - ]), - ); - - assert.deepStrictEqual(Array.from(resolved.unresolved), [ - "FLAG", - "FLAG_KEY", - "EMPTY_KEY", - "NO_ID", - ]); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe(Effect.provide(resolverLayer(spawner.layer, { [AWS_PROVIDER]: awsProvider() }))); - }); -}); - -const sourceOf = (variable: Parameters[0][number]) => { - const [decoded] = decodeEnvironment([variable]); - const source = decoded?.value; - if (source === undefined || typeof source === "string") { - throw new Error("expected a secret source"); - } - return source; -}; - -describe("ProviderSecretResolver.check", () => { - it.effect("reports a legacy 1Password read as resolved and primes the cache", () => { - const spawner = recordingOpSpawner({ stdout: "sk-live-token", stderr: "", code: 0 }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - const variable = onePasswordVariable("TOKEN", TOKEN_REFERENCE, HOME_ACCOUNT); - - const result = yield* resolver.check(sourceOf(variable)); - - assert.deepStrictEqual(result, { _tag: "Resolved" }); - assert.strictEqual(spawner.invocations.length, 1); - const resolved = yield* resolver.resolve(decodeEnvironment([variable])); - assert.deepStrictEqual(resolved.unresolved, []); - assert.strictEqual(spawner.invocations.length, 1); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("reports the op error text when 1Password exits non-zero", () => { - const spawner = recordingOpSpawner({ - stdout: "", - stderr: "[ERROR] could not read secret: item not found\n", - code: 1, - }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const result = yield* resolver.check( - sourceOf(onePasswordVariable("TOKEN", TOKEN_REFERENCE, HOME_ACCOUNT)), - ); - - assert.deepStrictEqual(result, { - _tag: "Failed", - detail: "[ERROR] could not read secret: item not found", - }); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("reports a named provider that is not in settings", () => { - const spawner = recordingOpSpawner({ stdout: "unused", stderr: "", code: 0 }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const result = yield* resolver.check( - sourceOf(namedVariable("TOKEN", VAULT_PROVIDER, "agents/codex#api_key")), - ); - - assert.deepStrictEqual(result, { - _tag: "Failed", - detail: "Secret provider is not configured", - }); - assert.strictEqual(spawner.invocations.length, 0); - }).pipe(Effect.provide(resolverLayer(spawner.layer))); - }); - - it.effect("reports the OpenBao errors on a 403", () => { - const spawner = recordingOpSpawner({ stdout: "", stderr: "", code: 0 }); - const http = recordingHttpClient(403, { errors: ["permission denied"] }); - return Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const result = yield* resolver.check( - sourceOf(namedVariable("TOKEN", VAULT_PROVIDER, "agents/codex#api_key")), - ); - - assert.deepStrictEqual(result, { _tag: "Failed", detail: "permission denied" }); - }).pipe( - Effect.provide( - resolverLayer( - spawner.layer, - { [VAULT_PROVIDER]: openBaoProvider({ token: "s.t" }) }, - http.client, - ), - ), - ); - }); - - it.effect("fails without a secret store on the default resolver", () => - Effect.gen(function* () { - const resolver = yield* ProviderSecretResolver; - - const result = yield* resolver.check( - sourceOf(onePasswordVariable("TOKEN", TOKEN_REFERENCE, HOME_ACCOUNT)), - ); - - assert.deepStrictEqual(result, { - _tag: "Failed", - detail: "Secret stores are not available on this server", - }); - }), - ); -}); diff --git a/apps/server/src/provider/ProviderSecretResolverLive.ts b/apps/server/src/provider/ProviderSecretResolverLive.ts deleted file mode 100644 index 60645ccfd6659..0000000000000 --- a/apps/server/src/provider/ProviderSecretResolverLive.ts +++ /dev/null @@ -1,757 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off -- the op inject separator only needs a random token at this subprocess boundary -/** - * Reads each secret store through its HTTP API or CLI. Providers are looked up in - * `settings.secretProviders` at read time. Failures are cached so a locked vault does not - * re-prompt; recovery is the refresh button. - * @module provider/ProviderSecretResolverLive - */ -import { - type AwsSecretsManagerSecretProviderConfig, - OnePasswordAccount, - OnePasswordAccountSummary, - OnePasswordSecretProviderConfig, - isFlatSecretProviderConfig, - type OpenBaoSecretProviderConfig, - type SecretProviderId, -} from "@t3tools/contracts"; -import * as Cache from "effect/Cache"; -import * as Duration from "effect/Duration"; -import * as Effect from "effect/Effect"; -import * as Exit from "effect/Exit"; -import * as Layer from "effect/Layer"; -import * as FileSystem from "effect/FileSystem"; -import * as Option from "effect/Option"; -import * as Schema from "effect/Schema"; -import * as NodeCrypto from "node:crypto"; -import * as HostProcess from "@t3tools/shared/HostProcess"; -import { HttpClient, HttpClientRequest } from "effect/http"; -import { ChildProcess, ChildProcessSpawner } from "effect/process"; -import { resolveSpawnCommand } from "@t3tools/shared/shell"; - -import type { ResolvedProviderEnvironmentVariable } from "@t3tools/provider-core/server/instanceEnvironment"; -import { - OnePasswordSecretReference, - providerSecretReference, - type NamedProviderSecretReference, - type ProviderSecretReference, -} from "./ProviderSecretReference.ts"; -import * as Settings from "../serverSettings.ts"; -import { spawnAndCollect } from "@t3tools/provider-core/server/snapshotProbe"; -import { - ProviderSecretResolver, - type ProviderSecretResolverShape, -} from "./ProviderSecretResolver.ts"; - -/** - * What one read of a secret store came back with. The failure carries the - * store's own reason so `check` can show it; `resolve` only needs to know - * whether there was a value. Details are the store's error text and never - * include a secret. - */ -type SecretRead = - | { readonly _tag: "Read"; readonly value: string } - | { readonly _tag: "Unreadable"; readonly detail: string }; - -const readValue = (value: string): SecretRead => ({ _tag: "Read", value }); -const unreadable = (detail: string): SecretRead => ({ _tag: "Unreadable", detail }); -const readNonEmpty = (value: string | undefined, emptyDetail: string) => - value === undefined || value.length === 0 ? unreadable(emptyDetail) : readValue(value); -const exitDetail = (result: { readonly code: number; readonly stderr: string }) => { - const stderr = result.stderr.trim(); - return stderr.length > 0 ? stderr : `Exited with code ${result.code}`; -}; - -const ONE_PASSWORD_BINARY = "op"; -const OPENBAO_BINARY = "bao"; -const AWS_BINARY = "aws"; - -/** - * Every `op` call skips its cache. The cache lives in an `op daemon` that `op` - * spawns on its own, which hangs on macOS permission prompts nobody can see - * from a background server and leaks zombie processes on Linux. Reads are - * already held in memory here, so the daemon only adds risk. - */ -const resolveOnePasswordCommand = (args: ReadonlyArray) => - resolveSpawnCommand(ONE_PASSWORD_BINARY, ["--cache=false", ...args]); - -/** - * Bound on a single `op read`. Long enough for a user to reach for the - * fingerprint reader, short enough that a vault that will never answer does - * not wedge the instance rebuild that is waiting on it. - */ -const SECRET_READ_TIMEOUT = Duration.seconds(45); -/** OpenBao never prompts, so an unreachable address should not hold a start for long. */ -const OPENBAO_REQUEST_TIMEOUT = Duration.seconds(10); - -/** - * References are small and few - one per provider credential. The cap only - * exists so a settings file that somehow accumulates hundreds cannot pin - * every secret it ever mentioned in memory. - */ -const SECRET_CACHE_CAPACITY = 64; - -/** - * Read many references from one account in one `op inject`, via a scoped temp file because `op` - * ignores piped stdin from Node. The separator is random per call so a secret cannot shift later - * values. Returns `undefined` when the batch fails as a whole; the caller reads one at a time. - */ -const readSecretsTogether = Effect.fn("readSecretsTogether")(function* ( - account: OnePasswordAccount, - references: ReadonlyArray, -) { - const fileSystem = yield* FileSystem.FileSystem; - const separator = `__t3-secret-${NodeCrypto.randomUUID()}__`; - const template = references.map((reference) => `{{ ${reference} }}`).join(separator); - const templatePath = yield* fileSystem.makeTempFileScoped({ prefix: "t3code-op-inject-" }); - yield* fileSystem.writeFileString(templatePath, template); - const spawnCommand = yield* resolveOnePasswordCommand([ - "inject", - "--account", - account, - "-i", - templatePath, - ]); - const result = yield* spawnAndCollect( - ONE_PASSWORD_BINARY, - ChildProcess.make(spawnCommand.command, spawnCommand.args, { - shell: spawnCommand.shell, - }), - ); - if (result.code !== 0) { - // `op` names the reference it could not resolve on stderr and never echoes - // a secret, so this is safe to log verbatim. - yield* Effect.logWarning("Could not batch-read provider secrets from 1Password", { - account, - references: references.length, - exitCode: result.code, - detail: result.stderr.trim(), - }); - return undefined; - } - const values = result.stdout.split(separator); - if (values.length !== references.length) { - yield* Effect.logWarning("1Password returned an unexpected number of provider secrets", { - expected: references.length, - received: values.length, - }); - return undefined; - } - return values.map((value) => { - const secret = value.trim(); - return secret.length > 0 ? secret : undefined; - }); -}); - -const readOnePasswordSecret = Effect.fn("readOnePasswordSecret")(function* ({ - reference, - account, -}: OnePasswordSecretReference) { - const spawnCommand = yield* resolveOnePasswordCommand([ - "read", - "--account", - account, - "--no-newline", - reference, - ]); - const result = yield* spawnAndCollect( - ONE_PASSWORD_BINARY, - ChildProcess.make(spawnCommand.command, spawnCommand.args, { shell: spawnCommand.shell }), - ); - if (result.code !== 0) { - // `op` reports "not signed in", "item not found", and friends on stderr - // and never echoes the secret itself, so this is safe to log verbatim. - yield* Effect.logWarning("Could not read provider secret from 1Password", { - reference, - account, - exitCode: result.code, - detail: result.stderr.trim(), - }); - return unreadable(exitDetail(result)); - } - return readNonEmpty(result.stdout.trim(), "1Password returned an empty value"); -}); - -const OnePasswordAccountListJson = Schema.fromJsonString( - Schema.Array(Schema.Struct({ url: OnePasswordAccount, email: Schema.String })), -); -const decodeOnePasswordAccountList = Schema.decodeUnknownEffect(OnePasswordAccountListJson); - -const readOnePasswordAccounts = Effect.fn("readOnePasswordAccounts")(function* () { - const spawnCommand = yield* resolveOnePasswordCommand(["account", "list", "--format", "json"]); - const result = yield* spawnAndCollect( - ONE_PASSWORD_BINARY, - ChildProcess.make(spawnCommand.command, spawnCommand.args, { shell: spawnCommand.shell }), - ); - if (result.code !== 0) { - yield* Effect.logWarning("Could not list 1Password accounts", { - exitCode: result.code, - detail: result.stderr.trim(), - }); - return []; - } - const accounts = yield* decodeOnePasswordAccountList(result.stdout); - return accounts.map(({ url, email }): OnePasswordAccountSummary => ({ account: url, email })); -}); - -/** - * Split an OpenBao reference into the secret's path inside the mount and the - * key to print. The path is handed to `bao` as a positional argument, so one - * starting with `-` would be read as a flag and is refused. - */ -function parseOpenBaoReference(reference: string) { - const separator = reference.indexOf("#"); - if (separator === -1) { - return undefined; - } - const path = reference.slice(0, separator); - const field = reference.slice(separator + 1); - if (path.length === 0 || field.length === 0 || path.startsWith("-")) { - return undefined; - } - return { path, field }; -} - -const OpenBaoErrorBody = Schema.Struct({ errors: Schema.optional(Schema.Array(Schema.String)) }); -const OpenBaoSecretBody = Schema.Struct({ - data: Schema.Struct({ data: Schema.Record(Schema.String, Schema.Unknown) }), -}); -const decodeOpenBaoErrorBody = Schema.decodeUnknownOption(OpenBaoErrorBody); -const decodeOpenBaoSecretBody = Schema.decodeUnknownOption(OpenBaoSecretBody); - -const nonEmpty = (value: string | undefined) => - value === undefined || value.length === 0 ? undefined : value; - -/** - * The token from the source the provider's `auth` names, with no fallback. `cli` asks - * `bao print token` with token variables withheld so it reports its own login. Returns the - * logged reason when that source has no token. - */ -const resolveOpenBaoToken = Effect.fn("resolveOpenBaoToken")(function* ( - provider: SecretProviderId, - config: OpenBaoSecretProviderConfig, -) { - switch (config.auth) { - case "token": { - const token = nonEmpty(config.token); - if (token === undefined) { - yield* Effect.logWarning("OpenBao provider has no stored token", { provider }); - return unreadable("OpenBao provider has no stored token"); - } - return readValue(token); - } - case "env": { - const env = yield* HostProcess.Environment; - const token = nonEmpty(env.BAO_TOKEN); - if (token === undefined) { - yield* Effect.logWarning("BAO_TOKEN is not set in the server environment", { provider }); - return unreadable("BAO_TOKEN is not set in the server environment"); - } - return readValue(token); - } - case "cli": { - const spawnCommand = yield* resolveSpawnCommand(OPENBAO_BINARY, ["print", "token"]); - const result = yield* spawnAndCollect( - OPENBAO_BINARY, - ChildProcess.make(spawnCommand.command, spawnCommand.args, { - shell: spawnCommand.shell, - env: { - BAO_ADDR: config.address, - ...(config.namespace === undefined ? {} : { BAO_NAMESPACE: config.namespace }), - // `bao` prefers these over its saved login, which would turn the - // CLI policy back into the environment one. Node drops a key whose - // value is undefined, so the child never sees them. - BAO_TOKEN: undefined, - VAULT_TOKEN: undefined, - }, - extendEnv: true, - }), - ); - const token = result.code === 0 ? nonEmpty(result.stdout.trim()) : undefined; - if (token === undefined) { - // `bao print token` reports a missing login on stderr and never echoes - // the token there, so this is safe to log verbatim. - yield* Effect.logWarning("Could not read an OpenBao token from the bao CLI", { - provider, - exitCode: result.code, - detail: result.stderr.trim(), - }); - return unreadable(exitDetail(result)); - } - return readValue(token); - } - } -}); - -/** Percent-encode each path segment and keep `/` as the separator. */ -const encodeOpenBaoPath = (path: string) => path.split("/").map(encodeURIComponent).join("/"); - -const readOpenBaoSecretOverApi = Effect.fn("readOpenBaoSecretOverApi")( - function* ( - provider: SecretProviderId, - config: OpenBaoSecretProviderConfig, - location: { readonly path: string; readonly field: string }, - reference: string, - ) { - const token = yield* resolveOpenBaoToken(provider, config); - if (token._tag === "Unreadable") { - return token; - } - const client = yield* HttpClient.HttpClient; - const url = `${config.address.replace(/\/+$/, "")}/v1/${encodeOpenBaoPath(config.mount)}/data/${encodeOpenBaoPath(location.path)}`; - const request = HttpClientRequest.get(url).pipe( - HttpClientRequest.setHeaders({ - "X-Vault-Token": token.value, - ...(config.namespace === undefined || config.namespace.length === 0 - ? {} - : { "X-Vault-Namespace": config.namespace }), - }), - ); - // Trace headers would tell the vault which of our spans made the request. - const response = yield* client - .execute(request) - .pipe( - Effect.provideService(HttpClient.TracerPropagationEnabled, false), - Effect.timeout(OPENBAO_REQUEST_TIMEOUT), - ); - const body: unknown = yield* response.json.pipe(Effect.orElseSucceed(() => undefined)); - if (response.status < 200 || response.status >= 300) { - // OpenBao's `errors` array names the failure and never carries a secret. - const errors = decodeOpenBaoErrorBody(body).pipe( - Option.flatMap((decoded) => Option.fromNullishOr(decoded.errors)), - Option.getOrElse((): ReadonlyArray => []), - ); - yield* Effect.logWarning("Could not read provider secret from OpenBao", { - provider, - reference, - status: response.status, - errorCount: errors.length, - }); - return unreadable(errors.length > 0 ? errors.join("; ") : `HTTP ${response.status}`); - } - const value = decodeOpenBaoSecretBody(body).pipe( - Option.map((decoded) => decoded.data.data[location.field]), - Option.getOrUndefined, - ); - const secret = - typeof value === "string" - ? value - : typeof value === "number" || typeof value === "boolean" - ? String(value) - : undefined; - if (secret === undefined || secret.length === 0) { - yield* Effect.logWarning("OpenBao secret has no usable value for the requested field", { - provider, - reference, - status: response.status, - }); - return unreadable("OpenBao secret has no usable value for the requested field"); - } - return readValue(secret); - }, - (effect, provider, _config, _location, reference) => - effect.pipe( - Effect.catch((error) => - Effect.logWarning("Could not read provider secret from OpenBao", { - provider, - reference, - detail: String(error), - }).pipe(Effect.as(unreadable(String(error)))), - ), - ), -); - -const readOpenBaoSecret = Effect.fn("readOpenBaoSecret")(function* ( - provider: SecretProviderId, - config: OpenBaoSecretProviderConfig, - reference: string, -) { - const location = parseOpenBaoReference(reference); - if (location === undefined) { - yield* Effect.logWarning("OpenBao secret reference must be path/to/secret#field", { - provider, - reference, - }); - return unreadable("OpenBao secret reference must be path/to/secret#field"); - } - return yield* readOpenBaoSecretOverApi(provider, config, location, reference); -}); - -/** - * Split an AWS Secrets Manager reference into the secret id and the optional - * JSON key. The id is handed to `aws` as the value of `--secret-id`, so one - * starting with `-` would be read as a flag and is refused. A secret name - * cannot contain `#`, so the first one always starts the key. - */ -function parseAwsReference(reference: string) { - const separator = reference.indexOf("#"); - const secretId = separator === -1 ? reference : reference.slice(0, separator); - const key = separator === -1 ? undefined : reference.slice(separator + 1); - if (secretId.length === 0 || secretId.startsWith("-") || key === "") { - return undefined; - } - return { secretId, key }; -} - -const AwsSecretValueJson = Schema.fromJsonString( - Schema.Struct({ - SecretString: Schema.optional(Schema.String), - SecretBinary: Schema.optional(Schema.String), - }), -); -const decodeAwsSecretValue = Schema.decodeUnknownOption(AwsSecretValueJson); -const decodeJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)); - -const readAwsSecret = Effect.fn("readAwsSecret")(function* ( - provider: SecretProviderId, - config: AwsSecretsManagerSecretProviderConfig, - reference: string, -) { - const location = parseAwsReference(reference); - if (location === undefined) { - yield* Effect.logWarning( - "AWS Secrets Manager reference must be name-or-arn or name-or-arn#key", - { provider, reference }, - ); - return unreadable("AWS Secrets Manager reference must be name-or-arn or name-or-arn#key"); - } - const spawnCommand = yield* resolveSpawnCommand(AWS_BINARY, [ - "secretsmanager", - "get-secret-value", - "--secret-id", - location.secretId, - "--region", - config.region, - ...(config.profile === undefined ? [] : ["--profile", config.profile]), - "--output", - "json", - "--no-cli-pager", - "--no-cli-auto-prompt", - ]); - const result = yield* spawnAndCollect( - AWS_BINARY, - ChildProcess.make(spawnCommand.command, spawnCommand.args, { shell: spawnCommand.shell }), - ); - if (result.code !== 0) { - // `aws` reports missing credentials, expired SSO sessions, and - // ResourceNotFound on stderr and never echoes the secret, so this is safe - // to log verbatim. Stdout carries the secret and is never logged. - yield* Effect.logWarning("Could not read provider secret from AWS Secrets Manager", { - provider, - reference, - exitCode: result.code, - detail: result.stderr.trim(), - }); - return unreadable(exitDetail(result)); - } - const body = Option.getOrUndefined(decodeAwsSecretValue(result.stdout)); - if (body === undefined) { - yield* Effect.logWarning("AWS Secrets Manager returned an unexpected response", { - provider, - reference, - }); - return unreadable("AWS Secrets Manager returned an unexpected response"); - } - if (body.SecretString === undefined) { - yield* Effect.logWarning("AWS Secrets Manager binary secrets are not supported", { - provider, - reference, - }); - return unreadable("AWS Secrets Manager binary secrets are not supported"); - } - if (location.key === undefined) { - return readNonEmpty(body.SecretString, "The secret string is empty"); - } - const parsed = Option.getOrUndefined(decodeJson(body.SecretString)); - const value = - typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) - ? Reflect.get(parsed, location.key) - : undefined; - if (typeof value === "string") { - return readNonEmpty(value, "The value for the key is empty"); - } - if (typeof value === "number" || typeof value === "boolean") { - return readValue(String(value)); - } - yield* Effect.logWarning("AWS Secrets Manager secret has no scalar value for the key", { - provider, - reference, - }); - return unreadable("The secret has no scalar value for the key"); -}); - -/** Log fields that identify a secret without carrying its value. */ -const describeSecret = (secret: ProviderSecretReference) => - secret._tag === "1password" - ? { reference: secret.reference, account: secret.account } - : { reference: secret.reference, provider: secret.provider }; - -// Fork follow-up: fold the flat `1password` kind into this envelope once upstream resolves secrets itself. -const decodeOnePasswordEnvelopeConfig = Schema.decodeUnknownOption(OnePasswordSecretProviderConfig); - -/** The account of an upstream `one_password` envelope, or `undefined` for any other kind or an undecodable config. */ -const envelopeOnePasswordAccount = (entry: { readonly kind: string; readonly config?: unknown }) => - entry.kind === "one_password" - ? Option.getOrUndefined( - Option.map(decodeOnePasswordEnvelopeConfig(entry.config), (config) => config.account), - ) - : undefined; - -export const ProviderSecretResolverLive = Layer.effect( - ProviderSecretResolver, - Effect.gen(function* () { - // The service tag declares `prime` as `Effect`, so the spawner and - // the filesystem it needs are captured here rather than asked of the - // caller, the same way the cache's own lookup captures the spawner. - const primeContext = yield* Effect.context< - ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem - >(); - const serverSettings = yield* Settings.ServerSettingsService; - - const secretProviderConfig = (provider: SecretProviderId) => - serverSettings.getSettings.pipe( - Effect.map((settings) => settings.secretProviders[provider]), - Effect.orElseSucceed(() => undefined), - ); - - const readNamedProviderSecret = Effect.fn("readNamedProviderSecret")(function* ({ - provider, - reference, - }: NamedProviderSecretReference) { - const config = yield* secretProviderConfig(provider); - if (config === undefined) { - yield* Effect.logWarning("Provider secret names a secret provider that is not configured", { - provider, - reference, - }); - return unreadable("Secret provider is not configured"); - } - if (!isFlatSecretProviderConfig(config)) { - const account = envelopeOnePasswordAccount(config); - if (account === undefined) { - yield* Effect.logWarning( - "Provider secret names a secret provider kind that is not supported", - { - provider, - kind: config.kind, - }, - ); - return unreadable("Secret provider kind is not supported"); - } - return yield* readOnePasswordSecret(new OnePasswordSecretReference({ reference, account })); - } - switch (config.kind) { - case "1password": - return yield* readOnePasswordSecret( - new OnePasswordSecretReference({ reference, account: config.account }), - ); - case "openbao": - return yield* readOpenBaoSecret(provider, config, reference); - case "aws-secrets-manager": - return yield* readAwsSecret(provider, config, reference); - } - }); - - const readSecret = (secret: ProviderSecretReference) => { - switch (secret._tag) { - case "1password": - return readOnePasswordSecret(secret); - case "secret-provider": - return readNamedProviderSecret(secret); - } - }; - - const readContext = - yield* Effect.context>>(); - - const readSecretBounded = (reference: ProviderSecretReference) => - readSecret(reference).pipe( - Effect.timeoutOption(SECRET_READ_TIMEOUT), - Effect.map(Option.getOrElse(() => unreadable("Timed out waiting for the secret store"))), - Effect.catch((error) => - Effect.logWarning("Could not run the secret store CLI to read a provider secret", { - ...describeSecret(reference), - detail: String(error), - }).pipe(Effect.as(unreadable(String(error)))), - ), - ); - - const cache = yield* Cache.makeWith( - (reference: ProviderSecretReference) => - readSecretBounded(reference).pipe( - Effect.map((read) => (read._tag === "Read" ? read.value : undefined)), - ), - { - capacity: SECRET_CACHE_CAPACITY, - // No time to live for a value: it is held until the user asks for a provider refresh. - // Expiring on a timer would reintroduce the surprise biometric prompt mid-session that the - // cache exists to remove. An unreadable secret is never held, so unlocking the store is - // enough for the next resolve to succeed. - timeToLive: (exit) => - Exit.isSuccess(exit) && exit.value !== undefined ? Duration.infinity : Duration.zero, - }, - ); - - const check: ProviderSecretResolverShape["check"] = Effect.fn("ProviderSecretResolver.check")( - function* (source) { - const reference = providerSecretReference(source); - if (reference === undefined) { - return { _tag: "Failed", detail: "This variable does not name a secret" } as const; - } - const read = yield* readSecretBounded(reference).pipe(Effect.provideContext(readContext)); - if (read._tag === "Unreadable") { - return { _tag: "Failed", detail: read.detail } as const; - } - yield* Cache.set(cache, reference, read.value); - return { _tag: "Resolved" } as const; - }, - ); - - const resolve: ProviderSecretResolverShape["resolve"] = Effect.fn( - "ProviderSecretResolver.resolve", - )(function* (environment) { - const resolved: Array = []; - const unresolved: Array = []; - for (const { name, value, sensitive } of environment ?? []) { - const reference = providerSecretReference(value); - if (reference === undefined) { - if (typeof value === "string") { - resolved.push({ name, value, sensitive }); - } - continue; - } - const secret = yield* Cache.get(cache, reference); - if (secret === undefined) { - unresolved.push(name); - continue; - } - resolved.push({ name, value: secret, sensitive: true }); - } - return { variables: resolved, unresolved }; - }); - - /** - * `wanted` maps each `op://` reference to every cache key that reads it, so - * a legacy source and a named 1Password provider pointing at the same item - * in the same account share one slot in the batch. - */ - const primeOnePassword = Effect.fn("primeOnePassword")(function* ( - account: OnePasswordAccount, - wanted: ReadonlyMap>, - ) { - // One reference costs one prompt whichever command reads it, so there - // is nothing to save and `op read` gives the better error. - if (wanted.size < 2) { - return; - } - const references = Array.from(wanted.keys()); - const values = yield* readSecretsTogether(account, references).pipe( - Effect.scoped, - Effect.timeoutOption(SECRET_READ_TIMEOUT), - Effect.map(Option.getOrUndefined), - Effect.catch((error) => - Effect.logWarning("Could not run 1Password to batch-read provider secrets", { - account, - references: references.length, - detail: String(error), - }).pipe(Effect.as(undefined)), - ), - ); - if (values === undefined) { - return; - } - yield* Effect.forEach( - references, - (reference, index) => - Effect.forEach( - wanted.get(reference) ?? [], - (key) => Cache.set(cache, key, values[index]), - { - discard: true, - }, - ), - { discard: true }, - ); - }); - - /** - * The 1Password account a reference reads from, or `undefined` when it is - * not a 1Password read. OpenBao has no unlock prompt to amortize, so its - * references are left for `resolve` to read one at a time. - */ - const onePasswordAccountOf = (reference: ProviderSecretReference) => { - switch (reference._tag) { - case "1password": - return Effect.succeed(reference.account); - case "secret-provider": - return secretProviderConfig(reference.provider).pipe( - Effect.map((config) => { - if (config === undefined) return undefined; - if (!isFlatSecretProviderConfig(config)) return envelopeOnePasswordAccount(config); - return config.kind === "1password" ? config.account : undefined; - }), - ); - } - }; - - const prime: ProviderSecretResolverShape["prime"] = Effect.fn("ProviderSecretResolver.prime")( - function* (references) { - // `op inject` reads every reference in its template from one account, - // so a batch is one call per account. - const onePasswordByAccount = new Map< - OnePasswordAccount, - Map> - >(); - for (const reference of references) { - if (yield* Cache.has(cache, reference)) { - continue; - } - const account = yield* onePasswordAccountOf(reference); - if (account === undefined) { - continue; - } - const group = onePasswordByAccount.get(account) ?? new Map(); - const keys = group.get(reference.reference) ?? []; - keys.push(reference); - group.set(reference.reference, keys); - onePasswordByAccount.set(account, group); - } - yield* Effect.forEach( - onePasswordByAccount, - ([account, wanted]) => primeOnePassword(account, wanted), - { discard: true }, - ); - }, - Effect.provideContext(primeContext), - ); - - const listOnePasswordAccounts: ProviderSecretResolverShape["listOnePasswordAccounts"] = - readOnePasswordAccounts().pipe( - Effect.timeoutOption(SECRET_READ_TIMEOUT), - Effect.map(Option.getOrElse(() => [])), - Effect.catch((error) => - Effect.logWarning("Could not run 1Password to list accounts", { - detail: String(error), - }).pipe(Effect.as([])), - ), - Effect.provideContext(primeContext), - ); - - const invalidate: ProviderSecretResolverShape["invalidate"] = (providers) => - providers === undefined - ? Cache.invalidateAll(cache) - : Effect.gen(function* () { - const stale = Array.from(yield* Cache.keys(cache)).filter( - (key) => key._tag === "secret-provider" && providers.has(key.provider), - ); - yield* Effect.forEach(stale, (key) => Cache.invalidate(cache, key), { discard: true }); - }); - - return { - resolve, - prime, - invalidate, - check, - listOnePasswordAccounts, - }; - }), -); diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index d5496ea8cfc57..409a917b9c81a 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5135, "codexAdditionalContextBytes": 6865, - "mcpToolListJsonBytes": 45907 + "mcpToolListJsonBytes": 46149 } diff --git a/apps/server/src/provider/providerCompatibility.test.ts b/apps/server/src/provider/providerCompatibility.test.ts index 013657ada8e7d..8575a73230f85 100644 --- a/apps/server/src/provider/providerCompatibility.test.ts +++ b/apps/server/src/provider/providerCompatibility.test.ts @@ -13,6 +13,7 @@ import * as ModelManifest from "./ModelManifest.ts"; import * as ProviderRegistry from "./ProviderRegistry.ts"; import * as ProviderLatestVersions from "@t3tools/provider-core/server/ProviderLatestVersions"; import * as ProviderInstanceRegistry from "./ProviderInstanceRegistry.ts"; +import * as ProviderInstanceRegistryMutator from "./ProviderInstanceRegistryMutator.ts"; import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; import { makeManualOnlyProviderMaintenanceCapabilities } from "@t3tools/provider-core/server/maintenanceResolver"; import * as Schema from "effect/Schema"; @@ -286,8 +287,6 @@ it.effect("a remote policy refresh preserves a newer health result on the regist listUnavailable: Effect.succeed([]), streamChanges: Stream.empty, subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), - rebuildInstanceWhen: () => Effect.succeed(false), - listEnvironments: Effect.succeed(new Map()), }), ServerConfig.layerTest(process.cwd(), { prefix: "compatibility-registry-test" }).pipe( Layer.provideMerge(NodeServices.layer), @@ -314,6 +313,15 @@ it.effect("a remote policy refresh preserves a newer health result on the regist assert.strictEqual(updated?.compatibilityAdvisory?.status, "supported"); assert.strictEqual(updated?.status, "error"); assert.strictEqual(updated?.message, "Authentication failed"); - }).pipe(Effect.provide(ProviderRegistry.layer.pipe(Layer.provide(layerDependencies)))); + }).pipe( + Effect.provide( + ProviderRegistry.layer.pipe( + Layer.provide(layerDependencies), + Layer.provide( + Layer.mock(ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator)({}), + ), + ), + ), + ); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index d8f6e1b7e5942..943928c4e18cf 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -192,8 +192,8 @@ function makeRegistry( getProviders: Ref.get(providersRef), refresh: () => Ref.get(providersRef), refreshInstance: () => Ref.get(providersRef), + reloadSecretsAndRefresh: () => Ref.get(providersRef), refreshAfterUpdate: () => Ref.get(providersRef), - checkSecretSource: () => Effect.succeed({ _tag: "Resolved" }), refreshWorkspaceSnapshot: () => Ref.get(providersRef), getProviderMaintenanceCapabilitiesForInstance: (_instanceId, provider) => Effect.succeed(lifecycleFor(provider)), diff --git a/apps/server/src/provider/testUtils/providerRegistryMock.ts b/apps/server/src/provider/testUtils/providerRegistryMock.ts index e37aab7ba9e85..a4008acc18efb 100644 --- a/apps/server/src/provider/testUtils/providerRegistryMock.ts +++ b/apps/server/src/provider/testUtils/providerRegistryMock.ts @@ -11,12 +11,12 @@ const makeProviderRegistryMock = ( getProviders: Effect.succeed(providers), refresh: () => Effect.succeed(providers), refreshInstance: () => Effect.succeed(providers), + reloadSecretsAndRefresh: () => Effect.succeed(providers), refreshAfterUpdate: () => Effect.succeed(providers), refreshWorkspaceSnapshot: () => Effect.succeed(providers), getProviderMaintenanceCapabilitiesForInstance: (_instanceId, provider) => Effect.succeed(makeManualOnlyProviderMaintenanceCapabilities({ provider, packageName: null })), setProviderMaintenanceActionState: () => Effect.succeed(providers), - checkSecretSource: () => Effect.succeed({ _tag: "Resolved" }), streamChanges: Stream.empty, }); diff --git a/apps/server/src/pullRequest/PullRequestService.test.ts b/apps/server/src/pullRequest/PullRequestService.test.ts index faa7320a081be..0f233030f4a16 100644 --- a/apps/server/src/pullRequest/PullRequestService.test.ts +++ b/apps/server/src/pullRequest/PullRequestService.test.ts @@ -2320,6 +2320,98 @@ it.effect("does not revive old summaries when project epochs are evicted", () => }), ); +it.effect("a refresh or a finished turn drops the provider's own change request reads", () => + Effect.gen(function* () { + const forgotten: Array<{ readonly repository: string; readonly number?: number } | undefined> = + []; + const service = yield* makeService({ + projects: [ + project({ + id: "p1", + title: "web", + workspaceRoot: "/a", + repository: "acme/web", + provider: "gitlab", + }), + ], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("gitlab"), { + forgetChangeRequestReads: (scope) => Effect.sync(() => void forgotten.push(scope)), + }), + ], + }); + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + + yield* service.invalidate({ reference }); + yield* service.refreshAfterTurn(reference.projectId); + yield* service.invalidate({}); + + assert.deepStrictEqual(forgotten, [ + { repository: "acme/web", number: 1 }, + undefined, + undefined, + ]); + }), +); + +it.effect("a read during a refresh cannot keep the provider's old change request", () => + Effect.gen(function* () { + let hostTitle = "old"; + let held: string | undefined; + let gate: + | { readonly entered: Deferred.Deferred; readonly release: Deferred.Deferred } + | undefined; + const service = yield* makeService({ + projects: [ + project({ + id: "p1", + title: "web", + workspaceRoot: "/a", + repository: "acme/web", + provider: "gitlab", + }), + ], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("gitlab"), { + getChangeRequest: () => + Effect.sync(() => { + held ??= hostTitle; + return { ...hostedChangeRequest("body"), title: held }; + }), + forgetChangeRequestReads: () => + Effect.gen(function* () { + if (gate !== undefined) { + yield* Deferred.succeed(gate.entered, undefined); + yield* Deferred.await(gate.release); + } + held = undefined; + }), + }), + ], + }); + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const refreshWithReadInBetween = (refresh: Effect.Effect) => + Effect.gen(function* () { + gate = { entered: yield* Deferred.make(), release: yield* Deferred.make() }; + const running = yield* refresh.pipe(Effect.forkChild()); + yield* Deferred.await(gate.entered); + yield* service.detail(reference); + yield* Deferred.succeed(gate.release, undefined); + yield* Fiber.join(running); + gate = undefined; + }); + + assert.strictEqual((yield* service.detail(reference)).title, "old"); + hostTitle = "after turn"; + yield* refreshWithReadInBetween(service.refreshAfterTurn(reference.projectId)); + assert.strictEqual((yield* service.detail(reference)).title, "after turn"); + + hostTitle = "after refresh"; + yield* refreshWithReadInBetween(service.invalidate({ reference })); + assert.strictEqual((yield* service.detail(reference)).title, "after refresh"); + }), +); + it.effect("close and reopen notify subscribed readers after invalidating their cached state", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/pullRequest/PullRequestService.ts b/apps/server/src/pullRequest/PullRequestService.ts index 94bc0f8ef0f4f..4e7d670245ddc 100644 --- a/apps/server/src/pullRequest/PullRequestService.ts +++ b/apps/server/src/pullRequest/PullRequestService.ts @@ -533,6 +533,9 @@ function withRateLimitBackoff( listChangeRequestStats: wrap("listChangeRequestStats", api.listChangeRequestStats), }), getChangeRequest: wrap("getChangeRequest", api.getChangeRequest), + ...(api.forgetChangeRequestReads === undefined + ? {} + : { forgetChangeRequestReads: api.forgetChangeRequestReads }), ...(api.getChangeRequestPreview === undefined ? {} : { getChangeRequestPreview: wrap("getChangeRequestPreview", api.getChangeRequestPreview) }), @@ -3457,6 +3460,13 @@ export const make = Effect.gen(function* () { return { stats: [...held, ...result.stats] }; }); + const forgetProviderReads = (scope?: { readonly repository: string; readonly number?: number }) => + Effect.forEach( + registry.kinds, + (kind) => registry.get(kind)?.forgetChangeRequestReads?.(scope) ?? Effect.void, + { discard: true }, + ); + const invalidate: PullRequestService["Service"]["invalidate"] = Effect.fn( "PullRequestService.invalidate", )(function* (input, options) { @@ -3472,13 +3482,15 @@ export const make = Effect.gen(function* () { if (reference !== undefined) { yield* canonicalRef(reference).pipe( Effect.flatMap((ref) => - readCache - .invalidate(refScope(ref)) - .pipe(Effect.andThen(Effect.sync(() => bumpRefEpoch(ref)))), + forgetProviderReads({ repository: ref.repository, number: ref.number }).pipe( + Effect.andThen(readCache.invalidate(refScope(ref))), + Effect.andThen(Effect.sync(() => bumpRefEpoch(ref))), + ), ), Effect.ignore, ); } else { + yield* forgetProviderReads(); listingsEpoch = ++epochCounter; everyFileRevisionEpoch = ++epochCounter; viewersByHost.clear(); @@ -3489,22 +3501,27 @@ export const make = Effect.gen(function* () { } }); - const refreshAfterTurn: PullRequestService["Service"]["refreshAfterTurn"] = (projectId) => - Effect.suspend(() => { - listingsEpoch = ++epochCounter; - projectEpochs.delete(projectId); - if (projectEpochs.size >= REF_EPOCH_CAPACITY) { - const oldest = projectEpochs.keys().next().value; - if (oldest !== undefined) { - projectEpochFloor = projectEpochs.get(oldest)!; - projectEpochs.delete(oldest); - } + const refreshAfterTurn: PullRequestService["Service"]["refreshAfterTurn"] = Effect.fn( + "PullRequestService.refreshAfterTurn", + )(function* (projectId) { + // Everything the provider holds, not just the project's repository: a turn can touch a + // change request reached through another repository, and no epoch may move until those + // reads are gone, or a reader in between keeps the old one under the new epoch. + yield* forgetProviderReads(); + listingsEpoch = ++epochCounter; + projectEpochs.delete(projectId); + if (projectEpochs.size >= REF_EPOCH_CAPACITY) { + const oldest = projectEpochs.keys().next().value; + if (oldest !== undefined) { + projectEpochFloor = projectEpochs.get(oldest)!; + projectEpochs.delete(oldest); } - projectEpochs.set(projectId, listingsEpoch); - return readCache - .invalidate(`project:${projectId}`) - .pipe(Effect.andThen(SubscriptionRef.set(pullRequestRefreshes, listingsEpoch))); - }); + } + projectEpochs.set(projectId, listingsEpoch); + const epoch = listingsEpoch; + yield* readCache.invalidate(`project:${projectId}`); + yield* SubscriptionRef.set(pullRequestRefreshes, epoch); + }); const invalidatedByMutation = ( diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 7f0e86bd6cf49..40c741fdd68ac 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -190,6 +190,8 @@ const makeTestRelay = Effect.fnUntraced(function* ( return { schemaVersion: 2, snapshotSequence: 1, threads: [], archivedThreads: [] }; }), listActiveSubagentThreadIds: unused, + countThreads: unused, + listActiveShellIndex: () => Effect.succeed([]), listOlderSettledThreads: unused, readShellSnapshot: unused, ensureLegacyTranscript: unused, @@ -217,6 +219,7 @@ const makeTestRelay = Effect.fnUntraced(function* ( settleThread: unused, interruptThread: unused, stopWorkflow: () => Effect.die("unused stopWorkflow"), + forceStopThread: () => Effect.die("unused forceStopThread"), awaitBackgroundTasks: unused, stopDelegatedTasks: unused, getThreadEventSequence: unused, diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index cbe7b35ef409e..54742a0af7d66 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -726,10 +726,15 @@ export const make = Effect.gen(function* () { return "unlinked" as const; } const environmentId = yield* serverEnvironment.getEnvironmentId; - const [projectSnapshot, shellSnapshot] = yield* Effect.all([ - projects.snapshot, - threads.getShellSnapshot(), - ]); + const projectSnapshot = yield* projects.snapshot; + // Subagents never publish awareness, and they are most threads. + const topLevelThreadIds = (yield* threads.listActiveShellIndex()).flatMap((thread) => + thread.lineage.relationshipToParent === "subagent" ? [] : [thread.id], + ); + const shellSnapshot = yield* threads.getShellSnapshot({ + location: "active", + threadIds: topLevelThreadIds, + }); const activeThreadIds = resolveAgentAwarenessRelayActiveThreadIds({ environmentId, startedAt, diff --git a/apps/server/src/review/ReviewService.ts b/apps/server/src/review/ReviewService.ts index 8f88b387bd454..39ce1554cea61 100644 --- a/apps/server/src/review/ReviewService.ts +++ b/apps/server/src/review/ReviewService.ts @@ -8,6 +8,8 @@ import * as Path from "effect/Path"; import { VcsRepositoryDetectionError, VcsUnsupportedOperationError, + type ReviewBranchCommitsInput, + type ReviewBranchCommitsResult, type ReviewDiffFileContentsInput, type ReviewDiffFileContentsResult, type ReviewDiffPreviewError, @@ -20,9 +22,18 @@ import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; import * as KnownWorkspaces from "../workspace/KnownWorkspaces.ts"; +const WORKSPACE_BOUND_SUBJECTS = { + "ReviewService.getBranchCommits": "commit list", + "ReviewService.getDiffPreview": "diff preview", + "ReviewService.getDiffFileContents": "diff file contents", +} as const; + export class ReviewService extends Context.Service< ReviewService, { + readonly getBranchCommits: ( + input: ReviewBranchCommitsInput, + ) => Effect.Effect; readonly getDiffPreview: ( input: ReviewDiffPreviewInput, ) => Effect.Effect; @@ -61,7 +72,7 @@ export const make = Effect.gen(function* () { }; const assertWorkspaceBoundCwd = Effect.fn("ReviewService.assertWorkspaceBoundCwd")(function* ( - operation: "ReviewService.getDiffPreview" | "ReviewService.getDiffFileContents", + operation: keyof typeof WORKSPACE_BOUND_SUBJECTS, cwd: string, ) { const [candidate, workspaceRoot] = yield* Effect.all([ @@ -78,13 +89,27 @@ export const make = Effect.gen(function* () { return yield* new VcsRepositoryDetectionError({ operation, cwd, - detail: - operation === "ReviewService.getDiffPreview" - ? "Review diff preview cwd must stay within the configured workspace root." - : "Review diff file contents cwd must stay within the configured workspace root.", + detail: `Review ${WORKSPACE_BOUND_SUBJECTS[operation]} cwd must stay within the configured workspace root.`, }); }); + const getBranchCommits: ReviewService["Service"]["getBranchCommits"] = Effect.fn( + "ReviewService.getBranchCommits", + )(function* (input) { + yield* assertWorkspaceBoundCwd("ReviewService.getBranchCommits", input.cwd); + + const handle = yield* vcsRegistry.detect({ cwd: input.cwd, requestedKind: "auto" }); + if (!handle) return { commits: [], truncated: false }; + if (handle.kind !== "git") { + return yield* new VcsUnsupportedOperationError({ + operation: "ReviewService.getBranchCommits", + kind: handle.kind, + detail: "Review commit lists require a Git repository.", + }); + } + return yield* git.getReviewBranchCommits(input); + }); + const getDiffPreview: ReviewService["Service"]["getDiffPreview"] = Effect.fn( "ReviewService.getDiffPreview", )(function* (input) { @@ -120,6 +145,7 @@ export const make = Effect.gen(function* () { }); return ReviewService.of({ + getBranchCommits, getDiffPreview, getDiffFileContents, }); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts index 09e1d7674044f..1070b49a0dcf2 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts @@ -3,6 +3,7 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { expect, it } from "@effect/vitest"; import { ScheduledTaskUpsertInput } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; @@ -167,3 +168,64 @@ it.effect("keeps the task list order when a toggle or edit rewrites updated_at", }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies)))); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it.effect( + "idle polls still wake for tasks saved later, recorded due times, and beside far-future tasks", + () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-07-01T09:00:00.000Z")); + const launched = yield* Deferred.make(); + const layerDependencies = Layer.mergeAll( + NodeCrypto.layer, + Scheduler.layer, + Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + launch: (input) => + Deferred.succeed(launched, input.title).pipe( + Effect.as({ threadId: "thread-idle-poll", resumed: false } as never), + ), + }), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + Layer.mock(ThreadManagementService.ThreadManagementService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + ); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + // The first poll finds no tasks and remembers that nothing is due. + yield* TestClock.adjust("6 seconds"); + // Its year-10000+ next run sorts first as text; it must not hide the task below. + yield* service.upsert( + yield* decodeUpsertInput({ + title: "Far future", + prompt: "Review the open pull requests.", + enabled: true, + schedule: { type: "interval", everyMs: 3e14 }, + projectId: "project-idle-poll", + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: "codex", model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + }), + ); + const created = yield* service.upsert( + yield* decodeUpsertInput({ + title: "Idle poll", + prompt: "Review the open pull requests.", + enabled: true, + schedule: { type: "interval", everyMs: 60_000 }, + projectId: "project-idle-poll", + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: "codex", model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + }), + ); + // This poll records the new task's due time; the later ones must still fire it. + yield* TestClock.adjust("6 seconds"); + yield* TestClock.adjust("60 seconds"); + expect(yield* Deferred.await(launched)).toBe(created.task.title); + }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies)))); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index b3aa9440944bd..f05811ed65110 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -396,6 +396,35 @@ export const listDueTasks = Effect.fn("ScheduledTaskService.listDueTasks")(funct return tasks; }); +// Backstop for a write path that forgets to invalidate; the tick still runs every few seconds. +const DUE_HINT_MAX_AGE_MS = 10 * 60 * 1_000; + +/** + * Remembers when a poller next has work so idle scheduler ticks skip SQL. + * `invalidate` after every write that can move work earlier; a tick records + * only if no write happened since it read `version`. + */ +const makeDueHint = () => { + let version = 0; + let hint: { readonly dueAtMs: number; readonly checkedAtMs: number } | null = null; + return { + invalidate: Effect.sync(() => { + version++; + hint = null; + }), + version: Effect.sync(() => version), + isIdle: (nowMs: number) => + Effect.sync( + () => + hint !== null && hint.dueAtMs > nowMs && nowMs - hint.checkedAtMs < DUE_HINT_MAX_AGE_MS, + ), + record: (readVersion: number, dueAtMs: number, nowMs: number) => + Effect.sync(() => { + if (readVersion === version) hint = { dueAtMs, checkedAtMs: nowMs }; + }), + }; +}; + export const layer = Layer.effect( ScheduledTaskService, Effect.gen(function* () { @@ -426,7 +455,13 @@ export const layer = Layer.effect( const activeRuns = yield* Ref.make>(new Set()); // Notifications re-emit the full list; slow subscribers need only the newest dirty signal. const changesPubSub = yield* PubSub.sliding(1); - const notifyChanged = PubSub.publish(changesPubSub, undefined).pipe(Effect.asVoid); + const dueTasks = makeDueHint(); + const pendingWebhooks = makeDueHint(); + // Every scheduled_tasks write that can change next_run_at, enabled or run state notifies. + const notifyChanged = dueTasks.invalidate.pipe( + Effect.andThen(PubSub.publish(changesPubSub, undefined)), + Effect.asVoid, + ); const selectAllRows = () => sql` @@ -972,6 +1007,28 @@ export const layer = Layer.effect( const runDueTasks = Effect.fn("ScheduledTaskService.runDueTasks")(function* () { const now = yield* localNow; + const nowMs = DateTime.toEpochMillis(now); + if (yield* dueTasks.isIdle(nowMs)) return; + const readVersion = yield* dueTasks.version; + const [next] = yield* sql<{ next_run_at: string | null }>` + SELECT MIN(next_run_at) AS next_run_at FROM scheduled_tasks + WHERE enabled = 1 AND next_run_at IS NOT NULL AND last_run_status <> 'running' + `.pipe( + NodeSqliteClient.readOnly, + Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })), + ); + // Compare as text like listDueTasks: expanded years (`+011533-…`) sort first but parse far ahead. + const nextRunAt = next?.next_run_at ?? null; + if (nextRunAt === null || nextRunAt > iso(now)) { + const nextDueAt = + nextRunAt === null + ? Option.some(Infinity) + : DateTime.make(nextRunAt).pipe(Option.map(DateTime.toEpochMillis)); + if (Option.isSome(nextDueAt) && nextDueAt.value > nowMs) { + yield* dueTasks.record(readVersion, nextDueAt.value, nowMs); + } + return; + } const tasks = yield* listDueTasks(now).pipe( Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })), ); @@ -1695,6 +1752,8 @@ export const layer = Layer.effect( const triggerWebhook: ScheduledTaskService["Service"]["triggerWebhook"] = (request) => triggerWebhookUnobserved(request).pipe( + // Accepted deliveries may have queued a coalescing window. + Effect.ensuring(pendingWebhooks.invalidate), Effect.tapError(() => observeDelivery(request, "error", undefined, undefined)), Metrics.withMetrics({ timer: Metrics.webhookDeliveryDuration }), Effect.withSpan("ScheduledTaskService.triggerWebhook", { @@ -2090,10 +2149,19 @@ export const layer = Layer.effect( const dispatchPending = Effect.fn("ScheduledTaskService.dispatchPendingWebhooks")(function* () { const now = DateTime.toEpochMillis(yield* localNow); + if (yield* pendingWebhooks.isIdle(now)) return; + const readVersion = yield* pendingWebhooks.version; yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'dispatching' WHERE state = 'pending' AND due_at <= ${now}`; const tasks = yield* sql<{ task_id: ScheduledTaskId }>` SELECT DISTINCT task_id FROM scheduled_task_webhook_pending WHERE state IN ('dispatching', 'running', 'completed', 'failed') `.pipe(NodeSqliteClient.readOnly); + // Workers can leave rows behind to retry, so keep polling while any are claimed. + if (tasks.length === 0) { + const [next] = yield* sql<{ due_at: number | null }>` + SELECT MIN(due_at) AS due_at FROM scheduled_task_webhook_pending WHERE state = 'pending' + `.pipe(NodeSqliteClient.readOnly); + yield* pendingWebhooks.record(readVersion, next?.due_at ?? Infinity, now); + } yield* Effect.forEach( tasks, ({ task_id: taskId }) => diff --git a/apps/server/src/secrets/SecretCheck.ts b/apps/server/src/secrets/SecretCheck.ts new file mode 100644 index 0000000000000..66cdfa17be081 --- /dev/null +++ b/apps/server/src/secrets/SecretCheck.ts @@ -0,0 +1,41 @@ +import type { SecretReference, SecretSourceCheckResult } from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; + +export class SecretCheck extends Context.Service< + SecretCheck, + { + /** + * Reads one secret fresh for the Settings "check" button and never returns the value. A + * readable secret also rebuilds the provider instances that depend on a changed value. + */ + readonly check: (reference: SecretReference) => Effect.Effect; + } +>()("t3/secrets/SecretCheck") {} + +export const layer = Layer.effect( + SecretCheck, + Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + const providerRegistry = yield* ProviderRegistry.ProviderRegistry; + + const check = Effect.fn("SecretCheck.check")(function* (reference: SecretReference) { + yield* resolver.invalidate([reference]); + const [value] = yield* resolver.read([reference]); + if (value === undefined) { + return { + _tag: "Failed", + detail: `Could not read this secret from "${reference.provider}". Check the provider's settings and that its store is unlocked.`, + } as const; + } + yield* providerRegistry.reloadSecretsAndRefresh(); + return { _tag: "Resolved" } as const; + }); + + return SecretCheck.of({ check }); + }), +); diff --git a/apps/server/src/secrets/SecretResolver.ts b/apps/server/src/secrets/SecretResolver.ts new file mode 100644 index 0000000000000..076e4a0780793 --- /dev/null +++ b/apps/server/src/secrets/SecretResolver.ts @@ -0,0 +1,20 @@ +import * as SecretResolverCore from "@t3tools/secret-provider-core/server/SecretResolver"; +import type { ServerSettings } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { BUILT_IN_SECRET_PROVIDER_DRIVER_LIST } from "./secretProviderDefinitions.ts"; +import * as Settings from "../serverSettings.ts"; + +export const layer = Layer.unwrap( + Effect.gen(function* () { + const serverSettings = yield* Settings.ServerSettingsService; + return SecretResolverCore.layer({ + drivers: BUILT_IN_SECRET_PROVIDER_DRIVER_LIST, + secretProviders: serverSettings.getSettings.pipe( + Effect.map((settings) => settings.secretProviders), + Effect.orElseSucceed((): ServerSettings["secretProviders"] => ({})), + ), + }); + }), +); diff --git a/apps/server/src/secrets/legacySecretSettings.ts b/apps/server/src/secrets/legacySecretSettings.ts new file mode 100644 index 0000000000000..4e15f8a24330f --- /dev/null +++ b/apps/server/src/secrets/legacySecretSettings.ts @@ -0,0 +1,160 @@ +type Json = Record; + +const isRecord = (value: unknown): value is Json => + value !== null && typeof value === "object" && !Array.isArray(value); + +const LEGACY_PROVIDER_KINDS: Readonly> = { + "1password": "one_password", + openbao: "openbao", + "aws-secrets-manager": "aws_secrets_manager", +}; + +const LEGACY_PROVIDER_FIELDS: Readonly>> = { + "1password": ["account"], + openbao: ["address", "mount", "namespace", "auth", "token"], + "aws-secrets-manager": ["region", "profile"], +}; + +const PROVIDER_ID_MAX_CHARS = 64; +const PROVIDER_LABEL_MAX_CHARS = 64; + +const pickFields = (entry: Json, fields: ReadonlyArray): Json => + Object.fromEntries( + fields.flatMap((field) => (entry[field] === undefined ? [] : [[field, entry[field]]])), + ); + +const migrateProvider = (entry: unknown): unknown => { + if (!isRecord(entry) || typeof entry.kind !== "string" || Object.hasOwn(entry, "config")) { + return entry; + } + const kind = LEGACY_PROVIDER_KINDS[entry.kind]; + if (kind === undefined) return entry; + return { + kind, + label: entry.label, + config: pickFields(entry, LEGACY_PROVIDER_FIELDS[entry.kind]!), + }; +}; + +const legacyReference = ( + value: unknown, +): + | { readonly provider?: string; readonly account?: string; readonly reference: string } + | undefined => { + if (!isRecord(value) || typeof value.reference !== "string") return undefined; + if (value.kind === "secret-provider" && typeof value.provider === "string") { + return { provider: value.provider, reference: value.reference }; + } + if (value.kind === "1password" && typeof value.account === "string") { + return { account: value.account, reference: value.reference }; + } + return undefined; +}; + +const accountSlug = (account: string) => + `one-password-${account.replace(/[^A-Za-z0-9_-]/g, "-")}`.slice(0, PROVIDER_ID_MAX_CHARS); + +/** + * Rewrites the fork's secret shapes in raw settings into the shared secret provider shapes: + * flat provider entries become `{ kind, label, config }` envelopes, and credential values that + * named a provider or a 1Password account become `{ provider, reference }`. Anything already in + * the new shape passes through untouched. + */ +export const migrateLegacySecretSettings = ( + settings: Json, +): { readonly settings: Json; readonly changed: boolean } => { + let changed = false; + + const secretProviders: Json = {}; + const rawProviders = isRecord(settings.secretProviders) ? settings.secretProviders : undefined; + if (rawProviders !== undefined) { + for (const [id, entry] of Object.entries(rawProviders)) { + const migrated = migrateProvider(entry); + if (migrated !== entry) changed = true; + secretProviders[id] = migrated; + } + } + + const accountProviders = new Map(); + for (const [id, entry] of Object.entries(secretProviders)) { + if (isRecord(entry) && entry.kind === "one_password" && isRecord(entry.config)) { + const account = entry.config.account; + if (typeof account === "string" && !accountProviders.has(account)) { + accountProviders.set(account, id); + } + } + } + + const providerForAccount = (account: string) => { + const existing = accountProviders.get(account); + if (existing !== undefined) return existing; + let id = accountSlug(account); + for (let suffix = 2; Object.hasOwn(secretProviders, id); suffix += 1) { + id = `${accountSlug(account).slice(0, PROVIDER_ID_MAX_CHARS - 4)}-${suffix}`; + } + secretProviders[id] = { + kind: "one_password", + label: `1Password ${account}`.slice(0, PROVIDER_LABEL_MAX_CHARS), + config: { account }, + }; + accountProviders.set(account, id); + return id; + }; + + const toSecretReference = (value: unknown) => { + const legacy = legacyReference(value); + if (legacy === undefined) return undefined; + changed = true; + return { + provider: legacy.provider ?? providerForAccount(legacy.account!), + reference: legacy.reference, + }; + }; + + const walk = (value: unknown): unknown => { + const reference = toSecretReference(value); + if (reference !== undefined) return reference; + if (Array.isArray(value)) return value.map(walk); + if (!isRecord(value)) return value; + return Object.fromEntries(Object.entries(value).map(([key, inner]) => [key, walk(inner)])); + }; + + const migrateInstance = (instance: unknown): unknown => { + if (!isRecord(instance)) return instance; + const { environment, ...rest } = instance; + if (!Array.isArray(environment)) return walk(instance); + return { + ...(walk(rest) as Json), + environment: environment.map((variable) => { + if (!isRecord(variable)) return variable; + const reference = toSecretReference(variable.value); + if (reference === undefined) return walk(variable); + const { value: _legacy, ...others } = variable; + return { ...(walk(others) as Json), value: "", secret: reference }; + }), + }; + }; + + const rest: Json = {}; + for (const [key, value] of Object.entries(settings)) { + if (key === "secretProviders") continue; + if (key === "providerInstances" && isRecord(value)) { + rest[key] = Object.fromEntries( + Object.entries(value).map(([id, instance]) => [id, migrateInstance(instance)]), + ); + continue; + } + rest[key] = walk(value); + } + + if (!changed) return { settings, changed: false }; + return { + settings: { + ...rest, + ...(rawProviders === undefined && Object.keys(secretProviders).length === 0 + ? {} + : { secretProviders }), + }, + changed: true, + }; +}; diff --git a/apps/server/src/secrets/secretProviderDefinitions.test.ts b/apps/server/src/secrets/secretProviderDefinitions.test.ts new file mode 100644 index 0000000000000..6431543da9877 --- /dev/null +++ b/apps/server/src/secrets/secretProviderDefinitions.test.ts @@ -0,0 +1,20 @@ +import { expect, it } from "@effect/vitest"; + +import { + BUILT_IN_SECRET_PROVIDER_DRIVER_LIST, + secretProviderDefinitions, +} from "./secretProviderDefinitions.ts"; + +it("runs a driver for every secret provider the server and clients read, in their order", () => { + expect(BUILT_IN_SECRET_PROVIDER_DRIVER_LIST.map((driver) => driver.definition)).toEqual( + secretProviderDefinitions.definitions, + ); +}); + +it("registers the built-in kinds in a stable order", () => { + expect(secretProviderDefinitions.definitions.map((definition) => definition.kind)).toEqual([ + "one_password", + "openbao", + "aws_secrets_manager", + ]); +}); diff --git a/apps/server/src/secrets/secretProviderDefinitions.ts b/apps/server/src/secrets/secretProviderDefinitions.ts new file mode 100644 index 0000000000000..2702a8753cf20 --- /dev/null +++ b/apps/server/src/secrets/secretProviderDefinitions.ts @@ -0,0 +1,13 @@ +import * as AwsDriver from "@t3tools/secret-provider-aws/server/driver"; +import { builtInSecretProviderDefinitions } from "@t3tools/secret-provider-definitions/registry"; +import * as OnePasswordDriver from "@t3tools/secret-provider-onepassword/server/driver"; +import * as OpenBaoDriver from "@t3tools/secret-provider-openbao/server/driver"; + +/** In the shared registry's order. */ +export const BUILT_IN_SECRET_PROVIDER_DRIVER_LIST = [ + OnePasswordDriver.driver, + OpenBaoDriver.driver, + AwsDriver.driver, +] as const; + +export const secretProviderDefinitions = builtInSecretProviderDefinitions; diff --git a/apps/server/src/secrets/settingsSecretValue.ts b/apps/server/src/secrets/settingsSecretValue.ts new file mode 100644 index 0000000000000..ed396cda2a615 --- /dev/null +++ b/apps/server/src/secrets/settingsSecretValue.ts @@ -0,0 +1,45 @@ +import { isSecretReference, type SettingsSecretValue } from "@t3tools/contracts"; +import type * as SecretResolverCore from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +type SecretResolverShape = SecretResolverCore.SecretResolver["Service"]; + +export class SettingsSecretUnresolvedError extends Schema.TaggedError()( + "SettingsSecretUnresolvedError", + { provider: Schema.String }, +) { + override get message(): string { + return `Could not read the credential from the secret provider "${this.provider}". Unlock or fix it in Settings, then try again.`; + } +} + +/** + * The secret a credential setting holds or names. A named secret that cannot be read, or reads + * empty, fails instead of passing for a missing token, so callers that would fall back to another + * credential cannot silently switch accounts. + */ +export const requireSettingsSecret = ( + resolver: SecretResolverShape, + value: SettingsSecretValue, +): Effect.Effect => + !isSecretReference(value) + ? Effect.succeed(value) + : resolver + .read([value]) + .pipe( + Effect.flatMap(([secret]) => + secret === undefined || secret.trim() === "" + ? Effect.fail(new SettingsSecretUnresolvedError({ provider: value.provider })) + : Effect.succeed(secret), + ), + ); + +/** An unreadable secret comes back as `""`, for credentials whose documented fallback is acceptable. */ +export const resolveSettingsSecret = ( + resolver: SecretResolverShape, + value: SettingsSecretValue, +): Effect.Effect => + requireSettingsSecret(resolver, value).pipe( + Effect.catchTags({ SettingsSecretUnresolvedError: () => Effect.succeed("") }), + ); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 2356178a13f87..34eb230185ca0 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -64,8 +64,9 @@ import * as SourceControlBuiltInDrivers from "./sourceControl/builtInDrivers.ts" import * as SourceControlSignIn from "./sourceControl/SourceControlSignIn.ts"; import { BUILT_IN_SOURCE_CONTROL_DRIVER_LIST } from "./sourceControl/sourceControlDefinitions.ts"; import * as TextGeneration from "./textGeneration/TextGeneration.ts"; +import * as SecretCheck from "./secrets/SecretCheck.ts"; +import * as SecretResolver from "./secrets/SecretResolver.ts"; import * as ProviderInstanceRegistryHydration from "./provider/ProviderInstanceRegistryHydration.ts"; -import { ProviderSecretResolverLive } from "./provider/ProviderSecretResolverLive.ts"; import * as ProjectSettingsOverrideSweep from "./provider/ProjectSettingsOverrideSweep.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import * as McpHttpServer from "./mcp/McpHttpServer.ts"; @@ -365,7 +366,11 @@ const layerRepositoryIdentityResolver = Layer.effect( }), }); }), -).pipe(Layer.provide(layerSourceControlProviderRegistry), Layer.provide(ProcessRunner.layer)); +).pipe( + Layer.provide(layerSourceControlProviderRegistry), + Layer.provide(SecretResolver.layer), + Layer.provide(ProcessRunner.layer), +); const layerPullRequestService = PullRequestService.layer.pipe( Layer.provide(PullRequestProviderRegistry.layer), @@ -373,6 +378,7 @@ const layerPullRequestService = PullRequestService.layer.pipe( Layer.provide(PullRequestReadCache.layer), Layer.provide(layerSourceControlProviderRegistry), Layer.provide(SourceControlRateLimit.layer), + Layer.provide(SecretResolver.layer), ); const layerGitManager = GitManager.layer.pipe( @@ -638,12 +644,9 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( Layer.provideMerge( Layer.mergeAll(Keybindings.layer, EnvironmentTheme.layer, UsageLimitSources.layer), ), + Layer.provideMerge(SecretCheck.layer), Layer.provideMerge(ProviderRegistry.layer), Layer.provideMerge(ProviderInstanceRegistryHydration.layer), - // Resolves secret-source environment values for both halves above: the instance - // registry reads secrets while building an instance, and - // `ProviderRegistry.layer` drops the cached values when the user refreshes. - Layer.provideMerge(ProviderSecretResolverLive), // Provider hydration builds the Claude runner, which reports agents to the same scanner the // terminal and preview layers share. Layer.provideMerge(layerPortScanner), diff --git a/apps/server/src/serverOwnership.ts b/apps/server/src/serverOwnership.ts index 1959735425515..6d89f45cb792e 100644 --- a/apps/server/src/serverOwnership.ts +++ b/apps/server/src/serverOwnership.ts @@ -44,7 +44,7 @@ export class ServerAlreadyRunningError extends Schema.TaggedError new ServerOwnershipError({ stateDir, cause }))); - yield* Effect.acquireRelease( + const lockPath = NodePath.join(stateDir, "server-owner.sqlite"); + const lock = yield* Effect.acquireRelease( Effect.try({ try: () => { NodeFS.mkdirSync(stateDir, { recursive: true }); - const db = new NodeSqlite.DatabaseSync(NodePath.join(stateDir, "server-owner.sqlite")); + let before = NodeFS.statSync(lockPath, { bigint: true, throwIfNoEntry: false }); + if (before === undefined) { + NodeFS.closeSync(NodeFS.openSync(lockPath, "a", 0o644)); + before = NodeFS.statSync(lockPath, { bigint: true }); + } + const db = new NodeSqlite.DatabaseSync(lockPath); try { db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); + const identity = NodeFS.statSync(lockPath, { bigint: true }); + return { + db, + identity, + replacedDuringAcquire: before.dev !== identity.dev || before.ino !== identity.ino, + }; } catch (cause) { db.close(); throw cause; } - return db; }, catch: (cause) => isSqliteBusy(cause) ? new ServerAlreadyRunningError({ stateDir }) : new ServerOwnershipError({ stateDir, cause }), }), - (db) => + ({ db }) => readPersistedServerRuntimeState(statePath).pipe( Effect.flatMap((state) => Option.isSome(state) && state.value.ownerId === ownerId @@ -233,6 +244,39 @@ export const acquireServerOwnership = Effect.fn("acquireServerOwnership")(functi return yield* new OlderBootServiceError({ baseDir: olderServiceHome }); } + yield* Effect.gen(function* () { + let lost = lock.replacedDuringAcquire; + let statWarningLogged = false; + while (!lost) { + yield* Effect.sleep("5 seconds"); + // Closing another descriptor for this inode releases SQLite's POSIX locks. Stat by path only. + const current = yield* Effect.sync(() => { + try { + return NodeFS.statSync(lockPath, { bigint: true, throwIfNoEntry: false }); + } catch (cause) { + return cause instanceof Error && "code" in cause ? String(cause.code) : "unknown"; + } + }); + if (typeof current === "string") { + if (!statWarningLogged) { + yield* Effect.logWarning( + `Cannot check server ownership lock at ${lockPath} (${current}).`, + ); + statWarningLogged = true; + } + continue; + } + statWarningLogged = false; + lost = + current === undefined || + current.dev !== lock.identity.dev || + current.ino !== lock.identity.ino; + } + yield* Effect.logError( + `Server ownership lock at ${lockPath} was deleted or replaced. Another server may now share this T3 home. Stop any other server using this home, then restart this server. Do not delete server-owner.sqlite while a server is running.`, + ); + }).pipe(Effect.forkScoped); + return { publish: (state: PersistedServerRuntimeState) => persistServerRuntimeState({ path: statePath, state: { ...state, ownerId } }), diff --git a/apps/server/src/serverRuntimeStartup.autoPull.test.ts b/apps/server/src/serverRuntimeStartup.autoPull.test.ts index 59f6b67f04509..549aae79e8be5 100644 --- a/apps/server/src/serverRuntimeStartup.autoPull.test.ts +++ b/apps/server/src/serverRuntimeStartup.autoPull.test.ts @@ -141,13 +141,7 @@ it.effect("parks automatic pull until activation without delaying command readin }), Layer.mock(ProjectService.ProjectService)({ snapshot: Effect.succeed(snapshot) }), Layer.mock(ThreadManagement.ThreadManagementService)({ - getShellSnapshot: () => - Effect.succeed({ - schemaVersion: 1, - snapshotSequence: 0, - threads: [], - archivedThreads: [], - }), + countThreads: () => Effect.succeed(0), }), Layer.mock(ThreadLaunch.ThreadLaunchService)({}), Layer.mock(ServiceLauncherClient.ServiceLauncherClient)({ diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.ts index 274b149c66efc..bef61bbca56fd 100644 --- a/apps/server/src/serverRuntimeStartup.test.ts +++ b/apps/server/src/serverRuntimeStartup.test.ts @@ -176,13 +176,8 @@ it.effect.each([ }), }), Layer.mock(ThreadManagement.ThreadManagementService)({ - getShellSnapshot: () => - Effect.succeed({ - threads: [], - archivedThreads: [], - schemaVersion: 1, - snapshotSequence: 0, - }), + countThreads: () => Effect.succeed(0), + listActiveShellIndex: () => Effect.succeed([]), }), Layer.mock(ThreadLaunch.ThreadLaunchService)({ launch: (input) => { diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index d8d62dc029c24..b812ad07fa6d6 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -153,11 +153,11 @@ const recordStartupHeartbeat = Effect.gen(function* () { const { threadCount, projectCount } = yield* Effect.all({ projects: projects.snapshot, - threads: threads.getShellSnapshot(), + threadCount: threads.countThreads(), }).pipe( - Effect.map(({ projects: projectSnapshot, threads: shellSnapshot }) => ({ + Effect.map(({ projects: projectSnapshot, threadCount }) => ({ projectCount: projectSnapshot.projects.length, - threadCount: shellSnapshot.threads.length + shellSnapshot.archivedThreads.length, + threadCount, })), Effect.catch((cause) => Effect.logWarning("failed to gather V2 startup counts for telemetry", { @@ -279,8 +279,7 @@ const resolveAutoBootstrapWelcomeTargets = Effect.gen(function* () { title: path.basename(serverConfig.cwd) || "project", workspaceRoot: serverConfig.cwd, }); - const shell = yield* threads.getShellSnapshot(); - const existingThread = shell.threads.find( + const existingThread = (yield* threads.listActiveShellIndex()).find( (thread) => thread.projectId === project.id && thread.lineage.relationshipToParent !== "subagent", ); diff --git a/apps/server/src/serverRuntimeState.test.ts b/apps/server/src/serverRuntimeState.test.ts index abafbc7e24f62..b75ab72594add 100644 --- a/apps/server/src/serverRuntimeState.test.ts +++ b/apps/server/src/serverRuntimeState.test.ts @@ -1,13 +1,20 @@ +// @effect-diagnostics nodeBuiltinImport:off - Injects a real file replacement during the native SQLite transaction. +import * as NodeFS from "node:fs"; +import * as NodeSqlite from "node:sqlite"; + import { assert, describe, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Runtime from "effect/Runtime"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as TestClock from "effect/testing/TestClock"; +import { vi } from "vite-plus/test"; import { renderBootServicePlist, renderBootServiceUnit } from "./cloud/bootService.ts"; import { ServiceLauncherHostProcess } from "./cloud/serviceLauncherClient.ts"; @@ -179,6 +186,7 @@ describe("server ownership", () => { Effect.flip, ); assert.equal(refused._tag, "ServerAlreadyRunningError"); + assert.include(refused.message, "do not delete it while a server is running"); assert.equal(yield* fs.readFileString(statePath), before); }), ); @@ -187,6 +195,89 @@ describe("server ownership", () => { }).pipe(Effect.provide(layerOwnership)), ); + describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("lock file checks", () => { + it.effect.each([ + "unchanged", + "unlink", + "replace", + "during acquisition", + "first acquisition", + // Root bypasses directory permissions, so chmod cannot force EACCES. + ...(process.getuid?.() === 0 ? [] : (["stat error"] as const)), + ] as const)("monitors ownership after %s", (operation) => { + const messages: unknown[] = []; + const warnings: unknown[] = []; + const logger = Logger.make(({ message, logLevel }) => + (logLevel === "Warn" ? warnings : messages).push(message), + ); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-lost-" }); + const statePath = path.join(root, "server-runtime.json"); + const lockPath = path.join(root, "server-owner.sqlite"); + const replacement = path.join(root, "replacement.sqlite"); + if (operation === "during acquisition" || operation === "first acquisition") { + if (operation === "during acquisition") yield* fs.writeFileString(lockPath, ""); + yield* fs.writeFileString(replacement, ""); + const exec = NodeSqlite.DatabaseSync.prototype.exec; + yield* Effect.acquireRelease( + Effect.sync(() => + vi.spyOn(NodeSqlite.DatabaseSync.prototype, "exec").mockImplementationOnce(function ( + this: NodeSqlite.DatabaseSync, + sql: string, + ) { + exec.call(this, sql); + NodeFS.renameSync(replacement, lockPath); + }), + ), + (spy) => Effect.sync(() => spy.mockRestore()), + ); + } + yield* ServerOwnership.acquireServerOwnership(statePath); + if (operation === "unlink") { + yield* fs.remove(lockPath); + } else if (operation === "replace") { + yield* fs.writeFileString(replacement, ""); + yield* fs.rename(replacement, lockPath); + } + if (operation === "stat error") { + yield* fs.chmod(root, 0); + yield* TestClock.adjust("15 seconds").pipe( + Effect.ensuring(fs.chmod(root, 0o700).pipe(Effect.orDie)), + ); + assert.deepEqual(warnings, [ + [`Cannot check server ownership lock at ${lockPath} (EACCES).`], + ]); + yield* TestClock.adjust("5 seconds"); + assert.deepEqual(messages, []); + yield* fs.remove(lockPath); + } + yield* TestClock.adjust("5 seconds"); + const expected = + operation === "unchanged" + ? [] + : [ + [ + `Server ownership lock at ${lockPath} was deleted or replaced. Another server may now share this T3 home. Stop any other server using this home, then restart this server. Do not delete server-owner.sqlite while a server is running.`, + ], + ]; + assert.deepEqual(messages, expected); + if (operation !== "unchanged") { + yield* fs.remove(`${lockPath}-journal`, { force: true }); + yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath)); + } + yield* TestClock.adjust("15 seconds"); + assert.deepEqual(messages, expected); + assert.lengthOf(warnings, operation === "stat error" ? 1 : 0); + }).pipe( + Effect.provide( + Layer.merge(layerOwnership, Logger.layer([logger], { mergeWithExisting: false })), + ), + ); + }); + }); + it.effect("refuses beside a live pre-lock server and ignores a stale record", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index bff7df677c874..517a64c7fb471 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -9,10 +9,11 @@ import { ProviderInstanceEnvironment, ProviderInstanceId, resolveProviderInstanceEnabled, + type SettingsSecretValue, SecretProviderId, - SecretProviderReference, + SecretProviderKind, ServerSettings, - type UsageLimitSourceConfig, + ServerSettingsPatch, UsageLimitSourceId, } from "@t3tools/contracts"; import { createModelSelection } from "@t3tools/shared/model"; @@ -34,10 +35,12 @@ import * as ServerConfig from "./config.ts"; import * as SqlitePersistence from "./persistence/Sqlite.ts"; import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; import * as ServerSettingsModule from "./serverSettings.ts"; +import { SECRET_REDACTED } from "./settingsSecrets.ts"; import { resolveProviderInstanceTerminalEnvironment } from "./terminal/Manager.ts"; const decodeServerSettings = Schema.decodeUnknownEffect(ServerSettings); const decodeServerSettingsJson = Schema.decodeUnknownEffect(Schema.fromJsonString(ServerSettings)); +const decodeServerSettingsPatch = Schema.decodeUnknownEffect(ServerSettingsPatch); const layerServerSettings = () => ServerSettingsModule.layer.pipe( @@ -217,6 +220,31 @@ it.layer(NodeServices.layer)("server settings", (it) => { }).pipe(Effect.provide(layerServerSettings())), ); + it.effect("keeps an override key a newer server saved when a project setting is edited", () => + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const service = yield* ServerSettingsModule.ServerSettingsService; + const projectId = ProjectId.make("project-a"); + yield* fs.writeFileString( + config.settingsPath, + `{ "projectSettingsOverrides": { "project-a": { "defaultAutoPull": true, "addedByNewerServer": 3 } } }`, + ); + const loaded = yield* service.getSettings; + + yield* service.updateSettings({ + projectSettingsOverrides: { + [projectId]: { ...loaded.projectSettingsOverrides[projectId], defaultAutoPull: false }, + }, + }); + + const persisted = JSON.parse(yield* fs.readFileString(config.settingsPath)); + assert.deepEqual(persisted.projectSettingsOverrides, { + [projectId]: { defaultAutoPull: false, addedByNewerServer: 3 }, + }); + }).pipe(Effect.provide(layerServerSettings())), + ); + it.effect("moves an idle compaction mode saved in the legacy switch to its own field", () => Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; @@ -246,47 +274,37 @@ it.layer(NodeServices.layer)("server settings", (it) => { secretProviders: { personal: { kind: "one_password", label: "Personal", config: { account: "my" } }, vault: { kind: "openbao", label: "Vault", config: { token: "bao-token" } }, + mystery: { kind: "mystery", label: "Mystery", config: { token: "mystery-token" } }, }, }); assert.deepStrictEqual( ServerSettingsModule.redactServerSettingsForClient(settings).secretProviders, { personal: { kind: "one_password", label: "Personal", config: { account: "my" } }, - vault: { kind: "openbao", label: "Vault" }, + vault: { kind: "openbao", label: "Vault", config: { token: "••••••" } }, + mystery: { kind: "mystery", label: "Mystery" }, }, ); }), ); - it.effect("keeps a fork flat entry's fields for clients but never its OpenBao token", () => + it.effect("stores no secret provider of an unloaded kind or with a config its kind rejects", () => Effect.gen(function* () { - const settings = yield* decodeServerSettings({ - secretProviders: { - home: { kind: "1password", label: "Home", account: "my" }, - homelab: { - kind: "openbao", - label: "Homelab", - address: "https://bao.example.test", - token: "bao-token", - }, - personal: { kind: "one_password", label: "Personal", config: { account: "my" } }, - }, - }); - const client: Readonly> = - ServerSettingsModule.redactServerSettingsForClient(settings).secretProviders; - assert.notInclude(JSON.stringify(client), "bao-token"); - assert.deepStrictEqual(client.home, { - kind: "1password", - label: "Home", - account: "my", - }); - assert.deepStrictEqual(client.personal, { - kind: "one_password", - label: "Personal", - config: { account: "my" }, - }); - assert.include(JSON.stringify(client.homelab), "https://bao.example.test"); - }), + const service = yield* ServerSettingsModule.ServerSettingsService; + const write = (entry: unknown) => + decodeServerSettingsPatch({ secretProviders: { personal: entry } }).pipe( + Effect.flatMap(service.updateSettings), + Effect.flip, + ); + for (const entry of [ + { kind: "one_password", label: "Personal", config: { account: "--help" } }, + { kind: "one_password", label: "Personal" }, + { kind: "openbao", label: "Vault", config: { address: "https://x" } }, + ]) { + assert.deepInclude(yield* write(entry), { operation: "write-secret-provider" }); + } + assert.deepStrictEqual((yield* service.getSettings).secretProviders, {}); + }).pipe(Effect.provide(layerServerSettings())), ); it.effect("saves through a symlinked settings file without replacing the link", () => @@ -1058,6 +1076,54 @@ it.layer(NodeServices.layer)("server settings", (it) => { }).pipe(Effect.provide(layerServerSettings())), ); + it.effect("reads settings secrets once per settings change", () => { + let reads = 0; + const layerSecret = Layer.effect( + ServerSecretStore.ServerSecretStore, + Effect.map(ServerSecretStore.ServerSecretStore, (store) => ({ + ...store, + get: (name: string) => + Effect.suspend(() => { + reads++; + return store.get(name); + }), + })), + ).pipe(Layer.provide(ServerSecretStore.layer)); + const layerSettings = ServerSettingsModule.layer.pipe( + Layer.provide(layerSecret), + Layer.provideMerge(Layer.fresh(SqlitePersistence.layerMemory)), + Layer.provideMerge( + Layer.fresh( + ServerConfig.layerTest(process.cwd(), { prefix: "t3code-settings-secret-cache-test-" }), + ), + ), + ); + return Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("codex_cached"); + const service = yield* ServerSettingsModule.ServerSettingsService; + const save = (value: string) => + service.updateSettings({ + providerInstances: { + [instanceId]: { + driver: ProviderDriverKind.make("codex"), + environment: [{ name: "API_TOKEN", value, sensitive: true }], + config: {}, + }, + }, + }); + const token = service.getSettings.pipe( + Effect.map((settings) => settings.providerInstances[instanceId]?.environment?.[0]?.value), + ); + yield* save("sk-first"); + reads = 0; + assert.equal(yield* token, "sk-first"); + assert.equal(yield* token, "sk-first"); + assert.equal(reads, 0); + yield* save("sk-second"); + assert.equal(yield* token, "sk-second"); + }).pipe(Effect.provide(layerSettings)); + }); + it.effect("keeps the inline value on disk when secret migration fails", () => { const cause = new ServerSecretStore.SecretStorePersistError({ resource: "provider environment secret", @@ -1243,12 +1309,14 @@ it.layer(NodeServices.layer)("server settings", (it) => { }, secretProviders: { [SecretProviderId.make("homelab")]: { - kind: "openbao", + kind: SecretProviderKind.make("openbao"), label: "Homelab", - address: "https://bao.example.test", - mount: "secret", - token: "openbao-secret", - auth: "token", + config: { + address: "https://bao.example.test", + mount: "secret", + token: "openbao-secret", + auth: "token", + }, }, }, easDeviceHosts: [ @@ -1343,6 +1411,27 @@ it.layer(NodeServices.layer)("server settings", (it) => { for (const name of secretNames) { assert.isTrue(Option.isSome(yield* store.get(name)), name); } + const secret = { + provider: SecretProviderId.make("personal"), + reference: "op://Private/Antigravity/token", + }; + const referenced = yield* service.updateProviderInstance({ + operation: "upsert", + instanceId, + instance: { + ...client.providerInstances[instanceId]!, + environment: [ + { name: "API_TOKEN", value: secret, sensitive: true, valueRedacted: true }, + ], + }, + }); + assert.isTrue(Option.isNone(yield* store.get(secretNames[1]!))); + assert.deepEqual( + ServerSettingsModule.redactServerSettingsForClient(referenced).providerInstances[ + instanceId + ]?.environment, + [{ name: "API_TOKEN", value: secret, sensitive: true }], + ); yield* service.updateSettings({ providerInstances: {}, sourceControlHosts: { @@ -1409,24 +1498,201 @@ it.layer(NodeServices.layer)("server settings", (it) => { }).pipe(Effect.provide(layerServerSettings())), ); - it.effect("stores 1Password secret sources as written and never as sensitive", () => + it.effect( + "rewrites the fork's secret shapes into secret provider shapes when settings load", + () => + Effect.gen(function* () { + const service = yield* ServerSettingsModule.ServerSettingsService; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const store = yield* ServerSecretStore.ServerSecretStore; + yield* store.set( + "secret-provider-aG9tZWxhYg-dG9rZW4", + new TextEncoder().encode("bao-secret"), + ); + yield* fs.writeFileString( + config.settingsPath, + JSON.stringify({ + secretProviders: { + personal: { kind: "1password", label: "Personal", account: "my.1password.com" }, + homelab: { + kind: "openbao", + label: "Homelab", + address: "https://bao.example.test", + mount: "secret", + auth: "token", + token: SECRET_REDACTED, + }, + aws: { + kind: "aws-secrets-manager", + label: "AWS", + region: "us-east-1", + profile: "dev", + }, + }, + providerInstances: { + codex_personal: { + driver: "codex", + environment: [ + { + name: "FROM_PROVIDER", + value: { kind: "secret-provider", provider: "homelab", reference: "codex#key" }, + sensitive: false, + }, + { + name: "FROM_ACCOUNT", + value: { + kind: "1password", + reference: "op://Home/Codex/key", + account: "other.1password.com", + }, + sensitive: false, + }, + { name: "PLAIN", value: "literal", sensitive: false }, + ], + }, + }, + usageLimitSources: { + hub: { + kind: "cliproxy", + url: "https://hub.example.test", + managementKey: { + kind: "1password", + reference: "op://Home/Hub/key", + account: "my.1password.com", + }, + enabled: true, + }, + }, + easDeviceHosts: [ + { + kind: "eas", + id: "eas", + label: "EAS Simulator", + accessToken: { + kind: "secret-provider", + provider: "homelab", + reference: "expo#token", + }, + maxSessionMinutes: 30, + idleMinutes: 10, + maxConcurrentSessions: 2, + }, + ], + sourceControlHosts: { + gitcafe: { + token: { kind: "secret-provider", provider: "aws", reference: "gitcafe" }, + }, + github: { + tokens: { + "github.com": { + kind: "secret-provider", + provider: "homelab", + reference: "gh#token", + }, + "ghe.acme.test": "ghp_literal", + }, + }, + }, + }), + ); + + const settings = yield* service.getSettings; + const synthesized = SecretProviderId.make("one-password-other-1password-com"); + assert.deepEqual(settings.secretProviders[SecretProviderId.make("personal")], { + kind: "one_password", + label: "Personal", + config: { account: "my.1password.com" }, + }); + assert.deepEqual(settings.secretProviders[SecretProviderId.make("aws")], { + kind: "aws_secrets_manager", + label: "AWS", + config: { region: "us-east-1", profile: "dev" }, + }); + assert.deepEqual(settings.secretProviders[synthesized], { + kind: "one_password", + label: "1Password other.1password.com", + config: { account: "other.1password.com" }, + }); + assert.deepEqual( + settings.providerInstances[ProviderInstanceId.make("codex_personal")]?.environment, + [ + { + name: "FROM_PROVIDER", + value: { provider: "homelab", reference: "codex#key" }, + sensitive: false, + }, + { + name: "FROM_ACCOUNT", + value: { provider: synthesized, reference: "op://Home/Codex/key" }, + sensitive: false, + }, + { name: "PLAIN", value: "literal", sensitive: false }, + ], + ); + assert.deepEqual( + settings.usageLimitSources[UsageLimitSourceId.make("hub")]?.managementKey, + { + provider: "personal", + reference: "op://Home/Hub/key", + }, + ); + assert.deepEqual(settings.easDeviceHosts[0]?.accessToken, { + provider: "homelab", + reference: "expo#token", + }); + assert.deepEqual(settings.sourceControlHosts.gitcafe?.token, { + provider: "aws", + reference: "gitcafe", + }); + assert.deepEqual( + (settings.sourceControlHosts.github?.tokens as Record)?.["github.com"], + { + provider: "homelab", + reference: "gh#token", + }, + ); + + const homelab = settings.secretProviders[SecretProviderId.make("homelab")]; + assert.deepEqual(homelab, { + kind: "openbao", + label: "Homelab", + config: { + address: "https://bao.example.test", + mount: "secret", + auth: "token", + token: "bao-secret", + }, + }); + const disk = yield* fs.readFileString(config.settingsPath); + assert.notInclude(disk, "bao-secret"); + assert.notInclude(disk, '"kind":"1password"'); + assert.notInclude(disk, "secret-provider"); + assert.notInclude( + JSON.stringify(ServerSettingsModule.redactServerSettingsForClient(settings)), + "bao-secret", + ); + }).pipe(Effect.provide(layerServerSettingsWithSecrets())), + ); + + it.effect("keeps an environment secret reference as written and out of the secret store", () => Effect.gen(function* () { const serverSettings = yield* ServerSettingsModule.ServerSettingsService; const serverConfig = yield* ServerConfig.ServerConfig; const fileSystem = yield* FileSystem.FileSystem; const instanceId = ProviderInstanceId.make("codex_personal"); const source = { - kind: "1password", + provider: SecretProviderId.make("personal"), reference: "op://Home Lab/Codex/api key", - account: "my.1password.com", - } as const; + }; const decodeEnvironment = Schema.decodeEffect(ProviderInstanceEnvironment); const environment = yield* decodeEnvironment([ - { name: "OPENAI_API_KEY", value: source, sensitive: true, valueRedacted: true }, - ]); - const expected = yield* decodeEnvironment([ - { name: "OPENAI_API_KEY", value: source, sensitive: false }, + { name: "OPENAI_API_KEY", value: "", secret: source, sensitive: true, valueRedacted: true }, ]); + const expectedEncoded = [ + { name: "OPENAI_API_KEY", value: "", secret: source, sensitive: true }, + ]; + const expected = yield* decodeEnvironment(expectedEncoded); const next = yield* serverSettings.updateSettings({ providerInstances: { @@ -1445,7 +1711,10 @@ it.layer(NodeServices.layer)("server settings", (it) => { expected, ); const raw = yield* fileSystem.readFileString(serverConfig.settingsPath); - assert.deepEqual(JSON.parse(raw).providerInstances.codex_personal.environment, expected); + assert.deepEqual( + JSON.parse(raw).providerInstances.codex_personal.environment, + expectedEncoded, + ); }).pipe(Effect.provide(layerServerSettings())), ); @@ -1520,9 +1789,8 @@ it.layer(NodeServices.layer)("server settings", (it) => { const serverConfig = yield* ServerConfig.ServerConfig; const fileSystem = yield* FileSystem.FileSystem; const source = (reference: string) => ({ - kind: "secret-provider" as const, provider: SecretProviderId.make("homelab"), - reference: SecretProviderReference.make(reference), + reference, }); const stored = (kind: string, field: string, serverHost: string | null) => secrets.get(sourceControlHostSecretName({ kind, field, serverHost })); @@ -1597,11 +1865,10 @@ it.layer(NodeServices.layer)("server settings", (it) => { const secrets = yield* ServerSecretStore.ServerSecretStore; const id = UsageLimitSourceId.make("hub"); const source = { - kind: "secret-provider" as const, provider: SecretProviderId.make("homelab"), - reference: SecretProviderReference.make("cliproxy#key"), + reference: "cliproxy#key", }; - const hub = (managementKey: UsageLimitSourceConfig["managementKey"]) => ({ + const hub = (managementKey: SettingsSecretValue) => ({ [id]: { kind: "cliproxy" as const, url: "https://hub.example.test", @@ -1759,6 +2026,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { const fileSystem = yield* FileSystem.FileSystem; let failRename = false; let settingsPathToFail: string | undefined; + let readDuringWrite: Effect.Effect = Effect.void; const writeFailure = PlatformError.systemError({ _tag: "PermissionDenied", module: "FileSystem", @@ -1769,7 +2037,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { ...fileSystem, rename: (fromPath, toPath) => failRename && toPath === settingsPathToFail - ? Effect.fail(writeFailure) + ? readDuringWrite.pipe(Effect.andThen(Effect.fail(writeFailure))) : fileSystem.rename(fromPath, toPath), }); const instanceId = ProviderInstanceId.make("antigravity_write_failure"); @@ -1791,6 +2059,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { }); failRename = true; + readDuringWrite = Effect.ignore(serverSettings.getSettings); const failedUpdate = yield* serverSettings .updateProviderInstance({ operation: "upsert", diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index bf0c13ea2440e..061024ec6a644 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -3,12 +3,11 @@ import { ModelSelection, ProjectId, ProjectScript, + keepUnknownOverrideKeys, ProjectSettingsOverrides, type ProviderInstanceConfig, type ProviderInstanceMutation, defaultInstanceIdForDriver, - isFlatSecretProviderConfig, - isKnownSecretProviderKind, isProviderDriverKind, ProviderDriverKind, ProviderInstanceId, @@ -53,6 +52,8 @@ import { isModelSelectionProviderEnabled, } from "@t3tools/shared/serverSettings"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; +import { migrateLegacySecretSettings } from "./secrets/legacySecretSettings.ts"; +import { secretProviderDefinitions } from "./secrets/secretProviderDefinitions.ts"; import { readSecretFieldValues, sourceControlHostSecretName, @@ -144,9 +145,7 @@ function withoutUnknownSecretProviderConfigs( ): ServerSettings["secretProviders"] { return Object.fromEntries( Object.entries(settings.secretProviders).map(([id, entry]) => { - if (isFlatSecretProviderConfig(entry) || isKnownSecretProviderKind(entry.kind)) { - return [id, entry]; - } + if (secretProviderConfigGuards.has(entry.kind)) return [id, entry]; const { config: _config, ...envelope } = entry; return [id, envelope]; }), @@ -166,6 +165,40 @@ export function applyProviderInstanceMutation( return { ...settings, providerInstances }; } +const secretProviderConfigGuards = new Map( + secretProviderDefinitions.definitions.map((definition) => [ + definition.kind as string, + Schema.is(definition.configSchema), + ]), +); + +// Rejects a kind this server has not loaded rather than storing a config it cannot check. +function ensureSecretProviderWritesAllowed( + patch: ServerSettingsPatch, + settingsPath: string, +): Effect.Effect { + for (const [id, entry] of Object.entries(patch.secretProviders ?? {})) { + if (entry === null) continue; + const isConfig = secretProviderConfigGuards.get(entry.kind); + const problem = + isConfig === undefined + ? `Unsupported secret provider kind: ${entry.kind}.` + : isConfig(entry.config) + ? undefined + : `Invalid ${entry.kind} config for secret provider ${id}.`; + if (problem !== undefined) { + return Effect.fail( + new ServerSettingsError({ + settingsPath, + operation: "write-secret-provider", + cause: new Error(problem), + }), + ); + } + } + return Effect.void; +} + function ensureProviderInstanceMutationAllowed( settings: ServerSettings, mutation: ProviderInstanceMutation, @@ -295,10 +328,12 @@ const ServerSettingsJson = fromLenientJson( ), projectSettingsOverrides: Schema.Record( ProjectId, - Schema.Struct({ - ...ProjectSettingsOverrides.fields, - responseStreamingMode: Schema.optionalKey(PersistedResponseStreamingMode), - }), + keepUnknownOverrideKeys( + Schema.Struct({ + ...ProjectSettingsOverrides.fields, + responseStreamingMode: Schema.optionalKey(PersistedResponseStreamingMode), + }), + ), ).pipe(Schema.withDecodingDefault(Effect.succeed({}))), }), ); @@ -311,6 +346,21 @@ const LegacyProviderSettingsJson = fromLenientJson( }), ); const decodeLegacyProviderSettingsJsonExit = Schema.decodeUnknownExit(LegacyProviderSettingsJson); +const decodeRawSettingsJsonExit = Schema.decodeUnknownExit(fromLenientJson(Schema.Unknown)); + +const migrateRawLegacySecrets = ( + raw: string, +): { readonly raw: string; readonly changed: boolean } => { + const parsed = decodeRawSettingsJsonExit(raw); + if (parsed._tag === "Failure" || parsed.value === null || typeof parsed.value !== "object") { + return { raw, changed: false }; + } + if (Array.isArray(parsed.value)) return { raw, changed: false }; + const migrated = migrateLegacySecretSettings(parsed.value as Record); + return migrated.changed + ? { raw: JSON.stringify(migrated.settings), changed: true } + : { raw, changed: false }; +}; const LEGACY_BITBUCKET_SECRET_NAMES: Readonly> = { accessToken: "bitbucket-access-token", @@ -668,9 +718,12 @@ const make = Effect.gen(function* () { let legacyProviders: Readonly> | undefined; let legacySourceControl: Parameters[1] = {}; let parseFailure = Option.none(); + let legacySecretsRewritten = false; if (yield* readConfigExists) { - const raw = yield* readRawConfig; + const migratedRaw = migrateRawLegacySecrets(yield* readRawConfig); + const raw = migratedRaw.raw; + legacySecretsRewritten = migratedRaw.changed; const decoded = decodeServerSettingsJsonExit(raw); const legacySettings = decodeLegacyProviderSettingsJsonExit(raw); if (legacySettings._tag === "Success") { @@ -793,6 +846,7 @@ const make = Effect.gen(function* () { if ( migrated !== loaded || (settingsFileTrusted && legacyProviders !== undefined) || + (settingsFileTrusted && legacySecretsRewritten) || hasLegacySourceControl ) { yield* writeSettingsAtomically(migrated); @@ -851,6 +905,31 @@ const make = Effect.gen(function* () { }, ); + // Background sweeps read settings every few seconds; reading secret files each time is wasted. + // Every secret write rewrites the settings file, so a new cached object implies fresh secrets. + // A read that overlaps a secret write may see files a rollback then restores, so it is not kept. + let materialized: { readonly source: ServerSettings; readonly value: ServerSettings } | null = + null; + let secretWriteGeneration = 0; + const invalidateMaterialized = Effect.sync(() => { + secretWriteGeneration++; + materialized = null; + }); + const getMaterializedSettings = getSettingsFromCache.pipe( + Effect.flatMap((source) => { + if (materialized?.source === source) return Effect.succeed(materialized.value); + const generation = secretWriteGeneration; + return materializeSettingsSecrets(source).pipe( + Effect.map(resolveTextGenerationProvider), + Effect.tap((value) => + Effect.sync(() => { + if (generation === secretWriteGeneration) materialized = { source, value }; + }), + ), + ); + }), + ); + const materializeChanges = (changes: Stream.Stream) => changes.pipe( Stream.mapEffect((settings) => @@ -1026,8 +1105,9 @@ const make = Effect.gen(function* () { const updated = yield* update(current); const persisted = yield* persistSettingsSecrets(current, updated); const next = yield* normalizeServerSettings(persisted.settings); - const materialized = yield* Effect.uninterruptibleMask(() => + const materializedNext = yield* Effect.uninterruptibleMask(() => Effect.gen(function* () { + yield* invalidateMaterialized; const rollbackSecretChanges = yield* applySettingsSecretChanges(persisted.changes); const materializedExit = yield* Effect.exit(materializeSettingsSecrets(next)); if (Exit.isFailure(materializedExit)) { @@ -1040,22 +1120,18 @@ const make = Effect.gen(function* () { return yield* Effect.failCause(writeExit.cause); } return materializedExit.value; - }), + }).pipe(Effect.ensuring(invalidateMaterialized)), ); yield* Cache.set(settingsCache, cacheKey, { settings: next, parseFailure: Option.none() }); + const resolved = resolveTextGenerationProvider(materializedNext); + materialized = { source: next, value: resolved }; yield* emitChange(next); - return resolveTextGenerationProvider(materialized); + return resolved; }), ); const withSettingsSnapshot: ServerSettingsService["Service"]["withSettingsSnapshot"] = (use) => - writeSemaphore.withPermits(1)( - getSettingsFromCache.pipe( - Effect.flatMap(materializeSettingsSecrets), - Effect.map(resolveTextGenerationProvider), - Effect.flatMap(use), - ), - ); + writeSemaphore.withPermits(1)(getMaterializedSettings.pipe(Effect.flatMap(use))); const revalidateAndEmit = writeSemaphore.withPermits(1)( Effect.gen(function* () { @@ -1160,17 +1236,17 @@ const make = Effect.gen(function* () { return { start, ready: Deferred.await(startedDeferred), - getSettings: getSettingsFromCache.pipe( - Effect.flatMap(materializeSettingsSecrets), - Effect.map(resolveTextGenerationProvider), - ), + getSettings: getMaterializedSettings, updateSettings: (patch) => updateAndPersistSettings((current) => - Effect.succeed(applyServerSettingsPatch(current, patch)), + ensureSecretProviderWritesAllowed(patch, settingsPath).pipe( + Effect.as(applyServerSettingsPatch(current, patch)), + ), ), updateProviderInstance: (mutation, patch = {}) => updateAndPersistSettings((current) => Effect.gen(function* () { + yield* ensureSecretProviderWritesAllowed(patch, settingsPath); yield* ensureProviderInstanceMutationAllowed(current, mutation, settingsPath); const patched = applyServerSettingsPatch(current, patch); return applyProviderInstanceMutation(patched, mutation); diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts index 91359e205af2d..9cadc433a70a3 100644 --- a/apps/server/src/settingsSecrets.ts +++ b/apps/server/src/settingsSecrets.ts @@ -3,9 +3,7 @@ import { ClaudeSettings, CodexSettings, EasDeviceHostConfig, - FlatSecretProviderConfig, - isFlatSecretProviderConfig, - isSecretProviderSecretSource, + isSecretReference, type ServerSettings, SimlockDeviceHostConfig, UsageLimitSourceConfig, @@ -36,11 +34,9 @@ const providerSecretFields = new Map([ ), ]); const usageLimitSecretFields = secretSettingsFields(UsageLimitSourceConfig); -const secretProviderSecretFields = new Map( - FlatSecretProviderConfig.members.map( - (member) => [member.fields.kind.literal, secretSettingsFields(member)] as const, - ), -); +const SECRET_PROVIDER_CREDENTIAL_FIELDS = new Map>([ + ["openbao", ["token"]], +]); const easDeviceHostSecretFields = secretSettingsFields(EasDeviceHostConfig); const simlockDeviceHostSecretFields = secretSettingsFields(SimlockDeviceHostConfig); const encode = (value: string) => Buffer.from(value, "utf8").toString("base64url"); @@ -68,7 +64,7 @@ export function mapSettingsSecrets( ) => { const next = { ...fields }; for (const field of secretFields) { - if (!(field in next) || isSecretProviderSecretSource(next[field])) continue; + if (!(field in next) || isSecretReference(next[field])) continue; next[field] = writeSecretFieldValues( next[field], readSecretFieldValues(next[field]).map(([serverHost, value]) => { @@ -107,21 +103,19 @@ export function mapSettingsSecrets( ...(instance.environment ? { environment: instance.environment.map((variable) => { - // A secret source names a secret without carrying it, so it is never - // sensitive, never redacted, and never touches the secret store. - if (typeof variable.value !== "string") { - return { name: variable.name, value: variable.value, sensitive: false }; - } + const { valueRedacted: _omit, ...rest } = variable; + // A secret reference is not itself secret; storage sees the value cleared. + const reference = typeof variable.value === "string" ? undefined : variable.value; const result = map({ secretName: `provider-env-${encode(instanceId)}-${encode(variable.name)}`, kind: "environment", - value: variable.value, - redacted: variable.valueRedacted === true, + value: typeof variable.value === "string" ? variable.value : "", + redacted: reference === undefined && variable.valueRedacted === true, sensitive: variable.sensitive, providerInstanceId: instanceId, environmentVariable: variable.name, }); - const { valueRedacted: _omit, ...rest } = variable; + if (reference !== undefined) return { ...rest, value: reference }; return { ...rest, value: result.value, @@ -160,18 +154,29 @@ export function mapSettingsSecrets( ) as ServerSettings["usageLimitSources"]; const secretProviders = Object.fromEntries( - Object.entries(settings.secretProviders).map(([id, provider]) => [ - id, - mapFields( - provider, - isFlatSecretProviderConfig(provider) - ? (secretProviderSecretFields.get(provider.kind) ?? []) - : [], - (field, serverHost) => - `secret-provider-${encode(id)}-${encode(field)}` + - (serverHost === null ? "" : `-${encode(serverHost)}`), - ), - ]), + Object.entries(settings.secretProviders).map(([id, provider]) => { + const config = provider.config; + const fields = SECRET_PROVIDER_CREDENTIAL_FIELDS.get(provider.kind); + if ( + fields === undefined || + config === null || + typeof config !== "object" || + Array.isArray(config) + ) { + return [id, provider]; + } + return [ + id, + { + ...provider, + config: mapFields( + config as Record, + fields, + (field) => `secret-provider-${encode(id)}-${encode(field)}`, + ), + }, + ]; + }), ) as ServerSettings["secretProviders"]; const easDeviceHosts = settings.easDeviceHosts.map( diff --git a/apps/server/src/sourceControl/ServerSourceControlHost.ts b/apps/server/src/sourceControl/ServerSourceControlHost.ts index a5e2a8fcc3019..4c2f52a67e1df 100644 --- a/apps/server/src/sourceControl/ServerSourceControlHost.ts +++ b/apps/server/src/sourceControl/ServerSourceControlHost.ts @@ -2,10 +2,8 @@ import * as SourceControlHost from "@t3tools/source-control-core/server/SourceCo import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import { - ProviderSecretResolver, - resolveSettingsSecret, -} from "../provider/ProviderSecretResolver.ts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import { resolveSettingsSecret } from "../secrets/settingsSecretValue.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; @@ -19,7 +17,7 @@ export const layer = Layer.effect( const process = yield* VcsProcess.VcsProcess; const git = yield* GitVcsDriver.GitVcsDriver; const vcsRegistry = yield* VcsDriverRegistry.VcsDriverRegistry; - const secrets = yield* ProviderSecretResolver; + const secrets = yield* SecretResolver.SecretResolver; return SourceControlHost.SourceControlHost.of({ settings: { get: serverSettings.getSettings.pipe( diff --git a/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts b/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts index 249bc07b3ad3f..05b6a1a8d0e42 100644 --- a/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts +++ b/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts @@ -118,7 +118,12 @@ function makeRegistry(input: { kind: provider.kind, label: provider.kind, installHint: "unused", - probeAuth: Effect.die("unused"), + probeAuth: Effect.succeed({ + status: "unknown", + account: Option.none(), + host: Option.none(), + detail: Option.none(), + }), }, })), ).pipe( @@ -440,3 +445,28 @@ it.effect("asks again once Settings change", () => assert.strictEqual((yield* resolveUnknown(registry, "c")).provider.kind, "github"); }), ); + +it.effect("answers discovery without waiting on a hosting API that stalls", () => + Effect.gen(function* () { + const registry = yield* makeRegistry({ + remotes: [], + discovery: [ + claimingCli("gitlab"), + { + type: "api", + kind: kind("bitbucket"), + label: "Bitbucket", + installHint: "unused", + probeAuth: Effect.never, + }, + ], + }); + const discovering = yield* registry.discover.pipe(Effect.forkChild); + yield* TestClock.adjust("3 seconds"); + const items = yield* Fiber.join(discovering); + const bitbucket = items.find((item) => item.kind === "bitbucket"); + assert.strictEqual(bitbucket?.status, "available"); + assert.strictEqual(bitbucket?.auth.status, "unknown"); + assert.strictEqual(items.find((item) => item.kind === "gitlab")?.status, "available"); + }), +); diff --git a/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts b/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts index 4a9105e9f5d9e..b0770b06d4830 100644 --- a/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts +++ b/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts @@ -383,6 +383,62 @@ it.effect.each([ }), ); +it.effect.each([ + ["a plain token", "s3cret"], + ["a token with an apostrophe", "s3'cret"], + ["a token with a double quote", 's3"cret'], + ["a host-only token with an at sign", "abc@secret"], + ["an unquoted token with an apostrophe", "s3'cret", true], + ["an unquoted token with a double quote", 's3"cret', true], + ["an unquoted URL with an apostrophe in its path", "path-token", true], +] as const)("keeps %s in the query out of a failed clone's error text", ([, token, unquoted]) => + Effect.gen(function* () { + const remote = + token === "abc@secret" + ? "https://github.com" + : token === "path-token" + ? "https://github.com/octocat/t3code's.git" + : "https://github.com/octocat/t3code.git"; + const error = yield* Effect.flip( + cloneInTemporaryDirectory( + { + execute: (input) => + Effect.gen(function* () { + const cloneUrl = input.args.at(-2)!; + // Git treats @ in a pathless URL's query as userinfo, losing the query delimiter. + const diagnosticUrl = + cloneUrl === "https://github.com?access_token=abc@secret" + ? "https://secret" + : cloneUrl; + yield* ( + input.progress?.onStderrLine?.( + unquoted + ? `fatal: ${diagnosticUrl}/info/refs not valid: is this a git repository?` + : `fatal: unable to access '${diagnosticUrl}/': Could not resolve host: github.com`, + ) ?? Effect.void + ); + return yield* new GitCommandError({ + operation: input.operation, + command: "git", + cwd: input.cwd, + detail: "Git command exited with a non-zero status.", + exitCode: 128, + }); + }), + }, + { remoteUrl: `${remote}?access_token=${token}` }, + ), + ); + + assert.strictEqual( + error.detail, + unquoted + ? `fatal: ${remote} not valid: is this a git repository?` + : `fatal: unable to access '${remote}${token === "abc@secret" ? "/" : ""}': Could not resolve host: github.com`, + ); + }), +); + it.effect("strips embedded credentials from the remote URL it reports", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/sourceControl/SourceControlRepositoryService.ts b/apps/server/src/sourceControl/SourceControlRepositoryService.ts index b6e7679037040..44d31a43b6978 100644 --- a/apps/server/src/sourceControl/SourceControlRepositoryService.ts +++ b/apps/server/src/sourceControl/SourceControlRepositoryService.ts @@ -110,7 +110,7 @@ function toRepositoryInfo( }; } -// Redact credentials from client clone snapshots; git still receives the original URL. +// Redact credentials from client clone snapshots; git still receives the credentials. function redactRemoteUrl(remoteUrl: string): string { try { const url = new URL(remoteUrl); @@ -128,10 +128,16 @@ function redactRemoteUrl(remoteUrl: string): string { } // Userinfo can contain @; remove through the last one before the host boundary. -const URL_WITH_USERINFO = /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/]+@/gi; +const URL_WITH_USERINFO = /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#]+@/gi; +// Git leaves quotes inside query tokens unescaped; consume through the closing quote. +const QUOTED_URL_QUERY = /(['"])([a-z][a-z0-9+.-]*:\/\/[^\s?]*)\?.*\1(?=[\s:]|$)/gi; +const URL_QUERY = /\b([a-z][a-z0-9+.-]*:\/\/[^\s?]*)\?[^\s]*/gi; function redactUrlCredentials(text: string): string { - return text.replace(URL_WITH_USERINFO, "$1"); + return text + .replace(QUOTED_URL_QUERY, "$1$2$1") + .replace(URL_QUERY, "$1") + .replace(URL_WITH_USERINFO, "$1"); } // The remote already exists after push failure; advise fixing access and pushing. @@ -340,11 +346,13 @@ export const make = Effect.gen(function* () { if (stderrTail.length > 4) stderrTail.shift(); }); }; + // Git misreads @ in a pathless URL's query as userinfo and echoes its suffix as the host. + const normalizedCloneUrl = cloneUrl.replace(/^([a-z][a-z0-9+.-]*:\/\/[^/?#]+)\?/i, "$1/?"); yield* git .execute({ operation: "SourceControlRepositoryService.cloneRepository", cwd: path.dirname(destinationPath), - args: ["clone", "--progress", "--", cloneUrl, path.basename(destinationPath)], + args: ["clone", "--progress", "--", normalizedCloneUrl, path.basename(destinationPath)], timeoutMs: options?.timeoutMs === undefined ? CLONE_TIMEOUT_MS : options.timeoutMs, // Buffered output is unused and progress can be huge; cap it while retaining line callbacks. maxOutputBytes: 256 * 1024, diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts index 05476194ca76f..1e890301169e8 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts @@ -1,22 +1,15 @@ import { assert, it } from "@effect/vitest"; -import { - type ProviderInstanceEnvironment, - SecretProviderId, - SecretProviderReference, -} from "@t3tools/contracts"; +import { SecretProviderId } from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; -import { - type ProviderSecretResolverShape, - ProviderSecretResolver, - resolveSettingsSecret, -} from "../provider/ProviderSecretResolver.ts"; +import { resolveSettingsSecret } from "../secrets/settingsSecretValue.ts"; import { resolveSourceControlHostSecrets } from "./sourceControlHostSecrets.ts"; const source = (reference: string) => ({ - kind: "secret-provider" as const, provider: SecretProviderId.make("homelab"), - reference: SecretProviderReference.make(reference), + reference, }); const secretsByReference = new Map([ @@ -24,27 +17,14 @@ const secretsByReference = new Map([ ["bitbucket#token", "bb_from_provider"], ]); -const fakeResolver = Effect.gen(function* () { - const fallback = yield* ProviderSecretResolver; - const resolve: ProviderSecretResolverShape["resolve"] = ( - environment: ProviderInstanceEnvironment | undefined, - ) => - Effect.sync(() => { - const variables = []; - const unresolved = []; - for (const variable of environment ?? []) { - const secret = - typeof variable.value === "object" && variable.value.kind === "secret-provider" - ? secretsByReference.get(variable.value.reference) - : undefined; - if (secret === undefined) unresolved.push(variable.name); - else variables.push({ name: variable.name, value: secret, sensitive: true }); - } - return { variables, unresolved }; - }); - return { ...fallback, resolve }; +const layerResolver = Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => + Effect.succeed(references.map((reference) => secretsByReference.get(reference.reference))), + invalidate: () => Effect.void, }); +const fakeResolver = SecretResolver.SecretResolver.use((resolver) => Effect.succeed(resolver)); + it.effect("replaces credential references with the secrets they name", () => Effect.gen(function* () { const resolver = yield* fakeResolver; @@ -61,7 +41,7 @@ it.effect("replaces credential references with the secrets they name", () => bitbucket: { email: "me@example.test", accessToken: "bb_from_provider" }, gitcafe: { token: "" }, }); - }), + }).pipe(Effect.provide(layerResolver)), ); it.effect("hands back the same settings when no credential names a secret", () => @@ -72,5 +52,5 @@ it.effect("hands back the same settings when no credential names a secret", () = resolveSettingsSecret(resolver, value), ); assert.strictEqual(resolved, hosts); - }), + }).pipe(Effect.provide(layerResolver)), ); diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.ts index 08ea2147d6392..e9bb2bbcb09cb 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.ts @@ -1,8 +1,4 @@ -import { - isSecretProviderSecretSource, - type SecretProviderSecretSource, - secretSettingsFields, -} from "@t3tools/contracts"; +import { isSecretReference, type SecretReference, secretSettingsFields } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import { sourceControlDefinitions } from "./sourceControlDefinitions.ts"; @@ -35,7 +31,7 @@ export function readSecretFieldValues( value: unknown, ): ReadonlyArray { if (typeof value === "string") return [[null, value]]; - if (isSecretProviderSecretSource(value)) return []; + if (isSecretReference(value)) return []; if (value === null || typeof value !== "object" || Array.isArray(value)) return []; return Object.entries(value).flatMap(([serverHost, entry]) => typeof entry === "string" ? [[serverHost.trim().toLowerCase(), entry] as const] : [], @@ -47,13 +43,11 @@ export function writeSecretFieldValues( values: ReadonlyArray, ): unknown { if (typeof original === "string") return values[0]?.[1] ?? ""; - if (isSecretProviderSecretSource(original)) return original; + if (isSecretReference(original)) return original; const sources = original !== null && typeof original === "object" && !Array.isArray(original) ? Object.entries(original).flatMap(([serverHost, entry]) => - isSecretProviderSecretSource(entry) - ? [[serverHost.trim().toLowerCase(), entry] as const] - : [], + isSecretReference(entry) ? [[serverHost.trim().toLowerCase(), entry] as const] : [], ) : []; return Object.fromEntries([ @@ -67,15 +61,15 @@ type SourceControlHostsSettings = Readonly( hosts: SourceControlHostsSettings, - resolve: (source: SecretProviderSecretSource) => Effect.Effect, + resolve: (source: SecretReference) => Effect.Effect, ): Effect.Effect { const resolveField = (value: unknown): Effect.Effect | null => { - if (isSecretProviderSecretSource(value)) return resolve(value); + if (isSecretReference(value)) return resolve(value); if (value === null || typeof value !== "object" || Array.isArray(value)) return null; const entries = Object.entries(value); - if (!entries.some(([, entry]) => isSecretProviderSecretSource(entry))) return null; + if (!entries.some(([, entry]) => isSecretReference(entry))) return null; return Effect.forEach(entries, ([serverHost, entry]) => - isSecretProviderSecretSource(entry) + isSecretReference(entry) ? resolve(entry).pipe(Effect.map((secret) => [serverHost, secret] as const)) : Effect.succeed([serverHost, entry] as const), ).pipe(Effect.map((resolved) => Object.fromEntries(resolved))); diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index 5b166d8b1452b..7e92567772950 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -10,20 +10,25 @@ import { TerminalProviderInstanceNotFoundError, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Base64Url from "effect/encoding/Base64Url"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; import * as Path from "effect/Path"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SynchronizedRef from "effect/SynchronizedRef"; +import * as TestClock from "effect/testing/TestClock"; import { expect, vi } from "vite-plus/test"; import * as ProcessRunner from "../processRunner.ts"; @@ -756,6 +761,154 @@ it.layer( }), ); + // Answers process-table queries after `queryDuration`, honoring the caller's + // timeout the way ProcessRunner's `timedOutResult` mode does. + const slowProcessTable = ( + platform: "linux" | "win32", + queryDuration: Duration.Input, + queries: Array<{ start: number; end?: number }>, + started: ReadonlyArray>, + ): ProcessRunner.ProcessRunner["Service"] => ({ + run: (input) => + Effect.gen(function* () { + const query: { start: number; end?: number } = { start: yield* Clock.currentTimeMillis }; + queries.push(query); + const deferred = started[queries.length - 1]; + if (deferred) yield* Deferred.succeed(deferred, undefined); + const completed = yield* Effect.sleep(queryDuration).pipe( + Effect.timeoutOption(input.timeout ?? "1 hour"), + ); + query.end = yield* Clock.currentTimeMillis; + return { + stdout: Option.isNone(completed) + ? "" + : platform === "win32" + ? "100|9000|node.exe\n" + : " 100 9000 node\n", + stderr: "", + code: Option.isSome(completed) ? ChildProcessSpawner.ExitCode(0) : null, + timedOut: Option.isNone(completed), + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + }; + }), + }); + + const createFallbackManager = ( + platform: "linux" | "win32", + runner: ProcessRunner.ProcessRunner["Service"], + ) => + createManager(5, { subprocessPollIntervalMs: 1_000, processKillGraceMs: 0 }).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.provide(layerWithHostPlatform(platform)), + ); + + it.effect.each(["linux", "win32"] as const)( + "reports activity from a slow but successful %s process-table query", + (platform) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const queries: Array<{ start: number; end?: number }> = []; + const { manager, getEvents } = yield* createFallbackManager( + platform, + slowProcessTable(platform, "5 seconds", queries, [started]), + ); + + yield* manager.open(openInput()); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(started); + yield* TestClock.adjust("5 seconds"); + + expect( + (yield* getEvents).some( + (event) => + event.type === "activity" && event.hasRunningSubprocess && event.label === "node", + ), + ).toBe(true); + yield* manager.close({ threadId: "thread-1" }); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("idles at least as long as a slow process-table query took before the next", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const queries: Array<{ start: number; end?: number }> = []; + const { manager } = yield* createFallbackManager( + "linux", + slowProcessTable("linux", "5 seconds", queries, [started]), + ); + + yield* manager.open(openInput()); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(started); + yield* TestClock.adjust("5 seconds"); + yield* TestClock.adjust(Duration.millis(19_999)); + + expect(queries).toHaveLength(1); + yield* TestClock.adjust("1 millis"); + const [slow, next] = queries; + expect(next?.start).toBe(slow!.end! + 20_000); + yield* manager.close({ threadId: "thread-1" }); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("ends a hung process-table query and paces the retry", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const queries: Array<{ start: number; end?: number }> = []; + const { manager } = yield* createFallbackManager( + "win32", + slowProcessTable("win32", "1 day", queries, [started]), + ); + + yield* manager.open(openInput()); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(started); + yield* TestClock.adjust("2 minutes"); + yield* TestClock.adjust("2 minutes"); + + const [hung, next] = queries; + expect(hung!.end! - hung!.start).toBe(15_000); + expect(next!.start - hung!.end!).toBe(60_000); + yield* manager.close({ threadId: "thread-1" }); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect.each([ + ["linux", 1_000], + ["win32", 1_500], + ] as const)( + "bounds closeIdle's %s query so other operations on the thread still run", + ([platform, budgetMs]) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const queries: Array<{ start: number; end?: number }> = []; + const { manager, ptyAdapter } = yield* createFallbackManager( + platform, + slowProcessTable(platform, "1 day", queries, [started]), + ); + yield* manager.open(openInput()); + + const closing = yield* manager + .closeIdle({ threadId: "thread-1" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(started); + const resizing = yield* manager + .resize({ threadId: "thread-1", terminalId: DEFAULT_TERMINAL_ID, cols: 80, rows: 30 }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(Duration.millis(budgetMs)); + + expect(closing.pollUnsafe()).toBeDefined(); + expect(resizing.pollUnsafe()).toBeDefined(); + expect(queries[0]!.end! - queries[0]!.start).toBe(budgetMs); + expect(ptyAdapter.processes[0]!.resizeCalls).toEqual([{ cols: 80, rows: 30 }]); + expect(ptyAdapter.processes[0]!.killed).toBe(false); + yield* manager.close({ threadId: "thread-1" }); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("keeps terminals that get input or output while closeIdle checks them", () => Effect.gen(function* () { const ptyAdapter = new FakePtyAdapter(); diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index c61e9ff5dc7b3..825e426587609 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -46,6 +46,7 @@ import { getTerminalLabel } from "@t3tools/shared/terminalLabels"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Base64Url from "effect/encoding/Base64Url"; import * as Equal from "effect/Equal"; @@ -55,6 +56,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; @@ -98,6 +100,12 @@ const MAX_HISTORY_CHUNK_LENGTH = 16 * 1024; const DEFAULT_PERSIST_DEBOUNCE_MS = 250; const DEFAULT_SUBPROCESS_POLL_INTERVAL_MS = 1_000; const MAX_SUBPROCESS_POLL_INTERVAL_MS = 60_000; +// `ps` and WMI can take seconds on a loaded host and their tables are still +// valid then; the timeout only ends a query that is truly stuck. +const PROCESS_TABLE_QUERY_TIMEOUT = "15 seconds"; +// closeIdle queries while holding the thread lock, so it keeps a short budget +// rather than stalling other operations on that thread behind a slow table. +const CLOSE_IDLE_QUERY_TIMEOUT = { posix: "1 second", win32: "1500 millis" } as const; // Discover children started without terminal input or output. const QUIET_SUBPROCESS_SNAPSHOT_EVERY_TICKS = 30; const DEFAULT_PROCESS_KILL_GRACE_MS = 1_000; @@ -678,8 +686,17 @@ interface TerminalProcessTableSnapshot { readonly commandById: ReadonlyMap; } -function subprocessSnapshotPollDelayMs(pollIntervalMs: number, failureCount: number): number { - return Math.min(pollIntervalMs * 2 ** failureCount, MAX_SUBPROCESS_POLL_INTERVAL_MS); +// Idles at least four times as long as the last fallback query took, so a slow +// process table never keeps the poll loop busy most of the time. +function subprocessSnapshotPollDelayMs( + pollIntervalMs: number, + failureCount: number, + queryElapsedMs: number, +): number { + return Math.min( + Math.max(pollIntervalMs * 2 ** failureCount, queryElapsedMs * 4), + MAX_SUBPROCESS_POLL_INTERVAL_MS, + ); } function parsePosixProcessTable(stdout: string): TerminalProcessTableSnapshot { @@ -767,6 +784,7 @@ const resolvePosixPsCommand = Effect.fn("terminal.resolvePosixPsCommand")(functi const posixProcessTableSnapshot = Effect.fn("terminal.posixProcessTableSnapshot")(function* ( psCommand: string, + timeout: Duration.Input, ): Effect.fn.Return< TerminalProcessTableSnapshot, TerminalSubprocessCheckError, @@ -777,7 +795,7 @@ const posixProcessTableSnapshot = Effect.fn("terminal.posixProcessTableSnapshot" .run({ command: psCommand, args: ["-eo", "pid=,ppid=,comm="], - timeout: "1 second", + timeout, maxOutputBytes: 524_288, outputMode: "truncate", timeoutBehavior: "timedOutResult", @@ -803,48 +821,46 @@ const posixProcessTableSnapshot = Effect.fn("terminal.posixProcessTableSnapshot" return parsePosixProcessTable(result.stdout); }); -const windowsProcessTableSnapshot = Effect.fn("terminal.windowsProcessTableSnapshot")( - function* (): Effect.fn.Return< - TerminalProcessTableSnapshot, - TerminalSubprocessCheckError, - ProcessRunner.ProcessRunner - > { - const processRunner = yield* ProcessRunner.ProcessRunner; - const command = - 'Get-CimInstance Win32_Process -ErrorAction Stop | ForEach-Object { Write-Output "$($_.ProcessId)|$($_.ParentProcessId)|$($_.Name)" }'; - const result = yield* processRunner - .run({ - command: "powershell.exe", - args: ["-NoProfile", "-NonInteractive", "-Command", command], - timeout: "1500 millis", - maxOutputBytes: 262_144, - outputMode: "truncate", - timeoutBehavior: "timedOutResult", - }) - .pipe( - Effect.mapError( - (cause) => new TerminalSubprocessCheckError({ cause, command: "powershell" }), - ), - ); - if (result.code !== 0 || result.timedOut || result.stdoutTruncated) { - return yield* new TerminalSubprocessCheckError({ - command: "powershell", - exitCode: result.code, - timedOut: result.timedOut, - stdoutTruncated: result.stdoutTruncated, - }); - } - const processes = result.stdout.split(/\r?\n/g).flatMap((line) => { - const [pidRaw, ppidRaw, name = ""] = line.trim().split("|", 3); - const pid = Number(pidRaw); - const ppid = Number(ppidRaw); - return Number.isInteger(pid) && pid > 0 && Number.isInteger(ppid) - ? [{ pid, ppid, name }] - : []; +const windowsProcessTableSnapshot = Effect.fn("terminal.windowsProcessTableSnapshot")(function* ( + timeout: Duration.Input, +): Effect.fn.Return< + TerminalProcessTableSnapshot, + TerminalSubprocessCheckError, + ProcessRunner.ProcessRunner +> { + const processRunner = yield* ProcessRunner.ProcessRunner; + const command = + 'Get-CimInstance Win32_Process -ErrorAction Stop | ForEach-Object { Write-Output "$($_.ProcessId)|$($_.ParentProcessId)|$($_.Name)" }'; + const result = yield* processRunner + .run({ + command: "powershell.exe", + args: ["-NoProfile", "-NonInteractive", "-Command", command], + timeout, + maxOutputBytes: 262_144, + outputMode: "truncate", + timeoutBehavior: "timedOutResult", + }) + .pipe( + Effect.mapError( + (cause) => new TerminalSubprocessCheckError({ cause, command: "powershell" }), + ), + ); + if (result.code !== 0 || result.timedOut || result.stdoutTruncated) { + return yield* new TerminalSubprocessCheckError({ + command: "powershell", + exitCode: result.code, + timedOut: result.timedOut, + stdoutTruncated: result.stdoutTruncated, }); - return processTableSnapshotFromProcesses(processes); - }, -); + } + const processes = result.stdout.split(/\r?\n/g).flatMap((line) => { + const [pidRaw, ppidRaw, name = ""] = line.trim().split("|", 3); + const pid = Number(pidRaw); + const ppid = Number(ppidRaw); + return Number.isInteger(pid) && pid > 0 && Number.isInteger(ppid) ? [{ pid, ppid, name }] : []; + }); + return processTableSnapshotFromProcesses(processes); +}); interface TerminalHistoryChunk { data: string; @@ -1563,54 +1579,74 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func }, ); // Share one process table per poll; per-terminal scans can exhaust PIDs on busy hosts. - const fallbackProcessTableSnapshot = ( - platform === "win32" - ? windowsProcessTableSnapshot() - : posixProcessTableSnapshot(yield* resolvePosixPsCommand()) - ).pipe(Effect.provideService(ProcessRunner.ProcessRunner, processRunner)); - const fetchProcessTableSnapshot: Effect.Effect< - { - readonly snapshot: TerminalProcessTableSnapshot; - readonly snapshotSucceeded: boolean; - }, - TerminalSubprocessCheckError - > = options.processTable - ? options.processTable.pipe( - Effect.map((entries) => ({ - snapshot: processTableSnapshotFromProcesses(entries), - snapshotSucceeded: true, - })), - Effect.catch(() => - fallbackProcessTableSnapshot.pipe( - Effect.map((snapshot) => ({ snapshot, snapshotSucceeded: false })), + // Timed on success and failure so the poll loop can pace the next query. + const posixPsCommand = platform === "win32" ? undefined : yield* resolvePosixPsCommand(); + const fallbackProcessTableSnapshot = (timeout: Duration.Input) => + (posixPsCommand === undefined + ? windowsProcessTableSnapshot(timeout) + : posixProcessTableSnapshot(posixPsCommand, timeout) + ).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, processRunner), + Effect.result, + Effect.timed, + Effect.map(([elapsed, snapshot]) => ({ + snapshot, + queryElapsedMs: Duration.toMillis(elapsed), + })), + ); + // `fallbackTimeout` only bounds the ps/WMI fallback; a native process table + // or custom inspector is unaffected. + const fetchProcessTableSnapshot = ( + fallbackTimeout: Duration.Input, + ): Effect.Effect<{ + readonly snapshot: Result.Result; + readonly snapshotSucceeded: boolean; + readonly queryElapsedMs: number; + }> => + options.processTable + ? options.processTable.pipe( + Effect.map((entries) => ({ + snapshot: Result.succeed(processTableSnapshotFromProcesses(entries)), + snapshotSucceeded: true, + queryElapsedMs: 0, + })), + Effect.catch(() => + fallbackProcessTableSnapshot(fallbackTimeout).pipe( + Effect.map((fallback) => ({ ...fallback, snapshotSucceeded: false })), + ), ), - ), - ) - : fallbackProcessTableSnapshot.pipe( - Effect.map((snapshot) => ({ snapshot, snapshotSucceeded: true })), - ); + ) + : fallbackProcessTableSnapshot(fallbackTimeout).pipe( + Effect.map((fallback) => ({ + ...fallback, + snapshotSucceeded: Result.isSuccess(fallback.snapshot), + })), + ); const customSubprocessInspector = options.subprocessInspector; - const acquireSubprocessInspector: Effect.Effect< - { - readonly inspector: TerminalSubprocessInspector; - readonly snapshotSucceeded: boolean; - }, - TerminalSubprocessCheckError - > = + const acquireSubprocessInspector = ( + fallbackTimeout: Duration.Input, + ): Effect.Effect<{ + readonly inspector: Result.Result; + readonly snapshotSucceeded: boolean; + readonly queryElapsedMs: number; + }> => customSubprocessInspector !== undefined - ? Effect.succeed({ inspector: customSubprocessInspector, snapshotSucceeded: true }) + ? Effect.succeed({ + inspector: Result.succeed(customSubprocessInspector), + snapshotSucceeded: true, + queryElapsedMs: 0, + }) : Effect.map( - fetchProcessTableSnapshot, - ({ - snapshot, - snapshotSucceeded, - }): { - readonly inspector: TerminalSubprocessInspector; - readonly snapshotSucceeded: boolean; - } => ({ - inspector: (terminalPid) => - Effect.succeed(deriveSubprocessInspectResult(snapshot, terminalPid, platform)), + fetchProcessTableSnapshot(fallbackTimeout), + ({ snapshot, snapshotSucceeded, queryElapsedMs }) => ({ + inspector: Result.map( + snapshot, + (table): TerminalSubprocessInspector => + (terminalPid) => + Effect.succeed(deriveSubprocessInspectResult(table, terminalPid, platform)), + ), snapshotSucceeded, + queryElapsedMs, }), ); const subprocessPollIntervalMs = @@ -2638,39 +2674,29 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func })); if (runningSessions.length === 0) { - return true; + return { snapshotSucceeded: true, queryElapsedMs: 0 }; } if ( runningSessions.every(({ session }) => isQuiet(session)) && skippedQuietTicks < QUIET_SUBPROCESS_SNAPSHOT_EVERY_TICKS - 1 ) { skippedQuietTicks += 1; - return true; + return { snapshotSucceeded: true, queryElapsedMs: 0 }; } skippedQuietTicks = 0; - const inspectorOption = yield* acquireSubprocessInspector.pipe( - Effect.asSome, - Effect.catch((reason) => - Effect.logWarning("failed to snapshot processes for terminal subprocess polling", { - reason, - }).pipe( - Effect.as( - Option.none<{ - readonly inspector: TerminalSubprocessInspector; - readonly snapshotSucceeded: boolean; - }>(), - ), - ), - ), + const { inspector, snapshotSucceeded, queryElapsedMs } = yield* acquireSubprocessInspector( + PROCESS_TABLE_QUERY_TIMEOUT, ); - - if (Option.isNone(inspectorOption)) { + if (Result.isFailure(inspector)) { + yield* Effect.logWarning("failed to snapshot processes for terminal subprocess polling", { + reason: inspector.failure, + }); quietSessions = new WeakMap(); - return false; + return { snapshotSucceeded: false, queryElapsedMs }; } - const { inspector: subprocessInspector, snapshotSucceeded } = inspectorOption.value; + const subprocessInspector = inspector.success; const checkSubprocessActivity = Effect.fn("terminal.checkSubprocessActivity")(function* ( candidate: (typeof runningSessions)[number], ) { @@ -2766,7 +2792,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func discard: true, }); - return snapshotSucceeded; + return { snapshotSucceeded, queryElapsedMs }; }); const hasRunningSessions = readManagerState.pipe( @@ -2781,13 +2807,14 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func Effect.flatMap((active) => active ? pollSubprocessActivity().pipe( - Effect.flatMap((snapshotSucceeded) => { + Effect.flatMap(({ snapshotSucceeded, queryElapsedMs }) => { subprocessSnapshotFailureCount = snapshotSucceeded ? 0 : Math.min(subprocessSnapshotFailureCount + 1, 30); const delayMs = subprocessSnapshotPollDelayMs( subprocessPollIntervalMs, subprocessSnapshotFailureCount, + queryElapsedMs, ); return Effect.sleep(delayMs); }), @@ -3423,7 +3450,10 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func running.map((session) => [session.terminalId, terminalActivityMark(session)]), ); // Inspect now so commands started after the last poll still protect their terminals. - const { inspector } = yield* acquireSubprocessInspector; + const { inspector: inspectorResult } = yield* acquireSubprocessInspector( + platform === "win32" ? CLOSE_IDLE_QUERY_TIMEOUT.win32 : CLOSE_IDLE_QUERY_TIMEOUT.posix, + ); + const inspector = yield* Effect.fromResult(inspectorResult); yield* Effect.forEach( running, (session) => diff --git a/apps/server/src/usage/UsageLimitSources.test.ts b/apps/server/src/usage/UsageLimitSources.test.ts index 3e9141bea55d8..1d955d497ef6b 100644 --- a/apps/server/src/usage/UsageLimitSources.test.ts +++ b/apps/server/src/usage/UsageLimitSources.test.ts @@ -1,18 +1,13 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { expect, it } from "@effect/vitest"; -import { - SecretProviderId, - SecretProviderReference, - ServerConfigStreamEvent, - UsageLimitSourceId, -} from "@t3tools/contracts"; +import { SecretProviderId, ServerConfigStreamEvent, UsageLimitSourceId } from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; import { HttpClient, HttpClientResponse } from "effect/http"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; -import { ProviderSecretResolver } from "../provider/ProviderSecretResolver.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as UsageLimitSources from "./UsageLimitSources.ts"; @@ -42,6 +37,10 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => ); }).pipe( Effect.provide([ + Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => Effect.succeed(references.map(() => undefined)), + invalidate: () => Effect.void, + }), ServerSettings.layerTest({ usageLimitSources: { [UsageLimitSourceId.make("local-hub")]: { @@ -93,9 +92,8 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => it.effect("reads a management key from its secret provider", () => { const authorizations: Array = []; const keySource = (reference: string) => ({ - kind: "secret-provider" as const, provider: SecretProviderId.make("homelab"), - reference: SecretProviderReference.make(reference), + reference, }); return Effect.gen(function* () { const sources = yield* UsageLimitSources.make; @@ -130,16 +128,14 @@ it.effect("reads a management key from its secret provider", () => { Layer.mock(BackgroundPolicy.BackgroundPolicy)({ shouldRunScopeWork: () => Effect.succeed(false), }), - Layer.mock(ProviderSecretResolver)({ - resolve: (environment) => - Effect.succeed({ - variables: (environment ?? []).flatMap(({ name, value, sensitive }) => - typeof value !== "string" && value.reference === "cliproxy#key" - ? [{ name, value: "resolved-key", sensitive }] - : [], + Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => + Effect.succeed( + references.map((reference) => + reference.reference === "cliproxy#key" ? "resolved-key" : undefined, ), - unresolved: [], - }), + ), + invalidate: () => Effect.void, }), Layer.succeed( HttpClient.HttpClient, diff --git a/apps/server/src/usage/UsageLimitSources.ts b/apps/server/src/usage/UsageLimitSources.ts index db8bed5ef5566..ec42033167c84 100644 --- a/apps/server/src/usage/UsageLimitSources.ts +++ b/apps/server/src/usage/UsageLimitSources.ts @@ -21,10 +21,8 @@ import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; -import { - ProviderSecretResolver, - resolveSettingsSecret, -} from "../provider/ProviderSecretResolver.ts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import { resolveSettingsSecret } from "../secrets/settingsSecretValue.ts"; import * as Settings from "../serverSettings.ts"; import { makeCliproxyApi } from "./cliproxyApi.ts"; @@ -60,7 +58,7 @@ function missingManagementKey(config: UsageLimitSourceConfig): string { export const make = Effect.gen(function* () { const api = yield* makeCliproxyApi; const settingsService = yield* Settings.ServerSettingsService; - const secrets = yield* ProviderSecretResolver; + const secrets = yield* SecretResolver.SecretResolver; const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy; const stateRef = yield* Ref.make>([]); const changes = yield* Effect.acquireRelease( diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index b77d3437bef78..33f2332dbd4a5 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -142,6 +142,7 @@ const layerService = (input: { ) => Effect.Effect; readonly environment?: NodeJS.ProcessEnv; readonly platform?: NodeJS.Platform; + readonly hostname?: string; }) => layerCursorUsageAccounts( CursorAccountReader.layer.pipe(Layer.provide(CursorKeychain.layer)), @@ -157,6 +158,7 @@ const layerService = (input: { ), Layer.provideMerge(NodeServices.layer), Layer.provideMerge(Layer.succeed(HostProcess.Platform, input.platform ?? "linux")), + Layer.provideMerge(Layer.succeed(HostProcess.Hostname, input.hostname ?? "usage-test-host")), Layer.provideMerge(ServerSettings.layerTest(input.settings)), Layer.provideMerge( Layer.succeed( @@ -465,6 +467,93 @@ describe("Codex usage history", () => { }); describe("UsageService", () => { + it.live("keeps the first host identity across recreation and explicit override removal", () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5))); + for (const [hostname, override, expected] of [ + ["container-a", undefined, "container-a"], + ["container-b", undefined, "container-a"], + ["container-c", "physical-host", "physical-host"], + ["container-d", undefined, "container-a"], + ] as const) { + const summary = yield* Effect.gen(function* () { + const service = yield* UsageService.make; + return yield* service.readSummary(WINDOW); + }).pipe( + Effect.scoped, + Effect.provide( + layerService({ + prefix: "usage-host-id", + home, + settings, + hostname, + environment: override === undefined ? {} : { T3CODE_HOST_ID: override }, + }), + ), + ); + const source = summary.sources.find((source) => source.fingerprint.provider === "claude"); + assert.strictEqual(source?.fingerprint.hostId, expected); + } + }).pipe(Effect.scoped), + ); + + it.live("concurrent initialization keeps one host identity across service recreation", () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5))); + const prefix = "usage-host-id-race"; + const hostIdPath = NodePath.join(home, prefix, "usage-host-id"); + const bothRead = yield* Deferred.make(); + let reads = 0; + const summaries = yield* Effect.forEach( + ["container-a", "container-b"], + (hostname) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + let firstRead = true; + const service = yield* UsageService.make.pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fileSystem, + readFileString: (path, encoding) => + Effect.gen(function* () { + if (path !== hostIdPath || !firstRead) { + return yield* fileSystem.readFileString(path, encoding); + } + firstRead = false; + // Both instances observe the missing file before either can create it. + const result = yield* Effect.exit(fileSystem.readFileString(path, encoding)); + reads += 1; + if (reads === 2) yield* Deferred.succeed(bothRead, undefined); + yield* Deferred.await(bothRead); + return yield* result; + }), + }), + ); + return yield* service.readSummary(WINDOW); + }).pipe( + Effect.scoped, + Effect.provide(layerService({ prefix, home, settings, hostname })), + ), + { concurrency: "unbounded" }, + ); + const persisted = yield* Effect.promise(() => NodeFSP.readFile(hostIdPath, "utf8")); + const hostId = persisted.trim(); + assert.include(["container-a", "container-b"], hostId); + const restarted = yield* Effect.gen(function* () { + const service = yield* UsageService.make; + return yield* service.readSummary(WINDOW); + }).pipe( + Effect.scoped, + Effect.provide(layerService({ prefix, home, settings, hostname: "container-c" })), + ); + for (const summary of [...summaries, restarted]) { + const source = summary.sources.find((source) => source.fingerprint.provider === "claude"); + assert.strictEqual(source?.fingerprint.hostId, hostId); + } + }).pipe(Effect.scoped), + ); + it.live( "repairs initial Codex prices after unchanged-file retries, persists evidence and corrections, and honors later native tiers", () => @@ -2210,7 +2299,7 @@ describe("UsageService", () => { 60 + 20 * (version === 4 ? 1 : 2), ); yield* upgraded.awaitPersisted; - assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v9.json"); + assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v10.json"); assert.strictEqual( yield* Effect.promise(() => NodeFSP.readFile(legacyPath, "utf8")), legacy, diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index c822f3c9eec23..e92e75bff32a0 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -1,5 +1,3 @@ -import * as NodeOS from "node:os"; - import { type ProviderInstanceConfig, ProviderInstanceId, @@ -158,6 +156,7 @@ export const make = Effect.gen(function* () { const settingsService = yield* ServerSettings.ServerSettingsService; const httpClient = yield* HttpClient.HttpClient; const hostEnvironment = yield* HostProcess.Environment; + const hostname = yield* HostProcess.Hostname; const readerContext = yield* Effect.context(); const fileCache: ScanCache = new Map(); @@ -180,6 +179,37 @@ export const make = Effect.gen(function* () { Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.provideService(Path.Path, path), ); + const hostIdPath = path.join(config.baseDir, "usage-host-id"); + const explicitHostId = hostEnvironment["T3CODE_HOST_ID"]?.trim(); + const hostId = + explicitHostId || + (yield* fileSystem.readFileString(hostIdPath).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => + Effect.scoped( + Effect.gen(function* () { + yield* fileSystem.makeDirectory(config.baseDir, { recursive: true }); + const tempDirectory = yield* fileSystem.makeTempDirectoryScoped({ + directory: config.baseDir, + prefix: "usage-host-id.", + }); + const tempFile = path.join(tempDirectory, "identity"); + yield* fileSystem.writeFileString(tempFile, `${hostname}\n`); + // Publish complete contents without replacing another initializer's identity. + yield* fileSystem.link(tempFile, hostIdPath).pipe( + Effect.catchIf( + (error) => error.reason._tag === "AlreadyExists", + () => Effect.void, + ), + ); + return yield* fileSystem.readFileString(hostIdPath); + }), + ), + ), + Effect.map((value) => value.trim() || hostname), + Effect.orElseSucceed(() => hostname), + )); let rates: RateTable = new Map(); let ratesFetchedAtMs: number | null = null; let ratesLastFailedAtMs: number | null = null; @@ -667,7 +697,6 @@ export const make = Effect.gen(function* () { const startedAtMs = yield* Clock.currentTimeMillis; yield* ensureScanCacheLoaded; - const hostId = NodeOS.hostname(); const windowStart = DateTime.make(`${input.sinceDay}T00:00:00Z`); if (Option.isNone(windowStart)) { return yield* new UsageReadError({ diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index 552c97585b87e..477b8bb6fc4f6 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -218,7 +218,11 @@ describe("scan cache round trip", () => { }; const restored = decodeScanCache(JSON.parse(JSON.stringify(legacy)), TEST_FORMAT_MAP); - expect(restored.get("/gone.jsonl")).toEqual(original.get("/gone.jsonl")); + expect(restored.get("/gone.jsonl")).toEqual({ + ...original.get("/gone.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }); expect(restored.get("/gone-codex.jsonl")).toEqual({ ...original.get("/gone-codex.jsonl"), size: -1, @@ -245,6 +249,18 @@ describe("scan cache round trip", () => { }, ); + it("re-parses a Claude transcript cached before advisor usage was counted", () => { + const original = cacheWith([["/claude.jsonl", 100, [record()]]]); + const v9 = { ...JSON.parse(makeScanCacheWriter()(original, {})), version: 9 }; + + const restored = decodeScanCache(JSON.parse(JSON.stringify(v9)), TEST_FORMAT_MAP); + expect(restored.get("/claude.jsonl")).toEqual({ + ...original.get("/claude.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }); + }); + it("rewrites only changed entries and still restores the whole cache", () => { const write = makeScanCacheWriter(); const cache = cacheWith([ diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index a74bc1840c025..70c0c3df17e3e 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -9,7 +9,7 @@ import * as Schema from "effect/Schema"; import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts"; import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; -const USAGE_SCAN_CACHE_VERSION = 9 as const; +const USAGE_SCAN_CACHE_VERSION = 10 as const; const SPEED_COMPATIBLE_SINCE_VERSION = 4; /** @@ -17,8 +17,9 @@ const SPEED_COMPATIBLE_SINCE_VERSION = 4; * replace it, dropping saved usage for deleted transcripts. Separate files keep * both. */ -export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v9.json"; +export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v10.json"; export const LEGACY_SCAN_CACHE_FILE_NAMES = [ + "usage-scan-cache-v9.json", "usage-scan-cache-v8.json", "usage-scan-cache-v7.json", "usage-scan-cache-v6.json", @@ -291,10 +292,12 @@ export function decodeScanCache( } // Keep history for deleted rollouts, but re-parse incompatible reducer state. // v9 repairs skipped initial Codex requests; clean v8 entries can still resume. + // v10 adds Claude advisor calls, which no older Claude entry holds. const legacy = - format.state !== undefined && - (version < 6 || - (entry.p === "codex" && (version < 8 || (version < 9 && malformedRecords > 0)))); + (entry.p === "claude" && version < 10) || + (format.state !== undefined && + (version < 6 || + (entry.p === "codex" && (version < 8 || (version < 9 && malformedRecords > 0))))); // A corrupt state disqualifies the entry: resuming with it would attach // appended usage to the wrong model or replay fork-copied history. if (!legacy && !isValidState.get(entry.p)?.(entry.cs)) continue; diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index c7a06c3bdc31f..88ca1eed8f5ea 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -1006,6 +1006,24 @@ it.effect("GitVcsDriver treats a missing git executable as outside any repositor }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "GitVcsDriver detects a repository whose path contains a newline", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parent = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-newline-repo-" }); + const cwd = path.join(parent, "multi\nline"); + yield* fileSystem.makeDirectory(cwd); + yield* runGit(cwd, ["init"]); + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + + const repository = yield* driver.detectRepository(cwd); + assert.strictEqual(repository?.rootPath, yield* fileSystem.realPath(cwd)); + assert.strictEqual(repository?.metadataPath, ".git"); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( "GitVcsDriver still fails detection when git exists but cannot be spawned", () => diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index acbf817462c12..74e409a8bc184 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -20,6 +20,8 @@ import { type VcsCreateRefResult, type VcsCreateWorktreeInput, type VcsCreateWorktreeResult, + type ReviewBranchCommitsInput, + type ReviewBranchCommitsResult, type ReviewDiffPreviewInput, type ReviewDiffPreviewResult, type ReviewDiffFileContentsInput, @@ -328,6 +330,9 @@ export class GitVcsDriver extends Context.Service< cwd: string, baseRef: string, ) => Effect.Effect; + readonly getReviewBranchCommits: ( + input: ReviewBranchCommitsInput, + ) => Effect.Effect; readonly getReviewDiffPreview: ( input: ReviewDiffPreviewInput, ) => Effect.Effect; @@ -587,25 +592,43 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const detectRepository: VcsDriver.VcsDriver["Service"]["detectRepository"] = Effect.fn( "detectRepository", )(function* (cwd) { - if (!(yield* isInsideWorkTree(cwd))) { - return null; - } - - const root = yield* gitCommand(vcsProcess, "GitVcsDriver.detectRepository.root", cwd, [ - "rev-parse", - "--show-toplevel", - ]); - const gitCommonDir = yield* gitCommand( + // One spawn answers all three; outside a work tree it exits non-zero or prints "false". + const result = yield* gitCommand( vcsProcess, - "GitVcsDriver.detectRepository.commonDir", + "GitVcsDriver.detectRepository", cwd, - ["rev-parse", "--git-common-dir"], - ).pipe(Effect.orElseSucceed(() => null)); + ["rev-parse", "--is-inside-work-tree", "--show-toplevel", "--git-common-dir"], + { + allowNonZeroExit: true, + timeoutMs: 5_000, + maxOutputBytes: 16_384, + }, + ).pipe(Effect.catchIf(isMissingExecutable, () => Effect.succeed(null))); + if (result === null || result.exitCode !== 0) return null; + const lines = result.stdout.replace(/\r?\n$/u, "").split(/\r?\n/u); + if (lines[0]?.trim() !== "true") return null; + // A newline inside a path breaks the line framing; ask for each path on its own. + if (lines.length !== 3) { + const root = yield* gitCommand(vcsProcess, "GitVcsDriver.detectRepository", cwd, [ + "rev-parse", + "--show-toplevel", + ]); + const gitCommonDir = yield* gitCommand(vcsProcess, "GitVcsDriver.detectRepository", cwd, [ + "rev-parse", + "--git-common-dir", + ]).pipe(Effect.orElseSucceed(() => null)); + return { + kind: "git" as const, + rootPath: root.stdout.trim(), + metadataPath: gitCommonDir?.stdout.trim() || null, + freshness: yield* nowFreshness(), + }; + } return { kind: "git" as const, - rootPath: root.stdout.trim(), - metadataPath: gitCommonDir?.stdout.trim() || null, + rootPath: lines[1]!.trim(), + metadataPath: lines[2]!.trim() || null, freshness: yield* nowFreshness(), }; }); diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index fed0740264c37..658c073fc0365 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -1,5 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off - realpathSync.native resolves Windows 8.3 short names, which the Effect realPath does not. import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodeURL from "node:url"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { assert, it, describe } from "@effect/vitest"; @@ -920,6 +922,54 @@ it.effect("retries a fetch that lost a ref update to another Git process", () => }).pipe(Effect.provide(layerServerConfig.pipe(Layer.provideMerge(NodeServices.layer)))), ); +it.effect("lets a worktree checkout that is still working run past five minutes", () => + Effect.gen(function* () { + const checkoutStarted = yield* Deferred.make(); + const checkoutFinished = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (!ChildProcess.isStandardCommand(command)) + return yield* Effect.die("expected Git command"); + if (!command.args.includes("worktree")) return makeSuccessfulHandle(""); + yield* Deferred.succeed(checkoutStarted, undefined); + return ChildProcessSpawner.makeHandle({ + ...makeSuccessfulHandle(""), + exitCode: Deferred.await(checkoutFinished).pipe( + Effect.as(ChildProcessSpawner.ExitCode(0)), + ), + stderr: Stream.encodeText( + Stream.concat( + Stream.make("Updating files: 40% (49052/122629)\n"), + Stream.fromEffect( + Deferred.await(checkoutFinished).pipe( + Effect.as("Updating files: 100% (122629/122629), done.\n"), + ), + ), + ), + ), + }); + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const path = yield* Path.Path; + const cwd = yield* makeTmpDir(); + const worktreePath = path.join(cwd, "worktree"); + + const creation = yield* driver + .createWorktree({ cwd, refName: "main", newRefName: "t3/slow-checkout", path: worktreePath }) + .pipe(Effect.forkChild); + yield* Deferred.await(checkoutStarted); + yield* TestClock.adjust("6 minutes"); + yield* Deferred.succeed(checkoutFinished, undefined); + + const result = yield* Fiber.join(creation); + assert.equal(result.worktree.path, worktreePath); + assert.equal(result.worktree.refName, "t3/slow-checkout"); + }).pipe(Effect.provide(layerServerConfig.pipe(Layer.provideMerge(NodeServices.layer)))), +); + describe("rejected git refs", () => { const remoteUrl = "https://user:token-123@git.example.invalid/org/secret-repo.git"; const failedPush = `error: failed to push some refs to '${remoteUrl}'`; @@ -1418,6 +1468,140 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); describe("review diff previews", () => { + it.effect("diffs a root commit against the empty tree but rejects a shallow boundary", () => + Effect.gen(function* () { + const upstream = yield* makeTmpDir(); + yield* initRepoWithCommit(upstream); + const driver = yield* GitVcsDriver.GitVcsDriver; + const emptyTree = yield* git(upstream, ["hash-object", "-t", "tree", NodeOS.devNull]); + const root = yield* git(upstream, ["rev-parse", "HEAD"]); + const rootSource = (yield* driver.getReviewDiffPreview({ cwd: upstream, commitSha: root })) + .sources[0]!; + assert.equal(rootSource.baseRef, emptyTree); + assert.include(rootSource.diff, "+# test"); + + yield* writeTextFile(upstream, "README.md", "# boundary\n"); + yield* git(upstream, ["commit", "-am", "boundary"]); + const sha = yield* git(upstream, ["rev-parse", "HEAD"]); + const cwd = yield* makeTmpDir(); + yield* git(upstream, ["clone", "--depth=1", NodeURL.pathToFileURL(upstream).href, cwd], { + GIT_CONFIG_GLOBAL: NodeOS.devNull, + GIT_CONFIG_NOSYSTEM: "1", + }); + assert.equal(yield* git(cwd, ["rev-parse", "--is-shallow-repository"]), "true"); + for (const request of [ + driver.getReviewDiffPreview({ cwd, commitSha: sha }), + driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd, { + sourceKind: "commit", + changeType: "new", + headRef: sha, + baseRef: emptyTree, + }), + ), + ]) { + const error = yield* Effect.asVoid(request).pipe( + Effect.flip, + ); + assert.include(error.detail, "parent"); + } + }), + ); + + it.effect("lists the branch's commits newest first, at most 100, and flags the rest", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(cwd, ["checkout", "-b", "feature/commits"]); + for (let index = 0; index < 102; index++) { + yield* git(cwd, ["commit", "--allow-empty", "-m", `feature ${index}`]); + } + + const result = yield* driver.getReviewBranchCommits({ cwd, baseRef: initialBranch }); + assert.strictEqual(result.truncated, true); + assert.strictEqual(result.commits.length, 100); + assert.strictEqual(result.commits[0]?.sha, yield* git(cwd, ["rev-parse", "HEAD"])); + assert.deepStrictEqual( + [result.commits[0]?.subject, result.commits[99]?.subject], + ["feature 101", "feature 2"], + ); + }), + ); + + it.effect("previews one selected commit, a merge against its first parent", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(cwd, ["checkout", "-b", "feature/merge-preview"]); + yield* git(cwd, ["checkout", "-b", "feature/side"]); + yield* writeTextFile(cwd, "README.md", "# side commit\n"); + yield* git(cwd, ["add", "README.md"]); + yield* git(cwd, ["commit", "-m", "side change"]); + yield* git(cwd, ["checkout", "feature/merge-preview"]); + yield* writeTextFile(cwd, "feature.txt", "feature commit\n"); + yield* git(cwd, ["add", "feature.txt"]); + yield* git(cwd, ["commit", "-m", "feature change"]); + const firstParent = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* git(cwd, ["merge", "--no-ff", "--no-edit", "feature/side"]); + const mergeSha = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* git(cwd, ["checkout", "-b", "feature/elsewhere", initialBranch]); + yield* writeTextFile(cwd, "elsewhere.txt", "elsewhere\n"); + yield* git(cwd, ["add", "elsewhere.txt"]); + yield* git(cwd, ["commit", "-m", "elsewhere change"]); + const elsewhereSha = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* git(cwd, ["checkout", "feature/merge-preview"]); + yield* writeTextFile(cwd, "README.md", "# dirty tree\n"); + + const elsewhere = yield* driver.getReviewDiffPreview({ cwd, commitSha: elsewhereSha }); + assert.deepStrictEqual( + elsewhere.sources + .find((candidate) => candidate.kind === "commit") + ?.files?.map((file) => file.path), + ["elsewhere.txt"], + ); + + const preview = yield* driver.getReviewDiffPreview({ + cwd, + baseRef: initialBranch, + commitSha: mergeSha, + }); + const source = preview.sources.find((candidate) => candidate.kind === "commit")!; + assert.strictEqual(source.baseRef, firstParent); + assert.strictEqual(source.headRef, mergeSha); + assert.deepStrictEqual( + source.files?.map((file) => file.path), + ["README.md"], + ); + assert.include(source.diff, "+# side commit"); + assert.notInclude(source.diff, "dirty tree"); + assert.notInclude(source.diff, "feature.txt"); + + const patch = yield* driver.getReviewDiffPreview({ + cwd, + baseRef: initialBranch, + commitSha: mergeSha, + file: { path: "README.md", previousPath: null, sourceKind: "commit" }, + }); + assert.include( + patch.sources.find((candidate) => candidate.kind === "commit")!.diff, + "+# side commit", + ); + const contents = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd, { + sourceKind: "commit", + baseRef: source.baseRef, + headRef: source.headRef, + }), + ); + assert.deepStrictEqual(contents, { + oldContents: "# test\n", + newContents: "# side commit\n", + }); + }), + ); + it.effect("loads repository-relative files from a nested project directory", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 8d51ba4b459ff..bb809e5f3d7a0 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -26,6 +26,7 @@ import { GitCommandError, type GitCommandFailureReason, T3_PROJECT_FILE_NAME, + type ReviewBranchCommitsInput, type ReviewDiffFileContentsInput, type ReviewDiffPreviewInput, type ReviewDiffFileStat, @@ -62,7 +63,6 @@ const branchBaseConfigKey = (branch: string) => `branch.${branch}.t3-merge-base` */ const legacyBranchBaseConfigKey = (branch: string) => `branch.${branch}.gh-merge-base`; const gitProcesses = Semaphore.makeUnsafe(8); -const WORKTREE_ADD_TIMEOUT_MS = 300_000; const WORKTREE_REMOVE_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000; const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]"; @@ -75,6 +75,9 @@ const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; // Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap. const REVIEW_UNTRACKED_MAX_FILES = 5_000; const REVIEW_DIFF_FILE_MAX_OUTPUT_BYTES = 1024 * 1024; +const REVIEW_BRANCH_COMMIT_LIMIT = 100; +const REVIEW_BRANCH_SUBJECT_LIMIT = 300; +const REVIEW_BRANCH_COMMIT_MAX_OUTPUT_BYTES = 128 * 1024; // Client patch parsers require a/ and b/ prefixes; override user diff prefix settings. export const PATCH_RENDER_PREFIX_ARGS = ["--src-prefix=a/", "--dst-prefix=b/"] as const; const REVIEW_DIFF_ARGS = [ @@ -2103,7 +2106,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return yield* lockError; } } - const statusResult = yield* executeGitWithStableDiagnostics( + const repositoryPaths = yield* resolveRepositoryPaths(cwd).pipe( + Effect.catchTags({ GitCommandError: () => Effect.succeed(null) }), + ); + const statusCacheKey = repositoryPaths?.gitCommonDir; + const readStatus = executeGitWithStableDiagnostics( "GitVcsDriver.statusDetails.status", cwd, statusArgs, @@ -2117,34 +2124,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* isMissingGitCwdError(error) ? Effect.succeed(null) : Effect.fail(error), }), ); - - if (statusResult === null) { - return NON_REPOSITORY_STATUS_DETAILS; - } - - if (statusResult.exitCode !== 0) { - if (isNonRepositoryGitStderr(statusResult.stderr)) { - return NON_REPOSITORY_STATUS_DETAILS; - } - return yield* new GitCommandError({ - ...gitCommandContext({ - operation: "GitVcsDriver.statusDetails.status", - cwd, - args: statusArgs, - }), - detail: "Git status failed.", - exitCode: statusResult.exitCode, - stdoutLength: statusResult.stdout.length, - stderrLength: statusResult.stderr.length, - }); - } - - const repositoryPaths = yield* resolveRepositoryPaths(cwd).pipe( - Effect.catchTags({ GitCommandError: () => Effect.succeed(null) }), - ); - const statusCacheKey = repositoryPaths?.gitCommonDir; - const [numstatEntries, defaultBranch, hasPrimaryRemote] = yield* Effect.all( + // diff-index does not read status output, so both spawn together; a non-repository discards it. + const [statusResult, numstatResult, defaultBranch, hasPrimaryRemote] = yield* Effect.all( [ + readStatus, // Porcelain diff rewrites indexes despite GIT_OPTIONAL_LOCKS=0; plumbing with -M preserves rename detection. executeGitWithStableDiagnostics( "GitVcsDriver.statusDetails.numstat", @@ -2190,6 +2173,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }), ); }), + Effect.result, ), statusCacheKey ? Cache.get(defaultBranchCache, statusCacheKey).pipe(Effect.orElseSucceed(() => null)) @@ -2200,6 +2184,28 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ], { concurrency: "unbounded" }, ); + if (statusResult === null) { + return NON_REPOSITORY_STATUS_DETAILS; + } + + if (statusResult.exitCode !== 0) { + if (isNonRepositoryGitStderr(statusResult.stderr)) { + return NON_REPOSITORY_STATUS_DETAILS; + } + return yield* new GitCommandError({ + ...gitCommandContext({ + operation: "GitVcsDriver.statusDetails.status", + cwd, + args: statusArgs, + }), + detail: "Git status failed.", + exitCode: statusResult.exitCode, + stdoutLength: statusResult.stdout.length, + stderrLength: statusResult.stderr.length, + }); + } + + const numstatEntries = yield* Effect.fromResult(numstatResult); const statusStdout = statusResult.stdout; let refName: string | null = null; @@ -2243,25 +2249,34 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } } - const fallbackAheadCount = - includeDivergence && !upstreamRef && refName - ? yield* computeAheadCountAgainstBase(cwd, refName).pipe(Effect.orElseSucceed(() => 0)) - : null; - - if (fallbackAheadCount !== null) { - aheadCount = fallbackAheadCount; - behindCount = 0; - } - const isDefaultBranch = refName !== null && (refName === defaultBranch || (defaultBranch === null && (refName === "main" || refName === "master"))); - if (includeDivergence && refName && !isDefaultBranch) { - aheadOfDefaultCount = - fallbackAheadCount !== null - ? fallbackAheadCount - : yield* computeAheadCountAgainstBase(cwd, refName).pipe(Effect.orElseSucceed(() => 0)); + const needsFallbackAhead = includeDivergence && !upstreamRef && refName !== null; + const needsAheadOfDefault = includeDivergence && refName !== null && !isDefaultBranch; + const [baseAheadCount, branchChanges] = yield* Effect.all( + [ + refName !== null && (needsFallbackAhead || needsAheadOfDefault) + ? computeAheadCountAgainstBase(cwd, refName, { + knownOriginDefaultBranch: defaultBranch, + }).pipe(Effect.orElseSucceed(() => 0)) + : Effect.succeed(0), + options?.includeBranchChanges + ? readBranchChangeTotals(repositoryPaths?.worktreeRoot ?? cwd, refName, { + knownOriginDefaultBranch: defaultBranch, + }).pipe(Effect.orElseSucceed(() => undefined)) + : Effect.undefined, + ], + { concurrency: "unbounded" }, + ); + + if (needsFallbackAhead) { + aheadCount = baseAheadCount; + behindCount = 0; + } + if (needsAheadOfDefault) { + aheadOfDefaultCount = baseAheadCount; } const fileStatMap = new Map(); @@ -2288,12 +2303,6 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } files.sort((a, b) => a.path.localeCompare(b.path)); - const branchChanges = options?.includeBranchChanges - ? yield* readBranchChangeTotals(repositoryPaths?.worktreeRoot ?? cwd, refName).pipe( - Effect.orElseSucceed(() => undefined), - ) - : undefined; - return { isRepo: true, hasOriginRemote: hasPrimaryRemote, @@ -3036,6 +3045,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* cwd: string, branch: string | null, explicitBaseRef?: string, + knownOriginDefaultBranch: string | null = null, ) { const base = explicitBaseRef !== undefined @@ -3047,9 +3057,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : explicitBaseRef, }) : branch - ? yield* resolveBaseBranchForNoUpstream(cwd, branch, { allowRemoteOfCurrent: true }).pipe( - Effect.orElseSucceed(() => null), - ) + ? yield* resolveBaseBranchForNoUpstream(cwd, branch, { + allowRemoteOfCurrent: true, + knownOriginDefaultBranch, + }).pipe(Effect.orElseSucceed(() => null)) : null; if (base === null) return { baseRef: null, mergeBase: "HEAD" }; const { baseRef, comparisonRef } = base; @@ -3080,8 +3091,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const readBranchChangeTotals = Effect.fn("readBranchChangeTotals")(function* ( cwd: string, branch: string | null, + options?: { readonly knownOriginDefaultBranch: string | null }, ) { - const { baseRef, mergeBase } = yield* resolveReviewMergeBase(cwd, branch); + const { baseRef, mergeBase } = yield* resolveReviewMergeBase( + cwd, + branch, + undefined, + options?.knownOriginDefaultBranch ?? null, + ); const untracked = yield* prepareUntrackedReviewIndex(cwd, []); if (untracked === null) { return yield* new GitCommandError({ @@ -3124,6 +3141,141 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return { baseRef, insertions, deletions }; }, Effect.scoped); + const reviewCommitError = (cwd: string, detail: string) => + new GitCommandError({ + operation: "GitVcsDriver.reviewCommit", + command: "git cat-file", + cwd, + detail, + }); + + const resolveReviewCommit = Effect.fn("resolveReviewCommit")(function* ( + cwd: string, + sha: string, + ) { + if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i.test(sha)) { + return yield* reviewCommitError(cwd, "Selected commit is invalid or no longer available."); + } + const present = yield* executeGit( + "GitVcsDriver.reviewCommit.present", + cwd, + ["cat-file", "-e", `${sha}^{commit}`], + { allowNonZeroExit: true, env: { GIT_NO_LAZY_FETCH: "1" } }, + ); + if (present.exitCode !== 0) { + return yield* reviewCommitError(cwd, "Selected commit is invalid or no longer available."); + } + // The raw object keeps the parents a shallow clone grafts away, so a boundary is not a root. + const object = yield* runGitStdout("GitVcsDriver.reviewCommit.parents", cwd, [ + "cat-file", + "commit", + sha, + ]); + const secondLine = object.split("\n", 2)[1]; + const firstParent = secondLine?.startsWith("parent ") ? secondLine.slice(7) : undefined; + if (firstParent !== undefined) { + const parent = yield* executeGit( + "GitVcsDriver.reviewCommit.parent", + cwd, + ["cat-file", "-e", `${firstParent}^{commit}`], + { allowNonZeroExit: true, env: { GIT_NO_LAZY_FETCH: "1" } }, + ); + if (parent.exitCode !== 0) { + return yield* reviewCommitError(cwd, "Selected commit's parent is not available locally."); + } + } + return { sha: sha.toLowerCase(), parent: firstParent ?? (yield* readEmptyTreeHash(cwd)) }; + }); + + const readReviewCommitMessage = Effect.fn("GitVcsDriver.readReviewCommitMessage")(function* ( + cwd: string, + sha: string, + ) { + const result = yield* executeGit( + "GitVcsDriver.readReviewCommitMessage", + cwd, + ["show", "-s", "--no-notes", "--no-show-signature", "--format=%B", sha], + { maxOutputBytes: 8_192, appendTruncationMarker: true }, + ); + const endOfSubject = result.stdout.indexOf("\n"); + return { + subject: result.stdout.slice(0, endOfSubject < 0 ? undefined : endOfSubject).slice(0, 300), + }; + }); + + const getReviewBranchCommits = Effect.fn("getReviewBranchCommits")(function* ( + input: ReviewBranchCommitsInput, + ) { + const repository = yield* resolveRepositoryPathsUncached(input.cwd); + if (!repository?.worktreeRoot) return { commits: [], truncated: false }; + const cwd = repository.worktreeRoot; + const head = yield* executeGit( + "GitVcsDriver.getReviewBranchCommits.head", + cwd, + ["rev-parse", "--verify", "--quiet", "HEAD"], + { allowNonZeroExit: true }, + ); + if (head.exitCode !== 0) { + if (head.exitCode === 1 && head.stderr.trim() === "") { + return { commits: [], truncated: false }; + } + return yield* reviewCommitError(cwd, "Could not resolve the current branch commit."); + } + const { mergeBase } = yield* resolveReviewMergeBase( + cwd, + repository.currentBranch, + input.baseRef, + ); + const result = yield* executeGit( + "GitVcsDriver.getReviewBranchCommits", + cwd, + [ + "log", + "-z", + "--no-notes", + "--no-show-signature", + "--format=%H%x00%s%x00%cI", + `--max-count=${REVIEW_BRANCH_COMMIT_LIMIT + 1}`, + `${mergeBase}..HEAD`, + ], + { + maxOutputBytes: REVIEW_BRANCH_COMMIT_MAX_OUTPUT_BYTES, + appendTruncationMarker: true, + }, + ); + if (!result.stdoutTruncated && result.stdout.length > 0 && !result.stdout.endsWith("\0")) { + return yield* reviewCommitError(cwd, "Could not read branch commit metadata."); + } + const fields = result.stdout.split("\0"); + const completeFieldCount = fields.length - 1; + const commits: Array<{ sha: string; subject: string; committedAt: string }> = []; + for ( + let index = 0; + index + 2 < completeFieldCount && commits.length <= REVIEW_BRANCH_COMMIT_LIMIT; + index += 3 + ) { + const sha = fields[index]; + const subject = fields[index + 1]; + const committedAt = fields[index + 2]; + if ( + !sha || + subject === undefined || + !committedAt || + !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i.test(sha) + ) { + return yield* reviewCommitError(cwd, "Could not read branch commit metadata."); + } + commits.push({ sha, subject: subject.slice(0, REVIEW_BRANCH_SUBJECT_LIMIT), committedAt }); + } + if (result.stdoutTruncated && commits.length === 0) { + return yield* reviewCommitError(cwd, "Branch commit metadata exceeds the response limit."); + } + return { + commits: commits.slice(0, REVIEW_BRANCH_COMMIT_LIMIT), + truncated: result.stdoutTruncated || commits.length > REVIEW_BRANCH_COMMIT_LIMIT, + }; + }); + const getReviewDiffPreview = Effect.fn("getReviewDiffPreview")(function* ( input: ReviewDiffPreviewInput, ) { @@ -3150,6 +3302,64 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } const cwd = repository.worktreeRoot; + const hashDiff = (diff: string, files: ReadonlyArray) => + crypto.digest("SHA-256", new TextEncoder().encode(JSON.stringify([diff, files]))).pipe( + Effect.map(Hex.encode), + Effect.mapError( + (cause) => + new GitCommandError({ + operation: "GitVcsDriver.getReviewDiffPreview.hash", + command: "crypto.digest SHA-256", + cwd, + detail: "Failed to hash review diff.", + cause, + }), + ), + ); + if (input.commitSha !== undefined || input.file?.sourceKind === "commit") { + if (!input.commitSha || (input.file && input.file.sourceKind !== "commit")) { + return yield* reviewCommitError(cwd, "Selected commit request is invalid."); + } + const commit = yield* resolveReviewCommit(cwd, input.commitSha); + const { subject } = yield* readReviewCommitMessage(cwd, commit.sha); + const diffArgs = [ + ...REVIEW_DIFF_ARGS, + ...(input.ignoreWhitespace ? ["--ignore-all-space"] : []), + ]; + const stat = yield* executeGit( + "GitVcsDriver.getReviewDiffPreview.commitStat", + cwd, + [...diffArgs, "--numstat", "-z", commit.parent, commit.sha, "--", ...pathArgs], + { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, + ); + const files = parseReviewNumstat(stat.stdout); + const patch = yield* executeGit( + "GitVcsDriver.getReviewDiffPreview.commitPatch", + cwd, + [...diffArgs, "--patch", commit.parent, commit.sha, "--", ...pathArgs], + { maxOutputBytes: patchLimit, appendTruncationMarker: true }, + ); + const diffHash = yield* hashDiff(patch.stdout, files); + return { + cwd: input.cwd, + generatedAt: yield* DateTime.now, + sources: [ + { + id: commit.sha, + kind: "commit" as const, + title: `Commit ${commit.sha.slice(0, 7)}`, + ...(subject.trim() ? { commitSubject: subject.trim() } : {}), + baseRef: commit.parent, + headRef: commit.sha, + diff: patch.stdout, + files, + diffHash, + truncated: patch.stdoutTruncated, + }, + ], + }; + } + const dirtyRef = input.file?.sourceKind === "branch-range" ? null : "HEAD"; const review = input.file?.sourceKind === "working-tree" @@ -3229,20 +3439,6 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const baseFiles = baseResult.files; const dirtyDiff = dirtyTrackedResult.stdout; const baseDiff = baseResult.stdout; - const hashDiff = (diff: string, files: ReadonlyArray) => - crypto.digest("SHA-256", new TextEncoder().encode(JSON.stringify([diff, files]))).pipe( - Effect.map(Hex.encode), - Effect.mapError( - (cause) => - new GitCommandError({ - operation: "GitVcsDriver.getReviewDiffPreview.hash", - command: "crypto.digest SHA-256", - cwd, - detail: "Failed to hash review diff.", - cause, - }), - ), - ); const [dirtyDiffHash, baseDiffHash] = yield* Effect.all([ hashDiff(dirtyDiff, dirtyFiles ?? []), hashDiff(baseDiff, baseFiles ?? []), @@ -3394,6 +3590,30 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* if (repositoryRoot.length === 0) { return yield* reviewDiffFileError(input, "Could not resolve the Git repository root."); } + if (input.sourceKind === "commit") { + if (!input.headRef) { + return yield* reviewDiffFileError(input, "Selected commit is no longer available."); + } + const commit = yield* resolveReviewCommit(input.cwd, input.headRef); + if (input.baseRef !== commit.parent) { + return yield* reviewDiffFileError( + input, + "Selected commit changed. Refresh the comparison.", + ); + } + const [oldContents, newContents] = yield* Effect.all( + [ + input.changeType === "new" + ? Effect.succeed("") + : readReviewFileAtRevision(input, commit.parent, input.oldPath), + input.changeType === "deleted" + ? Effect.succeed("") + : readReviewFileAtRevision(input, commit.sha, input.newPath), + ], + { concurrency: 2 }, + ); + return { oldContents, newContents }; + } const oldRevision = input.sourceKind === "working-tree" ? (input.baseRef ?? "HEAD") @@ -3885,7 +4105,9 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ["-c", `checkout.workers=${checkoutWorkers}`, ...args], { fallbackErrorDetail: "git worktree add failed", - timeoutMs: WORKTREE_ADD_TIMEOUT_MS, + // Large checkouts and post-checkout hooks can legitimately run for many + // minutes; a fixed deadline kills them and leaves an orphaned worktree. + timeoutMs: null, ...(onCheckoutProgress ? { // Git only prints checkout progress when stderr is a tty or the @@ -4656,6 +4878,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return yield* withListRefsInvalidation(cwd, pullCurrentBranch(cwd)); }), readRangeContext, + getReviewBranchCommits, getReviewDiffPreview, getReviewDiffFileContents, readConfigValue, diff --git a/apps/server/src/vcs/VcsDriverRegistry.test.ts b/apps/server/src/vcs/VcsDriverRegistry.test.ts index e5c8169774830..b6f7dae813744 100644 --- a/apps/server/src/vcs/VcsDriverRegistry.test.ts +++ b/apps/server/src/vcs/VcsDriverRegistry.test.ts @@ -21,6 +21,8 @@ const processOutput = (stdout: string): VcsProcess.VcsProcessOutput => ({ const normalizeGitArgs = (args: ReadonlyArray): ReadonlyArray => args[0] === "-C" && args.length >= 2 ? args.slice(2) : args; +const DETECT_COMMAND = "rev-parse --is-inside-work-tree --show-toplevel --git-common-dir"; + describe("VcsDriverRegistry", () => { const makeDiskBackedLayer = (rootDir: string, calls: string[]) => Layer.effect(VcsDriverRegistry.VcsDriverRegistry, VcsDriverRegistry.make).pipe( @@ -49,10 +51,10 @@ describe("VcsDriverRegistry", () => { stderr: "fatal: not a git repository", }; } - if (command === "rev-parse --is-inside-work-tree") return processOutput("true\n"); - if (command === "rev-parse --show-toplevel") return processOutput(`${repoDir}\n`); - if (command === "rev-parse --git-common-dir") { - return processOutput(`${path.relative(input.cwd, path.join(repoDir, ".git"))}\n`); + if (command === DETECT_COMMAND) { + return processOutput( + `true\n${repoDir}\n${path.relative(input.cwd, path.join(repoDir, ".git"))}\n`, + ); } return processOutput(""); }).pipe(Effect.provide(NodeServices.layer), Effect.orDie), @@ -75,7 +77,7 @@ describe("VcsDriverRegistry", () => { yield* fs.remove(path.join(repoDir, ".git"), { recursive: true }); assert.equal(yield* registry.detect({ cwd: repoDir }), null); - assert.equal(calls.filter((call) => call === "rev-parse --is-inside-work-tree").length, 2); + assert.equal(calls.filter((call) => call === DETECT_COMMAND).length, 2); }).pipe(Effect.provide(makeDiskBackedLayer(repoDir, calls))); }).pipe(Effect.provide(NodeServices.layer)), ); @@ -118,7 +120,7 @@ describe("VcsDriverRegistry", () => { yield* fs.makeDirectory(path.join(repoDir, ".git")); assert.equal((yield* registry.detect({ cwd: repoDir }))?.repository.rootPath, repoDir); - assert.equal(calls.filter((call) => call === "rev-parse --is-inside-work-tree").length, 2); + assert.equal(calls.filter((call) => call === DETECT_COMMAND).length, 2); }).pipe(Effect.provide(makeDiskBackedLayer(repoDir, calls))); }).pipe(Effect.provide(NodeServices.layer)), ); diff --git a/apps/server/src/vcs/VcsProcess.ts b/apps/server/src/vcs/VcsProcess.ts index e2055e3400531..00218397481ff 100644 --- a/apps/server/src/vcs/VcsProcess.ts +++ b/apps/server/src/vcs/VcsProcess.ts @@ -83,8 +83,7 @@ const gitMetricOperations = new Set([ "GitVcsDriver.checkpoints.resolveGitCommonDir", "GitVcsDriver.checkpoints.resolveHeadCommit", "GitVcsDriver.checkpoints.restoreCheckpoint", - "GitVcsDriver.detectRepository.commonDir", - "GitVcsDriver.detectRepository.root", + "GitVcsDriver.detectRepository", "GitVcsDriver.filterIgnoredPaths", "GitVcsDriver.initRepository", "GitVcsDriver.isInsideWorkTree", diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts index cc1b6d24e3873..042f8274543e5 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts @@ -76,6 +76,7 @@ function layerTestFor(state: { backgroundWorkEnabled?: boolean; automaticFetchInterval?: Effect.Effect; beforeRemoteStatus?: Effect.Effect; + beforeLocalStatus?: Effect.Effect; fileSystem?: FileSystem.FileSystem; pullCurrentBranch?: GitWorkflowService.GitWorkflowService["Service"]["pullCurrentBranch"]; }) { @@ -95,9 +96,11 @@ function layerTestFor(state: { Layer.provide( Layer.mock(GitWorkflowService.GitWorkflowService)({ localStatus: () => - Effect.sync(() => { + Effect.suspend(() => { state.localStatusCalls += 1; - return state.currentLocalStatus; + return (state.beforeLocalStatus ?? Effect.void).pipe( + Effect.as(state.currentLocalStatus), + ); }), currentBranch: () => Effect.sync(() => state.currentLocalStatus.refName), remoteStatus: (_input, options) => @@ -163,6 +166,47 @@ function layerBackgroundPolicy(shouldRunScopeWork: (scope: BackgroundScope) => b } describe("VcsStatusBroadcaster", () => { + it.effect("refreshes queued behind a running one share the next git status read", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + fileSystem: FileSystem.makeNoop({ realPath: (cwd) => Effect.succeed(cwd) }), + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + state.beforeLocalStatus = Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ); + const running = yield* broadcaster + .refreshLocalStatus("/repo") + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(started); + // Requested while the first read runs, so that read may predate their change. + state.currentLocalStatus = { ...baseLocalStatus, hasWorkingTreeChanges: true }; + const queued = yield* Effect.all( + [broadcaster.refreshLocalStatus("/repo"), broadcaster.refreshLocalStatus("/repo")], + { concurrency: "unbounded" }, + ).pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.succeed(release, undefined); + + assert.isFalse((yield* Fiber.join(running)).hasWorkingTreeChanges); + const [second, third] = yield* Fiber.join(queued); + assert.isTrue(second.hasWorkingTreeChanges); + assert.isTrue(third.hasWorkingTreeChanges); + assert.strictEqual(state.localStatusCalls, 2); + + yield* broadcaster.refreshLocalStatus("/repo"); + assert.strictEqual(state.localStatusCalls, 3); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }); + it.effect("publishes a terminal branch switch without reading remote status", () => { const state = { currentLocalStatus: baseLocalStatus, diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.ts b/apps/server/src/vcs/VcsStatusBroadcaster.ts index 8ce1b225d123d..a8839d0c1cd0c 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.ts @@ -359,11 +359,30 @@ export const make = Effect.gen(function* () { ); }); + // A refresh that started after a request saw every change that request could mean, so queued + // callers share it instead of each running git status again. + const localRefreshLocks = yield* KeyedLock.make(); + const localRefreshRequests = new Map(); + const localRefreshResults = new Map< + string, + { readonly startedAt: number; readonly local: VcsStatusLocalResult } + >(); const refreshLocalStatusCore = Effect.fn("VcsStatusBroadcaster.refreshLocalStatusCore")( function* (cwd: string) { - yield* workflow.invalidateLocalStatus(cwd); - const local = yield* workflow.localStatus({ cwd }); - return yield* updateCachedLocalStatus(cwd, local, { publish: true }); + const requestedAt = (localRefreshRequests.get(cwd) ?? 0) + 1; + localRefreshRequests.set(cwd, requestedAt); + return yield* localRefreshLocks.withLock( + cwd, + Effect.gen(function* () { + const shared = localRefreshResults.get(cwd); + if (shared !== undefined && shared.startedAt >= requestedAt) return shared.local; + const startedAt = localRefreshRequests.get(cwd) ?? requestedAt; + yield* workflow.invalidateLocalStatus(cwd); + const local = yield* workflow.localStatus({ cwd }); + localRefreshResults.set(cwd, { startedAt, local }); + return yield* updateCachedLocalStatus(cwd, local, { publish: true }); + }), + ); }, ); diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index bce9450778736..90b7ed28864ee 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -105,6 +105,7 @@ import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts"; import * as TriageService from "./triage/TriageService.ts"; import * as WorkspacePaths from "./workspace/WorkspacePaths.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; +import * as SecretCheck from "./secrets/SecretCheck.ts"; import * as ServerConfigModule from "./config.ts"; import * as Keybindings from "./keybindings.ts"; import * as ServerSettings from "./serverSettings.ts"; @@ -412,12 +413,23 @@ it.effect("shell snapshots omit idle children and older settled threads only on Effect.flatMap((stream) => stream.pipe(Stream.take(1), Stream.runCollect)), Effect.map(([item]) => (item?.kind === "snapshot" ? item.snapshot : null)), Effect.provideService(ThreadManagement.ThreadManagementService, { - readShellSnapshot: () => + listActiveShellIndex: () => + Effect.succeed( + threads.map((thread) => ({ + ...thread, + pinnedAt: null, + snoozedUntil: null, + mayBeLive: thread.activityRunStatus !== null, + })), + ), + readShellSnapshot: (options?: { readonly threadIds?: ReadonlyArray }) => Effect.succeed( Effect.succeed({ schemaVersion: 1, snapshotSequence: 0, - threads, + threads: threads.filter( + (thread) => options?.threadIds?.includes(thread.id) ?? true, + ), archivedThreads: [], }), ), @@ -468,6 +480,7 @@ it.effect("shell snapshots omit idle children and older settled threads only on const layerUnusedConfigRpcServices = Layer.mergeAll( Layer.mock(DocumentText.DocumentText)({}), Layer.mock(ServerSecretStore.ServerSecretStore)({}), + Layer.mock(SecretCheck.SecretCheck)({}), Layer.mock(WorkspacePaths.WorkspacePaths)({}), Layer.mock(TraceDiagnostics.TraceDiagnostics)({}), Layer.mock(TriageService.TriageService)({}), diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 338529ea0b23b..d2da4e9048c92 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -124,7 +124,6 @@ import * as ScheduledTasks from "./scheduledTasks/ScheduledTaskService.ts"; import * as SecretRequests from "./secrets/SecretRequests.ts"; import * as GuestShareService from "./guest/GuestShareService.ts"; import { - buildActiveShellSnapshot, bufferShellLiveStream, coalesceShellApplicationEvents, composeShellStreamWithEnrichment, @@ -134,7 +133,7 @@ import { shellStreamItemFromEnrichmentRefresh, shellStreamItemFromThreadShell, shellStreamItemsFromInitialSnapshot, - shellSnapshotForSubscriber, + loadSubscriberShellSnapshot, shellStreamItemsFromResumeSnapshot, shouldPublishShellApplicationEvent, skipUnchangedThreadShells, @@ -170,7 +169,9 @@ import * as OrchestrationEventStore from "./persistence/OrchestrationEventStore. import { userFacingDispatchErrorMessage } from "./orchestration-v2/UserFacingErrors.ts"; import * as ProviderRegistry from "./provider/ProviderRegistry.ts"; import * as ProviderInstanceRegistry from "./provider/ProviderInstanceRegistry.ts"; -import { ProviderSecretResolver } from "./provider/ProviderSecretResolver.ts"; +import { ChildProcessSpawner } from "effect/process"; +import * as SecretCheck from "./secrets/SecretCheck.ts"; +import * as OnePasswordDriver from "@t3tools/secret-provider-onepassword/server/driver"; import * as AcpRegistrySupport from "@t3tools/provider-acp-registry/server/AcpRegistrySupport"; import * as AcpRegistryRuntimeCoordinator from "@t3tools/provider-acp-registry/server/AcpRegistryRuntimeCoordinator"; import * as ModelManifest from "./provider/ModelManifest.ts"; @@ -1015,21 +1016,23 @@ export const subscribeOrchestrationV2Shell = Effect.fn("ws.orchestrationV2.subsc }, ); const loadSnapshot = Effect.fn("ws.orchestrationV2.loadShellSnapshot")(function* () { - const base = buildActiveShellSnapshot( - yield* loadShellSnapshotParts({ - sql, - readThreads: threadManagement.readShellSnapshot({ location: "active" }), - listProjects: projects.listShells(), - latestSequence: applicationEvents.latestApplicationSequence, - }), - ); - const forSubscriber = shellSnapshotForSubscriber(base, { - omitIdleSubagents: input.omitIdleSubagents === true, - omitOlderSettled: input.omitOlderSettled === true, - now: yield* DateTime.now, + const forSubscriber = yield* loadSubscriberShellSnapshot({ + sql, + listShellIndex: threadManagement.listActiveShellIndex(), + readThreads: (threadIds) => + threadManagement.readShellSnapshot({ + location: "active", + ...(threadIds === undefined ? {} : { threadIds }), + }), + listProjects: projects.listShells(), + latestSequence: applicationEvents.latestApplicationSequence, + options: { + omitIdleSubagents: input.omitIdleSubagents === true, + omitOlderSettled: input.omitOlderSettled === true, + }, }); subagentThreadIds = forSubscriber.subagentThreadIds; - const enriched = yield* enrichProjectShells(base.projects); + const enriched = yield* enrichProjectShells(forSubscriber.snapshot.projects); return { snapshot: { ...forSubscriber.snapshot, @@ -1363,7 +1366,8 @@ export const layerWsRpc = ( ); const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; const sourceControlDiscovery = yield* SourceControlDiscovery.SourceControlDiscovery; - const providerSecretResolver = yield* ProviderSecretResolver; + const secretCheck = yield* SecretCheck.SecretCheck; + const processSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const otlpEndpointCheck = yield* OtlpEndpointCheck.OtlpEndpointCheck; const sourceControlRepositories = yield* SourceControlRepositoryService.SourceControlRepositoryService; @@ -1798,6 +1802,7 @@ export const layerWsRpc = ( threadFind: true, threadFindProgressive: true, secretProviderSettings: true, + secretReferences: true, ...Option.match(scratchWorkspaceRoot, { onNone: () => ({}), onSome: (root) => ({ scratchWorkspaceRoot: root }), @@ -1919,6 +1924,8 @@ export const layerWsRpc = ( ), ), [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: (input) => threadManagement.stopWorkflow(input), + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: (input) => + threadManagement.forceStopThread(input), [ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: (input) => readWorkflowScript({ scriptPath: input.scriptPath }), [ORCHESTRATION_V2_WS_METHODS.getTurnItem]: (input) => @@ -2317,9 +2324,11 @@ export const layerWsRpc = ( cwd: input.cwd, fresh: input.fresh === true, }) - : input.instanceId !== undefined - ? providerRegistry.refreshInstance(input.instanceId) - : providerRegistry.refresh(); + : input.reloadSecrets === true + ? providerRegistry.reloadSecretsAndRefresh(input.instanceId) + : input.instanceId !== undefined + ? providerRegistry.refreshInstance(input.instanceId) + : providerRegistry.refresh(); if (input.refreshModels) { const instances = yield* providerInstances.listInstances; for (const instance of instances) { @@ -2521,9 +2530,10 @@ export const layerWsRpc = ( }), [WS_METHODS.serverDiscoverSourceControl]: (_input) => sourceControlDiscovery.discover, [WS_METHODS.serverListOnePasswordAccounts]: (_input) => - providerSecretResolver.listOnePasswordAccounts, - [WS_METHODS.serverCheckSecretSource]: (input) => - providerRegistry.checkSecretSource(input.source), + OnePasswordDriver.listAccounts().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, processSpawner), + ), + [WS_METHODS.serverCheckSecretSource]: (input) => secretCheck.check(input.source), [WS_METHODS.serverGetTraceDiagnostics]: (_input) => TraceDiagnostics.readTraceDiagnostics({ traceFilePath: config.serverTracePath, @@ -3005,6 +3015,7 @@ export const layerWsRpc = ( gitWorkflow.switchRef(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), [WS_METHODS.vcsInit]: (input) => vcsProvisioning.initRepository(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), + [WS_METHODS.reviewGetBranchCommits]: (input) => review.getBranchCommits(input), [WS_METHODS.reviewGetDiffPreview]: (input) => review.getDiffPreview(input), [WS_METHODS.reviewGetDiffFileContents]: (input) => review.getDiffFileContents(input), [WS_METHODS.terminalOpen]: (input) => terminalManager.open(input), diff --git a/apps/web/package.json b/apps/web/package.json index 2ee8fe51a06a3..22d8155b5b828 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -34,6 +34,10 @@ "@t3tools/client-runtime": "workspace:*", "@t3tools/contracts": "workspace:*", "@t3tools/provider-core": "workspace:*", + "@t3tools/secret-provider-aws": "workspace:*", + "@t3tools/secret-provider-definitions": "workspace:*", + "@t3tools/secret-provider-onepassword": "workspace:*", + "@t3tools/secret-provider-openbao": "workspace:*", "@t3tools/shared": "workspace:*", "@tanstack/react-pacer": "^0.19.4", "@tanstack/react-router": "^1.160.2", diff --git a/apps/web/src/browser/browserLinkTarget.ts b/apps/web/src/browser/browserLinkTarget.ts index 50ff1a4e351fc..f1c4e65a99913 100644 --- a/apps/web/src/browser/browserLinkTarget.ts +++ b/apps/web/src/browser/browserLinkTarget.ts @@ -1,6 +1,11 @@ -import type { BrowserLinkTarget, ScopedThreadRef } from "@t3tools/contracts"; +import { + AuthPreviewOperateScope, + type BrowserLinkTarget, + type ScopedThreadRef, +} from "@t3tools/contracts"; import { ensureClientSettingsHydrated, getClientSettings } from "~/hooks/useSettings"; +import { readEnvironmentScope } from "~/state/session"; import { isPreviewAvailableFor } from "./previewRuntime"; export interface ResolveLinkTargetInput { @@ -38,5 +43,9 @@ export async function resolveBrowserLinkTargetPreference(): Promise - + + {isElectron ? : null} ); diff --git a/apps/web/src/components/ChatMarkdown.tsx b/apps/web/src/components/ChatMarkdown.tsx index 6a8f80d6d1071..99c2ebbc8f40f 100644 --- a/apps/web/src/components/ChatMarkdown.tsx +++ b/apps/web/src/components/ChatMarkdown.tsx @@ -189,6 +189,7 @@ import { import { resolveLinkTarget } from "../browser/browserLinkTarget"; import { PullRequestLinkPreview } from "./pullRequest/PullRequestLinkPreview"; import { MarkdownCodeBlock, SuspenseShikiCodeBlock } from "./chat/MarkdownCodeBlock"; +import { splitMarkdownBlocks } from "./chat/markdownStreamBlocks"; import { MarkdownMermaidCodeBlock } from "./chat/MarkdownMermaidCodeBlock"; import { MarkdownTable } from "./chat/MarkdownTable"; import { MarkdownDetails } from "./chat/MarkdownDetails"; @@ -434,6 +435,44 @@ const markdownServerConfigFieldsAtom = Atom.family((environmentId: EnvironmentId }).pipe(Atom.withLabel("markdown-server-config-fields")), ); +function fileLinkSourcesKey( + text: string, + cwd: string | undefined, + imageBaseDir: string | undefined, +): string { + const hrefs = extractMarkdownLinkHrefs(renderCodexInlineDirectivesAsMarkdown(text)) + .filter((href) => !parseComposerContextHref(href)) + .map(normalizeMarkdownLinkHrefKey) + .filter((href) => resolveMarkdownFileLinkMeta(href, cwd, imageBaseDir ?? cwd)); + const spans = extractInlineCodeSpans(text).filter((span) => + resolveInlineCodeFileLinkMeta(span, cwd, imageBaseDir ?? cwd), + ); + return JSON.stringify([[...new Set(hrefs)], [...new Set(spans)]]); +} + +function resolveFileLinkSources( + key: string, + cwd: string | undefined, + imageBaseDir: string | undefined, +) { + const [hrefs, spans] = JSON.parse(key) as [string[], string[]]; + const markdownFileLinkMetaByHref = new Map(); + for (const href of hrefs) { + const meta = resolveMarkdownFileLinkMeta(href, cwd, imageBaseDir ?? cwd); + if (meta) markdownFileLinkMetaByHref.set(href, meta); + } + const inlineCodeFileLinkMetaByText = new Map(); + for (const span of spans) { + const meta = resolveInlineCodeFileLinkMeta(span, cwd, imageBaseDir ?? cwd); + if (meta) inlineCodeFileLinkMetaByText.set(span, meta); + } + const fileLinkParentSuffixByPath = buildFileLinkParentSuffixByPath([ + ...[...markdownFileLinkMetaByHref.values()].map((meta) => meta.filePath), + ...[...inlineCodeFileLinkMetaByText.values()].map((meta) => meta.filePath), + ]); + return { markdownFileLinkMetaByHref, inlineCodeFileLinkMetaByText, fileLinkParentSuffixByPath }; +} + function useChatMarkdownState({ text, inline = false, @@ -605,40 +644,6 @@ function useChatMarkdownState({ }); }; const diffThemeName = resolveDiffThemeName(resolvedTheme); - const markdownFileLinkMetaByHref = (() => { - const metaByHref = new Map< - string, - NonNullable> - >(); - for (const href of extractMarkdownLinkHrefs(renderCodexInlineDirectivesAsMarkdown(text))) { - if (parseComposerContextHref(href)) continue; - const normalizedHref = normalizeMarkdownLinkHrefKey(href); - if (metaByHref.has(normalizedHref)) continue; - const meta = resolveMarkdownFileLinkMeta(normalizedHref, cwd, imageBaseDir ?? cwd); - if (meta) { - metaByHref.set(normalizedHref, meta); - } - } - return metaByHref; - })(); - const inlineCodeFileLinkMetaByText = (() => { - const metaByText = new Map(); - for (const span of extractInlineCodeSpans(text)) { - if (metaByText.has(span)) continue; - const meta = resolveInlineCodeFileLinkMeta(span, cwd, imageBaseDir ?? cwd); - if (meta) { - metaByText.set(span, meta); - } - } - return metaByText; - })(); - const fileLinkParentSuffixByPath = (() => { - const filePaths = [ - ...[...markdownFileLinkMetaByHref.values()].map((meta) => meta.filePath), - ...[...inlineCodeFileLinkMetaByText.values()].map((meta) => meta.filePath), - ]; - return buildFileLinkParentSuffixByPath(filePaths); - })(); const markdownUrlTransform = (href: string) => { if (parseAssistantCitationHref(href)) return href; if (parseComposerContextHref(href)) return href; @@ -666,6 +671,12 @@ function useChatMarkdownState({ // Subscribe so hydrated settings apply before the anchor synchronously decides whether to intercept _blank. const linkTargetPreference = useClientSettings((settings) => settings.browserLinkTarget); const mathEnabled = useClientSettings((settings) => settings.mathRenderingEnabled); + // Keyed by the file links found, not the text, so streamed text that adds none keeps the + // renderer context, and every block reading it, unchanged. + // A template literal marks the key as a string, so the compiler memoizes the maps on its value. + const fileLinksKey = `${fileLinkSourcesKey(text, cwd, imageBaseDir)}`; + const { markdownFileLinkMetaByHref, inlineCodeFileLinkMetaByText, fileLinkParentSuffixByPath } = + resolveFileLinkSources(fileLinksKey, cwd, imageBaseDir); const resolveThreadPullRequest = ( href: string, ): (ThreadPullRequestKey & { readonly url: string }) | null => { @@ -1622,6 +1633,32 @@ const CHAT_MARKDOWN_COMPONENTS = { }, } satisfies Components; +const MarkdownSource = memo(function MarkdownSource({ + text, + remarkPlugins, + rehypePlugins, + urlTransform, +}: { + text: string; + remarkPlugins: NonNullable; + rehypePlugins: NonNullable; + urlTransform: ReactMarkdownOptions["urlTransform"]; +}) { + return ( + + + {text} + + + ); +}); + function ChatMarkdown({ text, className, @@ -1639,8 +1676,17 @@ function ChatMarkdown({ localMediaPreview, closeLocalMediaPreview, } = useChatMarkdownState({ text, inline, ...props }); + // Streaming re-renders only the last block; offsets for task edits need the whole text. + const streamingBlocks = + props.isStreaming === true && + !inline && + extraRemarkPlugins.length === 0 && + !props.onTaskListChange + ? splitMarkdownBlocks(text) + : null; const incrementalParsing = props.isStreaming === true && + streamingBlocks === null && extraRemarkPlugins.length === 0 && /(?:^|\n) {0,3}(?:`{3}|~{3})/.test(text); const Root = inline ? "span" : "div"; @@ -1668,21 +1714,20 @@ function ChatMarkdown({ onCopy={handleCopy} > - - ( + - {text} - - + /> + ))} {localMediaPreview ? ( diff --git a/apps/web/src/components/ChatView.logic.ts b/apps/web/src/components/ChatView.logic.ts index c7c0f77bc97fb..bef8f7ab405f1 100644 --- a/apps/web/src/components/ChatView.logic.ts +++ b/apps/web/src/components/ChatView.logic.ts @@ -3,6 +3,7 @@ import { deriveLatestThreadRun, deriveThreadRuntime, } from "@t3tools/client-runtime/state/thread-execution"; +import { checkpointFileTotals } from "@t3tools/client-runtime/state/thread-checkpoints"; import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import { latestUnheldRun, @@ -218,7 +219,7 @@ export function shouldOpenProactiveTurnDiff(input: { } export function resolveProactiveTurnDiffAction(input: { - checkpoint: Pick | undefined; + checkpoint: Pick | undefined; isGitRepo: boolean | undefined; }): "defer" | "ignore" | "open" { if (input.checkpoint === undefined || input.checkpoint.status === "missing") return "defer"; @@ -230,11 +231,8 @@ export function resolveProactiveTurnDiffAction(input: { ) { return "ignore"; } - const changedLines = input.checkpoint.files.reduce( - (total, file) => total + file.additions + file.deletions, - 0, - ); - return input.checkpoint.files.length >= 3 || changedLines >= 50 ? "open" : "ignore"; + const totals = checkpointFileTotals(input.checkpoint); + return totals.count >= 3 || totals.additions + totals.deletions >= 50 ? "open" : "ignore"; } /** Mark reactivated Diff tabs to preserve review scope. Already-active tabs must not leave a marker for diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 1bf6f9b0fc0e8..cf48ddcb842dd 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -1010,7 +1010,12 @@ export default function ChatView(props: ChatViewProps) { useLayoutEffect(() => { const explicitThreadRef = explicitDiffOpenRef.current; explicitDiffOpenRef.current = null; - if (diffOpen && activeThreadRef && explicitThreadRef !== activeThreadRef) { + if ( + diffOpen && + activeThreadRef && + explicitThreadRef !== activeThreadRef && + !useDiffPanelStore.getState().consumeKeepSelectionOnOpen(activeThreadRef) + ) { useDiffPanelStore.getState().selectGitScope(activeThreadRef, "branch"); } }, [activeThreadRef, diffOpen]); @@ -2349,7 +2354,8 @@ export default function ChatView(props: ChatViewProps) { const { acknowledgeActiveThreadWoke, activeBranchMismatchKey, - backgroundWorkNow, + attentionNow, + workNow, childInputNow, goalNow, handleSwitchCheckoutToThread, @@ -2993,10 +2999,7 @@ export default function ChatView(props: ChatViewProps) { const onExpandTimelineImageStable = useStableCallback(onExpandTimelineImage); const shownThreadSyncPhase = activeEnvironmentUnavailable ? null : delayedThreadSyncPhase; const hasThreadNow = - childInputNow !== null || - backgroundWorkNow !== null || - goalNow !== null || - shownThreadSyncPhase !== null; + attentionNow !== null || workNow !== null || goalNow !== null || shownThreadSyncPhase !== null; const queuedRunCount = serverProjection ? getUserQueuedThreadRuns(serverProjection).length : 0; // The queue control is mounted for every server thread but joins the footer only while it shows // something, by its own rule: an optimistic message stops counting once the server has it. @@ -3035,8 +3038,8 @@ export default function ChatView(props: ChatViewProps) { )} diff --git a/apps/web/src/components/DiffPanel.tsx b/apps/web/src/components/DiffPanel.tsx index ddce50db3cfe3..ef95c24d578a2 100644 --- a/apps/web/src/components/DiffPanel.tsx +++ b/apps/web/src/components/DiffPanel.tsx @@ -6,7 +6,7 @@ import { squashAtomCommandFailure, } from "@t3tools/client-runtime/state/runtime"; import { safeErrorLogAttributes } from "@t3tools/client-runtime/errors"; -import type { ScopedThreadRef, RunId } from "@t3tools/contracts"; +import type { ReviewBranchCommitsResult, ScopedThreadRef, RunId } from "@t3tools/contracts"; import { ArrowRightIcon, CheckIcon, @@ -20,7 +20,7 @@ import { } from "lucide-react"; import { ChevronDown, ChevronRight, ChevronsDownUp, ChevronsUpDown } from "lucide"; import * as Schema from "effect/Schema"; -import { useEffect, useRef, useState } from "react"; +import { type MouseEvent, useEffect, useRef, useState } from "react"; import { useCodeViewFileReveal } from "./diffs/useCodeViewFileReveal"; import { useFilesystemReadAccess } from "~/state/filesystem"; import { useOpenInPreferredEditor } from "../editorPreferences"; @@ -33,6 +33,7 @@ import { revisionedQuery } from "~/state/revisionedQuery"; import { cn } from "~/lib/utils"; import { resolveToolCallDiffTarget, + selectThreadBranchBaseRef, selectThreadDiffPanelSelection, useDiffPanelSessionStore, useDiffPanelStore, @@ -60,7 +61,7 @@ import { useQuietThreadVisibleTurnItems, } from "../state/entities"; import { useClientSettings, updateClientSettings } from "../hooks/useSettings"; -import { formatShortTimestamp } from "../timestampFormat"; +import { formatRelativeTimeLabel, formatShortTimestamp } from "../timestampFormat"; import { DiffFilePathCopyButton } from "./DiffFilePathCopyButton"; import { DiffPanelLoadingState, DiffPanelShell, type DiffPanelMode } from "./DiffPanelShell"; import { DiffStatLabel } from "./chat/DiffStatLabel"; @@ -83,6 +84,7 @@ import { import { DropdownMenu, DropdownMenuContent, + DropdownMenuItem, DropdownMenuRadioGroup, DropdownMenuRadioItem, DropdownMenuSub, @@ -103,6 +105,8 @@ import { useReviewFilePatches } from "./diffs/useReviewFilePatches"; import { DiffFileLoadingBoundary } from "./diffs/DiffFileLoadingBoundary"; import { DiffFileStatus } from "./diffs/DiffFileStatus"; import { useThreadRouteRef } from "../hooks/useThreadRouteTarget"; +import { projectCommitUrl } from "@t3tools/client-runtime/change-request-links"; +import { useOpenLink } from "../browser/useOpenLink"; type DiffThemeType = "light" | "dark"; const AUTOMATIC_BASE_REF = "__automatic_base_ref__"; @@ -223,6 +227,8 @@ export default function DiffPanel({ ); const [hideTestFiles, setHideTestFiles] = useHideTestFiles(); const [baseRefQuery, setBaseRefQuery] = useState(""); + // The open commit menu keeps the list it opened with, so rows do not move under the pointer. + const [openCommitMenu, setOpenCommitMenu] = useState(null); const [codeViewRevision, setCodeViewRevision] = useState(0); const routeThreadRef = useThreadRouteRef(); @@ -263,8 +269,14 @@ export default function DiffPanel({ const diffSelection = useDiffPanelStore((state) => selectThreadDiffPanelSelection(state.byThreadKey, routeThreadRef), ); + const rememberedBaseRef = useDiffPanelStore((state) => + selectThreadBranchBaseRef(state, routeThreadRef), + ); const toolCallSelection = diffSelection.kind === "tool-call" ? diffSelection : null; - const isGitSelection = diffSelection.kind === "branch" || diffSelection.kind === "unstaged"; + const isGitSelection = + diffSelection.kind === "branch" || + diffSelection.kind === "unstaged" || + diffSelection.kind === "commit"; const toolCallItems = useQuietThreadVisibleTurnItems(toolCallSelection ? routeThreadRef : null); const toolCallDetail = useTurnItemDetail( toolCallSelection && routeThreadRef @@ -294,8 +306,15 @@ export default function DiffPanel({ }, [diffSelection, orderedTurnDiffSummaries, routeThreadRef]); const selectedRunId = diffSelection.kind === "turn" ? diffSelection.turnId : null; - const selectedGitScope = diffSelection.kind === "unstaged" ? "unstaged" : "branch"; - const selectedBaseRef = diffSelection.kind === "branch" ? diffSelection.baseRef : null; + const selectedGitScope = + diffSelection.kind === "unstaged" || diffSelection.kind === "commit" + ? diffSelection.kind + : "branch"; + const selectedCommitSha = diffSelection.kind === "commit" ? diffSelection.sha : null; + const selectedBaseRef = + diffSelection.kind === "branch" || diffSelection.kind === "commit" + ? diffSelection.baseRef + : null; const selectedFilePath = diffSelection.kind === "turn" ? diffSelection.filePath : null; const selectedFileRevealRequestId = diffSelection.kind === "turn" ? diffSelection.revealRequestId : 0; @@ -306,31 +325,17 @@ export default function DiffPanel({ orderedTurnDiffSummaries[0]); const selectedCheckpointTurnCount = selectedTurn?.checkpointTurnCount; const latestTurn = orderedTurnDiffSummaries[0]; - const selectedScopeLabel = toolCallSelection - ? "Tool call" - : selectedRunId === null - ? selectedGitScope === "unstaged" - ? "Uncommitted" - : "Changes" - : selectedTurn?.runId === latestTurn?.runId - ? "Latest turn" - : `Turn ${selectedCheckpointTurnCount ?? "?"}`; const reviewSectionId = toolCallSelection ? `tool-call:${toolCallSelection.threadId}:${toolCallSelection.itemId}` : selectedTurn ? `turn:${selectedTurn.runId}` - : selectedGitScope; + : selectedCommitSha !== null + ? `commit:${selectedCommitSha}` + : selectedGitScope; const collapseScopeKey = routeThreadRef ? `${routeThreadRef.environmentId}:${routeThreadRef.threadId}:${reviewSectionId}` : null; const codeViewMountKey = `${collapseScopeKey ?? reviewSectionId}:${codeViewRevision}`; - const reviewSectionTitle = toolCallSelection - ? "Tool call" - : selectedTurn - ? `Turn ${selectedCheckpointTurnCount ?? "?"}` - : selectedGitScope === "unstaged" - ? "Uncommitted" - : "Changes"; const selectedCheckpointRange = typeof selectedCheckpointTurnCount === "number" ? { @@ -354,6 +359,7 @@ export default function DiffPanel({ ? { cwd: activeCwd, ...(selectedBaseRef ? { baseRef: selectedBaseRef } : {}), + ...(selectedCommitSha !== null ? { commitSha: selectedCommitSha } : {}), ignoreWhitespace: diffIgnoreWhitespace, } : undefined; @@ -372,13 +378,69 @@ export default function DiffPanel({ ) : null, ); + const branchCommitsQuery = useEnvironmentQuery( + visible && isGitRepo && canReadFiles && activeThread && activeCwd + ? reviewEnvironment.branchCommits({ + environmentId: activeThread.environmentId, + input: { + cwd: activeCwd, + ...(rememberedBaseRef ? { baseRef: rememberedBaseRef } : {}), + }, + }) + : null, + ); + const branchCommits = openCommitMenu ?? branchCommitsQuery.data; const activeThreadRefreshKey = routeThreadRef ? `${routeThreadRef.environmentId}:${routeThreadRef.threadId}` : null; const selectedGitSource = branchDiffPreview.data?.sources.find( - (source) => source.kind === (selectedGitScope === "unstaged" ? "working-tree" : "branch-range"), + (source) => + source.kind === + (selectedGitScope === "unstaged" + ? "working-tree" + : selectedGitScope === "commit" + ? "commit" + : "branch-range"), ); + const selectedCommitSubject = + selectedCommitSha !== null && selectedGitSource?.headRef === selectedCommitSha + ? selectedGitSource.commitSubject + : undefined; + const selectedGitScopeLabel = + selectedCommitSha !== null + ? (selectedCommitSubject ?? `Commit ${selectedCommitSha.slice(0, 7)}`) + : selectedGitScope === "unstaged" + ? "Uncommitted" + : "Changes"; + const selectedScopeLabel = toolCallSelection + ? "Tool call" + : selectedRunId === null + ? selectedCommitSha !== null + ? (selectedCommitSubject ?? "Commit") + : selectedGitScopeLabel + : selectedTurn?.runId === latestTurn?.runId + ? "Latest turn" + : `Turn ${selectedCheckpointTurnCount ?? "?"}`; + const reviewSectionTitle = toolCallSelection + ? "Tool call" + : selectedTurn + ? `Turn ${selectedCheckpointTurnCount ?? "?"}` + : selectedGitScopeLabel; + const headerCommitSha = selectedRunId === null ? selectedCommitSha : null; + const selectedCommitUrl = + headerCommitSha !== null + ? projectCommitUrl(activeProject?.repositoryIdentity, headerCommitSha) + : null; + const openLink = useOpenLink(routeThreadRef); + // A SHA link goes to the code host; it must not route back into this pane's commit selection. + const openCommitUrl = (event: MouseEvent, url: string) => { + event.stopPropagation(); + if (event.metaKey || event.ctrlKey || event.shiftKey || event.altKey || event.button === 1) + return; + event.preventDefault(); + void openLink(url, { event }); + }; const refreshPreviewQuery = branchDiffPreview.refresh; const refreshDiffFromUserAction = refreshPreviewQuery; @@ -512,7 +574,8 @@ export default function DiffPanel({ environmentId: activeThread?.environmentId, cwd: branchDiffPreview.data?.cwd, source: lazySource, - baseRef: lazySource?.baseRef ?? selectedBaseRef, + baseRef: + selectedCommitSha !== null ? selectedBaseRef : (lazySource?.baseRef ?? selectedBaseRef), ignoreWhitespace: diffIgnoreWhitespace, theme: resolvedTheme, previewInput: branchDiffPreviewInput, @@ -693,16 +756,23 @@ export default function DiffPanel({ useDiffPanelStore.getState().selectBranchBaseRef(routeThreadRef, baseRef); }; const selectedTurnValue = selectedTurn ? `turn:${selectedTurn.runId}` : ""; + const selectedCommitValue = selectedCommitSha !== null ? `commit:${selectedCommitSha}` : ""; const selectedScopeValue = toolCallSelection ? "" : selectedRunId === null - ? selectedGitScope + ? selectedCommitValue || selectedGitScope : selectedTurn?.runId === latestTurn?.runId ? "latest" : selectedTurnValue; const selectScopeValue = (value: string) => { if (value === "unstaged" || value === "branch") { selectGitScope(value); + } else if (value === selectedCommitValue) { + selectGitScope("branch"); + } else if (value.startsWith("commit:")) { + if (routeThreadRef) { + useDiffPanelStore.getState().selectCommit(routeThreadRef, value.slice("commit:".length)); + } } else if (value === "latest") { if (latestTurn) selectTurn(latestTurn.runId); } else { @@ -717,13 +787,37 @@ export default function DiffPanel({ } - className="max-w-full" + className={cn("max-w-full", headerCommitSha !== null && "min-w-0 shrink")} aria-label={`Diff scope: ${selectedScopeLabel}`} data-workspace-focus-default > {selectedScopeLabel} + {headerCommitSha !== null ? ( + selectedCommitUrl ? ( + + openCommitUrl(event, selectedCommitUrl)} + /> + } + > + {headerCommitSha.slice(0, 7)} + + {selectedCommitUrl} + + ) : ( + + {headerCommitSha.slice(0, 7)} + + ) + ) : null} @@ -760,6 +854,99 @@ export default function DiffPanel({ + { + setOpenCommitMenu(open ? (branchCommitsQuery.data ?? null) : null); + if (!open) return; + // The menu keeps the list it opened with; the refreshed one shows next time. + branchCommitsQuery.refresh(); + }} + > + Commits + + + {branchCommits?.commits.map((commit) => { + const commitUrl = projectCommitUrl( + activeProject?.repositoryIdentity, + commit.sha, + ); + return ( +
+ + + } + > + + {commit.subject} + + {formatRelativeTimeLabel(commit.committedAt)} + + + + + {commit.subject} + + + {commitUrl ? ( + + } + onClick={(event) => openCommitUrl(event, commitUrl)} + > + + {commit.sha.slice(0, 7)} + + + ) : ( + + {commit.sha.slice(0, 7)} + + )} +
+ ); + })} + + {branchCommitsQuery.error ? ( + {branchCommitsQuery.error} + ) : !canReadFiles ? ( + + {fileAccess.isPending + ? "Checking file access..." + : (fileAccess.error ?? "This connection cannot read local diffs.")} + + ) : branchCommits === null ? ( + Loading commits... + ) : branchCommits.commits.length === 0 ? ( + No commits on this branch. + ) : branchCommits.truncated ? ( + + {`Showing latest ${branchCommits.commits.length} commits`} + + ) : null} + + {isGitSelection && selectedGitScope === "branch" && selectedGitSource?.baseRef && ( @@ -894,7 +1081,12 @@ export default function DiffPanel({ )} -
- +
+ + {isElectron ? ( +
+ + +
+ ) : null} +
{pillLabel ? (
@@ -77,8 +94,10 @@ export const SidebarChromeHeader = memo(function SidebarChromeHeader({ }); export function SidebarBrandWidthProbe({ + isElectron, onWidthChange, }: { + isElectron: boolean; onWidthChange: (width: number) => void; }) { const observeWidth = (probe: HTMLDivElement) => { @@ -93,17 +112,60 @@ export function SidebarBrandWidthProbe({ className="pointer-events-none invisible fixed top-0 left-0 flex w-max border-r border-transparent pr-3" ref={observeWidth} > -
+
+ {isElectron ?
: null}
); } +function HistoryNavigationButton({ + direction, + onBackdrop, +}: { + direction: HistoryDirection; + onBackdrop: boolean; +}) { + const keybindings = useAtomValue(primaryServerKeybindingsAtom); + const enabled = useCanNavigateHistory(direction); + const label = direction === "back" ? "Go back" : "Go forward"; + const shortcutLabel = shortcutLabelForCommand( + keybindings, + direction === "back" ? "navigation.back" : "navigation.forward", + ); + const Icon = direction === "back" ? ArrowLeftIcon : ArrowRightIcon; + return ( + + navigateHistory(direction)} + size="icon-sm" + variant={onBackdrop ? "media-navigation" : "ghost"} + /> + } + > + + + + {label} + {shortcutLabel ? ` (${shortcutLabel})` : ""} + + + ); +} + function SidebarBrand({ onBackdrop }: { onBackdrop: boolean }) { return ( { ).toBe("Map to a different model."); }); }); + +describe("model mapping validation", () => { + const mapping = (model: string, alias: string): UsagePriceDraft => ({ + id: `model:${model}`, + model, + alias, + isNew: false, + values: {}, + }); + + it("rejects a cycle created by several mappings in one save", () => { + const edits = [mapping("a", "b"), mapping("b", "c"), mapping("c", "a")]; + const result = usagePriceTableChanges(target("local", {}), edits); + expect([...result.errors.keys()]).toEqual(["model:a", "model:b", "model:c"]); + }); + + it("checks each environment's saved chain when an edited mapping closes it", () => { + const edits = [mapping("c", "a")]; + const acyclic = target("one", {}, { a: "b" }); + const cyclic = target("two", {}, { a: "b", b: "c" }); + expect(usagePriceTableChanges(acyclic, edits).errors.size).toBe(0); + expect(usagePriceTableErrors([acyclic, cyclic], edits).get("model:c")).toBe( + "Model mappings cannot form a cycle on two.", + ); + }); + + it("validates the final batch so a mapping can be reversed in either row order", () => { + const environment = target("local", {}, { constructor: "__proto__" }); + const clear = mapping("constructor", ""); + const reverse = mapping("__proto__", "constructor"); + for (const edits of [ + [reverse, clear], + [clear, reverse], + ]) { + const result = usagePriceTableChanges(environment, edits); + expect(result.errors.size).toBe(0); + expect(result.changes).toEqual( + edits.map(({ model, alias }) => ({ model, alias: alias || null })), + ); + } + }); +}); diff --git a/apps/web/src/components/usage/usagePriceTable.ts b/apps/web/src/components/usage/usagePriceTable.ts index e6ad3d72da8bb..04b667c2ec30d 100644 --- a/apps/web/src/components/usage/usagePriceTable.ts +++ b/apps/web/src/components/usage/usagePriceTable.ts @@ -115,6 +115,26 @@ export function usagePriceTableChanges( if (USAGE_PRICE_FIELDS.some((field) => original[field.key] !== next[field.key])) changes.push(parsed); } + // Check the final batch so clearing a mapping and reversing it in one save stays valid. + const aliases = new Map(Object.entries(target.aliases ?? {})); + for (const change of changes) { + if (!("alias" in change)) continue; + if (change.alias === null) aliases.delete(change.model); + else aliases.set(change.model, change.alias); + } + for (const draft of drafts) { + if (draft.removed || errors.has(draft.id)) continue; + let model = draft.model.trim(); + const seen = new Set(); + while (aliases.has(model)) { + if (seen.has(model)) { + errors.set(draft.id, `Model mappings cannot form a cycle on ${target.label}.`); + break; + } + seen.add(model); + model = aliases.get(model)!; + } + } return { changes, errors }; } diff --git a/apps/web/src/diffPanelStore.ts b/apps/web/src/diffPanelStore.ts index 11d96a3909ce7..6bd7eb0e01bdd 100644 --- a/apps/web/src/diffPanelStore.ts +++ b/apps/web/src/diffPanelStore.ts @@ -17,6 +17,7 @@ export type DiffPanelSelection = | ({ kind: "tool-call" } & ToolCallDiffTarget) | { kind: "branch"; baseRef: string | null } | { kind: "unstaged" } + | { kind: "commit"; sha: string; baseRef: string | null } | { kind: "turn"; turnId: RunId; filePath: string | null; revealRequestId: number }; const DEFAULT_SELECTION: DiffPanelSelection = { kind: "branch", baseRef: null }; @@ -24,10 +25,14 @@ const DEFAULT_SELECTION: DiffPanelSelection = { kind: "branch", baseRef: null }; interface DiffPanelStoreState { byThreadKey: Record; branchBaseRefByThreadKey: Record; + /** A link that selected a commit before opening the closed pane; opening keeps that selection. */ + keepSelectionOnOpenThreadKey: string | null; selectGitScope: (ref: ScopedThreadRef, scope: "branch" | "unstaged") => void; selectBranchBaseRef: (ref: ScopedThreadRef, baseRef: string | null) => void; selectToolCall: (ref: ScopedThreadRef, target: ToolCallDiffTarget) => void; selectTurn: (ref: ScopedThreadRef, turnId: RunId, filePath?: string) => void; + selectCommit: (ref: ScopedThreadRef, sha: string, keepOnOpen?: boolean) => void; + consumeKeepSelectionOnOpen: (ref: ScopedThreadRef) => boolean; reconcileTurnSelection: (ref: ScopedThreadRef, availableTurnIds: ReadonlyArray) => void; removeThread: (ref: ScopedThreadRef) => void; } @@ -37,19 +42,27 @@ function normalizeBaseRef(baseRef: string | null): string | null { return normalized ? normalized : null; } +function branchBaseRef( + state: Pick, + threadKey: string, +): string | null { + const previous = state.byThreadKey[threadKey]; + return previous?.kind === "branch" || previous?.kind === "commit" + ? previous.baseRef + : (state.branchBaseRefByThreadKey[threadKey] ?? null); +} + export const useDiffPanelStore = create()( persist( - (set) => ({ + (set, get) => ({ byThreadKey: {}, branchBaseRefByThreadKey: {}, + keepSelectionOnOpenThreadKey: null, selectGitScope: (ref, scope) => set((state) => { const threadKey = scopedThreadKey(ref); const previous = state.byThreadKey[threadKey]; - const previousBaseRef = - previous?.kind === "branch" - ? previous.baseRef - : (state.branchBaseRefByThreadKey[threadKey] ?? null); + const previousBaseRef = branchBaseRef(state, threadKey); return { byThreadKey: { ...state.byThreadKey, @@ -59,7 +72,7 @@ export const useDiffPanelStore = create()( : { kind: "unstaged" }, }, branchBaseRefByThreadKey: - previous?.kind === "branch" + previous?.kind === "branch" || previous?.kind === "commit" ? { ...state.branchBaseRefByThreadKey, [threadKey]: previous.baseRef } : state.branchBaseRefByThreadKey, }; @@ -102,6 +115,26 @@ export const useDiffPanelStore = create()( }, }; }), + selectCommit: (ref, sha, keepOnOpen = false) => + set((state) => { + const threadKey = scopedThreadKey(ref); + const baseRef = branchBaseRef(state, threadKey); + return { + byThreadKey: { + ...state.byThreadKey, + [threadKey]: { kind: "commit", sha, baseRef }, + }, + branchBaseRefByThreadKey: { ...state.branchBaseRefByThreadKey, [threadKey]: baseRef }, + keepSelectionOnOpenThreadKey: keepOnOpen + ? threadKey + : state.keepSelectionOnOpenThreadKey, + }; + }), + consumeKeepSelectionOnOpen: (ref) => { + if (get().keepSelectionOnOpenThreadKey !== scopedThreadKey(ref)) return false; + set({ keepSelectionOnOpenThreadKey: null }); + return true; + }, reconcileTurnSelection: (ref, availableTurnIds) => set((state) => { const threadKey = scopedThreadKey(ref); @@ -178,6 +211,13 @@ export function selectThreadDiffPanelSelection( return byThreadKey[scopedThreadKey(ref)] ?? DEFAULT_SELECTION; } +export function selectThreadBranchBaseRef( + state: Pick, + ref: ScopedThreadRef | null | undefined, +): string | null { + return ref ? branchBaseRef(state, scopedThreadKey(ref)) : null; +} + export function resolveToolCallDiffTarget( target: ToolCallDiffTarget, items: ReadonlyArray, diff --git a/apps/web/src/hooks/useResizableWidth.ts b/apps/web/src/hooks/useResizableWidth.ts index 16bb19829135b..50fdba38a9fc5 100644 --- a/apps/web/src/hooks/useResizableWidth.ts +++ b/apps/web/src/hooks/useResizableWidth.ts @@ -29,6 +29,7 @@ export interface ResizableWidthHandlers { /** Persist on drag end to avoid a storage write on every animation frame. */ export function useResizableWidth(options: UseResizableWidthOptions): { readonly width: number; + readonly setWidth: (width: number) => void; readonly handlers: ResizableWidthHandlers; } { const { storageKey, defaultWidth, minWidth, maxWidth, edge } = options; @@ -55,6 +56,16 @@ export function useResizableWidth(options: UseResizableWidthOptions): { const clampedWidth = clamp(widthState.width); + const commitWidth = (nextStorageKey: string, nextWidth: number) => { + // Resizing remains usable for this session when browser storage is unavailable. + setWidthState({ storageKey: nextStorageKey, width: nextWidth }); + try { + setLocalStorageItem(nextStorageKey, nextWidth, WidthSchema); + } catch (error) { + console.error("Could not persist panel width.", error); + } + }; + const drag = useResizeDrag((event) => { const host = event.currentTarget.parentElement; // Disable drag transitions through transition-property; the host renders transition-duration. @@ -72,12 +83,7 @@ export function useResizableWidth(options: UseResizableWidthOptions): { return nextWidth; }, finish(finalWidth) { - setWidthState({ storageKey: startStorageKey, width: finalWidth }); - try { - setLocalStorageItem(startStorageKey, finalWidth, WidthSchema); - } catch (error) { - console.error("Could not persist panel width.", error); - } + commitWidth(startStorageKey, finalWidth); }, cleanup(committed) { host?.style.removeProperty("transition-property"); @@ -94,6 +100,7 @@ export function useResizableWidth(options: UseResizableWidthOptions): { return { width: clampedWidth, + setWidth: (value) => commitWidth(storageKey, clamp(value)), handlers: { ref: drag.ref, onPointerDown: drag.onPointerDown, diff --git a/apps/web/src/lib/openPullRequestLink.browser.test.tsx b/apps/web/src/lib/openPullRequestLink.browser.test.tsx new file mode 100644 index 0000000000000..a93f32a8902f7 --- /dev/null +++ b/apps/web/src/lib/openPullRequestLink.browser.test.tsx @@ -0,0 +1,160 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { scopeThreadRef, scopedThreadKey } from "@t3tools/client-runtime/environment"; +import { AsyncResult } from "effect/reactivity"; +import { expect, it, vi } from "vite-plus/test"; +import { page } from "vite-plus/test/browser"; + +import { act, render, stubGlobal } from "../../test/render"; + +const state = vi.hoisted(() => ({ preview: vi.fn(), openLink: vi.fn() })); +vi.mock("../state/use-atom-query-runner", () => ({ useAtomQueryRunner: () => state.preview })); +vi.mock("../browser/useOpenLink", () => ({ useOpenLink: () => state.openLink })); +vi.mock(import("../state/environments"), async (original) => ({ + ...(await original()), + usePrimaryEnvironmentId: () => EnvironmentId.make("local"), +})); +vi.mock("../state/entities", async (original) => ({ + ...(await original()), + readProjects: () => [project], + useProjects: () => [project], + readProject: () => project, + readThreadShell: () => ({ projectId: "project", worktreePath: null }), +})); +vi.mock("../state/server", async (original) => { + const { Atom } = await import("effect/reactivity"); + return { + ...(await original()), + environmentServerConfigsAtom: Atom.make(new Map([["local", {}]])), + }; +}); + +import { DEFAULT_CLIENT_SETTINGS } from "@t3tools/contracts/settings"; +import { __setClientSettingsForTests } from "../hooks/useSettings"; +import { ToastProvider } from "../components/ui/toast"; +import { AppAtomRegistryProvider } from "../rpc/atomRegistry"; +import { useRightPanelStore } from "../rightPanelStore"; +import { useDiffPanelStore } from "../diffPanelStore"; +import { useOpenChangeRequestLink } from "./openPullRequestLink"; + +const environmentId = EnvironmentId.make("local"); +const threadRef = scopeThreadRef(environmentId, ThreadId.make("thread")); +const project = { + id: "project", + environmentId, + workspaceRoot: "/repo", + repositoryIdentity: { + provider: "github", + canonicalKey: "github.com/effect-ts/effect", + displayName: "Effect-TS/effect", + locator: { + source: "git-remote", + remoteName: "origin", + remoteUrl: "https://github.com/Effect-TS/effect.git", + }, + }, +}; +const commitUrl = `https://github.com/Effect-TS/effect/commit/${"a".repeat(40)}?plain=1#diff-original`; +const pullRequestCommitUrl = `https://github.com/Effect-TS/effect/pull/1/commits/${"a".repeat(40)}`; + +function CommitLink({ url = commitUrl }: { url?: string }) { + const open = useOpenChangeRequestLink(threadRef); + return ( + open(event, url)}> + Commit + + ); +} + +function setup() { + state.preview.mockReset().mockResolvedValue(AsyncResult.success({ sources: [] })); + state.openLink.mockReset().mockResolvedValue(undefined); + __setClientSettingsForTests({ + ...DEFAULT_CLIENT_SETTINGS, + pullRequestLinksOpenInApp: true, + browserLinkTarget: "system", + }); + useDiffPanelStore.setState({ + byThreadKey: {}, + branchBaseRefByThreadKey: {}, + keepSelectionOnOpenThreadKey: null, + }); + useRightPanelStore.setState({ byThreadKey: {}, userActionRevisionByThreadKey: {} }); +} + +it.each([ + ["shiftKey", commitUrl], + ["altKey", commitUrl], + ["shiftKey", pullRequestCommitUrl], +] as const)("keeps %s clicks on %s native", async (modifier, url) => { + setup(); + await using screen = await render( + + + , + ); + const event = new MouseEvent("click", { bubbles: true, cancelable: true, [modifier]: true }); + const defaults: boolean[] = []; + screen.container.addEventListener( + "click", + (nativeEvent) => { + defaults.push(nativeEvent.defaultPrevented); + nativeEvent.preventDefault(); + }, + { once: true }, + ); + await act(() => screen.getByRole("link", { name: "Commit" }).element().dispatchEvent(event)); + expect(defaults).toEqual([false]); + expect(state.preview).not.toHaveBeenCalled(); + expect(useRightPanelStore.getState().byThreadKey).toEqual({}); +}); + +it.each(["system", "app"] as const)( + "offers the original unavailable commit URL when web popups are blocked (%s)", + async (browserLinkTarget) => { + setup(); + // Without preview access, the "app" preference opens a popup too. + __setClientSettingsForTests({ + ...DEFAULT_CLIENT_SETTINGS, + pullRequestLinksOpenInApp: true, + browserLinkTarget, + }); + using _desktop = stubGlobal("desktopBridge", undefined); + await using screen = await render( + + + + + , + ); + using open = vi.spyOn(window, "open").mockReturnValue(null); + await screen.getByRole("link", { name: "Commit" }).click(); + await expect + .element(page.getByRole("button", { name: "Open link", exact: true })) + .toBeVisible(); + expect(open).not.toHaveBeenCalled(); + await page.getByRole("button", { name: "Open link", exact: true }).click(); + expect(open).toHaveBeenCalledExactlyOnceWith(commitUrl, "_blank", "noopener,noreferrer"); + expect(state.openLink).not.toHaveBeenCalled(); + }, +); + +it("opens an available commit in the closed Diff pane without the pane resetting it", async () => { + setup(); + const sha = "a".repeat(40); + state.preview.mockResolvedValue( + AsyncResult.success({ sources: [{ kind: "commit", headRef: sha }] }), + ); + await using screen = await render( + + + , + ); + await screen.getByRole("link", { name: "Commit" }).click(); + await vi.waitFor(() => + expect(useDiffPanelStore.getState().byThreadKey).toEqual({ + [scopedThreadKey(threadRef)]: { kind: "commit", sha, baseRef: null }, + }), + ); + // ChatView resets a pane opened without an explicit request to Changes unless this is set. + expect(useDiffPanelStore.getState().consumeKeepSelectionOnOpen(threadRef)).toBe(true); +}); diff --git a/apps/web/src/lib/openPullRequestLink.ts b/apps/web/src/lib/openPullRequestLink.ts index 014728f058a43..31c04b6263e5f 100644 --- a/apps/web/src/lib/openPullRequestLink.ts +++ b/apps/web/src/lib/openPullRequestLink.ts @@ -8,15 +8,25 @@ import { resolvePullRequestPreviewTarget, } from "@t3tools/client-runtime/change-request-links"; +import { sourceControlClients } from "@t3tools/client-runtime/source-control-clients"; + +import { canOpenLinksInApp } from "../browser/browserLinkTarget"; import { useOpenLink } from "../browser/useOpenLink"; import { useClientSettings } from "../hooks/useSettings"; import { stackedThreadToast, toastManager } from "../components/ui/toast"; -import { useRightPanelStore } from "../rightPanelStore"; +import { selectActiveRightPanel, useRightPanelStore } from "../rightPanelStore"; +import { + selectThreadBranchBaseRef, + selectThreadDiffPanelSelection, + useDiffPanelStore, +} from "../diffPanelStore"; import { appAtomRegistry } from "../rpc/atomRegistry"; -import { readProjects, useProjects } from "../state/entities"; +import { readProject, readProjects, readThreadShell, useProjects } from "../state/entities"; import { environmentServerConfigsAtom, serverEnvironment } from "../state/server"; import { usePrimaryEnvironmentId } from "../state/environments"; +import { reviewEnvironment } from "../state/review"; +import { useAtomQueryRunner } from "../state/use-atom-query-runner"; export function usePullRequestPreviewTarget(environmentId: EnvironmentId | null, url: string) { const projects = useProjects(); @@ -43,71 +53,178 @@ export function useOpenChangeRequestLink( event: Pick< MouseEvent, "preventDefault" | "stopPropagation" | "metaKey" | "ctrlKey" - >, + > & + Partial, "button" | "shiftKey" | "altKey">>, targetUrl: string, targetThreadRef?: ScopedThreadRef, targetEnvironmentId?: EnvironmentId, + onCommitUnavailable?: () => void, ) => boolean { const navigate = useNavigate(); const primaryEnvironmentId = usePrimaryEnvironmentId(); const openInApp = useClientSettings((settings) => settings.pullRequestLinksOpenInApp); - return (event, targetUrl, targetThreadRef, targetEnvironmentId) => { + const ignoreWhitespace = useClientSettings((settings) => settings.diffIgnoreWhitespace); + const browserLinkTarget = useClientSettings((settings) => settings.browserLinkTarget); + const readDiffPreview = useAtomQueryRunner(reviewEnvironment.diffPreview, { + reportFailure: false, + refresh: true, + }); + const openLink = useOpenLink(threadRef); + return (event, targetUrl, targetThreadRef, targetEnvironmentId, onCommitUnavailable) => { if (!openInApp || shouldOpenPullRequestExternally(event)) return false; - // Read at click time: every provider status push replaces the config map, and - // subscribing would hand every rendered link a new handler. Projects likewise, so a - // link can resolve against a snapshot that arrived after this handler was built. - const serverConfigs = appAtomRegistry.get(environmentServerConfigsAtom); const resolvedThreadRef = targetThreadRef ?? threadRef; const resolvedPanelRef = panelRef ?? resolvedThreadRef; - const target = resolveChangeRequestLinkTarget({ - url: targetUrl, - projects: readProjects(), - configLoaded: (environmentId) => serverConfigs.has(environmentId), - environmentId: resolvedThreadRef?.environmentId ?? targetEnvironmentId, - primaryEnvironmentId, - lendHostProject: resolvedPanelRef !== undefined, - }); - if (target === null) return false; - const { project, repository, number, host } = target; - event.preventDefault(); - event.stopPropagation(); - const pullRequestsSearch = { - repository, - number, - ...(host === undefined ? {} : { selectedHost: host }), - selectedProjectId: project.id, - selectedEnvironmentId: project.environmentId, - }; - if (resolvedPanelRef) { - useRightPanelStore.getState().openPullRequest(resolvedPanelRef, { - // The standalone PR panel has a synthetic ref; each tab keeps its real environment. - ...(resolvedPanelRef.environmentId === project.environmentId - ? {} - : { environmentId: project.environmentId }), - projectId: project.id, - ...(host === undefined ? {} : { host }), - repository, + const openPullRequest = () => { + // Read at click time: every provider status push replaces the config map, and + // subscribing would hand every rendered link a new handler. Projects likewise, so a + // link can resolve against a snapshot that arrived after this handler was built. + const serverConfigs = appAtomRegistry.get(environmentServerConfigsAtom); + const target = resolveChangeRequestLinkTarget({ url: targetUrl, - number, + projects: readProjects(), + configLoaded: (environmentId) => serverConfigs.has(environmentId), + environmentId: resolvedThreadRef?.environmentId ?? targetEnvironmentId, + primaryEnvironmentId, + lendHostProject: resolvedPanelRef !== undefined, }); - if (!resolvedThreadRef) { - void navigate({ - to: "/pull-requests", - search: (previous) => ({ - ...previous, - involvement: previous.involvement ?? "all", - state: previous.state ?? "all", - ...pullRequestsSearch, - }), - replace: true, + if (target === null) return false; + const { project, repository, number, host } = target; + const pullRequestsSearch = { + repository, + number, + ...(host === undefined ? {} : { selectedHost: host }), + selectedProjectId: project.id, + selectedEnvironmentId: project.environmentId, + }; + if (resolvedPanelRef) { + useRightPanelStore.getState().openPullRequest(resolvedPanelRef, { + // The standalone PR panel has a synthetic ref; each tab keeps its real environment. + ...(resolvedPanelRef.environmentId === project.environmentId + ? {} + : { environmentId: project.environmentId }), + projectId: project.id, + ...(host === undefined ? {} : { host }), + repository, + url: targetUrl, + number, }); + if (!resolvedThreadRef) { + void navigate({ + to: "/pull-requests", + search: (previous) => ({ + ...previous, + involvement: previous.involvement ?? "all", + state: previous.state ?? "all", + ...pullRequestsSearch, + }), + replace: true, + }); + } + return true; } + void navigate({ + to: "/pull-requests", + search: { involvement: "all", state: "all", ...pullRequestsSearch }, + }); + return true; + }; + const commitThread = resolvedThreadRef ? readThreadShell(resolvedThreadRef) : null; + const commitProject = + commitThread && resolvedThreadRef + ? readProject({ + environmentId: resolvedThreadRef.environmentId, + projectId: commitThread.projectId, + }) + : null; + const identity = commitProject?.repositoryIdentity; + const parsedCommit = sourceControlClients.parseCommitUrl(targetUrl, identity?.provider); + // Shift and Alt keep their browser meaning (new window, download) on commit links. + if (parsedCommit !== null && (event.shiftKey || event.altKey)) return false; + if ( + parsedCommit !== null && + commitThread && + resolvedThreadRef && + commitProject?.environmentId === resolvedThreadRef.environmentId && + identity && + identity.provider === parsedCommit.definition.kind && + parsedCommit.definition.isChangeRequestInRepository(identity, parsedCommit.link) + ) { + const commit = parsedCommit.link; + event.preventDefault(); + event.stopPropagation(); + const userActionRevision = useRightPanelStore.getState().markUserAction(resolvedThreadRef); + const baseRef = selectThreadBranchBaseRef(useDiffPanelStore.getState(), resolvedThreadRef); + const selection = selectThreadDiffPanelSelection( + useDiffPanelStore.getState().byThreadKey, + resolvedThreadRef, + ); + void readDiffPreview({ + environmentId: resolvedThreadRef.environmentId, + input: { + cwd: commitThread.worktreePath ?? commitProject.workspaceRoot, + ...(baseRef ? { baseRef } : {}), + commitSha: commit.sha, + ignoreWhitespace, + }, + }) + .then((result) => { + if ( + useRightPanelStore.getState().getUserActionRevision(resolvedThreadRef) !== + userActionRevision || + selectThreadDiffPanelSelection( + useDiffPanelStore.getState().byThreadKey, + resolvedThreadRef, + ) !== selection + ) + return; + if ( + result._tag === "Success" && + result.value.sources.some( + (source) => source.kind === "commit" && source.headRef === commit.sha, + ) + ) { + const diffOpen = + selectActiveRightPanel( + useRightPanelStore.getState().byThreadKey, + resolvedThreadRef, + ) === "diff"; + useDiffPanelStore.getState().selectCommit(resolvedThreadRef, commit.sha, !diffOpen); + useRightPanelStore.getState().open(resolvedThreadRef, "diff"); + return; + } + if (openPullRequest()) return; + if (onCommitUnavailable) return onCommitUnavailable(); + // A popup opened after the lookup has lost the click's activation, so browsers block it. + if ( + window.desktopBridge === undefined && + (browserLinkTarget !== "app" || !canOpenLinksInApp(resolvedThreadRef)) + ) { + const toastId = toastManager.add( + stackedThreadToast({ + type: "info", + title: "This commit is not in the thread's workspace", + actionProps: { + children: "Open link", + onClick: () => { + toastManager.close(toastId); + window.open(targetUrl, "_blank", "noopener,noreferrer"); + }, + }, + }), + ); + return; + } + return openLink(targetUrl, { + event: { metaKey: event.metaKey, ctrlKey: event.ctrlKey, button: event.button }, + threadRef: resolvedThreadRef, + }); + }) + .catch((error: unknown) => console.error(error)); return true; } - void navigate({ - to: "/pull-requests", - search: { involvement: "all", state: "all", ...pullRequestsSearch }, - }); + if (!openPullRequest()) return false; + event.preventDefault(); + event.stopPropagation(); return true; }; } diff --git a/apps/web/src/lib/runtime.test.ts b/apps/web/src/lib/runtime.test.ts index 22e362ddd2e81..5a34519a97f17 100644 --- a/apps/web/src/lib/runtime.test.ts +++ b/apps/web/src/lib/runtime.test.ts @@ -12,12 +12,12 @@ describe("web runtime relay telemetry policy", () => { it.each([ { surface: "desktop opt-out", - desktopBridge: { getRelayTelemetryEnabled: () => false }, + desktopBridge: { getAppBranding: () => null, getRelayTelemetryEnabled: () => false }, exports: false, }, { surface: "desktop enabled", - desktopBridge: { getRelayTelemetryEnabled: () => true }, + desktopBridge: { getAppBranding: () => null, getRelayTelemetryEnabled: () => true }, exports: true, }, { diff --git a/apps/web/src/lib/surfaceShortcutKey.ts b/apps/web/src/lib/surfaceShortcutKey.ts new file mode 100644 index 0000000000000..98c4d6660da11 --- /dev/null +++ b/apps/web/src/lib/surfaceShortcutKey.ts @@ -0,0 +1,14 @@ +type SurfaceKeyEvent = Pick & + Partial>; + +export function surfaceShortcutKey(event: SurfaceKeyEvent): string | null { + // AltGr text must stay text even when the browser omits Ctrl and Alt. + if (event.getModifierState?.("AltGraph") && !/^[a-z0-9]$/i.test(event.key)) return null; + const typed = event.key.toLowerCase(); + // Preserve Latin layouts, punctuation and dead keys; non-Latin letters/marks use the physical key. + const physical = + /^[\p{L}\p{M}]+$/u.test(typed) && !/\p{Script=Latin}/u.test(typed) + ? event.code?.match(/^Key([A-Z])$/)?.[1]?.toLowerCase() + : undefined; + return physical ?? typed; +} diff --git a/apps/web/src/lib/turnDiffTree.ts b/apps/web/src/lib/turnDiffTree.ts index 810a04755e895..e841e18ce24d2 100644 --- a/apps/web/src/lib/turnDiffTree.ts +++ b/apps/web/src/lib/turnDiffTree.ts @@ -98,20 +98,6 @@ function toTreeNodes(directory: MutableDirectoryNode): TurnDiffTreeNode[] { return [...subdirectories, ...files]; } -export function summarizeTurnDiffStats(files: ReadonlyArray): TurnDiffStat { - return files.reduce( - (acc, file) => { - const stat = readStat(file); - if (!stat) return acc; - return { - additions: acc.additions + stat.additions, - deletions: acc.deletions + stat.deletions, - }; - }, - { additions: 0, deletions: 0 }, - ); -} - export function buildTurnDiffTree(files: ReadonlyArray): TurnDiffTreeNode[] { const root: MutableDirectoryNode = { name: "", diff --git a/apps/web/src/navigationHistory.ts b/apps/web/src/navigationHistory.ts new file mode 100644 index 0000000000000..e062bb8fd7618 --- /dev/null +++ b/apps/web/src/navigationHistory.ts @@ -0,0 +1,22 @@ +import { useSyncExternalStore } from "react"; + +export type HistoryDirection = "back" | "forward"; + +function subscribeToNavigationHistory(onChange: () => void): () => void { + window.navigation.addEventListener("currententrychange", onChange); + return () => window.navigation.removeEventListener("currententrychange", onChange); +} + +export function canNavigateHistory(direction: HistoryDirection): boolean { + return direction === "back" ? window.navigation.canGoBack : window.navigation.canGoForward; +} + +export function navigateHistory(direction: HistoryDirection) { + if (direction === "back") window.history.back(); + else window.history.forward(); +} + +/** Only for Electron: the Navigation API is Chromium-only. */ +export function useCanNavigateHistory(direction: HistoryDirection): boolean { + return useSyncExternalStore(subscribeToNavigationHistory, () => canNavigateHistory(direction)); +} diff --git a/apps/web/src/rightPanelStore.ts b/apps/web/src/rightPanelStore.ts index 33c537a3cf942..ea043c0298bdb 100644 --- a/apps/web/src/rightPanelStore.ts +++ b/apps/web/src/rightPanelStore.ts @@ -109,6 +109,7 @@ interface RightPanelStoreState { userActionRevisionByThreadKey: Record; closeRevisionByThreadKey: Record; getUserActionRevision: (ref: ScopedThreadRef) => number; + markUserAction: (ref: ScopedThreadRef) => number; /** Refused if the user chose a panel after expectedUserActionRevision was read. */ openProactive: ( ref: ScopedThreadRef, @@ -598,6 +599,16 @@ export const useRightPanelStore = create()( closeRevisionByThreadKey: {}, getUserActionRevision: (ref) => get().userActionRevisionByThreadKey[scopedThreadKey(ref)] ?? 0, + markUserAction: (ref) => { + const revision = get().getUserActionRevision(ref) + 1; + set((state) => ({ + userActionRevisionByThreadKey: { + ...state.userActionRevisionByThreadKey, + [scopedThreadKey(ref)]: revision, + }, + })); + return revision; + }, openProactive: (ref, request, expectedUserActionRevision, activateExisting = false) => { let opened = false; set((state) => { diff --git a/apps/web/src/session-logic.test.ts b/apps/web/src/session-logic.test.ts index 9c4e38c30e510..df3eb85c3db38 100644 --- a/apps/web/src/session-logic.test.ts +++ b/apps/web/src/session-logic.test.ts @@ -1307,6 +1307,23 @@ describe("HTML renders in the timeline", () => { expect(rowsFor(entries, false)).toEqual(["message", "turn-fold", "mcp-app", "message"]); }); + it("hosts an MCP app whose call failed, as a form-deferring write does", () => { + const mcpApp = { + attachmentId: "render-thread-app-html", + server: "github", + tool: "create_pull_request", + resourceUri: "ui://github-mcp-server/pr-create", + }; + const call = renderCall("failed", { + t3McpApp: mcpApp, + result: { content: [], isError: true }, + }); + const entries = turn( + call.type === "dynamic_tool" ? { ...call, toolName: "github.create_pull_request" } : call, + ); + expect(entries.find((entry) => entry.kind === "mcp-app")).toMatchObject({ mcpApp }); + }); + it("keeps a render visible when its superseded attempt folds", () => { const attempt: OrchestrationV2RunAttempt = { id: RunAttemptId.make("attempt-superseded"), diff --git a/apps/web/src/session-logic.ts b/apps/web/src/session-logic.ts index 7b79dea6f7401..6cf2761d3d6d3 100644 --- a/apps/web/src/session-logic.ts +++ b/apps/web/src/session-logic.ts @@ -711,10 +711,7 @@ export function deriveTimelineEntriesFromVisibleTurnItems( continue; } - const mcpApp = - item.type === "dynamic_tool" && item.status === "completed" - ? mcpAppFromToolItem(item) - : undefined; + const mcpApp = item.type === "dynamic_tool" ? mcpAppFromToolItem(item) : undefined; if (mcpApp !== undefined) { entries.push({ id: item.id, diff --git a/apps/web/src/state/entities.ts b/apps/web/src/state/entities.ts index 54b367df15b8b..0f94280768d4b 100644 --- a/apps/web/src/state/entities.ts +++ b/apps/web/src/state/entities.ts @@ -65,6 +65,9 @@ const EMPTY_VISIBLE_TURN_ITEMS_ATOM = Atom.make(EMPTY_VISIBLE_TURN_ITEMS).pipe( const EMPTY_CHECKPOINT_SUMMARIES_ATOM = Atom.make>( Object.freeze([]), ).pipe(Atom.withLabel("web-thread-checkpoint-summaries:empty")); +const EMPTY_ASSISTANT_MESSAGE_TEXT_ATOM = Atom.make(null).pipe( + Atom.withLabel("web-assistant-message-text:empty"), +); const EMPTY_LAST_VISITED_AT_ATOM = Atom.make(null).pipe( Atom.withLabel("web-thread-last-visited:empty"), ); @@ -212,6 +215,29 @@ export function useThreadVisibleTurnItems( ); } +/** Visible turn items held through streamed assistant text; see useAssistantMessageText. */ +export function useTimelineVisibleTurnItems( + ref: ScopedThreadRef | null, +): ReadonlyArray { + return useAtomValue( + ref === null + ? EMPTY_VISIBLE_TURN_ITEMS_ATOM + : environmentThreadDetails.timelineVisibleTurnItemsAtom(ref), + ); +} + +/** One assistant message's live text, or null when the thread does not hold the item. */ +export function useAssistantMessageText( + ref: ScopedThreadRef | null, + itemId: string | undefined, +): string | null { + return useAtomValue( + ref === null || itemId === undefined + ? EMPTY_ASSISTANT_MESSAGE_TEXT_ATOM + : environmentThreadDetails.assistantMessageTextAtom(ref, itemId), + ); +} + /** Visible turn items without their streamed text; see ThreadStructure. */ export function useQuietThreadVisibleTurnItems( ref: ScopedThreadRef | null, diff --git a/apps/web/test/appScenarios.renders.tsx b/apps/web/test/appScenarios.renders.tsx index 9c1a7d520eeb1..459f3b9791d7f 100644 --- a/apps/web/test/appScenarios.renders.tsx +++ b/apps/web/test/appScenarios.renders.tsx @@ -182,7 +182,7 @@ it("app-stream-assistant-turn", async () => { const counted = await measure(async () => { let text = ""; for (let delta = 0; delta < STREAM_DELTAS; delta++) { - text += `word${delta} `; + text += `word${delta}\n\n`; await act(() => emitThreadEvent(LIVE, { type: "turn-item.updated", @@ -201,6 +201,13 @@ it("app-stream-assistant-turn", async () => { .element(page.getByText(`word${STREAM_DELTAS - 1}`, { exact: false })) .toBeVisible(); }); + // Only the streaming row changes. The timeline renders for the new item and its layout, not + // per delta, and history rows not at all. + expect(counted.components.ConversationTimeline).toBeLessThanOrEqual(3); + expect(counted.components.UserTimelineRow).toBeUndefined(); + expect(counted.components.AssistantTimelineRow).toBe(STREAM_DELTAS); + // Each delta adds a paragraph; finished blocks keep their render. + expect(counted.components.MarkdownParagraph).toBe(STREAM_DELTAS); await report("app-stream-assistant-turn", [counted], { commits: false }); }); diff --git a/docs/fork/0016-provider-secrets-live-in-1password.md b/docs/fork/0016-provider-secrets-live-in-1password.md deleted file mode 100644 index 63046b13907ed..0000000000000 --- a/docs/fork/0016-provider-secrets-live-in-1password.md +++ /dev/null @@ -1,58 +0,0 @@ -# 0016: Provider secrets can live in 1Password - -- PR: [TrogonStack/t3code#27](https://github.com/TrogonStack/t3code/pull/27) -- Status: active - -## What you can do now - -- Give a provider its credential without giving T3 Code the credential. Set any - provider instance's environment variable to a 1Password secret reference - plus the account it lives in, and T3 Code reads the value from the 1Password - CLI when it starts the agent. What gets saved is the reference; the secret itself is - never written to the settings file or to T3 Code's secret store. -- Unlock your vault once instead of all day. Each reference is read a single - time and held in memory for the life of the server, so starting a thread, - sending a message, and the background status check all reuse it. One approval - covers every reference across every provider, so a machine running five - providers on references costs the same single unlock that one provider does, - both at startup and on a refresh. -- Rotate a secret and pick it up without restarting anything. Refreshing - provider status in Settings goes back to 1Password and rebuilds only the - providers that read from it. Threads already running keep working. -- Tell when a vault is locked. A reference T3 Code cannot read marks the - provider unavailable and names the variable, instead of starting with a blank - credential and failing on your first message. Nothing is remembered about the - failed read, so unlocking the vault and refreshing, or running the Settings - check on that variable, brings the provider back. - -## Why - -Running several accounts of the same provider side by side means several -long-lived tokens, and every one of them had to be copied out of the password -manager and pasted into a second store to be usable. That is the copy nobody -rotates: it outlives the original, it has no expiry anyone tracks, and it sits -in a file that gets synced, backed up, and occasionally shared with a support -thread. - -Pointing at the secret instead of duplicating it keeps one copy under the -policy that was chosen for it, and makes rotation a vault edit plus a refresh -rather than a hunt for everywhere the value was pasted. - -The in-memory hold is what makes it usable rather than merely correct. Reading -the vault on every thread start turns a background biometric prompt into a -constant interruption, which is the kind of friction that ends with the token -pasted in plaintext again to make it stop. - -## Upstream considerations - -Worth proposing upstream, though it is a product decision rather than a bug -fix, so it may not be wanted in that shape. The 1Password-specific piece is -deliberately confined to one layer behind a resolver service, which is the -seam another secret store would plug into, so the argument upstream is about -whether to carry any secret-store integration at all rather than about -1Password specifically. - -Rebase burden is moderate. Resolution happens once, in the provider instance -registry, before a driver is created, so drivers are untouched. A sync that -reshapes how the registry builds or rebuilds instances will conflict, and the -conflicts are shallow. diff --git a/docs/fork/0033-more-secret-stores-and-checking-them.md b/docs/fork/0033-more-secret-stores-and-checking-them.md new file mode 100644 index 0000000000000..64dcda8e6db71 --- /dev/null +++ b/docs/fork/0033-more-secret-stores-and-checking-them.md @@ -0,0 +1,40 @@ +# 0033: More secret stores, and checking them + +- PR: [TrogonStack/t3code#106](https://github.com/TrogonStack/t3code/pull/106) +- Status: active + +## What you can do now + +- Read from OpenBao. An OpenBao provider reads over its HTTP API with one explicit token source: + its own stored token, `BAO_TOKEN`, or the `bao` CLI login. +- Read from AWS Secrets Manager. An AWS provider reads through the `aws` CLI in one region, with + an optional named profile, so credentials stay in the CLI's own configuration. +- Read a provider's own setup fields, such as the Cursor API key, from a secret provider. Pick the + provider next to the field instead of adding the same variable by hand under Environment. +- Edit a provider in place. Change its region, address, mount, namespace, or token from the same + dialog. The name stays, so variables stay pointed at it, though a reference written for the old + mount or region may need updating. +- Check a reference before saving it. A check button on each secret variable reads the reference + through its provider right now and shows whether it resolved, with the store's error when it did + not. A readable reference also rebuilds the providers that read it. +- Keep settings saved by earlier fork builds. Flat provider entries and variables saved with the + earlier source shapes are converted when settings load, so they still resolve with nothing to + migrate. + +## Why + +Upstream reads secrets from 1Password only. People also run OpenBao and AWS Secrets Manager, and +a reference that cannot be tried before it is saved only fails when an agent starts, far from the +place the mistake was made. Naming the provider once and letting every variable point at it keeps +connection details out of each variable, and a check turns a wrong reference into an answer on the +spot. + +## Upstream considerations + +The pieces are built on upstream's secret provider architecture: OpenBao and AWS are drivers +beside the 1Password one, so they could be proposed as separate drivers. The check button, in-place +edit, and the Cursor setup-field source are small additions to upstream's secret UI. + +The rebase burden is small. New kinds are separate packages registered in one list, and the +conversion of earlier settings is one pass before settings decode. A sync that reshapes the secret +provider dialog or the settings secrets store needs the additions carried forward. diff --git a/docs/fork/0033-secret-providers-have-names.md b/docs/fork/0033-secret-providers-have-names.md deleted file mode 100644 index d845eb93dca95..0000000000000 --- a/docs/fork/0033-secret-providers-have-names.md +++ /dev/null @@ -1,57 +0,0 @@ -# 0033: Secret providers have names - -- PR: [TrogonStack/t3code#106](https://github.com/TrogonStack/t3code/pull/106) -- Status: active - -## What you can do now - -- Connect more than one secret store and give each a name. Under Settings, - Secrets, add a provider for the 1Password CLI or for OpenBao, and keep as many - as you need side by side. -- Pick a provider by name on any provider variable. The variable keeps only the - provider name and the reference inside it, so switching a variable to another - store is a change of source, not a rewrite. -- Read from OpenBao as well as 1Password. An OpenBao provider reads over its HTTP API - with one explicit token source: its own stored token, `BAO_TOKEN`, or the `bao` CLI login. -- Read from AWS Secrets Manager. An AWS provider reads through the `aws` CLI in one region, with - an optional named profile, so credentials stay in the CLI's own configuration. -- Read a provider's own setup fields, such as the Cursor API key, from a secret provider. Pick the - provider next to the field instead of adding the same variable by hand under Environment. -- Edit a provider in place. Change its region, address, mount, namespace, or token from the same - dialog. The name stays, so variables stay pointed at it, though a reference written for the old - mount or region may need updating. -- Check a reference before saving it. A check button on each secret variable reads the reference - through its provider right now and shows whether it resolved, with the store's error when it did - not. -- Keep variables saved with the earlier 1Password source. They still resolve and - still open as before, with nothing to migrate. - -## Why - -Reading credentials from 1Password solved one store for one account. People -also run OpenBao, and some work across several 1Password accounts, so a variable -that hard-codes "1Password plus an account" stops fitting. Naming the provider -once, in one place, lets every variable point at it and keeps connection details -out of each variable. - -## Upstream considerations - -Upstream now carries a first slice of named secret providers: settings-only -1Password providers stored as a kind, a label, and a kind-specific config. It -does not read secrets from them yet and has no OpenBao or AWS kinds. The fork -accepts both shapes. Providers it already stores keep their flat shape and -behave as before, and providers written in upstream's shape are listed and read -from the same way, so settings move freely between the fork and upstream. - -The flat kinds are a pending convergence: once upstream resolves secrets and -adds more kinds, the fork should store its providers in upstream's shape and -drop the flat one. Until then, the Secrets page is always on in the fork, -where upstream gates it behind a feature flag. - -Beyond that, this builds on the secret-store integration we already carry -(0016) and shares its argument: whether to carry any secret-store integration -at all. A new kind of store is a new kind of provider, not a new concept. - -The rebase burden is small beyond what 0016 already costs. It adds a settings -section and a page, so a sync that reshapes settings navigation or the provider -variables editor needs the new pieces carried forward. diff --git a/docs/fork/README.md b/docs/fork/README.md index 064715bead966..c5c8e48ca47e1 100644 --- a/docs/fork/README.md +++ b/docs/fork/README.md @@ -33,7 +33,6 @@ Each entry uses these sections: active, [#19](https://github.com/TrogonStack/t3code/pull/19) - **0012** [The timeline scrolls only as far as its content](./0012-timeline-scrolls-only-as-far-as-its-content.md) active, [#21](https://github.com/TrogonStack/t3code/pull/21) -- **0016** [Provider secrets can live in 1Password](./0016-provider-secrets-live-in-1password.md) active, [#27](https://github.com/TrogonStack/t3code/pull/27) - **0017** [A revoked Claude token reads as revoked](./0017-a-revoked-claude-token-reads-as-revoked.md) active, [#28](https://github.com/TrogonStack/t3code/pull/28) @@ -43,5 +42,5 @@ Each entry uses these sections: submitted, [#97](https://github.com/TrogonStack/t3code/pull/97) - **0032** [The model picker reopens where you last picked from](./0032-model-picker-reopens-where-you-left-it.md) active, [#103](https://github.com/TrogonStack/t3code/pull/103) -- **0033** [Secret providers have names](./0033-secret-providers-have-names.md) +- **0033** [More secret stores, and checking them](./0033-more-secret-stores-and-checking-them.md) active, [#106](https://github.com/TrogonStack/t3code/pull/106) diff --git a/docs/internals/devices.md b/docs/internals/devices.md index 5273f7942e9e0..28fa103feb5cb 100644 --- a/docs/internals/devices.md +++ b/docs/internals/devices.md @@ -36,13 +36,13 @@ its sessions. Sessions it starts are recorded under the state directory and stopped by the next server if this one dies; sessions other tools started on the same account are never touched. -Each session brings its own serve-sim preview host and agent-device daemon, -with their own tokens. The proxy asks the host to route each request -(`DeviceHostReady.upstream`) and adds the session token server-side, so +Each session brings its own preview host (serve-sim, or serve-emu for Android) +and agent-device daemon, with their own tokens. The proxy asks the host to +route each request (`DeviceHostReady.upstream`) and adds the session token server-side, so clients never hold it. Agent config files are written per device (`DeviceOperations.agentEndpoint`), which is why `device_open` opens the -device before targeting it. Device settings run `simctl` on the simulator's -machine, which EAS does not expose, so they are unavailable there. +device before targeting it. Device settings and tools run `simctl` or `adb` on +the device's machine, which EAS does not expose, so they are unavailable there. ## Simlock hosts lease through the agent-device front @@ -147,8 +147,8 @@ decoder then retains frames for seconds despite `optimizeForLatency` and an empt `decodeQueueSize`. The iOS AVCC viewer restores the missing metadata before configuring the decoder so hardware decoding can remain enabled. WebCodecs is secure-context only, so the viewer probes `isConfigSupported` and falls back to -MJPEG on iOS when decoding is unavailable. Android has no MJPEG; there the panel -reports that it cannot decode. +MJPEG on iOS when decoding is unavailable. Android's SEMU socket has no MJPEG; +there the panel reports that it cannot decode. serve-sim can also be locked to WebRTC, as on EAS Simulator, and then refuses AVCC with a 409 naming the transport. The viewer still asks for AVCC first, so @@ -156,3 +156,6 @@ hubs that serve it pay nothing, and switches on that 409, or when its MJPEG image fails in a browser that has `RTCPeerConnection`. Offers are not trickled: serve-sim takes one offer with every candidate. The video element only feeds the same canvas sink, so the panel and the mobile WebView keep one surface. +Android asks serve-emu's `/api` before opening its video socket and streams +WebRTC when it is advertised (EAS Simulator), keeping a separate `video=0` +socket for input so input can reconnect without restarting video. diff --git a/docs/internals/providers.md b/docs/internals/providers.md index f1537f5a86ba0..1fb348bfdb743 100644 --- a/docs/internals/providers.md +++ b/docs/internals/providers.md @@ -145,100 +145,30 @@ current client support. ## Secret references in provider environments -A provider instance's `environment` can hold secret references: a `ProviderSecretSource` value such -as `{ kind: "secret-provider", provider, reference }`, or the older `{ kind: "1password", reference, account }`. A plain string is always a literal, including one -that starts with `op://`. -[`ProviderSecretResolver`](../../apps/server/src/provider/ProviderSecretResolver.ts) swaps -each one for the value its secret store returns. This happens once per instance in -[`ProviderInstanceRegistry`](../../apps/server/src/provider/ProviderInstanceRegistry.ts), -before `driver.create`, so drivers and the orchestration v2 adapters built from them only ever see -resolved values; `ResolvedProviderEnvironment` is the type that enforces it. The registry keeps the -raw config, which is what makes a later rebuild possible. Paths that build a process environment -without the resolver (terminals, usage, installation, session scanning) use +A provider instance's environment variable is either a literal `value` or a `secret`, a +`SecretReference` naming an entry under the `secretProviders` server setting. The registry resolves +each reference through +[`SecretResolver`](../../packages/secret-provider-core/src/server/SecretResolver.ts) once per +instance, before `driver.create`, so drivers only ever see resolved values. Paths that build a +process environment without the resolver (terminals, usage, installation, session scanning) use `literalProviderInstanceEnvironment`, which leaves references out rather than passing them through. -The parsing half lives in -[`ProviderSecretReference.ts`](../../apps/server/src/provider/ProviderSecretReference.ts) and knows -nothing about how a secret is fetched, so the registry can ask "does this instance read from a secret -store?" without depending on the resolver. -[`ProviderSecretResolverLive`](../../apps/server/src/provider/ProviderSecretResolverLive.ts) -is the half that fetches values. A `secret-provider` source names an entry under the `secretProviders` -server setting, and the resolver dispatches on that entry's kind: the 1Password CLI for `op`, the -OpenBao HTTP API with the token from one explicit source (the stored token, `BAO_TOKEN`, or the `bao` CLI login), or -the `aws` CLI for AWS Secrets Manager, which owns its own credentials so none are stored. The older 1Password source resolves as if it named a 1Password provider for its -account. The provider id or account is part of the cache key, because the same reference in two -stores is two different secrets. A reference naming a provider that no longer exists is unresolved. -Changing `secretProviders` drops only the cached secrets read through the providers that changed -and rebuilds the instances that read from them, since their own config is unchanged. Editing an -OpenBao provider must not cost a 1Password prompt. - -Three decisions are load-bearing: - -- **A failed read unsets the variable.** It never substitutes an empty string, because an empty - `ANTHROPIC_API_KEY` reads to the provider as a configured-but-broken credential rather than an - absent one, and the status badge would go back to lying about it. Unsetting is stronger than - leaving the name out of the resolved list: the child environment starts from the server's own, so - a name left alone keeps whatever the server inherited under it, and the agent would quietly run - as a different account than the one the instance names. The registry therefore hands the driver a - `HostProcessEnvironment` with every unresolved name removed. -- **Reads are batched across instances, and sequential within one.** The store charges an unlock per - `op` invocation, not per secret, and every instance resolves its own environment as it is built, - so a fleet would otherwise cost one prompt per provider. `prime` reads the whole set with one - `op inject` per account before the builds start, called from the settings watcher (which covers boot) and from - `reloadSecretBackedInstances` (which covers the refresh button). Whatever `prime` misses, - `resolve` still walks with a plain loop rather than `Effect.forEach` with concurrency, so it - produces one prompt rather than several simultaneous ones. -- **Priming can only ever help.** It is best effort and never fails: `op inject` resolves the whole - template or fails it, so one bad reference would take the batch down with it. A batch that does - not come back leaves the cache exactly as cold as it found it and `resolve` falls back to reading - one reference at a time, which is both where the per-variable failure isolation lives and how the - user learns which reference is the broken one. A separator is generated per call because a secret - can contain anything, newlines included. -- **Failures are cached alongside successes.** The `Cache` holds Exits, so a locked vault costs one - prompt per refresh cycle instead of one per thread start. Recovery is the refresh button, not a - timeout: the cache is built without a `timeToLive`. - -### Why a refresh has to rebuild the instance - -An instance's environment is resolved once, when the registry builds it, and -`makeManagedServerProvider` re-probes using that captured `processEnv`. Dropping the cached secret -therefore changes nothing on its own, because the running instance still holds the value it was -built with. Sessions follow a rebuild without extra wiring, because the v2 adapter registry looks up -the instance's adapter on every request. - -So `ProviderRegistry.reloadSecretBackedInstances` invalidates the cache and then calls -[`rebuildInstanceWhen`](../../apps/server/src/provider/ProviderInstanceRegistry.ts) on each -instance, passing `hasProviderSecretReference` as the predicate. The registry owns the child scopes -and already stores each instance's config, so it can -close and rebuild one entry in place; passing a predicate rather than exposing the entries keeps -secret-store policy in `ProviderRegistry` and keeps the instance registry ignorant of 1Password. -`reconcile` cannot do this job, because it diffs the config envelope and a rotated secret leaves -that envelope byte-identical. - -A rebuild takes as long as the secret read does, and a locked vault can park it on a person at a -biometric prompt. Three things follow from that window being long: - -- **The instance leaves the map before its scope closes.** Otherwise every lookup for the whole - window hands back a bundle whose scope is already gone. Its last snapshot stands in for it - meanwhile, because `ProviderRegistry` prunes ids it finds in neither list, and the card must not - blink out of Settings while 1Password waits on a fingerprint. -- **Rebuilds and `reconcile` take turns.** Both read the instance map, do slow work, then write it - back, so a settings change landing mid-rebuild would be overwritten by the map the rebuild read - before it. Reads stay outside the lock: `getInstance` and `listInstances` never wait on a - 1Password prompt. -- **A rebuild that fails is retryable.** The registry keeps the config envelope of an instance it - could not bring back, so the next refresh retries it, and a refresh with no explicit target - covers the unavailable instances as well as the live ones. Without both halves, a vault that - happened to be locked would cost the user the instance until settings changed. The envelope is - recorded before the build starts and the map writes around the build are uninterruptible, so a - refresh whose caller walked away mid-read is retryable on the same terms as one that failed. - -This hangs off the three refresh entry points (`refreshAll`, the kind-scoped `refresh`, and -`refreshInstance`), all of which are reached only by a user action: the Settings refresh button, and -the post-update verification in `providerMaintenanceRunner`. The periodic provider health loop is -not one of them. It lives inside `makeManagedServerProvider` and calls `refreshSnapshot` directly, -which is what keeps a resolved secret alive between refreshes instead of re-reading it every few -minutes. +Each secret provider kind is a driver package: `secret-provider-onepassword`, +`secret-provider-openbao` (KV v2 over HTTP, with the token from its stored config, `BAO_TOKEN`, or +the `bao` login), and `secret-provider-aws` (the `aws` CLI, which owns its own credentials). The +fork carries the last two. The resolver dispatches on the entry's kind and holds every read, failed +ones included, until a refresh invalidates it, so a locked vault costs one prompt per refresh rather +than one per thread start. An OpenBao `config.token` is moved into the secret store on write and +redacted on read, like other secret settings. + +The Settings check button calls `SecretCheck`, which invalidates one reference, reads it again, and +on success reloads the instances that read it. A failed read unsets the variable rather than +substituting an empty string: an empty `ANTHROPIC_API_KEY` reads to the provider as a configured but +broken credential, and a name left alone would keep whatever value the server inherited. + +Settings saved by earlier fork builds (flat provider entries, variables whose `value` held a source +object) are rewritten into this shape when settings load, in +[`legacySecretSettings.ts`](../../apps/server/src/secrets/legacySecretSettings.ts). ## Claude credential liveness diff --git a/docs/operations/development.md b/docs/operations/development.md index bab511e04fb69..9076134c8c6ae 100644 --- a/docs/operations/development.md +++ b/docs/operations/development.md @@ -211,8 +211,12 @@ files to packages with [ci-changes.cjs](../../.github/scripts/ci-changes.cjs), a compares against the merge base with `origin/main`, or with a ref you pass. CI still runs the full test suites and the jobs the summary lists, so a local pass does not promise a green CI run. -The steps run side by side, so a run takes about as long as its slowest step, usually knip's -exports pass at about 10 s. When the files are exactly those of the last passing run, it prints that +The steps run side by side, as many at once as the machine allows: one per two cores and per 6 GiB +of available memory. On a large machine a run takes about as long as its slowest step, usually +knip's exports pass at about 10 s. Pass `--concurrency ` or set `CHECK_CHANGED_CONCURRENCY` to +change the limit (`1` runs the steps one by one). A step that runs past `--timeout ` or +`CHECK_CHANGED_TIMEOUT` (default 900) is killed with every process it started and reported as +`TIMEOUT`, not as a failing check. When the files are exactly those of the last passing run, it prints that run's summary instead of running again. Any change to a file, the install, or a generated file runs the checks again. diff --git a/docs/user/background-service.md b/docs/user/background-service.md index be4ad66ff3ab4..f9ab85fcac4c3 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -90,6 +90,24 @@ Windows background services are not supported. T3 Connect can offer service installation during setup, but the two are managed separately. Signing out of T3 Connect does not stop or uninstall the service. +## Keep the server responsive when agents use all memory + +On Linux, agents already run with lower priority than the server. When they +fill the machine's memory, though, the kernel may still slow down or stop the +server, because only an administrator can reserve memory for it. Run, as your +normal user: + +```sh +t3 service protect +``` + +It shows the system settings it will change, asks before applying them, and +then asks for your sudo password. If the machine runs Tailscale or SSH, it +also offers to protect them, so you can still reach the machine. Memory +protection takes effect at once. The server's protection from the kernel's +out-of-memory killer takes effect after `t3 service restart`. `t3 service status` reports whether +the protection is in place, and `t3 service protect --undo` removes it. + ## Troubleshooting Start with `t3 service status` on the host. It prints the log path and, on Linux, diff --git a/docs/user/devices.md b/docs/user/devices.md index ce5a7d8d06d05..46a5a427cc87c 100644 --- a/docs/user/devices.md +++ b/docs/user/devices.md @@ -94,7 +94,7 @@ started from then on; your own Device panel is unaffected. The device stream goes through the environment server, so it works over the local network, Tailscale, and T3 Connect. Live video needs a secure page (HTTPS or localhost); on a plain-HTTP remote origin iOS falls back to a slower -still-image stream and Android cannot show video. +still-image stream and Android cannot show video, except on EAS Simulator. ## SSH device hosts diff --git a/docs/user/provider-secrets.md b/docs/user/provider-secrets.md index f6ce5dfdddd33..b797a31fd7808 100644 --- a/docs/user/provider-secrets.md +++ b/docs/user/provider-secrets.md @@ -41,9 +41,9 @@ Plain values are used exactly as typed, so mixing literal variables and referenc provider is fine. A plain value is never read from a secret manager, even one that starts with `op://`. -Variables saved earlier with a 1Password account as their source keep working. A sensitive `op://` -value is hidden from the app, so it cannot be recognized; add it again with a named provider as its -source. +Variables saved earlier with a 1Password account as their source are converted to a named provider +when settings load, and keep working. A sensitive `op://` value is hidden from the app, so it cannot +be recognized; add it again with a named provider as its source. ## What Does A Reference Look Like diff --git a/docs/user/source-control.md b/docs/user/source-control.md index 44ceb793416db..c209758e61a68 100644 --- a/docs/user/source-control.md +++ b/docs/user/source-control.md @@ -10,6 +10,16 @@ rebases, the files Git brought in are left out. A file stays in the list when th committed it, or fixed a conflict in it. Use the branch comparison to review everything that changed against your base branch. Restore still returns the complete saved workspace. +On web and desktop, open the Diff pane source picker and choose **Commits** to review one local +commit at a time. The list shows the newest 100 commits in the branch comparison. A merge commit +shows its changes against its first parent. + +A commit link from a supported code host opens in the Diff pane when the thread's repository has +that commit locally, on any branch. Otherwise a pull request commit link opens the pull request. Other +commit links follow your **Open links in** setting; on web, when that opens a new tab, T3 Code first +shows an **Open link** button. Command/Ctrl+Click always opens the link in your browser. Commit links follow the **Open pull requests in T3 Code** +setting in **Settings → Diffs**. + ## Connect an account Install Git and configure authentication on the machine running your T3 Code server. For a remote diff --git a/docs/user/usage.md b/docs/user/usage.md index 155175351c550..af1fc02352f83 100644 --- a/docs/user/usage.md +++ b/docs/user/usage.md @@ -43,6 +43,10 @@ Use absolute paths or `~/` paths in the account's environment settings; relative depend on each project's working directory and cannot be reliably discovered by Usage. Accounts sharing a history directory count once. +The server saves its first hostname in `/usage-host-id`, so a recreated container keeps +its usage identity when T3 home is persisted. Set `T3CODE_HOST_ID` to choose the identity +explicitly. + When your app and server support different providers, usage totals may cover only the providers your app understands. Update the app to include newly supported providers. diff --git a/evals/thread-titles/run.ts b/evals/thread-titles/run.ts index 9195d0ed520a2..f67172b7e0986 100644 --- a/evals/thread-titles/run.ts +++ b/evals/thread-titles/run.ts @@ -20,6 +20,7 @@ import { import * as ThreadTitleLinks from "../../apps/server/src/textGeneration/ThreadTitleLinks.ts"; import * as SourceControlProviderRegistry from "../../apps/server/src/sourceControl/SourceControlProviderRegistry.ts"; import * as SourceControlBuiltInDrivers from "../../apps/server/src/sourceControl/builtInDrivers.ts"; +import * as SecretResolver from "../../apps/server/src/secrets/SecretResolver.ts"; import * as ServerSettings from "../../apps/server/src/serverSettings.ts"; import * as VcsProcess from "../../apps/server/src/vcs/VcsProcess.ts"; import * as VcsDriverRegistry from "../../apps/server/src/vcs/VcsDriverRegistry.ts"; @@ -143,6 +144,7 @@ await Effect.runPromise( ProcessRunner.layer, SourceControlProviderRegistry.layer.pipe( Layer.provide(SourceControlBuiltInDrivers.layer), + Layer.provide(SecretResolver.layer), Layer.provide(ServerSettings.layerTest()), Layer.provide(VcsDriverRegistry.layer.pipe(Layer.provide(VcsProjectConfig.layer))), Layer.provide(GitVcsDriver.layer), diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 01230171be9ba..96661fad42a61 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -343,6 +343,7 @@ function makeTunnelLimits( } return yield* reserve; }), + getOverride: () => Effect.die("unused"), }); } diff --git a/infra/relay/src/environments/ManagedTunnelLimits.test.ts b/infra/relay/src/environments/ManagedTunnelLimits.test.ts index 6aca885b85b87..6b8dd817ee01c 100644 --- a/infra/relay/src/environments/ManagedTunnelLimits.test.ts +++ b/infra/relay/src/environments/ManagedTunnelLimits.test.ts @@ -121,6 +121,27 @@ const reserve = ( ), ); +const getOverride = (override?: number) => + Effect.gen(function* () { + const limits = yield* ManagedTunnelLimits.ManagedTunnelLimits; + return yield* limits.getOverride({ userId: "user-1" }); + }).pipe( + Effect.provide( + ManagedTunnelLimits.layer.pipe( + Layer.provide( + Layer.succeed( + RelayDb.RelayDb, + makeFakeDb({ + otherSlots: () => 0, + ...(override === undefined ? {} : { override }), + }), + ), + ), + Layer.provide(t3ChatLayer("free")), + ), + ), + ); + describe("ManagedTunnelLimits", () => { it.effect("rejects concurrent reservations past the limit", () => { const reserved: Array = []; @@ -383,7 +404,9 @@ describe("ManagedTunnelLimits", () => { reserve({ override, activeTunnels: override, plan: "premier", lookups }), ); expect(error).toMatchObject({ _tag: "ManagedTunnelLimitExceeded", maxTunnels: override }); + expect(yield* getOverride(override)).toBe(override); } + expect(yield* getOverride()).toBeNull(); expect(lookups).toEqual([]); }), ); diff --git a/infra/relay/src/environments/ManagedTunnelLimits.ts b/infra/relay/src/environments/ManagedTunnelLimits.ts index aa373ad27ffb2..3af90f86c4163 100644 --- a/infra/relay/src/environments/ManagedTunnelLimits.ts +++ b/infra/relay/src/environments/ManagedTunnelLimits.ts @@ -106,6 +106,10 @@ export class ManagedTunnelLimits extends Context.Service< input: { readonly userId: string; readonly environmentId: string }, reserve: Effect.Effect, ) => Effect.Effect; + /** The user's manual cap, which replaces the default and any T3 Chat plan cap. */ + readonly getOverride: (input: { + readonly userId: string; + }) => Effect.Effect; } >()("t3code-relay/environments/ManagedTunnelLimits") {} @@ -126,7 +130,24 @@ const make = Effect.gen(function* () { ), ); + const loadOverride = (userId: string) => + db + .select({ maxTunnels: relayManagedTunnelLimits.maxTunnels }) + .from(relayManagedTunnelLimits) + .where(eq(relayManagedTunnelLimits.userId, userId)) + .limit(1) + .pipe( + Effect.map((rows) => rows[0]?.maxTunnels ?? null), + Effect.mapError( + (cause) => + new ManagedTunnelLimitPersistenceError({ operation: "load-limit", userId, cause }), + ), + ); + return ManagedTunnelLimits.of({ + getOverride: Effect.fn("relay.managed_tunnel_limits.get_override")(function* (input) { + return yield* loadOverride(input.userId); + }), withCapacity: Effect.fn("relay.managed_tunnel_limits.with_capacity")(function* ( input: { readonly userId: string; readonly environmentId: string }, reserve: Effect.Effect, @@ -148,22 +169,8 @@ const make = Effect.gen(function* () { ), ); - const overrides = yield* db - .select({ maxTunnels: relayManagedTunnelLimits.maxTunnels }) - .from(relayManagedTunnelLimits) - .where(eq(relayManagedTunnelLimits.userId, input.userId)) - .limit(1) - .pipe( - Effect.mapError( - (cause) => - new ManagedTunnelLimitPersistenceError({ - operation: "load-limit", - userId: input.userId, - cause, - }), - ), - ); - const baseMaxTunnels = overrides[0]?.maxTunnels ?? DEFAULT_MANAGED_TUNNEL_LIMIT; + const override = yield* loadOverride(input.userId); + const baseMaxTunnels = override ?? DEFAULT_MANAGED_TUNNEL_LIMIT; const pendingCutoff = DateTime.formatIso( DateTime.subtract(yield* DateTime.now, { @@ -196,7 +203,7 @@ const make = Effect.gen(function* () { // A manual override is the exact cap; a T3 Chat plan raises only the default. // Runs under the user lock, bounded by the T3 Chat request timeout. const maxTunnels = - t3Chat.available && overrides.length === 0 + t3Chat.available && override === null ? Math.max(baseMaxTunnels, yield* t3ChatTunnelLimit(input.userId)) : baseMaxTunnels; if (activeTunnels >= maxTunnels) { diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index a20989cd3c025..e6db7fc8b7aa0 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -64,6 +64,7 @@ import * as HookInbox from "../hooks/HookInbox.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts"; +import * as ManagedTunnelLimits from "../environments/ManagedTunnelLimits.ts"; import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher.ts"; import * as EnvironmentPublishSignatures from "../environments/EnvironmentPublishSignatures.ts"; import * as T3ChatAccounts from "../t3chat/T3ChatAccounts.ts"; @@ -130,6 +131,7 @@ describe("managed endpoint provisioning errors", () => { Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}), Layer.mock(Devices.Devices, {}), Layer.mock(T3ChatAccounts.T3ChatAccounts, { available: false }), + Layer.mock(ManagedTunnelLimits.ManagedTunnelLimits, {}), ), ), Layer.provideMerge( @@ -241,6 +243,7 @@ describe("device listing compatibility", () => { }, }), Layer.mock(T3ChatAccounts.T3ChatAccounts, { available: false }), + Layer.mock(ManagedTunnelLimits.ManagedTunnelLimits, {}), ), ), Layer.provideMerge( @@ -322,6 +325,7 @@ describe("Cloudflare rate-limit retry hint", () => { Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}), Layer.mock(Devices.Devices, {}), Layer.mock(T3ChatAccounts.T3ChatAccounts, { available: false }), + Layer.mock(ManagedTunnelLimits.ManagedTunnelLimits, {}), ), ), Layer.provideMerge( diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 5a4d77c28d3c2..390f3c18f785f 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -74,6 +74,7 @@ import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher import * as EnvironmentConnector from "../environments/EnvironmentConnector.ts"; import * as EnvironmentLinker from "../environments/EnvironmentLinker.ts"; import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts"; +import * as ManagedTunnelLimits from "../environments/ManagedTunnelLimits.ts"; import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; import * as EnvironmentPublishSignatures from "../environments/EnvironmentPublishSignatures.ts"; import * as MobileRegistrations from "../agentActivity/MobileRegistrations.ts"; @@ -825,6 +826,7 @@ export const layerClientApi = HttpApiBuilder.group( const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider; const devices = yield* Devices.Devices; const t3Chat = yield* T3ChatAccounts.T3ChatAccounts; + const tunnelLimits = yield* ManagedTunnelLimits.ManagedTunnelLimits; return handlers .handle( "listEnvironments", @@ -996,12 +998,20 @@ export const layerClientApi = HttpApiBuilder.group( "getT3ChatLink", Effect.fn("relay.api.client.getT3ChatLink")(function* () { yield* appendRelayCredentialResponseHeaders; - if (!t3Chat.available) return { available: false, link: null }; const { userId } = yield* RelayClientPrincipal; - const link = yield* t3Chat - .getLink({ userId }) - .pipe(Effect.catch(() => relayInternalErrorResponse("upstream_unavailable"))); - return { available: true, link }; + const link = t3Chat.available + ? yield* t3Chat + .getLink({ userId }) + .pipe(Effect.catch(() => relayInternalErrorResponse("upstream_unavailable"))) + : null; + const maxTunnelsOverride = yield* tunnelLimits + .getOverride({ userId }) + .pipe(Effect.catch(() => relayInternalErrorResponse("internal_error"))); + return { + available: t3Chat.available, + link, + ...(maxTunnelsOverride === null ? {} : { maxTunnelsOverride }), + }; }, mapRelayCommonApiErrors("not_authorized")), ) .handle( diff --git a/package.json b/package.json index 1c4ffa679029c..584aed70cb1dd 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "report:renders": "node scripts/render-metrics.ts", "lint": "vp lint --report-unused-disable-directives", "knip": "knip --preprocessor ./scripts/knip-schemas.ts", - "knip:check": "knip --include files,dependencies --no-config-hints && knip --workspace apps/server --workspace apps/desktop --workspace apps/web --workspace packages/client-runtime --workspace packages/contracts --workspace packages/effect-acp --workspace packages/effect-codex-app-server --workspace packages/shared --workspace packages/ssh --workspace packages/tailscale --workspace packages/provider-core --workspace packages/provider-acp --workspace packages/provider-acp-registry --workspace packages/provider-testing --workspace packages/provider-pi --workspace packages/provider-muse --workspace packages/provider-opencode --workspace packages/provider-cursor --workspace packages/provider-fx --workspace packages/provider-grok --workspace packages/source-control-core --workspace packages/source-control-definitions --workspace packages/source-control-azure-devops --workspace packages/source-control-bitbucket --workspace packages/source-control-forgejo --workspace packages/source-control-gitcafe --workspace packages/source-control-github --workspace packages/source-control-gitlab --workspace packages/source-control-testing --exports --preprocessor ./scripts/knip-schemas.ts --no-config-hints", + "knip:check": "knip --include files,dependencies --no-config-hints && knip --workspace apps/server --workspace apps/desktop --workspace apps/web --workspace packages/client-runtime --workspace packages/contracts --workspace packages/effect-acp --workspace packages/effect-codex-app-server --workspace packages/shared --workspace packages/ssh --workspace packages/tailscale --workspace packages/provider-core --workspace packages/provider-acp --workspace packages/provider-acp-registry --workspace packages/provider-testing --workspace packages/provider-pi --workspace packages/provider-muse --workspace packages/provider-opencode --workspace packages/provider-cursor --workspace packages/provider-fx --workspace packages/provider-grok --workspace packages/secret-provider-aws --workspace packages/secret-provider-core --workspace packages/secret-provider-definitions --workspace packages/secret-provider-onepassword --workspace packages/secret-provider-openbao --workspace packages/source-control-core --workspace packages/source-control-definitions --workspace packages/source-control-azure-devops --workspace packages/source-control-bitbucket --workspace packages/source-control-forgejo --workspace packages/source-control-gitcafe --workspace packages/source-control-github --workspace packages/source-control-gitlab --workspace packages/source-control-testing --exports --preprocessor ./scripts/knip-schemas.ts --no-config-hints", "knip:production": "knip --production --preprocessor ./scripts/knip-schemas.ts", "lint:mobile": "node scripts/mobile-native-static-check.ts", "mobile:install": "node scripts/mobile-native-client.ts ensure", diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 5808641a5606a..9600ad7ad356d 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -203,6 +203,10 @@ "types": "./src/state/orchestration.ts", "default": "./src/state/orchestration.ts" }, + "./state/thread-history-merge": { + "types": "./src/state/threadHistoryMerge.ts", + "default": "./src/state/threadHistoryMerge.ts" + }, "./state/orchestration-v2-projection": { "types": "./src/state/orchestrationV2Projection.ts", "default": "./src/state/orchestrationV2Projection.ts" @@ -415,6 +419,10 @@ "types": "./src/threadReference.ts", "default": "./src/threadReference.ts" }, + "./pull-request-project-filter": { + "types": "./src/pullRequestProjectFilter.ts", + "default": "./src/pullRequestProjectFilter.ts" + }, "./thread-pull-request-compatibility": { "types": "./src/threadPullRequestCompatibility.ts", "default": "./src/threadPullRequestCompatibility.ts" diff --git a/packages/client-runtime/src/changeRequestLinks.ts b/packages/client-runtime/src/changeRequestLinks.ts index 2983647f3e26f..76f60a5953f2b 100644 --- a/packages/client-runtime/src/changeRequestLinks.ts +++ b/packages/client-runtime/src/changeRequestLinks.ts @@ -150,3 +150,21 @@ export function projectChangeRequestUrl( ...(ownRepository && identity.webUrl !== undefined ? { webUrl: identity.webUrl } : {}), }); } + +export function projectCommitUrl( + identity: RepositoryIdentity | null | undefined, + sha: string, +): string | null { + if (identity?.provider === undefined) return null; + const [host, ...path] = identity.canonicalKey.split("/"); + if (!host) return null; + return ( + sourceControlClients.get(identity.provider).commitUrl?.({ + host, + repository: path.join("/"), + sha, + remoteUrl: identity.locator.remoteUrl.trim(), + ...(identity.webUrl !== undefined ? { webUrl: identity.webUrl } : {}), + }) ?? null + ); +} diff --git a/packages/client-runtime/src/connection/driver.ts b/packages/client-runtime/src/connection/driver.ts index 0fd08c63364e7..2cdf012187efb 100644 --- a/packages/client-runtime/src/connection/driver.ts +++ b/packages/client-runtime/src/connection/driver.ts @@ -40,6 +40,8 @@ export interface EnvironmentConnectionLease { } export type RouteCheck = "answered" | "silent" | "unchecked"; +/** "unprepared" means the route answered its check but could not be prepared. */ +export type RoutePreflight = "ready" | "unprepared" | "silent" | "unchecked"; const ROUTE_CHECK_TIMEOUT_MS = 2_500; // Preferred direct routes share one head start, so a relay connection does not @@ -57,7 +59,7 @@ export class ConnectionDriver extends Context.Service< readonly preflight: ( entry: ConnectionCatalogEntry, route: ConnectionRoute, - ) => Effect.Effect; + ) => Effect.Effect; /** Asks the host behind a live session to answer a fresh request on the session's own origin. */ readonly checkHost: (prepared: PreparedConnection) => Effect.Effect; } @@ -204,13 +206,13 @@ export const make = Effect.gen(function* () { ? Effect.succeed("silent" as const) : checkRoute(entry, route) ).pipe( - Effect.flatMap((check) => - check === "answered" - ? resolver.prepare(routeEntry(entry, route)).pipe( - Effect.as(true), - Effect.orElseSucceed(() => false), - ) - : Effect.succeed(false), + Effect.filterOrElse( + (check): check is Exclude => check !== "answered", + () => + resolver.prepare(routeEntry(entry, route)).pipe( + Effect.as("ready"), + Effect.orElseSucceed((): RoutePreflight => "unprepared"), + ), ), Effect.withSpan("ConnectionDriver.preflight"), ); diff --git a/packages/client-runtime/src/connection/registry.test.ts b/packages/client-runtime/src/connection/registry.test.ts index c99e950200b23..ec8863949cef7 100644 --- a/packages/client-runtime/src/connection/registry.test.ts +++ b/packages/client-runtime/src/connection/registry.test.ts @@ -431,7 +431,11 @@ const makeHarness = Effect.fn("TestEnvironmentRegistry.makeHarness")(function* ( ), ), preflight: (_entry, route) => - checkRoute(route).pipe(Effect.map((check) => check === "answered")), + checkRoute(route).pipe( + Effect.map((check): ConnectionDriver.RoutePreflight => + check === "answered" ? "ready" : check, + ), + ), checkHost: () => Effect.succeed("unchecked"), }); diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index 347bde9cc44c8..66d151043b115 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -148,6 +148,7 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: readonly initialConfig?: (attempt: number) => Effect.Effect; readonly checkRoute?: (route: ConnectionRoute) => Effect.Effect; readonly checkHost?: (prepared: PreparedConnection) => Effect.Effect; + readonly preflight?: (route: ConnectionRoute) => Effect.Effect; readonly inboundFrames?: Effect.Effect; }) { const networkStatus = yield* SubscriptionRef.make( @@ -244,7 +245,12 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: ConnectionDriver.ConnectionDriver.of({ connect, preflight: (_entry, route) => - checkRoute(route).pipe(Effect.map((check) => check === "answered")), + options?.preflight?.(route) ?? + checkRoute(route).pipe( + Effect.map((check): ConnectionDriver.RoutePreflight => + check === "answered" ? "ready" : check, + ), + ), checkHost: (prepared) => options?.checkHost?.(prepared) ?? Effect.succeed("unchecked"), @@ -1614,6 +1620,133 @@ describe("EnvironmentSupervisor routes", () => { }), ); + it.effect("checks a dead LAN route less often until the network changes", () => + Effect.gen(function* () { + const lanReachable = yield* Ref.make(false); + const lanChecks = yield* Ref.make(0); + const harness = yield* makeHarness({ + checkRoute: (route) => + route.target._tag === "BearerConnectionTarget" + ? Ref.update(lanChecks, (count) => count + 1).pipe( + Effect.andThen(Ref.get(lanReachable)), + Effect.map((reachable) => (reachable ? "answered" : "silent")), + ) + : Effect.succeed("unchecked"), + prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), + }); + const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + yield* Ref.set(lanChecks, 0); + + // Checked at 1, 3 and 7 minutes instead of every minute. + for (let minute = 0; minute < 10; minute += 1) { + yield* TestClock.adjust("60 seconds"); + } + expect(yield* Ref.get(lanChecks)).toBe(3); + + yield* Ref.set(lanReachable, true); + yield* harness.wake("network-changed"); + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 2, + ); + expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( + "BearerConnectionTarget", + ); + }), + ); + + it.effect("keeps checking a LAN route every minute when it answers but cannot be prepared", () => + Effect.gen(function* () { + const lanPreflights = yield* Ref.make(0); + const harness = yield* makeHarness({ + checkRoute: (route) => + Effect.succeed(route.target._tag === "BearerConnectionTarget" ? "silent" : "unchecked"), + preflight: (route) => + route.target._tag === "BearerConnectionTarget" + ? Ref.update(lanPreflights, (count) => count + 1).pipe(Effect.as("unprepared" as const)) + : Effect.succeed("unchecked"), + prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), + }); + const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + for (let minute = 0; minute < 10; minute += 1) { + yield* TestClock.adjust("60 seconds"); + } + expect(yield* Ref.get(lanPreflights)).toBe(10); + }), + ); + + it.effect("ignores a dead LAN check that was running when the network changed", () => + Effect.gen(function* () { + const lanChecks = yield* Ref.make(0); + const checkStarted = yield* Deferred.make(); + const finishCheck = yield* Deferred.make(); + const woke = yield* Ref.make(false); + const wakeupHandled = yield* Deferred.make(); + const harness = yield* makeHarness({ + checkRoute: (route) => + route.target._tag === "BearerConnectionTarget" + ? Ref.updateAndGet(lanChecks, (count) => count + 1).pipe( + Effect.flatMap((check) => + check === 1 + ? Effect.succeed("silent" as const) + : check === 2 + ? Deferred.succeed(checkStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishCheck)), + Effect.as("silent" as const), + ) + : Effect.succeed("answered" as const), + ), + ) + : Effect.succeed("unchecked"), + prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), + // The wakeup probes the session only after it forgets the silent routes. + probe: () => + Ref.get(woke).pipe( + Effect.flatMap((afterWake) => + afterWake ? Deferred.succeed(wakeupHandled, undefined) : Effect.void, + ), + Effect.asVoid, + ), + }); + const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + yield* TestClock.adjust("60 seconds"); + yield* Deferred.await(checkStarted); + yield* Ref.set(woke, true); + yield* harness.wake("network-changed"); + yield* Deferred.await(wakeupHandled); + yield* Deferred.succeed(finishCheck, undefined); + + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 2, + ); + expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( + "BearerConnectionTarget", + ); + }), + ); + it.effect("moves off a dead LAN socket when the phone switches to cellular", () => Effect.gen(function* () { const lanReachable = yield* Ref.make(true); diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index ce7632df0f52b..212381921961b 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -58,7 +58,11 @@ const SILENT_SOCKET_RECHECK_EVERY = "1 second"; // instead of timing out. const REMOTE_SILENT_SOCKET_AFTER = "2500 millis"; const BACKOFF_RESET_AFTER_MS = 30_000; -const BETTER_ROUTE_CHECK_INTERVAL = "60 seconds"; +const BETTER_ROUTE_CHECK_INTERVAL_MS = 60_000; +// Each check of a dead route can wait out the full probe timeout, so a route +// that keeps failing is checked half as often each time, until the network +// changes and it may answer again. +const SILENT_ROUTE_BACKOFF_MAX_MS = 1_800_000; // A better route that answered the check but then failed to connect is not // tried again for this long, so a flaky LAN cannot bounce the connection. const BETTER_ROUTE_COOLDOWN_MS = 300_000; @@ -139,6 +143,20 @@ function retryDelayMs(failureCount: number, random: number): number { return Math.round(ceiling / 2 + (ceiling / 2) * random); } +interface SilentRoute { + readonly failures: number; + readonly until: bigint; +} + +function backOffSilentRoute(previous: SilentRoute | undefined, now: bigint): SilentRoute { + const failures = (previous?.failures ?? 0) + 1; + const delayMs = Math.min( + SILENT_ROUTE_BACKOFF_MAX_MS, + BETTER_ROUTE_CHECK_INTERVAL_MS * 2 ** failures, + ); + return { failures, until: now + BigInt(delayMs) * 1_000_000n }; +} + function annotateTarget(target: ConnectionTarget) { return Effect.annotateCurrentSpan({ "environment.id": target.environmentId, @@ -288,6 +306,16 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( const currentRoutes = Ref.get(currentEntry).pipe(Effect.map(connectionRoutes)); const preferredRouteId = yield* Ref.make(Option.none()); const routeCooldowns = yield* Ref.make>(new Map()); + // The generation keeps a check that started before a reset from recording + // its stale failures after it. + const silentRoutes = yield* Ref.make({ + generation: 0, + routes: new Map() as ReadonlyMap, + }); + const forgetSilentRoutes = Ref.update(silentRoutes, ({ generation }) => ({ + generation: generation + 1, + routes: new Map(), + })); const attemptEntry = Effect.gen(function* () { const latest = yield* Ref.get(currentEntry); @@ -338,15 +366,29 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( if (current <= 0) return; const now = yield* Clock.monotonicTimeNanos; const cooldowns = yield* Ref.get(routeCooldowns); + const silent = yield* Ref.get(silentRoutes); const better = connectionRoutes(latest) .slice(0, current) - .filter((route) => (cooldowns.get(connectionRouteId(route.target)) ?? 0n) <= now); - const passed = yield* Effect.forEach( + .filter((route) => { + const id = connectionRouteId(route.target); + return (cooldowns.get(id) ?? 0n) <= now && (silent.routes.get(id)?.until ?? 0n) <= now; + }); + const results = yield* Effect.forEach( better, (route: ConnectionRoute) => driver.preflight(latest, route), { concurrency: "unbounded" }, ); - const index = passed.indexOf(true); + yield* Ref.update(silentRoutes, (previous) => { + if (previous.generation !== silent.generation) return previous; + const routes = new Map(previous.routes); + for (const [position, route] of better.entries()) { + const id = connectionRouteId(route.target); + if (results[position] !== "silent") routes.delete(id); + else routes.set(id, backOffSilentRoute(previous.routes.get(id), now)); + } + return { generation: previous.generation, routes }; + }); + const index = results.indexOf("ready"); if (index === -1) return; const live = yield* SubscriptionRef.get(session); if (Option.isNone(live) || live.value !== lease.session) return; @@ -668,7 +710,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( ); // Ticks always run: the check skips the preferred route itself, and the // route list can grow while connected (learned routes). - yield* Stream.tick(BETTER_ROUTE_CHECK_INTERVAL).pipe( + yield* Stream.tick(Duration.millis(BETTER_ROUTE_CHECK_INTERVAL_MS)).pipe( Stream.drop(1), Stream.runForEach(() => requestBetterRouteCheck(lease)), Effect.forkScoped, @@ -1114,14 +1156,24 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( current.network === network ? [false, current] : ([true, { ...current, network }] as const), ).pipe( Effect.flatMap((changed) => - changed ? signal({ _tag: "NetworkChanged", network }) : Effect.void, + changed + ? (network === "offline" ? Effect.void : forgetSilentRoutes).pipe( + Effect.andThen(signal({ _tag: "NetworkChanged", network })), + ) + : Effect.void, ), ), ), Effect.forkScoped, ); yield* wakeups.changes.pipe( - Stream.runForEach((reason) => signal({ _tag: "Wakeup", reason })), + Stream.runForEach((reason) => + // A long suspension can hide a network change the platform never reported. + (reason === "network-changed" || reason === "application-active-long" + ? forgetSilentRoutes + : Effect.void + ).pipe(Effect.andThen(signal({ _tag: "Wakeup", reason }))), + ), Effect.forkScoped, ); yield* Queue.take(betterRouteChecks).pipe( diff --git a/packages/client-runtime/src/device/stream.test.ts b/packages/client-runtime/src/device/stream.test.ts index 737daf5158c04..09f04c52780e2 100644 --- a/packages/client-runtime/src/device/stream.test.ts +++ b/packages/client-runtime/src/device/stream.test.ts @@ -75,7 +75,7 @@ describe("native device stream transport", () => { } } vi.stubGlobal("WebSocket", FakeSocket); - const fetch = vi.fn(() => Promise.resolve(new Response("frame"))); + const fetch = vi.fn(() => Promise.resolve(Response.json({}))); vi.stubGlobal("fetch", fetch); const events = { onStatus: vi.fn(), @@ -251,6 +251,7 @@ function recoveryFixture( const fetch = vi.fn((url: string, init: RequestInit) => { const signal = init.signal!; signals.push(signal); + if (url.includes("/api?")) return Promise.resolve(Response.json({})); if (!url.includes("stream.avcc")) return Promise.resolve(new Response("prime")); return Promise.resolve( new Response( @@ -450,9 +451,11 @@ describe("shared device stream readiness and recovery", () => { }), ); client.start(); + await vi.advanceTimersByTimeAsync(0); sockets[0]!.onmessage?.({ data: sps }); client.stop(); client.start(); + await vi.advanceTimersByTimeAsync(0); resolveSupport({ supported: true }); await vi.advanceTimersByTimeAsync(0); expect(decoders).toHaveLength(0); @@ -491,3 +494,136 @@ describe("shared device stream readiness and recovery", () => { expect(vi.getTimerCount()).toBe(0); }); }); + +describe("Android WebRTC device streams", () => { + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + function setup() { + vi.useFakeTimers(); + const sockets: Socket[] = []; + class Socket { + static OPEN = 1; + readyState = 1; + binaryType = ""; + onopen: (() => void) | null = null; + onmessage: ((event: { data: string }) => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = null; + send = vi.fn(); + close = vi.fn(); + readonly url: string; + constructor(url: string) { + this.url = url; + sockets.push(this); + } + } + const videos: Video[] = []; + class Video { + muted = false; + playsInline = false; + style = { cssText: "" }; + srcObject: unknown = null; + videoWidth = 400; + videoHeight = 800; + setAttribute = vi.fn(); + remove = vi.fn(); + play = vi.fn(async () => {}); + callback: (() => void) | null = null; + requestVideoFrameCallback(callback: () => void) { + this.callback = callback; + return 0; + } + constructor() { + videos.push(this); + } + } + const peers: Peer[] = []; + class Peer { + iceGatheringState = "complete"; + connectionState = "connected"; + localDescription = { type: "offer", sdp: "offer-sdp" }; + ontrack: ((event: { streams: unknown[]; track: unknown }) => void) | null = null; + onconnectionstatechange: (() => void) | null = null; + close = vi.fn(); + createOffer = vi.fn(async () => this.localDescription); + setLocalDescription = vi.fn(async () => {}); + setRemoteDescription = vi.fn(async () => {}); + addTransceiver = vi.fn(() => ({ setCodecPreferences: vi.fn() })); + addEventListener = vi.fn(); + removeEventListener = vi.fn(); + constructor() { + peers.push(this); + } + } + vi.stubGlobal("WebSocket", Socket); + vi.stubGlobal("RTCPeerConnection", Peer); + vi.stubGlobal("RTCRtpReceiver", { getCapabilities: () => ({ codecs: [] }) }); + vi.stubGlobal("document", { createElement: () => new Video(), body: { append: vi.fn() } }); + const fetch = vi.fn((url: string, _init?: RequestInit) => + Promise.resolve( + url.includes("/api?") + ? Response.json({ serial: "emulator-5554", stream: { transport: "webrtc" } }) + : Response.json({ type: "answer", sdp: "answer-sdp" }), + ), + ); + vi.stubGlobal("fetch", fetch); + const events = { + onStatus: vi.fn(), + onScreen: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + onInputConnected: vi.fn(), + }; + const present = vi.fn(() => true); + const client = createDeviceStreamClient( + { + platform: "android", + deviceId: "session-1", + access: { + httpBase: "https://test/api/device-hub", + wsBase: "wss://test/api/device-hub", + credentials: true, + query: { hostId: "eas" }, + }, + }, + { present }, + events, + ); + return { client, events, fetch, peers, videos, sockets, present }; + } + + it("streams over WebRTC when serve-emu advertises it, with input that reconnects on its own", async () => { + const { client, events, fetch, peers, videos, sockets, present } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(0); + expect(fetch.mock.calls.map(([url]) => url)).toEqual([ + "https://test/api/device-hub/vendor/serve-emu/api?device=session-1&hostId=eas", + "https://test/api/device-hub/vendor/serve-emu/webrtc/offer?device=session-1&hostId=eas", + ]); + expect(peers[0]?.setRemoteDescription).toHaveBeenCalledWith({ + type: "answer", + sdp: "answer-sdp", + }); + peers[0]?.ontrack?.({ streams: [{}], track: {} }); + videos[0]?.callback?.(); + expect(present).toHaveBeenCalledWith(videos[0], 400, 800); + expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined); + + expect(sockets.map((socket) => socket.url)).toEqual([ + "wss://test/api/device-hub/vendor/serve-emu/ws?device=session-1&video=0&hostId=eas", + ]); + sockets[0]!.onopen?.(); + client.pressButton("back"); + expect(sockets[0]!.send).toHaveBeenCalledWith(JSON.stringify({ type: "back" })); + events.onStatus.mockClear(); + sockets[0]!.onclose?.({ code: 1001, reason: "going away" }); + await vi.advanceTimersByTimeAsync(1_000); + expect(sockets[1]?.url).toContain("video=0"); + expect(peers).toHaveLength(1); + expect(peers[0]?.close).not.toHaveBeenCalled(); + expect(events.onStatus).not.toHaveBeenCalled(); + client.stop(); + }); +}); diff --git a/packages/client-runtime/src/device/stream.ts b/packages/client-runtime/src/device/stream.ts index c4b8078597245..a8913d5a41037 100644 --- a/packages/client-runtime/src/device/stream.ts +++ b/packages/client-runtime/src/device/stream.ts @@ -836,9 +836,10 @@ export function createDeviceStreamClient( /** * serve-sim can be locked to WebRTC (EAS Simulator always is), and then - * refuses AVCC and MJPEG. An unreadable answer keeps the AVCC path. + * refuses AVCC and MJPEG; serve-emu advertises WebRTC the same way. An + * unreadable answer keeps the default path. */ - const probeIosTransport = async (): Promise => { + const probeWebRtc = async (): Promise => { const probe = new AbortController(); const timeout = setTimeout(() => probe.abort(), TRANSPORT_PROBE_TIMEOUT_MS); try { @@ -858,6 +859,9 @@ export function createDeviceStreamClient( } }; + const webRtcUrl = (action: "offer" | "close") => + httpUrl(avcc ? `/helper/${device}/webrtc/${action}` : `/webrtc/${action}?device=${device}`); + const closePeer = () => { const current = peer; if (!current) return; @@ -865,8 +869,8 @@ export function createDeviceStreamClient( current.connection.close(); current.video.srcObject = null; current.video.remove(); - // Releases serve-sim's capture session; best effort because the page may be unloading. - void fetch(httpUrl(`/helper/${device}/webrtc/close`), { + // Releases the capture session; best effort because the page may be unloading. + void fetch(webRtcUrl("close"), { method: "POST", keepalive: true, headers: { "content-type": "application/json" }, @@ -875,7 +879,7 @@ export function createDeviceStreamClient( }).catch(() => {}); }; - const readIosWebRtc = async (settings: WebRtcSettings) => { + const readWebRtc = async (settings: WebRtcSettings) => { const lifecycle = generation; const feedGeneration = ++videoGeneration; closePeer(); @@ -920,7 +924,7 @@ export function createDeviceStreamClient( if (!isCurrent()) return; closePeer(); connecting(detail); - scheduleRetry("video", () => void readIosWebRtc(settings)); + scheduleRetry("video", () => void readWebRtc(settings)); }; const presentFrame = () => { if (!isCurrent()) { @@ -988,7 +992,7 @@ export function createDeviceStreamClient( iceGatheringDeadlineMs = Math.min(iceGatheringDeadlineMs * 2, ICE_GATHERING_MAX_TIMEOUT_MS); return retry("Device stream found no network candidates yet. Retrying…"); } - const response = await fetch(httpUrl(`/helper/${device}/webrtc/offer`), { + const response = await fetch(webRtcUrl("offer"), { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ @@ -1023,8 +1027,7 @@ export function createDeviceStreamClient( const switchToWebRtc = async (failure: string, locked: boolean) => { const session = generation; if (typeof RTCPeerConnection === "undefined") return fail(failure); - const settings = - (await probeIosTransport()) ?? (locked ? { transport: "webrtc" as const } : null); + const settings = (await probeWebRtc()) ?? (locked ? { transport: "webrtc" as const } : null); if (stopped || generation !== session) return; if (!settings) return fail(failure); if (mjpeg) { @@ -1036,7 +1039,7 @@ export function createDeviceStreamClient( // MJPEG may have shown frames already: say the stream is connecting again // rather than leave a blank canvas until the first WebRTC frame. connecting("Switching to WebRTC."); - void readIosWebRtc(settings); + void readWebRtc(settings); }; const startDuoVideo = (panels: DuoPanelSinks) => { @@ -1161,18 +1164,21 @@ export function createDeviceStreamClient( ws.onerror = () => ws.close(); }; - const connectAndroid = () => { + /** `inputOnly` keeps a socket for input beside a WebRTC feed, so its reconnects leave video alone. */ + const connectAndroid = (inputOnly = false) => { if (stopped) return; - const ws = new WebSocket(wsUrl(`/ws?device=${device}&frame-meta=1`)); + const ws = new WebSocket( + wsUrl(`/ws?device=${device}&${inputOnly ? "video=0" : "frame-meta=1"}`), + ); ws.binaryType = "arraybuffer"; socket = ws; ws.onopen = () => { if (stopped || socket !== ws) return; - connecting(); + if (!inputOnly) connecting(); events.onInputConnected(true); }; ws.onmessage = (event) => { - if (stopped || socket !== ws) return; + if (stopped || socket !== ws || inputOnly) return; if (typeof event.data === "string") { // The encoder restarts at a new size when the device rotates; the // next keyframe carries a fresh SPS, so the decoder is rebuilt from it. @@ -1224,13 +1230,29 @@ export function createDeviceStreamClient( (event.code === 1006 && access.query.wsTicket) ) return handleUnauthorized(); - configuring = false; - connecting(event.reason || undefined); - scheduleRetry("input", connectAndroid); + if (!inputOnly) { + configuring = false; + connecting(event.reason || undefined); + } + // A full socket re-probes, so one failed /api read does not pin SEMU on a WebRTC host. + scheduleRetry("input", () => (inputOnly ? connectAndroid(true) : void startAndroid())); }; ws.onerror = () => ws.close(); }; + const startAndroid = async () => { + const session = generation; + const settings = await probeWebRtc(); + if (stopped || generation !== session) return; + if (settings) { + if (typeof RTCPeerConnection === "undefined") + return fail("This browser cannot receive the device's WebRTC stream."); + if (!target.videoOnly) connectAndroid(true); + void readWebRtc(settings); + } else if (useWebCodecs) connectAndroid(); + else fail("This browser cannot decode the Android stream (WebCodecs unavailable)."); + }; + const start = () => { if (!stopped) return; stopped = false; @@ -1247,11 +1269,7 @@ export function createDeviceStreamClient( if (!target.videoOnly) void connectIosInput(); if (useWebCodecs) void readIosVideo(); else fallBackToMjpeg(); - } else if (useWebCodecs) { - connectAndroid(); - } else { - fail("This browser cannot decode the Android stream (WebCodecs unavailable)."); - } + } else void startAndroid(); }; const stop = () => { diff --git a/packages/client-runtime/src/providerSkills.test.ts b/packages/client-runtime/src/providerSkills.test.ts index 7539861026165..ccc5bf6ef2807 100644 --- a/packages/client-runtime/src/providerSkills.test.ts +++ b/packages/client-runtime/src/providerSkills.test.ts @@ -199,6 +199,63 @@ describe("provider workspace snapshot refresh atom", () => { unmount(); }); + it.each(["already expired", "expires during discovery"])( + "retries an unchanged catalog that %s after the short cooldown", + async (expiry) => { + const start = Date.parse("2026-01-01T00:05:00.000Z"); + vi.useFakeTimers({ now: expiry === "expires during discovery" ? start - 1 : start }); + const stale = withSnapshot("2026-01-01T00:00:00.000Z"); + const { registry, atom, responses, run, unmount } = setup( + expiry === "expires during discovery" + ? withSnapshot("2026-01-01T00:00:00.000Z", true) + : stale, + [ + () => { + if (expiry === "expires during discovery") vi.setSystemTime(start + 1); + return stale; + }, + ], + ); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(9_999); + registry.refresh(atom); + expect(run).toHaveBeenCalledTimes(1); + responses.push(() => withSnapshot("2026-01-01T00:05:10.000Z")); + await vi.advanceTimersByTimeAsync(1); + registry.refresh(atom); + expect(run).toHaveBeenCalledTimes(2); + unmount(); + }, + ); + + it("keeps the TTL guard for a newly scanned catalog when the client clock is ahead", async () => { + vi.useFakeTimers({ now: Date.parse("2026-01-01T01:00:00.000Z") }); + const { registry, atom, run, unmount } = setup(withSnapshot("2026-01-01T00:00:00.000Z"), [ + () => withSnapshot("2026-01-01T00:01:00.000Z"), + ]); + await vi.advanceTimersByTimeAsync(10_000); + registry.refresh(atom); + expect(run).toHaveBeenCalledTimes(1); + unmount(); + }); + + it("bounds retries when discovery repeatedly returns the same expired catalog", async () => { + vi.useFakeTimers({ now: Date.parse("2026-01-01T00:05:00.000Z") }); + const stale = withSnapshot("2026-01-01T00:00:00.000Z"); + const { registry, atom, run, unmount } = setup(stale, [() => stale, () => stale, () => stale]); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(10_000); + registry.refresh(atom); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(10_000); + registry.refresh(atom); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(10_000); + registry.refresh(atom); + expect(run).toHaveBeenCalledTimes(3); + unmount(); + }); + it("does not rescan a current snapshot, and rescans once it ages past the TTL", () => { vi.useFakeTimers({ now: Date.parse("2026-01-01T00:01:00.000Z") }); const { registry, atom, run, unmount } = setup(withSnapshot("2026-01-01T00:00:00.000Z")); diff --git a/packages/client-runtime/src/providerSkills.ts b/packages/client-runtime/src/providerSkills.ts index e0e15450669f8..63ded7a47dc5c 100644 --- a/packages/client-runtime/src/providerSkills.ts +++ b/packages/client-runtime/src/providerSkills.ts @@ -144,7 +144,7 @@ interface ProviderWorkspaceSnapshotRefreshState { readonly requestedAt: number | null; /** A failed scan is not retried before this time. */ readonly retryNotBefore: number | null; - /** Scans in a row that ended without a complete snapshot. */ + /** Failed scans within the current request TTL. */ readonly failedAttempts: number; readonly hadSnapshot: boolean; } @@ -183,8 +183,10 @@ function planProviderWorkspaceSnapshotRefresh( const state = { requestedAt, retryNotBefore, - // A complete snapshot ends the run of failures. - failedAttempts: hasSnapshot ? 0 : previous.failedAttempts, + failedAttempts: + lost || (requestedAt !== null && now - requestedAt >= PROVIDER_WORKSPACE_SNAPSHOT_TTL_MS) + ? 0 + : previous.failedAttempts, hadSnapshot: hasSnapshot, }; const refresh = @@ -305,9 +307,18 @@ export function createProviderWorkspaceSnapshotRefreshAtomFamily(input: { { environmentId, input: { instanceId, cwd } }, { reportFailure: false }, ).then((result) => { + const refreshedProvider = + result._tag === "Success" ? findProvider(result.value.providers) : undefined; + const snapshot = + refreshedProvider && resolveProviderWorkspaceSnapshot(refreshedProvider, cwd); + // A new server timestamp proves progress even when the client clock is ahead. + // @effect-diagnostics-next-line globalDate:off + const completedAt = Date.now(); const refreshed = - result._tag === "Success" && - hasCompleteProviderWorkspaceSnapshot(findProvider(result.value.providers), cwd); + hasCompleteProviderWorkspaceSnapshot(refreshedProvider, cwd) && + (hasCurrentProviderWorkspaceSnapshot(refreshedProvider, cwd, completedAt) || + snapshot?.checkedAt !== + (provider && resolveProviderWorkspaceSnapshot(provider, cwd))?.checkedAt); if (!refreshed) retryLater(); }, retryLater); return plan.state; diff --git a/apps/web/src/components/pullRequest/pullRequestProjectFilter.logic.ts b/packages/client-runtime/src/pullRequestProjectFilter.ts similarity index 82% rename from apps/web/src/components/pullRequest/pullRequestProjectFilter.logic.ts rename to packages/client-runtime/src/pullRequestProjectFilter.ts index 040f3537446cf..03e47dcafab16 100644 --- a/apps/web/src/components/pullRequest/pullRequestProjectFilter.logic.ts +++ b/packages/client-runtime/src/pullRequestProjectFilter.ts @@ -1,8 +1,9 @@ -import type { EnvironmentId } from "@t3tools/contracts"; +import type { EnvironmentId, ProjectId } from "@t3tools/contracts"; -import type { AssignableProject } from "./pullRequestProjectAssignment.logic"; - -interface FilterProject extends AssignableProject { +interface FilterProject { + readonly id: ProjectId; + readonly environmentId: EnvironmentId; + readonly repositoryIdentity?: { readonly canonicalKey?: string | undefined } | null | undefined; readonly title: string; readonly workspaceRoot: string; } @@ -25,7 +26,7 @@ function distinguishTitles( export function pullRequestFilterProjects( projects: ReadonlyArray, environmentLabels: ReadonlyMap, - selectedProject?: Pick, + selectedProject?: Pick, ) { const byRepository = new Map(); for (const project of projects) { @@ -45,7 +46,7 @@ export function pullRequestFilterProjects( ); const byPath = distinguishTitles(byServer, (project) => project.workspaceRoot); const byEnvironment = distinguishTitles(byPath, (project) => project.environmentId); - return distinguishTitles(byEnvironment, (project) => project.id).toSorted((left, right) => + return distinguishTitles(byEnvironment, (project) => project.id).sort((left, right) => left.title.localeCompare(right.title), ); } diff --git a/packages/client-runtime/src/state/orchestration.ts b/packages/client-runtime/src/state/orchestration.ts index 4e6d3022b4812..5f5c2ce3020ac 100644 --- a/packages/client-runtime/src/state/orchestration.ts +++ b/packages/client-runtime/src/state/orchestration.ts @@ -107,6 +107,10 @@ export function createOrchestrationEnvironmentAtoms( label: "environment-data:orchestration:stop-workflow", tag: ORCHESTRATION_V2_WS_METHODS.stopWorkflow, }), + forceStopThread: createEnvironmentRpcCommand(runtime, { + label: "environment-data:orchestration:force-stop-thread", + tag: ORCHESTRATION_V2_WS_METHODS.forceStopThread, + }), turnItem: createEnvironmentRpcQueryAtomFamily(runtime, { label: "environment-data:orchestration:turn-item", tag: ORCHESTRATION_V2_WS_METHODS.getTurnItem, diff --git a/packages/client-runtime/src/state/review.ts b/packages/client-runtime/src/state/review.ts index 691212aedd501..6700be9af657a 100644 --- a/packages/client-runtime/src/state/review.ts +++ b/packages/client-runtime/src/state/review.ts @@ -51,6 +51,11 @@ export function createReviewEnvironmentAtoms( }); }); return { + branchCommits: createEnvironmentRpcQueryAtomFamily(runtime, { + label: "environment-data:review:branch-commits", + tag: WS_METHODS.reviewGetBranchCommits, + staleTimeMs: 5_000, + }), diffPreview, diffFilePatch: createEnvironmentQueryAtomFamily(runtime, { label: "environment-data:review:diff-file-patch", diff --git a/packages/client-runtime/src/state/runtime.test.ts b/packages/client-runtime/src/state/runtime.test.ts index 963b3a7bfdeb7..519ccd025b070 100644 --- a/packages/client-runtime/src/state/runtime.test.ts +++ b/packages/client-runtime/src/state/runtime.test.ts @@ -1212,6 +1212,35 @@ describe("runtime command runner", () => { }); describe("persisted environment queries", () => { + it.effect("keeps an in-memory cached page pending until revalidation settles", () => + Effect.scoped( + Effect.gen(function* () { + const refreshStarted = Latch.makeUnsafe(); + const finishRefresh = Latch.makeUnsafe(); + let executions = 0; + const harness = yield* makeEnvironmentQueryHarness( + Effect.suspend(() => { + if (++executions === 1) return Effect.succeed("cached"); + refreshStarted.openUnsafe(); + return finishRefresh.await.pipe(Effect.as("fresh")); + }), + { persistence: { load: () => Effect.succeedNone, save: () => Effect.void } }, + ); + const registry = yield* mountEnvironmentQuery(harness.atom); + expect(yield* readEnvironmentQuery(registry, harness.atom)).toBe("cached"); + registry.refresh(harness.atom); + yield* refreshStarted.await; + expect(registry.get(harness.atom)).toMatchObject({ + _tag: "Success", + value: "cached", + waiting: true, + }); + finishRefresh.openUnsafe(); + expect(yield* readEnvironmentQuery(registry, harness.atom)).toBe("fresh"); + }), + ), + ); + it.effect( "restores cached data in a new registry while offline, then refreshes on reconnect", () => diff --git a/packages/client-runtime/src/state/session.test.ts b/packages/client-runtime/src/state/session.test.ts new file mode 100644 index 0000000000000..740ab2229bdb4 --- /dev/null +++ b/packages/client-runtime/src/state/session.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from "@effect/vitest"; +import { EnvironmentId, type AuthSessionState } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import * as SubscriptionRef from "effect/SubscriptionRef"; +import * as TestClock from "effect/testing/TestClock"; +import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity"; + +import { + AVAILABLE_CONNECTION_STATE, + PrimaryConnectionTarget, + type PreparedConnection, +} from "../connection/model.ts"; +import * as EnvironmentRegistry from "../connection/registry.ts"; +import * as EnvironmentSupervisor from "../connection/supervisor.ts"; +import * as RpcHttp from "../rpc/http.ts"; +import type { RpcSession } from "../rpc/session.ts"; +import { createEnvironmentSessionAtoms } from "./session.ts"; + +const TARGET = new PrimaryConnectionTarget({ + environmentId: EnvironmentId.make("environment-1"), + label: "Test environment", + httpBaseUrl: "https://environment.example.test", + wsBaseUrl: "wss://environment.example.test", +}); +const PREPARED: PreparedConnection = { + environmentId: TARGET.environmentId, + label: TARGET.label, + httpBaseUrl: "https://environment.example.test", + socketUrl: "wss://environment.example.test/ws", + httpAuthorization: { _tag: "Bearer", token: "bearer-token" }, + target: TARGET, +}; +const SESSION = { + authenticated: true, + auth: { + policy: "remote-reachable", + bootstrapMethods: [], + sessionMethods: [], + sessionCookieName: "t3_session", + }, + scopes: ["orchestration:read"], +} satisfies AuthSessionState; + +const rpcSession = (): RpcSession => ({ closed: Effect.never }) as unknown as RpcSession; + +const makeHarness = Effect.fn("TestSessionState.makeHarness")(function* ( + reply: (requestNumber: number) => Promise, +) { + let requestCount = 0; + const transport = yield* SubscriptionRef.make(Option.some(rpcSession())); + const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({ + target: TARGET, + state: yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE), + session: transport, + prepared: yield* SubscriptionRef.make(Option.some(PREPARED)), + connect: Effect.void, + disconnect: Effect.void, + retryNow: Effect.void, + } satisfies EnvironmentSupervisor.EnvironmentSupervisor["Service"]); + const followStream: EnvironmentRegistry.EnvironmentRegistry["Service"]["followStream"] = ( + _environmentId, + stream, + ) => Stream.provideService(stream, EnvironmentSupervisor.EnvironmentSupervisor, supervisor); + const runtime = Atom.runtime( + Layer.mergeAll( + Layer.succeed( + EnvironmentRegistry.EnvironmentRegistry, + EnvironmentRegistry.EnvironmentRegistry.of({ + followStream, + } as unknown as EnvironmentRegistry.EnvironmentRegistry["Service"]), + ), + RpcHttp.layerRemoteHttpClient(async () => reply(++requestCount)), + // The test's clock, so the retry wait moves with `TestClock.adjust`. + Layer.succeed(Clock.Clock, yield* Clock.Clock), + ), + ); + const registry = yield* Effect.acquireRelease(Effect.sync(AtomRegistry.make), (registry) => + Effect.sync(() => registry.dispose()), + ); + const results = yield* Queue.unbounded>(); + const atom = createEnvironmentSessionAtoms(runtime).sessionStateAtom(TARGET.environmentId); + const unsubscribe = registry.subscribe(atom, (result) => Queue.offerUnsafe(results, result), { + immediate: true, + }); + yield* Effect.addFinalizer(() => Effect.sync(unsubscribe)); + const nextSettled = Effect.fn("TestSessionState.nextSettled")(function* () { + for (;;) { + const result = yield* Queue.take(results); + if (!result.waiting && result._tag !== "Initial") return result; + } + }); + return { transport, nextSettled }; +}); + +const unreachable = () => Promise.reject(new TypeError("Network request failed")); + +describe("environment session state", () => { + it.effect("fetches permissions again when the connection is re-established", () => + Effect.gen(function* () { + const harness = yield* makeHarness((requestNumber) => + requestNumber === 1 ? unreachable() : Promise.resolve(Response.json(SESSION)), + ); + expect((yield* harness.nextSettled())._tag).toBe("Failure"); + + yield* SubscriptionRef.set(harness.transport, Option.some(rpcSession())); + + expect(yield* harness.nextSettled()).toMatchObject({ _tag: "Success", value: SESSION }); + }), + ); + + it.effect("retries a failed permission check with backoff that restarts per connection", () => + Effect.gen(function* () { + const harness = yield* makeHarness((requestNumber) => + requestNumber <= 3 ? unreachable() : Promise.resolve(Response.json(SESSION)), + ); + expect((yield* harness.nextSettled())._tag).toBe("Failure"); + + yield* TestClock.adjust("2 seconds"); + expect((yield* harness.nextSettled())._tag).toBe("Failure"); + + yield* SubscriptionRef.set(harness.transport, Option.some(rpcSession())); + expect((yield* harness.nextSettled())._tag).toBe("Failure"); + + yield* TestClock.adjust("2 seconds"); + expect(yield* harness.nextSettled()).toMatchObject({ _tag: "Success", value: SESSION }); + }), + ); +}); diff --git a/packages/client-runtime/src/state/session.ts b/packages/client-runtime/src/state/session.ts index efdf0b37c0a09..90afa29f9d05d 100644 --- a/packages/client-runtime/src/state/session.ts +++ b/packages/client-runtime/src/state/session.ts @@ -14,6 +14,7 @@ import type { PreparedConnection } from "../connection/model.ts"; import * as EnvironmentSupervisor from "../connection/supervisor.ts"; import * as ManagedRelay from "../relay/managedRelay.ts"; import type { RemoteEnvironmentRequestError } from "../rpc/http.ts"; +import type { RpcSession } from "../rpc/session.ts"; import { safeErrorLogAttributes } from "../errors/safeLog.ts"; import { executeAuthenticatedEnvironmentHttpRequest } from "./environmentHttpAuth.ts"; import { followStreamInEnvironment } from "./environmentStreams.ts"; @@ -39,6 +40,8 @@ function initialConfigOption( const DEFAULT_SESSION_STATE_TIMEOUT_MS = 6_000; const MAX_SESSION_STATE_TIMEOUT_MS = 60_000; +const SESSION_STATE_RETRY_BASE_MS = 2_000; +const SESSION_STATE_RETRY_MAX_MS = 60_000; export const fetchEnvironmentSessionState = Effect.fn( "clientRuntime.state.fetchEnvironmentSessionState", @@ -136,13 +139,38 @@ function makeEnvironmentSessionAtoms( ).pipe(Atom.withLabel(`environment-prepared-connection:${environmentId}`)), ); - const sessionStateAtom = Atom.family((environmentId: EnvironmentId) => - runtime + const transportSessionAtom = Atom.family((environmentId: EnvironmentId) => + runtime.atom( + followStreamInEnvironment( + environmentId, + Stream.unwrap( + EnvironmentSupervisor.EnvironmentSupervisor.pipe( + Effect.map((supervisor) => SubscriptionRef.changes(supervisor.session)), + ), + ), + ), + { initialValue: Option.none() }, + ), + ); + + const sessionStateAtom = Atom.family((environmentId: EnvironmentId) => { + let consecutiveFailures = 0; + let backoffConnection: RpcSession | null = null; + return runtime .atom((get) => { const prepared = Option.getOrNull(get(preparedConnectionValueAtom(environmentId))); - if (prepared === null) { + // Each established connection fetches anew, and a failure retries while it stays up, + // so a fetch that failed mid-reconnect cannot outlive the socket that recovered. + const connection = AsyncResult.getOrElse(get(transportSessionAtom(environmentId)), () => + Option.none(), + ); + if (prepared === null || Option.isNone(connection)) { return Effect.never; } + if (connection.value !== backoffConnection) { + backoffConnection = connection.value; + consecutiveFailures = 0; + } return Effect.gen(function* () { const signer = yield* Effect.serviceOption(ManagedRelay.ManagedRelayDpopSigner); const remoteAuthorization = yield* Effect.serviceOption( @@ -155,14 +183,31 @@ function makeEnvironmentSessionAtoms( signer, remoteAuthorization, }).pipe(Effect.provideService(HttpClient.HttpClient, client.value)); - }); + }).pipe( + Effect.tap(() => + Effect.sync(() => { + consecutiveFailures = 0; + }), + ), + Effect.tapError(() => { + const delayMs = Math.min( + SESSION_STATE_RETRY_BASE_MS * 2 ** consecutiveFailures, + SESSION_STATE_RETRY_MAX_MS, + ); + consecutiveFailures += 1; + return Effect.sleep(delayMs).pipe( + Effect.andThen(Effect.sync(() => get.refreshSelf())), + Effect.forkScoped, + ); + }), + ); }) .pipe( Atom.swr({ staleTime: 30_000, revalidateOnMount: true }), Atom.setIdleTTL(5 * 60_000), Atom.withLabel(`environment-session-state:${environmentId}`), - ), - ); + ); + }); const sessionStateValueAtom = Atom.family((environmentId: EnvironmentId) => Atom.make( diff --git a/packages/client-runtime/src/state/threadCheckpoints.ts b/packages/client-runtime/src/state/threadCheckpoints.ts index 4501007e20053..ee2fe45d9c756 100644 --- a/packages/client-runtime/src/state/threadCheckpoints.ts +++ b/packages/client-runtime/src/state/threadCheckpoints.ts @@ -3,6 +3,7 @@ import type { CheckpointRef, CheckpointScopeId, MessageId, + OrchestrationV2CheckpointFileTotals, OrchestrationV2ThreadProjection, RunId, } from "@t3tools/contracts"; @@ -15,12 +16,14 @@ export interface ThreadCheckpointSummary { readonly checkpointTurnCount: number; readonly checkpointRef: CheckpointRef; readonly status: "ready" | "missing" | "error" | "stale"; + /** A prefix of the changed files when `filesTotal` is set; see `checkpointFileTotals`. */ readonly files: ReadonlyArray<{ readonly path: string; readonly kind: string; readonly additions: number; readonly deletions: number; }>; + readonly filesTotal?: OrchestrationV2CheckpointFileTotals; readonly assistantMessageId: MessageId | null; readonly completedAt: string; } @@ -43,9 +46,23 @@ export function deriveThreadCheckpointSummaries( checkpointRef: checkpoint.ref, status: checkpoint.status, files: checkpoint.files, + ...(checkpoint.filesTotal === undefined ? {} : { filesTotal: checkpoint.filesTotal }), assistantMessageId, completedAt: DateTime.formatIso(checkpoint.capturedAt), }, ]; }); } + +export function checkpointFileTotals( + checkpoint: Pick, +): OrchestrationV2CheckpointFileTotals { + if (checkpoint.filesTotal !== undefined) return checkpoint.filesTotal; + let additions = 0; + let deletions = 0; + for (const file of checkpoint.files) { + additions += file.additions; + deletions += file.deletions; + } + return { count: checkpoint.files.length, additions, deletions }; +} diff --git a/packages/client-runtime/src/state/threadDetail.test.ts b/packages/client-runtime/src/state/threadDetail.test.ts index 8fc01839183fe..3e66202d0bb4b 100644 --- a/packages/client-runtime/src/state/threadDetail.test.ts +++ b/packages/client-runtime/src/state/threadDetail.test.ts @@ -197,6 +197,42 @@ describe("quietThreadAtom", () => { }); }); +describe("timelineVisibleTurnItemsAtom", () => { + const ref = { environmentId: EnvironmentId.make("environment-1"), threadId: v2ThreadId }; + + it("holds through streamed assistant text while the message reads its own text", () => { + const stateAtom = Atom.make( + AsyncResult.success({ + ...EMPTY_ENVIRONMENT_THREAD_STATE, + data: Option.some(withItem(assistant)), + }), + ); + const details = createEnvironmentThreadDetailAtoms(() => stateAtom); + const timelineAtom = details.timelineVisibleTurnItemsAtom(ref); + const textAtom = details.assistantMessageTextAtom(ref, assistant.id); + const registry = AtomRegistry.make(); + registry.mount(timelineAtom); + registry.mount(textAtom); + const push = (item: OrchestrationV2TurnItem) => + registry.set( + stateAtom, + AsyncResult.success({ + ...EMPTY_ENVIRONMENT_THREAD_STATE, + data: Option.some(withItem(item, later)), + }), + ); + const first = registry.get(timelineAtom); + + push({ ...assistant, text: "Hello", updatedAt: later }); + expect(registry.get(timelineAtom)).toBe(first); + expect(registry.get(textAtom)).toBe("Hello"); + + push({ ...assistant, text: "Hello", streaming: false, status: "completed" }); + expect(registry.get(timelineAtom)[0]?.item).toMatchObject({ text: "Hello", streaming: false }); + registry.dispose(); + }); +}); + it("only compares replaced entries in a streamed reducer delta", () => { const size = 100; let slotReads = 0; diff --git a/packages/client-runtime/src/state/threadDetail.ts b/packages/client-runtime/src/state/threadDetail.ts index e1f30998caefb..eeeda6a2f348f 100644 --- a/packages/client-runtime/src/state/threadDetail.ts +++ b/packages/client-runtime/src/state/threadDetail.ts @@ -14,6 +14,7 @@ import { type ThreadQueueWorkflowState, } from "./threadWorkflows.ts"; import { deriveThreadTurnSubagents, type ThreadTurnSubagents } from "./threadSubagents.ts"; +import { presentRunStop, type RunStopPresentation } from "./threadExecution.ts"; import { deriveThreadCheckpointSummaries, type ThreadCheckpointSummary, @@ -147,6 +148,26 @@ function sameVisibleTurnItemStructure(before: VisibleTurnItem, after: VisibleTur ); } +function isStreamingAssistantMessage(visibleItem: VisibleTurnItem): boolean { + return visibleItem.item.type === "assistant_message" && visibleItem.item.streaming; +} + +/** Whether `next` only streams more text into assistant messages that were already streaming. */ +function sameApartFromStreamedAssistantText( + previous: OrchestrationV2ThreadProjection["visibleTurnItems"], + next: OrchestrationV2ThreadProjection["visibleTurnItems"], +): boolean { + return sameProjectionArray( + previous, + next, + (before, after) => + before === after || + (isStreamingAssistantMessage(before) && + isStreamingAssistantMessage(after) && + sameVisibleTurnItemStructure(before, after)), + ); +} + /** * Whether `previous` and `next` have the same ThreadStructure: they differ * only by text streamed into existing assistant and reasoning items or by a @@ -266,6 +287,32 @@ export function createEnvironmentThreadDetailAtoms( ); }); + // Holds while only streamed assistant text moves, so the timeline re-renders on structure; + // each streaming message reads its text through assistantMessageTextAtom. + const timelineVisibleTurnItemsAtomFamily = Atom.family((key: string) => { + let previous = EMPTY_VISIBLE_TURN_ITEMS; + return Atom.make((get) => { + const next = get(visibleTurnItemsAtomFamily(key)); + if (!sameApartFromStreamedAssistantText(previous, next)) previous = next; + return previous; + }).pipe( + Atom.setIdleTTL(0), + Atom.withLabel(`environment-thread-timeline-visible-turn-items:${key}`), + ); + }); + + const assistantMessageTextAtomFamily = Atom.family((key: string) => { + const separator = key.lastIndexOf("\u0000"); + const thread = key.slice(0, separator); + const itemId = key.slice(separator + 1); + return Atom.make((get) => { + const item = get(visibleTurnItemsAtomFamily(thread)).find( + (visibleItem) => visibleItem.item.id === itemId, + )?.item; + return item?.type === "assistant_message" ? item.text : null; + }).pipe(Atom.setIdleTTL(0), Atom.withLabel(`environment-thread-assistant-message-text:${key}`)); + }); + const queueWorkflowAtomFamily = Atom.family((key: string) => { let previous: Pick< OrchestrationV2ThreadProjection, @@ -294,6 +341,32 @@ export function createEnvironmentThreadDetailAtoms( return value; }).pipe(Atom.setIdleTTL(0), Atom.withLabel(`environment-thread-queue:${key}`)); }); + const runStopAtomFamily = Atom.family((key: string) => { + let previous: Pick | null = null; + let stop: RunStopPresentation | null = null; + return Atom.make((get) => { + const projection = Option.getOrNull(get(threadStateValueAtomFamily(key)).data); + if (projection === null) { + previous = null; + stop = null; + } else if ( + projection.runs !== previous?.runs || + projection.turnItems !== previous?.turnItems + ) { + const { runs, turnItems } = projection; + previous = { runs, turnItems }; + const next = presentRunStop(previous); + if ( + next?.state !== stop?.state || + next?.runId !== stop?.runId || + next?.requestedAt !== stop?.requestedAt || + next?.background !== stop?.background + ) + stop = next; + } + return stop; + }).pipe(Atom.setIdleTTL(0), Atom.withLabel(`environment-thread-run-stop:${key}`)); + }); const queuedCountAtomFamily = Atom.family((key: string) => { let previous: Pick | null = null; let count = 0; @@ -435,6 +508,7 @@ export function createEnvironmentThreadDetailAtoms( pendingRequestsAtom: (ref: ScopedThreadRef) => pendingRequestsAtomFamily(threadKey(ref)), queueWorkflowAtom: (ref: ScopedThreadRef) => queueWorkflowAtomFamily(threadKey(ref)), queuedCountAtom: (ref: ScopedThreadRef) => queuedCountAtomFamily(threadKey(ref)), + runStopAtom: (ref: ScopedThreadRef) => runStopAtomFamily(threadKey(ref)), turnSubagentsAtom: (ref: ScopedThreadRef) => turnSubagentsAtomFamily(threadKey(ref)), checkpointSummariesAtom: (ref: ScopedThreadRef) => checkpointSummariesAtomFamily(threadKey(ref)), @@ -444,6 +518,10 @@ export function createEnvironmentThreadDetailAtoms( visibleTurnItemsAtom: (ref: ScopedThreadRef) => visibleTurnItemsAtomFamily(threadKey(ref)), quietVisibleTurnItemsAtom: (ref: ScopedThreadRef) => quietVisibleTurnItemsAtomFamily(threadKey(ref)), + timelineVisibleTurnItemsAtom: (ref: ScopedThreadRef) => + timelineVisibleTurnItemsAtomFamily(threadKey(ref)), + assistantMessageTextAtom: (ref: ScopedThreadRef, itemId: string) => + assistantMessageTextAtomFamily(`${threadKey(ref)}\u0000${itemId}`), statusAtom: (ref: ScopedThreadRef) => statusAtomFamily(threadKey(ref)), sequenceAtom: (ref: ScopedThreadRef) => sequenceAtomFamily(threadKey(ref)), errorAtom: (ref: ScopedThreadRef) => errorAtomFamily(threadKey(ref)), diff --git a/packages/client-runtime/src/state/threadExecution.test.ts b/packages/client-runtime/src/state/threadExecution.test.ts index 8ef37f959cd40..3f8ef2a7f9d31 100644 --- a/packages/client-runtime/src/state/threadExecution.test.ts +++ b/packages/client-runtime/src/state/threadExecution.test.ts @@ -25,6 +25,7 @@ import { deriveRunlessWorkStartedAt, deriveThreadActivityRun, deriveThreadRuntime, + presentRunStop, threadRuntimeHasInterruptibleRun, } from "./threadExecution.ts"; @@ -259,6 +260,74 @@ describe("thread execution presentation", () => { }); }); +describe("presentRunStop", () => { + const signal = (type: "run_interrupt_request" | "run_interrupt_result", runId: RunId) => ({ + id: TurnItemId.make(`${type}:${runId}`), + threadId: v2Projection.thread.id, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: type === "run_interrupt_result" ? ("interrupted" as const) : ("completed" as const), + title: "Interrupt", + startedAt: now, + completedAt: now, + updatedAt: now, + type, + message: "Stopped by user", + }); + const running = run("run-stopping", 1, "running"); + const request = signal("run_interrupt_request", running.id); + + it.each([ + { name: "no Stop", runs: [running], turnItems: [], state: null }, + { + name: "a Stop the agent is winding down", + runs: [running], + turnItems: [request], + state: "stopping", + }, + { + name: "a Stop the provider never confirmed", + runs: [running], + turnItems: [{ ...request, unresponsiveAt: now }], + state: "unresponsive", + }, + { + name: "a Stop that ended the run", + runs: [running], + turnItems: [request, signal("run_interrupt_result", running.id)], + state: null, + }, + { + name: "a settled run's background work still stopping", + runs: [{ ...running, status: "completed" as const, completedAt: now }], + turnItems: [request], + state: null, + }, + { + name: "a settled run whose background work ignored Stop", + runs: [{ ...running, status: "completed" as const, completedAt: now }], + turnItems: [{ ...request, unresponsiveAt: now }], + state: "unresponsive", + }, + { + name: "a settled run whose background work ignored Stop, with a queued follow-up", + runs: [ + { ...running, status: "completed" as const, completedAt: now }, + run("run-queued", 2, "queued"), + ], + turnItems: [{ ...request, unresponsiveAt: now }], + state: "unresponsive", + }, + ])("presents $name", ({ runs, turnItems, state }) => { + expect(presentRunStop({ runs, turnItems })?.state ?? null).toBe(state); + }); +}); + describe("deriveRunlessWorkStartedAt", () => { const later = DateTime.makeUnsafe("2026-07-28T10:05:00.000Z"); const rootTurn = ( diff --git a/packages/client-runtime/src/state/threadExecution.ts b/packages/client-runtime/src/state/threadExecution.ts index ba87930bebbd5..52d07babfbb61 100644 --- a/packages/client-runtime/src/state/threadExecution.ts +++ b/packages/client-runtime/src/state/threadExecution.ts @@ -294,6 +294,55 @@ const BACKGROUND_WORK_KINDS: Record< background_task: { order: 3, singular: "background task", plural: "background tasks" }, }; +export interface RunStopPresentation { + /** `unresponsive`: the provider did not confirm the stop in time, so Force stop is offered. */ + readonly state: "stopping" | "unresponsive"; + readonly runId: ThreadStructure["runs"][number]["id"]; + readonly requestedAt: string; + /** The run already ended; the Stop targeted its background work. */ + readonly background: boolean; +} + +/** + * A Stop the provider has not finished yet. It stays `stopping` while the agent winds down, + * since some providers confirm the request before the turn actually ends. On a run that + * already ended, only an unanswered Stop of its background work counts. + */ +export function presentRunStop( + projection: Pick, +): RunStopPresentation | null { + let active: ThreadStructure["runs"][number] | null = null; + let latest: ThreadStructure["runs"][number] | null = null; + for (const candidate of projection.runs) { + // Queued follow-ups never ran, so they hold no Stop of their own. + if (candidate.status !== "queued" && candidate.ordinal > (latest?.ordinal ?? -1)) + latest = candidate; + if ( + INTERRUPTIBLE_RUN_STATUSES.has(candidate.status) && + candidate.ordinal > (active?.ordinal ?? -1) + ) + active = candidate; + } + const run = active ?? latest; + if (run === null) return null; + let request: + | Extract + | undefined; + for (const item of projection.turnItems) { + if (item.runId !== run.id) continue; + if (item.type === "run_interrupt_result") return null; + if (item.type === "run_interrupt_request") request = item; + } + if (request === undefined || (active === null && request.unresponsiveAt === undefined)) + return null; + return { + state: request.unresponsiveAt === undefined ? "stopping" : "unresponsive", + runId: run.id, + requestedAt: DateTime.formatIso(request.startedAt ?? request.updatedAt), + background: active === null, + }; +} + export interface PendingBackgroundWorkItem { readonly taskId: string; readonly kind: BackgroundWorkKind; diff --git a/packages/client-runtime/src/state/threadVisits.test.ts b/packages/client-runtime/src/state/threadVisits.test.ts index a99417534ea7a..adcac7139e82a 100644 --- a/packages/client-runtime/src/state/threadVisits.test.ts +++ b/packages/client-runtime/src/state/threadVisits.test.ts @@ -111,6 +111,24 @@ it("keeps a mark-unread on the open thread until new activity lands", () => { expect(t.visits.slice(1)).toEqual([{ threadId: t.ref.threadId, visitedAt: at(30) }]); }); +it("keeps mark unread sticky after an expired wake is retired", () => { + using t = setup("thread-unread-retired-wake", { + updatedAt: at(10), + lastVisitedAt: at(0), + latestRun: { completedAt: at(9) }, + wokeAt: at(15), + }); + t.mount(); + expect(t.visits).toEqual([{ threadId: t.ref.threadId, visitedAt: at(15), opened: true }]); + t.update({ lastVisitedAt: at(15) }); + t.update({ lastVisitedAt: at(8), wokeAt: null }); + vi.advanceTimersByTime(VISIT_DISPATCH_THROTTLE_MS); + expect(t.visits).toHaveLength(1); + + t.update({ updatedAt: at(30) }); + expect(t.visits.slice(1)).toEqual([{ threadId: t.ref.threadId, visitedAt: at(30) }]); +}); + it("waits for the shell and records local visits for servers without tracking", () => { using t = setup("thread-local", null); t.mount(); diff --git a/packages/client-runtime/src/state/threadVisits.ts b/packages/client-runtime/src/state/threadVisits.ts index 08d05909a2dad..c08f47ed01436 100644 --- a/packages/client-runtime/src/state/threadVisits.ts +++ b/packages/client-runtime/src/state/threadVisits.ts @@ -115,7 +115,8 @@ export function createThreadVisitTracker(options: ThreadVisitTrackerOptions) { } const lastVisitedAtMs = shell.lastVisitedAt ? Date.parse(shell.lastVisitedAt) : NaN; if (!Number.isNaN(lastVisitedAtMs) && lastVisitedAtMs >= updatedAtMs) return; - if (lastDispatchedUpdatedAt === visitedAt) return; + if (lastDispatchedUpdatedAt !== null && Date.parse(lastDispatchedUpdatedAt) >= updatedAtMs) + return; const completedAtMs = shell.latestRun?.completedAt ? Date.parse(shell.latestRun.completedAt) diff --git a/packages/client-runtime/src/userMessage.test.ts b/packages/client-runtime/src/userMessage.test.ts index 0447dc54d1b26..207939cf2f39d 100644 --- a/packages/client-runtime/src/userMessage.test.ts +++ b/packages/client-runtime/src/userMessage.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vite-plus/test"; import { GuestMessageId } from "@t3tools/contracts"; +import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences"; import { resolveUserMessagePresentation } from "./userMessage.ts"; @@ -21,6 +22,19 @@ describe("resolveUserMessagePresentation", () => { } }); + it("hides the prompt that stands in for an attachment-only message", () => { + for (const text of [ + ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, + `Ultrathink:\n${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`, + ]) { + expect(resolveUserMessagePresentation({ role: "user", text })).toEqual({ + text: "", + attribution: null, + scheduledTaskId: undefined, + }); + } + }); + it("shows a guest's own words and name, not the attributed text the agent reads", () => { expect( resolveUserMessagePresentation({ diff --git a/packages/client-runtime/src/userMessage.ts b/packages/client-runtime/src/userMessage.ts index d5d7c5ac08956..8a80f31b155d0 100644 --- a/packages/client-runtime/src/userMessage.ts +++ b/packages/client-runtime/src/userMessage.ts @@ -5,6 +5,7 @@ import { type OrchestrationV2IdleCompaction, ScheduledTaskId, } from "@t3tools/contracts"; +import { isAttachmentOnlyBootstrapPrompt } from "@t3tools/shared/composerContextReferences"; /** The working row while T3 Code's own compaction runs. */ export const IDLE_COMPACTING_LABELS = { @@ -68,7 +69,8 @@ export function resolveUserMessagePresentation(message: { }; } return { - text: message.text, + // The agent needs words to start a turn; people sent only the attachments. + text: isAttachmentOnlyBootstrapPrompt(message.text) ? "" : message.text, attribution: message.createdBy !== "agent" ? null diff --git a/packages/contracts/src/clientRpcPermissions.ts b/packages/contracts/src/clientRpcPermissions.ts index 4f376d54052ef..be656ebffae66 100644 --- a/packages/contracts/src/clientRpcPermissions.ts +++ b/packages/contracts/src/clientRpcPermissions.ts @@ -61,6 +61,8 @@ export const CLIENT_GUARDED_RPC_SCOPES = { [WS_METHODS.scheduledTasksRunNow]: AuthOrchestrationOperateScope, [WS_METHODS.scheduledTasksRotateWebhookToken]: AuthOrchestrationOperateScope, [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: AuthOrchestrationOperateScope, + // Force stop kills agent processes, as Stop on a terminal does. + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: AuthOrchestrationOperateScope, // A guest link hands thread access to someone outside, like a pairing link. [WS_METHODS.guestShareCreate]: AuthAccessWriteScope, [WS_METHODS.peerLinksLink]: AuthAccessWriteScope, diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts index d5698c007ec00..72b0cb5567b4b 100644 --- a/packages/contracts/src/device.ts +++ b/packages/contracts/src/device.ts @@ -3,7 +3,7 @@ import * as Schema from "effect/Schema"; import * as SchemaTransformation from "effect/SchemaTransformation"; import { ThreadId, TrimmedNonEmptyString, TrimmedString } from "./baseSchemas.ts"; -import { SettingsSecretValue } from "./secretProvider.ts"; +import { SettingsSecretValue } from "./settingsSecretValue.ts"; import { secretSetting } from "./settingsSecret.ts"; export const DevicePlatform = Schema.Literals(["ios", "android"]); diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 75a546bed875f..a10c6c2a6bce7 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -99,6 +99,8 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ /** Hide the switch when absent; older servers silently drop per-project provider patches. */ projectProviderInstanceOverrides: Schema.optionalKey(Schema.Boolean), threadSnooze: Schema.optionalKey(Schema.Boolean), + /** Serves `orchestration.forceStopThread`; hide Force stop where it is absent. */ + threadForceStop: Schema.optionalKey(Schema.Boolean), /** Every protocol-2 server sends this; older mobile builds still read it. */ environmentThemes: Schema.optionalKey(Schema.Boolean), usageLimitSources: Schema.optionalKey(Schema.Boolean), diff --git a/packages/contracts/src/guestShare.ts b/packages/contracts/src/guestShare.ts index fd991fe1e4c3b..715a837daeae3 100644 --- a/packages/contracts/src/guestShare.ts +++ b/packages/contracts/src/guestShare.ts @@ -129,6 +129,11 @@ export const guestSessionCookieName = (shareId: string) => `t3_guest_${shareId}` export const GUEST_RENDER_ROUTE = `${GUEST_ROUTE_PREFIX}/render`; /** One item of the shared thread, as an expanded row loads it: `?share=&item=`. */ export const GUEST_ITEM_ROUTE = `${GUEST_ROUTE_PREFIX}/item`; +/** + * Older turns of the shared thread, as the owner's "Load earlier turns" pages them: + * `?share=&cursor=`. + */ +export const GUEST_HISTORY_ROUTE = `${GUEST_ROUTE_PREFIX}/history`; /** Server-sent output frames of one command in the shared thread: `?share=&item=`. */ export const GUEST_COMMAND_OUTPUT_ROUTE = `${GUEST_ROUTE_PREFIX}/command-output`; /** diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 197a16e125ea6..f00976d243b37 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -15,9 +15,6 @@ export * from "./remoteAccess.ts"; export * from "./ipc.ts"; export * from "./terminal.ts"; export * from "./provider.ts"; -export * from "./onePassword.ts"; -export * from "./secretProvider.ts"; -export * from "./secretProviderId.ts"; export * from "./providerInstance.ts"; export * from "./providerSetup.ts"; export * from "./providerRuntime.ts"; @@ -70,4 +67,6 @@ export * from "./resourceTelemetry.ts"; export * from "./rpc.ts"; export * from "./worktreeSetup.ts"; export * from "./secretRequest.ts"; +export * from "./secretReference.ts"; +export * from "./settingsSecretValue.ts"; export * from "./clientRpcPermissions.ts"; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index f460619f11355..f14452fec5a92 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1018,6 +1018,7 @@ export interface DesktopBridge { getClientPlatform: () => string; setNotificationBadge: (badge: { count: number; image: string | null }) => Promise; onNotificationBadgeClear: (listener: () => void) => () => void; + onTrackpadScrollBegin: (listener: () => void) => () => void; onTrackpadScrollEnd: (listener: () => void) => () => void; /** * Renderer `window.focus()` cannot raise an Electron window, and on Wayland diff --git a/packages/contracts/src/onePassword.test.ts b/packages/contracts/src/onePassword.test.ts deleted file mode 100644 index f1f8cff9e5501..0000000000000 --- a/packages/contracts/src/onePassword.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { describe, expect, it } from "vite-plus/test"; -import * as Schema from "effect/Schema"; - -import { OnePasswordSecretSource } from "./onePassword.ts"; - -const decodeSource = Schema.decodeUnknownSync(OnePasswordSecretSource); - -describe("OnePasswordSecretSource", () => { - const onePassword = (reference: string, account = "my.1password.com") => ({ - kind: "1password", - reference, - account, - }); - - it.each([ - "op://Private/claude-code/credential", - "op://Home Lab/Claude Code/API Key", - "op://Private/claude-code/login section/password", - "op://Private/github/one-time password?attribute=otp", - ])("accepts the reference %s", (reference) => { - expect(decodeSource(onePassword(reference))).toEqual(onePassword(reference)); - }); - - it.each([ - ["leading whitespace", " op://Private/item/field"], - ["trailing newline", "op://Private/item/field\n"], - ["control character", "op://Private/it\u0007em/field"], - ["missing scheme", "Private/item/field"], - ["nothing after the scheme", "op://"], - ["template delimiter", "op://Private/item}}/field"], - ])("rejects a reference with %s", (_label, reference) => { - expect(() => decodeSource(onePassword(reference))).toThrow(); - }); - - it.each(["my", "my.1password.com", "team-acme.1password.eu", "ABCDEFGHIJKLMNOPQRSTUVWXYZ"])( - "accepts the account %s", - (account) => { - expect(decodeSource(onePassword("op://Private/item/field", account))).toMatchObject({ - account, - }); - }, - ); - - it.each([ - ["empty", ""], - ["a space", "my account"], - ["a leading dash", "--help"], - ["surrounding whitespace", " my "], - ])("rejects an account with %s", (_label, account) => { - expect(() => decodeSource(onePassword("op://Private/item/field", account))).toThrow(); - }); - - it("rejects a source without an account", () => { - expect(() => - decodeSource({ kind: "1password", reference: "op://Private/item/field" }), - ).toThrow(); - }); -}); diff --git a/packages/contracts/src/onePassword.ts b/packages/contracts/src/onePassword.ts deleted file mode 100644 index c3e957b93fdc5..0000000000000 --- a/packages/contracts/src/onePassword.ts +++ /dev/null @@ -1,76 +0,0 @@ -import * as Schema from "effect/Schema"; - -const ONE_PASSWORD_SECRET_REFERENCE_PREFIX = "op://"; -const ONE_PASSWORD_SECRET_REFERENCE_MAX_CHARS = 1024; -const ONE_PASSWORD_ACCOUNT_MAX_CHARS = 253; -const ONE_PASSWORD_ACCOUNT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; - -function hasControlCharacter(value: string): boolean { - for (let index = 0; index < value.length; index++) { - const code = value.charCodeAt(index); - if (code < 0x20 || code === 0x7f) return true; - } - return false; -} - -function onePasswordSecretReferenceIssue(value: string): string | undefined { - if (value !== value.trim()) { - return "1Password secret reference must not start or end with whitespace."; - } - if (hasControlCharacter(value)) { - return "1Password secret reference must not contain control characters."; - } - if (!value.startsWith(ONE_PASSWORD_SECRET_REFERENCE_PREFIX)) { - return "1Password secret reference must start with op://."; - } - // The resolver embeds references in an `op inject` template, where `{{` and - // `}}` delimit a reference. - if (value.includes("{{") || value.includes("}}")) { - return "1Password secret reference must not contain {{ or }}."; - } - if (value.length === ONE_PASSWORD_SECRET_REFERENCE_PREFIX.length) { - return "1Password secret reference must name a vault, item, and field after op://."; - } - return undefined; -} - -/** - * A 1Password secret reference such as `op://vault/item/field`. Only what - * T3 Code itself depends on is checked here; `op` validates the rest and - * reports which part it could not resolve. - */ -export const OnePasswordSecretReference = Schema.String.check( - Schema.isMaxLength(ONE_PASSWORD_SECRET_REFERENCE_MAX_CHARS), - Schema.makeFilter((value: string) => onePasswordSecretReferenceIssue(value) ?? true), -).pipe(Schema.brand("OnePasswordSecretReference")); -export type OnePasswordSecretReference = typeof OnePasswordSecretReference.Type; - -/** - * The 1Password account a reference is read from, in any form `op --account` - * accepts: account shorthand, sign-in address (`my.1password.com`), account ID, - * or user ID. It is passed to the CLI as an argument, so it can never start - * with `-`. - */ -export const OnePasswordAccount = Schema.String.check( - Schema.isMaxLength(ONE_PASSWORD_ACCOUNT_MAX_CHARS), - Schema.makeFilter((value: string) => - ONE_PASSWORD_ACCOUNT_PATTERN.test(value) - ? true - : "1Password account must be a shorthand, sign-in address, or ID with no spaces.", - ), -).pipe(Schema.brand("OnePasswordAccount")); -export type OnePasswordAccount = typeof OnePasswordAccount.Type; - -export const OnePasswordSecretSource = Schema.Struct({ - kind: Schema.Literal("1password"), - reference: OnePasswordSecretReference, - account: OnePasswordAccount, -}); -export type OnePasswordSecretSource = typeof OnePasswordSecretSource.Type; - -/** A 1Password account signed in on the server, as `op account list` reports it. */ -export const OnePasswordAccountSummary = Schema.Struct({ - account: OnePasswordAccount, - email: Schema.String, -}); -export type OnePasswordAccountSummary = typeof OnePasswordAccountSummary.Type; diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index c75c687ad94cd..a84ff27e73c75 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1350,6 +1350,14 @@ export const OrchestrationV2CheckpointFileSummary = Schema.Struct({ }); export type OrchestrationV2CheckpointFileSummary = typeof OrchestrationV2CheckpointFileSummary.Type; +/** Sent only when the wire carries a prefix of `files`; the totals cover the full list. */ +export const OrchestrationV2CheckpointFileTotals = Schema.Struct({ + count: NonNegativeInt, + additions: NonNegativeInt, + deletions: NonNegativeInt, +}); +export type OrchestrationV2CheckpointFileTotals = typeof OrchestrationV2CheckpointFileTotals.Type; + export const OrchestrationV2Checkpoint = Schema.Struct({ id: CheckpointId, threadId: ThreadId, @@ -1362,6 +1370,7 @@ export const OrchestrationV2Checkpoint = Schema.Struct({ ref: CheckpointRef, status: Schema.Literals(["ready", "missing", "error", "stale"]), files: Schema.Array(OrchestrationV2CheckpointFileSummary), + filesTotal: Schema.optionalKey(OrchestrationV2CheckpointFileTotals), capturedAt: Schema.DateTimeUtc, }); export type OrchestrationV2Checkpoint = typeof OrchestrationV2Checkpoint.Type; @@ -1660,6 +1669,7 @@ export const OrchestrationV2TurnItem = Schema.Union([ checkpointId: CheckpointId, scopeId: CheckpointScopeId, files: Schema.Array(OrchestrationV2CheckpointFileSummary), + filesTotal: Schema.optionalKey(OrchestrationV2CheckpointFileTotals), /** Same as the files of checkpoint `checkpointId`. */ filesOmitted: DedupedBodyMarker, }), @@ -1667,6 +1677,8 @@ export const OrchestrationV2TurnItem = Schema.Union([ ...OrchestrationV2TurnItemBaseFields, type: Schema.Literal("run_interrupt_request"), message: Schema.String, + /** Set when the provider did not confirm the stop in time; clients then offer Force stop. */ + unresponsiveAt: Schema.optionalKey(Schema.DateTimeUtc), }), Schema.Struct({ ...OrchestrationV2TurnItemBaseFields, @@ -2491,6 +2503,7 @@ export const OrchestrationV2TurnItemJson = Schema.Union([ ...OrchestrationV2TurnItemJsonBaseFields, type: Schema.Literal("run_interrupt_request"), message: Schema.String, + unresponsiveAt: Schema.optionalKey(Schema.DateTimeUtcFromString), }), Schema.Struct({ ...OrchestrationV2TurnItemJsonBaseFields, @@ -3429,12 +3442,22 @@ const OrchestrationV2InternalCommand = Schema.Union([ threadId: ThreadId, requestId: RuntimeRequestId, }), + /** The provider did not confirm a Stop of this turn in time. */ + Schema.Struct({ + type: Schema.Literal("thread.interrupt.unresponsive"), + commandId: CommandId, + threadId: ThreadId, + providerThreadId: ProviderThreadId, + providerTurnId: ProviderTurnId, + }), Schema.Struct({ type: Schema.Literal("thread.background-work.settle"), commandId: CommandId, threadId: ThreadId, providerThreadId: ProviderThreadId, providerTurnId: ProviderTurnId, + /** Replaces the Stop's own reason on the result, as Force stop does. */ + reason: Schema.optional(Schema.String), subagent: Schema.optional( Schema.Struct({ id: NodeId, @@ -3514,6 +3537,7 @@ export const ORCHESTRATION_V2_WS_METHODS = { getThreadProjection: "orchestration.getThreadProjection", getWorkflowScript: "orchestration.getWorkflowScript", stopWorkflow: "orchestration.stopWorkflow", + forceStopThread: "orchestration.forceStopThread", getTurnItem: "orchestration.getTurnItem", launchThread: "orchestration.launchThread", subscribeShell: "orchestration.subscribeShell", @@ -3901,6 +3925,41 @@ export class OrchestrationV2StopWorkflowError extends Schema.TaggedError()( + "OrchestrationV2ForceStopThreadError", + { + threadId: ThreadId, + /** `superseded`: a newer run started since the user asked; nothing was stopped. */ + reason: Schema.optional(Schema.Literal("superseded")), + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return this.reason === "superseded" + ? "A new turn started, so nothing was force stopped." + : "Failed to force stop the agent."; + } +} + export const OrchestrationV2GetWorkflowScriptInput = Schema.Struct({ threadId: ThreadId, scriptPath: TrimmedNonEmptyString, @@ -4087,6 +4146,10 @@ export const OrchestrationV2RpcSchemas = { input: OrchestrationV2StopWorkflowInput, output: Schema.Void, }, + forceStopThread: { + input: OrchestrationV2ForceStopThreadInput, + output: OrchestrationV2ForceStopThreadResult, + }, getWorkflowScript: { input: OrchestrationV2GetWorkflowScriptInput, output: OrchestrationV2GetWorkflowScriptResult, diff --git a/packages/contracts/src/providerInstance.test.ts b/packages/contracts/src/providerInstance.test.ts index f7f63b13945ae..ccc2f1de34301 100644 --- a/packages/contracts/src/providerInstance.test.ts +++ b/packages/contracts/src/providerInstance.test.ts @@ -5,17 +5,7 @@ import { ProviderInstanceEnvironmentVariable } from "./providerInstance.ts"; const decodeEnvironmentVariable = Schema.decodeUnknownSync(ProviderInstanceEnvironmentVariable); -describe("ProviderInstanceEnvironmentVariable secret sources", () => { - it("decodes a 1Password source", () => { - const value = { kind: "1password", reference: "op://Private/item/field", account: "my" }; - expect(decodeEnvironmentVariable({ name: "API_KEY", value }).value).toEqual(value); - }); - - it("decodes a named secret provider source", () => { - const value = { kind: "secret-provider", provider: "homelab", reference: "apps/claude#key" }; - expect(decodeEnvironmentVariable({ name: "API_KEY", value }).value).toEqual(value); - }); - +describe("ProviderInstanceEnvironmentVariable literals", () => { it("decodes a string value as a literal, even one that looks like a reference", () => { expect( decodeEnvironmentVariable({ name: "API_KEY", value: "op://Private/item/field" }).value, diff --git a/packages/contracts/src/providerInstance.ts b/packages/contracts/src/providerInstance.ts index 205caf99120f2..61fee930c3193 100644 --- a/packages/contracts/src/providerInstance.ts +++ b/packages/contracts/src/providerInstance.ts @@ -1,8 +1,8 @@ import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; +import * as SchemaTransformation from "effect/SchemaTransformation"; import { TrimmedNonEmptyString } from "./baseSchemas.ts"; -import { OnePasswordSecretSource } from "./onePassword.ts"; -import { SecretProviderSecretSource } from "./secretProvider.ts"; +import { SecretReference } from "./secretReference.ts"; const PROVIDER_SLUG_MAX_CHARS = 64; const PROVIDER_SLUG_PATTERN = /^[a-zA-Z][a-zA-Z0-9_-]*$/; @@ -29,33 +29,48 @@ export const ProviderInstanceEnvironmentVariableName = TrimmedNonEmptyString.che export type ProviderInstanceEnvironmentVariableName = typeof ProviderInstanceEnvironmentVariableName.Type; -/** - * An environment value read from an external secret store at the moment the - * provider process starts, instead of a literal. Discriminated on `kind`. - */ -export const ProviderSecretSource = Schema.Union([ - OnePasswordSecretSource, - SecretProviderSecretSource, -]); -export type ProviderSecretSource = typeof ProviderSecretSource.Type; +const ProviderInstanceEnvironmentVariableValue = Schema.Struct({ + name: ProviderInstanceEnvironmentVariableName, + value: Schema.Union([Schema.String, SecretReference]), + sensitive: Schema.Boolean, + valueRedacted: Schema.optionalKey(Schema.Boolean), +}); -/** - * `sensitive` and `valueRedacted` only describe literal string values; a - * secret source is not itself a secret and is stored as written. - */ -export const ProviderInstanceEnvironmentVariable = Schema.Struct({ +// A reference is encoded beside an empty value so older servers and clients still decode it. +const StoredProviderInstanceEnvironmentVariable = Schema.Struct({ name: ProviderInstanceEnvironmentVariableName, - value: Schema.Union([Schema.String, ProviderSecretSource]).pipe( - Schema.withDecodingDefault(Effect.succeed("")), - ), + value: Schema.String.pipe(Schema.withDecodingDefault(Effect.succeed(""))), sensitive: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), valueRedacted: Schema.optionalKey(Schema.Boolean), + secret: Schema.optionalKey(SecretReference), }); + +export const ProviderInstanceEnvironmentVariable = StoredProviderInstanceEnvironmentVariable.pipe( + Schema.decodeTo( + Schema.toType(ProviderInstanceEnvironmentVariableValue), + SchemaTransformation.transform< + typeof ProviderInstanceEnvironmentVariableValue.Type, + typeof StoredProviderInstanceEnvironmentVariable.Type + >({ + decode: ({ secret, ...variable }) => + secret === undefined ? variable : { ...variable, value: secret }, + encode: ({ value, ...variable }) => + typeof value === "string" + ? { ...variable, value } + : { ...variable, value: "", secret: value }, + }), + ), +); export type ProviderInstanceEnvironmentVariable = typeof ProviderInstanceEnvironmentVariable.Type; export const ProviderInstanceEnvironment = Schema.Array(ProviderInstanceEnvironmentVariable); export type ProviderInstanceEnvironment = typeof ProviderInstanceEnvironment.Type; +/** What drivers receive: every secret reference has been read into a plain value. */ +export type ResolvedProviderInstanceEnvironment = ReadonlyArray< + Omit & { readonly value: string } +>; + /** Drivers decode their own config; unknown-driver envelopes round-trip verbatim across versions. */ export const ProviderInstanceConfig = Schema.Struct({ driver: ProviderDriverKind, diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 4dbb6a199f472..8ebc9df23a0bb 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -981,6 +981,8 @@ export const RelayT3ChatLinkResponse = Schema.Struct({ // False when this relay has no T3 Chat secret, so clients hide the feature. available: Schema.Boolean, link: Schema.NullOr(RelayT3ChatLink), + // A manual cap that replaces the plan's; T3 Chat upgrades do not raise it. + maxTunnelsOverride: Schema.optionalKey(Schema.Number), }); export type RelayT3ChatLinkResponse = typeof RelayT3ChatLinkResponse.Type; diff --git a/packages/contracts/src/review.ts b/packages/contracts/src/review.ts index 5a95442917c9a..a40bddfc1e33e 100644 --- a/packages/contracts/src/review.ts +++ b/packages/contracts/src/review.ts @@ -6,18 +6,41 @@ import { VcsError } from "./vcs.ts"; export const ReviewDiffPreviewInput = Schema.Struct({ cwd: TrimmedNonEmptyString, baseRef: Schema.optional(TrimmedNonEmptyString), + commitSha: Schema.optional(TrimmedNonEmptyString), ignoreWhitespace: Schema.optionalKey(Schema.Boolean), file: Schema.optionalKey( Schema.Struct({ path: Schema.NonEmptyString, previousPath: Schema.NullOr(Schema.NonEmptyString), - sourceKind: Schema.Literals(["working-tree", "branch-range"]), + sourceKind: Schema.Literals(["working-tree", "branch-range", "commit"]), }), ), }); export type ReviewDiffPreviewInput = typeof ReviewDiffPreviewInput.Type; -export const ReviewDiffPreviewSourceKind = Schema.Literals(["working-tree", "branch-range"]); +export const ReviewBranchCommitsInput = Schema.Struct({ + cwd: TrimmedNonEmptyString, + baseRef: Schema.optional(TrimmedNonEmptyString), +}); +export type ReviewBranchCommitsInput = typeof ReviewBranchCommitsInput.Type; + +export const ReviewBranchCommitsResult = Schema.Struct({ + commits: Schema.Array( + Schema.Struct({ + sha: Schema.String, + subject: Schema.String, + committedAt: Schema.String, + }), + ), + truncated: Schema.Boolean, +}); +export type ReviewBranchCommitsResult = typeof ReviewBranchCommitsResult.Type; + +export const ReviewDiffPreviewSourceKind = Schema.Literals([ + "working-tree", + "branch-range", + "commit", +]); export type ReviewDiffPreviewSourceKind = typeof ReviewDiffPreviewSourceKind.Type; export const ReviewDiffFileStat = Schema.Struct({ @@ -32,6 +55,7 @@ export const ReviewDiffPreviewSource = Schema.Struct({ id: TrimmedNonEmptyString, kind: ReviewDiffPreviewSourceKind, title: TrimmedNonEmptyString, + commitSubject: Schema.optionalKey(TrimmedNonEmptyString), baseRef: Schema.NullOr(TrimmedNonEmptyString), headRef: Schema.NullOr(TrimmedNonEmptyString), diff: Schema.String, diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 115e270a77ec0..ca6f1905f2ab7 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -38,7 +38,8 @@ import * as Rpc from "effect/rpc/Rpc"; import * as RpcGroup from "effect/rpc/RpcGroup"; import * as RpcMiddleware from "effect/rpc/RpcMiddleware"; import { NonNegativeInt, TrimmedNonEmptyString } from "./baseSchemas.ts"; -import { OnePasswordAccountSummary } from "./onePassword.ts"; +import { OnePasswordAccountSummary, SecretSourceCheckResult } from "./settingsSecretValue.ts"; +import { SecretReference } from "./secretReference.ts"; import { ProviderAuthCancelInput, ProviderAuthCompleteInput, @@ -150,6 +151,8 @@ import { VcsStatusStreamEvent, } from "./git.ts"; import { + ReviewBranchCommitsInput, + ReviewBranchCommitsResult, ReviewDiffFileContentsInput, ReviewDiffFileContentsResult, ReviewDiffPreviewError, @@ -163,12 +166,7 @@ import { ProviderUploadFeedbackInput, ProviderUploadFeedbackResult, } from "./provider.ts"; -import { - ProviderInstanceId, - ProviderInstanceMutation, - ProviderSecretSource, -} from "./providerInstance.ts"; -import { SecretSourceCheckResult } from "./secretProvider.ts"; +import { ProviderInstanceId, ProviderInstanceMutation } from "./providerInstance.ts"; import { GuestShareCreateError, GuestShareCreateInput, @@ -226,6 +224,7 @@ import { ORCHESTRATION_V2_WS_METHODS, OrchestrationGetWorkflowScriptError, OrchestrationV2StopWorkflowError, + OrchestrationV2ForceStopThreadError, OrchestrationV2DispatchCommandError, OrchestrationV2GetShellSnapshotError, OrchestrationV2CommandOutputError, @@ -460,6 +459,7 @@ export const WS_METHODS = { gitResolvePullRequest: "git.resolvePullRequest", gitPreparePullRequestThread: "git.preparePullRequestThread", + reviewGetBranchCommits: "review.getBranchCommits", reviewGetDiffPreview: "review.getDiffPreview", reviewGetDiffFileContents: "review.getDiffFileContents", @@ -641,6 +641,8 @@ export const WsServerRefreshProvidersRpc = Rpc.make(WS_METHODS.serverRefreshProv fresh: Schema.optional(Schema.Boolean), /** Bypass model caches; explicit rediscovery may open sessions, background refreshes must not. */ refreshModels: Schema.optional(Schema.Boolean), + /** Read secret references again; unlocking a vault can prompt on the host, so only explicit user refreshes ask. */ + reloadSecrets: Schema.optional(Schema.Boolean), /** * Reply without snapshots, or with only the requested instance's cwd snapshot. Snapshots then * reach the client only through these replies, so another client's scan shows on its next @@ -830,7 +832,7 @@ const WsServerListOnePasswordAccountsRpc = Rpc.make(WS_METHODS.serverListOnePass }); const WsServerCheckSecretSourceRpc = Rpc.make(WS_METHODS.serverCheckSecretSource, { - payload: Schema.Struct({ source: ProviderSecretSource }), + payload: Schema.Struct({ source: SecretReference }), success: SecretSourceCheckResult, error: EnvironmentAuthorizationError, }); @@ -1517,6 +1519,12 @@ const WsVcsInitRpc = Rpc.make(WS_METHODS.vcsInit, { error: Schema.Union([VcsError, EnvironmentAuthorizationError]), }); +const WsReviewGetBranchCommitsRpc = Rpc.make(WS_METHODS.reviewGetBranchCommits, { + payload: ReviewBranchCommitsInput, + success: ReviewBranchCommitsResult, + error: Schema.Union([ReviewDiffPreviewError, EnvironmentAuthorizationError]), +}); + const WsReviewGetDiffPreviewRpc = Rpc.make(WS_METHODS.reviewGetDiffPreview, { payload: ReviewDiffPreviewInput, success: ReviewDiffPreviewResult, @@ -1790,6 +1798,12 @@ const WsOrchestrationV2StopWorkflowRpc = Rpc.make(ORCHESTRATION_V2_WS_METHODS.st error: Schema.Union([OrchestrationV2StopWorkflowError, EnvironmentAuthorizationError]), }); +const WsOrchestrationV2ForceStopThreadRpc = Rpc.make(ORCHESTRATION_V2_WS_METHODS.forceStopThread, { + payload: OrchestrationV2RpcSchemas.forceStopThread.input, + success: OrchestrationV2RpcSchemas.forceStopThread.output, + error: Schema.Union([OrchestrationV2ForceStopThreadError, EnvironmentAuthorizationError]), +}); + const WsOrchestrationV2GetWorkflowScriptRpc = Rpc.make( ORCHESTRATION_V2_WS_METHODS.getWorkflowScript, { @@ -2105,6 +2119,7 @@ export const WsRpcGroup = RpcGroup.make( WsVcsCreateRefRpc, WsVcsSwitchRefRpc, WsVcsInitRpc, + WsReviewGetBranchCommitsRpc, WsReviewGetDiffPreviewRpc, WsReviewGetDiffFileContentsRpc, WsTerminalOpenRpc, @@ -2144,6 +2159,7 @@ export const WsRpcGroup = RpcGroup.make( WsOrchestrationV2DispatchCommandRpc, WsOrchestrationV2GetWorkflowScriptRpc, WsOrchestrationV2StopWorkflowRpc, + WsOrchestrationV2ForceStopThreadRpc, WsOrchestrationV2GetTurnItemRpc, WsOrchestrationV2GetTurnDiffRpc, WsOrchestrationV2GetFullThreadDiffRpc, diff --git a/packages/contracts/src/secretProvider.test.ts b/packages/contracts/src/secretProvider.test.ts deleted file mode 100644 index 38c7b1ed06666..0000000000000 --- a/packages/contracts/src/secretProvider.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { describe, expect, it } from "vite-plus/test"; -import * as Schema from "effect/Schema"; - -import { - AwsSecretsManagerSecretProviderConfig, - OpenBaoSecretProviderConfig, - FlatSecretProviderConfig, - SettingsSecretValue, -} from "./secretProvider.ts"; -import { secretSettingsFields } from "./settingsSecret.ts"; -import { ProviderInstanceEnvironmentVariable } from "./providerInstance.ts"; -import { EasDeviceHostConfig } from "./device.ts"; - -describe("SettingsSecretValue", () => { - const decode = Schema.decodeUnknownSync(SettingsSecretValue); - const source = { kind: "secret-provider", provider: "homelab", reference: "expo#token" }; - - it("decodes a literal and a secret provider reference", () => { - expect(decode(" literal-token ")).toBe("literal-token"); - expect(decode(source)).toEqual(source); - }); - - it("rejects a reference without a provider", () => { - expect(() => decode({ kind: "secret-provider", reference: "expo#token" })).toThrow(); - }); - - it("lets an EAS host keep either shape as its access token", () => { - const decodeHost = Schema.decodeUnknownSync(EasDeviceHostConfig); - const host = { kind: "eas", id: "eas", label: "EAS Simulator" }; - expect(decodeHost({ ...host, accessToken: "expo-literal" }).accessToken).toBe("expo-literal"); - expect(decodeHost({ ...host, accessToken: source }).accessToken).toEqual(source); - expect(decodeHost(host).accessToken).toBe(""); - expect(secretSettingsFields(EasDeviceHostConfig)).toContain("accessToken"); - }); -}); - -const decodeConfig = Schema.decodeUnknownSync(FlatSecretProviderConfig); -const decodeVariable = Schema.decodeUnknownSync(ProviderInstanceEnvironmentVariable); - -describe("FlatSecretProviderConfig", () => { - it("decodes a 1Password provider", () => { - const config = { kind: "1password", label: "Personal", account: "my.1password.com" }; - expect(decodeConfig(config)).toEqual(config); - }); - - it("defaults the OpenBao mount and keeps the optional fields", () => { - expect( - decodeConfig({ kind: "openbao", label: " Homelab ", address: "https://bao.example:8200" }), - ).toEqual({ - kind: "openbao", - label: "Homelab", - address: "https://bao.example:8200", - mount: "secret", - auth: "token", - }); - const full = { - kind: "openbao", - label: "Homelab", - address: "http://127.0.0.1:8200", - mount: "kv/team", - namespace: "admin", - token: "s.token", - auth: "cli", - }; - expect(decodeConfig(full)).toEqual(full); - }); - - it("marks only the OpenBao token as a stored secret", () => { - expect(secretSettingsFields(OpenBaoSecretProviderConfig)).toEqual(["token"]); - }); - - it.each([ - ["an empty label", { label: "" }], - ["a long label", { label: "x".repeat(65) }], - ["a non-http address", { address: "file:///etc/passwd" }], - ["an unparsable address", { address: "bao.example" }], - ["a mount starting with a dash", { mount: "-secret" }], - ["a mount with a space", { mount: "my secret" }], - ["an empty namespace", { namespace: "" }], - ])("rejects an OpenBao provider with %s", (_label, override) => { - expect(() => - decodeConfig({ - kind: "openbao", - label: "Homelab", - address: "https://bao.example:8200", - ...override, - }), - ).toThrow(); - }); - - it("decodes an AWS Secrets Manager provider with and without a profile", () => { - const withoutProfile = { - kind: "aws-secrets-manager", - label: "Production", - region: "us-east-1", - }; - expect(decodeConfig(withoutProfile)).toEqual(withoutProfile); - const withProfile = { ...withoutProfile, region: "ap-southeast-2", profile: "production" }; - expect(decodeConfig(withProfile)).toEqual(withProfile); - }); - - it("stores no secret fields for an AWS Secrets Manager provider", () => { - expect(secretSettingsFields(AwsSecretsManagerSecretProviderConfig)).toEqual([]); - }); - - it.each([ - ["a missing region", { region: undefined }], - ["a region with uppercase", { region: "US-EAST-1" }], - ["a region without a number", { region: "us-east" }], - ["an unrelated region", { region: "mars" }], - ["a profile starting with a dash", { profile: "-production" }], - ["a profile with whitespace", { profile: "my profile" }], - ["an empty profile", { profile: "" }], - ["a long profile", { profile: "x".repeat(65) }], - ])("rejects an AWS Secrets Manager provider with %s", (_label, override) => { - expect(() => - decodeConfig({ - kind: "aws-secrets-manager", - label: "Production", - region: "us-east-1", - ...override, - }), - ).toThrow(); - }); - - it("rejects a 1Password provider without an account", () => { - expect(() => decodeConfig({ kind: "1password", label: "Personal" })).toThrow(); - }); -}); - -describe("SecretProviderSecretSource", () => { - const source = (reference: string) => ({ - kind: "secret-provider", - provider: "homelab", - reference, - }); - - it.each(["op://Private/item/field", "apps/claude#api_key", "team/app config#token"])( - "decodes the reference %s as an environment value", - (reference) => { - expect(decodeVariable({ name: "API_KEY", value: source(reference) }).value).toEqual( - source(reference), - ); - }, - ); - - it.each([ - ["empty", ""], - ["leading whitespace", " apps/claude#key"], - ["a trailing newline", "apps/claude#key\n"], - ["a control character", "apps/cl\u0007aude#key"], - ["a template delimiter", "apps/{{claude}}#key"], - ["too many characters", "x".repeat(1025)], - ])("rejects a reference with %s", (_label, reference) => { - expect(() => decodeVariable({ name: "API_KEY", value: source(reference) })).toThrow(); - }); - - it("still decodes the legacy 1Password source", () => { - const value = { kind: "1password", reference: "op://Private/item/field", account: "my" }; - expect(decodeVariable({ name: "API_KEY", value }).value).toEqual(value); - }); -}); diff --git a/packages/contracts/src/secretProvider.ts b/packages/contracts/src/secretProvider.ts deleted file mode 100644 index f5ac2aefd69e6..0000000000000 --- a/packages/contracts/src/secretProvider.ts +++ /dev/null @@ -1,268 +0,0 @@ -import * as Effect from "effect/Effect"; -import * as Schema from "effect/Schema"; -import { TrimmedNonEmptyString, TrimmedString } from "./baseSchemas.ts"; -import { OnePasswordAccount } from "./onePassword.ts"; -import { SecretProviderId, SecretProviderKind } from "./secretProviderId.ts"; -import { secretSetting } from "./settingsSecret.ts"; - -const SECRET_PROVIDER_LABEL_MAX_CHARS = 64; -const SECRET_PROVIDER_REFERENCE_MAX_CHARS = 1024; -const OPENBAO_ADDRESS_MAX_CHARS = 1024; -const OPENBAO_PATH_MAX_CHARS = 253; -const OPENBAO_MOUNT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._/-]*$/; -const AWS_REGION_PATTERN = /^[a-z]{2}(-[a-z]+)+-\d+$/; -const AWS_PROFILE_MAX_CHARS = 64; - -function hasControlCharacter(value: string): boolean { - for (let index = 0; index < value.length; index++) { - const code = value.charCodeAt(index); - if (code < 0x20 || code === 0x7f) return true; - } - return false; -} - -function isHttpUrl(value: string): boolean { - try { - const url = new URL(value); - return url.protocol === "http:" || url.protocol === "https:"; - } catch { - return false; - } -} - -function secretProviderReferenceIssue(value: string): string | undefined { - if (value.length === 0) return "Secret reference must not be empty."; - if (value !== value.trim()) return "Secret reference must not start or end with whitespace."; - if (hasControlCharacter(value)) return "Secret reference must not contain control characters."; - // 1Password references are resolved through an `op inject` template, where - // `{{` and `}}` delimit a reference. - if (value.includes("{{") || value.includes("}}")) { - return "Secret reference must not contain {{ or }}."; - } - return undefined; -} - -export const SecretProviderLabel = TrimmedNonEmptyString.check( - Schema.isMaxLength(SECRET_PROVIDER_LABEL_MAX_CHARS), -); -export type SecretProviderLabel = typeof SecretProviderLabel.Type; - -export const OpenBaoAddress = TrimmedNonEmptyString.check( - Schema.isMaxLength(OPENBAO_ADDRESS_MAX_CHARS), - Schema.makeFilter((value: string) => - isHttpUrl(value) ? true : "OpenBao address must be an http or https URL.", - ), -); -export type OpenBaoAddress = typeof OpenBaoAddress.Type; - -/** A KV secrets engine mount. It is passed to the CLI as an argument, so it can never start with `-`. */ -export const OpenBaoMount = TrimmedString.check( - Schema.isMaxLength(OPENBAO_PATH_MAX_CHARS), - Schema.makeFilter((value: string) => - OPENBAO_MOUNT_PATTERN.test(value) - ? true - : "OpenBao mount must start with a letter or digit and contain no spaces.", - ), -); -export type OpenBaoMount = typeof OpenBaoMount.Type; - -export const OpenBaoNamespace = TrimmedNonEmptyString.check( - Schema.isMaxLength(OPENBAO_PATH_MAX_CHARS), -); -export type OpenBaoNamespace = typeof OpenBaoNamespace.Type; - -export const OpenBaoToken = TrimmedString; -export type OpenBaoToken = typeof OpenBaoToken.Type; - -/** - * Exactly one place the server takes the token from: the provider's stored - * token, the `BAO_TOKEN` variable of the server process, or the login the - * `bao` CLI holds on that machine. Nothing falls back to another source. - */ -export const OpenBaoAuth = Schema.Literals(["token", "env", "cli"]); -export type OpenBaoAuth = typeof OpenBaoAuth.Type; - -/** The region the provider reads in, such as `us-east-1`. Secrets Manager is regional, so one provider is one region. */ -export const AwsRegion = TrimmedNonEmptyString.check( - Schema.makeFilter((value: string) => - AWS_REGION_PATTERN.test(value) ? true : "AWS region must look like us-east-1.", - ), -); -export type AwsRegion = typeof AwsRegion.Type; - -/** A named profile from the server's AWS CLI configuration. It is passed to the CLI as an argument, so it can never start with `-`. */ -export const AwsProfile = TrimmedNonEmptyString.check( - Schema.isMaxLength(AWS_PROFILE_MAX_CHARS), - Schema.makeFilter((value: string) => - !value.startsWith("-") && !/\s/.test(value) && !hasControlCharacter(value) - ? true - : "AWS profile must not start with - or contain whitespace.", - ), -); -export type AwsProfile = typeof AwsProfile.Type; - -/** The fork's flat 1Password entry; upstream's envelope carries the same account as `OnePasswordSecretProviderConfig`. */ -export const FlatOnePasswordSecretProviderConfig = Schema.Struct({ - kind: Schema.Literal("1password"), - label: SecretProviderLabel, - account: OnePasswordAccount, -}); -export type FlatOnePasswordSecretProviderConfig = typeof FlatOnePasswordSecretProviderConfig.Type; - -export const OpenBaoSecretProviderConfig = Schema.Struct({ - kind: Schema.Literal("openbao"), - label: SecretProviderLabel, - address: OpenBaoAddress, - mount: OpenBaoMount.pipe(Schema.withDecodingDefault(Effect.succeed("secret"))), - namespace: Schema.optionalKey(OpenBaoNamespace), - token: Schema.optionalKey(OpenBaoToken).pipe(secretSetting), - auth: OpenBaoAuth.pipe(Schema.withDecodingDefault(Effect.succeed("token" as const))), -}); -export type OpenBaoSecretProviderConfig = typeof OpenBaoSecretProviderConfig.Type; - -/** - * Credentials are never stored here: the `aws` CLI on the server resolves them - * the way it does for any other command, from the named profile when one is - * set, otherwise from its default chain (environment, default profile, SSO, - * instance role). - */ -export const AwsSecretsManagerSecretProviderConfig = Schema.Struct({ - kind: Schema.Literal("aws-secrets-manager"), - label: SecretProviderLabel, - region: AwsRegion, - profile: Schema.optionalKey(AwsProfile), -}); -export type AwsSecretsManagerSecretProviderConfig = - typeof AwsSecretsManagerSecretProviderConfig.Type; - -// Fork follow-up: the flat kinds below should converge on the `{ kind, label, config }` envelope -// (`one_password` and friends) once upstream resolves them and adds OpenBao and AWS kinds. -export const FlatSecretProviderConfig = Schema.Union([ - FlatOnePasswordSecretProviderConfig, - OpenBaoSecretProviderConfig, - AwsSecretsManagerSecretProviderConfig, -]); -export type FlatSecretProviderConfig = typeof FlatSecretProviderConfig.Type; - -export const FlatSecretProviderKind = Schema.Literals([ - "1password", - "openbao", - "aws-secrets-manager", -]); -export type FlatSecretProviderKind = FlatSecretProviderConfig["kind"]; - -const isFlatSecretProviderKindValue = Schema.is(FlatSecretProviderKind); - -function isFlatSecretProviderKind(kind: string): kind is FlatSecretProviderKind { - return isFlatSecretProviderKindValue(kind); -} - -/** A flat entry keeps its fields beside `kind`; a `config` key makes it an envelope even when the kind collides. */ -export function isFlatSecretProviderConfig( - config: StoredSecretProviderConfig, -): config is FlatSecretProviderConfig { - return isFlatSecretProviderKind(config.kind) && !Object.hasOwn(config, "config"); -} - -export const OnePasswordSecretProviderConfig = Schema.Struct({ - account: OnePasswordAccount, -}); -export type OnePasswordSecretProviderConfig = typeof OnePasswordSecretProviderConfig.Type; - -/** Kinds decode their own config; unknown-kind envelopes round-trip verbatim across versions. */ -export const SecretProviderConfig = Schema.Struct({ - kind: SecretProviderKind, - label: TrimmedNonEmptyString, - config: Schema.optionalKey(Schema.Unknown), -}); -export type SecretProviderConfig = typeof SecretProviderConfig.Type; - -/** Kinds this build can resolve. Their `config` holds no credentials, so clients may read it. */ -const SECRET_PROVIDER_KIND_CONFIG_GUARDS: Readonly boolean>> = { - one_password: Schema.is(OnePasswordSecretProviderConfig), -}; - -export function isKnownSecretProviderKind(kind: string): boolean { - return Object.hasOwn(SECRET_PROVIDER_KIND_CONFIG_GUARDS, kind); -} - -// Limits that may change later are enforced on writes only, never on the stored envelope. -// Writes must be a kind this build knows with a config it accepts, so an older server -// rejects a newer kind instead of storing what it cannot check. -export const SecretProviderConfigWrite = Schema.Struct({ - ...SecretProviderConfig.fields, - label: SecretProviderLabel, -}).check( - Schema.makeFilter((entry) => { - if (!isKnownSecretProviderKind(entry.kind)) { - return `Unsupported secret provider kind: ${entry.kind}.`; - } - return SECRET_PROVIDER_KIND_CONFIG_GUARDS[entry.kind]!(entry.config) - ? true - : `Invalid ${entry.kind} secret provider config.`; - }), -); - -// A fork kind that fails its own member must fail here instead of degrading into an envelope -// that drops its fields. Only an entry carrying `config` is a genuine envelope of a colliding kind. -const SecretProviderEnvelope = SecretProviderConfig.check( - Schema.makeFilter((entry) => - isFlatSecretProviderKind(entry.kind) && entry.config === undefined - ? `Invalid ${entry.kind} secret provider config.` - : true, - ), -); - -/** What `settings.secretProviders` stores: the fork's flat entries, tried first, or an upstream envelope. */ -export const StoredSecretProviderConfig = Schema.Union([ - ...FlatSecretProviderConfig.members, - SecretProviderEnvelope, -]); -export type StoredSecretProviderConfig = typeof StoredSecretProviderConfig.Type; - -/** What a settings patch may write for one provider. */ -export const SecretProviderConfigPatch = Schema.Union([ - ...FlatSecretProviderConfig.members, - SecretProviderConfigWrite, -]); -export type SecretProviderConfigPatch = typeof SecretProviderConfigPatch.Type; - -/** - * A secret's location inside a named provider: `op://vault/item/field` for - * 1Password, `path/to/secret#field` for OpenBao, and `name-or-arn` or - * `name-or-arn#key` for AWS Secrets Manager. The format is checked when the - * reference is resolved, against the provider it names. - */ -export const SecretProviderReference = Schema.String.check( - Schema.isMaxLength(SECRET_PROVIDER_REFERENCE_MAX_CHARS), - Schema.makeFilter((value: string) => secretProviderReferenceIssue(value) ?? true), -).pipe(Schema.brand("SecretProviderReference")); -export type SecretProviderReference = typeof SecretProviderReference.Type; - -export const SecretSourceCheckResult = Schema.Union([ - Schema.TaggedStruct("Resolved", {}), - Schema.TaggedStruct("Failed", { detail: Schema.String }), -]); -export type SecretSourceCheckResult = typeof SecretSourceCheckResult.Type; - -export const SecretProviderSecretSource = Schema.Struct({ - kind: Schema.Literal("secret-provider"), - provider: SecretProviderId, - reference: SecretProviderReference, -}); -export type SecretProviderSecretSource = typeof SecretProviderSecretSource.Type; - -/** - * A credential setting: the literal, which is kept in the secret store and redacted for - * clients, or the secret provider to read it from, which is not itself a secret and is - * stored and shown as written. - */ -export const SettingsSecretValue = Schema.Union([TrimmedString, SecretProviderSecretSource]); -export type SettingsSecretValue = typeof SettingsSecretValue.Type; - -export const isSecretProviderSecretSource = Schema.is(SecretProviderSecretSource); - -/** The literal a credential setting holds, or `""` when it names a secret instead. */ -export function settingsSecretLiteral(value: SettingsSecretValue | undefined): string { - return typeof value === "string" ? value : ""; -} diff --git a/packages/contracts/src/secretProviderId.ts b/packages/contracts/src/secretProviderId.ts deleted file mode 100644 index 78b645b253bea..0000000000000 --- a/packages/contracts/src/secretProviderId.ts +++ /dev/null @@ -1,20 +0,0 @@ -import * as Schema from "effect/Schema"; -import { TrimmedNonEmptyString } from "./baseSchemas.ts"; - -// Final once shipped: older builds drop entries whose key fails here, and erase them on their next save. -// Unlike provider slugs it may start with a digit, so a provider named "1Password" fits. -const SecretProviderSlug = TrimmedNonEmptyString.check( - Schema.isMaxLength(64), - Schema.isPattern(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), -); - -/** - * Key of one `settings.secretProviders` entry. Lives in its own module so - * both the settings and the provider instance contracts can import it without - * importing each other. - */ -export const SecretProviderId = SecretProviderSlug.pipe(Schema.brand("SecretProviderId")); -export type SecretProviderId = typeof SecretProviderId.Type; - -export const SecretProviderKind = SecretProviderSlug.pipe(Schema.brand("SecretProviderKind")); -export type SecretProviderKind = typeof SecretProviderKind.Type; diff --git a/packages/contracts/src/secretReference.ts b/packages/contracts/src/secretReference.ts new file mode 100644 index 0000000000000..4e03f8a675688 --- /dev/null +++ b/packages/contracts/src/secretReference.ts @@ -0,0 +1,27 @@ +import * as Schema from "effect/Schema"; + +import { TrimmedNonEmptyString } from "./baseSchemas.ts"; + +// Separate module avoids an import cycle between settings and provider instance contracts. +// Final once shipped: older builds drop entries whose key fails here, and erase them on their next save. +// Unlike provider slugs it may start with a digit, so a provider named "1Password" fits. +export const SecretProviderSlug = TrimmedNonEmptyString.check( + Schema.isMaxLength(64), + Schema.isPattern(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), +); + +export const SecretProviderId = SecretProviderSlug.pipe(Schema.brand("SecretProviderId")); +export type SecretProviderId = typeof SecretProviderId.Type; + +// Provider-specific syntax is the secret provider driver's to check. +export const SecretReferencePath = TrimmedNonEmptyString.check( + Schema.isMaxLength(1024), + Schema.isPattern(/^[^\r\n]+$/), +); + +/** Points at a secret held by a configured secret provider; the secret itself is never stored. */ +export const SecretReference = Schema.Struct({ + provider: SecretProviderId, + reference: SecretReferencePath, +}); +export type SecretReference = typeof SecretReference.Type; diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index 9fd3813abf5fa..ab79a9dea60f5 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -640,6 +640,8 @@ export const ServerConfig = Schema.Struct({ threadFindProgressive: Schema.optionalKey(Schema.Boolean), /** Settings patches may carry `secretProviders`; older servers drop the key without an error. */ secretProviderSettings: Schema.optionalKey(Schema.Boolean), + /** Secret settings may point at a secret provider; older servers drop the reference. */ + secretReferences: Schema.optionalKey(Schema.Boolean), scratchWorkspaceRoot: Schema.optionalKey(TrimmedNonEmptyString), newProjectsRoot: Schema.optionalKey(TrimmedNonEmptyString), /** diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts index f1aea820544a6..859644a54c72f 100644 --- a/packages/contracts/src/settings.test.ts +++ b/packages/contracts/src/settings.test.ts @@ -2,7 +2,11 @@ import { describe, expect, it } from "vite-plus/test"; import * as Schema from "effect/Schema"; import { type CommandId, ProjectId } from "./baseSchemas.ts"; -import { ProviderDriverKind, ProviderInstanceId } from "./providerInstance.ts"; +import { + ProviderDriverKind, + ProviderInstanceEnvironment, + ProviderInstanceId, +} from "./providerInstance.ts"; import { ClientSettingsSchema, ClientSettingsPatch, @@ -23,8 +27,17 @@ const decodeClientSettings = Schema.decodeUnknownSync(ClientSettingsSchema); const decodeClientSettingsPatch = Schema.decodeUnknownSync(ClientSettingsPatch); const encodeClientSettings = Schema.encodeSync(ClientSettingsSchema); const decodeServerSettings = Schema.decodeUnknownSync(ServerSettings); +const encodeProviderInstanceEnvironment = Schema.encodeSync(ProviderInstanceEnvironment); +const decodeProviderInstanceEnvironment = Schema.decodeSync(ProviderInstanceEnvironment); +// The environment variable shape builds released before secret references decode. +const decodeStringOnlyEnvironment = Schema.decodeUnknownSync( + Schema.Array( + Schema.Struct({ name: Schema.String, value: Schema.String, sensitive: Schema.Boolean }), + ), +); const decodeServerSettingsPatch = Schema.decodeUnknownSync(ServerSettingsPatch); const encodeServerSettings = Schema.encodeSync(ServerSettings); +const encodeServerSettingsPatch = Schema.encodeSync(ServerSettingsPatch); const decodeClaudeSettings = Schema.decodeUnknownSync(ClaudeSettings); describe("storage cleanup settings", () => { @@ -135,37 +148,6 @@ describe("ServerSettings secret providers", () => { }); }); - it("decodes a fork flat entry beside an upstream envelope and keeps every flat field", () => { - const openbao = { - kind: "openbao", - label: "Homelab", - address: "https://bao.example:8200", - mount: "secret", - auth: "token", - token: "s.secret", - }; - const aws = { kind: "aws-secrets-manager", label: "Prod", region: "us-east-1" }; - const flatOnePassword = { kind: "1password", label: "Home", account: "my.1password.com" }; - const settings = { - secretProviders: { - homelab: openbao, - prod: aws, - home: flatOnePassword, - personal: onePassword("my.1password.com"), - }, - }; - const decoded = decodeServerSettings(settings); - expect(decoded.secretProviders).toEqual(settings.secretProviders); - expect(encodeServerSettings(decoded).secretProviders).toEqual(settings.secretProviders); - expect(decodeServerSettingsPatch(settings).secretProviders).toEqual(settings.secretProviders); - }); - - it("never lets an invalid fork entry degrade into an envelope that drops its fields", () => { - const invalid = { kind: "openbao", label: "Homelab", address: "not a url" }; - expect(() => decodeServerSettings({ secretProviders: { homelab: invalid } })).toThrow(); - expect(() => decodeServerSettingsPatch({ secretProviders: { homelab: invalid } })).toThrow(); - }); - it("accepts slug ids and null removals but no id that is not a slug", () => { expect( decodeServerSettingsPatch({ @@ -184,14 +166,45 @@ describe("ServerSettings secret providers", () => { ).toThrow(); }); - it("rejects writes of an unknown kind or an account op could misread", () => { - const write = (entry: unknown) => () => - decodeServerSettingsPatch({ secretProviders: { personal: entry } }); - for (const account of ["", "my account", "--help"]) { - expect(write(onePassword(account))).toThrow(); + it("stores a provider environment secret reference in a layout older builds still decode", () => { + const providerInstances = (secret: unknown) => ({ + codex: { + driver: "codex", + environment: [ + { name: "OPENAI_API_KEY", sensitive: true, value: "", secret }, + { name: "BASE_URL", value: "https://example.test" }, + ], + }, + }); + const secret = { provider: "personal", reference: "op://Private/OpenAI/credential" }; + const environment = decodeServerSettings({ providerInstances: providerInstances(secret) }) + .providerInstances[ProviderInstanceId.make("codex")]?.environment; + expect(environment).toEqual([ + { name: "OPENAI_API_KEY", value: secret, sensitive: true }, + { name: "BASE_URL", value: "https://example.test", sensitive: false }, + ]); + + const encoded = encodeProviderInstanceEnvironment(environment ?? []); + expect(encoded).toEqual([ + { name: "OPENAI_API_KEY", value: "", sensitive: true, secret }, + { name: "BASE_URL", value: "https://example.test", sensitive: false }, + ]); + expect(decodeProviderInstanceEnvironment(encoded)).toEqual(environment); + expect(decodeStringOnlyEnvironment(encoded)).toEqual([ + { name: "OPENAI_API_KEY", value: "", sensitive: true }, + { name: "BASE_URL", value: "https://example.test", sensitive: false }, + ]); + + for (const invalid of [ + { provider: "My Vault", reference: "op://a/b/c" }, + { provider: "personal", reference: " " }, + { provider: "personal", reference: "op://Private/OpenAI\n/credential" }, + { provider: "personal", reference: "x".repeat(1025) }, + ]) { + expect(() => + decodeServerSettings({ providerInstances: providerInstances(invalid) }), + ).toThrow(); } - expect(write({ kind: "one_password", label: "Personal" })).toThrow(); - expect(write({ kind: "openbao", label: "Vault", config: { address: "https://x" } })).toThrow(); }); }); @@ -403,6 +416,27 @@ it("validates remote device hosts and rejects ambiguous host ids", () => { expect(() => decodeDeviceHostSettings({ deviceHosts: [{ ...host, port: 0 }] })).toThrow(); }); +describe("ServerSettings project overrides across versions", () => { + it("sends back an override key from a newer server when an older client edits the entry", () => { + const projectId = ProjectId.make("project-a"); + // As an older client reads a newer server's settings, with one key it does not know. + const entry = decodeServerSettings({ + projectSettingsOverrides: { + [projectId]: { defaultAutoPull: true, addedByNewerServer: { mode: "kept" } }, + }, + }).projectSettingsOverrides[projectId]; + + const sent = encodeServerSettingsPatch({ + projectSettingsOverrides: { [projectId]: { ...entry, defaultAutoPull: false } }, + }); + + expect(decodeServerSettingsPatch(sent).projectSettingsOverrides?.[projectId]).toEqual({ + defaultAutoPull: false, + addedByNewerServer: { mode: "kept" }, + }); + }); +}); + describe("resolveProjectProviderInstanceEnabled", () => { const projectId = "project-a" as ProjectId; const instanceId = ProviderInstanceId.make("codex_work"); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 182893a9cce02..b27bf725c8677 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -20,12 +20,8 @@ import { TrimmedString, } from "./baseSchemas.ts"; import { UsageLimitSourceId } from "./usageLimitSourceId.ts"; -import { - SecretProviderConfigPatch, - SettingsSecretValue, - StoredSecretProviderConfig, -} from "./secretProvider.ts"; -import { SecretProviderId } from "./secretProviderId.ts"; +import { SecretProviderId, SecretProviderSlug } from "./secretReference.ts"; +import { SettingsSecretValue } from "./settingsSecretValue.ts"; import { secretSetting } from "./settingsSecret.ts"; import { EnvironmentMachineKind, ThreadEnvMode, WorktreeSubmodules } from "./environment.ts"; import { KeybindingShortcut } from "./keybindings.ts"; @@ -849,6 +845,26 @@ export const UsageLimitSourceConfig = Schema.Struct({ }); export type UsageLimitSourceConfig = typeof UsageLimitSourceConfig.Type; +export const SecretProviderKind = SecretProviderSlug.pipe(Schema.brand("SecretProviderKind")); +export type SecretProviderKind = typeof SecretProviderKind.Type; + +export const SecretProviderLabel = TrimmedNonEmptyString.check(Schema.isMaxLength(64)); + +/** Kinds decode their own config; unknown-kind envelopes round-trip verbatim across versions. */ +export const SecretProviderConfig = Schema.Struct({ + kind: SecretProviderKind, + label: TrimmedNonEmptyString, + config: Schema.optionalKey(Schema.Unknown), +}); +export type SecretProviderConfig = typeof SecretProviderConfig.Type; + +// Limits that may change later are enforced on writes only, never on the stored envelope. The +// server also checks the kind and its config against the secret providers it has loaded. +const SecretProviderConfigWrite = Schema.Struct({ + ...SecretProviderConfig.fields, + label: SecretProviderLabel, +}); + export const SourceControlHostSettings = Schema.Record(Schema.String, Schema.Unknown); export type SourceControlHostSettings = typeof SourceControlHostSettings.Type; @@ -1057,6 +1073,16 @@ export const ProjectSettingsOverrides = Schema.Struct({ } satisfies Record); export type ProjectSettingsOverrides = typeof ProjectSettingsOverrides.Type; +// Whole-entry writers must round-trip unknown keys; keep the type closed to catch misspellings. +export const keepUnknownOverrideKeys = >( + entry: Entry, +): Schema.Codec< + Entry["Type"], + Entry["Encoded"], + Entry["DecodingServices"], + Entry["EncodingServices"] +> => Schema.StructWithRest(entry, [Schema.Record(Schema.String, Schema.Unknown)]); + /** True if null stores an override; otherwise null removes it. */ export function isNullableProjectSettingsOverride(key: ProjectScopedServerSettingKey): boolean { return NULLABLE_PROJECT_SETTINGS_OVERRIDES.has(key); @@ -1183,9 +1209,10 @@ export const ServerSettings = Schema.Struct({ Schema.withDecodingDefault(Effect.succeed(DEFAULT_RUNTIME_MODE)), ), - projectSettingsOverrides: Schema.Record(ProjectId, ProjectSettingsOverrides).pipe( - Schema.withDecodingDefault(Effect.succeed({})), - ), + projectSettingsOverrides: Schema.Record( + ProjectId, + keepUnknownOverrideKeys(ProjectSettingsOverrides), + ).pipe(Schema.withDecodingDefault(Effect.succeed({}))), /** Fold once so restarts cannot undo later UI resets. */ projectSettingsFolded: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), @@ -1304,7 +1331,7 @@ export const ServerSettings = Schema.Struct({ Schema.withDecodingDefault(Effect.succeed({})), ), /** Named secret stores that provider environment variables read from, keyed by a user-chosen id. */ - secretProviders: Schema.Record(SecretProviderId, StoredSecretProviderConfig).pipe( + secretProviders: Schema.Record(SecretProviderId, SecretProviderConfig).pipe( Schema.withDecodingDefault(Effect.succeed({})), ), cursorKeychainUsageEnabled: Schema.Boolean.pipe( @@ -1436,6 +1463,7 @@ export const ServerSettingsOperation = Schema.Literals([ "remove-secret", "remove-stale-secret", "write-secret", + "write-secret-provider", "write-file", "prepare-directory", ]); @@ -1547,10 +1575,12 @@ export const ServerSettingsPatch = Schema.Struct({ Schema.Record( ProjectId, Schema.NullOr( - Schema.Struct({ - ...ProjectSettingsOverrides.fields, - sourceControlWritingStyle: Schema.optionalKey(SourceControlWritingStyleSettings), - }), + keepUnknownOverrideKeys( + Schema.Struct({ + ...ProjectSettingsOverrides.fields, + sourceControlWritingStyle: Schema.optionalKey(SourceControlWritingStyleSettings), + }), + ), ), ), ), @@ -1622,7 +1652,7 @@ export const ServerSettingsPatch = Schema.Struct({ ), /** Per-entry like `usageLimitSources`; `null` removes a provider. */ secretProviders: Schema.optionalKey( - Schema.Record(SecretProviderId, Schema.NullOr(SecretProviderConfigPatch)), + Schema.Record(SecretProviderId, Schema.NullOr(SecretProviderConfigWrite)), ), cursorKeychainUsageEnabled: Schema.optionalKey(Schema.Boolean), diff --git a/packages/contracts/src/settingsSecretValue.ts b/packages/contracts/src/settingsSecretValue.ts new file mode 100644 index 0000000000000..897289d24ef15 --- /dev/null +++ b/packages/contracts/src/settingsSecretValue.ts @@ -0,0 +1,32 @@ +import * as Schema from "effect/Schema"; + +import { TrimmedNonEmptyString, TrimmedString } from "./baseSchemas.ts"; +import { SecretReference } from "./secretReference.ts"; + +/** + * A credential setting: the literal, which is kept in the secret store and redacted for + * clients, or the secret reference to read it from, which is not itself a secret and is + * stored and shown as written. + */ +export const SettingsSecretValue = Schema.Union([TrimmedString, SecretReference]); +export type SettingsSecretValue = typeof SettingsSecretValue.Type; + +export const isSecretReference = Schema.is(SecretReference); + +/** The literal a credential setting holds, or `""` when it names a secret instead. */ +export function settingsSecretLiteral(value: SettingsSecretValue | undefined): string { + return typeof value === "string" ? value : ""; +} + +export const SecretSourceCheckResult = Schema.Union([ + Schema.TaggedStruct("Resolved", {}), + Schema.TaggedStruct("Failed", { detail: Schema.String }), +]); +export type SecretSourceCheckResult = typeof SecretSourceCheckResult.Type; + +/** A 1Password account signed in on the server, as `op account list` reports it. */ +export const OnePasswordAccountSummary = Schema.Struct({ + account: TrimmedNonEmptyString, + email: Schema.String, +}); +export type OnePasswordAccountSummary = typeof OnePasswordAccountSummary.Type; diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index 398eac2170e70..c895abe3f5ef8 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -224,6 +224,7 @@ export interface AcpAdapterV2Flavor { readonly promptCapabilities: EffectAcpSchema.AgentCapabilities["promptCapabilities"]; }) => Effect.Effect, EffectAcpErrors.AcpError>; readonly promptFailure?: (cause: unknown) => OrchestrationV2ProviderFailure; + readonly requiresSignIn?: (cause: unknown) => boolean; /** * Runs before attachment paths, T3 instructions and native slash-command detection, * so a rewrite can turn the user's text into a native command. @@ -1323,6 +1324,8 @@ interface SnapshotMessageState { loadingIndex: number; } +/** How long a Stop waits for the agent to confirm before the user is offered Force stop. */ +const ACP_STOP_CONFIRM_TIMEOUT_MS = 10_000; const TOOL_UPDATE_PERSIST_EVERY = 10; // Agents can resend whole calls while streaming arguments. @@ -7988,7 +7991,9 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( return yield* Effect.uninterruptibleMask((restore) => Effect.gen(function* () { yield* restore(runtimeTransitionPermit.take(1)); - return yield* Effect.gen(function* () { + // Stop's caller times out on this call. The transition runs in its own fiber so + // a stalled hard teardown keeps its state and permit without holding the caller. + const transitionFiber = yield* Effect.gen(function* () { const transition = yield* Effect.gen(function* () { const interruptContext = yield* Ref.get(activeTurn); if ( @@ -7999,8 +8004,12 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ) { // ACP cancel leaves native children alive; close confirms session-wide Stop. pendingStopCloseTurn = interruptContext; - const closed = yield* runtime.closeSession().pipe( - Effect.timeoutOption("10 seconds"), + // Interruptible, or neither this timeout nor the caller's could end a hung close. + const closed = yield* restore( + runtime + .closeSession() + .pipe(Effect.timeoutOption(ACP_STOP_CONFIRM_TIMEOUT_MS)), + ).pipe( Effect.mapError( (cause) => new ProviderAdapter.ProviderAdapterProtocolError({ @@ -8012,10 +8021,11 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ), ); if (Option.isNone(closed)) { - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({ driver, - detail: - "The agent did not confirm session close; background work may still run", + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + timeoutMs: ACP_STOP_CONFIRM_TIMEOUT_MS, }); } yield* runtimeCallbackPermit.withPermit( @@ -8245,15 +8255,16 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( } } const stopped = yield* Deferred.await(context.completed).pipe( - Effect.timeoutOption("10 seconds"), + Effect.timeoutOption(ACP_STOP_CONFIRM_TIMEOUT_MS), ); + // The turn stays open: the agent may still be working, and the user + // decides whether to Force stop it. if (Option.isNone(stopped)) { - if (!context.finalized) { - yield* finalizeTurn(context, "interrupted"); - } - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({ driver, - detail: `ACP provider turn ${turnInput.providerTurnId} did not acknowledge cancellation before the interrupt timeout`, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + timeoutMs: ACP_STOP_CONFIRM_TIMEOUT_MS, }); } }); @@ -8274,20 +8285,29 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( }), ), ); - }).pipe(Effect.ensuring(runtimeTransitionPermit.release(1))); + }).pipe( + Effect.ensuring(runtimeTransitionPermit.release(1)), + Effect.forkIn(sessionScope, { startImmediately: true, uninterruptible: true }), + ); + return yield* restore(Fiber.join(transitionFiber)).pipe( + Effect.onInterrupt(() => + Fiber.interrupt(transitionFiber).pipe(Effect.forkDetach, Effect.asVoid), + ), + ); }), ); }, (effect, turnInput) => effect.pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterInterruptError({ - driver, - providerThreadId: turnInput.providerThread.id, - providerTurnId: turnInput.providerTurnId, - cause, - }), + Effect.mapError((cause) => + cause._tag === "ProviderAdapterInterruptTimeoutError" + ? cause + : new ProviderAdapter.ProviderAdapterInterruptError({ + driver, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + cause, + }), ), ), ), @@ -8555,6 +8575,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( new ProviderAdapter.ProviderAdapterOpenSessionError({ driver, providerSessionId: input.providerSessionId, + ...(flavor.requiresSignIn?.(cause) === true ? { requiresSignIn: true } : {}), cause, }), ), diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 9dc7ec1f62285..0b5459f647f8d 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -183,6 +183,7 @@ export class ProviderAdapterOpenSessionError extends Schema.TaggedError()( + "ProviderAdapterInterruptTimeoutError", + { + driver: ProviderDriverKind, + providerThreadId: ProviderThreadId, + providerTurnId: ProviderTurnId, + timeoutMs: Schema.Number, + }, +) { + override get message(): string { + return `The ${this.driver} provider did not confirm stopping turn ${this.providerTurnId} within ${this.timeoutMs} ms.`; + } +} + export class ProviderAdapterRuntimeRequestResponseError extends Schema.TaggedError()( "ProviderAdapterRuntimeRequestResponseError", { @@ -387,6 +403,7 @@ export const ProviderAdapterV2Error = Schema.Union([ ProviderAdapterSteerRunUnsupportedError, ProviderAdapterSteerRunError, ProviderAdapterInterruptError, + ProviderAdapterInterruptTimeoutError, ProviderAdapterRuntimeRequestResponseError, ProviderAdapterEventStreamError, ProviderAdapterProtocolError, diff --git a/packages/provider-core/src/server/driver.ts b/packages/provider-core/src/server/driver.ts index 72841df06e2e7..a1771e2e748e9 100644 --- a/packages/provider-core/src/server/driver.ts +++ b/packages/provider-core/src/server/driver.ts @@ -6,6 +6,7 @@ import type { AcpRegistryOperationError, AcpRegistrySetProviderInput, ProviderDriverKind, + ResolvedProviderInstanceEnvironment, ProviderInstanceId, ServerProvider, ServerProviderWorkspaceSnapshot, @@ -17,7 +18,6 @@ import type * as Scope from "effect/Scope"; import type * as ProviderAdapter from "./ProviderAdapter.ts"; import type { ProviderAuthController } from "./auth.ts"; import type { ProviderDriverError } from "./errors.ts"; -import type { ResolvedProviderEnvironment } from "./instanceEnvironment.ts"; import type { ManagedServerProvider } from "./snapshot.ts"; import type { ProviderTextGeneration } from "./textGeneration.ts"; import type { ProviderUsageReader } from "./usage.ts"; @@ -96,7 +96,7 @@ export interface ProviderDriverCreateInput { readonly instanceId: ProviderInstanceId; readonly displayName: string | undefined; readonly accentColor?: string | undefined; - readonly environment: ResolvedProviderEnvironment; + readonly environment: ResolvedProviderInstanceEnvironment; readonly enabled: boolean; readonly config: Config; } diff --git a/packages/provider-core/src/server/instanceEnvironment.test.ts b/packages/provider-core/src/server/instanceEnvironment.test.ts index c496f247e574c..d4c5a2361fc37 100644 --- a/packages/provider-core/src/server/instanceEnvironment.test.ts +++ b/packages/provider-core/src/server/instanceEnvironment.test.ts @@ -1,5 +1,6 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; +import { SecretProviderId } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Path from "effect/Path"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -71,12 +72,21 @@ describe("mergeProviderInstanceEnvironment", () => { [ { name: "OPENROUTER_API_KEY", value: "sk-or-test", sensitive: true }, { name: "ANTHROPIC_API_KEY", value: "", sensitive: false }, + { + name: "OPENAI_API_KEY", + value: { + provider: SecretProviderId.make("personal"), + reference: "op://Private/OpenAI/credential", + }, + sensitive: true, + }, ], - { ANTHROPIC_API_KEY: "inherited", PATH: "/bin" }, + { ANTHROPIC_API_KEY: "inherited", OPENAI_API_KEY: "inherited", PATH: "/bin" }, ), ).toMatchObject({ OPENROUTER_API_KEY: "sk-or-test", ANTHROPIC_API_KEY: "", + OPENAI_API_KEY: "inherited", PATH: "/bin", }); }), @@ -160,14 +170,7 @@ describe("literalProviderInstanceEnvironment", () => { const environment = decodeProviderInstanceEnvironment([ { name: "CODEX_HOME", value: "~/.codex-work" }, { name: "REFERENCE_LOOKALIKE", value: "op://Private/item/field" }, - { - name: "TOKEN", - value: { kind: "1password", reference: "op://Private/item/field", account: "my" }, - }, - { - name: "VAULT_TOKEN", - value: { kind: "secret-provider", provider: "team-vault", reference: "agents/codex#key" }, - }, + { name: "TOKEN", secret: { provider: "team-vault", reference: "agents/codex#key" } }, ]); expect(literalProviderInstanceEnvironment(environment)).toEqual([ diff --git a/packages/provider-core/src/server/instanceEnvironment.ts b/packages/provider-core/src/server/instanceEnvironment.ts index dd8a643474768..b322ad36c8b8d 100644 --- a/packages/provider-core/src/server/instanceEnvironment.ts +++ b/packages/provider-core/src/server/instanceEnvironment.ts @@ -1,23 +1,13 @@ import type { ProviderInstanceEnvironment, - ProviderInstanceEnvironmentVariableName, + ResolvedProviderInstanceEnvironment, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; import { expandHomePath } from "./pathExpansion.ts"; -/** - * A variable ready for a child process. Only `ProviderSecretResolver` turns a secret source - * into this shape, so a source cannot reach a process unresolved. Secret-store values are sensitive. - */ -export interface ResolvedProviderEnvironmentVariable { - readonly name: ProviderInstanceEnvironmentVariableName; - readonly value: string; - readonly sensitive: boolean; -} - -export type ResolvedProviderEnvironment = ReadonlyArray; +export type ResolvedProviderEnvironment = ResolvedProviderInstanceEnvironment; /** * The configured variables that hold literal values, for callers that build a @@ -27,7 +17,7 @@ export type ResolvedProviderEnvironment = ReadonlyArray = []; + const literals: Array = []; for (const { name, value, sensitive } of environment ?? []) { if (typeof value === "string") { literals.push({ name, value, sensitive }); @@ -94,7 +84,7 @@ const withLaunchedByAttribute = (environment: NodeJS.ProcessEnv): NodeJS.Process * inherited value instead of extending it. */ export const overlayProviderInstanceEnvironment = Effect.fn(function* ( - environment: ResolvedProviderEnvironment | undefined, + environment: ProviderInstanceEnvironment | ResolvedProviderEnvironment | undefined, baseEnv: NodeJS.ProcessEnv, ) { if (!environment || environment.length === 0) { @@ -104,6 +94,8 @@ export const overlayProviderInstanceEnvironment = Effect.fn(function* ( const home = yield* HostProcess.HomeDirectory; const next: NodeJS.ProcessEnv = { ...baseEnv }; for (const variable of environment) { + // An unresolved secret reference carries no value; the registry resolves it for the provider. + if (typeof variable.value !== "string") continue; next[variable.name] = variable.name === "CODEX_HOME" || variable.name === "CLAUDE_CONFIG_DIR" ? expandHomePath(variable.value, home) @@ -113,7 +105,7 @@ export const overlayProviderInstanceEnvironment = Effect.fn(function* ( }); export const mergeProviderInstanceEnvironment = Effect.fn(function* ( - environment: ResolvedProviderEnvironment | undefined, + environment: ProviderInstanceEnvironment | ResolvedProviderEnvironment | undefined, baseEnv: NodeJS.ProcessEnv = process.env, ) { return withLaunchedByAttribute(yield* overlayProviderInstanceEnvironment(environment, baseEnv)); diff --git a/packages/provider-core/src/server/maintenanceResolver.ts b/packages/provider-core/src/server/maintenanceResolver.ts index 2e3aad166c58e..8a8e11e2ec0d7 100644 --- a/packages/provider-core/src/server/maintenanceResolver.ts +++ b/packages/provider-core/src/server/maintenanceResolver.ts @@ -87,7 +87,9 @@ export interface PackageManagedProviderMaintenanceDefinition { readonly isCommandPath?: (commandPath: string) => boolean; /** The updater cannot detect installs outside `isCommandPath`, so leave those manual-only. */ readonly requiresCommandPath?: boolean; - readonly env?: NodeJS.ProcessEnv; + readonly env?: + | NodeJS.ProcessEnv + | ((context: ProviderMaintenanceResolutionContext) => NodeJS.ProcessEnv); } | null; } @@ -446,6 +448,7 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( } const nativeUpdate = definition.nativeUpdate; + const nativeEnv = nativeUpdate?.env; const native = nativeUpdate ? makeProviderMaintenanceCapabilities({ provider: definition.provider, @@ -454,7 +457,9 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( updateArgs: nativeUpdate.args, updateLockKey: `${definition.provider}-native`, platform: context.platform, - ...(nativeUpdate.env ? { env: nativeUpdate.env } : {}), + ...(nativeEnv + ? { env: typeof nativeEnv === "function" ? nativeEnv(context) : nativeEnv } + : {}), }) : manual; // A native-looking launcher can still delegate to a mise-owned install. diff --git a/packages/provider-opencode/src/server/adapter.test.ts b/packages/provider-opencode/src/server/adapter.test.ts index 41da6bd0ff435..52f7590e63f42 100644 --- a/packages/provider-opencode/src/server/adapter.test.ts +++ b/packages/provider-opencode/src/server/adapter.test.ts @@ -1,3 +1,8 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off - Exercise HTTP authentication with the real SDK. +import * as NodeHttp from "node:http"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as OpenCodeServerLedger from "./OpenCodeServerLedger.ts"; import { assert, describe, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import type { OpencodeClient, ToolPart } from "@opencode-ai/sdk/v2"; @@ -50,6 +55,7 @@ import { import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; const encodeUnknownJson = Schema.encodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); +const decodeOpenCodeSettings = Schema.decodeEffect(OpenCodeSettings); const OPEN_CODE_TEST_SETTINGS = Schema.decodeSync(OpenCodeSettings)({ serverUrl: "http://test.invalid", }); @@ -2671,3 +2677,123 @@ it.effect.each([false, true])( Effect.scoped, ), ); + +it.live.each([true, false])("opens an authenticated OpenCode session (external=%s)", (external) => + Effect.gen(function* () { + const password = external ? "settings-password" : "environment-password"; + const requests: string[] = []; + let eventConnected!: () => void; + const eventRequest = new Promise((resolve) => { + eventConnected = resolve; + }); + const server = yield* Effect.acquireRelease( + Effect.promise( + () => + new Promise>((resolve) => { + const server = NodeHttp.createServer((request, response) => { + if ( + request.headers.authorization !== + `Basic ${Buffer.from(`opencode:${password}`).toString("base64")}` + ) { + response.writeHead(401).end("Unauthorized"); + return; + } + const url = new URL(request.url!, "http://localhost"); + requests.push(url.pathname); + if (url.pathname === "/event") { + response.writeHead(200, { "Content-Type": "text/event-stream" }); + response.write(": connected\n\n"); + eventConnected(); + return; + } + response.writeHead(200, { "Content-Type": "application/json" }); + response.end( + JSON.stringify( + url.pathname.endsWith("/children") + ? [] + : url.pathname === "/session" + ? { id: "native-auth", time: { created: 1, updated: 1 } } + : { healthy: true, version: "1.14.19" }, + ), + ); + }); + server.listen(0, "127.0.0.1", () => resolve(server)); + }), + ), + (server) => + Effect.promise( + () => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + ), + ); + const address = server.address(); + assert.isNotNull(address); + if (!address || typeof address === "string") return; + const url = `http://127.0.0.1:${address.port}`; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + let managedStarts = 0; + const runtimeLayer = OpenCodeRuntime.layer.pipe( + Layer.provide( + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => { + assert.isTrue(ChildProcess.isStandardCommand(command)); + if (!ChildProcess.isStandardCommand(command)) return Effect.die("Unexpected pipeline"); + assert.include(command.args, "serve"); + assert.equal(command.options.env?.OPENCODE_SERVER_PASSWORD, password); + managedStarts += 1; + return spawner.spawn( + ChildProcess.make( + process.execPath, + [ + "-e", + `console.log("server listening on " + process.argv[1]); setInterval(() => {}, 1000)`, + url, + ], + command.options, + ), + ); + }), + ), + ), + Layer.provide(OpenCodeServerLedger.layerTest), + Layer.provideMerge(NodeServices.layer), + ); + const instanceId = ProviderInstanceId.make("opencode-auth"); + const adapter = yield* makeOpenCodeAdapterV2({ + instanceId, + settings: yield* decodeOpenCodeSettings({ + serverUrl: external ? url : "", + ...(external ? { serverPassword: password } : {}), + }), + environment: { OPENCODE_SERVER_PASSWORD: "environment-password" }, + }).pipe(Effect.provide(runtimeLayer)); + const session = yield* adapter.openSession({ + threadId: ThreadId.make("thread-opencode-auth"), + providerSessionId: ProviderSessionId.make("session-opencode-auth"), + modelSelection: { instanceId, model: "anthropic/claude-sonnet", options: [] }, + runtimePolicy: { runtimeMode: "full-access", interactionMode: "default", cwd: null }, + }); + yield* session.ensureThread({ + threadId: ThreadId.make("thread-opencode-auth"), + modelSelection: { instanceId, model: "anthropic/claude-sonnet", options: [] }, + runtimePolicy: { runtimeMode: "full-access", interactionMode: "default", cwd: null }, + }); + assert.include(requests, "/session"); + yield* Effect.promise(() => eventRequest); + assert.include(requests, "/event"); + assert.include(requests, "/global/health"); + assert.equal(managedStarts, external ? 0 : 1); + }).pipe( + Effect.scoped, + Effect.provide([ + NodeServices.layer, + TestProviderHost.layer().pipe(Layer.provide(NodeServices.layer)), + IdAllocator.layer, + McpProviderSessions.layer, + ]), + ), +); diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts index 2a851188d076e..7762c3b64503d 100644 --- a/packages/provider-opencode/src/server/adapter.ts +++ b/packages/provider-opencode/src/server/adapter.ts @@ -936,13 +936,16 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function binaryPath: options.settings.binaryPath, directory: cwd, serverUrl: options.settings.serverUrl, + ...(options.settings.serverPassword + ? { serverPassword: options.settings.serverPassword } + : {}), environment: options.environment, }); const client = runtime.createOpenCodeSdkClient({ baseUrl: connection.url, directory: cwd, - ...(connection.external && options.settings.serverPassword - ? { serverPassword: options.settings.serverPassword } + ...(connection.serverPassword !== undefined + ? { serverPassword: connection.serverPassword } : {}), }); diff --git a/packages/secret-provider-aws/package.json b/packages/secret-provider-aws/package.json new file mode 100644 index 0000000000000..01840a0b8999d --- /dev/null +++ b/packages/secret-provider-aws/package.json @@ -0,0 +1,32 @@ +{ + "name": "@t3tools/secret-provider-aws", + "private": true, + "type": "module", + "exports": { + "./shared/definition": { + "types": "./src/shared/definition.ts", + "import": "./src/shared/definition.ts" + }, + "./server/driver": { + "types": "./src/server/driver.ts", + "import": "./src/server/driver.ts" + } + }, + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vp test run --config ../../vite.config.ts --dir ." + }, + "dependencies": { + "@t3tools/contracts": "workspace:*", + "@t3tools/provider-core": "workspace:*", + "@t3tools/secret-provider-core": "workspace:*", + "@t3tools/shared": "workspace:*", + "effect": "catalog:" + }, + "devDependencies": { + "@effect/platform-node": "catalog:", + "@effect/vitest": "catalog:", + "@types/node": "catalog:", + "vite-plus": "catalog:" + } +} diff --git a/packages/secret-provider-aws/src/server/driver.test.ts b/packages/secret-provider-aws/src/server/driver.test.ts new file mode 100644 index 0000000000000..d2d3f7dceaf92 --- /dev/null +++ b/packages/secret-provider-aws/src/server/driver.test.ts @@ -0,0 +1,127 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeFileSystem from "@effect/platform-node/NodeFileSystem"; +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { ChildProcessSpawner } from "effect/process"; + +import { driver } from "./driver.ts"; + +const encoder = new TextEncoder(); +const config = { region: "us-east-1" }; + +function fakeAws( + answer: (args: ReadonlyArray) => { stdout?: string; stderr?: string; code?: number }, +) { + const invocations: Array> = []; + const layer = Layer.mergeAll( + NodeFileSystem.layer, + NodeCrypto.layer, + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.sync(() => { + const { args } = command as unknown as { readonly args: ReadonlyArray }; + invocations.push(args); + const result = answer(args); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result.code ?? 0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.make(encoder.encode(result.stdout ?? "")), + stderr: Stream.make(encoder.encode(result.stderr ?? "")), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ), + ), + ); + return { layer, invocations }; +} + +const body = (value: { SecretString?: string; SecretBinary?: string }) => + JSON.stringify({ ARN: "arn:aws:secretsmanager:us-east-1:000000000000:secret:prod", ...value }); + +describe("AWS Secrets Manager driver", () => { + it.effect("reads the whole secret string in the provider's region", () => { + const aws = fakeAws(() => ({ stdout: body({ SecretString: "sk-aws" }) })); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* driver.read(config, ["prod/openai"]), ["sk-aws"]); + assert.deepStrictEqual(aws.invocations, [ + [ + "secretsmanager", + "get-secret-value", + "--secret-id", + "prod/openai", + "--region", + "us-east-1", + "--output", + "json", + "--no-cli-pager", + "--no-cli-auto-prompt", + ], + ]); + }).pipe(Effect.provide(aws.layer)); + }); + + it.effect("passes the named profile", () => { + const aws = fakeAws(() => ({ stdout: body({ SecretString: "x" }) })); + return Effect.gen(function* () { + yield* driver.read({ ...config, profile: "work" }, ["prod/openai"]); + const args = aws.invocations[0]!; + assert.deepStrictEqual(args.slice(args.indexOf("--profile"), args.indexOf("--profile") + 2), [ + "--profile", + "work", + ]); + }).pipe(Effect.provide(aws.layer)); + }); + + it.effect("reads one key of a JSON secret and reports a missing key unreadable", () => { + const aws = fakeAws(() => ({ + stdout: body({ SecretString: JSON.stringify({ api_key: "sk", port: 8200, nested: {} }) }), + })); + return Effect.gen(function* () { + assert.deepStrictEqual( + yield* driver.read(config, [ + "prod/openai#api_key", + "prod/openai#port", + "prod/openai#nested", + "prod/openai#missing", + ]), + ["sk", "8200", undefined, undefined], + ); + }).pipe(Effect.provide(aws.layer)); + }); + + it.effect("reads a binary-only secret as undefined", () => { + const aws = fakeAws(() => ({ stdout: body({ SecretBinary: "AAEC" }) })); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* driver.read(config, ["prod/openai"]), [undefined]); + }).pipe(Effect.provide(aws.layer)); + }); + + it.effect("reads a failing aws as undefined", () => { + const aws = fakeAws(() => ({ code: 254, stderr: "ResourceNotFoundException" })); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* driver.read(config, ["prod/openai"]), [undefined]); + }).pipe(Effect.provide(aws.layer)); + }); + + it.effect("refuses a flag-shaped id and an empty key without spawning", () => { + const aws = fakeAws(() => ({ stdout: body({ SecretString: "x" }) })); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* driver.read(config, ["--profile", "prod#"]), [ + undefined, + undefined, + ]); + assert.strictEqual(aws.invocations.length, 0); + }).pipe(Effect.provide(aws.layer)); + }); +}); diff --git a/packages/secret-provider-aws/src/server/driver.ts b/packages/secret-provider-aws/src/server/driver.ts new file mode 100644 index 0000000000000..832c2aec1cba0 --- /dev/null +++ b/packages/secret-provider-aws/src/server/driver.ts @@ -0,0 +1,94 @@ +import { spawnAndCollect } from "@t3tools/provider-core/server/snapshotProbe"; +import { defineSecretProviderDriver } from "@t3tools/secret-provider-core/server/driver"; +import { resolveSpawnCommand } from "@t3tools/shared/shell"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { ChildProcess } from "effect/process"; + +import { + definition, + parseAwsReference, + type AwsSecretsManagerSecretProviderConfig, +} from "../shared/definition.ts"; + +const AWS_BINARY = "aws"; +const READ_TIMEOUT = Duration.seconds(45); + +const SecretValueJson = Schema.fromJsonString( + Schema.Struct({ + SecretString: Schema.optional(Schema.String), + SecretBinary: Schema.optional(Schema.String), + }), +); +const decodeSecretValue = Schema.decodeUnknownOption(SecretValueJson); +const decodeJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)); + +const nonEmpty = (value: string) => (value.length === 0 ? undefined : value); + +const readOne = Effect.fn("AwsSecretsManagerDriver.readOne")(function* ( + config: AwsSecretsManagerSecretProviderConfig, + reference: string, +) { + const location = parseAwsReference(reference); + if (location === undefined) return undefined; + const spawnCommand = yield* resolveSpawnCommand(AWS_BINARY, [ + "secretsmanager", + "get-secret-value", + "--secret-id", + location.secretId, + "--region", + config.region, + ...(config.profile === undefined ? [] : ["--profile", config.profile]), + "--output", + "json", + "--no-cli-pager", + "--no-cli-auto-prompt", + ]); + const result = yield* spawnAndCollect( + AWS_BINARY, + ChildProcess.make(spawnCommand.command, spawnCommand.args, { shell: spawnCommand.shell }), + ); + if (result.code !== 0) { + // `aws` reports credential and not-found failures on stderr and never echoes the secret. + yield* Effect.logWarning("Could not read secret from AWS Secrets Manager", { + exitCode: result.code, + detail: result.stderr.trim(), + }); + return undefined; + } + const body = Option.getOrUndefined(decodeSecretValue(result.stdout)); + if (body?.SecretString === undefined) { + yield* Effect.logWarning("AWS Secrets Manager returned no secret string"); + return undefined; + } + if (location.key === undefined) return nonEmpty(body.SecretString); + const parsed = Option.getOrUndefined(decodeJson(body.SecretString)); + const value = + typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) + ? Reflect.get(parsed, location.key) + : undefined; + if (typeof value === "string") return nonEmpty(value); + if (typeof value === "number" || typeof value === "boolean") return String(value); + yield* Effect.logWarning("AWS Secrets Manager secret has no scalar value for the key"); + return undefined; +}); + +export const driver = defineSecretProviderDriver({ + kind: definition.kind, + definition, + read: Effect.fn("AwsSecretsManagerDriver.read")(function* (config, references) { + return yield* Effect.forEach(references, (reference) => + readOne(config, reference).pipe( + Effect.timeoutOption(READ_TIMEOUT), + Effect.map(Option.getOrUndefined), + Effect.catch((error) => + Effect.logWarning("Could not run aws", { error: String(error) }).pipe( + Effect.as(undefined), + ), + ), + ), + ); + }), +}); diff --git a/packages/secret-provider-aws/src/shared/definition.test.ts b/packages/secret-provider-aws/src/shared/definition.test.ts new file mode 100644 index 0000000000000..ae4118d21d236 --- /dev/null +++ b/packages/secret-provider-aws/src/shared/definition.test.ts @@ -0,0 +1,29 @@ +import * as Schema from "effect/Schema"; +import { describe, expect, it } from "@effect/vitest"; + +import { AwsSecretsManagerSecretProviderConfig, definition } from "./definition.ts"; + +describe("AWS Secrets Manager definition", () => { + it("accepts a name or ARN with an optional JSON key", () => { + for (const reference of [ + "prod/openai", + "prod/openai#api_key", + "arn:aws:secretsmanager:us-east-1:000000000000:secret:prod", + ]) { + expect(definition.isReference(reference), reference).toBe(true); + } + }); + + it("rejects an empty id, an empty key, and a flag-shaped id", () => { + for (const reference of ["", "#key", "prod#", "--help", " prod", "prod\n"]) { + expect(definition.isReference(reference), reference).toBe(false); + } + }); + + it("validates region and profile", () => { + const decode = Schema.decodeUnknownExit(AwsSecretsManagerSecretProviderConfig); + expect(decode({ region: "us-east-1", profile: "work" })._tag).toBe("Success"); + expect(decode({ region: "nowhere" })._tag).toBe("Failure"); + expect(decode({ region: "us-east-1", profile: "-x" })._tag).toBe("Failure"); + }); +}); diff --git a/packages/secret-provider-aws/src/shared/definition.ts b/packages/secret-provider-aws/src/shared/definition.ts new file mode 100644 index 0000000000000..4a081fac15114 --- /dev/null +++ b/packages/secret-provider-aws/src/shared/definition.ts @@ -0,0 +1,68 @@ +import { SecretProviderKind, TrimmedNonEmptyString } from "@t3tools/contracts"; +import { defineSecretProvider } from "@t3tools/secret-provider-core/shared/definition"; +import * as Schema from "effect/Schema"; + +const AWS_REGION_PATTERN = /^[a-z]{2}(-[a-z]+)+-\d+$/; +const AWS_PROFILE_MAX_CHARS = 64; +const MAX_REFERENCE_LENGTH = 1024; + +function hasControlCharacter(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code < 0x20 || code === 0x7f) return true; + } + return false; +} + +/** Secrets Manager is regional, so one provider is one region. */ +export const AwsRegion = TrimmedNonEmptyString.check( + Schema.makeFilter((value: string) => + AWS_REGION_PATTERN.test(value) ? true : "AWS region must look like us-east-1.", + ), +); +export type AwsRegion = typeof AwsRegion.Type; + +/** A named profile from the server's AWS CLI configuration. It is passed to the CLI as an argument, so it can never start with `-`. */ +export const AwsProfile = TrimmedNonEmptyString.check( + Schema.isMaxLength(AWS_PROFILE_MAX_CHARS), + Schema.makeFilter((value: string) => + !value.startsWith("-") && !/\s/.test(value) && !hasControlCharacter(value) + ? true + : "AWS profile must not start with - or contain whitespace.", + ), +); +export type AwsProfile = typeof AwsProfile.Type; + +/** + * Credentials are never stored: the `aws` CLI on the server resolves them from the named profile + * when one is set, otherwise from its default chain. + */ +export const AwsSecretsManagerSecretProviderConfig = Schema.Struct({ + region: AwsRegion, + profile: Schema.optionalKey(AwsProfile), +}); +export type AwsSecretsManagerSecretProviderConfig = + typeof AwsSecretsManagerSecretProviderConfig.Type; + +/** `name-or-arn` or `name-or-arn#key`; the id follows `--secret-id`, so it can never start with `-`. A secret name has no `#`, so the first one starts the key. */ +export function parseAwsReference(reference: string) { + const separator = reference.indexOf("#"); + const secretId = separator === -1 ? reference : reference.slice(0, separator); + const key = separator === -1 ? undefined : reference.slice(separator + 1); + if (secretId.length === 0 || secretId.startsWith("-") || key === "") return undefined; + return { secretId, key }; +} + +export const definition = defineSecretProvider({ + kind: SecretProviderKind.make("aws_secrets_manager"), + label: "AWS Secrets Manager", + configSchema: AwsSecretsManagerSecretProviderConfig, + isReference: (reference) => + reference.length <= MAX_REFERENCE_LENGTH && + reference === reference.trim() && + !hasControlCharacter(reference) && + parseAwsReference(reference) !== undefined, + referencePlaceholder: "prod/openai#api_key", + invalidReferenceMessage: + "Use an AWS Secrets Manager reference like name-or-arn or name-or-arn#key.", +}); diff --git a/packages/secret-provider-aws/tsconfig.json b/packages/secret-provider-aws/tsconfig.json new file mode 100644 index 0000000000000..374bac55202dd --- /dev/null +++ b/packages/secret-provider-aws/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src"] +} diff --git a/packages/secret-provider-core/package.json b/packages/secret-provider-core/package.json new file mode 100644 index 0000000000000..3edea7a879b97 --- /dev/null +++ b/packages/secret-provider-core/package.json @@ -0,0 +1,33 @@ +{ + "name": "@t3tools/secret-provider-core", + "private": true, + "type": "module", + "exports": { + "./shared/definition": { + "types": "./src/shared/definition.ts", + "import": "./src/shared/definition.ts" + }, + "./server/driver": { + "types": "./src/server/driver.ts", + "import": "./src/server/driver.ts" + }, + "./server/SecretResolver": { + "types": "./src/server/SecretResolver.ts", + "import": "./src/server/SecretResolver.ts" + } + }, + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vp test run --config ../../vite.config.ts --dir ." + }, + "dependencies": { + "@t3tools/contracts": "workspace:*", + "effect": "catalog:" + }, + "devDependencies": { + "@effect/platform-node": "catalog:", + "@effect/vitest": "catalog:", + "@types/node": "catalog:", + "vite-plus": "catalog:" + } +} diff --git a/packages/secret-provider-core/src/server/SecretResolver.test.ts b/packages/secret-provider-core/src/server/SecretResolver.test.ts new file mode 100644 index 0000000000000..d3c44aa14eaf7 --- /dev/null +++ b/packages/secret-provider-core/src/server/SecretResolver.test.ts @@ -0,0 +1,170 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import { + SecretProviderId, + SecretProviderKind, + type SecretReference, + type ServerSettings, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import { defineSecretProviderDriver } from "./driver.ts"; +import * as SecretResolver from "./SecretResolver.ts"; +import { defineSecretProvider } from "../shared/definition.ts"; + +const FAKE_KIND = SecretProviderKind.make("fake"); +const FakeConfig = Schema.Struct({ account: Schema.String }); + +const personal = SecretProviderId.make("personal"); +const work = SecretProviderId.make("work"); +const fake = (account: string) => ({ + kind: FAKE_KIND, + label: account, + config: { account }, +}); +const ref = (reference: string, provider = personal): SecretReference => ({ provider, reference }); + +function fakeDriver(answer: (account: string, reference: string) => string | undefined) { + const reads: Array<{ readonly account: string; readonly references: ReadonlyArray }> = []; + const driver = defineSecretProviderDriver({ + kind: FAKE_KIND, + definition: defineSecretProvider({ + kind: FAKE_KIND, + label: "Fake", + configSchema: FakeConfig, + isReference: (reference) => reference.startsWith("fake://"), + referencePlaceholder: "fake://item", + invalidReferenceMessage: "Use fake://item.", + }), + read: ({ account }, references) => + Effect.sync(() => { + reads.push({ account, references }); + return references.map((reference) => answer(account, reference)); + }), + }); + return { driver, reads }; +} + +const resolverLayer = ( + driver: ReturnType["driver"], + secretProviders: () => ServerSettings["secretProviders"] = () => ({ [personal]: fake("a") }), +) => + SecretResolver.layer({ + drivers: [driver], + secretProviders: Effect.sync(secretProviders), + }).pipe(Layer.provide(NodeServices.layer)); + +describe("SecretResolver", () => { + it.effect("reads a reference once and holds it", () => { + const { driver, reads } = fakeDriver(() => "sk-live"); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + const token = ref("fake://openai"); + + assert.deepStrictEqual(yield* resolver.read([token]), ["sk-live"]); + assert.deepStrictEqual(yield* resolver.read([token]), ["sk-live"]); + assert.deepStrictEqual(reads, [{ account: "a", references: ["fake://openai"] }]); + }).pipe(Effect.provide(resolverLayer(driver))); + }); + + it.effect("holds a failed read until invalidated, then reads again", () => { + let locked = true; + const { driver, reads } = fakeDriver(() => (locked ? undefined : "sk-live")); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + const token = ref("fake://openai"); + + assert.deepStrictEqual(yield* resolver.read([token]), [undefined]); + locked = false; + assert.deepStrictEqual(yield* resolver.read([token]), [undefined]); + assert.strictEqual(reads.length, 1); + + yield* resolver.invalidate([token]); + assert.deepStrictEqual(yield* resolver.read([token]), ["sk-live"]); + assert.strictEqual(reads.length, 2); + }).pipe(Effect.provide(resolverLayer(driver))); + }); + + it.effect("invalidates every reference when none are named", () => { + const { driver, reads } = fakeDriver(() => "sk-live"); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + const token = ref("fake://openai"); + + yield* resolver.read([token]); + yield* resolver.invalidate(); + yield* resolver.read([token]); + assert.strictEqual(reads.length, 2); + }).pipe(Effect.provide(resolverLayer(driver))); + }); + + it.effect("reads each distinct reference of a provider in one call, in input order", () => { + const { driver, reads } = fakeDriver((_, reference) => reference.toUpperCase()); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + + const values = yield* resolver.read([ref("fake://a"), ref("fake://b"), ref("fake://a")]); + + assert.deepStrictEqual(values, ["FAKE://A", "FAKE://B", "FAKE://A"]); + assert.deepStrictEqual(reads, [{ account: "a", references: ["fake://a", "fake://b"] }]); + }).pipe(Effect.provide(resolverLayer(driver))); + }); + + it.effect("reads each provider separately", () => { + const { driver, reads } = fakeDriver((account, reference) => `${account}:${reference}`); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + + const values = yield* resolver.read([ref("fake://a"), ref("fake://a", work)]); + + assert.deepStrictEqual(values, ["a:fake://a", "b:fake://a"]); + assert.strictEqual(reads.length, 2); + }).pipe( + Effect.provide(resolverLayer(driver, () => ({ [personal]: fake("a"), [work]: fake("b") }))), + ); + }); + + it.effect("reads again when the provider's config changes", () => { + let account = "a"; + const { driver, reads } = fakeDriver((account, reference) => `${account}:${reference}`); + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + const token = ref("fake://openai"); + + assert.deepStrictEqual(yield* resolver.read([token]), ["a:fake://openai"]); + account = "b"; + assert.deepStrictEqual(yield* resolver.read([token]), ["b:fake://openai"]); + assert.deepStrictEqual( + reads.map((read) => read.account), + ["a", "b"], + ); + }).pipe(Effect.provide(resolverLayer(driver, () => ({ [personal]: fake(account) })))); + }); + + it.effect("does not call a driver for what it cannot read, and reads it later", () => { + const { driver, reads } = fakeDriver(() => "sk-live"); + const providers: ServerSettings["secretProviders"] = { + [personal]: fake("a"), + [SecretProviderId.make("vault")]: { + kind: SecretProviderKind.make("openbao"), + label: "Vault", + }, + [SecretProviderId.make("broken")]: { kind: FAKE_KIND, label: "Broken", config: {} }, + }; + return Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + + const values = yield* resolver.read([ + ref("fake://openai", SecretProviderId.make("missing")), + ref("fake://openai", SecretProviderId.make("vault")), + ref("fake://openai", SecretProviderId.make("broken")), + ref("openai"), + ]); + + assert.deepStrictEqual(values, [undefined, undefined, undefined, undefined]); + assert.strictEqual(reads.length, 0); + }).pipe(Effect.provide(resolverLayer(driver, () => providers))); + }); +}); diff --git a/packages/secret-provider-core/src/server/SecretResolver.ts b/packages/secret-provider-core/src/server/SecretResolver.ts new file mode 100644 index 0000000000000..28f5816f953b9 --- /dev/null +++ b/packages/secret-provider-core/src/server/SecretResolver.ts @@ -0,0 +1,125 @@ +// Reads secrets that settings point at through a configured secret provider. Each read, including +// a failed one, is held for the server's life so rebuilds never re-prompt an unlock; `invalidate` +// (the Settings refresh) is the only way to read again. +import type { SecretReference, ServerSettings } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +import type { SecretProviderDriver, SecretProviderDriverServices } from "./driver.ts"; + +export class SecretResolver extends Context.Service< + SecretResolver, + { + /** Values in input order; `undefined` marks a reference that could not be read. */ + readonly read: ( + references: ReadonlyArray, + ) => Effect.Effect>; + /** Forgets the given references, or every reference, so the next read asks the provider again. */ + readonly invalidate: (references?: ReadonlyArray) => Effect.Effect; + } +>()("@t3tools/secret-provider-core/server/SecretResolver") {} + +const cacheKey = (reference: SecretReference) => + `${reference.provider}\u0000${reference.reference}`; + +interface CachedSecret { + // A provider whose config changed, such as another account, reads again. + readonly config: string; + readonly value: string | undefined; +} + +const decodeConfigWith = Schema.decodeUnknownOption>; + +interface ReadablePlan { + readonly driver: SecretProviderDriver; + readonly config: unknown; + readonly configKey: string; +} + +export const make = Effect.fn("SecretResolver.make")(function* (options: { + readonly drivers: ReadonlyArray; + readonly secretProviders: Effect.Effect; +}) { + const context = yield* Effect.context(); + const driversByKind = new Map( + options.drivers.map((driver) => [ + driver.kind, + { driver, decodeConfig: decodeConfigWith(driver.definition.configSchema) }, + ]), + ); + const cache = new Map(); + const semaphore = yield* Semaphore.make(1); + + // Only a provider this build can read is cached; anything else fails fast and is checked again + // on the next read. + const planFor = ( + providers: ServerSettings["secretProviders"], + reference: SecretReference, + ): ReadablePlan | undefined => { + const provider = providers[reference.provider]; + const entry = provider === undefined ? undefined : driversByKind.get(provider.kind); + if (provider === undefined || entry === undefined) return undefined; + const { driver, decodeConfig } = entry; + if (!driver.definition.isReference(reference.reference)) return undefined; + const config = decodeConfig(provider.config); + return Option.isNone(config) + ? undefined + : { driver, config: config.value, configKey: JSON.stringify(config.value) }; + }; + + const read: SecretResolver["Service"]["read"] = Effect.fn("SecretResolver.read")( + function* (references) { + const providers = yield* options.secretProviders; + const plans = references.map((reference) => planFor(providers, reference)); + + const wanted = new Map< + string, + { readonly plan: ReadablePlan; readonly byReference: Map> } + >(); + references.forEach((reference, index) => { + const plan = plans[index]; + if (plan === undefined || cache.get(cacheKey(reference))?.config === plan.configKey) return; + const group = wanted.get(reference.provider) ?? { plan, byReference: new Map() }; + group.byReference.set(reference.reference, [ + ...(group.byReference.get(reference.reference) ?? []), + reference, + ]); + wanted.set(reference.provider, group); + }); + for (const { plan, byReference } of wanted.values()) { + const values = yield* plan.driver.read(plan.config, [...byReference.keys()]); + [...byReference.values()].forEach((sameReference, index) => { + for (const reference of sameReference) { + cache.set(cacheKey(reference), { config: plan.configKey, value: values[index] }); + } + }); + } + + return references.map((reference, index) => { + const cached = cache.get(cacheKey(reference)); + return plans[index] !== undefined && cached?.config === plans[index].configKey + ? cached.value + : undefined; + }); + }, + (effect) => semaphore.withPermits(1)(effect), + Effect.provideContext(context), + ); + + const invalidate: SecretResolver["Service"]["invalidate"] = (references) => + semaphore.withPermits(1)( + Effect.sync(() => { + if (references === undefined) cache.clear(); + else for (const reference of references) cache.delete(cacheKey(reference)); + }), + ); + + return SecretResolver.of({ read, invalidate }); +}); + +export const layer = (options: Parameters[0]) => + Layer.effect(SecretResolver, make(options)); diff --git a/packages/secret-provider-core/src/server/driver.ts b/packages/secret-provider-core/src/server/driver.ts new file mode 100644 index 0000000000000..85deb93d8938b --- /dev/null +++ b/packages/secret-provider-core/src/server/driver.ts @@ -0,0 +1,46 @@ +import type { SecretProviderKind } from "@t3tools/contracts"; +import type * as Crypto from "effect/Crypto"; +import * as Effect from "effect/Effect"; +import type * as FileSystem from "effect/FileSystem"; +import * as Schema from "effect/Schema"; +import type { ChildProcessSpawner } from "effect/process"; + +import type { SecretProviderDefinition } from "../shared/definition.ts"; + +/** What the server gives every driver. A driver builds anything else it needs itself. */ +export type SecretProviderDriverServices = + | ChildProcessSpawner.ChildProcessSpawner + | FileSystem.FileSystem + | Crypto.Crypto; + +export interface SecretProviderDriver { + readonly kind: SecretProviderKind; + readonly definition: SecretProviderDefinition; + /** + * Values in input order; `undefined` marks a reference that could not be read. `config` has + * decoded with the definition's `configSchema` and every reference passed its `isReference`. + */ + readonly read: ( + config: unknown, + references: ReadonlyArray, + ) => Effect.Effect, never, SecretProviderDriverServices>; +} + +export const defineSecretProviderDriver = (driver: { + readonly kind: SecretProviderKind; + readonly definition: SecretProviderDefinition; + readonly read: ( + config: Config, + references: ReadonlyArray, + ) => Effect.Effect, never, SecretProviderDriverServices>; +}): SecretProviderDriver => { + const isConfig = Schema.is(driver.definition.configSchema); + return { + kind: driver.kind, + definition: driver.definition, + read: (config, references) => + isConfig(config) + ? driver.read(config, references) + : Effect.succeed(references.map(() => undefined)), + }; +}; diff --git a/packages/secret-provider-core/src/shared/definition.ts b/packages/secret-provider-core/src/shared/definition.ts new file mode 100644 index 0000000000000..7f4320dd0c3ea --- /dev/null +++ b/packages/secret-provider-core/src/shared/definition.ts @@ -0,0 +1,35 @@ +import type { SecretProviderKind } from "@t3tools/contracts"; +import type * as Schema from "effect/Schema"; + +/** What clients and the server both need to know about one kind of secret provider. */ +export interface SecretProviderDefinition { + readonly kind: SecretProviderKind; + readonly label: string; + readonly configSchema: Schema.Codec; + /** Whether `reference` is syntax this kind can read. */ + readonly isReference: (reference: string) => boolean; + readonly referencePlaceholder: string; + readonly invalidReferenceMessage: string; +} + +export const defineSecretProvider = ( + definition: SecretProviderDefinition, +): SecretProviderDefinition => definition; + +export interface SecretProviderDefinitionRegistry { + readonly definitions: ReadonlyArray; + readonly find: (kind: string) => SecretProviderDefinition | undefined; +} + +export function makeSecretProviderDefinitionRegistry( + definitions: ReadonlyArray, +): SecretProviderDefinitionRegistry { + const byKind = new Map(); + for (const definition of definitions) { + if (byKind.has(definition.kind)) { + throw new Error(`Secret provider '${definition.kind}' is defined more than once.`); + } + byKind.set(definition.kind, definition); + } + return { definitions, find: (kind) => byKind.get(kind) }; +} diff --git a/packages/secret-provider-core/tsconfig.json b/packages/secret-provider-core/tsconfig.json new file mode 100644 index 0000000000000..374bac55202dd --- /dev/null +++ b/packages/secret-provider-core/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src"] +} diff --git a/packages/secret-provider-definitions/package.json b/packages/secret-provider-definitions/package.json new file mode 100644 index 0000000000000..388759b289856 --- /dev/null +++ b/packages/secret-provider-definitions/package.json @@ -0,0 +1,20 @@ +{ + "name": "@t3tools/secret-provider-definitions", + "private": true, + "type": "module", + "exports": { + "./registry": { + "types": "./src/registry.ts", + "import": "./src/registry.ts" + } + }, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@t3tools/secret-provider-aws": "workspace:*", + "@t3tools/secret-provider-core": "workspace:*", + "@t3tools/secret-provider-onepassword": "workspace:*", + "@t3tools/secret-provider-openbao": "workspace:*" + } +} diff --git a/packages/secret-provider-definitions/src/registry.ts b/packages/secret-provider-definitions/src/registry.ts new file mode 100644 index 0000000000000..d6379134a625d --- /dev/null +++ b/packages/secret-provider-definitions/src/registry.ts @@ -0,0 +1,10 @@ +import { makeSecretProviderDefinitionRegistry } from "@t3tools/secret-provider-core/shared/definition"; +import * as Aws from "@t3tools/secret-provider-aws/shared/definition"; +import * as OnePassword from "@t3tools/secret-provider-onepassword/shared/definition"; +import * as OpenBao from "@t3tools/secret-provider-openbao/shared/definition"; + +export const builtInSecretProviderDefinitions = makeSecretProviderDefinitionRegistry([ + OnePassword.definition, + OpenBao.definition, + Aws.definition, +]); diff --git a/packages/secret-provider-definitions/tsconfig.json b/packages/secret-provider-definitions/tsconfig.json new file mode 100644 index 0000000000000..564a5990051fd --- /dev/null +++ b/packages/secret-provider-definitions/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"] +} diff --git a/packages/secret-provider-onepassword/package.json b/packages/secret-provider-onepassword/package.json new file mode 100644 index 0000000000000..062495c90895f --- /dev/null +++ b/packages/secret-provider-onepassword/package.json @@ -0,0 +1,32 @@ +{ + "name": "@t3tools/secret-provider-onepassword", + "private": true, + "type": "module", + "exports": { + "./shared/definition": { + "types": "./src/shared/definition.ts", + "import": "./src/shared/definition.ts" + }, + "./server/driver": { + "types": "./src/server/driver.ts", + "import": "./src/server/driver.ts" + } + }, + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vp test run --config ../../vite.config.ts --dir ." + }, + "dependencies": { + "@t3tools/contracts": "workspace:*", + "@t3tools/provider-core": "workspace:*", + "@t3tools/secret-provider-core": "workspace:*", + "@t3tools/shared": "workspace:*", + "effect": "catalog:" + }, + "devDependencies": { + "@effect/platform-node": "catalog:", + "@effect/vitest": "catalog:", + "@types/node": "catalog:", + "vite-plus": "catalog:" + } +} diff --git a/packages/secret-provider-onepassword/src/server/driver.test.ts b/packages/secret-provider-onepassword/src/server/driver.test.ts new file mode 100644 index 0000000000000..f77ffe9144b06 --- /dev/null +++ b/packages/secret-provider-onepassword/src/server/driver.test.ts @@ -0,0 +1,189 @@ +// @effect-diagnostics nodeBuiltinImport:off -- the fake `op` reads the template the driver wrote. +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeFileSystem from "@effect/platform-node/NodeFileSystem"; +import { assert, describe, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; +import { ChildProcessSpawner } from "effect/process"; +import * as NodeFS from "node:fs"; + +import { OnePasswordAccount } from "../shared/definition.ts"; +import { driver, listAccounts } from "./driver.ts"; + +const encoder = new TextEncoder(); +const account = OnePasswordAccount.make("my.1password.com"); + +type OpResult = { + readonly stdout?: string; + readonly stderr?: string; + readonly code?: number; + readonly hang?: boolean; +}; + +function fakeOp(answer: (args: ReadonlyArray) => OpResult) { + const invocations: Array> = []; + const spawned = Deferred.makeUnsafe(); + const layer = Layer.mergeAll( + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.sync(() => { + const { args } = command as unknown as { readonly args: ReadonlyArray }; + invocations.push(args); + Deferred.doneUnsafe(spawned, Effect.void); + const result = answer(args); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: result.hang + ? Effect.never + : Effect.succeed(ChildProcessSpawner.ExitCode(result.code ?? 0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.make(encoder.encode(result.stdout ?? "")), + stderr: Stream.make(encoder.encode(result.stderr ?? "")), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ), + ), + NodeFileSystem.layer, + NodeCrypto.layer, + ); + return { layer, invocations, spawned }; +} + +// Answers `op inject` by substituting each `{{ reference }}` in the template file it was handed. +const injectFrom = + (secrets: Readonly>) => + (args: ReadonlyArray): OpResult => { + const template = NodeFS.readFileSync(args[args.indexOf("-i") + 1]!, "utf8"); + return { stdout: template.replace(/\{\{ (\S+) \}\}/g, (_, reference) => secrets[reference]!) }; + }; + +describe("1Password driver", () => { + it.effect("reads one reference from the account", () => { + const op = fakeOp(() => ({ stdout: "sk-live" })); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* driver.read({ account }, ["op://Private/OpenAI/credential"]), [ + "sk-live", + ]); + assert.deepStrictEqual(op.invocations, [ + [ + "--cache=false", + "read", + "--account=my.1password.com", + "--no-newline", + "op://Private/OpenAI/credential", + ], + ]); + }).pipe(Effect.provide(op.layer)); + }); + + it.effect("reads a failed reference as undefined", () => { + const op = fakeOp(() => ({ code: 1, stderr: "[ERROR] account is locked" })); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* driver.read({ account }, ["op://Private/OpenAI/credential"]), [ + undefined, + ]); + }).pipe(Effect.provide(op.layer)); + }); + + it.effect("reads every reference in a single call", () => { + const op = fakeOp( + injectFrom({ + "op://Private/OpenAI/credential": "sk-a", + "op://Private/Anthropic/key": " sk-b\n", + }), + ); + return Effect.gen(function* () { + const values = yield* driver.read({ account }, [ + "op://Private/OpenAI/credential", + "op://Private/Anthropic/key", + ]); + + assert.deepStrictEqual(values, ["sk-a", " sk-b\n"]); + assert.strictEqual(op.invocations.length, 1); + assert.deepStrictEqual(op.invocations[0]?.slice(0, 3), [ + "--cache=false", + "inject", + "--account=my.1password.com", + ]); + }).pipe(Effect.provide(op.layer)); + }); + + it.effect("falls back to one read at a time when the batch fails", () => { + const op = fakeOp((args) => + args.includes("inject") + ? { code: 1, stderr: '[ERROR] could not resolve "op://Private/Gone/key"' } + : args.at(-1) === "op://Private/Gone/key" + ? { code: 1, stderr: "[ERROR] item not found" } + : { stdout: "sk-a" }, + ); + return Effect.gen(function* () { + const values = yield* driver.read({ account }, [ + "op://Private/OpenAI/credential", + "op://Private/Gone/key", + ]); + + assert.deepStrictEqual(values, ["sk-a", undefined]); + assert.strictEqual(op.invocations.length, 3); + }).pipe(Effect.provide(op.layer)); + }); + + it.effect("does not read one at a time after the batch times out", () => { + const op = fakeOp(() => ({ hang: true })); + return Effect.gen(function* () { + const reading = yield* driver + .read({ account }, ["op://Private/OpenAI/credential", "op://Private/Anthropic/key"]) + .pipe(Effect.forkChild); + yield* Deferred.await(op.spawned); + yield* TestClock.adjust("1 minute"); + + assert.deepStrictEqual(yield* Fiber.join(reading), [undefined, undefined]); + assert.strictEqual(op.invocations.length, 1); + }).pipe(Effect.provide(op.layer)); + }); +}); + +describe("1Password listAccounts", () => { + it.effect("offers each signed-in account by the address `op --account` accepts", () => { + const op = fakeOp(() => ({ + stdout: JSON.stringify([ + { url: "my.1password.com", email: "me@example.com", user_uuid: "U1" }, + { url: "acme.1password.com", email: "me@acme.example", user_uuid: "U2" }, + ]), + })); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* listAccounts(), [ + { account: "my.1password.com", email: "me@example.com" }, + { account: "acme.1password.com", email: "me@acme.example" }, + ]); + assert.deepStrictEqual(op.invocations, [ + ["--cache=false", "account", "list", "--format", "json"], + ]); + }).pipe(Effect.provide(op.layer)); + }); + + it.effect("offers no accounts when `op` fails", () => { + const op = fakeOp(() => ({ code: 1, stderr: "not configured" })); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* listAccounts(), []); + }).pipe(Effect.provide(op.layer)); + }); + + it.effect("offers no accounts when `op` prints something unexpected", () => { + const op = fakeOp(() => ({ stdout: "not json" })); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* listAccounts(), []); + }).pipe(Effect.provide(op.layer)); + }); +}); diff --git a/packages/secret-provider-onepassword/src/server/driver.ts b/packages/secret-provider-onepassword/src/server/driver.ts new file mode 100644 index 0000000000000..fd442b543a540 --- /dev/null +++ b/packages/secret-provider-onepassword/src/server/driver.ts @@ -0,0 +1,137 @@ +import { spawnAndCollect } from "@t3tools/provider-core/server/snapshotProbe"; +import { defineSecretProviderDriver } from "@t3tools/secret-provider-core/server/driver"; +import { resolveSpawnCommand } from "@t3tools/shared/shell"; +import * as Crypto from "effect/Crypto"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import { ChildProcess } from "effect/process"; + +import type { OnePasswordAccountSummary } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; + +import { definition, OnePasswordAccount } from "../shared/definition.ts"; + +const ONE_PASSWORD_BINARY = "op"; +// Bounds a read long enough to reach for a fingerprint reader, short enough not to wedge a rebuild. +const READ_TIMEOUT = Duration.seconds(45); + +// `--cache=false` skips the `op daemon`, which hangs on prompts a background server cannot show. +const runOnePassword = Effect.fn("OnePasswordDriver.runOnePassword")(function* ( + args: ReadonlyArray, +) { + const spawnCommand = yield* resolveSpawnCommand(ONE_PASSWORD_BINARY, ["--cache=false", ...args]); + return yield* spawnAndCollect( + ONE_PASSWORD_BINARY, + ChildProcess.make(spawnCommand.command, spawnCommand.args, { shell: spawnCommand.shell }), + ); +}); + +// A secret is passed on verbatim; surrounding whitespace can be part of it. +const nonEmpty = (value: string) => (value.trim().length > 0 ? value : undefined); + +const readOne = Effect.fn("OnePasswordDriver.readOne")(function* ( + account: OnePasswordAccount, + reference: string, +) { + const result = yield* runOnePassword(["read", `--account=${account}`, "--no-newline", reference]); + if (result.code !== 0) { + yield* Effect.logWarning("Could not read secret from 1Password", { + stage: "read", + exitCode: result.code, + }); + return undefined; + } + return nonEmpty(result.stdout); +}); + +// One `op inject` costs one unlock for every reference in the template. `op` ignores piped stdin +// from Node, so the template goes through a temp file. A random separator after every reference +// keeps one secret from shifting the others, and leaves any trailing newline outside the values. +const readTogether = Effect.fn("OnePasswordDriver.readTogether")(function* ( + account: OnePasswordAccount, + references: ReadonlyArray, +) { + const fileSystem = yield* FileSystem.FileSystem; + const separator = `__t3-secret-${yield* (yield* Crypto.Crypto).randomUUIDv4()}__`; + const templatePath = yield* fileSystem.makeTempFileScoped({ prefix: "t3code-op-inject-" }); + yield* fileSystem.writeFileString( + templatePath, + references.map((reference) => `{{ ${reference} }}${separator}`).join(""), + ); + const result = yield* runOnePassword(["inject", `--account=${account}`, "-i", templatePath]); + const values = result.stdout.split(separator); + const trailing = values.pop(); + if ( + result.code !== 0 || + values.length !== references.length || + trailing === undefined || + trailing.trim().length > 0 + ) { + yield* Effect.logWarning("Could not batch-read secrets from 1Password", { + stage: "inject", + exitCode: result.code, + references: references.length, + }); + return undefined; + } + return values.map(nonEmpty); +}, Effect.scoped); + +/** `None` when 1Password did not answer in time; a failed run reads as `Some(undefined)`. */ +const bounded = ( + stage: "read" | "inject", + effect: Effect.Effect, +) => + effect.pipe( + Effect.timeoutOption(READ_TIMEOUT), + Effect.tap((outcome) => + Option.isNone(outcome) + ? Effect.logWarning("1Password did not answer in time", { stage }) + : Effect.void, + ), + Effect.catch((error) => + Effect.logWarning("Could not run 1Password", { stage, error: error._tag }).pipe( + Effect.as(Option.some(undefined)), + ), + ), + ); + +export const driver = defineSecretProviderDriver({ + kind: definition.kind, + definition, + read: Effect.fn("OnePasswordDriver.read")(function* ({ account }, references) { + if (references.length > 1) { + const together = yield* bounded("inject", readTogether(account, references)); + // Reading one by one after a timeout would only wait that long again for each reference. + if (Option.isNone(together)) return references.map(() => undefined); + if (together.value !== undefined) return together.value; + } + return yield* Effect.forEach(references, (reference) => + bounded("read", readOne(account, reference)).pipe(Effect.map(Option.getOrUndefined)), + ); + }), +}); + +const AccountListJson = Schema.fromJsonString( + Schema.Array(Schema.Struct({ url: OnePasswordAccount, email: Schema.String })), +); +const decodeAccountList = Schema.decodeUnknownOption(AccountListJson); + +/** Signed-in accounts by the address `op --account` accepts; empty when `op` cannot answer. */ +export const listAccounts = Effect.fn("OnePasswordDriver.listAccounts")(function* () { + const result = yield* runOnePassword(["account", "list", "--format", "json"]).pipe( + Effect.timeoutOption(READ_TIMEOUT), + Effect.orElseSucceed(() => Option.none()), + ); + if (Option.isNone(result) || result.value.code !== 0) { + yield* Effect.logWarning("Could not list 1Password accounts"); + return []; + } + return Option.match(decodeAccountList(result.value.stdout), { + onNone: (): ReadonlyArray => [], + onSome: (accounts) => + accounts.map(({ url, email }): OnePasswordAccountSummary => ({ account: url, email })), + }); +}); diff --git a/packages/secret-provider-onepassword/src/shared/definition.test.ts b/packages/secret-provider-onepassword/src/shared/definition.test.ts new file mode 100644 index 0000000000000..0daefcf6acca1 --- /dev/null +++ b/packages/secret-provider-onepassword/src/shared/definition.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { definition } from "./definition.ts"; + +describe("1Password definition", () => { + it("accepts vault, item, optional section, and field references", () => { + for (const reference of [ + "op://Private/OpenAI/credential", + "op://Private/OpenAI/Section 1/credential", + "op://Private/OpenAI/credential?attribute=otp", + ]) { + expect(definition.isReference(reference), reference).toBe(true); + } + }); + + it("rejects anything that is not a plain reference", () => { + for (const reference of [ + "--help", + "Private/OpenAI/credential", + "op://Private/OpenAI", + "op://Private/OpenAI/credential }} {{ op://Private/B/c", + "op://Private/OpenAI\n/credential", + `op://Private/OpenAI/${"x".repeat(1024)}`, + ]) { + expect(definition.isReference(reference), reference).toBe(false); + } + }); +}); diff --git a/packages/secret-provider-onepassword/src/shared/definition.ts b/packages/secret-provider-onepassword/src/shared/definition.ts new file mode 100644 index 0000000000000..ff5164b1e00b2 --- /dev/null +++ b/packages/secret-provider-onepassword/src/shared/definition.ts @@ -0,0 +1,41 @@ +import { SecretProviderKind } from "@t3tools/contracts"; +import { defineSecretProvider } from "@t3tools/secret-provider-core/shared/definition"; +import * as Schema from "effect/Schema"; + +const ACCOUNT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +/** + * Any value `op --account` accepts: shorthand, sign-in address, account ID, or user ID. + * It is passed to the CLI as an argument, so it can never start with `-`. + */ +export const OnePasswordAccount = Schema.String.check( + Schema.isMaxLength(253), + Schema.makeFilter((value: string) => + ACCOUNT_PATTERN.test(value) + ? true + : "1Password account must be a shorthand, sign-in address, or ID with no spaces.", + ), +).pipe(Schema.brand("OnePasswordAccount")); +export type OnePasswordAccount = typeof OnePasswordAccount.Type; + +export const OnePasswordSecretProviderConfig = Schema.Struct({ + account: OnePasswordAccount, +}); +export type OnePasswordSecretProviderConfig = typeof OnePasswordSecretProviderConfig.Type; + +// 1Password secret reference syntax: names outside this charset must be written as item ids. +// It also keeps `{{`, `}}`, and line breaks out of the `op inject` template the server builds. +const NAME = String.raw`[A-Za-z0-9._-](?:[A-Za-z0-9 ._-]*[A-Za-z0-9._-])?`; +const REFERENCE = new RegExp( + String.raw`^op://${NAME}(?:/${NAME}){2,3}(?:\?[a-z-]+=[A-Za-z0-9_-]+)?$`, +); +const MAX_REFERENCE_LENGTH = 1024; + +export const definition = defineSecretProvider({ + kind: SecretProviderKind.make("one_password"), + label: "1Password", + configSchema: OnePasswordSecretProviderConfig, + isReference: (reference) => reference.length <= MAX_REFERENCE_LENGTH && REFERENCE.test(reference), + referencePlaceholder: "op://vault/item/field", + invalidReferenceMessage: "Use a 1Password reference like op://vault/item/field.", +}); diff --git a/packages/secret-provider-onepassword/tsconfig.json b/packages/secret-provider-onepassword/tsconfig.json new file mode 100644 index 0000000000000..374bac55202dd --- /dev/null +++ b/packages/secret-provider-onepassword/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src"] +} diff --git a/packages/secret-provider-openbao/package.json b/packages/secret-provider-openbao/package.json new file mode 100644 index 0000000000000..bade0d47ff74b --- /dev/null +++ b/packages/secret-provider-openbao/package.json @@ -0,0 +1,32 @@ +{ + "name": "@t3tools/secret-provider-openbao", + "private": true, + "type": "module", + "exports": { + "./shared/definition": { + "types": "./src/shared/definition.ts", + "import": "./src/shared/definition.ts" + }, + "./server/driver": { + "types": "./src/server/driver.ts", + "import": "./src/server/driver.ts" + } + }, + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vp test run --config ../../vite.config.ts --dir ." + }, + "dependencies": { + "@t3tools/contracts": "workspace:*", + "@t3tools/provider-core": "workspace:*", + "@t3tools/secret-provider-core": "workspace:*", + "@t3tools/shared": "workspace:*", + "effect": "catalog:" + }, + "devDependencies": { + "@effect/platform-node": "catalog:", + "@effect/vitest": "catalog:", + "@types/node": "catalog:", + "vite-plus": "catalog:" + } +} diff --git a/packages/secret-provider-openbao/src/server/driver.test.ts b/packages/secret-provider-openbao/src/server/driver.test.ts new file mode 100644 index 0000000000000..793a6318685b8 --- /dev/null +++ b/packages/secret-provider-openbao/src/server/driver.test.ts @@ -0,0 +1,179 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeFileSystem from "@effect/platform-node/NodeFileSystem"; +import { assert, describe, it } from "@effect/vitest"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientResponse } from "effect/http"; +import { ChildProcessSpawner } from "effect/process"; + +import { makeDriver } from "./driver.ts"; + +const encoder = new TextEncoder(); +const ADDRESS = "https://bao.example"; + +function fakeBao(result: { stdout?: string; stderr?: string; code?: number } = {}) { + const commands: Array = []; + const layer = Layer.mergeAll( + NodeFileSystem.layer, + NodeCrypto.layer, + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.sync(() => { + commands.push((command as unknown as { command: string }).command); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result.code ?? 0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.make(encoder.encode(result.stdout ?? "")), + stderr: Stream.make(encoder.encode(result.stderr ?? "")), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ), + ), + ); + return { layer, commands }; +} + +function fakeHttp(status: number, body: unknown) { + const requests: Array<{ url: string; headers: Record }> = []; + const layer = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.sync(() => { + requests.push({ url: request.url, headers: { ...request.headers } }); + return HttpClientResponse.fromWeb(request, new Response(JSON.stringify(body), { status })); + }), + ), + ); + return { layer, requests }; +} + +const kv = { data: { data: { api_key: "sk-api", port: 8200, nested: {} } } }; +const base = { address: ADDRESS, mount: "kv", auth: "token" as const, token: "s.configured" }; + +describe("OpenBao driver", () => { + it.effect("reads the KV v2 field with the stored token and namespace", () => { + const http = fakeHttp(200, kv); + const driver = makeDriver(http.layer); + return Effect.gen(function* () { + assert.deepStrictEqual( + yield* driver.read({ ...base, namespace: "team" }, [ + "agents/codex#api_key", + "agents/codex#port", + ]), + ["sk-api", "8200"], + ); + assert.deepStrictEqual(http.requests[0], { + url: `${ADDRESS}/v1/kv/data/agents/codex`, + headers: { "x-vault-token": "s.configured", "x-vault-namespace": "team" }, + }); + }).pipe(Effect.provide(fakeBao().layer)); + }); + + it.effect("joins a trailing slash address and encodes unsafe path characters", () => { + const http = fakeHttp(200, kv); + const driver = makeDriver(http.layer); + return Effect.gen(function* () { + yield* driver.read({ ...base, address: `${ADDRESS}/` }, ["my agents/co?dex#api_key"]); + assert.strictEqual(http.requests[0]?.url, `${ADDRESS}/v1/kv/data/my%20agents/co%3Fdex`); + }).pipe(Effect.provide(fakeBao().layer)); + }); + + it.effect("reads nothing without a request when the stored token is missing", () => { + const http = fakeHttp(200, kv); + const driver = makeDriver(http.layer); + return Effect.gen(function* () { + assert.deepStrictEqual( + yield* driver.read({ address: ADDRESS, mount: "kv", auth: "token" }, ["a#b"]), + [undefined], + ); + assert.strictEqual(http.requests.length, 0); + }).pipe(Effect.provide(fakeBao().layer)); + }); + + it.effect("uses BAO_TOKEN from the server environment when auth is env", () => { + const http = fakeHttp(200, kv); + const driver = makeDriver(http.layer); + return Effect.gen(function* () { + assert.deepStrictEqual( + yield* driver.read({ address: ADDRESS, mount: "kv", auth: "env" }, [ + "agents/codex#api_key", + ]), + ["sk-api"], + ); + assert.strictEqual(http.requests[0]?.headers["x-vault-token"], "s.env"); + }).pipe( + Effect.provideService(HostProcess.Environment, { BAO_TOKEN: "s.env" }), + Effect.provide(fakeBao().layer), + ); + }); + + it.effect("reads nothing when auth is env and BAO_TOKEN is unset", () => { + const http = fakeHttp(200, kv); + const driver = makeDriver(http.layer); + return Effect.gen(function* () { + assert.deepStrictEqual( + yield* driver.read({ address: ADDRESS, mount: "kv", auth: "env" }, ["a#b"]), + [undefined], + ); + assert.strictEqual(http.requests.length, 0); + }).pipe(Effect.provideService(HostProcess.Environment, {}), Effect.provide(fakeBao().layer)); + }); + + it.effect("uses the output of `bao print token` when auth is cli", () => { + const http = fakeHttp(200, kv); + const bao = fakeBao({ stdout: "s.from-cli\n" }); + const driver = makeDriver(http.layer); + return Effect.gen(function* () { + assert.deepStrictEqual( + yield* driver.read({ address: ADDRESS, mount: "kv", auth: "cli" }, [ + "agents/codex#api_key", + ]), + ["sk-api"], + ); + assert.deepStrictEqual(bao.commands, ["bao"]); + assert.strictEqual(http.requests[0]?.headers["x-vault-token"], "s.from-cli"); + }).pipe(Effect.provide(bao.layer)); + }); + + it.effect("reads nothing when `bao print token` fails", () => { + const http = fakeHttp(200, kv); + const driver = makeDriver(http.layer); + return Effect.gen(function* () { + assert.deepStrictEqual( + yield* driver.read({ address: ADDRESS, mount: "kv", auth: "cli" }, ["a#b"]), + [undefined], + ); + assert.strictEqual(http.requests.length, 0); + }).pipe(Effect.provide(fakeBao({ code: 2, stderr: "no token" }).layer)); + }); + + it.effect("reads a 403 as undefined", () => { + const http = fakeHttp(403, { errors: ["permission denied"] }); + const driver = makeDriver(http.layer); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* driver.read(base, ["agents/codex#api_key"]), [undefined]); + }).pipe(Effect.provide(fakeBao().layer)); + }); + + it.effect("reads a missing or non-scalar field as undefined", () => { + const http = fakeHttp(200, kv); + const driver = makeDriver(http.layer); + return Effect.gen(function* () { + assert.deepStrictEqual( + yield* driver.read(base, ["agents/codex#missing", "agents/codex#nested"]), + [undefined, undefined], + ); + }).pipe(Effect.provide(fakeBao().layer)); + }); +}); diff --git a/packages/secret-provider-openbao/src/server/driver.ts b/packages/secret-provider-openbao/src/server/driver.ts new file mode 100644 index 0000000000000..f962ff8d2785f --- /dev/null +++ b/packages/secret-provider-openbao/src/server/driver.ts @@ -0,0 +1,157 @@ +import { spawnAndCollect } from "@t3tools/provider-core/server/snapshotProbe"; +import { defineSecretProviderDriver } from "@t3tools/secret-provider-core/server/driver"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { resolveSpawnCommand } from "@t3tools/shared/shell"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; +import { ChildProcess } from "effect/process"; + +import { + definition, + parseOpenBaoReference, + type OpenBaoSecretProviderConfig, +} from "../shared/definition.ts"; + +const OPENBAO_BINARY = "bao"; +// OpenBao never prompts, so an unreachable address should not hold a rebuild for long. +const REQUEST_TIMEOUT = Duration.seconds(10); + +const SecretBody = Schema.Struct({ + data: Schema.Struct({ data: Schema.Record(Schema.String, Schema.Unknown) }), +}); +const decodeSecretBody = Schema.decodeUnknownOption(SecretBody); +const ErrorBody = Schema.Struct({ errors: Schema.optional(Schema.Array(Schema.String)) }); +const decodeErrorBody = Schema.decodeUnknownOption(ErrorBody); + +const nonEmpty = (value: string | undefined) => + value === undefined || value.length === 0 ? undefined : value; + +// `cli` asks `bao print token` with token variables withheld so it reports its own login. +const resolveToken = Effect.fn("OpenBaoDriver.resolveToken")(function* ( + config: OpenBaoSecretProviderConfig, +) { + switch (config.auth) { + case "token": + return nonEmpty(config.token); + case "env": + return nonEmpty((yield* HostProcess.Environment).BAO_TOKEN); + case "cli": { + const spawnCommand = yield* resolveSpawnCommand(OPENBAO_BINARY, ["print", "token"]); + const result = yield* spawnAndCollect( + OPENBAO_BINARY, + ChildProcess.make(spawnCommand.command, spawnCommand.args, { + shell: spawnCommand.shell, + env: { + BAO_ADDR: config.address, + ...(config.namespace === undefined ? {} : { BAO_NAMESPACE: config.namespace }), + // `bao` prefers these over its saved login; Node drops keys whose value is undefined. + BAO_TOKEN: undefined, + VAULT_TOKEN: undefined, + }, + extendEnv: true, + }), + ); + if (result.code !== 0) { + yield* Effect.logWarning("Could not read an OpenBao token from the bao CLI", { + exitCode: result.code, + detail: result.stderr.trim(), + }); + return undefined; + } + return nonEmpty(result.stdout.trim()); + } + } +}); + +const encodePath = (path: string) => path.split("/").map(encodeURIComponent).join("/"); + +const readOne = Effect.fn("OpenBaoDriver.readOne")(function* ( + config: OpenBaoSecretProviderConfig, + token: string, + reference: string, +) { + const location = parseOpenBaoReference(reference); + if (location === undefined) return undefined; + const client = yield* HttpClient.HttpClient; + const url = `${config.address.replace(/\/+$/, "")}/v1/${encodePath(config.mount)}/data/${encodePath(location.path)}`; + const request = HttpClientRequest.get(url).pipe( + HttpClientRequest.setHeaders({ + "X-Vault-Token": token, + ...(config.namespace === undefined ? {} : { "X-Vault-Namespace": config.namespace }), + }), + ); + // Trace headers would tell the vault which of our spans made the request. + const response = yield* client + .execute(request) + .pipe( + Effect.provideService(HttpClient.TracerPropagationEnabled, false), + Effect.timeout(REQUEST_TIMEOUT), + ); + const body: unknown = yield* response.json.pipe(Effect.orElseSucceed(() => undefined)); + if (response.status < 200 || response.status >= 300) { + // OpenBao's `errors` array names the failure and never carries a secret. + const errors = decodeErrorBody(body).pipe( + Option.flatMap((decoded) => Option.fromNullishOr(decoded.errors)), + Option.getOrElse((): ReadonlyArray => []), + ); + yield* Effect.logWarning("Could not read secret from OpenBao", { + status: response.status, + errorCount: errors.length, + }); + return undefined; + } + const value = decodeSecretBody(body).pipe( + Option.map((decoded) => decoded.data.data[location.field]), + Option.getOrUndefined, + ); + const secret = + typeof value === "string" + ? value + : typeof value === "number" || typeof value === "boolean" + ? String(value) + : undefined; + if (secret === undefined || secret.length === 0) { + yield* Effect.logWarning("OpenBao secret has no usable value for the requested field"); + return undefined; + } + return secret; +}); + +export const makeDriver = ( + httpClient: Layer.Layer = FetchHttpClient.layer, +) => + defineSecretProviderDriver({ + kind: definition.kind, + definition, + read: Effect.fn("OpenBaoDriver.read")(function* (config, references) { + const token = yield* resolveToken(config).pipe( + Effect.catch((error) => + Effect.logWarning("Could not run bao", { detail: String(error) }).pipe( + Effect.as(undefined), + ), + ), + ); + if (token === undefined) { + yield* Effect.logWarning("OpenBao provider has no token to read with", { + auth: config.auth, + }); + return references.map(() => undefined); + } + return yield* Effect.forEach(references, (reference) => + readOne(config, token, reference).pipe( + Effect.provide(httpClient), + Effect.catch((error) => + Effect.logWarning("Could not read secret from OpenBao", { + detail: String(error), + }).pipe(Effect.as(undefined)), + ), + ), + ); + }), + }); + +export const driver = makeDriver(); diff --git a/packages/secret-provider-openbao/src/shared/definition.test.ts b/packages/secret-provider-openbao/src/shared/definition.test.ts new file mode 100644 index 0000000000000..dbd83056e8115 --- /dev/null +++ b/packages/secret-provider-openbao/src/shared/definition.test.ts @@ -0,0 +1,35 @@ +import * as Schema from "effect/Schema"; +import { assert, describe, expect, it } from "@effect/vitest"; + +import { definition, OpenBaoSecretProviderConfig } from "./definition.ts"; + +const decodeConfig = Schema.decodeSync(OpenBaoSecretProviderConfig); +const decodeUnknownConfig = Schema.decodeUnknownExit(OpenBaoSecretProviderConfig); + +describe("OpenBao definition", () => { + it("accepts path#field references", () => { + for (const reference of ["agents/codex#api_key", "a#b"]) { + expect(definition.isReference(reference), reference).toBe(true); + } + }); + + it("rejects references without a path or field, flag-shaped paths, and control characters", () => { + for (const reference of ["agents/codex", "#key", "agents/codex#", "-x#key", " a#b", "a#b\n"]) { + expect(definition.isReference(reference), reference).toBe(false); + } + }); + + it("defaults the mount and auth", () => { + assert.deepStrictEqual(decodeConfig({ address: "https://bao.example" }), { + address: "https://bao.example", + mount: "secret", + auth: "token", + }); + }); + + it("rejects a non-http address and a flag-shaped mount", () => { + const decode = decodeUnknownConfig; + expect(decode({ address: "ftp://bao" })._tag).toBe("Failure"); + expect(decode({ address: "https://bao.example", mount: "-x" })._tag).toBe("Failure"); + }); +}); diff --git a/packages/secret-provider-openbao/src/shared/definition.ts b/packages/secret-provider-openbao/src/shared/definition.ts new file mode 100644 index 0000000000000..56e5cbbd9753c --- /dev/null +++ b/packages/secret-provider-openbao/src/shared/definition.ts @@ -0,0 +1,87 @@ +import { SecretProviderKind, TrimmedNonEmptyString, TrimmedString } from "@t3tools/contracts"; +import { defineSecretProvider } from "@t3tools/secret-provider-core/shared/definition"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +const ADDRESS_MAX_CHARS = 1024; +const PATH_MAX_CHARS = 253; +const MOUNT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._/-]*$/; +const MAX_REFERENCE_LENGTH = 1024; + +function isHttpUrl(value: string): boolean { + try { + const url = new URL(value); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } +} + +export const OpenBaoAddress = TrimmedNonEmptyString.check( + Schema.isMaxLength(ADDRESS_MAX_CHARS), + Schema.makeFilter((value: string) => + isHttpUrl(value) ? true : "OpenBao address must be an http or https URL.", + ), +); +export type OpenBaoAddress = typeof OpenBaoAddress.Type; + +/** A KV secrets engine mount. It is passed to the CLI as an argument, so it can never start with `-`. */ +export const OpenBaoMount = TrimmedString.check( + Schema.isMaxLength(PATH_MAX_CHARS), + Schema.makeFilter((value: string) => + MOUNT_PATTERN.test(value) + ? true + : "OpenBao mount must start with a letter or digit and contain no spaces.", + ), +); +export type OpenBaoMount = typeof OpenBaoMount.Type; + +export const OpenBaoNamespace = TrimmedNonEmptyString.check(Schema.isMaxLength(PATH_MAX_CHARS)); +export type OpenBaoNamespace = typeof OpenBaoNamespace.Type; + +/** + * Exactly one place the server takes the token from: the provider's stored token, the + * `BAO_TOKEN` variable of the server process, or the login the `bao` CLI holds on that machine. + */ +export const OpenBaoAuth = Schema.Literals(["token", "env", "cli"]); +export type OpenBaoAuth = typeof OpenBaoAuth.Type; + +export const OpenBaoSecretProviderConfig = Schema.Struct({ + address: OpenBaoAddress, + mount: OpenBaoMount.pipe(Schema.withDecodingDefault(Effect.succeed("secret"))), + namespace: Schema.optionalKey(OpenBaoNamespace), + token: Schema.optionalKey(TrimmedString), + auth: OpenBaoAuth.pipe(Schema.withDecodingDefault(Effect.succeed("token" as const))), +}); +export type OpenBaoSecretProviderConfig = typeof OpenBaoSecretProviderConfig.Type; + +/** `path/to/secret#field`; the path is a positional argument, so it can never start with `-`. */ +export function parseOpenBaoReference(reference: string) { + const separator = reference.indexOf("#"); + if (separator === -1) return undefined; + const path = reference.slice(0, separator); + const field = reference.slice(separator + 1); + if (path.length === 0 || field.length === 0 || path.startsWith("-")) return undefined; + return { path, field }; +} + +function hasControlCharacter(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code < 0x20 || code === 0x7f) return true; + } + return false; +} + +export const definition = defineSecretProvider({ + kind: SecretProviderKind.make("openbao"), + label: "OpenBao", + configSchema: OpenBaoSecretProviderConfig, + isReference: (reference) => + reference.length <= MAX_REFERENCE_LENGTH && + reference === reference.trim() && + !hasControlCharacter(reference) && + parseOpenBaoReference(reference) !== undefined, + referencePlaceholder: "path/to/secret#field", + invalidReferenceMessage: "Use an OpenBao reference like path/to/secret#field.", +}); diff --git a/packages/secret-provider-openbao/tsconfig.json b/packages/secret-provider-openbao/tsconfig.json new file mode 100644 index 0000000000000..374bac55202dd --- /dev/null +++ b/packages/secret-provider-openbao/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src"] +} diff --git a/packages/shared/src/AgentScope.test.ts b/packages/shared/src/AgentScope.test.ts index cf3ca64acce0a..2d9aca5ac1129 100644 --- a/packages/shared/src/AgentScope.test.ts +++ b/packages/shared/src/AgentScope.test.ts @@ -1,9 +1,12 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; import { describe, expect, it } from "@effect/vitest"; -import { AGENT_OOM_SCORE_ADJ, agentScopeCommand } from "./AgentScope.ts"; +import { AGENT_OOM_SCORE_ADJ, agentScopeCommand, agentScopeLauncherScript } from "./AgentScope.ts"; import * as HostProcess from "./HostProcess.ts"; describe.skipIf(HostProcess.Platform.defaultValue() !== "linux")("agentScopeCommand", () => { @@ -25,5 +28,20 @@ describe.skipIf(HostProcess.Platform.defaultValue() !== "linux")("agentScopeComm expect(Number(pid)).toBe(child.pid); expect(Number(score)).toBe(AGENT_OOM_SCORE_ADJ); expect(args).toEqual(["a b", "$HOME"]); + + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-launcher-")); + try { + const launcher = NodePath.join(directory, "launch's"); + NodeFS.writeFileSync(launcher, agentScopeLauncherScript(launch), { mode: 0o700 }); + const launched = NodeChildProcess.spawnSync(launcher, ["--flag", "it's"], { + encoding: "utf8", + }); + const [launchedPid, launchedScore, ...launchedArgs] = launched.stdout.trimEnd().split("\n"); + expect(Number(launchedPid)).toBe(launched.pid); + expect(Number(launchedScore)).toBe(AGENT_OOM_SCORE_ADJ); + expect(launchedArgs).toEqual(["a b", "$HOME", "--flag", "it's"]); + } finally { + NodeFS.rmSync(directory, { recursive: true, force: true }); + } }); }); diff --git a/packages/shared/src/AgentScope.ts b/packages/shared/src/AgentScope.ts index 361bfc49108c6..5f0ffca217172 100644 --- a/packages/shared/src/AgentScope.ts +++ b/packages/shared/src/AgentScope.ts @@ -22,6 +22,12 @@ export interface AgentScopeShape { * descendants outside its process group. Preserves systemd OOM diagnosis. */ readonly stop: (unit: string) => Effect.Effect; + /** + * SIGKILLs every process in the thread's own scope, which works even while + * the kernel throttles them. Shared scopes serve other threads and are never + * killed here. Succeeds with whether a scope was killed. + */ + readonly kill: (threadId: string) => Effect.Effect; /** * Forgets the thread's scope. The session manager calls it before it opens * a provider session, so a scope only answers for the session that launched @@ -42,6 +48,7 @@ export const AgentScope = Context.Reference("@t3tools/shared/Ag wrap: ({ command, args }) => Effect.succeed({ command, args }), oomKilled: () => Effect.succeed(false), stop: () => Effect.void, + kill: () => Effect.succeed(false), clear: () => Effect.void, }), }); @@ -74,6 +81,14 @@ export interface AgentSystemdScope { readonly options: ReadonlyArray; } +const quoteShell = (value: string) => `'${value.replaceAll("'", `'\\''`)}'`; + +/** An executable file for launchers that take a path, not a command line, such as Playwright. */ +export function agentScopeLauncherScript(launch: AgentScopeCommand): string { + const command = [launch.command, ...launch.args].map(quoteShell).join(" "); + return `#!/bin/sh\nexec ${command} "$@"\n`; +} + /** Exec at every step preserves the agent PID, stdio, and signals; systemd-run places it and descendants in the scope. */ export function agentScopeCommand(input: { readonly command: string; diff --git a/packages/shared/src/authScopeOptions.ts b/packages/shared/src/authScopeOptions.ts index d443e50326d28..9a2261a09f3e9 100644 --- a/packages/shared/src/authScopeOptions.ts +++ b/packages/shared/src/authScopeOptions.ts @@ -104,3 +104,19 @@ export const AUTH_SCOPE_OPTIONS: ReadonlyArray<{ description: "Change managed tunnel connectivity.", }, ]; + +/** Shown instead of the granted scopes while they are unknown, so every surface says the same thing. */ +export function sessionPermissionsUnavailableMessage(input: { + readonly connected: boolean; + /** The check errored; the client retries it while connected. */ + readonly failed: boolean; + /** The environment answered that this session is not signed in; retrying will not change that. */ + readonly rejected: boolean; +}): string { + if (!input.connected) return "Connect to this environment to view this session’s permissions."; + if (input.rejected) + return "This environment did not accept your session. Reconnect to try again."; + return input.failed + ? "Could not verify your permissions. Retrying automatically." + : "Checking permissions…"; +} diff --git a/packages/shared/src/composerContextReferences.ts b/packages/shared/src/composerContextReferences.ts index e1b25bc094a66..88361606a9a64 100644 --- a/packages/shared/src/composerContextReferences.ts +++ b/packages/shared/src/composerContextReferences.ts @@ -132,6 +132,11 @@ export function collectComposerContextReferences( export const ATTACHMENT_ONLY_BOOTSTRAP_PROMPT = "[User attached one or more files without additional text. Respond using the conversation context and the attached files.]"; +/** Claude's ultrathink effort sends the prompt as `Ultrathink:\n`. */ +export function isAttachmentOnlyBootstrapPrompt(text: string): boolean { + return text.replace(/^Ultrathink:\s*/i, "") === ATTACHMENT_ONLY_BOOTSTRAP_PROMPT; +} + export function isImageReferencesOnly(text: string): boolean { const references = collectComposerContextReferences(text); return ( diff --git a/packages/shared/src/editor.ts b/packages/shared/src/editor.ts index dd9f11b0751e4..57bc67afdcfa6 100644 --- a/packages/shared/src/editor.ts +++ b/packages/shared/src/editor.ts @@ -34,7 +34,8 @@ export const resolveEditorCommand = Effect.fn("editor.resolveEditorCommand")(fun const platform = yield* HostProcess.Platform; // Prefer Zed's macOS bundle because OpenZFS also installs a zed executable. const preferBundle = platform === "darwin" && editor.id === "zed"; - if (!preferBundle) { + const preferZeditor = platform === "linux" && editor.id === "zed"; + if (!preferBundle && !preferZeditor) { for (const command of editor.commands) { if (yield* isCommandAvailable(command, { env })) return Option.some({ command, baseArgs }); } @@ -129,8 +130,15 @@ export const resolveEditorCommand = Effect.fn("editor.resolveEditorCommand")(fun const dataHome = env.XDG_DATA_HOME || (home ? path.join(home, ".local/share") : undefined); if (dataHome) dirs.push(path.join(dataHome, "JetBrains/Toolbox/scripts")); } - for (const dir of dirs) { - for (const name of editor.commands) candidates.push(path.join(dir, name)); + if (preferZeditor) { + // Linux packages use zeditor because OpenZFS owns the zed command. + for (const name of ["zeditor", "zed"]) { + candidates.push(name, ...dirs.map((dir) => path.join(dir, name))); + } + } else { + for (const dir of dirs) { + for (const name of editor.commands) candidates.push(path.join(dir, name)); + } } } diff --git a/packages/shared/src/htmlRender.ts b/packages/shared/src/htmlRender.ts index 572441ffe5366..12af0f39336b2 100644 --- a/packages/shared/src/htmlRender.ts +++ b/packages/shared/src/htmlRender.ts @@ -232,6 +232,7 @@ export const HTML_RENDER_THEME_GUIDE = [ export const HTML_RENDER_LAYOUT_GUIDE = [ `Frame width is reply width (${HTML_RENDER_COLUMN_WIDTH}px desktop, about 360px phones), on the thread background.`, "Leave html, body, and the outer element with no background color, no outer card, border, banner title, or side padding. This wins over any rule that sets a page background.", + "T3 adds an overridable 8px html gutter for selection rings and shadows. Use fluid widths within it.", "A box that needs its own background gets 16px padding and var(--radius) corners. Give charts fixed pixel heights.", "Let content set the height. Do not use vh units or height:100% on html or body. Both are rejected or loop.", ].join(" "); @@ -353,7 +354,7 @@ export function htmlRenderThemeMessage(theme: HtmlRenderTheme) { } const BASE_CSS = - "html{background:var(--background);color:var(--foreground);font-family:var(--font-sans);font-size:14px;line-height:1.5;-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:100%;scrollbar-width:none}" + + "html{box-sizing:border-box;padding:8px;background:var(--background);color:var(--foreground);font-family:var(--font-sans);font-size:14px;line-height:1.5;-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:100%;scrollbar-width:none}" + "html::-webkit-scrollbar{display:none}body{margin:0}code,kbd,pre,samp{font-family:var(--font-mono)}" + // Focus outlines paint inside the element, so the frame edge never clips them. ":where(:focus-visible){outline:2px solid var(--ring)}:focus-visible{outline-width:2px!important;outline-offset:-2px!important}" + diff --git a/packages/shared/src/nodeSqliteClient.test.ts b/packages/shared/src/nodeSqliteClient.test.ts index 99eb8b37a215b..ac786993fa0bf 100644 --- a/packages/shared/src/nodeSqliteClient.test.ts +++ b/packages/shared/src/nodeSqliteClient.test.ts @@ -238,6 +238,27 @@ it.effect("lets the writer commit while a read-only transaction is open", () => ), ); +it.effect("serves another read while a read-only transaction is open", () => + withReaderClient((sql) => + Effect.gen(function* () { + const opened = yield* Deferred.make(); + const other = yield* Effect.forkChild( + Deferred.await(opened).pipe(Effect.andThen(countEntries(sql).pipe(SqliteClient.readOnly))), + ); + yield* sql + .withTransaction( + Effect.gen(function* () { + yield* countEntries(sql); + yield* Deferred.succeed(opened, undefined); + // One shared reader would make this join wait on the transaction forever. + assert.equal(yield* Fiber.join(other), 1); + }), + ) + .pipe(SqliteClient.readOnly); + }), + ), +); + // regression: #19226 read-only transactions run on the separate reader connection it.effect("rejects a write in a read-only transaction", () => withReaderClient((sql) => @@ -333,6 +354,25 @@ it.effect("uses the writer when the reader cannot open the database", () => assert.equal(yield* sql.withTransaction(countEntries(sql)).pipe(SqliteClient.readOnly), 1); assert.equal(yield* sql.withTransaction(countEntries(sql)).pipe(SqliteClient.readOnly), 1); + + // A read beside an open read transaction must not start a worker on the next slot. + const held = yield* Deferred.make(); + const release = yield* Deferred.make(); + const holder = yield* sql + .withTransaction( + countEntries(sql).pipe( + Effect.andThen(Deferred.succeed(held, undefined)), + Effect.andThen(Deferred.await(release)), + ), + ) + .pipe(SqliteClient.readOnly, Effect.forkChild); + yield* Deferred.await(held); + const other = yield* countEntries(sql).pipe(SqliteClient.readOnly, (effect) => + Effect.forkChild(effect, { startImmediately: true }), + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(holder); + assert.equal(yield* Fiber.join(other), 1); assert.equal(startedWorkers.length, startedBefore + 1); }), ), diff --git a/packages/shared/src/nodeSqliteClient.ts b/packages/shared/src/nodeSqliteClient.ts index a42943c065ac0..c0edfdb868d1f 100644 --- a/packages/shared/src/nodeSqliteClient.ts +++ b/packages/shared/src/nodeSqliteClient.ts @@ -293,8 +293,14 @@ const failed = (cause: unknown, message: string, operation: string) => // Read-transaction cleanup succeeds after worker exit because it can no longer change anything. const endsReadTransaction = (sql: string) => /^\s*ROLLBACK\b|^\s*COMMIT\b/i.test(sql); +// WAL readers run concurrently; one worker serialized every read behind the slowest transaction. +const READER_POOL_SIZE = 4; + // Bind transactions to one worker; fail after its exit instead of resuming outside the transaction on a replacement. -const makeReader = Effect.fnUntraced(function* (options: SqliteClientConfig) { +const makeReader = Effect.fnUntraced(function* ( + options: SqliteClientConfig, + pool: { unavailable: boolean }, +) { let current: | { readonly worker: NodeWorkerThreads.Worker; @@ -302,7 +308,6 @@ const makeReader = Effect.fnUntraced(function* (options: SqliteClientConfig) { readonly started: Promise; } | undefined; - let unavailable = false; let nextRequestId = 0; const start = () => { @@ -439,8 +444,8 @@ const makeReader = Effect.fnUntraced(function* (options: SqliteClientConfig) { void reader.worker.terminate(); } // @effect-diagnostics-next-line effectSucceedWithVoid:off - The connection result distinguishes undefined from void. - if (unavailable) return Effect.succeed(undefined); - unavailable = true; + if (pool.unavailable) return Effect.succeed(undefined); + pool.unavailable = true; return Effect.logWarning("SQLite reader worker unavailable; reads use the main connection", { reason, }).pipe(Effect.as(undefined)); @@ -448,7 +453,7 @@ const makeReader = Effect.fnUntraced(function* (options: SqliteClientConfig) { return Effect.suspend((): Effect.Effect => { // @effect-diagnostics-next-line effectSucceedWithVoid:off - The connection result distinguishes undefined from void. - if (unavailable) return Effect.succeed(undefined); + if (pool.unavailable) return Effect.succeed(undefined); let reader: NonNullable; try { reader = current ?? start(); @@ -467,7 +472,7 @@ const makeReader = Effect.fnUntraced(function* (options: SqliteClientConfig) { }), (startup): Effect.Effect => // A late ready event must not hand out a worker retired by another acquire timeout. - unavailable || current !== reader + pool.unavailable || current !== reader ? // @effect-diagnostics-next-line effectSucceedWithVoid:off - The connection result distinguishes undefined from void. Effect.succeed(undefined) : Option.isSome(startup) && startup.value.type === "ready" @@ -1073,21 +1078,48 @@ const make = Effect.fn("makeWithDatabase")(function* ( const semaphore = yield* Semaphore.make(1); const connection = yield* makeConnection; - const acquireReader = - options.readerWorker && isFileDatabase ? yield* makeReader(options) : undefined; - const readerSemaphore = yield* Semaphore.make(1); + // A reader that cannot start turns off the whole pool, so routing never picks a dead slot. + const readerPool = { unavailable: false }; + const readers = + options.readerWorker && isFileDatabase + ? yield* Effect.forEach(Array.from({ length: READER_POOL_SIZE }), () => + Effect.map( + Effect.all([makeReader(options, readerPool), Semaphore.make(1)]), + ([acquire, lock]) => ({ + acquire, + lock, + claims: 0, + }), + ), + ) + : []; const writer = { name: "writer" as const, lock: semaphore, connection }; let activeWriter: ConnectionOwner | undefined; - const route = Effect.withFiber((fiber) => - acquireReader !== undefined && Context.get(fiber.context, ReadOnly) - ? Effect.map(acquireReader, (readerConnection) => + // Callers must run route uninterruptibly and pass `claimed` a finalizer that runs `release` once. + const route = (claimed: (release: Effect.Effect) => Effect.Effect) => + Effect.withFiber((fiber) => { + if (readers.length === 0 || readerPool.unavailable || !Context.get(fiber.context, ReadOnly)) { + return Effect.succeed(writer); + } + // Least claimed wins and ties go to the lowest slot, so idle slots never start a worker. + let slot = readers[0]!; + for (const candidate of readers) if (candidate.claims < slot.claims) slot = candidate; + slot.claims++; + const reader = slot; + return Effect.andThen( + claimed( + Effect.sync(() => { + reader.claims--; + }), + ), + Effect.map(reader.acquire, (readerConnection) => readerConnection === undefined ? writer - : { name: "reader" as const, lock: readerSemaphore, connection: readerConnection }, - ) - : Effect.succeed(writer), - ); + : { name: "reader" as const, lock: reader.lock, connection: readerConnection }, + ), + ); + }); const acquired = ( target: { readonly name: Acquired["connection"] }, @@ -1101,88 +1133,110 @@ const make = Effect.fn("makeWithDatabase")(function* ( const writerBorrowBatchSize = 16; let writerBorrowsSinceYield = 0; const borrower: Borrower = (use) => - Effect.flatMap(route, (target) => - Effect.withFiber((fiber) => { - const clock = fiber.getRef(Clock.Clock); - const start = clock.monotonicTimeNanosUnsafe(); - const ownerAtWaitStart = target.name === "writer" ? activeWriter : undefined; - const owner = target.name === "writer" ? connectionOwner(fiber) : undefined; - const useConnection = Effect.suspend(() => { - if (target.name === "writer") activeWriter = owner; - if ( - fiber.getRef(References.TracerEnabled) && - fiber.getRef(Statement.SpanPropagationEnabled) - ) { - annotateAcquired(fiber, acquired(target, clock, start), ownerAtWaitStart); - } - return use(target.connection); - }); - const useOwnedConnection = - target.name === "writer" - ? Effect.onExit(useConnection, () => - Effect.sync(() => { - if (activeWriter === owner) activeWriter = undefined; - }), - ) - : useConnection; - if (!options.checkpointWorker || target.name !== "writer") { - return target.lock.withPermits(1)(useOwnedConnection); - } - const writerUse = Effect.onExit(useOwnedConnection, (exit) => - Effect.sync(() => { - writerBorrowsSinceYield = Exit.hasInterrupts(exit) - ? 0 - : Math.min(writerBorrowsSinceYield + 1, writerBorrowBatchSize); - }), - ); - return Effect.flatMap(target.lock.withPermitsIfAvailable(1)(writerUse), (result) => - Option.isSome(result) - ? Effect.succeed(result.value) - : target.lock.withPermits(1)( - Effect.andThen( - // Failed SQL also consumes the batch; cancellation hands off immediately. - Effect.suspend(() => { - if (writerBorrowsSinceYield < writerBorrowBatchSize) return Effect.void; - writerBorrowsSinceYield = 0; - return Effect.callback((resume) => { - const immediate = setImmediate(() => resume(Effect.void)); - return Effect.sync(() => clearImmediate(immediate)); - }); - }), - writerUse, - ), - ), - ); - }), - ); - const transactionAcquirer = Effect.flatMap(route, (target) => Effect.uninterruptibleMask((restore) => { - const fiber = Fiber.getCurrent()!; - const scope = Context.getUnsafe(fiber.context, Scope.Scope); + let release: Effect.Effect = Effect.void; + const claimed = (next: Effect.Effect) => + Effect.sync(() => { + release = next; + }); + return Effect.flatMap(route(claimed), (target) => restore(borrowTarget(target, use))).pipe( + Effect.ensuring(Effect.suspend(() => release)), + ); + }); + const borrowTarget = ( + target: { + readonly name: Acquired["connection"]; + readonly lock: Semaphore.Semaphore; + readonly connection: Connection; + }, + use: (connection: Connection) => Effect.Effect, + ): Effect.Effect => + Effect.withFiber((fiber) => { const clock = fiber.getRef(Clock.Clock); const start = clock.monotonicTimeNanosUnsafe(); - const pending = fiber.getRef(TransactionAcquire); const ownerAtWaitStart = target.name === "writer" ? activeWriter : undefined; const owner = target.name === "writer" ? connectionOwner(fiber) : undefined; - const acquire = Effect.tap((pending?.restore ?? restore)(target.lock.take(1)), () => { + const useConnection = Effect.suspend(() => { if (target.name === "writer") activeWriter = owner; - if (pending !== undefined) { - pending.acquired = acquired(target, clock, start); - pending.ownerAtWaitStart = ownerAtWaitStart; + if ( + fiber.getRef(References.TracerEnabled) && + fiber.getRef(Statement.SpanPropagationEnabled) + ) { + annotateAcquired(fiber, acquired(target, clock, start), ownerAtWaitStart); } - return Scope.addFinalizer( - scope, - Effect.sync(() => { - if (target.name === "writer" && activeWriter === owner) activeWriter = undefined; - }).pipe(Effect.andThen(target.lock.release(1))), - ); + return use(target.connection); }); - return Effect.as( - // A scheduler yield after taking the permit must not precede its release finalizer. - Effect.provideService(acquire, Scheduler.PreventSchedulerYield, true), - target.connection, + const useOwnedConnection = + target.name === "writer" + ? Effect.onExit(useConnection, () => + Effect.sync(() => { + if (activeWriter === owner) activeWriter = undefined; + }), + ) + : useConnection; + if (!options.checkpointWorker || target.name !== "writer") { + return target.lock.withPermits(1)(useOwnedConnection); + } + const writerUse = Effect.onExit(useOwnedConnection, (exit) => + Effect.sync(() => { + writerBorrowsSinceYield = Exit.hasInterrupts(exit) + ? 0 + : Math.min(writerBorrowsSinceYield + 1, writerBorrowBatchSize); + }), ); - }), + return Effect.flatMap(target.lock.withPermitsIfAvailable(1)(writerUse), (result) => + Option.isSome(result) + ? Effect.succeed(result.value) + : target.lock.withPermits(1)( + Effect.andThen( + // Failed SQL also consumes the batch; cancellation hands off immediately. + Effect.suspend(() => { + if (writerBorrowsSinceYield < writerBorrowBatchSize) return Effect.void; + writerBorrowsSinceYield = 0; + return Effect.callback((resume) => { + const immediate = setImmediate(() => resume(Effect.void)); + return Effect.sync(() => clearImmediate(immediate)); + }); + }), + writerUse, + ), + ), + ); + }); + const claimedUntilScopeCloses = (release: Effect.Effect) => + Effect.withFiber((fiber) => + Scope.addFinalizer(Context.getUnsafe(fiber.context, Scope.Scope), release), + ); + const transactionAcquirer = Effect.uninterruptible(route(claimedUntilScopeCloses)).pipe( + Effect.flatMap((target) => + Effect.uninterruptibleMask((restore) => { + const fiber = Fiber.getCurrent()!; + const scope = Context.getUnsafe(fiber.context, Scope.Scope); + const clock = fiber.getRef(Clock.Clock); + const start = clock.monotonicTimeNanosUnsafe(); + const pending = fiber.getRef(TransactionAcquire); + const ownerAtWaitStart = target.name === "writer" ? activeWriter : undefined; + const owner = target.name === "writer" ? connectionOwner(fiber) : undefined; + const acquire = Effect.tap((pending?.restore ?? restore)(target.lock.take(1)), () => { + if (target.name === "writer") activeWriter = owner; + if (pending !== undefined) { + pending.acquired = acquired(target, clock, start); + pending.ownerAtWaitStart = ownerAtWaitStart; + } + return Scope.addFinalizer( + scope, + Effect.sync(() => { + if (target.name === "writer" && activeWriter === owner) activeWriter = undefined; + }).pipe(Effect.andThen(target.lock.release(1))), + ); + }); + return Effect.as( + // A scheduler yield after taking the permit must not precede its release finalizer. + Effect.provideService(acquire, Scheduler.PreventSchedulerYield, true), + target.connection, + ); + }), + ), ); const client = yield* Client.make({ diff --git a/packages/shared/src/orchestrationTiming.ts b/packages/shared/src/orchestrationTiming.ts index 187db6c6d97d4..1f43f99453456 100644 --- a/packages/shared/src/orchestrationTiming.ts +++ b/packages/shared/src/orchestrationTiming.ts @@ -22,10 +22,7 @@ export function deriveSubagentElapsedMs( export function formatDuration(durationMs: number): string { if (!Number.isFinite(durationMs) || durationMs < 0) return "0ms"; if (durationMs < 1_000) return `${Math.max(1, Math.round(durationMs))}ms`; - if (durationMs < 10_000) { - const tenths = Math.round(durationMs / 100) / 10; - return tenths >= 10 ? "10s" : `${tenths.toFixed(1)}s`; - } + if (durationMs < 10_000) return `${Math.round(durationMs / 100) / 10}s`; const totalSeconds = Math.round(durationMs / 1_000); // Choose the unit after rounding so 59.5 seconds never renders as "60s". if (totalSeconds < 60) return `${totalSeconds}s`; diff --git a/packages/shared/src/serverSettings.test.ts b/packages/shared/src/serverSettings.test.ts index 0838ab9e97df0..16870c049cdf0 100644 --- a/packages/shared/src/serverSettings.test.ts +++ b/packages/shared/src/serverSettings.test.ts @@ -1,15 +1,17 @@ import { DEFAULT_SERVER_SETTINGS, - OnePasswordAccount, ProjectId, ProviderDriverKind, ProviderInstanceId, SecretProviderId, SecretProviderKind, + ServerSettings, + ServerSettingsPatch, UsageLimitSourceId, type ServerProvider, } from "@t3tools/contracts"; import * as Duration from "effect/Duration"; +import * as Schema from "effect/Schema"; import { describe, expect, it } from "vite-plus/test"; import { resolveServerBackgroundActivitySettings } from "./backgroundActivitySettings.ts"; import { createModelSelection } from "./model.ts"; @@ -24,6 +26,9 @@ import { resolveSourceControlWriterModelSelection, } from "./serverSettings.ts"; +const decodeServerSettings = Schema.decodeUnknownSync(ServerSettings); +const decodeServerSettingsPatch = Schema.decodeUnknownSync(ServerSettingsPatch); + const FOLDED_SERVER_SETTINGS = { ...DEFAULT_SERVER_SETTINGS, projectSettingsFolded: true }; describe("fallbackTextGenerationModelSelection", () => { @@ -499,6 +504,37 @@ describe("serverSettings helpers", () => { }); }); + // regression: #18334 a newer client must still see this server drop a key it does not know + it("drops an override key this server does not know from a patch and keeps one it stored", () => { + const projectId = ProjectId.make("project-a"); + const current = decodeServerSettings({ + projectSettingsFolded: true, + projectSettingsOverrides: { + [projectId]: { defaultAutoPull: true, addedByNewerServer: 3 }, + }, + }); + const patch = decodeServerSettingsPatch({ + projectSettingsOverrides: { + [projectId]: { defaultAutoPull: false, addedByNewerServer: 4, addedByNewerClient: "x" }, + }, + }); + + const updated = applyServerSettingsPatch(current, patch); + expect(updated.projectSettingsOverrides[projectId]).toEqual({ + defaultAutoPull: false, + addedByNewerServer: 3, + }); + const reset = applyServerSettingsPatch(updated, { + projectSettingsOverrides: { [projectId]: {} }, + }); + expect(reset.projectSettingsOverrides[projectId]).toEqual({ addedByNewerServer: 3 }); + expect( + applyServerSettingsPatch(reset, { + projectSettingsOverrides: { [projectId]: null }, + }).projectSettingsOverrides[projectId], + ).toBeUndefined(); + }); + it("clears a project's providerInstances override when a patch entry sends an empty map", () => { const instanceId = ProviderInstanceId.make("codex_work"); const projectId = ProjectId.make("project-a"); @@ -554,15 +590,15 @@ describe("serverSettings helpers", () => { const onePassword = (label: string, account: string) => ({ kind: SecretProviderKind.make("one_password"), label, - config: { account: OnePasswordAccount.make(account) }, + config: { account }, }); const current = { ...DEFAULT_SERVER_SETTINGS, secretProviders: { [personal]: { - kind: "1password" as const, + kind: SecretProviderKind.make("one_password"), label: "Personal", - account: OnePasswordAccount.make("my.1password.com"), + config: { account: "my.1password.com" }, }, }, }; @@ -570,11 +606,9 @@ describe("serverSettings helpers", () => { const added = applyServerSettingsPatch(current, { secretProviders: { [homelab]: { - kind: "openbao", + kind: SecretProviderKind.make("openbao"), label: "Homelab", - address: "https://bao.example:8200", - mount: "secret", - auth: "token", + config: { address: "https://bao.example:8200", mount: "secret", auth: "token" }, }, [work]: onePassword("Work", "team.1password.com"), }, diff --git a/packages/shared/src/serverSettings.ts b/packages/shared/src/serverSettings.ts index 11eecc6fe3df8..e936c9f83acba 100644 --- a/packages/shared/src/serverSettings.ts +++ b/packages/shared/src/serverSettings.ts @@ -14,7 +14,7 @@ import { type ModelSelection, type ProjectId, type ProjectScopedServerSettingKey, - type ProjectSettingsOverrides, + ProjectSettingsOverrides, ProviderDriverKind, ProviderInstanceId, type ServerProvider, @@ -226,6 +226,16 @@ function mergeSettingsEntries( return Object.fromEntries(next); } +const knownProjectOverrideKeys = new Set(Object.keys(ProjectSettingsOverrides.fields)); + +function pickOverrideKeys(entry: ProjectSettingsOverrides | undefined, known: boolean) { + return Object.fromEntries( + Object.entries(entry ?? {}).filter( + ([key]) => knownProjectOverrideKeys.has(key) === known, + ), + ); +} + // Older clients omit providerInstances during whole-entry replacement; preserve it to avoid wiping overrides. function mergeProjectSettingsOverridesPatch( current: Readonly>, @@ -239,7 +249,9 @@ function mergeProjectSettingsOverridesPatch( } const storedProviderInstances = current[projectId]?.providerInstances; next.set(projectId, { - ...entry, + // Preserve downgraded storage, but reject new unknown writes so clients can detect unsupported settings. + ...pickOverrideKeys(current[projectId], false), + ...pickOverrideKeys(entry, true), ...(Object.hasOwn(entry, "providerInstances") || storedProviderInstances === undefined ? {} : { providerInstances: storedProviderInstances }), diff --git a/packages/shared/src/threadFindText.test.ts b/packages/shared/src/threadFindText.test.ts index 0dd6a87ae1dea..77a0874c21336 100644 --- a/packages/shared/src/threadFindText.test.ts +++ b/packages/shared/src/threadFindText.test.ts @@ -1,6 +1,7 @@ import { EnvironmentId, MessageId, ThreadId } from "@t3tools/contracts"; import { expect, it } from "vite-plus/test"; import { serializeAssistantCitation } from "./assistantCitations.ts"; +import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "./composerContextReferences.ts"; import { searchableMessageSegments } from "./threadFindText.ts"; it("excludes review attachments rendered as context chips", () => { @@ -31,6 +32,16 @@ it("excludes structured context chips without joining text across them", () => { ).toEqual(["before", "after"]); }); +it("excludes the hidden prompt of an attachment-only message", () => { + expect( + searchableMessageSegments({ + role: "user", + streaming: false, + text: ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, + }), + ).toEqual([]); +}); + it("does not upgrade literal legacy tags in messages with structured context", () => { const text = "\n- Terminal 1 line 12:\n 12 | visible output\n"; diff --git a/packages/shared/src/threadFindText.ts b/packages/shared/src/threadFindText.ts index a2e5b4dbb982d..3afec6e04729e 100644 --- a/packages/shared/src/threadFindText.ts +++ b/packages/shared/src/threadFindText.ts @@ -21,7 +21,10 @@ import { resolveMarkdownFileLinkTarget, } from "./markdownLinks.ts"; import { upgradeLegacyContextMessage } from "./composerContextLegacy.ts"; -import { parseComposerContextHref } from "./composerContextReferences.ts"; +import { + isAttachmentOnlyBootstrapPrompt, + parseComposerContextHref, +} from "./composerContextReferences.ts"; import { assistantCitationLabel, parseAssistantCitationHref } from "./assistantCitations.ts"; import type { OrchestrationV2ConversationMessage } from "@t3tools/contracts"; import { proposedPlanTitle, stripDisplayedPlanMarkdown } from "./proposedPlanText.ts"; @@ -216,6 +219,8 @@ export function searchableMessageSegments( skills: readonly InlineSkill[] = [], ): readonly string[] | null { if (message.role === "user") { + // Clients hide this prompt, so find must not match text nobody can see. + if (isAttachmentOnlyBootstrapPrompt(message.text)) return []; const text = message.context ? message.text : upgradeLegacyContextMessage(message.text).text; return markdownThreadFindText(text, true, cwd, skills); } diff --git a/packages/source-control-azure-devops/src/shared/definition.ts b/packages/source-control-azure-devops/src/shared/definition.ts index b43adccedc846..2dd036674747b 100644 --- a/packages/source-control-azure-devops/src/shared/definition.ts +++ b/packages/source-control-azure-devops/src/shared/definition.ts @@ -1,15 +1,18 @@ import { SourceControlProviderKind } from "@t3tools/contracts"; import { + buildCommitUrl, changeRequestUrlBelow, defineSourceControl, isChangeRequestAuthorityOfProject, parseChangeRequestPath, + parseCommitPath, remoteWebOrigin, } from "@t3tools/source-control-core/shared/definition"; import { icon } from "./icon.ts"; // Azure DevOps Server runs on any hostname; its `_git` route identifies it. const CHANGE_REQUEST_PATH = /^\/((?:[^/]+\/)*_git\/[^/]+)\/pullrequest\/(\d+)(?:\/|$)/u; +const COMMIT_PATH = /^\/((?:[^/]+\/)*_git\/[^/]+)\/commit\/([0-9a-f]+)\/?$/iu; const CHECKOUT_COMMAND = /^az\s+repos\s+pr\s+checkout\s+(.+)$/i; const CHANGE_REQUEST_REFERENCE = /^https:\/\/(?:dev\.azure\.com\/[^/\s]+\/[^/\s]+|[^/\s]+\.visualstudio\.com\/[^/\s]+)\/_git\/[^/\s]+\/pullrequest\/(\d+)(?:[/?#].*)?$/i; @@ -88,6 +91,12 @@ export const definition = defineSourceControl({ const [canonicalHost = host, ...path] = (canonicalRepositoryKey(key) ?? key).split("/"); return `${remoteWebOrigin(canonicalHost, remoteUrl)}/${path.join("/")}/pullrequest/${number}`; }, + commitUrl: (input) => { + const key = `${input.host}/${input.repository}`.toLowerCase(); + const [host = input.host, ...path] = (canonicalRepositoryKey(key) ?? key).split("/"); + return buildCommitUrl({ ...input, host, repository: path.join("/") }, "commit"); + }, + parseCommitUrl: (url) => parseCommitPath(url, COMMIT_PATH), // Servers on different ports are different servers, so a port stays in the link. parseChangeRequestUrl: (url) => { const link = parseChangeRequestPath(url, CHANGE_REQUEST_PATH); diff --git a/packages/source-control-bitbucket/src/server/BitbucketApi.test.ts b/packages/source-control-bitbucket/src/server/BitbucketApi.test.ts index 26891c3548974..c0464276abf24 100644 --- a/packages/source-control-bitbucket/src/server/BitbucketApi.test.ts +++ b/packages/source-control-bitbucket/src/server/BitbucketApi.test.ts @@ -513,7 +513,7 @@ it.effect("never puts a saved token that is unsafe for an HTTP header on the wir }); it.effect("reports saved credentials as configured when Bitbucket cannot confirm them", () => { - const { layer } = makeLayer({ + const { execute, layer } = makeLayer({ response: () => new Response(null, { status: 401 }), env: { T3CODE_BITBUCKET_API_BASE_URL: "https://api.test.local/2.0" }, }); @@ -523,6 +523,7 @@ it.effect("reports saved credentials as configured when Bitbucket cannot confirm const settings = yield* TestSourceControlHost.TestSourceControlHostSettings; assert.strictEqual((yield* bitbucket.probeAuth).status, "unauthenticated"); + assert.strictEqual(execute.mock.calls.length, 0); yield* settings.update({ sourceControlHosts: { bitbucket: { accessToken: "saved-access-token" } }, diff --git a/packages/source-control-bitbucket/src/server/BitbucketApi.ts b/packages/source-control-bitbucket/src/server/BitbucketApi.ts index a5acd5e412eb4..9f6de0e96d757 100644 --- a/packages/source-control-bitbucket/src/server/BitbucketApi.ts +++ b/packages/source-control-bitbucket/src/server/BitbucketApi.ts @@ -991,18 +991,25 @@ export const make = Effect.gen(function* () { return BitbucketApi.of({ request, - probeAuth: executeJson( - "probeAuth", - HttpClientRequest.get(apiUrl("/user")), - BitbucketUserSchema, - ).pipe( - Effect.map((user) => ({ - status: "authenticated" as const, - account: nonEmpty(user.username ?? user.display_name ?? user.account_id), - host: Option.some("bitbucket.org"), - detail: Option.none(), - })), - Effect.catch(() => currentCredential.pipe(Effect.map(authFromCredential))), + // Bitbucket answers anonymous /user with 401, so without a credential there is nothing to ask. + probeAuth: currentCredential.pipe( + Effect.flatMap((credential) => + credential === null + ? Effect.succeed(authFromCredential(null)) + : executeJson( + "probeAuth", + HttpClientRequest.get(apiUrl("/user")), + BitbucketUserSchema, + ).pipe( + Effect.map((user) => ({ + status: "authenticated" as const, + account: nonEmpty(user.username ?? user.display_name ?? user.account_id), + host: Option.some("bitbucket.org"), + detail: Option.none(), + })), + Effect.catch(() => currentCredential.pipe(Effect.map(authFromCredential))), + ), + ), ), listPullRequests: Effect.fn("BitbucketApi.listPullRequests")(function (input) { return resolveRepository(input).pipe( diff --git a/packages/source-control-bitbucket/src/shared/definition.ts b/packages/source-control-bitbucket/src/shared/definition.ts index 2f46d02ec3b5d..86522f4d8c960 100644 --- a/packages/source-control-bitbucket/src/shared/definition.ts +++ b/packages/source-control-bitbucket/src/shared/definition.ts @@ -7,11 +7,13 @@ import { import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import { + buildCommitUrl, changeRequestUrlBelow, defineSourceControl, isChangeRequestInProjectRepository, isChangeRequestOnProjectHost, parseChangeRequestPath, + parseCommitPath, projectChangeRequestHost, } from "@t3tools/source-control-core/shared/definition"; import { icon } from "./icon.ts"; @@ -20,6 +22,7 @@ const safeShellArgument = /^[A-Za-z0-9._/@+=,-]+$/; const repositoryName = /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/; const CHANGE_REQUEST_PATH = /^\/([^/]+\/[^/]+)\/pull-requests\/(\d+)(?:\/|$)/u; +const COMMIT_PATH = /^\/([^/]+\/[^/]+)\/commits\/([0-9a-f]+)\/?$/iu; const KIND = SourceControlProviderKind.make("bitbucket"); @@ -94,6 +97,9 @@ export const definition = defineSourceControl({ : `https://${host}/${repository}/pull-requests/${number}`, parseChangeRequestUrl: (url) => isBitbucketHost(url.hostname) ? parseChangeRequestPath(url, CHANGE_REQUEST_PATH) : null, + commitUrl: (input) => buildCommitUrl(input, "commits"), + parseCommitUrl: (url) => + isBitbucketHost(url.hostname) ? parseCommitPath(url, COMMIT_PATH) : null, changeRequestUrlFromReferenceAutolink: () => null, // No endpoint reopens a declined pull request, and nothing documented moves one in or out of // draft, so neither is offered rather than failing when pressed. diff --git a/packages/source-control-core/src/server/PullRequestProvider.ts b/packages/source-control-core/src/server/PullRequestProvider.ts index f587d95e22c31..ed0f5a1580572 100644 --- a/packages/source-control-core/src/server/PullRequestProvider.ts +++ b/packages/source-control-core/src/server/PullRequestProvider.ts @@ -361,6 +361,16 @@ export interface PullRequestProviderApi { input: ProviderRepositoryRef & { readonly number: number }, ) => Effect.Effect; + /** + * For a provider that holds change request reads of its own: called when the service drops its + * reads, so a refresh or a finished turn reaches the host. No scope forgets everything; a scope + * without a number forgets the repository's change requests. + */ + readonly forgetChangeRequestReads?: (scope?: { + readonly repository: string; + readonly number?: number; + }) => Effect.Effect; + readonly getChangeRequestPreview?: ( input: ProviderRepositoryRef & { readonly number: number }, ) => Effect.Effect< diff --git a/packages/source-control-core/src/server/discovery.ts b/packages/source-control-core/src/server/discovery.ts index 66889ddc4f2ab..2f018bfa10ca3 100644 --- a/packages/source-control-core/src/server/discovery.ts +++ b/packages/source-control-core/src/server/discovery.ts @@ -75,6 +75,8 @@ type SourceControlCliRemoteRefinementSpec = SourceControlCliDiscoverySpec & { }; const DEFAULT_PROBE_TIMEOUT_MS = 5_000; +/** An API host that stalls must not hold the whole discovery answer behind it. */ +const API_PROBE_TIMEOUT = "3 seconds"; function probeTimeoutMs(spec: SourceControlCliDiscoverySpec): number { return spec.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS; @@ -236,6 +238,12 @@ export function probeSourceControlProvider(input: { if (input.spec.type === "managed-cli") return input.spec.probe(input.cwd); if (input.spec.type === "api") { return input.spec.probeAuth.pipe( + Effect.timeoutOption(API_PROBE_TIMEOUT), + Effect.map( + Option.getOrElse(() => + unknownAuth("The hosting service did not answer in time. Refresh to try again."), + ), + ), Effect.map( (auth) => ({ @@ -253,20 +261,15 @@ export function probeSourceControlProvider(input: { const spec = input.spec; - return probeCli({ - spec, - process: input.process, - cwd: input.cwd, - }).pipe( - Effect.flatMap((item) => { - if (item.status !== "available") { - return Effect.succeed({ - ...item, - auth: unknownAuth("Hosting integration command was not found on the server PATH."), - } satisfies SourceControlProviderDiscoveryItem); - } - - return input.process + // The auth check does not depend on the version output, so both spawn at once. + return Effect.all( + [ + probeCli({ + spec, + process: input.process, + cwd: input.cwd, + }), + input.process .run({ operation: "source-control.discovery.auth", command: spec.executable, @@ -278,21 +281,24 @@ export function probeSourceControlProvider(input: { appendTruncationMarker: true, }) .pipe( - Effect.map( - (result) => - ({ - ...item, - auth: spec.parseAuth(result), - }) satisfies SourceControlProviderDiscoveryItem, - ), + Effect.map(spec.parseAuth), Effect.catch((cause) => - Effect.succeed({ - ...item, - auth: unknownAuth(Option.getOrUndefined(detailFromCause(cause))), - } satisfies SourceControlProviderDiscoveryItem), + Effect.succeed(unknownAuth(Option.getOrUndefined(detailFromCause(cause)))), ), - ); - }), + ), + ], + { concurrency: "unbounded" }, + ).pipe( + Effect.map( + ([item, auth]) => + ({ + ...item, + auth: + item.status === "available" + ? auth + : unknownAuth("Hosting integration command was not found on the server PATH."), + }) satisfies SourceControlProviderDiscoveryItem, + ), ); } diff --git a/packages/source-control-core/src/shared/definition.ts b/packages/source-control-core/src/shared/definition.ts index 024925c60a0b8..61b4752ba6c2c 100644 --- a/packages/source-control-core/src/shared/definition.ts +++ b/packages/source-control-core/src/shared/definition.ts @@ -31,6 +31,17 @@ export interface ParsedChangeRequestUrl extends ChangeRequestLink { readonly repositoryUrl: string; } +export interface CommitLink { + readonly host: string; + readonly repository: string; + readonly sha: string; + readonly authority?: string; +} + +export interface CommitUrlInput extends Omit { + readonly sha: string; +} + export interface ChangeRequestTerminology { readonly shortLabel: string; /** Spelled out because the article follows how the label is said: "an MR". */ @@ -239,6 +250,9 @@ export interface SourceControlDefinition { readonly newRepositoryOwner: (account: string | null) => { readonly owner: string | null } | null; readonly defaultCloneTransport: "https" | "ssh"; readonly changeRequestUrl: (input: ChangeRequestUrlInput) => string | null; + readonly commitUrl?: (input: CommitUrlInput) => string | null; + /** `known` is set when the caller already knows this host serves the URL's repository. */ + readonly parseCommitUrl?: (url: URL, known: boolean) => CommitLink | null; /** Null when the URL is not one of this host's change requests. */ readonly parseChangeRequestUrl: (url: URL) => ParsedChangeRequestUrl | null; /** @@ -291,7 +305,7 @@ export interface SourceControlDefinition { readonly isChangeRequestReference: (url: string) => boolean; readonly isChangeRequestInRepository: ( identity: ChangeRequestProjectIdentity, - link: ChangeRequestLink, + link: ChangeRequestLink | CommitLink, ) => boolean; readonly canReadChangeRequestOnHost: ( identity: ChangeRequestProjectIdentity, @@ -336,6 +350,42 @@ export type SourceControlHostSettingsSchema = { readonly fields: Readonly>; } & Schema.Decoder; +const FULL_COMMIT_SHA = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/iu; + +export function parseCommitPath(url: URL, route: RegExp): CommitLink | null { + if (url.protocol !== "https:" && url.protocol !== "http:") return null; + const match = route.exec(url.pathname); + const repository = match?.[1]; + const sha = match?.[2]; + return repository && sha && FULL_COMMIT_SHA.test(sha) + ? { + host: url.hostname.toLowerCase(), + repository: repository.toLowerCase(), + sha: sha.toLowerCase(), + ...(url.port ? { authority: url.host.toLowerCase() } : {}), + } + : null; +} + +export function buildCommitUrl( + { host, repository, sha, remoteUrl, webUrl }: CommitUrlInput, + route: string, +): string | null { + if (!repository || !FULL_COMMIT_SHA.test(sha)) return null; + try { + const url = new URL(webUrl || `${remoteWebOrigin(host, remoteUrl)}/${repository}`); + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + url.pathname = changeRequestUrlBelow(url.pathname, `${route}/${sha.toLowerCase()}`); + url.username = ""; + url.password = ""; + url.search = ""; + url.hash = ""; + return url.toString(); + } catch { + return null; + } +} + /** * Reads a change request from `url` with `route`, whose first group is the repository path and * second the number. @@ -384,7 +434,7 @@ export function defineSourceControl( export function isChangeRequestOnProjectHost( projectHost: string, identity: ChangeRequestProjectIdentity, - link: ChangeRequestLink, + link: ChangeRequestLink | CommitLink, ): boolean { if (!isChangeRequestAuthorityOfProject(identity, link)) return false; return projectHost === link.host.toLowerCase() || projectHost === link.authority; @@ -427,7 +477,7 @@ function projectRepositoryPath(identity: ChangeRequestProjectIdentity): string | export function isChangeRequestInProjectRepository( projectHost: string, identity: ChangeRequestProjectIdentity, - link: ChangeRequestLink, + link: ChangeRequestLink | CommitLink, ): boolean { const repository = projectRepositoryPath(identity); return ( @@ -534,6 +584,11 @@ export interface ParsedChangeRequest { readonly link: ParsedChangeRequestUrl; } +export interface ParsedCommit { + readonly definition: SourceControlDefinition; + readonly link: CommitLink; +} + export interface SourceControlMedia extends SourceControlMediaUrl { readonly definition: SourceControlDefinition; } @@ -550,6 +605,7 @@ export interface SourceControlDefinitionRegistry { readonly findByPublicDomain: (hostname: string) => SourceControlDefinition | undefined; /** The first host, in registry order, that reads `url` as one of its change requests. */ readonly parseChangeRequestUrl: (url: string) => ParsedChangeRequest | null; + readonly parseCommitUrl: (url: string, kind?: SourceControlProviderKind) => ParsedCommit | null; readonly findByChangeRequestUrl: (url: string) => SourceControlDefinition | undefined; readonly hostLabelForChangeRequestUrl: (url: string) => string; readonly isSameChangeRequest: (left: string, right: string) => boolean; @@ -649,6 +705,16 @@ export function makeSourceControlDefinitionRegistry( ); }, parseChangeRequestUrl, + parseCommitUrl: (value, kind) => { + const url = httpUrl(value); + if (url === null) return null; + const preferred = kind === undefined ? undefined : byKind.get(kind); + for (const definition of kind === undefined ? claimOrder : preferred ? [preferred] : []) { + const link = definition.parseCommitUrl?.(url, preferred !== undefined); + if (link != null) return { definition, link }; + } + return null; + }, findByChangeRequestUrl: (url) => parseChangeRequestUrl(url)?.definition, hostLabelForChangeRequestUrl: (url) => parseChangeRequestUrl(url)?.definition.label ?? "the host", diff --git a/packages/source-control-definitions/src/registry.ts b/packages/source-control-definitions/src/registry.ts index 0e42360855350..e44a6bb281717 100644 --- a/packages/source-control-definitions/src/registry.ts +++ b/packages/source-control-definitions/src/registry.ts @@ -23,7 +23,7 @@ export const builtInSourceControlDefinitions = makeSourceControlDefinitionRegist AzureDevOps.definition, ], { - claimChangeRequestUrlsFirst: [GitCafe.definition.kind], + claimChangeRequestUrlsFirst: [GitCafe.definition.kind, AzureDevOps.definition.kind], recognizeRemotesIn: [ GitCafe.definition.kind, Forgejo.definition.kind, diff --git a/packages/source-control-forgejo/src/shared/definition.ts b/packages/source-control-forgejo/src/shared/definition.ts index 32258854c79da..2cf8ac5bb5838 100644 --- a/packages/source-control-forgejo/src/shared/definition.ts +++ b/packages/source-control-forgejo/src/shared/definition.ts @@ -1,5 +1,6 @@ import { SourceControlProviderKind } from "@t3tools/contracts"; import { + buildCommitUrl, changeRequestUrlBelow, type ChangeRequestHostIdentity, type ChangeRequestProjectIdentity, @@ -8,6 +9,7 @@ import { isChangeRequestInProjectRepository, isChangeRequestOnProjectHost, parseChangeRequestPath, + parseCommitPath, projectChangeRequestHost, remoteWebOrigin, } from "@t3tools/source-control-core/shared/definition"; @@ -19,6 +21,7 @@ const CHANGE_REQUEST_REFERENCE = // Forgejo and Gitea run on any hostname, possibly mounted below a path. const CHANGE_REQUEST_PATH = /^\/([^/]+(?:\/[^/]+)+)\/pulls\/(\d+)(?:\/|$)/u; +const COMMIT_PATH = /^\/([^/]+(?:\/[^/]+)+)\/commit\/([0-9a-f]+)\/?$/iu; const KIND = SourceControlProviderKind.make("forgejo"); @@ -82,6 +85,8 @@ export const definition = defineSourceControl({ return link === null ? null : { ...link, authority: url.host.toLowerCase() }; }, changeRequestUrlFromReferenceAutolink: () => null, + commitUrl: (input) => buildCommitUrl(input, "commit"), + parseCommitUrl: (url) => parseCommitPath(url, COMMIT_PATH), changeRequestActions: new Set(["merge", "close", "reopen", "update-branch"] as const), // Neither `fj` nor `tea` checks out by number. checkoutCommand: ({ number, repositoryUrl }) => diff --git a/packages/source-control-gitcafe/src/server/GitCafePullRequestProvider.test.ts b/packages/source-control-gitcafe/src/server/GitCafePullRequestProvider.test.ts index 7c257f037003e..113a388e853bb 100644 --- a/packages/source-control-gitcafe/src/server/GitCafePullRequestProvider.test.ts +++ b/packages/source-control-gitcafe/src/server/GitCafePullRequestProvider.test.ts @@ -1226,43 +1226,70 @@ describe("GitCafePullRequestProvider", () => { it.effect("recounts summary lines only when the compared commits move", () => { const landedBase = "fedcba9876543210fedcba9876543210fedcba98"; + const mainTip = "0123012301230123012301230123012301230123"; let comparisonBaseOid = baseOid; + let tips = { targetBranch: "main", observedBaseOid: baseOid }; let pullReads = 0; + let listingCut = true; + let hunksDown = false; + const hunks = (request: SentRequest) => ({ + version: 4, + headOid, + comparisonBaseOid, + items: (request.body as { paths: ReadonlyArray }).paths.map((path) => ({ + path, + additions: comparisonBaseOid === baseOid ? 3 : 1, + deletions: 1, + })), + }); const server = fakeGitCafe({ // A comment moves the version without moving the compared commits. - "GET /repos/owner/repo/pulls/7": () => ({ ...pull, version: 4 + pullReads++ }), - "GET /repos/owner/repo/pulls/7/changes": () => ({ - version: 4, - headOid, - comparisonBaseOid, - items: [{ path: "a.ts" }, { path: "b.ts" }], - next: null, - }), - "POST /repos/owner/repo/pulls/7/diff-files": (request: SentRequest) => ({ + "GET /repos/owner/repo/pulls/7": () => ({ ...pull, ...tips, version: 4 + pullReads++ }), + "GET /repos/owner/repo/pulls/7/changes": (request: SentRequest) => ({ version: 4, headOid, comparisonBaseOid, - items: (request.body as { paths: ReadonlyArray }).paths.map((path) => ({ - path, - additions: comparisonBaseOid === baseOid ? 3 : 1, - deletions: 1, - })), + ...(!listingCut + ? { items: [{ path: "a.ts" }, { path: "b.ts" }], next: null } + : request.query.has("after") + ? { items: [], next: "more" } + : { items: [{ path: "a.ts" }], next: "more" }), }), + get "POST /repos/owner/repo/pulls/7/diff-files"() { + return hunksDown + ? new Response(encodeJson({ type: "https://cafe.sh/errors/conflict" }), { status: 409 }) + : hunks; + }, }); - const hunkReads = () => - server.sent.filter((request) => request.path.endsWith("/diff-files")).length; + const reads = (suffix: string) => + server.sent.filter((request) => request.path.endsWith(suffix)).length; return Effect.gen(function* () { const provider = yield* GitCafePullRequestProvider.make; + // A listing that stops before its last page holds nothing for its commits. + yield* provider.getChangeRequestSummary!(target); + listingCut = false; + hunksDown = true; + // Unread hunks leave the lines unknown, and are read again next time. + const unread = yield* provider.getChangeRequestSummary!(target); + assert.strictEqual(unread.additions, undefined); + hunksDown = false; const first = yield* provider.getChangeRequestSummary!(target); const second = yield* provider.getChangeRequestSummary!(target); assert.deepInclude(first, { additions: 6, deletions: 2, changedFiles: 2 }); assert.deepInclude(second, { additions: 6, deletions: 2, changedFiles: 2 }); - assert.strictEqual(hunkReads(), 1); - // A landed parent layer moves the comparison base under the same head. - comparisonBaseOid = landedBase; + // Neither the head nor the target tip moved, so the file list is not read again. + assert.deepStrictEqual([reads("/changes"), reads("/diff-files")], [4, 3]); + // The target branch advancing keeps the merge base, so only the file list is read. + tips = { ...tips, observedBaseOid: mainTip }; const third = yield* provider.getChangeRequestSummary!(target); - assert.deepInclude(third, { additions: 2, deletions: 2, changedFiles: 2 }); - assert.strictEqual(hunkReads(), 2); + assert.deepInclude(third, { additions: 6, deletions: 2, changedFiles: 2 }); + assert.deepStrictEqual([reads("/changes"), reads("/diff-files")], [5, 3]); + // A landed parent layer retargets the pull and moves the comparison base under the same head. + comparisonBaseOid = landedBase; + tips = { targetBranch: "trunk", observedBaseOid: mainTip }; + const fourth = yield* provider.getChangeRequestSummary!(target); + assert.deepInclude(fourth, { additions: 2, deletions: 2, changedFiles: 2 }); + assert.deepStrictEqual([reads("/changes"), reads("/diff-files")], [6, 4]); }).pipe(Effect.provide(server.layer)); }); diff --git a/packages/source-control-gitcafe/src/server/GitCafePullRequestProvider.ts b/packages/source-control-gitcafe/src/server/GitCafePullRequestProvider.ts index 1d4b7338f3c12..e87a4e154d69b 100644 --- a/packages/source-control-gitcafe/src/server/GitCafePullRequestProvider.ts +++ b/packages/source-control-gitcafe/src/server/GitCafePullRequestProvider.ts @@ -636,11 +636,19 @@ export const make = Effect.gen(function* () { // so hunk totals are held per compared pair of commits. The file list that names the pair is // cheap; a retarget or a landed parent layer moves the base without moving the head. const lineTotals = new Map(); + // The compared pair follows from the head and the target tip the pull reports, so while neither + // moves the stats are reused without listing the files again. + const statsByTips = new Map< + string, + { readonly changedFiles: number; readonly additions?: number; readonly deletions?: number } + >(); + const remember = (cache: Map, key: K, value: V) => { + if (cache.size >= LINE_STATS_CACHE_SIZE) cache.delete(cache.keys().next().value!); + cache.set(key, value); + }; - const readLineStats = Effect.fn("GitCafePullRequestProvider.readLineStats")(function* ( - input: PullRef, - version: number, - ) { + // readLineStats owns the span. + const countLines = Effect.fnUntraced(function* (input: PullRef, version: number) { const path = yield* pullPath("getChangeRequest", input); const files: Array = []; let revision: { version: number; headOid: string; baseOid: string } | undefined; @@ -667,13 +675,23 @@ export const make = Effect.gen(function* () { after = batch.next; if (after === null || after === previous || batch.items.length === 0) break; } - if (revision === undefined) return { changedFiles: 0, additions: 0, deletions: 0 }; + if (revision === undefined) + return { + headOid: null, + complete: true, + stats: { changedFiles: 0, additions: 0, deletions: 0 }, + }; const key = `${input.host} ${input.repository.toLowerCase()} ${input.number} ${revision.baseOid} ${revision.headOid}`; const held = lineTotals.get(key); - if (held !== undefined) return { changedFiles: files.length, ...held }; + const { headOid } = revision; + // A listing that stopped before its last page counts only part of the diff, so nothing is held. + const listed = after === null; + if (held !== undefined) + return { headOid, complete: listed, stats: { changedFiles: files.length, ...held } }; + const tooMany = files.length > LINE_STATS_MAX_FILES; const details = files.some((file) => file.additions !== undefined) ? Option.some(files) - : files.length > LINE_STATS_MAX_FILES + : tooMany ? Option.none() : yield* Effect.forEach( chunk(detailablePaths(files), DIFF_FILES_BATCH), @@ -684,16 +702,33 @@ export const make = Effect.gen(function* () { Effect.option, ); // Unread hunks leave the lines unknown rather than zero; the file count still stands. - if (Option.isNone(details)) return { changedFiles: files.length }; + if (Option.isNone(details)) + return { headOid, complete: listed && tooMany, stats: { changedFiles: files.length } }; const counted = details.value; const totals = { additions: counted.reduce((total, file) => total + (file.additions ?? 0), 0), deletions: counted.reduce((total, file) => total + (file.deletions ?? 0), 0), }; - if (lineTotals.size >= LINE_STATS_CACHE_SIZE) - lineTotals.delete(lineTotals.keys().next().value!); - lineTotals.set(key, totals); - return { changedFiles: files.length, ...totals }; + if (listed) remember(lineTotals, key, totals); + return { headOid, complete: listed, stats: { changedFiles: files.length, ...totals } }; + }); + + const readLineStats = Effect.fn("GitCafePullRequestProvider.readLineStats")(function* ( + input: PullRef, + pull: Json.GitCafePullDetail, + ) { + const tipsKey = + pull.headOid === null || pull.observedBaseOid == null + ? null + : `${input.host} ${input.repository.toLowerCase()} ${input.number} ${pull.targetBranch} ${pull.observedBaseOid} ${pull.headOid}`; + const heldStats = tipsKey === null ? undefined : statsByTips.get(tipsKey); + if (heldStats !== undefined) return heldStats; + const stats = yield* countLines(input, pull.version); + // A failed hunk read is retried next time rather than held as unknown, and a head that moved + // between the two reads would hold another pair's stats. + if (tipsKey !== null && stats.complete && stats.headOid === pull.headOid) + remember(statsByTips, tipsKey, stats.stats); + return stats.stats; }); const merge = Effect.fn("GitCafePullRequestProvider.merge")(function* ( @@ -870,7 +905,7 @@ export const make = Effect.gen(function* () { const changes = pull.headOid === null ? null - : yield* readLineStats(input, pull.version).pipe(Effect.orElseSucceed(() => null)); + : yield* readLineStats(input, pull).pipe(Effect.orElseSucceed(() => null)); // A count that could not be read is absent rather than zero. const { additions: _additions, @@ -904,7 +939,7 @@ export const make = Effect.gen(function* () { ), pull.headOid === null ? Effect.succeed(null) - : readLineStats(input, pull.version).pipe(Effect.orElseSucceed(() => null)), + : readLineStats(input, pull).pipe(Effect.orElseSucceed(() => null)), // Reviewers need `pr:review:read`, which a contents-only token lacks. readPages("getChangeRequest", input, `${path}/reviewers`, Json.GitCafeActorEntries).pipe( // @effect-diagnostics-next-line catchConditionalRefailToCatchIf:off - catchIf widens this paginated result to any in tsc-rs. diff --git a/packages/source-control-gitcafe/src/shared/definition.ts b/packages/source-control-gitcafe/src/shared/definition.ts index 1ed7fcfd873d1..fce36d4a0b52d 100644 --- a/packages/source-control-gitcafe/src/shared/definition.ts +++ b/packages/source-control-gitcafe/src/shared/definition.ts @@ -7,11 +7,13 @@ import { import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import { + buildCommitUrl, changeRequestUrlBelow, defineSourceControl, isChangeRequestInProjectRepository, isChangeRequestOnProjectHost, parseChangeRequestPath, + parseCommitPath, projectChangeRequestHost, } from "@t3tools/source-control-core/shared/definition"; import { icon } from "./icon.ts"; @@ -21,6 +23,7 @@ const GITCAFE_HOSTS = new Set(["git.cafe", "staging.git.cafe"]); /** GitCafe serves pull requests at `/pull/` and redirects the older `/pulls/` there. */ const CHANGE_REQUEST_PATH = /^\/([^/]+\/[^/]+)\/pulls?\/(\d+)(?:\/|$)/u; +const COMMIT_PATH = /^\/([^/]+\/[^/]+)\/commit\/([0-9a-f]+)\/?$/iu; function parseChangeRequestUrl(url: URL) { return GITCAFE_HOSTS.has(url.hostname.toLowerCase()) @@ -68,6 +71,9 @@ export const definition = defineSourceControl({ // `/pull/` and `/pulls/` are GitHub's and Forgejo's shapes too, so only GitCafe's own hosts // claim them. parseChangeRequestUrl, + commitUrl: (input) => buildCommitUrl(input, "commit"), + parseCommitUrl: (url) => + GITCAFE_HOSTS.has(url.hostname.toLowerCase()) ? parseCommitPath(url, COMMIT_PATH) : null, changeRequestUrlFromReferenceAutolink: () => null, changeRequestActions: new Set([ "ready", diff --git a/packages/source-control-github/src/server/gitHubChangeRequestTemplate.test.ts b/packages/source-control-github/src/server/gitHubChangeRequestTemplate.test.ts index 58b094063ecba..29bf922020d0d 100644 --- a/packages/source-control-github/src/server/gitHubChangeRequestTemplate.test.ts +++ b/packages/source-control-github/src/server/gitHubChangeRequestTemplate.test.ts @@ -227,3 +227,41 @@ it.effect("bounds template reads and marks truncated content", () => }), ), ); + +it.effect.each([ + ".github/pull_request_template.txt", + "PULL_REQUEST_TEMPLATE.TXT", + "docs/PuLl_ReQuEsT_TeMpLaTe.Md", + ".github/PULL_REQUEST_TEMPLATE/template.TxT", +])("reads committed text and mixed-case template $0 from a subdirectory", (relativePath) => + runWithTempDirectory((cwd) => + Effect.gen(function* () { + const path = yield* Path.Path; + yield* runGit(cwd, ["config", "grep.patternType", "fixed"]); + yield* runGit(cwd, ["config", "color.grep", "always"]); + yield* writeTemplate(cwd, relativePath, "committed template"); + yield* writeTemplate(cwd, "nested/README.md", "project"); + for (let index = 0; index < 600; index += 1) { + yield* writeTemplate(cwd, `docs/${index}-${"a".repeat(180)}.md`, "guide"); + } + yield* commitTemplates(cwd); + yield* writeTemplate(cwd, relativePath, "uncommitted template"); + + assert.strictEqual( + Option.getOrUndefined(yield* detectTemplate(path.join(cwd, "nested"))), + "committed template", + ); + }), + ), +); + +it.effect("does not guess between Markdown and text directory templates", () => + runWithTempDirectory((cwd) => + Effect.gen(function* () { + yield* writeTemplate(cwd, ".github/PULL_REQUEST_TEMPLATE/a.md", "markdown"); + yield* writeTemplate(cwd, ".github/PULL_REQUEST_TEMPLATE/b.TXT", "text"); + yield* commitTemplates(cwd); + assert.isTrue(Option.isNone(yield* detectTemplate(cwd))); + }), + ), +); diff --git a/packages/source-control-github/src/server/gitHubChangeRequestTemplate.ts b/packages/source-control-github/src/server/gitHubChangeRequestTemplate.ts index c6af80389d1f7..9edbccdd504dc 100644 --- a/packages/source-control-github/src/server/gitHubChangeRequestTemplate.ts +++ b/packages/source-control-github/src/server/gitHubChangeRequestTemplate.ts @@ -11,10 +11,16 @@ const TRUNCATION_MARKER = "[truncated]"; const TEMPLATE_PATHS = [ ".github/pull_request_template.md", ".github/PULL_REQUEST_TEMPLATE.md", + ".github/pull_request_template.txt", + ".github/PULL_REQUEST_TEMPLATE.txt", "pull_request_template.md", "PULL_REQUEST_TEMPLATE.md", + "pull_request_template.txt", + "PULL_REQUEST_TEMPLATE.txt", "docs/pull_request_template.md", "docs/PULL_REQUEST_TEMPLATE.md", + "docs/pull_request_template.txt", + "docs/PULL_REQUEST_TEMPLATE.txt", ] as const; const TEMPLATE_DIRECTORIES = [ @@ -23,8 +29,6 @@ const TEMPLATE_DIRECTORIES = [ "docs/PULL_REQUEST_TEMPLATE", ] as const; -const TREE_PATHS = [...TEMPLATE_PATHS, ...TEMPLATE_DIRECTORIES] as const; - type ExecuteGit = SourceControlHost.SourceControlHost["Service"]["git"]["execute"]; interface TemplateTreeEntry { @@ -105,7 +109,7 @@ const readTemplateDirectory = Effect.fn("gitHubChangeRequestTemplate.readTemplat return false; } const relativePath = entry.path.slice(prefix.length); - return !relativePath.includes("/") && relativePath.toLowerCase().endsWith(".md"); + return !relativePath.includes("/") && /\.(?:md|txt)$/i.test(relativePath); }); const templates: string[] = []; @@ -132,10 +136,50 @@ export const detect = Effect.fn("gitHubChangeRequestTemplate.detect")(function* ) { return yield* Effect.gen(function* () { // Read committed regular blobs; worktree paths can become symlinks between validation and open. + // ls-tree does not support case-insensitive pathspecs; grep discovers only matching names. + const paths = yield* executeGit({ + operation: "gitHubChangeRequestTemplate.findTemplatePaths", + cwd, + args: [ + "grep", + "-G", + "--no-color", + "-l", + "-z", + "--full-name", + "--no-recurse-submodules", + "-e", + "^", + treeish, + "--", + ...TEMPLATE_PATHS.map((path) => `:(top,icase,literal)${path}`), + ], + maxOutputBytes: TREE_LIST_MAX_BYTES, + }).pipe( + Effect.map((result) => + result.stdoutTruncated + ? [] + : result.stdout + .split("\0") + .filter((path) => path.startsWith(`${treeish}:`)) + .map((path) => path.slice(treeish.length + 1)), + ), + // grep exits with status 1 when no nonempty templates match. + Effect.orElseSucceed(() => []), + ); const result = yield* executeGit({ operation: "gitHubChangeRequestTemplate.listTemplates", cwd, - args: ["ls-tree", "-r", "-z", "--full-tree", treeish, "--", ...TREE_PATHS], + args: [ + "ls-tree", + "-r", + "-z", + "--full-tree", + treeish, + "--", + ...new Set([...TEMPLATE_PATHS, ...paths]), + ...TEMPLATE_DIRECTORIES, + ], maxOutputBytes: TREE_LIST_MAX_BYTES, appendTruncationMarker: true, }); @@ -144,12 +188,20 @@ export const detect = Effect.fn("gitHubChangeRequestTemplate.detect")(function* } const entries = parseTemplateTreeEntries(result.stdout); - const entriesByPath = new Map(entries.map((entry) => [entry.path, entry])); - for (const templatePath of TEMPLATE_PATHS) { - const entry = entriesByPath.get(templatePath); - if (!entry) { - continue; - } + const candidates = [".github/", "", "docs/"].flatMap((location) => { + const paths = TEMPLATE_PATHS.filter( + (path) => path.slice(0, path.lastIndexOf("/") + 1) === location, + ); + return [ + ...paths.flatMap((path) => entries.filter((entry) => entry.path === path)), + ...entries.filter( + (entry) => + !paths.some((path) => path === entry.path) && + paths.some((path) => path.toLowerCase() === entry.path.toLowerCase()), + ), + ]; + }); + for (const entry of candidates) { const template = yield* readTemplateBlob({ cwd, executeGit, entry }); if (Option.isSome(template)) { return template; diff --git a/packages/source-control-github/src/shared/definition.ts b/packages/source-control-github/src/shared/definition.ts index 965ed42814c19..6ac8aa305f3cd 100644 --- a/packages/source-control-github/src/shared/definition.ts +++ b/packages/source-control-github/src/shared/definition.ts @@ -1,11 +1,13 @@ import { makeProviderSettingsSchema, SourceControlProviderKind } from "@t3tools/contracts"; import { + buildCommitUrl, changeRequestUrlBelow, defineSourceControl, hostCredentialSettingsFields, isChangeRequestInProjectRepository, isChangeRequestOnProjectHost, parseChangeRequestPath, + parseCommitPath, projectChangeRequestHost, remoteWebOrigin, type SourceControlAppSignIn, @@ -22,15 +24,14 @@ const CHANGE_REQUEST_REFERENCE = // GitHub Enterprise Server runs on any hostname, so the `//pull/` route alone // identifies a GitHub pull request. const CHANGE_REQUEST_PATH = /^\/([^/]+\/[^/]+)\/pull\/(\d+)(?:\/|$)/u; +const COMMIT_PATH = /^\/([^/]+\/[^/]+)\/(?:commit|pull\/\d+\/commits)\/([0-9a-f]+)\/?$/iu; const ISSUE_PATH = /^\/([^/]+\/[^/]+)\/issues\/(\d+)(?:\/|$)/u; const KIND = SourceControlProviderKind.make("github"); function isGitHubHost(hostname: string): boolean { const host = hostname.toLowerCase(); - return ( - host === "github.com" || host.endsWith(".github.com") || host.split(".").includes("github") - ); + return host === "github.com" || host.endsWith(".ghe.com") || host.split(".").includes("github"); } function isRepositoryPath(repository: string): boolean { @@ -91,6 +92,11 @@ export const definition = defineSourceControl({ return null; } }, + commitUrl: (input) => + isRepositoryPath(input.repository) ? buildCommitUrl(input, "commit") : null, + // A known project's host can be any GitHub Enterprise Server name. + parseCommitUrl: (url, known) => + known || isGitHubHost(url.hostname) ? parseCommitPath(url, COMMIT_PATH) : null, parseChangeRequestUrl: (url) => parseChangeRequestPath(url, CHANGE_REQUEST_PATH), // GitHub writes every bare reference through `/issues/`, including pull requests. changeRequestUrlFromReferenceAutolink: (url) => { @@ -133,9 +139,7 @@ export const definition = defineSourceControl({ changeRequestHeadRef: (number) => `refs/pull/${number}/head`, // `*.ghe.com` is GitHub Enterprise Cloud with data residency, one subdomain per enterprise. recognizeRemote: ({ host, hostname }) => - hostname === "github.com" || - hostname.endsWith(".ghe.com") || - hostname.split(".").includes("github") + isGitHubHost(hostname) ? { kind: KIND, name: hostname === "github.com" ? "GitHub" : "GitHub Self-Hosted", diff --git a/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.test.ts b/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.test.ts index 6eec9ccd09142..cc6bab6c53f42 100644 --- a/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.test.ts +++ b/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.test.ts @@ -1,6 +1,7 @@ import { assert, describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as TestClock from "effect/testing/TestClock"; import * as GitLabCli from "./GitLabCli.ts"; import * as GitLabPullRequestCli from "./GitLabPullRequestCli.ts"; @@ -122,6 +123,157 @@ describe("getChangeRequest base freshness", () => { ); }); +describe("merge request reads", () => { + const reference = { cwd: "/w", repository: "acme/web", host: "gitlab.com", number: 7 }; + const detail = { + number: 7, + title: "Merge request 7", + url: "https://gitlab.com/acme/web/-/merge_requests/7", + author: null, + headBranch: "feat/page", + baseBranch: "main", + state: "open" as const, + isDraft: false, + mergeability: "mergeable" as const, + additions: 0, + deletions: 0, + createdAt: "2026-07-01T00:00:00Z", + updatedAt: "2026-07-02T00:00:00Z", + reviewRequestLogins: [], + labels: [], + body: "", + changedFiles: 1, + mergedAt: null, + closedAt: null, + reviewers: [], + checks: [], + viewerCanMerge: true, + reviewerIds: [], + }; + + const countingProvider = Effect.gen(function* () { + const calls = { detail: 0, mergeCapabilities: 0 }; + const host = { viewerCanMerge: true }; + const provider = yield* make.pipe( + Effect.provide( + Layer.mock(GitLabPullRequestCli.GitLabPullRequestCli)({ + getMergeRequestDetail: (input) => + Effect.sync(() => { + calls.detail += 1; + return { ...detail, number: input.number, viewerCanMerge: host.viewerCanMerge }; + }), + getProjectMergeCapabilities: () => + Effect.sync(() => { + calls.mergeCapabilities += 1; + return { merge: true, squash: true, rebase: true }; + }), + runMergeRequestAction: () => Effect.void, + }), + ), + ); + const checks = provider.getChangeRequestChecks; + const forget = provider.forgetChangeRequestReads; + if (checks === undefined || forget === undefined) { + return yield* Effect.die("GitLab answers checks and forgets its reads"); + } + return { provider, checks, forget, calls, host }; + }); + + it.effect("asks for a project's merge methods once across its merge requests", () => + Effect.gen(function* () { + const { provider, calls } = yield* countingProvider; + + yield* provider.getChangeRequest(reference); + yield* provider.getChangeRequest(reference); + yield* provider.getChangeRequest({ ...reference, number: 8 }); + + expect(calls.mergeCapabilities).toBe(1); + }), + ); + + it.effect("answers checks from the merge request the detail just read", () => + Effect.gen(function* () { + const { provider, checks, calls } = yield* countingProvider; + + yield* provider.getChangeRequest(reference); + yield* checks(reference); + + expect(calls.detail).toBe(1); + }), + ); + + it.effect("reads a merge request again after a write to it", () => + Effect.gen(function* () { + const { provider, checks, calls } = yield* countingProvider; + + yield* provider.getChangeRequest(reference); + yield* provider.runAction({ ...reference, action: "ready" }); + yield* checks(reference); + + expect(calls.detail).toBe(2); + }), + ); + + it.effect("does not share reads between checkouts", () => + Effect.gen(function* () { + const { provider, calls } = yield* countingProvider; + + yield* provider.getChangeRequest(reference); + yield* provider.getChangeRequest({ ...reference, cwd: "/other" }); + + expect(calls).toEqual({ detail: 2, mergeCapabilities: 2 }); + }), + ); + + it.effect("reads a merge request again once a minute has passed", () => + Effect.gen(function* () { + const { provider, calls } = yield* countingProvider; + + yield* provider.getChangeRequest(reference); + yield* TestClock.adjust("61 seconds"); + yield* provider.getChangeRequest(reference); + + expect(calls.detail).toBe(2); + }), + ); + + it.effect("answers viewer permissions from the host, not the page's read", () => + Effect.gen(function* () { + const { provider, host } = yield* countingProvider; + + host.viewerCanMerge = false; + yield* provider.getChangeRequest(reference); + host.viewerCanMerge = true; + const granted = yield* provider.getViewerPermissions(reference); + host.viewerCanMerge = false; + const revoked = yield* provider.getViewerPermissions(reference); + + expect(granted.actions).toContain("merge"); + expect(revoked.actions).not.toContain("merge"); + }), + ); + + it.effect("reads a merge request again once its reads are forgotten", () => + Effect.gen(function* () { + const { provider, checks, forget, calls } = yield* countingProvider; + const other = { ...reference, repository: "acme/api" }; + + yield* provider.getChangeRequest(reference); + yield* provider.getChangeRequest(other); + yield* forget({ repository: "ACME/web", number: 7 }); + yield* checks(reference); + yield* checks(other); + expect(calls.detail).toBe(3); + + yield* forget({ repository: "acme/web" }); + yield* checks(reference); + yield* forget(); + yield* checks(other); + expect(calls.detail).toBe(5); + }), + ); +}); + describe("optional activity reads", () => { const reference = { cwd: "/w", repository: "acme/web", host: "gitlab.com", number: 7 }; const comment = { diff --git a/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.ts b/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.ts index a9856f907ebb8..79c3d43c01966 100644 --- a/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.ts +++ b/packages/source-control-gitlab/src/server/GitLabPullRequestProvider.ts @@ -1,5 +1,8 @@ import { SourceControlProviderKind } from "@t3tools/contracts"; +import * as Cache from "effect/Cache"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import type { PullRequestCapabilities, PullRequestReaction, @@ -72,6 +75,16 @@ export function gitLabViewerPermissions(input: { }; } +/** + * One page view asks for the detail and then its checks within a second, and the poll that + * follows asks again. A minute keeps those to one `glab` call without hiding a pipeline for long; + * a refresh or a finished turn drops it sooner through `forgetChangeRequestReads`. + */ +const MERGE_REQUEST_DETAIL_TTL = Duration.minutes(1); +/** A project's merge methods change in its settings, which nobody does between page views. */ +const PROJECT_MERGE_CAPABILITIES_TTL = Duration.minutes(15); +const READ_CACHE_CAPACITY = 256; + function gitLabProviderFailure( error: GitLabPullRequestCli.GitLabPullRequestCliError, ): PullRequestProviderFailure { @@ -85,6 +98,55 @@ function gitLabProviderFailure( export const make = Effect.gen(function* () { const cli = yield* GitLabPullRequestCli.GitLabPullRequestCli; + // Keyed on the checkout as well: `glab` picks the host and the account to answer as from the + // checkout's remote and config, so two checkouts may read the same project as different people. + const mergeRequestDetails = yield* Cache.makeWith( + (key: string) => { + const [cwd, repository, number] = JSON.parse(key) as [string, string, number]; + return cli.getMergeRequestDetail({ cwd, repository, number }); + }, + { + capacity: READ_CACHE_CAPACITY, + timeToLive: (exit) => (Exit.isSuccess(exit) ? MERGE_REQUEST_DETAIL_TTL : Duration.zero), + }, + ); + const projectMergeCapabilities = yield* Cache.makeWith( + (key: string) => { + const [cwd, repository] = JSON.parse(key) as [string, string]; + return cli.getProjectMergeCapabilities({ cwd, repository }); + }, + { + capacity: READ_CACHE_CAPACITY, + timeToLive: (exit) => (Exit.isSuccess(exit) ? PROJECT_MERGE_CAPABILITIES_TTL : Duration.zero), + }, + ); + + const detailKey = (input: { + readonly cwd: string; + readonly repository: string; + readonly number: number; + }) => JSON.stringify([input.cwd, input.repository, input.number]); + const mergeRequestDetail = (input: { + readonly cwd: string; + readonly repository: string; + readonly number: number; + }) => Cache.get(mergeRequestDetails, detailKey(input)); + + // Dropped on both sides, as the service does with its own reads: a read that starts while the + // write is in flight must not outlive it. + const writing = + < + I extends { readonly cwd: string; readonly repository: string; readonly number: number }, + A, + E, + >( + write: (input: I) => Effect.Effect, + ) => + (input: I): Effect.Effect => { + const invalidate = Cache.invalidate(mergeRequestDetails, detailKey(input)); + return invalidate.pipe(Effect.andThen(write(input)), Effect.ensuring(invalidate)); + }; + const fail = (operation: string) => (error: GitLabPullRequestCli.GitLabPullRequestCliError) => new PullRequestProviderError({ provider: SourceControlProviderKind.make("gitlab"), @@ -138,7 +200,7 @@ export const make = Effect.gen(function* () { ), getChangeRequestChecks: (input) => - cli.getMergeRequestDetail(input).pipe( + mergeRequestDetail(input).pipe( Effect.map(({ state, checks }) => ({ state, checks })), Effect.mapError(fail("getChangeRequestChecks")), ), @@ -153,8 +215,8 @@ export const make = Effect.gen(function* () { getChangeRequest: (input) => Effect.all( [ - cli.getMergeRequestDetail(input), - cli.getProjectMergeCapabilities({ cwd: input.cwd, repository: input.repository }), + mergeRequestDetail(input), + Cache.get(projectMergeCapabilities, JSON.stringify([input.cwd, input.repository])), ], { concurrency: 2 }, ).pipe( @@ -209,11 +271,31 @@ export const make = Effect.gen(function* () { })), ), + // Straight from the host, never the shared read: this is what a write is refused or let + // through on, and access granted or withdrawn within the minute must count. getViewerPermissions: (input) => cli .getMergeRequestDetail(input) .pipe(Effect.mapError(fail("getViewerPermissions")), Effect.map(gitLabViewerPermissions)), + forgetChangeRequestReads: (scope) => + Cache.keys(mergeRequestDetails).pipe( + Effect.flatMap((keys) => + Effect.forEach( + Array.from(keys).filter((key) => { + if (scope === undefined) return true; + const [, repository, number] = JSON.parse(key) as [string, string, number]; + return ( + repository.toLowerCase() === scope.repository.toLowerCase() && + (scope.number === undefined || number === scope.number) + ); + }), + (key) => Cache.invalidate(mergeRequestDetails, key), + { discard: true }, + ), + ), + ), + getDiff: (input) => cli.getMergeRequestDiff(input).pipe(Effect.mapError(fail("getDiff"))), getDiffFileContents: (input) => @@ -245,7 +327,7 @@ export const make = Effect.gen(function* () { .listProjectUsers({ cwd: input.cwd, repository: input.repository, search: input.query }) .pipe(Effect.mapError(fail("searchRepositoryAuthors"))), - setReviewerRequest: (input) => + setReviewerRequest: writing((input) => cli .setReviewerRequest({ cwd: input.cwd, @@ -255,8 +337,9 @@ export const make = Effect.gen(function* () { requested: input.requested, }) .pipe(Effect.mapError(fail("setReviewerRequest"))), + ), - runAction: (input) => + runAction: writing((input) => cli .runMergeRequestAction({ cwd: input.cwd, @@ -266,8 +349,9 @@ export const make = Effect.gen(function* () { ...(input.mergeMethod === undefined ? {} : { mergeMethod: input.mergeMethod }), }) .pipe(Effect.mapError(fail("runAction"))), + ), - updateChangeRequest: (input) => + updateChangeRequest: writing((input) => cli .updateMergeRequest({ cwd: input.cwd, @@ -277,10 +361,13 @@ export const make = Effect.gen(function* () { ...(input.body === undefined ? {} : { description: input.body }), }) .pipe(Effect.mapError(fail("updateChangeRequest"))), + ), - comment: (input) => cli.commentOnMergeRequest(input).pipe(Effect.mapError(fail("comment"))), + comment: writing((input) => + cli.commentOnMergeRequest(input).pipe(Effect.mapError(fail("comment"))), + ), - updateComment: (input) => + updateComment: writing((input) => cli .updateNote({ cwd: input.cwd, @@ -290,10 +377,13 @@ export const make = Effect.gen(function* () { body: input.body, }) .pipe(Effect.mapError(fail("updateComment"))), + ), - submitReview: (input) => cli.submitReview(input).pipe(Effect.mapError(fail("submitReview"))), + submitReview: writing((input) => + cli.submitReview(input).pipe(Effect.mapError(fail("submitReview"))), + ), - replyToThread: (input) => + replyToThread: writing((input) => cli .replyToDiscussion({ cwd: input.cwd, @@ -303,8 +393,9 @@ export const make = Effect.gen(function* () { body: input.body, }) .pipe(Effect.mapError(fail("replyToThread"))), + ), - setReaction: (input) => + setReaction: writing((input) => cli .setReaction({ cwd: input.cwd, @@ -315,8 +406,9 @@ export const make = Effect.gen(function* () { reacted: input.reacted, }) .pipe(Effect.mapError(fail("setReaction"))), + ), - setThreadResolution: (input) => + setThreadResolution: writing((input) => cli .setDiscussionResolution({ cwd: input.cwd, @@ -326,6 +418,7 @@ export const make = Effect.gen(function* () { resolved: input.resolved, }) .pipe(Effect.mapError(fail("setThreadResolution"))), + ), }; return provider; diff --git a/packages/source-control-gitlab/src/shared/definition.ts b/packages/source-control-gitlab/src/shared/definition.ts index 437320c0d0419..2d280f01c62f2 100644 --- a/packages/source-control-gitlab/src/shared/definition.ts +++ b/packages/source-control-gitlab/src/shared/definition.ts @@ -1,10 +1,12 @@ import { SourceControlProviderKind } from "@t3tools/contracts"; import { + buildCommitUrl, changeRequestUrlBelow, defineSourceControl, isChangeRequestInProjectRepository, isChangeRequestOnProjectHost, parseChangeRequestPath, + parseCommitPath, projectChangeRequestHost, } from "@t3tools/source-control-core/shared/definition"; import { icon } from "./icon.ts"; @@ -15,6 +17,7 @@ const CHANGE_REQUEST_REFERENCE = // Self-managed GitLab runs on any hostname and nests projects in groups. const CHANGE_REQUEST_PATH = /^\/([^/]+(?:\/[^/]+)+)\/-\/merge_requests\/(\d+)(?:\/|$)/u; +const COMMIT_PATH = /^\/([^/]+(?:\/[^/]+)+)\/-\/commit\/([0-9a-f]+)\/?$/iu; const KIND = SourceControlProviderKind.make("gitlab"); @@ -40,6 +43,8 @@ export const definition = defineSourceControl({ ? changeRequestUrlBelow(webUrl, `-/merge_requests/${number}`) : `https://${host}/${repository}/-/merge_requests/${number}`, parseChangeRequestUrl: (url) => parseChangeRequestPath(url, CHANGE_REQUEST_PATH), + commitUrl: (input) => buildCommitUrl(input, "-/commit"), + parseCommitUrl: (url) => parseCommitPath(url, COMMIT_PATH), changeRequestUrlFromReferenceAutolink: () => null, changeRequestActions: new Set([ "merge", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9d603115d6b4a..b311c44049778 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -955,6 +955,21 @@ importers: '@t3tools/provider-testing': specifier: workspace:* version: link:../../packages/provider-testing + '@t3tools/secret-provider-aws': + specifier: workspace:* + version: link:../../packages/secret-provider-aws + '@t3tools/secret-provider-core': + specifier: workspace:* + version: link:../../packages/secret-provider-core + '@t3tools/secret-provider-definitions': + specifier: workspace:* + version: link:../../packages/secret-provider-definitions + '@t3tools/secret-provider-onepassword': + specifier: workspace:* + version: link:../../packages/secret-provider-onepassword + '@t3tools/secret-provider-openbao': + specifier: workspace:* + version: link:../../packages/secret-provider-openbao '@t3tools/shared': specifier: workspace:* version: link:../../packages/shared @@ -1060,6 +1075,18 @@ importers: '@t3tools/provider-core': specifier: workspace:* version: link:../../packages/provider-core + '@t3tools/secret-provider-aws': + specifier: workspace:* + version: link:../../packages/secret-provider-aws + '@t3tools/secret-provider-definitions': + specifier: workspace:* + version: link:../../packages/secret-provider-definitions + '@t3tools/secret-provider-onepassword': + specifier: workspace:* + version: link:../../packages/secret-provider-onepassword + '@t3tools/secret-provider-openbao': + specifier: workspace:* + version: link:../../packages/secret-provider-openbao '@t3tools/shared': specifier: workspace:* version: link:../../packages/shared @@ -1890,6 +1917,136 @@ importers: specifier: 24.12.4 version: 24.12.4 + packages/secret-provider-aws: + dependencies: + '@t3tools/contracts': + specifier: workspace:* + version: link:../contracts + '@t3tools/provider-core': + specifier: workspace:* + version: link:../provider-core + '@t3tools/secret-provider-core': + specifier: workspace:* + version: link:../secret-provider-core + '@t3tools/shared': + specifier: workspace:* + version: link:../shared + effect: + specifier: 4.0.5 + version: 4.0.5 + devDependencies: + '@effect/platform-node': + specifier: 4.0.5 + version: 4.0.5(bufferutil@4.1.0)(effect@4.0.5)(redis@6.2.1)(utf-8-validate@6.0.6) + '@effect/vitest': + specifier: 4.0.5 + version: 4.0.5(patch_hash=2731c9c3761c595f25ddc625a7d61ca376b389d5b3049ecb837c8940f21cfa02)(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(effect@4.0.5)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + '@types/node': + specifier: 24.12.4 + version: 24.12.4 + vite-plus: + specifier: 1.4.0 + version: 1.4.0(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + + packages/secret-provider-core: + dependencies: + '@t3tools/contracts': + specifier: workspace:* + version: link:../contracts + effect: + specifier: 4.0.5 + version: 4.0.5 + devDependencies: + '@effect/platform-node': + specifier: 4.0.5 + version: 4.0.5(bufferutil@4.1.0)(effect@4.0.5)(redis@6.2.1)(utf-8-validate@6.0.6) + '@effect/vitest': + specifier: 4.0.5 + version: 4.0.5(patch_hash=2731c9c3761c595f25ddc625a7d61ca376b389d5b3049ecb837c8940f21cfa02)(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(effect@4.0.5)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + '@types/node': + specifier: 24.12.4 + version: 24.12.4 + vite-plus: + specifier: 1.4.0 + version: 1.4.0(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + + packages/secret-provider-definitions: + dependencies: + '@t3tools/secret-provider-aws': + specifier: workspace:* + version: link:../secret-provider-aws + '@t3tools/secret-provider-core': + specifier: workspace:* + version: link:../secret-provider-core + '@t3tools/secret-provider-onepassword': + specifier: workspace:* + version: link:../secret-provider-onepassword + '@t3tools/secret-provider-openbao': + specifier: workspace:* + version: link:../secret-provider-openbao + + packages/secret-provider-onepassword: + dependencies: + '@t3tools/contracts': + specifier: workspace:* + version: link:../contracts + '@t3tools/provider-core': + specifier: workspace:* + version: link:../provider-core + '@t3tools/secret-provider-core': + specifier: workspace:* + version: link:../secret-provider-core + '@t3tools/shared': + specifier: workspace:* + version: link:../shared + effect: + specifier: 4.0.5 + version: 4.0.5 + devDependencies: + '@effect/platform-node': + specifier: 4.0.5 + version: 4.0.5(bufferutil@4.1.0)(effect@4.0.5)(redis@6.2.1)(utf-8-validate@6.0.6) + '@effect/vitest': + specifier: 4.0.5 + version: 4.0.5(patch_hash=2731c9c3761c595f25ddc625a7d61ca376b389d5b3049ecb837c8940f21cfa02)(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(effect@4.0.5)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + '@types/node': + specifier: 24.12.4 + version: 24.12.4 + vite-plus: + specifier: 1.4.0 + version: 1.4.0(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + + packages/secret-provider-openbao: + dependencies: + '@t3tools/contracts': + specifier: workspace:* + version: link:../contracts + '@t3tools/provider-core': + specifier: workspace:* + version: link:../provider-core + '@t3tools/secret-provider-core': + specifier: workspace:* + version: link:../secret-provider-core + '@t3tools/shared': + specifier: workspace:* + version: link:../shared + effect: + specifier: 4.0.5 + version: 4.0.5 + devDependencies: + '@effect/platform-node': + specifier: 4.0.5 + version: 4.0.5(bufferutil@4.1.0)(effect@4.0.5)(redis@6.2.1)(utf-8-validate@6.0.6) + '@effect/vitest': + specifier: 4.0.5 + version: 4.0.5(patch_hash=2731c9c3761c595f25ddc625a7d61ca376b389d5b3049ecb837c8940f21cfa02)(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(effect@4.0.5)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + '@types/node': + specifier: 24.12.4 + version: 24.12.4 + vite-plus: + specifier: 1.4.0 + version: 1.4.0(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + packages/shared: dependencies: '@noble/curves': diff --git a/scripts/build-npm-launcher.test.ts b/scripts/build-npm-launcher.test.ts index 749e27bd5454a..4c7edd4763359 100644 --- a/scripts/build-npm-launcher.test.ts +++ b/scripts/build-npm-launcher.test.ts @@ -195,6 +195,118 @@ it.layer(NodeServices.layer)("build-npm-launcher", (it) => { VERSION, ]); + // A peer's executable and sidecar stay available after every later rename or removal, + // even when it publishes after our missing-marker read. + const staleHome = path.join(root, "stale-home"); + const staleDir = path.join(staleHome, "runtime/versions", VERSION); + const peerDir = path.join(root, "peer-runtime"); + yield* fs.makeDirectory(staleDir, { recursive: true }); + yield* fs.writeFileString(path.join(staleDir, "t3"), ""); + yield* fs.makeDirectory(peerDir); + yield* fs.writeFileString(path.join(peerDir, "t3"), '#!/bin/sh\necho "peer $*"\nexit 7\n'); + yield* fs.chmod(path.join(peerDir, "t3"), 0o755); + yield* fs.writeFileString(path.join(peerDir, ".install-complete"), `${VERSION}\n`); + yield* fs.writeFileString(path.join(peerDir, "sidecar"), "peer sidecar"); + const peerChecks = path.join(root, "peer-checks.txt"); + // The peer publishes after the launcher's rename fails and it reads the marker. + const peerHook = path.join(root, "peer-publishes.mjs"); + yield* fs.writeFileString( + peerHook, + `import fs from "node:fs"; +import { spawnSync } from "node:child_process"; +const dir = ${JSON.stringify(staleDir)}; +const { readFileSync, renameSync, rmSync } = fs; +let checking = false; +let published = false; +const checkPeer = () => { + const child = spawnSync(dir + "/t3", ["--version"], { encoding: "utf8" }); + let sidecar; + try { + sidecar = readFileSync(dir + "/sidecar", "utf8"); + } catch (error) { + sidecar = error.code; + } + fs.appendFileSync(${JSON.stringify(peerChecks)}, + [child.status, child.error?.code ?? child.stdout.trim(), sidecar].join("|") + "\\n"); +}; +fs.renameSync = (from, to) => { + try { + return renameSync(from, to); + } catch (error) { + checking ||= to === dir; + throw error; + } finally { + if (published) checkPeer(); + } +}; +fs.rmSync = (...args) => { + try { + return rmSync(...args); + } finally { + if (published) checkPeer(); + } +}; +fs.readFileSync = (file, ...rest) => { + try { + return readFileSync(file, ...rest); + } finally { + if (checking && file === dir + "/.install-complete") { + checking = false; + rmSync(dir, { recursive: true }); + renameSync(${JSON.stringify(peerDir)}, dir); + published = true; + checkPeer(); + } + } +}; +`, + ); + const recovered = yield* run( + process.execPath, + ["--import", peerHook, launcher, "--version"], + { cwd: root, env: { ...env, T3CODE_HOME: staleHome } }, + ); + const checks = (yield* fs.readFileString(peerChecks)).trim().split("\n"); + assert.isAtLeast(checks.length, 2); + assert.deepStrictEqual([...new Set(checks)], ["7|peer --version|peer sidecar"]); + assert.equal(recovered.stdout.trim(), "peer --version", recovered.stderr); + assert.equal(recovered.exitCode, 7); + assert.deepStrictEqual(yield* fs.readDirectory(path.join(staleHome, "runtime/versions")), [ + VERSION, + ]); + + // While another launch holds the lock, a stale folder is left alone and the error says + // how to clear a lock left by a crash. + const staleOnlyHome = path.join(root, "stale-only-home"); + const staleOnlyDir = path.join(staleOnlyHome, "runtime/versions", VERSION); + yield* fs.makeDirectory(staleOnlyDir, { recursive: true }); + yield* fs.writeFileString(path.join(staleOnlyDir, "t3"), "incomplete"); + yield* fs.makeDirectory(`${staleOnlyDir}.lock`); + const locked = yield* run(process.execPath, [launcher, "--version"], { + cwd: root, + env: { ...env, T3CODE_HOME: staleOnlyHome }, + }); + assert.equal(locked.exitCode, 1); + assert.include(locked.stderr, `delete ${staleOnlyDir}.lock`); + assert.equal(yield* fs.readFileString(path.join(staleOnlyDir, "t3")), "incomplete"); + + // Without a peer, the stale folder is replaced by our own download. + yield* fs.remove(`${staleOnlyDir}.lock`, { recursive: true }); + const replaced = yield* run(process.execPath, [launcher, "--version"], { + cwd: root, + env: { ...env, T3CODE_HOME: staleOnlyHome }, + }); + assert.equal( + replaced.stdout.trim(), + `stub --version launcher=${launcher}`, + replaced.stderr, + ); + assert.equal(replaced.exitCode, 7); + assert.deepStrictEqual( + yield* fs.readDirectory(path.join(staleOnlyHome, "runtime/versions")), + [VERSION], + ); + release.corruptChecksum(); const corrupt = yield* run(process.execPath, [launcher, "--version"], { cwd: root, diff --git a/scripts/check-changed.test.ts b/scripts/check-changed.test.ts new file mode 100644 index 0000000000000..2bcd623a0b4b6 --- /dev/null +++ b/scripts/check-changed.test.ts @@ -0,0 +1,154 @@ +import { assert, describe, it } from "@effect/vitest"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { ChildProcessSpawner } from "effect/process"; +import * as TestClock from "effect/testing/TestClock"; + +import { defaultConcurrency, gitLines, runSteps, type Step } from "./check-changed.ts"; + +const GiB = 1024 ** 3; + +describe("check:changed", () => { + it("bounds the default concurrency by the scarcer of cores and available memory", () => { + assert.strictEqual(defaultConcurrency(32, 128 * GiB), 16); + assert.strictEqual(defaultConcurrency(32, 20 * GiB), 3); + assert.strictEqual(defaultConcurrency(4, 15 * GiB), 2); + assert.strictEqual(defaultConcurrency(4, 2 * GiB), 1); + assert.strictEqual(defaultConcurrency(1, 64 * GiB), 1); + }); + + it.effect( + "keeps weighted steps within the budget, in order, and serial steps one at a time", + () => + Effect.gen(function* () { + const startedAt: Record = {}; + let active = 0; + let maxActive = 0; + let serialActive = 0; + let maxSerial = 0; + const step = ( + name: string, + options: { weight?: number; serial?: boolean } = {}, + ): Step => { + const slots = Math.min(options.weight ?? 1, 3); + return { + name, + ...options, + run: Effect.gen(function* () { + startedAt[name] = (yield* Clock.currentTimeMillis) / 1000; + active += slots; + maxActive = Math.max(maxActive, active); + if (options.serial) maxSerial = Math.max(maxSerial, ++serialActive); + yield* Effect.sleep("1 second"); + active -= slots; + if (options.serial) serialActive--; + return { ok: true, output: "" }; + }), + }; + }; + const fiber = yield* runSteps( + [ + step("light 1"), + // Above the budget, so it holds every slot rather than never starting. + step("heavy", { weight: 5 }), + step("pair", { weight: 2 }), + step("light 2"), + step("test 1", { serial: true }), + step("test 2", { serial: true }), + ], + { concurrency: 3, timeoutSeconds: 60 }, + ).pipe(Effect.forkChild); + yield* TestClock.adjust("10 seconds"); + const results = yield* Fiber.join(fiber); + + assert.deepStrictEqual( + results.map((result) => result.outcome), + ["pass", "pass", "pass", "pass", "pass", "pass"], + ); + assert.strictEqual(maxActive, 3); + // Lighter steps queued behind the heavy one never start ahead of it. + assert.deepStrictEqual(startedAt, { + "light 1": 0, + heavy: 1, + pair: 2, + "light 2": 2, + "test 1": 3, + "test 2": 4, + }); + assert.strictEqual(maxSerial, 1); + }), + ); + + it.effect("interrupts a step at its timeout and reports TIMEOUT, apart from FAIL", () => + Effect.gen(function* () { + let interrupted = false; + const fiber = yield* runSteps( + [ + { + name: "hangs", + run: Effect.never.pipe( + Effect.onInterrupt(() => Effect.sync(() => (interrupted = true))), + ), + }, + { name: "fails", run: Effect.succeed({ ok: false, output: "error" }) }, + { name: "passes", run: Effect.succeed({ ok: true, output: "" }) }, + ], + { concurrency: 3, timeoutSeconds: 10 }, + ).pipe(Effect.forkChild); + yield* TestClock.adjust("9 seconds"); + assert.isFalse(interrupted); + yield* TestClock.adjust("1 second"); + const results = yield* Fiber.join(fiber); + + assert.isTrue(interrupted); + assert.deepStrictEqual( + results.map(({ name, outcome, time }) => [name, outcome, time]), + [ + ["hangs", "TIMEOUT", "killed after 10s"], + ["fails", "FAIL", "0.0s"], + ["passes", "pass", "0.0s"], + ], + ); + }), + ); + + it.effect("fails instead of returning an empty file list when git exits nonzero", () => { + const spawner = (exitCode: number) => + ChildProcessSpawner.make(() => + Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.encodeText(Stream.make(exitCode === 0 ? "a.ts\nb.ts\n" : "")), + stderr: Stream.encodeText( + Stream.make(exitCode === 0 ? "" : "fatal: index file corrupt"), + ), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ), + ); + return Effect.gen(function* () { + const listed = yield* gitLines("/repo", ["ls-files"]).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner(0)), + ); + assert.deepStrictEqual(listed, ["a.ts", "b.ts"]); + const error = yield* gitLines("/repo", ["ls-files"]).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner(128)), + Effect.flip, + ); + assert.strictEqual( + error, + "check:changed: git ls-files exited with 128: fatal: index file corrupt", + ); + }); + }); +}); diff --git a/scripts/check-changed.ts b/scripts/check-changed.ts index b854cf0a4e539..dfa4f3a833bbe 100644 --- a/scripts/check-changed.ts +++ b/scripts/check-changed.ts @@ -1,4 +1,4 @@ -// @effect-diagnostics nodeBuiltinImport:off globalConsole:off +// @effect-diagnostics nodeBuiltinImport:off - hashes inputs with node:crypto and sizes the budget with node:os. // Runs CI's fast checks for this branch's changes: format and lint of the // changed files, knip, generated GraphQL operations, typecheck of the affected // packages and their dependents, the test files beside the changed files, and @@ -7,369 +7,551 @@ // A dependency change also gets the unused-patch install check below. CI still // runs the full test suites and the jobs listed in the summary. // -// vp run check:changed compare against the merge base with origin/main -// vp run check:changed compare against the merge base with +// vp run check:changed compare against the merge base with origin/main +// vp run check:changed compare against the merge base with +// vp run check:changed --concurrency 1 run the steps one at a time // // Uncommitted and untracked files count. Affected packages come from // .github/scripts/ci-changes.cjs, the same mapping CI uses to gate its jobs. -// The steps run side by side; each prints its output when it finishes. A -// rerun on exactly the files of the last passing run prints that run's summary. -import * as NodeChildProcess from "node:child_process"; +// The steps run side by side within a budget derived from the machine's cores +// and available memory, and each prints its output when it finishes. A step +// running past the timeout is killed with its process group and reported as +// TIMEOUT. A rerun on exactly the files of the last passing run prints that +// run's summary. import * as NodeCrypto from "node:crypto"; -import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; -import * as NodePath from "node:path"; + +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Clock from "effect/Clock"; +import * as Config from "effect/Config"; +import * as Console from "effect/Console"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import type * as PlatformError from "effect/PlatformError"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import { Argument, Command, Flag } from "effect/cli"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + import { decide, readWorkspace } from "../.github/scripts/ci-changes.cjs"; -const root = NodePath.resolve(import.meta.dirname, ".."); +const GiB = 1024 ** 3; + +// A typecheck or test step can take several cores and gigabytes of memory, so +// allow one step per two cores and per 6 GiB of available memory. +export const defaultConcurrency = (cores: number, availableBytes: number) => + Math.max(1, Math.min(Math.floor(cores / 2), Math.floor(availableBytes / (6 * GiB)))); + +export type Outcome = "pass" | "FAIL" | "TIMEOUT"; + +export interface Step { + readonly name: string; + readonly run: Effect.Effect<{ readonly ok: boolean; readonly output: string }, never, R>; + // Budget slots the step holds: one per worker it runs in parallel. + readonly weight?: number; + // Vitest runs started together race to rebuild node_modules/.vitest-cache (ENOTEMPTY). + readonly serial?: boolean; +} + +// Interrupting a step at its timeout closes the scopes of the processes it +// spawned, which kills each one's process group. +export const runSteps = Effect.fn("checkChanged.runSteps")(function* ( + steps: ReadonlyArray>, + options: { readonly concurrency: number; readonly timeoutSeconds: number }, +) { + const budget = yield* Semaphore.make(options.concurrency); + // Steps claim budget one at a time, in order: Semaphore lets a light request + // take free slots ahead of a waiting heavier one, which would delay typecheck. + const admission = yield* Semaphore.make(1); + const serial = yield* Semaphore.make(1); + return yield* Effect.forEach( + steps, + (step) => { + const timed = Effect.gen(function* () { + const started = yield* Clock.currentTimeMillis; + const result = yield* Effect.timeoutOption(step.run, `${options.timeoutSeconds} seconds`); + const seconds = ((yield* Clock.currentTimeMillis) - started) / 1000; + const outcome: Outcome = Option.match(result, { + onNone: () => "TIMEOUT", + onSome: ({ ok }) => (ok ? "pass" : "FAIL"), + }); + const time = + outcome === "TIMEOUT" + ? `killed after ${options.timeoutSeconds}s` + : `${seconds.toFixed(1)}s`; + yield* Console.log(`\n> vp ${step.name} ${outcome} ${time}`); + const output = Option.getOrUndefined(result)?.output.trimEnd(); + if (output) yield* Console.log(output); + return { name: step.name, outcome, time }; + }); + const slots = Math.min(step.weight ?? 1, options.concurrency); + const limited = Effect.acquireUseRelease( + admission.withPermit(budget.take(slots)), + () => timed, + () => budget.release(slots), + ); + return step.serial ? serial.withPermit(limited) : limited; + }, + { concurrency: "unbounded" }, + ); +}); + +const CacheEntry = Schema.fromJsonString( + Schema.Struct({ key: Schema.String, summary: Schema.Array(Schema.String) }), +); +const decodeCacheEntry = Schema.decodeEffect(CacheEntry); +const encodeCacheEntry = Schema.encodeEffect(CacheEntry); +const decodeRootPackageJson = Schema.decodeEffect( + Schema.fromJsonString(Schema.Struct({ scripts: Schema.Struct({ "knip:check": Schema.String }) })), +); + +const collectBytes = (stream: Stream.Stream) => + Stream.runCollect(stream).pipe(Effect.map((chunks) => Buffer.concat(chunks))); + +// Fails on a nonzero exit: an empty file list from a broken index would +// otherwise pass as "no changes". quotePath=false keeps non-ASCII paths as real +// paths instead of quoted escapes. +export const git = Effect.fn("checkChanged.git")(function* ( + cwd: string, + args: ReadonlyArray, +) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* Effect.scoped( + Effect.gen(function* () { + const handle = yield* spawner.spawn( + ChildProcess.make("git", ["-c", "core.quotePath=false", ...args], { cwd }), + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [collectBytes(handle.stdout), collectBytes(handle.stderr), handle.exitCode], + { concurrency: "unbounded" }, + ); + if (exitCode !== 0) { + return yield* Effect.fail( + `check:changed: git ${args.join(" ")} exited with ${exitCode}: ${stderr.toString().trim()}`, + ); + } + return stdout; + }), + ); +}); +export const gitLines = (cwd: string, args: ReadonlyArray) => + git(cwd, args).pipe(Effect.map((stdout) => stdout.toString().split("\n").filter(Boolean))); + const CODE = /\.[cm]?[jt]sx?$/; +const TEST = /\.test\.[cm]?[jt]sx?$/; +const escapeRegExp = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -// quotePath=false keeps non-ASCII paths as real paths instead of quoted escapes. -const git = (...args: Array) => - NodeChildProcess.execFileSync("git", ["-c", "core.quotePath=false", ...args], { - cwd: root, - encoding: "utf8", - maxBuffer: 64 * 1024 * 1024, - }) - .split("\n") - .filter(Boolean); - -const ref = process.argv[2] ?? "origin/main"; -const mergeBase = NodeChildProcess.spawnSync("git", ["merge-base", "HEAD", ref], { - cwd: root, - encoding: "utf8", +// A flag wins over its environment variable; either must be a positive integer. +const override = Effect.fn("checkChanged.override")(function* ( + flagName: string, + flag: Option.Option, + envName: string, +) { + const [name, value] = Option.isSome(flag) + ? [`--${flagName}`, flag] + : [envName, yield* Config.option(Config.Int(envName))]; + if (Option.isSome(value) && value.value < 1) { + return yield* Effect.fail(`check:changed: ${name} must be a positive integer.`); + } + return value; }); -if (mergeBase.status !== 0) { - console.error( - `check:changed: no merge base with ${ref}. Run \`git fetch origin\` or pass a ref.`, + +const checkChanged = Effect.fn("checkChanged")(function* (input: { + readonly ref: string; + readonly concurrency: Option.Option; + readonly timeout: Option.Option; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* path.fromFileUrl(new URL("..", import.meta.url)); + const inRoot = (file: string) => path.join(root, file); + + // macOS counts its file cache as used, so available memory there would + // starve a large Mac. Elsewhere availableMemory() honors a cgroup limit. + const platform = yield* HostProcess.Platform; + const concurrency = Option.getOrElse( + yield* override("concurrency", input.concurrency, "CHECK_CHANGED_CONCURRENCY"), + () => + defaultConcurrency( + NodeOS.availableParallelism(), + platform === "darwin" ? NodeOS.totalmem() : process.availableMemory(), + ), ); - process.exit(1); -} -const base = mergeBase.stdout.trim(); -// --no-renames lists a rename as its old and new path, as CI does. -const untracked = git("ls-files", "--others", "--exclude-standard"); -const files = [ - ...new Set([...git("diff", "--name-only", "--no-renames", base, "--"), ...untracked]), -].sort(); - -if (files.length === 0) { - console.log(`check:changed: no changes since ${base.slice(0, 12)}.`); - process.exit(0); -} -console.log(`check:changed: ${files.length} files since ${base.slice(0, 12)}`); - -// Ignored generated sources, declarations, env files, and install state affect check results. -// Include them in the key, but skip large ignored folders that no check reads. -// Recompute after a passing run so edits made during it are never cached as passing. -const inputKey = () => { - const hash = NodeCrypto.createHash("sha256"); - // CI changes test behavior (vite.config.ts rejects .only under CI). - hash.update(`${process.version}\0${base}\0${process.env.CI ?? ""}\0`); - hash.update( - NodeChildProcess.execFileSync( - "git", - ["diff", "--binary", "--no-renames", "--no-ext-diff", "--no-textconv", base, "--"], - { cwd: root, maxBuffer: 1024 * 1024 * 1024 }, - ), + const timeoutSeconds = Option.getOrElse( + yield* override("timeout", input.timeout, "CHECK_CHANGED_TIMEOUT"), + () => 900, ); - const ignored = git( - "ls-files", - "--others", - "--ignored", - "--exclude-standard", - "--", - ":(glob)**/.generated/**", - ":(glob)**/.astro/**", - ":(glob)**/*.d.ts", - ":(glob)**/.env*", - ":(exclude,glob)**/node_modules/**", + + const lines = (...args: Array) => gitLines(root, args); + + const base = yield* git(root, ["merge-base", "HEAD", input.ref]).pipe( + Effect.map((stdout) => stdout.toString().trim()), + Effect.mapError( + () => + `check:changed: no merge base with ${input.ref}. Run \`git fetch origin\` or pass a ref.`, + ), ); - for (const file of [ - // Raw bytes of changed files too: .gitattributes normalizes line endings in - // the diff, but the formatter reads the file as it is on disk. - ...git("diff", "--name-only", "--no-renames", base, "--"), - ...git("ls-files", "--others", "--exclude-standard"), - ...ignored, - "node_modules/.modules.yaml", - "node_modules/.pnpm/lock.yaml", - ]) { - hash.update(`\0${file}\0`); - const path = NodePath.join(root, file); - // Untracked nested repositories are listed as directories. - if (NodeFS.statSync(path, { throwIfNoEntry: false })?.isFile()) { - hash.update(NodeFS.readFileSync(path)); - } + // --no-renames lists a rename as its old and new path, as CI does. + const changedFiles = lines("diff", "--name-only", "--no-renames", base, "--"); + const untrackedFiles = lines("ls-files", "--others", "--exclude-standard"); + const files = [...new Set([...(yield* changedFiles), ...(yield* untrackedFiles)])].sort(); + + if (files.length === 0) { + yield* Console.log(`check:changed: no changes since ${base.slice(0, 12)}.`); + return true; } - return hash.digest("hex"); -}; -const key = inputKey(); -const cachePath = NodePath.join(root, "node_modules/.cache/check-changed.json"); -// A missing or unreadable cache is a miss, never a failure. -const readCache = () => { - try { - return JSON.parse(NodeFS.readFileSync(cachePath, "utf8")) as { - key: string; - summary: Array; - }; - } catch { - return undefined; + yield* Console.log(`check:changed: ${files.length} files since ${base.slice(0, 12)}`); + + // Ignored generated sources, declarations, env files, and install state affect check results. + // Include them in the key, but skip large ignored folders that no check reads. + // Recompute after a passing run so edits made during it are never cached as passing. + const ci = Option.getOrElse(yield* Config.option(Config.String("CI")), () => ""); + const inputKey = Effect.gen(function* () { + const hash = NodeCrypto.createHash("sha256"); + // CI changes test behavior (vite.config.ts rejects .only under CI). + hash.update(`${process.version}\0${base}\0${ci}\0`); + hash.update( + yield* git(root, [ + "diff", + "--binary", + "--no-renames", + "--no-ext-diff", + "--no-textconv", + base, + "--", + ]), + ); + const ignored = yield* lines( + "ls-files", + "--others", + "--ignored", + "--exclude-standard", + "--", + ":(glob)**/.generated/**", + ":(glob)**/.astro/**", + ":(glob)**/*.d.ts", + ":(glob)**/.env*", + ":(exclude,glob)**/node_modules/**", + ); + for (const file of [ + // Raw bytes of changed files too: .gitattributes normalizes line endings in + // the diff, but the formatter reads the file as it is on disk. + ...(yield* changedFiles), + ...(yield* untrackedFiles), + ...ignored, + "node_modules/.modules.yaml", + "node_modules/.pnpm/lock.yaml", + ]) { + hash.update(`\0${file}\0`); + // Untracked nested repositories are listed as directories. + const info = yield* Effect.option(fs.stat(inRoot(file))); + if (Option.isSome(info) && info.value.type === "File") { + hash.update(yield* fs.readFile(inRoot(file))); + } + } + return hash.digest("hex"); + }); + const key = yield* inputKey; + const cachePath = inRoot("node_modules/.cache/check-changed.json"); + // A missing or unreadable cache is a miss, never a failure. + const cached = yield* fs + .readFileString(cachePath) + .pipe(Effect.flatMap(decodeCacheEntry), Effect.option); + if (Option.isSome(cached) && cached.value.key === key) { + yield* Console.log("Nothing changed since the last passing run.\n"); + for (const line of cached.value.summary) yield* Console.log(line); + return true; } -}; -const cached = readCache(); -if (cached?.key === key) { - console.log("Nothing changed since the last passing run.\n"); - for (const line of cached.summary) console.log(line); - process.exit(0); -} -const present = files.filter((file) => NodeFS.existsSync(NodePath.join(root, file))); -const code = present.filter((file) => CODE.test(file)); -const packages = readWorkspace(); -const { affected, outputs } = decide(files, packages, false); + const present = yield* Effect.filter(files, (file) => fs.exists(inRoot(file))); + const code = present.filter((file) => CODE.test(file)); + const packages = readWorkspace(); + const { affected, outputs } = decide(files, packages, false); -// Changed test files, and the test files beside a changed source file -// (foo.ts -> foo.test.ts, foo.browser.test.tsx). Vitest's import-graph -// `related` reaches most of a package: one leaf server file pulled in 252 of -// 406 server test files and took 7 minutes. CI runs the rest. -const TEST = /\.test\.[cm]?[jt]sx?$/; -const tests = new Set(code.filter((file) => TEST.test(file))); -for (const file of code) { - const dir = NodePath.dirname(file); - const prefix = `${NodePath.basename(file).replace(CODE, "")}.`; - for (const entry of NodeFS.readdirSync(NodePath.join(root, dir))) { - if (entry.startsWith(prefix) && TEST.test(entry)) tests.add(NodePath.join(dir, entry)); + // Changed test files, and the test files beside a changed source file + // (foo.ts -> foo.test.ts, foo.browser.test.tsx). Vitest's import-graph + // `related` reaches most of a package: one leaf server file pulled in 252 of + // 406 server test files and took 7 minutes. CI runs the rest. + const tests = new Set(code.filter((file) => TEST.test(file))); + for (const file of code) { + const dir = path.dirname(file); + const prefix = `${path.basename(file).replace(CODE, "")}.`; + for (const entry of yield* fs.readDirectory(inRoot(dir))) { + if (entry.startsWith(prefix) && TEST.test(entry)) tests.add(path.join(dir, entry)); + } } -} -// It reads files other than the ones beside it, so no changed file selects it. -if (affected.includes("t3")) tests.add("apps/server/src/provider/AgentContextBudget.test.ts"); + // It reads files other than the ones beside it, so no changed file selects it. + if (affected.includes("t3")) tests.add("apps/server/src/provider/AgentContextBudget.test.ts"); -// React Compiler coverage compiles every client source file, so run only the -// changed files' cases, in one extra step: a -t filter in the main test step -// would drop every other test. An app is skipped when the main step already -// runs its coverage file unfiltered. File filters mirror each coverage test. -const escapeRegExp = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -const compilerApps = [ - { - test: "apps/web/src/reactCompilerCoverage.test.ts", - root: "apps/web/src/", - // Test names keep the repo-relative path. - name: (file: string) => file, - compiled: (file: string) => - /\.tsx?$/.test(file) && !/\.(test|browser)\.tsx?$/.test(file) && !file.endsWith(".gen.ts"), - // Web has only per-file cases. - always: [], - }, - { - test: "apps/mobile/src/reactCompilerCoverage.test.ts", - root: "apps/mobile/src/", - name: (file: string) => file.slice("apps/mobile/".length), - compiled: (file: string) => /\.tsx?$/.test(file) && !/\.(test\.tsx?|d\.ts)$/.test(file), - // The Metro, cache, and stale-allowlist cases check more than one file, deletions included. - always: ["^(?!.*compiles )"], - }, -].flatMap(({ test, root: appRoot, name, compiled, always }) => { - const changed = files.filter((file) => file.startsWith(appRoot) && compiled(file)); - if (changed.length === 0 || tests.has(test)) return []; - const names = changed - .filter((file) => present.includes(file)) - .map((file) => `compiles ${escapeRegExp(name(file))}$`); - const pattern = [...names, ...always]; - return pattern.length > 0 ? [{ test, names, pattern }] : []; -}); -const compilerCoverage = compilerApps.length > 0 && { - name: `react compiler (${compilerApps.reduce((count, app) => count + app.names.length, 0)} files)`, - args: [ - "test", - "run", - ...compilerApps.map((app) => app.test), - "-t", - compilerApps.flatMap((app) => app.pattern).join("|"), - ], -}; - -// knip:check cannot be scoped to a package. Any code change can leave a file, -// export, or dependency unused (removing the last import of a helper does), so -// it runs whenever code or a manifest changes. -const runKnip = files.some( - (file) => CODE.test(file) || NodePath.basename(file) === "package.json" || file === "knip.jsonc", -); -// CI runs knip:check as one script of `knip ... && knip ...` passes. The passes -// do not depend on each other, so they run side by side here. -const knipCheck: string = JSON.parse( - NodeFS.readFileSync(NodePath.join(root, "package.json"), "utf8"), -).scripts["knip:check"]; -const knipPasses = knipCheck.split(" && ").map((command, index, all) => { - const [bin, ...args] = command.trim().split(/\s+/); - if (bin !== "knip" || /["'$\\|;&<>]/.test(command)) { - throw new Error(`check:changed cannot split knip:check: ${command}`); + // React Compiler coverage compiles every client source file, so run only the + // changed files' cases, in one extra step: a -t filter in the main test step + // would drop every other test. An app is skipped when the main step already + // runs its coverage file unfiltered. File filters mirror each coverage test. + const compilerApps = [ + { + test: "apps/web/src/reactCompilerCoverage.test.ts", + root: "apps/web/src/", + // Test names keep the repo-relative path. + name: (file: string) => file, + compiled: (file: string) => + /\.tsx?$/.test(file) && !/\.(test|browser)\.tsx?$/.test(file) && !file.endsWith(".gen.ts"), + // Web has only per-file cases. + always: [], + }, + { + test: "apps/mobile/src/reactCompilerCoverage.test.ts", + root: "apps/mobile/src/", + name: (file: string) => file.slice("apps/mobile/".length), + compiled: (file: string) => /\.tsx?$/.test(file) && !/\.(test\.tsx?|d\.ts)$/.test(file), + // The Metro, cache, and stale-allowlist cases check more than one file, deletions included. + always: ["^(?!.*compiles )"], + }, + ].flatMap(({ test, root: appRoot, name, compiled, always }) => { + const changed = files.filter((file) => file.startsWith(appRoot) && compiled(file)); + if (changed.length === 0 || tests.has(test)) return []; + const names = changed + .filter((file) => present.includes(file)) + .map((file) => `compiles ${escapeRegExp(name(file))}$`); + const pattern = [...names, ...always]; + return pattern.length > 0 ? [{ test, names, pattern }] : []; + }); + + // knip:check cannot be scoped to a package. Any code change can leave a file, + // export, or dependency unused (removing the last import of a helper does), so + // it runs whenever code or a manifest changes. + const runKnip = files.some( + (file) => CODE.test(file) || path.basename(file) === "package.json" || file === "knip.jsonc", + ); + // CI runs knip:check as one script of `knip ... && knip ...` passes. The passes + // do not depend on each other, so they run side by side here. + const rootPackage = yield* fs + .readFileString(inRoot("package.json")) + .pipe(Effect.flatMap(decodeRootPackageJson), Effect.orDie); + const knipCheck = rootPackage.scripts["knip:check"]; + const knipPasses: Array> = []; + for (const command of knipCheck.split(" && ")) { + const [bin, ...args] = command.trim().split(/\s+/); + if (bin !== "knip" || /["'$\\|;&<>]/.test(command)) { + return yield* Effect.fail(`check:changed cannot split knip:check: ${command}`); + } + knipPasses.push(["exec", "knip", ...args]); } - return { name: `knip ${index + 1}/${all.length}`, args: ["exec", "knip", ...args] }; -}); -// A version bump can leave a patch unused, causing pnpm to reject the install -// with ERR_PNPM_UNUSED_PATCH. Check it here as Release Smoke does in CI. -const runPatchCheck = files.some( - (file) => - file === "pnpm-workspace.yaml" || - file === "pnpm-lock.yaml" || - file === ".pnpmfile.cjs" || - file.startsWith("patches/") || - file.startsWith("apps/mobile/deps/") || - NodePath.basename(file) === "package.json", -); + // A version bump can leave a patch unused, causing pnpm to reject the install + // with ERR_PNPM_UNUSED_PATCH. Check it here as Release Smoke does in CI. + const runPatchCheck = files.some( + (file) => + file === "pnpm-workspace.yaml" || + file === "pnpm-lock.yaml" || + file === ".pnpmfile.cjs" || + file.startsWith("patches/") || + file.startsWith("apps/mobile/deps/") || + path.basename(file) === "package.json", + ); + + // Output is piped, so keep the colors a terminal would get. + const env = process.stdout.isTTY ? { FORCE_COLOR: "1" } : {}; + const vp = (args: ReadonlyArray, cwd = root) => + Effect.scoped( + Effect.gen(function* () { + const handle = yield* spawner.spawn( + ChildProcess.make("vp", args, { cwd, env, extendEnv: true, forceKillAfter: "5 seconds" }), + ); + const [output, exitCode] = yield* Effect.all( + [Stream.mkString(Stream.decodeText(handle.all)), handle.exitCode], + { concurrency: 2 }, + ); + return { ok: exitCode === 0, output }; + }), + ).pipe(Effect.catch((error) => Effect.succeed({ ok: false, output: error.message }))); -// Resolve without the lockfile in a temp copy to preserve the working tree and lockfile. -// Copy mobile modules too: they are file: dependencies, not workspace packages. -// Cached registry metadata limits this to patches stranded by local changes, not registry drift. -const checkPatches = (): boolean => { - const temp = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-patch-check-")); - try { - const copy = (relative: string) => { - const dest = NodePath.join(temp, relative); - NodeFS.mkdirSync(NodePath.dirname(dest), { recursive: true }); - NodeFS.cpSync(NodePath.join(root, relative), dest, { recursive: true }); - }; - copy("package.json"); - copy("pnpm-workspace.yaml"); - for (const pkg of packages) copy(`${pkg.dir}/package.json`); - const modulesDir = NodePath.join(root, "apps/mobile/modules"); - if (NodeFS.existsSync(modulesDir)) { - for (const entry of NodeFS.readdirSync(modulesDir)) { - const manifest = `apps/mobile/modules/${entry}/package.json`; - if (NodeFS.existsSync(NodePath.join(root, manifest))) copy(manifest); + // Resolve without the lockfile in a temp copy to preserve the working tree and lockfile. + // Copy mobile modules too: they are file: dependencies, not workspace packages. + // Cached registry metadata limits this to patches stranded by local changes, not registry drift. + const checkPatches = Effect.scoped( + Effect.gen(function* () { + const temp = yield* fs.makeTempDirectoryScoped({ prefix: "t3-patch-check-" }); + const copy = Effect.fnUntraced(function* (relative: string) { + const dest = path.join(temp, relative); + yield* fs.makeDirectory(path.dirname(dest), { recursive: true }); + yield* fs.copy(inRoot(relative), dest); + }); + yield* copy("package.json"); + yield* copy("pnpm-workspace.yaml"); + for (const pkg of packages) yield* copy(`${pkg.dir}/package.json`); + const modulesDir = "apps/mobile/modules"; + if (yield* fs.exists(inRoot(modulesDir))) { + for (const entry of yield* fs.readDirectory(inRoot(modulesDir))) { + const manifest = `${modulesDir}/${entry}/package.json`; + if (yield* fs.exists(inRoot(manifest))) yield* copy(manifest); + } } - } - for (const extra of ["apps/mobile/deps", "patches", ".pnpmfile.cjs"]) { - if (NodeFS.existsSync(NodePath.join(root, extra))) copy(extra); - } - const { status, stdout, stderr, error } = NodeChildProcess.spawnSync( - "vp", - ["install", "--lockfile-only", "--ignore-scripts"], - { cwd: temp, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }, - ); - if (status === 0) return true; - if (stdout) console.log(stdout); - if (stderr) console.error(stderr); - if (error) console.error(error.message); - if (`${stdout ?? ""}${stderr ?? ""}`.includes("ERR_PNPM_UNUSED_PATCH")) { - console.error( - "Unused dependency patches: remove each patch named above from " + + for (const extra of ["apps/mobile/deps", "patches", ".pnpmfile.cjs"]) { + if (yield* fs.exists(inRoot(extra))) yield* copy(extra); + } + const result = yield* vp(["install", "--lockfile-only", "--ignore-scripts"], temp); + if (result.ok || !result.output.includes("ERR_PNPM_UNUSED_PATCH")) return result; + return { + ok: false, + output: + `${result.output}\nUnused dependency patches: remove each patch named above from ` + "patchedDependencies in pnpm-workspace.yaml and from patches/, " + "update it to the version the install resolves, " + "or pin the dependency that pulls it in to the patched version.", - ); - } - return false; - } finally { - NodeFS.rmSync(temp, { recursive: true, force: true }); - } -}; + }; + }), + ).pipe(Effect.catch((error) => Effect.succeed({ ok: false, output: error.message }))); -// CI's Lint job checks that generated GraphQL operations match their sources. -const graphql = ["apps/server", "packages/source-control-github"].filter((dir) => - files.some((file) => file.startsWith(`${dir}/`)), -); + // CI's Lint job checks that generated GraphQL operations match their sources. + const graphql = ["apps/server", "packages/source-control-github"].filter((dir) => + files.some((file) => file.startsWith(`${dir}/`)), + ); + const filters = (names: ReadonlyArray) => names.flatMap((name) => ["--filter", name]); + const typecheckWorkers = Math.min(concurrency, affected.length); + const command = (name: string, args: ReadonlyArray) => ({ name, run: vp(args) }); + const compilerFiles = compilerApps.reduce((count, app) => count + app.names.length, 0); -const filters = (names: ReadonlyArray) => names.flatMap((name) => ["--filter", name]); -const steps = [ - // Runs first; it takes about a second. - runPatchCheck && { name: "unused patches", run: checkPatches }, - present.length > 0 && { - name: "format", - args: ["fmt", "--check", "--no-error-on-unmatched-pattern", ...present], - }, - code.length > 0 && { name: "lint", args: ["lint", ...code] }, - ...(runKnip ? knipPasses : []), - ...graphql.map((dir) => ({ - name: `graphql (${dir})`, - args: ["run", "--filter", `./${dir}`, "generate", "--check"], - })), - affected.length > 0 && { - name: `typecheck (${affected.length} packages)`, - // `tsc --noEmit` writes nothing another package reads, so dependency order - // only made dependents wait: --parallel cut a contracts change from 13 s to 9 s. - args: [ - "run", - "--cache", - "--parallel", - "--concurrency-limit", - "4", - ...filters(affected), - "typecheck", - ], - }, - // From the repo root, the root vite.config.ts hands each file to its package's config. - tests.size > 0 && { - name: `test (${tests.size} files)`, - args: ["test", "run", "--passWithNoTests", ...tests], - }, - compilerCoverage, -].filter((step) => step !== false); - -// Output is piped, so keep the colors a terminal would get. -const env = process.stdout.isTTY ? { ...process.env, FORCE_COLOR: "1" } : process.env; -const runStep = (step: (typeof steps)[number]) => - new Promise<{ name: string; ok: boolean; seconds: string }>((resolve) => { - const started = performance.now(); - const finish = (ok: boolean, output?: Buffer) => { - const seconds = ((performance.now() - started) / 1000).toFixed(1); - console.log(`\n> vp ${step.name} ${ok ? "pass" : "FAIL"} ${seconds}s`); - if (output) process.stdout.write(output); - resolve({ name: step.name, ok, seconds }); - }; - // In-process steps print their own output. The patch check is first and - // takes about a second, so the commands start right after it. - if ("run" in step) return finish(step.run()); - const output: Array = []; - const child = NodeChildProcess.spawn("vp", step.args, { cwd: root, env }); - child.stdout.on("data", (chunk: Buffer) => output.push(chunk)); - child.stderr.on("data", (chunk: Buffer) => output.push(chunk)); - // A failed spawn emits "error", then "close" with a nonzero code. - child.on("error", (error) => output.push(Buffer.from(`${error.message}\n`))); - child.on("close", (status) => finish(status === 0, Buffer.concat(output))); - }); + const steps: Array> = [ + ...(runPatchCheck ? [{ name: "unused patches", run: checkPatches }] : []), + ...(present.length > 0 + ? [command("format", ["fmt", "--check", "--no-error-on-unmatched-pattern", ...present])] + : []), + ...(code.length > 0 ? [command("lint", ["lint", ...code])] : []), + ...(runKnip + ? knipPasses.map((args, index) => command(`knip ${index + 1}/${knipPasses.length}`, args)) + : []), + ...graphql.map((dir) => + command(`graphql (${dir})`, ["run", "--filter", `./${dir}`, "generate", "--check"]), + ), + ...(affected.length > 0 + ? [ + { + // `tsc --noEmit` writes nothing another package reads, so dependency order + // only made dependents wait: --parallel cut a contracts change from 13 s to 9 s. + ...command(`typecheck (${affected.length} packages)`, [ + "run", + "--cache", + "--parallel", + "--concurrency-limit", + String(typecheckWorkers), + ...filters(affected), + "typecheck", + ]), + weight: typecheckWorkers, + }, + ] + : []), + // From the repo root, the root vite.config.ts hands each file to its package's config. + ...(tests.size > 0 + ? [ + { + ...command(`test (${tests.size} files)`, [ + "test", + "run", + "--passWithNoTests", + ...tests, + ]), + serial: true, + }, + ] + : []), + ...(compilerApps.length > 0 + ? [ + { + ...command(`react compiler (${compilerFiles} files)`, [ + "test", + "run", + ...compilerApps.map((app) => app.test), + "-t", + compilerApps.flatMap((app) => app.pattern).join("|"), + ]), + serial: true, + }, + ] + : []), + ]; -// Vitest runs started together race to rebuild node_modules/.vitest-cache after -// an install (ENOTEMPTY), so they run one after another. -let previousVitest: Promise = Promise.resolve(); -const results = await Promise.all( - steps.map((step) => { - if (!("args" in step) || step.args[0] !== "test") return runStep(step); - const result = previousVitest.then(() => runStep(step)); - previousVitest = result; - return result; - }), -); + const results = yield* runSteps(steps, { concurrency, timeoutSeconds }); -const summary = ["check:changed summary"]; -for (const result of results) { - summary.push(` ${result.ok ? "pass" : "FAIL"} ${result.name} ${result.seconds}s`); -} -const ciOnly = Object.entries({ - "full test suites": outputs.test || outputs.test_web || outputs.test_server, - build: outputs.build, - rust: outputs.rust, - "mobile native lint": outputs.mobile_native, - "release smoke": outputs.release_smoke, -}).flatMap(([job, runs]) => (runs ? [job] : [])); -if (ciOnly.length > 0) summary.push(` CI also runs: ${ciOnly.join(", ")}`); -if (!runPatchCheck) summary.push(" unused patch check skipped: no dependency files changed"); -if (!runKnip) summary.push(" knip skipped: no code or manifest changed"); -console.log(`\n${summary.join("\n")}`); -// exitCode, not exit(): piped stdout is asynchronous on macOS, and exit() -// would cut off a failing step's output. -if (results.some((result) => !result.ok)) { - process.exitCode = 1; -} else if (inputKey() !== key) { - console.log(" pass not cached: files changed during the run"); -} else { - try { - // Write then rename, so a concurrent run never reads half a file. - const temp = `${cachePath}.${process.pid}`; - NodeFS.mkdirSync(NodePath.dirname(cachePath), { recursive: true }); - NodeFS.writeFileSync(temp, JSON.stringify({ key, summary })); - NodeFS.renameSync(temp, cachePath); - } catch (error) { - console.warn(` pass not cached: ${error instanceof Error ? error.message : String(error)}`); + const summary = ["check:changed summary"]; + for (const result of results) { + summary.push(` ${result.outcome} ${result.name} ${result.time}`); + } + if (results.some((result) => result.outcome === "TIMEOUT")) { + summary.push( + ` TIMEOUT: ran past ${timeoutSeconds}s and was killed; this is not a check failure. ` + + "Raise --timeout or lower --concurrency.", + ); + } + const ciOnly = Object.entries({ + "full test suites": outputs.test || outputs.test_web || outputs.test_server, + build: outputs.build, + rust: outputs.rust, + "mobile native lint": outputs.mobile_native, + "release smoke": outputs.release_smoke, + }).flatMap(([job, runs]) => (runs ? [job] : [])); + if (ciOnly.length > 0) summary.push(` CI also runs: ${ciOnly.join(", ")}`); + if (!runPatchCheck) summary.push(" unused patch check skipped: no dependency files changed"); + if (!runKnip) summary.push(" knip skipped: no code or manifest changed"); + yield* Console.log(`\n${summary.join("\n")}`); + + if (results.some((result) => result.outcome !== "pass")) return false; + if ((yield* inputKey) !== key) { + yield* Console.log(" pass not cached: files changed during the run"); + return true; } + // Write then rename, so a concurrent run never reads half a file. + const temp = `${cachePath}.${process.pid}`; + yield* Effect.gen(function* () { + yield* fs.makeDirectory(path.dirname(cachePath), { recursive: true }); + yield* fs.writeFileString(temp, yield* encodeCacheEntry({ key, summary })); + yield* fs.rename(temp, cachePath); + }).pipe(Effect.catch((error) => Console.warn(` pass not cached: ${error.message}`))); + return true; +}); + +export const checkChangedCommand = Command.make( + "check-changed", + { + ref: Argument.String("ref").pipe( + Argument.withDescription("Compare against the merge base with this ref."), + Argument.withDefault("origin/main"), + ), + concurrency: Flag.Int("concurrency").pipe( + Flag.withDescription( + "Budget of parallel steps and typecheck workers; also CHECK_CHANGED_CONCURRENCY. Defaults to one per two cores and per 6 GiB of available memory.", + ), + Flag.optional, + ), + timeout: Flag.Int("timeout").pipe( + Flag.withDescription( + "Seconds before a step is killed; also CHECK_CHANGED_TIMEOUT. Defaults to 900.", + ), + Flag.optional, + ), + }, + (input) => + checkChanged(input).pipe( + Effect.catch((message) => Console.error(message).pipe(Effect.as(false))), + // exitCode, not a failed exit: runMain calls process.exit() on failure, and + // piped stdout is asynchronous on macOS, so it would cut off step output. + Effect.flatMap((ok) => + Effect.sync(() => { + if (!ok) process.exitCode = 1; + }), + ), + ), +).pipe(Command.withDescription("Run CI's fast checks for this branch's changes.")); + +if (import.meta.main) { + Command.run(checkChangedCommand, { version: "0.0.0" }).pipe( + Effect.provide(NodeServices.layer), + NodeRuntime.runMain, + ); } diff --git a/scripts/npm-launcher/t3.mjs b/scripts/npm-launcher/t3.mjs index 4973cf10a423a..f8828210862e3 100644 --- a/scripts/npm-launcher/t3.mjs +++ b/scripts/npm-launcher/t3.mjs @@ -109,11 +109,36 @@ const isComplete = (dir) => { function publish(staging, dir) { try { fs.renameSync(staging, dir); + return; } catch (error) { if (isComplete(dir)) return; if (!fs.existsSync(dir)) throw error; - fs.rmSync(dir, { recursive: true, force: true }); - fs.renameSync(staging, dir); + } + const lock = `${dir}.lock`; + try { + fs.mkdirSync(lock); + } catch (error) { + if (error.code !== "EEXIST") throw error; + throw new Error( + `another t3 is installing ${version}. If none is running, delete ${lock} and try again.`, + { cause: error }, + ); + } + const stale = `${staging}.stale`; + try { + // A peer may have published between our marker check and taking the lock. + if (isComplete(dir)) return; + // Move it aside rather than deleting in place: once the folder is empty, a + // peer's first rename can land in it. + fs.renameSync(dir, stale); + try { + fs.renameSync(staging, dir); + } catch (error) { + if (!isComplete(dir)) throw error; + } + } finally { + fs.rmSync(lock, { recursive: true, force: true }); + fs.rmSync(stale, { recursive: true, force: true }); } } diff --git a/scripts/release-smoke.ts b/scripts/release-smoke.ts index 12ef21824014d..b8823befa59e9 100644 --- a/scripts/release-smoke.ts +++ b/scripts/release-smoke.ts @@ -39,6 +39,11 @@ const workspaceFiles = [ "packages/provider-grok/package.json", "packages/provider-muse/package.json", "packages/provider-opencode/package.json", + "packages/secret-provider-aws/package.json", + "packages/secret-provider-core/package.json", + "packages/secret-provider-definitions/package.json", + "packages/secret-provider-onepassword/package.json", + "packages/secret-provider-openbao/package.json", "packages/source-control-azure-devops/package.json", "packages/source-control-bitbucket/package.json", "packages/source-control-core/package.json",
+
{codeViewFiles.length > 0 || (isGitSelection && selectedGitSource?.files?.length) ? ( ) : ( diff --git a/apps/web/src/components/GitActionsControl.tsx b/apps/web/src/components/GitActionsControl.tsx index cae352e6d6b26..a125ceacaed24 100644 --- a/apps/web/src/components/GitActionsControl.tsx +++ b/apps/web/src/components/GitActionsControl.tsx @@ -89,7 +89,7 @@ import { useVcsInitAction, useVcsPullAction, } from "../state/sourceControlActions"; -import { useThreadProjection, useQuietThreadShell } from "~/state/entities"; +import { useQuietThreadProjection, useQuietThreadShell } from "~/state/entities"; import { readEnvironmentScope, useEnvironmentScope } from "~/state/session"; import { useEnvironmentQuery } from "~/state/query"; import { serverEnvironment } from "~/state/server"; @@ -958,7 +958,7 @@ export default function GitActionsControl({ ? store.getDraftThreadByRef(activeThreadRef) : null, ); - const activeServerThreadProjection = useThreadProjection( + const activeServerThreadProjection = useQuietThreadProjection( activeDraftThread !== null && activeServerThreadShell === null ? null : activeThreadRef, ); const activeServerThread = diff --git a/apps/web/src/components/HistorySwipeNavigation.test.ts b/apps/web/src/components/HistorySwipeNavigation.test.ts new file mode 100644 index 0000000000000..9eef35f0dc61e --- /dev/null +++ b/apps/web/src/components/HistorySwipeNavigation.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { isMomentumTail } from "./HistorySwipeNavigation"; + +describe("isMomentumTail", () => { + it.each([ + ["steady momentum decay", [40, 37, 34, 31.3, 28.8], true], + ["fingers still moving", [12, 18, 9, 15, 11], false], + ["fingers slowing unevenly", [30, 22, 20, 12, 11], false], + ["reversing", [10, 8, 6, 4, -2], false], + ["too few samples", [40, 37, 34], false], + ])("%s", (_name, deltas, expected) => { + expect(isMomentumTail(deltas)).toBe(expected); + }); +}); diff --git a/apps/web/src/components/HistorySwipeNavigation.tsx b/apps/web/src/components/HistorySwipeNavigation.tsx new file mode 100644 index 0000000000000..c020878b25363 --- /dev/null +++ b/apps/web/src/components/HistorySwipeNavigation.tsx @@ -0,0 +1,189 @@ +import { ArrowLeftIcon, ArrowRightIcon } from "lucide-react"; +import { useEffect, useState } from "react"; + +import { cn } from "../lib/utils"; +import { canNavigateHistory, navigateHistory, type HistoryDirection } from "../navigationHistory"; + +const SWIPE_TRIGGER_DISTANCE = 120; +// Movement needed before the gesture picks an axis; first events are often diagonal. +const SWIPE_AXIS_LOCK_DISTANCE = 8; +const MOMENTUM_SAMPLE_SIZE = 5; + +type SwipeBounds = { left: number; top: number; width: number; height: number }; +type SwipeGesture = + | { kind: "pending"; target: EventTarget | null; y: number; sumX: number; sumY: number } + | { + kind: "tracking"; + direction: HistoryDirection; + distance: number; + deltas: number[]; + y: number; + bounds: SwipeBounds; + } + // Not a swipe, or one already decided: the rest of its events are ignored. + | { kind: "ignored" }; +type SwipeIndicatorState = { + direction: HistoryDirection; + progress: number; + y: number; + bounds: SwipeBounds; + tracking: boolean; +}; + +function readMainContentBounds(): SwipeBounds { + const main = document.querySelector('[data-slot="sidebar-inset"]'); + if (!main) return { left: 0, top: 0, width: window.innerWidth, height: window.innerHeight }; + const { left, top, width, height } = main.getBoundingClientRect(); + return { left, top, width, height }; +} + +function canScrollHorizontally(target: EventTarget | null, deltaX: number): boolean { + for ( + let element = target instanceof Element ? target : null; + element; + element = element.parentElement + ) { + if (element.scrollWidth <= element.clientWidth) continue; + const { overflowX } = getComputedStyle(element); + if (overflowX !== "auto" && overflowX !== "scroll") continue; + const canScroll = + deltaX < 0 + ? element.scrollLeft > 0 + : element.scrollLeft + element.clientWidth < element.scrollWidth - 1; + if (canScroll) return true; + } + return false; +} + +function lockSwipeAxis( + pending: Extract, +): Exclude { + const { target, sumX, sumY } = pending; + if (Math.abs(sumX) <= Math.abs(sumY)) return { kind: "ignored" }; + if (target instanceof Element && target.closest('[role="dialog"]')) return { kind: "ignored" }; + if (canScrollHorizontally(target, sumX)) return { kind: "ignored" }; + const direction = sumX < 0 ? "back" : "forward"; + if (!canNavigateHistory(direction)) return { kind: "ignored" }; + return { + kind: "tracking", + direction, + distance: Math.abs(sumX), + deltas: [], + y: pending.y, + bounds: readMainContentBounds(), + }; +} + +/** + * macOS momentum after the fingers lift decays by a near-constant ratio per + * event; finger movement is noisier. Deltas are signed along the swipe. + */ +export function isMomentumTail(deltas: readonly number[]): boolean { + if (deltas.length < MOMENTUM_SAMPLE_SIZE) return false; + const recent = deltas.slice(-MOMENTUM_SAMPLE_SIZE); + if (recent.some((delta) => delta <= 0)) return false; + const ratios = recent.slice(1).map((delta, index) => delta / recent[index]!); + return ( + ratios.every((ratio) => ratio >= 0.7 && ratio < 1) && + Math.max(...ratios) - Math.min(...ratios) < 0.1 + ); +} + +/** Two-finger horizontal trackpad swipes go back and forward, like a browser. */ +export function HistorySwipeNavigation() { + const [swipe, setSwipe] = useState(null); + + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- window wheel and desktop gesture listeners track trackpad swipes + useEffect(() => { + let gesture: SwipeGesture | null = null; + const hideIndicator = () => + setSwipe((current) => (current === null ? null : { ...current, tracking: false })); + // The fingers lifted: navigate if the swipe went far enough, then ignore its momentum. + const release = () => { + if (gesture?.kind === "tracking" && gesture.distance >= SWIPE_TRIGGER_DISTANCE) { + navigateHistory(gesture.direction); + } + gesture = { kind: "ignored" }; + hideIndicator(); + }; + // Chromium brackets each trackpad scroll, momentum included, with these. + const unsubscribeScrollBegin = window.desktopBridge?.onTrackpadScrollBegin(() => { + if (gesture?.kind === "ignored") gesture = null; + }); + const unsubscribeScrollEnd = window.desktopBridge?.onTrackpadScrollEnd(() => { + if (gesture?.kind === "tracking") release(); + gesture = null; + }); + const onWheel = (event: WheelEvent) => { + if (gesture === null) { + gesture = + event.ctrlKey || event.deltaMode !== WheelEvent.DOM_DELTA_PIXEL + ? { kind: "ignored" } + : { kind: "pending", target: event.target, y: event.clientY, sumX: 0, sumY: 0 }; + } + if (gesture.kind === "ignored") return; + if (gesture.kind === "pending") { + gesture.sumX += event.deltaX; + gesture.sumY += event.deltaY; + if (Math.abs(gesture.sumX) + Math.abs(gesture.sumY) < SWIPE_AXIS_LOCK_DISTANCE) return; + gesture = lockSwipeAxis(gesture); + if (gesture.kind === "ignored") return; + } else { + const delta = gesture.direction === "back" ? -event.deltaX : event.deltaX; + if (event.deltaX === 0 && event.deltaY === 0) return release(); + gesture.deltas = [...gesture.deltas.slice(1 - MOMENTUM_SAMPLE_SIZE), delta]; + if (isMomentumTail(gesture.deltas)) return release(); + gesture.distance = Math.max(0, gesture.distance + delta); + } + setSwipe({ + direction: gesture.direction, + progress: Math.min(1, gesture.distance / SWIPE_TRIGGER_DISTANCE), + y: gesture.y, + bounds: gesture.bounds, + tracking: true, + }); + }; + window.addEventListener("wheel", onWheel, { capture: true, passive: true }); + return () => { + unsubscribeScrollBegin?.(); + unsubscribeScrollEnd?.(); + window.removeEventListener("wheel", onWheel, { capture: true }); + }; + }, []); + + return swipe === null ? null : ; +} + +// A circle pinned to the edge of the main content that peeks in up to half its width. +function SwipeIndicator({ direction, progress, y, bounds, tracking }: SwipeIndicatorState) { + const armed = progress >= 1; + const Icon = direction === "back" ? ArrowLeftIcon : ArrowRightIcon; + const hidden = 100 - progress * 50; + return ( + + ); +} diff --git a/apps/web/src/components/RightPanelTabs.test.tsx b/apps/web/src/components/RightPanelTabs.test.tsx index fa9cbf0141c3e..88793cce44850 100644 --- a/apps/web/src/components/RightPanelTabs.test.tsx +++ b/apps/web/src/components/RightPanelTabs.test.tsx @@ -26,8 +26,62 @@ describe("surface shortcuts", () => { const actions = [ { shortcut: "B", available: true, label: "Browser" }, { shortcut: "D", available: false, label: "Diff" }, + { shortcut: "T", available: true, label: "Terminal" }, + { shortcut: "M", available: true, label: "Device" }, ] as const; + it("falls back to the physical key when the layout types a non-Latin letter or mark", () => { + // Ukrainian layout: the B key types "и". + expect(surfaceShortcutActionForKey(actions, shortcutEvent("\u0438", { code: "KeyB" }))).toBe( + actions[0], + ); + // Greek layout: the B key types "β". + expect(surfaceShortcutActionForKey(actions, shortcutEvent("\u03b2", { code: "KeyB" }))).toBe( + actions[0], + ); + // Thai layout: the B key types the vowel sign "\u0e34", a combining mark. + expect(surfaceShortcutActionForKey(actions, shortcutEvent("\u0e34", { code: "KeyB" }))).toBe( + actions[0], + ); + // Arabic (101) layout: the B key types lam alef "\u0644\u0627", two code points. + expect( + surfaceShortcutActionForKey(actions, shortcutEvent("\u0644\u0627", { code: "KeyB" })), + ).toBe(actions[0]); + // Ukrainian layout: the D key types "\u0432", but the Diff surface is unavailable. + expect( + surfaceShortcutActionForKey(actions, shortcutEvent("\u0432", { code: "KeyD" })), + ).toBeNull(); + }); + + it("follows the typed letter on Latin layouts that move it", () => { + // Dvorak: the key at QWERTY B types "x", and "b" sits on QWERTY N. + expect(surfaceShortcutActionForKey(actions, shortcutEvent("x", { code: "KeyB" }))).toBeNull(); + // The same key with Shift or Caps Lock types "X". + expect(surfaceShortcutActionForKey(actions, shortcutEvent("X", { code: "KeyB" }))).toBeNull(); + expect(surfaceShortcutActionForKey(actions, shortcutEvent("b", { code: "KeyN" }))).toBe( + actions[0], + ); + }); + + it("keeps accented Latin letters typed on a letter key", () => { + // Bépo: the T key types "è". + expect( + surfaceShortcutActionForKey(actions, shortcutEvent("\u00e8", { code: "KeyT" })), + ).toBeNull(); + }); + + it("keeps punctuation typed on a letter key", () => { + // French AZERTY: the M key types ",". + expect(surfaceShortcutActionForKey(actions, shortcutEvent(",", { code: "KeyM" }))).toBeNull(); + }); + + it("keeps a dead key on a letter key", () => { + // A dead key starts an accented letter on a Latin layout. + expect( + surfaceShortcutActionForKey(actions, shortcutEvent("Dead", { code: "KeyT" })), + ).toBeNull(); + }); + it("leaves modified, composing, and already-handled key events alone", () => { expect(surfaceShortcutActionForKey(actions, shortcutEvent("b", { metaKey: true }))).toBeNull(); expect( @@ -37,6 +91,21 @@ describe("surface shortcuts", () => { surfaceShortcutActionForKey(actions, shortcutEvent("b", { defaultPrevented: true })), ).toBeNull(); }); + + it("leaves AltGr text alone unless it is a plain letter", () => { + const getModifierState = (key: string) => key === "AltGraph"; + // German layout: AltGr+M types "\u00b5" and sets only AltGraph on Linux and Windows. + expect( + surfaceShortcutActionForKey( + actions, + shortcutEvent("\u00b5", { code: "KeyM", getModifierState }), + ), + ).toBeNull(); + // Finnish layout: AltGr+B types "b", which still matches, as it does in keybindings.ts. + expect( + surfaceShortcutActionForKey(actions, shortcutEvent("b", { code: "KeyB", getModifierState })), + ).toBe(actions[0]); + }); }); describe("surface shortcut typing contexts", () => { diff --git a/apps/web/src/components/RightPanelTabs.tsx b/apps/web/src/components/RightPanelTabs.tsx index dd62a684e92e4..d74e03ea842ab 100644 --- a/apps/web/src/components/RightPanelTabs.tsx +++ b/apps/web/src/components/RightPanelTabs.tsx @@ -92,6 +92,7 @@ import { PierreEntryIcon } from "./chat/PierreEntryIcon"; import { resolvePullRequestState } from "./pullRequest/pullRequestPresentation"; import { PullRequestGlyph } from "~/components/pullRequest/pullRequestIcons"; import { observeResize } from "~/lib/observeResize"; +import { surfaceShortcutKey } from "~/lib/surfaceShortcutKey"; export interface RightPanelTabsProps { mode: PreviewPanelMode; @@ -238,17 +239,17 @@ function tabAudioState(overlay: DesktopPreviewOverlay | null): TabAudioState { type SurfaceShortcutEvent = Pick< KeyboardEvent, "altKey" | "ctrlKey" | "defaultPrevented" | "isComposing" | "key" | "metaKey" ->; +> & + Partial>; export function surfaceShortcutActionForKey< const Action extends { available: boolean; shortcut: string }, >(actions: readonly Action[], event: SurfaceShortcutEvent): Action | null { if (event.defaultPrevented || event.isComposing) return null; if (event.metaKey || event.ctrlKey || event.altKey) return null; + const key = surfaceShortcutKey(event); return ( - actions.find( - (action) => action.available && action.shortcut.toLowerCase() === event.key.toLowerCase(), - ) ?? null + actions.find((action) => action.available && action.shortcut.toLowerCase() === key) ?? null ); } diff --git a/apps/web/src/components/chat/ChangedFilesTree.tsx b/apps/web/src/components/chat/ChangedFilesTree.tsx index 0b569cbc0c563..868484f37b474 100644 --- a/apps/web/src/components/chat/ChangedFilesTree.tsx +++ b/apps/web/src/components/chat/ChangedFilesTree.tsx @@ -1,11 +1,7 @@ -import { type RunId } from "@t3tools/contracts"; +import { type OrchestrationV2CheckpointFileTotals, type RunId } from "@t3tools/contracts"; import { type MouseEvent, type ReactNode, memo, useLayoutEffect, useRef, useState } from "react"; import { type TurnDiffFileChange } from "../../types"; -import { - buildTurnDiffTree, - summarizeTurnDiffStats, - type TurnDiffTreeNode, -} from "../../lib/turnDiffTree"; +import { buildTurnDiffTree, type TurnDiffTreeNode } from "../../lib/turnDiffTree"; import { ChevronRightIcon, FileDiffIcon } from "lucide-react"; import { ChevronsDownUp, ChevronsUpDown, Folder, FolderClosed } from "lucide"; import { cn } from "~/lib/utils"; @@ -30,7 +26,9 @@ type ChangedFileContextMenuHandler = (filePath: string, event: MouseEvent) => vo export const ChangedFilesCard = memo(function ChangedFilesCard(props: { runId: RunId; + /** May be a prefix of the changed files; `totals` covers all of them. */ files: ReadonlyArray; + totals: OrchestrationV2CheckpointFileTotals; allDirectoriesExpanded: boolean; resolvedTheme: "light" | "dark"; onToggleAllDirectories: () => void; @@ -40,13 +38,14 @@ export const ChangedFilesCard = memo(function ChangedFilesCard(props: { const { runId, files, + totals, allDirectoriesExpanded, resolvedTheme, onToggleAllDirectories, onOpenTurnDiff, onFileContextMenu, } = props; - const summaryStat = summarizeTurnDiffStats(files); + const hiddenFileCount = totals.count - files.length; const hasDirectories = files.some((file) => /[/\\]/.test(file.path)); return ( @@ -60,12 +59,12 @@ export const ChangedFilesCard = memo(function ChangedFilesCard(props: { >
- {files.length} changed file{files.length === 1 ? "" : "s"} + {totals.count.toLocaleString()} changed file{totals.count === 1 ? "" : "s"} - {hasNonZeroStat(summaryStat) && ( + {hasNonZeroStat(totals) && ( @@ -126,6 +125,19 @@ export const ChangedFilesCard = memo(function ChangedFilesCard(props: { onOpenTurnDiff={onOpenTurnDiff} onFileContextMenu={onFileContextMenu} /> + {hiddenFileCount > 0 && ( +
+ +
+ )}
); }); diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts index 8f4b15cc11f91..bd74694dc46af 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts @@ -1841,10 +1841,10 @@ describe("deriveMessagesTimelineRows", () => { const settled = rows({ resume: "completed", working: false }); expect(shape(settled)).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:launch-answer", "user:resume", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:resume-answer", ]); @@ -1857,17 +1857,17 @@ describe("deriveMessagesTimelineRows", () => { }); expect(shape(expanded)).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "work-toggle", "assistant:launch-answer", "user:resume", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:resume-answer", ]); expect(shape(rows({ resume: "running", working: true }))).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:launch-answer", "user:resume", "working", @@ -1876,7 +1876,7 @@ describe("deriveMessagesTimelineRows", () => { expect(shape(rows({ resume: "failed", working: false }))).toEqual([ "user:launch", - "fold:Worked for 8.0s", + "fold:Worked for 8s", "assistant:launch-answer", "user:resume", "work", diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index a9cab1554a3d3..faa733c76e896 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -659,12 +659,11 @@ const ConversationTimeline = memo(function ConversationTimeline({ ? new Set(paintedExpandedAttemptIds).add(activeFindAttemptId) : paintedExpandedAttemptIds; - const pendingApprovalsByRequestId = new Map( - pendingApprovals - .filter((approval) => !heldApprovalRequestIds?.has(approval.requestId)) - .map((approval) => [approval.requestId, approval]), + const isVisiblePendingApproval = (approval: PendingApproval) => + !heldApprovalRequestIds?.has(approval.requestId); + const pendingApprovalRequestIds = new Set( + pendingApprovals.filter(isVisiblePendingApproval).map((approval) => approval.requestId), ); - const pendingApprovalRequestIds = new Set(pendingApprovalsByRequestId.keys()); const rawRows = useIncrementalTimelineRows(`${listIdentityKey}\0${workspaceRoot ?? ""}`, { timelineEntries, pendingApprovalRequestIds, @@ -683,17 +682,19 @@ const ConversationTimeline = memo(function ConversationTimeline({ worktreeSetup, }); const rows = useStableRows(rawRows, listIdentityKey); - const unmatchedPendingApprovals = new Map(pendingApprovalsByRequestId); - if (unmatchedPendingApprovals.size > 0) { - for (const row of rows) { - const entries = - row.kind === "work" ? row.groupedEntries : row.kind === "work-live" ? [row.entry] : []; - for (const entry of entries) { - const item = entry.projectedItem?.item; - if (item?.type === "approval_request") unmatchedPendingApprovals.delete(item.requestId); - } + const matchedApprovalRequestIds = new Set(); + for (const row of pendingApprovalRequestIds.size > 0 ? rows : []) { + const entries = + row.kind === "work" ? row.groupedEntries : row.kind === "work-live" ? [row.entry] : []; + for (const entry of entries) { + const item = entry.projectedItem?.item; + if (item?.type === "approval_request") matchedApprovalRequestIds.add(item.requestId); } } + const unmatchedPendingApprovals = pendingApprovals.filter( + (approval) => + isVisiblePendingApproval(approval) && !matchedApprovalRequestIds.has(approval.requestId), + ); // Run status/timestamps change per stream event; keep them out of shared row context. const runs = useStableHandoffRuns(runsProp); const minimapItems = deriveTimelineMinimapItems(rows); @@ -706,7 +707,7 @@ const ConversationTimeline = memo(function ConversationTimeline({ restoringThreadPosition && restoreRowIndex >= 0 ? { indices: [restoreRowIndex] } : undefined; const showsEmptyState = rows.length === 0 && - unmatchedPendingApprovals.size === 0 && + unmatchedPendingApprovals.length === 0 && !isWorking && parentThreadLink === null && historyControls === undefined && @@ -1000,6 +1001,13 @@ const ConversationTimeline = memo(function ConversationTimeline({ }; }; + // Built from props after the last hook so the compiler can memoize the row context; a map + // made before a hook would rebuild it, and re-render every row, on each streamed update. + const pendingApprovalsByRequestId = new Map( + pendingApprovals + .filter(isVisiblePendingApproval) + .map((approval) => [approval.requestId, approval]), + ); const sharedState: TimelineRowSharedState = { pendingApprovals: pendingApprovalsByRequestId, respondingRequestIds, @@ -1238,12 +1246,12 @@ const ConversationTimeline = memo(function ConversationTimeline({ onGrow={onFooterGrow} > {footer} - {unmatchedPendingApprovals.size > 0 ? ( + {unmatchedPendingApprovals.length > 0 ? (
- {[...unmatchedPendingApprovals.values()].map((approval) => ( + {unmatchedPendingApprovals.map((approval) => ( ))} diff --git a/apps/web/src/components/chat/QueuedRunsControl.tsx b/apps/web/src/components/chat/QueuedRunsControl.tsx index 866612bc11009..ca98e919ff93d 100644 --- a/apps/web/src/components/chat/QueuedRunsControl.tsx +++ b/apps/web/src/components/chat/QueuedRunsControl.tsx @@ -1,7 +1,10 @@ import { scopeThreadRef } from "@t3tools/client-runtime/environment"; import { deriveThreadQueueWorkflowState } from "@t3tools/client-runtime/state/thread-workflows"; import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations"; -import { replaceComposerContextReferences } from "@t3tools/shared/composerContextReferences"; +import { + isAttachmentOnlyBootstrapPrompt, + replaceComposerContextReferences, +} from "@t3tools/shared/composerContextReferences"; import type { ChatAttachment as ContractChatAttachment, EnvironmentId, @@ -237,11 +240,13 @@ export function QueuedRunsControl({
    {visibleItems.map((item, position) => { - const previewText = assistantCitationsToPlainText( - replaceComposerContextReferences(item.text, (reference) => - reference.kind === "image" && item.thumbnails.length > 0 ? "" : reference.label, - ), - ).trim(); + const previewText = isAttachmentOnlyBootstrapPrompt(item.text) + ? "" + : assistantCitationsToPlainText( + replaceComposerContextReferences(item.text, (reference) => + reference.kind === "image" && item.thumbnails.length > 0 ? "" : reference.label, + ), + ).trim(); const isEditing = item.runId !== null && item.runId === props.editingRunId; return (
  1. void; +}): ThreadNowItem { + const unresponsive = input.stop.state === "unresponsive"; + const requestedAtMs = Date.parse(input.stop.requestedAt); + return { + id: `run-stop:${input.threadId}`, + icon: unresponsive ? ( + + ) : ( + + ), + title: unresponsive ? "Agent not responding to Stop" : "Stopping", + description: unresponsive + ? "It may be short on memory. Force stop kills its processes." + : undefined, + actions: ( + <> + formatDuration(Math.max(1_000, nowMs - requestedAtMs)), + true, + )} + className="tabular-nums" + /> + {unresponsive && input.canForceStop ? ( + + {input.forceStopping ? "Force stopping..." : "Force stop"} + + ) : null} + + ), + }; +} + /** Background commands and subagents, for the end of the timeline. */ export function backgroundWorkNowItem(input: { readonly threadId: ThreadId; diff --git a/apps/web/src/components/chat/ThreadTimeline.tsx b/apps/web/src/components/chat/ThreadTimeline.tsx index 7d1db86178cf4..a68d494d8b0a0 100644 --- a/apps/web/src/components/chat/ThreadTimeline.tsx +++ b/apps/web/src/components/chat/ThreadTimeline.tsx @@ -2,7 +2,7 @@ import type { ScopedThreadRef } from "@t3tools/contracts"; import { type ComponentProps, useLayoutEffect } from "react"; import { useIncrementalTimelineEntries } from "../../hooks/useIncrementalTimelineEntries"; import type { TimelineEntriesInput, TimelineEntry } from "../../session-logic"; -import { useThreadVisibleTurnItems } from "../../state/entities"; +import { useTimelineVisibleTurnItems } from "../../state/entities"; import { peekRememberedThreadTimeline, rememberReadyThreadTimeline, @@ -14,7 +14,8 @@ import { MessagesTimeline } from "./MessagesTimeline"; /** * The timeline with the streamed text. ChatView reads the thread through its quiet * atoms, which hold through text deltas; this reads the live turn items, so a - * delta re-renders the timeline without re-rendering ChatView. + * reasoning delta re-renders the timeline without re-rendering ChatView. Streamed + * assistant text reaches only its own row, through useAssistantMessageText. */ export function ThreadTimeline({ threadDetailRef, @@ -34,7 +35,7 @@ export function ThreadTimeline({ syncing: boolean; detailLoading: boolean; }) { - const visibleTurnItems = useThreadVisibleTurnItems(threadDetailRef); + const visibleTurnItems = useTimelineVisibleTurnItems(threadDetailRef); const serverEntries = useIncrementalTimelineEntries(activeThreadKey, { ...entriesInput, visibleTurnItems, diff --git a/apps/web/src/components/chat/TimelineAssistantRows.tsx b/apps/web/src/components/chat/TimelineAssistantRows.tsx index 89b52ae5c5cba..458d84b9e982b 100644 --- a/apps/web/src/components/chat/TimelineAssistantRows.tsx +++ b/apps/web/src/components/chat/TimelineAssistantRows.tsx @@ -9,9 +9,10 @@ import { Button } from "../ui/button"; import { ProposedPlanCard } from "./ProposedPlanCard"; import { HtmlRenderFrame } from "./HtmlRenderFrame"; import { McpAppFrame } from "./McpAppFrame"; +import { checkpointFileTotals } from "@t3tools/client-runtime/state/thread-checkpoints"; import { ChangedFilesCard } from "./ChangedFilesTree"; import { useFileContextMenuHandler } from "../../fileContextMenu"; -import { useProject, useThreadShell } from "../../state/entities"; +import { useAssistantMessageText, useProject, useThreadShell } from "../../state/entities"; import { MessageCopyButton } from "./MessageCopyButton"; import { AssistantCitationSource } from "./AssistantCitationSource"; import { resolveAssistantMessageCopyState } from "./MessagesTimeline.logic"; @@ -25,7 +26,13 @@ import { MESSAGE_HEADING_LEVEL, MessageAuthorHeading } from "./TimelineRowParts" export function AssistantTimelineRow({ row }: { row: Extract }) { const ctx = use(TimelineRowCtx); - const messageText = row.message.text || (row.message.streaming ? "" : "(empty response)"); + // The timeline holds a streaming message's row while only its text grows. + const liveText = useAssistantMessageText( + row.message.streaming ? ctx.threadRef : null, + row.projectedItem?.item.id, + ); + const text = liveText ?? row.message.text; + const messageText = text || (row.message.streaming ? "" : "(empty response)"); return ( <> @@ -347,6 +354,7 @@ function AssistantChangedFilesSectionInner({ diff --git a/apps/web/src/components/chat/TimelineUserMessageRow.tsx b/apps/web/src/components/chat/TimelineUserMessageRow.tsx index 66cbb0bcc5ed4..c4363a1b45983 100644 --- a/apps/web/src/components/chat/TimelineUserMessageRow.tsx +++ b/apps/web/src/components/chat/TimelineUserMessageRow.tsx @@ -81,6 +81,10 @@ import { import { UserMessageContextReferenceChip } from "./UserMessageContextChips"; import { resolvePreviewAnnotationImage } from "./TimelineUserMessageRow.logic"; +// Drawn inside the media so dark screenshots keep an edge on the dark background. +const MEDIA_EDGE_CLASS_NAME = + "relative after:pointer-events-none after:absolute after:inset-0 after:z-10 after:rounded-[inherit] after:ring-1 after:ring-inset after:ring-black/9 dark:after:ring-white/12"; + const BUBBLE_CHIP_CLASS_NAME = "[&_[data-slot=context-chip]]:[--context-chip-ink:var(--context-chip-message,var(--context-chip-accent))]"; @@ -327,9 +331,7 @@ function UserTimelineRow({ row }: { row: Extract image.name.startsWith("preview-annotation-")); const revertTurnCount = row.revertTurnCount; const chippedAttachmentIds = new Set( @@ -340,6 +342,27 @@ function UserTimelineRow({ row }: { row: Extract !image.name.startsWith("preview-annotation-")); const unchippedFiles = otherUserFiles.filter((file) => !chippedAttachmentIds.has(file.id)); + // Images or videos sent without words show as media, not as a bubble around them. + const mediaOnly = + !folded && + userMessage.attribution !== "guest" && + resolvedContext.text.trim() === "" && + userAudio.length === 0 && + unchippedFiles.length === 0 && + unknownAttachments.length === 0 && + regularImages.length + userVideos.length > 0; + // A lone image keeps its own shape; a 4:3 crop hides most of a wide screenshot. + const loneImage = + mediaOnly && + userVideos.length === 0 && + regularImages.length === 1 && + regularImages[0]!.source?.kind !== "snap-shot"; + // Several images sent without words share one rounded shape, split by hairlines. + const collage = + mediaOnly && + userVideos.length === 0 && + regularImages.length > 1 && + regularImages.every((image) => image.source?.kind !== "snap-shot"); const annotationRecordIds = resolvedContext.records .filter((record) => record.kind === "preview-annotation") .map((record) => record.contextId); @@ -469,8 +492,10 @@ function UserTimelineRow({ row }: { row: Extract @@ -502,15 +527,45 @@ function UserTimelineRow({ row }: { row: Extract {(regularImages.length > 0 || userVideos.length > 0) && ( -
    - {regularImages.map((image) => ( +
    + {regularImages.map((image, index) => (
    3 && + regularImages.length % 2 === 1 && + index === regularImages.length - 1 && + "col-span-2", + ) + : "aspect-[4/3] overflow-hidden rounded-lg border border-border/80", )} > {image.previewUrl ? ( @@ -532,11 +587,23 @@ function UserTimelineRow({ row }: { row: Extract ) : ( -
    +
    {image.name}
    )} diff --git a/apps/web/src/components/chat/WorktreeSetupCard.browser.test.tsx b/apps/web/src/components/chat/WorktreeSetupCard.browser.test.tsx index 1a34f537efd15..eddd6691884ef 100644 --- a/apps/web/src/components/chat/WorktreeSetupCard.browser.test.tsx +++ b/apps/web/src/components/chat/WorktreeSetupCard.browser.test.tsx @@ -53,13 +53,13 @@ it("ticks the running stage while finished stages keep their duration", async () // Assert synchronously: expect.element polls on timers these tests fake. const section = screen.getByRole("region", { name: "Worktree setup" }); - expect(section.element().textContent).toContain("Fetch base branch3.0s"); - expect(section.element().textContent).toContain("Check out files7.0s"); + expect(section.element().textContent).toContain("Fetch base branch3s"); + expect(section.element().textContent).toContain("Check out files7s"); await act(async () => { await vi.advanceTimersByTimeAsync(5_000); }); - expect(section.element().textContent).toContain("Fetch base branch3.0s"); + expect(section.element().textContent).toContain("Fetch base branch3s"); expect(section.element().textContent).toContain("Check out files12s"); }); @@ -70,9 +70,9 @@ it("stops a stage interrupted by the setup ending at the setup's end", async () await using screen = await render(card(failed)); const section = screen.getByRole("region", { name: "Worktree setup" }); - expect(section.element().textContent).toContain("Check out files2.0s"); + expect(section.element().textContent).toContain("Check out files2s"); await act(async () => { await vi.advanceTimersByTimeAsync(5_000); }); - expect(section.element().textContent).toContain("Check out files2.0s"); + expect(section.element().textContent).toContain("Check out files2s"); }); diff --git a/apps/web/src/components/chat/composerFocusRedirect.ts b/apps/web/src/components/chat/composerFocusRedirect.ts index 4bbd8df7fe46d..0010e783d7098 100644 --- a/apps/web/src/components/chat/composerFocusRedirect.ts +++ b/apps/web/src/components/chat/composerFocusRedirect.ts @@ -1,3 +1,5 @@ +import { surfaceShortcutKey } from "~/lib/surfaceShortcutKey"; + export const TYPE_TO_FOCUS_EDITABLE_SELECTOR = [ "input", "textarea", @@ -48,11 +50,13 @@ const SURFACE_LAUNCHER_KEYS_ATTRIBUTE = "data-surface-launcher-keys"; // The right-panel launcher registers its own window capture listener when it // mounts. When the panel opens after page load that listener runs after ours, // so we must leave its letters alone or the composer takes them first. -function isSurfaceLauncherKey(key: string): boolean { +function isSurfaceLauncherKey(event: KeyboardEvent): boolean { const launcherKeys = document .querySelector(`[${SURFACE_LAUNCHER_KEYS_ATTRIBUTE}]`) - ?.getAttribute(SURFACE_LAUNCHER_KEYS_ATTRIBUTE); - return launcherKeys?.toLowerCase().includes(key.toLowerCase()) ?? false; + ?.getAttribute(SURFACE_LAUNCHER_KEYS_ATTRIBUTE) + ?.toLowerCase(); + const key = surfaceShortcutKey(event); + return key !== null && (launcherKeys?.includes(key) ?? false); } export function shouldRedirectInputToComposer(event: Event): boolean { @@ -70,7 +74,7 @@ export function shouldTypeToFocusComposer(event: KeyboardEvent): boolean { if (event.metaKey || event.ctrlKey || event.altKey) return false; if (event.key.length !== 1) return false; if (!shouldRedirectInputToComposer(event)) return false; - if (isSurfaceLauncherKey(event.key)) return false; + if (isSurfaceLauncherKey(event)) return false; return true; } diff --git a/apps/web/src/components/chat/markdownStreamBlocks.test.tsx b/apps/web/src/components/chat/markdownStreamBlocks.test.tsx new file mode 100644 index 0000000000000..ae81316580bbd --- /dev/null +++ b/apps/web/src/components/chat/markdownStreamBlocks.test.tsx @@ -0,0 +1,61 @@ +import { + CHAT_MARKDOWN_REHYPE_PLUGINS, + CHAT_MARKDOWN_REMARK_PLUGINS, +} from "@t3tools/shared/markdownPipeline"; +import { renderToStaticMarkup } from "react-dom/server"; +import ReactMarkdown from "react-markdown"; +import { describe, expect, it } from "vite-plus/test"; + +import { splitMarkdownBlocks } from "./markdownStreamBlocks"; + +// Whitespace between top-level blocks collapses in the rendered message. +const html = (source: string) => + renderToStaticMarkup( + + {source} + , + ) + .replace(/>\s+<") + .trim(); + +describe("splitMarkdownBlocks", () => { + it.each([ + "# Title\n\nFirst paragraph with `code` and **bold**.\n\nSecond [link](https://example.com).", + "- one\n- two\n\n- loose three\n\n continued\n\nAfter the list.", + "1. first\n\n2. second\n\nText\n\n3) restarted", + "```ts\nconst a = 1;\n\n\nconst b = 2;\n```\n\nafter\n\n~~~\nx\n\n~~~~\n\ny\n~~~\n\nend", + "> quote\n\n> another quote\n\n> [!NOTE]\n> alert", + "| a | b |\n| - | - |\n| 1 | 2 |\n\nAfter the table.", + "para\n\n indented code\n\nnext", + "Setext\n===\n\nnext\n---\n\n- [ ] task\n- [x] done", + ])("renders each block as the whole text renders it, at every streamed prefix: %j", (text) => { + for (let end = 0; end <= text.length; end++) { + const source = text.slice(0, end); + const blocks = splitMarkdownBlocks(source); + expect(blocks?.join("")).toBe(source); + expect(blocks!.map(html).join("")).toBe(html(source)); + } + }); + + it("splits only between closed blocks", () => { + expect(splitMarkdownBlocks("a\n\n```\nb\n\nc\n```\n\n- d\n\n- e\n\nf")).toEqual([ + "a\n\n", + "```\nb\n\nc\n```\n\n- d\n\n- e\n\n", + "f", + ]); + }); + + it.each([ + "[ref]\n\n[ref]: /later", + "a[^x]\n\n[^x]: note", + "
    \n\nbody\n\n
    ", + ":::note\n\ncontent\n\n:::", + "$$\na\n\nb\n$$", + "a\r\n\r\nb", + ])("keeps text with document-wide syntax whole: %j", (text) => { + expect(splitMarkdownBlocks(text)).toBeNull(); + }); +}); diff --git a/apps/web/src/components/chat/markdownStreamBlocks.ts b/apps/web/src/components/chat/markdownStreamBlocks.ts new file mode 100644 index 0000000000000..52415ae71977b --- /dev/null +++ b/apps/web/src/components/chat/markdownStreamBlocks.ts @@ -0,0 +1,43 @@ +// Definitions, footnotes, raw HTML, container directives and display math can tie blocks together. +const DOCUMENT_WIDE_SYNTAX = /^ {0,3}(?:\[[^\]\n]+\]:|<[A-Za-z!?/]|:::|\$\$)|\[\^/m; +const FENCE_PATTERN = /^ {0,3}(`{3,}|~{3,})/; +// After a blank line these still continue the previous block: a list item or indented content. +const CONTINUATION_PATTERN = /^(?:[ \t]|(?:[-*+]|\d{1,9}[.)])(?:[ \t]|$))/; + +/** + * Cuts Markdown at blank lines that close every open block, so each piece renders exactly as it + * would inside the whole text. Returns null when the text uses syntax that spans blocks. + */ +export function splitMarkdownBlocks(text: string): string[] | null { + if (text.includes("\r") || DOCUMENT_WIDE_SYNTAX.test(text)) return null; + const blocks: string[] = []; + let blockStart = 0; + let lineStart = 0; + let fence: string | null = null; + let afterBlankLine = false; + for (const line of text.split("\n")) { + const opener = FENCE_PATTERN.exec(line)?.[1]; + if (fence !== null) { + if ( + opener && + opener[0] === fence[0] && + opener.length >= fence.length && + line.trim() === opener + ) { + fence = null; + } + } else if (line.trim() === "") { + afterBlankLine = true; + } else { + if (afterBlankLine && lineStart > blockStart && !CONTINUATION_PATTERN.test(line)) { + blocks.push(text.slice(blockStart, lineStart)); + blockStart = lineStart; + } + afterBlankLine = false; + fence = opener ?? null; + } + lineStart += line.length + 1; + } + blocks.push(text.slice(blockStart)); + return blocks; +} diff --git a/apps/web/src/components/chat/useChatShortcuts.browser.test.tsx b/apps/web/src/components/chat/useChatShortcuts.browser.test.tsx index de2f9f764d7e3..0bdba6a94027a 100644 --- a/apps/web/src/components/chat/useChatShortcuts.browser.test.tsx +++ b/apps/web/src/components/chat/useChatShortcuts.browser.test.tsx @@ -2,6 +2,7 @@ import { useEffect, type ReactNode } from "react"; import { expect, it } from "vite-plus/test"; import { userEvent } from "vite-plus/test/browser"; +import { surfaceShortcutKey } from "~/lib/surfaceShortcutKey"; import { render } from "../../../test/render"; import type { ChatComposerHandle } from "./ChatComposer"; import { useChatShortcuts } from "./useChatShortcuts"; @@ -25,7 +26,8 @@ function Launcher(props: { keys: string; received: string[] }) { useEffect(() => { const claim = (event: KeyboardEvent) => { if (event.defaultPrevented) return; - if (!props.keys.toLowerCase().split("").includes(event.key.toLowerCase())) return; + const key = surfaceShortcutKey(event); + if (key === null || !props.keys.toLowerCase().includes(key)) return; event.preventDefault(); props.received.push(event.key); }; @@ -200,3 +202,38 @@ function claimEscape(): Disposable { window.addEventListener("keydown", claim, true); return { [Symbol.dispose]: () => window.removeEventListener("keydown", claim, true) }; } +it("leaves a non-Latin launcher letter to the launcher by its physical key", async () => { + const composerKeys: string[] = []; + const received: string[] = []; + await using screen = await render(, options); + await screen.rerender(); + + // Playwright cannot press a key its US layout lacks, so send what a Ukrainian T key sends. + const event = new KeyboardEvent("keydown", { + key: "\u0435", + code: "KeyT", + bubbles: true, + cancelable: true, + }); + (document.activeElement ?? document.body).dispatchEvent(event); + + expect(composerKeys).toEqual([]); + expect(received).toEqual(["\u0435"]); +}); + +it.each([ + { key: "b", code: "KeyN", composer: [], launcher: ["b"] }, + { key: "x", code: "KeyB", composer: ["x"], launcher: [] }, + { key: "è", code: "KeyT", composer: ["è"], launcher: [] }, +])("routes $key on $code by the typed letter", async ({ key, code, composer, launcher }) => { + const composerKeys: string[] = []; + const received: string[] = []; + await using screen = await render(, options); + await screen.rerender(); + + const event = new KeyboardEvent("keydown", { key, code, bubbles: true, cancelable: true }); + (document.activeElement ?? document.body).dispatchEvent(event); + + expect(composerKeys).toEqual(composer); + expect(received).toEqual(launcher); +}); diff --git a/apps/web/src/components/chat/useForceStopThread.ts b/apps/web/src/components/chat/useForceStopThread.ts new file mode 100644 index 0000000000000..00876c758a8b6 --- /dev/null +++ b/apps/web/src/components/chat/useForceStopThread.ts @@ -0,0 +1,59 @@ +import { useAtomValue } from "@effect/atom-react"; +import type { EnvironmentId, RunId, ThreadId } from "@t3tools/contracts"; +import { useState } from "react"; + +import { useServerConfigs } from "../../state/entities"; +import { orchestrationEnvironment } from "../../state/orchestration"; +import { + isAtomCommandInterrupted, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { toastManager } from "../ui/toast"; + +/** Force stop for an agent that no longer answers Stop; shared by every place that offers it. */ +export function useForceStopThread(environmentId: EnvironmentId) { + const forceStop = useAtomCommand(orchestrationEnvironment.forceStopThread); + const permitted = useAtomValue( + orchestrationEnvironment.forceStopThread.permissionAtom(environmentId), + ); + const supported = + useServerConfigs().get(environmentId)?.environment.capabilities.threadForceStop === true; + const canForceStop = permitted && supported; + const [stoppingThreadId, setStoppingThreadId] = useState(null); + /** Pass the run that ignored Stop, so a Force stop left open cannot hit a newer run. */ + const forceStopThread = async (threadId: ThreadId, runId?: RunId) => { + if (!canForceStop || stoppingThreadId !== null) return; + setStoppingThreadId(threadId); + const result = await forceStop({ + environmentId, + input: runId === undefined ? { threadId } : { threadId, runId }, + }); + setStoppingThreadId(null); + if (result._tag === "Failure") { + if (isAtomCommandInterrupted(result)) return; + const error = squashAtomCommandFailure(result); + toastManager.add({ + type: "error", + title: "Could not force stop the agent", + description: error instanceof Error ? error.message : undefined, + }); + return; + } + const others = result.value.sharedWithThreadIds.length; + if (result.value.outcome === "releasing") { + toastManager.add({ + type: "info", + title: "Turn stopped; the agent is still shutting down", + }); + } + if (result.value.outcome === "shared") { + toastManager.add({ + type: "info", + title: "Turn stopped; the agent process kept running", + description: `It is shared with ${others === 0 ? "other" : others} working ${others === 1 ? "thread" : "threads"}. Stop those to free it.`, + }); + } + }; + return { canForceStop, stoppingThreadId, forceStopThread }; +} diff --git a/apps/web/src/components/chat/useThreadLifecycle.tsx b/apps/web/src/components/chat/useThreadLifecycle.tsx index b1cf8a4a6af4c..0ad8acacf71ce 100644 --- a/apps/web/src/components/chat/useThreadLifecycle.tsx +++ b/apps/web/src/components/chat/useThreadLifecycle.tsx @@ -58,9 +58,22 @@ import { appAtomRegistry } from "../../rpc/atomRegistry"; import { readEnvironmentScope } from "../../state/session"; import { useAtomCommand } from "../../state/use-atom-command"; import { useOrchestrationCommand } from "../../state/use-orchestration-command"; -import { backgroundWorkNowItem, childInputNowItem, goalNowItem } from "./ThreadActivityBanners"; +import { + backgroundWorkNowItem, + childInputNowItem, + goalNowItem, + runStopNowItem, +} from "./ThreadActivityBanners"; +import { useForceStopThread } from "./useForceStopThread"; +import { useAtomValue } from "@effect/atom-react"; +import type { RunStopPresentation } from "@t3tools/client-runtime/state/thread-execution"; +import { Atom } from "effect/reactivity"; +import { environmentThreadDetails } from "../../state/threads"; import type { useLocalDispatchState } from "./useLocalDispatchState"; const RELATED_THREAD_SHELL_WAIT_MS = 1_000; +const NO_RUN_STOP_ATOM = Atom.make(null).pipe( + Atom.withLabel("web-thread-run-stop:empty"), +); type PendingThreadRequests = ReturnType; type LocalDispatchState = ReturnType; @@ -509,6 +522,22 @@ export function useThreadLifecycle(input: ThreadLifecycleInput) { }) : null; + const forceStop = useForceStopThread(environmentId); + const anyRunStop = useAtomValue( + activeThreadRef ? environmentThreadDetails.runStopAtom(activeThreadRef) : NO_RUN_STOP_ATOM, + ); + // A background Stop matters only while that work is still pending. + const runStop = anyRunStop?.background && activeBackgroundTasks.length === 0 ? null : anyRunStop; + const runStopNow = + activeThread && runStop !== null + ? runStopNowItem({ + threadId: activeThread.id, + stop: runStop, + canForceStop: forceStop.canForceStop, + forceStopping: forceStop.stoppingThreadId === activeThread.id, + onForceStop: () => void forceStop.forceStopThread(activeThread.id, runStop.runId), + }) + : null; const backgroundWorkNow = activeThread ? backgroundWorkNowItem({ threadId: activeThread.id, @@ -592,7 +621,9 @@ export function useThreadLifecycle(input: ThreadLifecycleInput) { shortcuts, acknowledgeActiveThreadWoke, activeBranchMismatchKey, - backgroundWorkNow, + // A Stop that went unanswered needs the user; one in progress is the thread's work. + attentionNow: runStop?.state === "unresponsive" ? runStopNow : childInputNow, + workNow: runStop?.state === "stopping" ? runStopNow : backgroundWorkNow, childInputNow, goalNow, handleSwitchCheckoutToThread, diff --git a/apps/web/src/components/desktop/DesktopWebLinkCoordinator.browser.test.tsx b/apps/web/src/components/desktop/DesktopWebLinkCoordinator.browser.test.tsx index a2a0f874d6e90..31ad2028c5f51 100644 --- a/apps/web/src/components/desktop/DesktopWebLinkCoordinator.browser.test.tsx +++ b/apps/web/src/components/desktop/DesktopWebLinkCoordinator.browser.test.tsx @@ -1,11 +1,16 @@ -import { EnvironmentId, ProjectId } from "@t3tools/contracts"; -import type { Atom } from "effect/reactivity"; +import { EnvironmentId, ProjectId, ThreadId } from "@t3tools/contracts"; +import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import type { EnvironmentProject } from "@t3tools/client-runtime/state/shell"; +import { AsyncResult, type Atom } from "effect/reactivity"; import { expect, it, vi } from "vite-plus/test"; import { act, render, stubGlobal } from "../../../test/render"; const state = vi.hoisted(() => ({ - projects: [] as never[], + projects: [] as EnvironmentProject[], + routeParams: {} as Record, + preview: vi.fn(), + openLink: vi.fn(async () => undefined), navigate: vi.fn(async (): Promise => undefined), openScratchProject: vi.fn(async () => null), confirm: vi.fn(async (_message: string) => true), @@ -18,7 +23,10 @@ const state = vi.hoisted(() => ({ vi.mock(import("@tanstack/react-router"), async (original) => ({ ...(await original()), useNavigate: (() => state.navigate) as never, - useRouter: (() => ({ navigate: state.navigate, state: { matches: [] } })) as never, + useRouter: (() => ({ + navigate: state.navigate, + state: { matches: [{ params: state.routeParams }] }, + })) as never, })); vi.mock(import("../../state/environments"), async (original) => ({ ...(await original()), @@ -47,11 +55,17 @@ vi.mock(import("../../state/server"), async (original) => { }); vi.mock(import("../../state/shell"), async (original) => { const { Atom } = await import("effect/reactivity"); - return { ...(await original()), allReachableShellsLiveAtom: Atom.make(false) }; + return { + ...(await original()), + allReachableShellsLiveAtom: Atom.make(false).pipe(Atom.keepAlive), + }; }); vi.mock(import("../../state/entities"), async (original) => ({ ...(await original()), readProjects: () => state.projects, + readThreadShell: (ref: { threadId: string }) => + ref.threadId === "thread" ? ({ projectId: "effect", worktreePath: null } as never) : null, + readProject: () => state.projects[0] ?? null, })); vi.mock(import("../../hooks/useScratchProject"), async (original) => ({ ...(await original()), @@ -61,6 +75,10 @@ vi.mock(import("../../hooks/useScratchProject"), async (original) => ({ })) as never, })); +vi.mock("../../state/use-atom-query-runner", () => ({ + useAtomQueryRunner: () => state.preview, +})); +vi.mock("../../browser/useOpenLink", () => ({ useOpenLink: () => state.openLink })); vi.mock(import("../../confirmDialog"), async (original) => ({ ...(await original()), requestConfirmDialog: state.confirm, @@ -82,6 +100,12 @@ vi.mock(import("../../state/use-atom-command"), async (original) => { import { AppAtomRegistryProvider, appAtomRegistry } from "../../rpc/atomRegistry"; import { allReachableShellsLiveAtom } from "../../state/shell"; +import { selectThreadDiffPanelSelection, useDiffPanelStore } from "../../diffPanelStore"; +import { + selectActiveRightPanel, + selectActiveRightPanelSurface, + useRightPanelStore, +} from "../../rightPanelStore"; import { DesktopWebLinkCoordinator } from "./DesktopWebLinkCoordinator"; const discoverySettled = allReachableShellsLiveAtom as Atom.Writable; @@ -89,6 +113,7 @@ const environmentId = EnvironmentId.make("local"); const projectId = ProjectId.make("effect"); const project = { id: projectId, + workspaceRoot: "/workspace/effect", environmentId, repositoryIdentity: { provider: "github", @@ -100,7 +125,7 @@ const project = { remoteUrl: "https://github.com/Effect-TS/effect.git", }, }, -} as never; +} as EnvironmentProject; const pullRequestUrl = "https://github.com/Effect-TS/effect/pull/9339"; const pullRequestView = { to: "/pull-requests", @@ -123,6 +148,12 @@ function setup() { state.confirm.mockResolvedValue(true); state.localEnvironmentDisabled = false; state.projects = []; + state.routeParams = {}; + state.preview + .mockReset() + .mockResolvedValue(AsyncResult.success({ sources: [{ kind: "commit", headRef: sha }] })); + useDiffPanelStore.setState({ byThreadKey: {}, branchBaseRefByThreadKey: {} }); + useRightPanelStore.setState({ byThreadKey: {}, userActionRevisionByThreadKey: {} }); appAtomRegistry.set(discoverySettled, false); } @@ -172,6 +203,80 @@ it("holds a launch PR link until project discovery settles, then opens the PR vi expect(state.openScratchProject).not.toHaveBeenCalled(); }); +const sha = "a".repeat(40); +const commitUrl = `https://github.com/Effect-TS/effect/pull/9339/commits/${sha}`; +const threadRef = scopeThreadRef(environmentId, ThreadId.make("thread")); + +it.each([ + ["github.com", "pull/9339/commits"], + ["code.example.com:8443", "commit"], +])("opens a %s/%s commit in the active thread's Diff pane after discovery", async (host, path) => { + setup(); + state.routeParams = { environmentId: "local", threadId: "thread" }; + await using _coordinator = await mount(`https://${host}/Effect-TS/effect/${path}/${sha}`); + expect(state.preview).not.toHaveBeenCalled(); + + state.projects = [ + { + ...project, + repositoryIdentity: { + ...project.repositoryIdentity, + canonicalKey: `${host}/effect-ts/effect`, + locator: { + ...project.repositoryIdentity?.locator, + remoteUrl: `https://${host}/Effect-TS/effect.git`, + }, + }, + } as EnvironmentProject, + ]; + await act(async () => appAtomRegistry.set(discoverySettled, true)); + await vi.waitFor(() => { + expect(selectActiveRightPanel(useRightPanelStore.getState().byThreadKey, threadRef)).toBe( + "diff", + ); + }); + expect( + selectThreadDiffPanelSelection(useDiffPanelStore.getState().byThreadKey, threadRef), + ).toMatchObject({ kind: "commit", sha }); + expect(state.preview).toHaveBeenCalledExactlyOnceWith({ + environmentId, + input: expect.objectContaining({ cwd: "/workspace/effect", commitSha: sha }), + }); + expect(state.navigate).not.toHaveBeenCalled(); + expect(state.openLink).not.toHaveBeenCalled(); +}); + +it("opens a PR commit the workspace lacks in the thread's PR panel", async () => { + setup(); + state.routeParams = { environmentId: "local", threadId: "thread" }; + state.projects = [project]; + state.preview.mockResolvedValueOnce(AsyncResult.success({ sources: [] })); + appAtomRegistry.set(discoverySettled, true); + await using _coordinator = await mount(commitUrl); + await vi.waitFor(() => + expect( + selectActiveRightPanelSurface(useRightPanelStore.getState().byThreadKey, threadRef), + ).toMatchObject({ kind: "pull-request", repository: "effect-ts/effect", number: 9339 }), + ); + expect(useDiffPanelStore.getState().byThreadKey).toEqual({}); + expect(state.openLink).not.toHaveBeenCalled(); +}); + +it("opens a plain commit the workspace lacks like any other delivered link", async () => { + setup(); + state.routeParams = { environmentId: "local", threadId: "thread" }; + state.projects = [project]; + state.preview.mockResolvedValueOnce(AsyncResult.success({ sources: [] })); + appAtomRegistry.set(discoverySettled, true); + const plainCommitUrl = `https://github.com/Effect-TS/effect/commit/${sha}`; + await using _coordinator = await mount(plainCommitUrl); + await vi.waitFor(() => expect(state.openScratchProject).toHaveBeenCalledOnce()); + // Handing it to the OS browser would loop when T3 Code is the default browser. + expect(state.openLink).not.toHaveBeenCalled(); + expect(useDiffPanelStore.getState().byThreadKey).toEqual({}); + expect(useRightPanelStore.getState().byThreadKey).toEqual({}); +}); + it("adds the backend from a pairing link after the user confirms its host", async () => { setup(); await using _coordinator = await mount(pairLink); diff --git a/apps/web/src/components/desktop/DesktopWebLinkCoordinator.tsx b/apps/web/src/components/desktop/DesktopWebLinkCoordinator.tsx index 4ae0b6cdb25b4..ddd892d02bd12 100644 --- a/apps/web/src/components/desktop/DesktopWebLinkCoordinator.tsx +++ b/apps/web/src/components/desktop/DesktopWebLinkCoordinator.tsx @@ -27,6 +27,7 @@ import { selectActiveRightPanelSurface, useRightPanelStore } from "../../rightPa import { appAtomRegistry } from "../../rpc/atomRegistry"; import { resolveThreadRouteTarget } from "../../threadRoutes"; import { assetEnvironment } from "../../state/assets"; +import { readProject, readThreadShell } from "../../state/entities"; import { useEnvironmentHttpBaseUrl, usePrimaryEnvironment } from "../../state/environments"; import { previewEnvironment } from "../../state/preview"; import { allReachableShellsLiveAtom } from "../../state/shell"; @@ -163,25 +164,14 @@ export function DesktopWebLinkCoordinator() { return threadRef; }; - const openLink = useEffectEvent(async (url: string) => { - const pairingUrl = readPairingAppLink(url); - if (pairingUrl !== null) { - await pairFromLink(pairingUrl); - return; - } - const appPath = promptLinkAppPath(url); - if (appPath !== null) { - await router.navigate({ href: appPath }); - return; - } - if (openChangeRequest(externalClick, url)) return; + const openInBrowser = useEffectEvent(async (url: string) => { + const routeParams = router.state.matches.at(-1)?.params ?? {}; const isFile = url.startsWith("file:"); // Only this machine's environment serves files, so fail before starting a thread. if (isFile && httpBaseUrl === null) { reportFailure(url); return; } - const routeParams = router.state.matches.at(-1)?.params ?? {}; const threadRef = threadShowingBrowser( routeParams, @@ -210,15 +200,49 @@ export function DesktopWebLinkCoordinator() { } }); + const openLink = useEffectEvent(async (url: string) => { + const pairingUrl = readPairingAppLink(url); + if (pairingUrl !== null) { + await pairFromLink(pairingUrl); + return; + } + const appPath = promptLinkAppPath(url); + if (appPath !== null) { + await router.navigate({ href: appPath }); + return; + } + const routeParams = router.state.matches.at(-1)?.params ?? {}; + const routeTarget = resolveThreadRouteTarget(routeParams); + const activeThreadRef = routeTarget?.kind === "server" ? routeTarget.threadRef : undefined; + const activeThread = activeThreadRef ? readThreadShell(activeThreadRef) : null; + const activeProject = + activeThread && activeThreadRef + ? readProject(scopeProjectRef(activeThreadRef.environmentId, activeThread.projectId)) + : null; + const commitThreadRef = + activeThread && + sourceControlClients.parseCommitUrl(url, activeProject?.repositoryIdentity?.provider) !== null + ? activeThreadRef + : undefined; + // A commit the thread's workspace lacks continues here, after its lookup. + const openUnavailableCommit = () => { + queueRef.current = queueRef.current.then(() => openInBrowser(url)).catch(() => undefined); + }; + if (openChangeRequest(externalClick, url, commitThreadRef, undefined, openUnavailableCommit)) + return; + await openInBrowser(url); + }); + const receiveLink = useEffectEvent((url: string) => { - // A connected server publishes its config before its project snapshot, so a PR link - // delivered at launch would miss its project and fall through to the browser. Hold PR - // links, and only those, until discovery settles; an ordinary link never waits for it. + // A connected server publishes its config before its project snapshot, so a source control + // link delivered at launch would miss its project and fall through to the browser. Hold + // those links until discovery settles; an ordinary link never waits for it. // Discovery never settles without the local environment, so those links do not wait. const hold = openPullRequestsInApp && !localEnvironmentDisabled && - sourceControlClients.parseChangeRequestUrl(url) !== null + (sourceControlClients.parseChangeRequestUrl(url) !== null || + sourceControlClients.parseCommitUrl(url) !== null) ? projectsSettled() : Promise.resolve(); void hold.then(() => { diff --git a/apps/web/src/components/device/DeviceControlsRail.tsx b/apps/web/src/components/device/DeviceControlsRail.tsx index 3ed2b3a4fa544..740c342b2cf9f 100644 --- a/apps/web/src/components/device/DeviceControlsRail.tsx +++ b/apps/web/src/components/device/DeviceControlsRail.tsx @@ -1,4 +1,4 @@ -import type { DevicePlatform } from "@t3tools/contracts"; +import type { DeviceHostSummary, DevicePlatform } from "@t3tools/contracts"; import { Camera, ChevronLeft, @@ -36,6 +36,7 @@ import type { DeviceControls } from "./useDeviceControls"; export function DeviceControlsRail(props: { platform: DevicePlatform; + hostKind: DeviceHostSummary["kind"] | undefined; handle: DeviceStreamHandle | null; view: DeviceViewControls; controls: DeviceControls; @@ -46,13 +47,12 @@ export function DeviceControlsRail(props: { onFloat: () => void; onClose: () => void; onPowerOff: () => void; - /** The host takes input but runs no device commands (Simlock): no rotation, settings, tools, or screenshots. */ - inputOnly?: boolean; }) { const { view, handle, controls } = props; const popupSide = "left"; const settings = controls.detail?.settings; const inputDisabled = !handle?.inputConnected; + const runsHostCommands = props.hostKind === "local" || props.hostKind === "ssh"; const nextAppearance = settings?.appearance === "dark" ? "light" : "dark"; return (
    {showExplorer ? ( - + ) : null}
    diff --git a/apps/web/src/components/guest/GuestPage.tsx b/apps/web/src/components/guest/GuestPage.tsx index 21c35447ddf8c..7e41e9a8ca297 100644 --- a/apps/web/src/components/guest/GuestPage.tsx +++ b/apps/web/src/components/guest/GuestPage.tsx @@ -1,6 +1,7 @@ import { EnvironmentId, GUEST_MESSAGE_MAX_LENGTH, + GUEST_HISTORY_ROUTE, GUEST_NAME_MAX_LENGTH, GUEST_ROUTE_PREFIX, GUEST_SHARE_PARAM, @@ -8,6 +9,7 @@ import { type GuestRedeemRequest, type GuestShareView, type GuestThreadModel, + OrchestrationV2ThreadHistoryPage, type OrchestrationV2ThreadProjection, type ThreadGuestMessage, type ThreadGuestMessageStatus, @@ -46,7 +48,7 @@ import { GuestMessageAttachments } from "../chat/GuestMessageAttachments"; import { HostImageUrlContext } from "../chat/hostImageUrl"; import { PierreEntryIcon } from "../chat/PierreEntryIcon"; import { GuestMessageName, guestBubbleClassName } from "../chat/guestMessageStyle"; -import { MessagesTimeline } from "../chat/MessagesTimeline"; +import { MessagesTimeline, type MessagesTimelineHistoryControls } from "../chat/MessagesTimeline"; import { ProviderInstanceIcon } from "../chat/ProviderInstanceIcon"; import { getDisplayModelName, getTriggerDisplayModelName } from "../chat/providerIconUtils"; import { ScrollToEndButton } from "../chat/TimelineOverlays"; @@ -131,6 +133,32 @@ function takeGuestEntry(): GuestEntry { } const decodeErrorBody = Schema.decodeUnknownOption(Schema.Struct({ message: Schema.String })); +const decodeHistoryPage = Schema.decodeUnknownOption( + Schema.toCodecJson(OrchestrationV2ThreadHistoryPage), +); + +/** Resolves to the page, or to a message the guest can read. */ +async function fetchGuestHistoryPage( + shareId: string, + cursor: string, +): Promise { + const failed = "Could not load earlier turns. Try again."; + try { + const response = await fetch( + `${GUEST_HISTORY_ROUTE}?${GUEST_SHARE_PARAM}=${encodeURIComponent(shareId)}&cursor=${encodeURIComponent(cursor)}`, + ); + const json: unknown = await response.json().catch(() => null); + if (!response.ok) { + return Option.match(decodeErrorBody(json), { + onNone: () => failed, + onSome: ({ message }) => message, + }); + } + return Option.getOrElse(decodeHistoryPage(json), () => failed); + } catch { + return "Could not reach the server."; + } +} /** Resolves to null on success, or to a message the guest can read. */ async function postGuest( @@ -241,6 +269,32 @@ function GuestSession({ shareId }: { readonly shareId: string }) { // missing, invalid, or ended session. Network drops reconnect on their own. const [rejected, setRejected] = useState(false); + const [historyLoad, setHistoryLoad] = useState<{ + readonly cursor: string; + readonly error: string | null; + } | null>(null); + const historyCursor = stream.historyCursor; + // A load for an older cursor is stale: a page or a reconnect snapshot moved the cursor on. + const currentLoad = historyLoad?.cursor === historyCursor ? historyLoad : null; + const loadingHistory = currentLoad !== null && currentLoad.error === null; + const historyError = currentLoad?.error ?? null; + const loadEarlier = async () => { + if (historyCursor === null || loadingHistory) return; + setHistoryLoad({ cursor: historyCursor, error: null }); + const result = await fetchGuestHistoryPage(shareId, historyCursor); + if (typeof result === "string") setHistoryLoad({ cursor: historyCursor, error: result }); + else dispatch({ type: "history-page", cursor: historyCursor, page: result }); + }; + const historyControls: MessagesTimelineHistoryControls | undefined = + historyCursor === null + ? undefined + : { + hasMoreHistory: true, + loading: loadingHistory, + error: historyError, + onLoadEarlier: () => void loadEarlier(), + }; + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- sets document.title to the shared thread's title useEffect(() => { if (stream.view) document.title = stream.view.title; @@ -273,6 +327,7 @@ function GuestSession({ shareId }: { readonly shareId: string }) { view={stream.view} projection={stream.projection} awaitingRowIds={stream.awaitingRowIds} + historyControls={historyControls} /> ); } @@ -358,10 +413,12 @@ function GuestThread({ view: { shareId, title, expiresAt, messages, model }, projection, awaitingRowIds, + historyControls, }: { readonly view: GuestShareView; readonly projection: OrchestrationV2ThreadProjection; readonly awaitingRowIds: GuestStreamState["awaitingRowIds"]; + readonly historyControls: MessagesTimelineHistoryControls | undefined; }) { // No ":" so the timeline finds no environment thread behind the key. const threadKey = `guest-${shareId}`; @@ -467,6 +524,7 @@ function GuestThread({ onFileDownload={openFile} renderHtmlRender={renderHtmlRender} turnItemSource={turnItemSource} + historyControls={historyControls} activeThreadEnvironmentId={GUEST_ENVIRONMENT_ID} markdownCwd={undefined} resolvedTheme={resolvedTheme} diff --git a/apps/web/src/components/guest/guestStream.logic.test.ts b/apps/web/src/components/guest/guestStream.logic.test.ts index fbdf9ce1927c2..674227dc171ff 100644 --- a/apps/web/src/components/guest/guestStream.logic.test.ts +++ b/apps/web/src/components/guest/guestStream.logic.test.ts @@ -112,6 +112,45 @@ describe("applyGuestStreamEvent", () => { expect(apply([itemEvent(2, reply("a1", "Changed"))], once)).toBe(once); }); + it("merges older pages in front, only for the cursor the thread is at", () => { + const windowed: GuestStreamEvent = { + type: "thread", + item: { + kind: "snapshot", + snapshotSequence: 1, + projection, + historyCursor: "cursor-1", + hasMoreHistory: true, + }, + }; + const page = (id: string, nextCursor: string | null) => ({ + snapshotSequence: 1, + items: [ + { + position: 0, + visibility: "local" as const, + sourceThreadId: projection.thread.id, + sourceItemId: TurnItemId.make(id), + item: reply(id, "Earlier"), + }, + ], + nextCursor, + hasMoreHistory: nextCursor !== null, + }); + const live = apply([windowed, itemEvent(2, reply("a1", "Hi"))]); + expect(live.historyCursor).toBe("cursor-1"); + // A page fetched before a reconnect moved the cursor changes nothing. + const stale = { type: "history-page" as const, cursor: "cursor-0", page: page("old", null) }; + expect(applyGuestStreamEvent(live, stale)).toBe(live); + const merged = applyGuestStreamEvent(live, { + type: "history-page", + cursor: "cursor-1", + page: page("o1", null), + }); + expect(itemIds(merged)).toEqual(["o1", "a1"]); + expect(merged.historyCursor).toBeNull(); + }); + it("replaces the thread on a new snapshot, as after a reconnect", () => { const before = apply([snapshot(1), itemEvent(2, reply("a1", "Hi")), { type: "ended" }]); const after = apply([snapshot(1)], { ...before, ended: false }); diff --git a/apps/web/src/components/guest/guestStream.logic.ts b/apps/web/src/components/guest/guestStream.logic.ts index 0b37d66eeec84..03eafb6d348ee 100644 --- a/apps/web/src/components/guest/guestStream.logic.ts +++ b/apps/web/src/components/guest/guestStream.logic.ts @@ -9,11 +9,13 @@ import { type GuestMessageId, type GuestShareView, type OrchestrationV2ProjectedTurnItem, + type OrchestrationV2ThreadHistoryPage, type OrchestrationV2ThreadProjection, type OrchestrationV2TurnItem, type ThreadGuestMessage, } from "@t3tools/contracts"; import { applyOrchestrationV2ProjectionEvent } from "@t3tools/client-runtime/state/orchestration-v2-projection"; +import { mergeOlderHistoryIntoProjection } from "@t3tools/client-runtime/state/thread-history-merge"; import { boundedSnapshotProjection } from "@t3tools/shared/orchestrationV2BoundedSnapshot"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; @@ -25,6 +27,8 @@ export interface GuestStreamState { readonly sequence: number; /** Set while the snapshot is a window of a longer history, as the owner's client tracks it. */ readonly partial: { readonly latestLocalTurnOrdinal: number | null } | null; + /** Where "Load earlier turns" continues, or null once the whole thread is loaded. */ + readonly historyCursor: string | null; readonly ended: boolean; /** * Messages this page saw waiting that have no transcript row yet. Survives reconnects, and a @@ -38,6 +42,7 @@ export const initialGuestStreamState: GuestStreamState = { projection: null, sequence: 0, partial: null, + historyCursor: null, ended: false, awaitingRowIds: new Set(), }; @@ -76,16 +81,32 @@ const decodeGuestStreamEvent = Schema.decodeUnknownOption( export const parseGuestStreamEvent = (data: string): GuestStreamEvent | null => Option.getOrNull(decodeGuestStreamEvent(data)); +/** A page of older turns, fetched from `cursor`. */ +export type GuestHistoryPageLoaded = { + readonly type: "history-page"; + readonly cursor: string; + readonly page: OrchestrationV2ThreadHistoryPage; +}; + /** * Applies the owner's thread stream the way the owner's client does (see * EnvironmentThreadState in client-runtime): a snapshot replaces the thread, and later events - * pass through the same projection reducer, once each. + * pass through the same projection reducer, once each. Older pages merge in front, as they do in + * the owner's view. */ export function applyGuestStreamEvent( state: GuestStreamState, - event: GuestStreamEvent, + event: GuestStreamEvent | GuestHistoryPageLoaded, ): GuestStreamState { switch (event.type) { + case "history-page": + // A snapshot since the request replaced the thread and its cursor. + if (state.projection === null || state.historyCursor !== event.cursor) return state; + return { + ...state, + projection: mergeOlderHistoryIntoProjection(state.projection, event.page.items), + historyCursor: event.page.hasMoreHistory ? event.page.nextCursor : null, + }; case "view": // The page follows the share it names; a view of another one means the link changed. if (state.view !== null && state.view.shareId !== event.view.shareId) { @@ -108,6 +129,7 @@ export function applyGuestStreamEvent( item.hasMoreHistory === true || (item.historyCursor ?? null) !== null ? { latestLocalTurnOrdinal: item.latestLocalTurnOrdinal ?? null } : null, + historyCursor: item.hasMoreHistory === true ? (item.historyCursor ?? null) : null, awaitingRowIds: withoutPlaced( state.awaitingRowIds, projection.visibleTurnItems.map((row) => row.item), diff --git a/apps/web/src/components/preview/RightPanelResizeHandle.tsx b/apps/web/src/components/preview/RightPanelResizeHandle.tsx index dd644ccf0c822..851daaebe5eab 100644 --- a/apps/web/src/components/preview/RightPanelResizeHandle.tsx +++ b/apps/web/src/components/preview/RightPanelResizeHandle.tsx @@ -1,16 +1,23 @@ +import type { ComponentProps } from "react"; import type { ResizableWidthHandlers } from "~/hooks/useResizableWidth"; import { cn } from "~/lib/utils"; -interface Props { +interface Props extends Pick< + ComponentProps<"div">, + "tabIndex" | "onKeyDown" | "aria-valuenow" | "aria-valuemin" | "aria-valuemax" +> { handlers: ResizableWidthHandlers; className?: string; + label?: string; } -export function RightPanelResizeHandle({ handlers, className }: Props) { +export function RightPanelResizeHandle({ handlers, className, label, ...separatorProps }: Props) { return (
    ); diff --git a/apps/web/src/components/pullRequest/guide/useGuideDiffFiles.browser.test.tsx b/apps/web/src/components/pullRequest/guide/useGuideDiffFiles.browser.test.tsx new file mode 100644 index 0000000000000..cdb7d4099e5c7 --- /dev/null +++ b/apps/web/src/components/pullRequest/guide/useGuideDiffFiles.browser.test.tsx @@ -0,0 +1,70 @@ +import { RegistryContext } from "@effect/atom-react"; +import { EnvironmentId, ProjectId, type PullRequestDiffResult } from "@t3tools/contracts"; +import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity"; +import { expect, it, vi } from "vite-plus/test"; + +import { act, render } from "../../../../test/render"; + +const state = vi.hoisted(() => ({ + pages: new Map>>(), +})); +vi.mock("~/state/pullRequests", () => ({ + pullRequestEnvironment: { + diff: ({ input }: { input: { cursor?: string } }) => state.pages.get(input.cursor ?? null)!, + }, +})); + +import { useGuideDiffFiles } from "./useGuideDiffFiles"; + +function Probe({ wantedPaths }: { wantedPaths: string[] }) { + const diff = useGuideDiffFiles({ + environmentId: EnvironmentId.make("test"), + reference: { projectId: ProjectId.make("project"), repository: "owner/repo", number: 1 }, + headSha: "new-head", + wantedPaths, + }); + return ( + + {JSON.stringify({ + loading: diff.loading, + files: [...diff.files].map(([path, file]) => [path, file.additionLines.join("")]), + })} + + ); +} + +function page(path: string, value: string, nextCursor: string | null): PullRequestDiffResult { + return { + patch: `diff --git a/${path} b/${path}\n--- a/${path}\n+++ b/${path}\n@@ -1 +1 @@\n-old\n+${value}\n`, + truncated: false, + nextCursor, + }; +} + +it.each([ + { name: "contains the requested file", path: "wanted.ts", cursor: null }, + { name: "ends before a newly added file", path: "other.ts", cursor: null }, + { name: "has an obsolete next cursor", path: "other.ts", cursor: "old-next" }, +])("waits for a cached page that $name to settle", async ({ path, cursor }) => { + state.pages.clear(); + await using stack = new AsyncDisposableStack(); + const registry = stack.adopt(AtomRegistry.make(), (value) => value.dispose()); + const first = Atom.make>( + AsyncResult.success(page(path, "STALE", cursor), { waiting: true }), + ); + state.pages.set(null, first); + state.pages.set( + "old-next", + Atom.make>(AsyncResult.initial(true)), + ); + await using screen = await render( + + + , + ); + const result = () => JSON.parse(screen.container.querySelector("output")!.textContent!); + expect(result()).toEqual({ loading: true, files: [] }); + + await act(() => registry.set(first, AsyncResult.success(page("wanted.ts", "FRESH", null)))); + expect(result()).toEqual({ loading: false, files: [["wanted.ts", "FRESH"]] }); +}); diff --git a/apps/web/src/components/pullRequest/guide/useGuideDiffFiles.ts b/apps/web/src/components/pullRequest/guide/useGuideDiffFiles.ts index c905488949cff..21765c68d18e8 100644 --- a/apps/web/src/components/pullRequest/guide/useGuideDiffFiles.ts +++ b/apps/web/src/components/pullRequest/guide/useGuideDiffFiles.ts @@ -51,7 +51,7 @@ export function useGuideDiffFiles({ ); const data = query.data; - if (data !== null && !current.done && current.folded !== data) { + if (data !== null && !query.isPending && !current.done && current.folded !== data) { const parsed = getRenderablePatch( data.patch, `pull-request-guide:${key}:${current.cursor ?? "first"}`, diff --git a/apps/web/src/components/pullRequest/pullRequestProjectFilter.logic.test.ts b/apps/web/src/components/pullRequest/pullRequestProjectFilter.logic.test.ts index ec3319f010843..59d7e0c83ad12 100644 --- a/apps/web/src/components/pullRequest/pullRequestProjectFilter.logic.test.ts +++ b/apps/web/src/components/pullRequest/pullRequestProjectFilter.logic.test.ts @@ -2,7 +2,7 @@ import { EnvironmentId, ProjectId } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; import { findScopedProject } from "./pullRequestList.logic"; -import { pullRequestFilterProjects } from "./pullRequestProjectFilter.logic"; +import { pullRequestFilterProjects } from "@t3tools/client-runtime/pull-request-project-filter"; const cups = EnvironmentId.make("env-cups"); const nucbox = EnvironmentId.make("env-nucbox"); diff --git a/apps/web/src/components/pullRequest/usePullRequestListView.ts b/apps/web/src/components/pullRequest/usePullRequestListView.ts index c4e917fa7134e..fe22fe5af2838 100644 --- a/apps/web/src/components/pullRequest/usePullRequestListView.ts +++ b/apps/web/src/components/pullRequest/usePullRequestListView.ts @@ -36,7 +36,7 @@ import { import type { PullRequestExpectedHost } from "./PullRequestListFilters"; import type { PullRequestListPreferences, PullRequestListSort } from "./pullRequestListPreferences"; import { assignProjectsToEnvironments } from "./pullRequestProjectAssignment.logic"; -import { pullRequestFilterProjects } from "./pullRequestProjectFilter.logic"; +import { pullRequestFilterProjects } from "@t3tools/client-runtime/pull-request-project-filter"; import { pullRequestListEntryId } from "./pullRequestListGroups.logic"; export interface PullRequestsSearch extends PullRequestListPreferences { diff --git a/apps/web/src/components/settings/AddProviderInstanceDialog.environment.browser.test.tsx b/apps/web/src/components/settings/AddProviderInstanceDialog.environment.browser.test.tsx index 90307d3470707..2f3414683e077 100644 --- a/apps/web/src/components/settings/AddProviderInstanceDialog.environment.browser.test.tsx +++ b/apps/web/src/components/settings/AddProviderInstanceDialog.environment.browser.test.tsx @@ -1,5 +1,4 @@ import { EnvironmentId, ProviderDriverKind, type AcpRegistrySearchAgent } from "@t3tools/contracts"; -import type { ReactNode } from "react"; import { userEvent } from "vite-plus/test/browser"; import { describe, expect, it, vi } from "vite-plus/test"; @@ -11,7 +10,7 @@ const actions = vi.hoisted(() => ({ canManageProviders: true, })); const settingsHooks = vi.hoisted(() => ({ - read: vi.fn(() => ({ providerInstances: {}, secretProviders: {} })), + read: vi.fn(() => ({ providerInstances: {} })), mutate: vi.fn(), useMutation: vi.fn(), })); @@ -33,12 +32,6 @@ vi.mock(import("@t3tools/client-runtime/state/runtime"), async (original) => ({ ...(await original()), squashAtomCommandFailure: () => new Error("The settings update failed."), })); -vi.mock(import("@tanstack/react-router"), async (original) => ({ - ...(await original()), - Link: (({ children }: { children: ReactNode }) => ( -     {children} - )) as never, -})); vi.mock(import("../../hooks/useSettings"), async (original) => ({ ...(await original()), useEnvironmentSettings: settingsHooks.read as never, @@ -150,7 +143,7 @@ function setup() { actions.canManageProviders = true; actions.toast.mockReset(); actions.onOpenChange.mockReset(); - settingsHooks.read.mockReset().mockReturnValue({ providerInstances: {}, secretProviders: {} }); + settingsHooks.read.mockReset().mockReturnValue({ providerInstances: {} }); settingsHooks.mutate.mockReset().mockResolvedValue({ _tag: "Success", value: {} }); settingsHooks.useMutation.mockReset().mockReturnValue(settingsHooks.mutate); } @@ -160,7 +153,6 @@ describe("AddProviderInstanceDialog environment routing", () => { setup(); settingsHooks.read.mockReturnValue({ providerInstances: { codex_2: { driver: "codex", enabled: false } }, - secretProviders: {}, }); await using _screen = await render(dialog()); await click("Configure manually"); diff --git a/apps/web/src/components/settings/AddProviderInstanceDialog.tsx b/apps/web/src/components/settings/AddProviderInstanceDialog.tsx index 76140ca42544e..40cc259cd34e6 100644 --- a/apps/web/src/components/settings/AddProviderInstanceDialog.tsx +++ b/apps/web/src/components/settings/AddProviderInstanceDialog.tsx @@ -55,6 +55,7 @@ import { resolveOfficialAcpRegistryIconUrl } from "./AcpRegistryIcon"; import { AddManagedCodexAccountDialog } from "./CodexSetupSection"; import { MultipleAccountsNote } from "./MultipleAccountsNote"; import { ProviderEnvironmentSection } from "./ProviderInstanceCard"; +import { useSecretSources } from "./SecretSourceInput"; /** Limit the slug to 48 chars so the composed ProviderInstanceId fits its 64-char cap. */ function slugifyLabel(value: string): string { @@ -160,6 +161,7 @@ export function AddProviderInstanceDialog({ onCreated, }: AddProviderInstanceDialogProps) { const settings = useEnvironmentSettings(environmentId); + const secretSources = useSecretSources(environmentId); const persistProviderInstance = usePersistEnvironmentProviderInstanceMutation(environmentId); const canManageProviders = useEnvironmentScope(environmentId, AuthProvidersManageScope); @@ -534,10 +536,10 @@ export function AddProviderInstanceDialog({ /> {isLocalAcp ? ( ) : null} diff --git a/apps/web/src/components/settings/AddUsageLimitSourceDialog.logic.test.ts b/apps/web/src/components/settings/AddUsageLimitSourceDialog.logic.test.ts new file mode 100644 index 0000000000000..33192bbaab71f --- /dev/null +++ b/apps/web/src/components/settings/AddUsageLimitSourceDialog.logic.test.ts @@ -0,0 +1,61 @@ +import { DEFAULT_SERVER_SETTINGS, type ServerSettings } from "@t3tools/contracts"; +import { applyServerSettingsPatch } from "@t3tools/shared/serverSettings"; +import { describe, expect, it } from "vite-plus/test"; + +import { sourceIdFromUrl } from "./AddUsageLimitSourceDialog.logic"; + +describe("adding usage limit sources", () => { + it.each([ + ["http://proxy.test:8318/hub-a", "http://proxy.test:8318/hub-b"], + ["localhost:8318", "localhost:9318"], + ])("retains saved configurations for %s and %s", (firstUrl, secondUrl) => { + let settings: ServerSettings = DEFAULT_SERVER_SETTINGS; + const sources = [ + [firstUrl, "First hub", "first-key"], + [secondUrl, "Second hub", "second-key"], + [`${secondUrl}/extra`, "Third hub", "third-key"], + ] as const; + for (const [url, label, managementKey] of sources) { + const id = sourceIdFromUrl(url, settings.usageLimitSources); + settings = applyServerSettingsPatch(settings, { + usageLimitSources: { + [id]: { kind: "cliproxy", url, label, managementKey, enabled: true }, + }, + }); + } + expect(Object.values(settings.usageLimitSources)).toEqual( + sources.map(([url, label, managementKey]) => ({ + kind: "cliproxy", + url, + label, + managementKey, + enabled: true, + })), + ); + }); + + it("updates a saved hub when its URL is added again", () => { + let settings: ServerSettings = DEFAULT_SERVER_SETTINGS; + for (const managementKey of ["old-key", "new-key"]) { + const url = "http://proxy.test:8318/hub-a"; + settings = applyServerSettingsPatch(settings, { + usageLimitSources: { + [sourceIdFromUrl(url, settings.usageLimitSources)]: { + kind: "cliproxy", + url, + managementKey, + enabled: true, + }, + }, + }); + } + expect(Object.values(settings.usageLimitSources)).toEqual([ + { + kind: "cliproxy", + url: "http://proxy.test:8318/hub-a", + managementKey: "new-key", + enabled: true, + }, + ]); + }); +}); diff --git a/apps/web/src/components/settings/AddUsageLimitSourceDialog.logic.ts b/apps/web/src/components/settings/AddUsageLimitSourceDialog.logic.ts new file mode 100644 index 0000000000000..67ccabdefefc9 --- /dev/null +++ b/apps/web/src/components/settings/AddUsageLimitSourceDialog.logic.ts @@ -0,0 +1,24 @@ +import { type UnifiedSettings, UsageLimitSourceId } from "@t3tools/contracts"; + +export function sourceIdFromUrl( + url: string, + sources: UnifiedSettings["usageLimitSources"], +): UsageLimitSourceId { + const saved = Object.entries(sources).find(([, source]) => source.url === url); + if (saved) return UsageLimitSourceId.make(saved[0]); + let host = url; + try { + host = new URL(url).host; + } catch {} + const slug = host + .toLowerCase() + .replace(/[^a-z0-9.-]+/g, "-") + .replace(/^-+|-+$/g, ""); + // Settings merge by ID, so adding a different hub must not replace an existing entry. + const base = `cliproxy-${slug || "hub"}`; + let id = base; + for (let suffix = 2; id in sources; suffix += 1) { + id = `${base}-${suffix}`; + } + return UsageLimitSourceId.make(id); +} diff --git a/apps/web/src/components/settings/AddUsageLimitSourceDialog.tsx b/apps/web/src/components/settings/AddUsageLimitSourceDialog.tsx index ede36f266d4ae..1abfacc467733 100644 --- a/apps/web/src/components/settings/AddUsageLimitSourceDialog.tsx +++ b/apps/web/src/components/settings/AddUsageLimitSourceDialog.tsx @@ -1,8 +1,7 @@ -import { type EnvironmentId, UsageLimitSourceId } from "@t3tools/contracts"; -import * as Result from "effect/Result"; +import type { EnvironmentId, SecretReference, UnifiedSettings } from "@t3tools/contracts"; import { useState } from "react"; -import { useEnvironmentSettings, useUpdateEnvironmentSettings } from "../../hooks/useSettings"; +import { useUpdateEnvironmentSettings } from "../../hooks/useSettings"; import { Button } from "../ui/button"; import { Dialog, @@ -14,61 +13,42 @@ import { DialogTitle, } from "../ui/dialog"; import { Input } from "../ui/input"; +import { sourceIdFromUrl } from "./AddUsageLimitSourceDialog.logic"; +import { SecretSourceInput, useSecretSources } from "./SecretSourceInput"; import { Label } from "../ui/label"; -import { - EMPTY_SECRET_FIELD_DRAFT, - SecretField, - secretFieldSubmission, -} from "./SecretSourceControls"; - -function sourceIdFromUrl(url: string): UsageLimitSourceId { - let host = url; - try { - host = new URL(url).host; - } catch {} - const slug = host - .toLowerCase() - .replace(/[^a-z0-9.-]+/g, "-") - .replace(/^-+|-+$/g, ""); - return UsageLimitSourceId.make(`cliproxy-${slug || "hub"}`); -} -/** - * A literal management key is sent once to server secret storage and settings retain only a - * redaction marker; a secret provider reference is stored as written. - */ +/** Send the management key once to server secret storage; settings retain only a redaction marker. */ export function AddUsageLimitSourceDialog({ open, onOpenChange, environmentId, environmentLabel, + sources, }: { readonly open: boolean; readonly onOpenChange: (open: boolean) => void; readonly environmentId: EnvironmentId; readonly environmentLabel: string; + readonly sources: UnifiedSettings["usageLimitSources"]; }) { const updateSettings = useUpdateEnvironmentSettings(environmentId); const [label, setLabel] = useState(""); const [url, setUrl] = useState(""); - const secretProviders = useEnvironmentSettings( - environmentId, - (settings) => settings.secretProviders, - ); - const [keyDraft, setKeyDraft] = useState(EMPTY_SECRET_FIELD_DRAFT); - const managementKey = Result.getOrUndefined(secretFieldSubmission(keyDraft, secretProviders)); + const secretSources = useSecretSources(environmentId); + const [keyDraft, setKeyDraft] = useState(""); + const managementKey = typeof keyDraft === "string" ? keyDraft.trim() : keyDraft; const trimmedUrl = url.trim(); - const canSave = trimmedUrl.length > 0 && managementKey !== undefined; + const canSave = trimmedUrl.length > 0 && managementKey !== ""; const reset = () => { setLabel(""); setUrl(""); - setKeyDraft(EMPTY_SECRET_FIELD_DRAFT); + setKeyDraft(""); }; const save = () => { if (!canSave) return; - const id = sourceIdFromUrl(trimmedUrl); + const id = sourceIdFromUrl(trimmedUrl, sources); updateSettings({ usageLimitSources: { [id]: { @@ -120,14 +100,14 @@ export function AddUsageLimitSourceDialog({
    - setKeyDraft(event.target.value)} + onSecretChange={(secret) => setKeyDraft(secret ?? "")} />
    diff --git a/apps/web/src/components/settings/CodexSetupSection.tsx b/apps/web/src/components/settings/CodexSetupSection.tsx index bab910d5eda82..e7ef1c9a20404 100644 --- a/apps/web/src/components/settings/CodexSetupSection.tsx +++ b/apps/web/src/components/settings/CodexSetupSection.tsx @@ -914,7 +914,7 @@ function ManagedCodexSetup({ title="ChatGPT account" description={accountDescription} control={ -
    +
    {authActive ? ( <> {waitingControl} @@ -1001,6 +1001,16 @@ function ManagedCodexSetup({ Use a different account ) : null} + {allowExistingCli ? ( + + ) : null} )}
    diff --git a/apps/web/src/components/settings/DeviceSourcesSettings.tsx b/apps/web/src/components/settings/DeviceSourcesSettings.tsx index d065c924f6a28..09ccc32fd61b9 100644 --- a/apps/web/src/components/settings/DeviceSourcesSettings.tsx +++ b/apps/web/src/components/settings/DeviceSourcesSettings.tsx @@ -31,7 +31,6 @@ import { toastManager } from "../ui/toast"; import { DeviceHostEditor } from "./DeviceHostEditor"; import { updateDeviceHosts, groupDeviceHosts } from "./deviceHostsSettings.logic"; import { EasDeviceHostEditor } from "./EasDeviceHostEditor"; -import { EMPTY_SECRET_PROVIDERS } from "./SecretSourceControls"; import { easSessionLimits, newEasDeviceHost, @@ -586,11 +585,6 @@ export function DeviceSourcesSettings(props: { environmentId={editing.environmentId} host={editing.host} isNew={editing.isNew} - secretProviders={ - connectedEnvironments.find( - (candidate) => candidate.environmentId === editing.environmentId, - )?.serverConfig?.settings.secretProviders ?? EMPTY_SECRET_PROVIDERS - } busy={busy} onSave={(host) => void saveEas(host, false, editing.isNew ? undefined : editing.host)} onClose={() => setEditing(null)} diff --git a/apps/web/src/components/settings/DiagnosticsSettings.tsx b/apps/web/src/components/settings/DiagnosticsSettings.tsx index 613074b2d914b..0fcea4fa5de00 100644 --- a/apps/web/src/components/settings/DiagnosticsSettings.tsx +++ b/apps/web/src/components/settings/DiagnosticsSettings.tsx @@ -48,7 +48,7 @@ function formatCount(value: number): string { function formatDuration(value: number): string { if (value < 1_000) return `${Math.round(value)} ms`; - return `${(value / 1_000).toFixed(value >= 10_000 ? 1 : 2)} s`; + return `${(value / 1_000).toFixed(value >= 10_000 ? 1 : 2).replace(/\.0+$/, "")} s`; } function formatBytes(value: number): string { @@ -438,10 +438,10 @@ const RESOURCE_HISTORY_WINDOWS = [ ] as const; function formatCpuTime(seconds: number): string { - if (seconds < 60) return `${seconds.toFixed(seconds >= 10 ? 1 : 2)}s`; + if (seconds < 60) return `${seconds.toFixed(seconds >= 10 ? 1 : 2).replace(/\.0+$/, "")}s`; const minutes = seconds / 60; - if (minutes < 60) return `${minutes.toFixed(minutes >= 10 ? 1 : 2)}m`; - return `${(minutes / 60).toFixed(2)}h`; + if (minutes < 60) return `${minutes.toFixed(minutes >= 10 ? 1 : 2).replace(/\.0+$/, "")}m`; + return `${(minutes / 60).toFixed(2).replace(/\.0+$/, "")}h`; } function formatShortProcessName(command: string): string { diff --git a/apps/web/src/components/settings/EasDeviceHostEditor.tsx b/apps/web/src/components/settings/EasDeviceHostEditor.tsx index 18fb8767d7755..70037c1ac7aae 100644 --- a/apps/web/src/components/settings/EasDeviceHostEditor.tsx +++ b/apps/web/src/components/settings/EasDeviceHostEditor.tsx @@ -2,11 +2,12 @@ import { useAtomValue } from "@effect/atom-react"; import { useId, useRef, useState } from "react"; import * as Cause from "effect/Cause"; import * as Option from "effect/Option"; -import * as Result from "effect/Result"; import { + isSecretReference, type DeviceEasAccount, type EasDeviceHostConfig, type EnvironmentId, + type SettingsSecretValue, } from "@t3tools/contracts"; import { XIcon } from "lucide-react"; @@ -26,13 +27,7 @@ import { Input } from "../ui/input"; import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; import { Spinner } from "../ui/spinner"; import { parseEasDeviceHostDraft } from "./easDeviceHostsSettings.logic"; -import { - EMPTY_SECRET_FIELD_DRAFT, - SecretField, - secretFieldDraft, - secretFieldSubmission, - type SecretProviders, -} from "./SecretSourceControls"; +import { SecretSourceInput, useSecretSources } from "./SecretSourceInput"; const FROM_THREAD_PROJECT = "thread-project"; @@ -57,7 +52,6 @@ export function EasDeviceHostEditor(props: { environmentId: EnvironmentId; host: EasDeviceHostConfig; isNew: boolean; - secretProviders: SecretProviders; busy: boolean; onSave: (host: EasDeviceHostConfig) => void; onClose: () => void; @@ -65,20 +59,11 @@ export function EasDeviceHostEditor(props: { const check = useAtomCommand(deviceEnvironment.easAccount, { reportFailure: false }); const [draft, setDraft] = useState(props.host); const tokenId = useId(); - const savedLiteral = - typeof props.host.accessToken === "string" && props.host.accessToken.length > 0; - const [tokenDraft, setTokenDraft] = useState(() => secretFieldDraft(props.host.accessToken)); - const [cleared, setCleared] = useState(false); - const tokenSubmission = secretFieldSubmission(tokenDraft, props.secretProviders); - const referenceIssue = Result.isFailure(tokenSubmission) ? tokenSubmission.failure : null; - // A saved token arrives redacted, so an empty literal keeps it until Clear drops it. - const keepsSavedToken = - savedLiteral && - !cleared && - Result.isSuccess(tokenSubmission) && - tokenSubmission.success === undefined; - const tokenCleared = - !keepsSavedToken && tokenDraft.kind === "literal" && tokenDraft.text.trim() === ""; + const secretSources = useSecretSources(props.environmentId); + // A saved token arrives redacted; only a replacement or a clear changes it. + const [token, setToken] = useState< + { kind: "saved" } | { kind: "new"; value: SettingsSecretValue } | { kind: "cleared" } + >(props.host.accessToken ? { kind: "saved" } : { kind: "cleared" }); // Checking on open names the saved project and shows a rejected credential // before anything is changed. A manual check or a token edit replaces it. const openCheck = useAtomValue( @@ -102,19 +87,16 @@ export function EasDeviceHostEditor(props: { const next: EasDeviceHostConfig = { ...draft, label: draft.label.trim(), - accessToken: keepsSavedToken - ? props.host.accessToken - : (Result.getOrUndefined(tokenSubmission) ?? ""), + accessToken: + token.kind === "saved" ? props.host.accessToken : token.kind === "new" ? token.value : "", }; - const valid = referenceIssue === null && Option.isSome(parseEasDeviceHostDraft(next)); + const valid = Option.isSome(parseEasDeviceHostDraft(next)); // Why Save is off, for the limits the fields' ranges do not show. const problem = valid ? null - : referenceIssue !== null - ? `Expo access token: ${referenceIssue}` - : draft.maxSessionMinutes > 0 && draft.idleMinutes >= draft.maxSessionMinutes - ? "Idle minutes must be smaller than session minutes." - : "Give the host a label and limits within the ranges shown."; + : draft.maxSessionMinutes > 0 && draft.idleMinutes >= draft.maxSessionMinutes + ? "Idle minutes must be smaller than session minutes." + : "Give the host a label and limits within the ranges shown."; const apps = account.status === "signed-in" ? account.account.apps : []; // A check answers for the credential it was asked about: a later check or a @@ -128,13 +110,16 @@ export function EasDeviceHostEditor(props: { const run = ++latestCheck.current; const result = await check({ environmentId: props.environmentId, - // A cleared token is sent as empty, so the check does not fall back to - // the saved one that Save is about to drop. - input: !keepsSavedToken - ? { accessToken: next.accessToken } - : props.isNew - ? {} - : { hostId: props.host.id }, + input: { + ...(token.kind === "saved" && !props.isNew ? { hostId: props.host.id } : {}), + // A cleared token is sent as empty, so the check does not fall back to + // the saved one that Save is about to drop. + ...(token.kind === "new" + ? { accessToken: token.value } + : token.kind === "cleared" + ? { accessToken: "" } + : {}), + }, }); if (run !== latestCheck.current) return; setAccount( @@ -209,32 +194,49 @@ export function EasDeviceHostEditor(props: { -
    - + { + setToken(secret ? { kind: "new", value: secret } : { kind: "cleared" }); + resetAccount(); + }} disabled={props.busy} placeholder="Optional" - value={props.host.accessToken} - configured={keepsSavedToken} - draft={tokenDraft} - onDraftChange={(next) => { - setTokenDraft(next); + onChange={(event) => { + const value = event.target.value; + // Emptying the field is not clearing the saved token; the + // Clear button is, so a saved token comes back here. + setToken( + value + ? { kind: "new", value } + : props.host.accessToken + ? { kind: "saved" } + : { kind: "cleared" }, + ); resetAccount(); }} /> - {!tokenCleared ? ( + {token.kind !== "cleared" ? ( - {isSecret ? ( - <> + + + ); + const sensitiveToggle = + typeof variable.value === "string" ? ( + + { + const sensitive = !variable.sensitive; + updateVariable(variable.id, { + sensitive, + ...(sensitive && variable.valueRedacted === undefined + ? {} + : { valueRedacted: sensitive ? variable.valueRedacted : false }), + }); + }} + aria-pressed={variable.sensitive} + aria-label={`Mark environment variable ${variable.name || index + 1} as sensitive`} + > + + + } + /> + + {variable.sensitive ? "Sensitive, stored separately" : "Plain text"} + + + ) : null; + const layout = (source: ReactNode, value: ReactNode) => ( +
    +
    + {nameInput} + {source === null ? null :
    {source}
    } + {removeButton} +
    +
    + {value} + {sensitiveToggle} +
    +
    + ); + return ( + + {variable.sensitive ? ( + updateVariable(variable.id, { value })} + placeholder="value" + configured={variable.valueRedacted === true} + spellCheck={false} + aria-label={`Environment variable value ${index + 1}`} + sources={props.secretSources ?? []} + environmentId={props.environmentId} + onSecretChange={(secret) => setVariableSecret(variable.id, secret)} + > + {(parts) => layout(parts.source, parts.value)} + + ) : ( + layout( + null, - updateVariable(variable.id, { reference: reference.trim() }) + className="min-w-0 flex-1" + value={ + variable.valueRedacted || typeof variable.value !== "string" + ? "" + : variable.value } - placeholder={referencePlaceholder} + onCommit={(value) => updateVariable(variable.id, { value })} + autoComplete="off" + placeholder="value" spellCheck={false} - aria-invalid={sourceIssue !== undefined || undefined} - aria-label={`Environment variable secret reference ${index + 1}`} - /> - {isLegacyOnePassword ? ( - updateVariable(variable.id, { account })} - placeholder="my.1password.com" - spellCheck={false} - aria-invalid={sourceIssue !== undefined || undefined} - aria-label={`Environment variable 1Password account ${index + 1}`} - /> - ) : null} - {canCheckSecrets && props.environmentId !== undefined ? ( - { - if (draftSource !== undefined && Result.isSuccess(draftSource)) { - void secretChecks.run(variable.id, draftSource.success); - } - }} - /> - ) : null} - - ) : ( - <> - {variable.sensitive ? ( - updateVariable(variable.id, { value })} - placeholder="value" - configured={variable.valueRedacted === true} - spellCheck={false} - aria-label={`Environment variable value ${index + 1}`} - /> - ) : ( - updateVariable(variable.id, { value })} - autoComplete="off" - placeholder="value" - spellCheck={false} - aria-label={`Environment variable value ${index + 1}`} - /> - )} - - { - const sensitive = !variable.sensitive; - updateVariable(variable.id, { - sensitive, - ...(sensitive && variable.valueRedacted === undefined - ? {} - : { - valueRedacted: sensitive ? variable.valueRedacted : false, - }), - }); - }} - aria-pressed={variable.sensitive} - aria-label={`Mark environment variable ${variable.name || index + 1} as sensitive`} - > - - - } - /> - - {variable.sensitive ? "Sensitive, stored separately" : "Plain text"} - - - + aria-label={`Environment variable value ${index + 1}`} + />, + ) )} - {sourceIssue !== undefined ? ( -

    {sourceIssue}

    - ) : null} -
    + ); })}

    - Sensitive values are stored separately and never returned to the app. Values from a - secret provider are read on the server each time the provider starts. + Sensitive values are stored separately and never returned to the app.

    - {hasSecretProviders ? null : ( -

    - Add a secret provider in{" "} - - Settings → Secrets - {" "} - to read values from 1Password, OpenBao, or AWS Secrets Manager. -

    - )}
    ) : null} @@ -858,6 +657,7 @@ interface ProviderInstanceCardProps { readonly isUpdating?: boolean | undefined; readonly onAcceptUrlAuth?: ((action: AcpRegistryUrlAuthAction) => void) | undefined; readonly environmentId?: EnvironmentId | undefined; + readonly secretSources?: ReadonlyArray; readonly projectOverride?: ProjectOverrideControl | undefined; readonly acpProjects?: | ReadonlyArray<{ @@ -866,10 +666,10 @@ interface ProviderInstanceCardProps { readonly workspaceRoot: string; }> | undefined; - readonly secretProviders?: SecretProviders | undefined; } const EMPTY_ACP_PROJECTS: NonNullable = []; +const NO_SECRET_SOURCES: ReadonlyArray = []; const NO_CUSTOM_MODELS: ReadonlyArray = []; /** Unknown drivers stay read-only to preserve fork config. projectOverride switches affect only the @@ -902,8 +702,8 @@ export function ProviderInstanceCard({ onAcceptUrlAuth, environmentId, acpProjects = EMPTY_ACP_PROJECTS, + secretSources = NO_SECRET_SOURCES, projectOverride, - secretProviders = EMPTY_SECRET_PROVIDERS, }: ProviderInstanceCardProps) { const enabled = projectOverride?.effectiveEnabled ?? resolveProviderInstanceEnabled(instance); const effectiveProvider = liveProvider && { ...liveProvider, enabled }; @@ -1042,26 +842,27 @@ export function ProviderInstanceCard({ }; const environmentFields = driverOption?.environmentFields ?? []; const environmentFieldNames = new Set(environmentFields.map((field) => field.name)); - const { dedicated: dedicatedEnvironment, generic: genericEnvironment } = - splitDedicatedProviderEnvironment(instance.environment, environmentFieldNames); + const genericEnvironment = providerEnvironmentWithoutNames( + instance.environment, + environmentFieldNames, + ); const updateGenericEnvironment = ( environment: ReadonlyArray, ) => { - const genericNames = new Set(environment.map((variable) => variable.name)); - updateEnvironment([ - ...providerEnvironmentWithoutNames(dedicatedEnvironment, genericNames), - ...environment, - ]); - }; - const updateEnvironmentFieldSource = ( - field: ProviderEnvironmentField, - source: ProviderSecretSource, - ) => { - updateEnvironment(nextProviderEnvironmentWithFieldSource(instance.environment, field, source)); + const dedicatedEnvironment = (instance.environment ?? []).filter((variable) => + environmentFieldNames.has(variable.name), + ); + updateEnvironment([...dedicatedEnvironment, ...environment]); }; const updateEnvironmentField = (field: ProviderEnvironmentField, value: string) => { updateEnvironment(nextProviderEnvironmentWithFieldValue(instance.environment, field, value)); }; + const updateEnvironmentFieldSecret = ( + field: ProviderEnvironmentField, + secret: SecretReference | undefined, + ) => { + updateEnvironment(nextProviderEnvironmentWithFieldSecret(instance.environment, field, secret)); + }; const removeEnvironmentField = (field: ProviderEnvironmentField) => { updateEnvironment(providerEnvironmentWithoutNames(instance.environment, new Set([field.name]))); }; @@ -1462,11 +1263,11 @@ export function ProviderInstanceCard({ field={field} variable={readProviderEnvironmentVariable(instance.environment, field.name)} idPrefix={`provider-instance-${instanceId}`} - environmentId={environmentId} - secretProviders={secretProviders} onCommit={updateEnvironmentField} - onCommitSource={updateEnvironmentFieldSource} + onSecretChange={updateEnvironmentFieldSecret} onRemove={removeEnvironmentField} + secretSources={secretSources} + environmentId={environmentId} /> ))}
    @@ -1507,10 +1308,10 @@ export function ProviderInstanceCard({ className={readOnly ? "opacity-50 select-none" : undefined} > {environmentId !== undefined && liveProvider?.driver === "acpRegistry" ? ( diff --git a/apps/web/src/components/settings/ProviderSettingsForm.tsx b/apps/web/src/components/settings/ProviderSettingsForm.tsx index 9ff95b3ef6284..1a1dc3363de9c 100644 --- a/apps/web/src/components/settings/ProviderSettingsForm.tsx +++ b/apps/web/src/components/settings/ProviderSettingsForm.tsx @@ -2,15 +2,14 @@ import { useState, type ReactNode } from "react"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import { - isSecretProviderSecretSource, + isSecretReference, isSecretSetting, type EnvironmentId, type ProviderSettingsFormAnnotation, type ProviderSettingsFormControl, type ProviderSettingsFormOption, type ProviderSettingsFormSchemaAnnotation, - type SecretProviderSecretSource, - type SettingsSecretValue, + type SecretReference, } from "@t3tools/contracts"; import { PlusIcon, XIcon } from "lucide-react"; @@ -19,7 +18,7 @@ import { Button } from "../ui/button"; import { DraftInput } from "../ui/draft-input"; import { Input } from "../ui/input"; import { SecretInput } from "./SecretInput"; -import { hasSavedCredential, SecretField, type SecretProviders } from "./SecretSourceControls"; +import { SecretSourceInput, useSecretSources } from "./SecretSourceInput"; import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; import { Switch } from "../ui/switch"; import { Textarea } from "../ui/textarea"; @@ -286,7 +285,7 @@ function readProviderConfigBoolean(config: unknown, key: string, defaultValue = export function nextProviderConfigWithFieldValue( config: unknown, field: ProviderSettingsFieldModel, - value: string | boolean | SecretProviderSecretSource, + value: string | boolean | SecretReference, ): Record | undefined { const base: Record = config !== null && typeof config === "object" ? { ...(config as Record) } : {}; @@ -321,13 +320,8 @@ interface ProviderSettingsFormProps { readonly idPrefix: string; readonly variant: "card" | "dialog" | "settings"; readonly onChange: (nextConfig: Record | undefined) => void; - /** Lets secret fields in the settings variant read from a configured secret provider. */ - readonly secretSources?: SecretSourceOptions | undefined; -} - -interface SecretSourceOptions { - readonly environmentId: EnvironmentId; - readonly secretProviders: SecretProviders; + /** Lets secret fields in the settings variant read from a secret provider on this environment. */ + readonly environmentId?: EnvironmentId | undefined; } function SecretSettingsControl({ @@ -335,42 +329,46 @@ function SecretSettingsControl({ value, inputId, descriptionId, - secretSources, + environmentId, onChange, }: { readonly field: ProviderSettingsFieldModel; readonly value: unknown; readonly inputId: string; readonly descriptionId: string | undefined; - readonly secretSources: SecretSourceOptions; + readonly environmentId: EnvironmentId; readonly onChange: ProviderSettingsFormProps["onChange"]; }) { + const sources = useSecretSources(environmentId); const saved = value !== null && typeof value === "object" ? (value as Record)[field.key] : undefined; - const stored = - typeof saved === "string" || isSecretProviderSecretSource(saved) ? saved : undefined; - const commit = (next: SettingsSecretValue) => - onChange(nextProviderConfigWithFieldValue(value, field, next)); + const reference = isSecretReference(saved) ? saved : undefined; + const configured = reference !== undefined || (typeof saved === "string" && saved.length > 0); return (
    - onChange(nextProviderConfigWithFieldValue(value, field, next))} + onSecretChange={(secret) => + onChange(nextProviderConfigWithFieldValue(value, field, secret ?? "")) + } /> - {hasSavedCredential(saved) ? ( + {configured ? (
    + {draft.kind === "one_password" ? ( +
    + + item.account} + mode="none" + openOnInputClick + value={draft.account} + onValueChange={(account) => update({ account })} + > + 0} + aria-invalid={accountError !== undefined} + aria-describedby={ + accountError !== undefined ? "secret-provider-account-error" : undefined + } + /> + {listedAccounts.length > 0 ? ( + + + {listedAccounts.map((item) => ( + + + {item.account} + + + {item.email} + + + ))} + + + ) : null} + +

    - Variables pick this provider by name, so it cannot change. + Accounts the op CLI is signed into are suggested. Type any other shorthand, + sign-in address without https://, or ID.

    - ) : null} -
    - {kindFields()} +
+ ) : null} + {draft.kind === "openbao" ? ( + <> +
+ + update({ address: event.target.value })} + spellCheck={false} + aria-invalid={addressError !== undefined} + aria-describedby={ + addressError !== undefined ? "secret-provider-address-error" : undefined + } + /> + +
+
+ + update({ mount: event.target.value })} + spellCheck={false} + aria-invalid={mountError !== undefined} + aria-describedby={ + mountError !== undefined ? "secret-provider-mount-error" : undefined + } + /> + +
+
+ + update({ namespace: event.target.value })} + spellCheck={false} + aria-invalid={namespaceError !== undefined} + aria-describedby={ + namespaceError !== undefined ? "secret-provider-namespace-error" : undefined + } + /> + +
+
+ + +

{authOption?.description}

+
+ {draft.auth === "token" ? ( +
+ + 0} + onChange={(event) => update({ token: event.target.value })} + aria-invalid={tokenError !== undefined} + aria-describedby={ + tokenError !== undefined ? "secret-provider-token-error" : undefined + } + /> + +
+ ) : null} + + ) : null} + {draft.kind === "aws_secrets_manager" ? ( + <> +
+ + update({ region: event.target.value })} + spellCheck={false} + aria-invalid={regionError !== undefined} + aria-describedby={ + regionError !== undefined ? "secret-provider-region-error" : undefined + } + /> + +
+
+ + update({ profile: event.target.value })} + spellCheck={false} + aria-invalid={profileError !== undefined} + aria-describedby={ + profileError !== undefined ? "secret-provider-profile-error" : undefined + } + /> + +

+ Credentials come from the aws CLI configuration on the server: this profile, or + its default chain when empty. +

+
+ + ) : null} - - diff --git a/apps/web/src/components/settings/SecretProviderKindIcon.tsx b/apps/web/src/components/settings/SecretProviderKindIcon.tsx index 79d27b8acef14..3c45e7f24c465 100644 --- a/apps/web/src/components/settings/SecretProviderKindIcon.tsx +++ b/apps/web/src/components/settings/SecretProviderKindIcon.tsx @@ -1,18 +1,36 @@ -import type { FlatSecretProviderKind } from "@t3tools/contracts"; -import type { SVGProps } from "react"; +import { KeyRoundIcon } from "lucide-react"; +import type { ComponentType } from "react"; import { AwsIcon, OnePasswordIcon, OpenBaoIcon } from "../Icons"; +type KindIcon = ComponentType<{ readonly className?: string }>; + +const ICONS: Readonly> = { + one_password: OnePasswordIcon, + openbao: OpenBaoIcon, + aws_secrets_manager: AwsIcon, +}; + +/** Unknown kinds keep a generic key so a row never renders without one. */ +export function secretProviderKindIcon(kind: string): KindIcon { + return Object.hasOwn(ICONS, kind) ? ICONS[kind]! : KeyRoundIcon; +} + export function SecretProviderKindIcon({ kind, - ...props -}: SVGProps & { readonly kind: FlatSecretProviderKind }) { + className, +}: { + readonly kind: string; + readonly className?: string; +}) { switch (kind) { - case "1password": - return ; + case "one_password": + return ; case "openbao": - return ; - case "aws-secrets-manager": - return ; + return ; + case "aws_secrets_manager": + return ; + default: + return ; } } diff --git a/apps/web/src/components/settings/SecretSourceControls.tsx b/apps/web/src/components/settings/SecretSourceControls.tsx deleted file mode 100644 index e491e0ed62fcf..0000000000000 --- a/apps/web/src/components/settings/SecretSourceControls.tsx +++ /dev/null @@ -1,578 +0,0 @@ -import { useAtomValue } from "@effect/atom-react"; -import { - OnePasswordAccount, - OnePasswordSecretReference, - SecretProviderId, - SecretProviderReference, - isSecretProviderSecretSource, - type EnvironmentId, - type ProviderSecretSource, - type FlatSecretProviderKind, - type SecretProviderSecretSource, - type SettingsSecretValue, - type StoredSecretProviderConfig, - type UnifiedSettings, -} from "@t3tools/contracts"; -import * as Result from "effect/Result"; -import * as Schema from "effect/Schema"; -import { useNavigate } from "@tanstack/react-router"; -import { - AlertTriangleIcon, - CheckIcon, - KeyRoundIcon, - PlusIcon, - RefreshCwIcon, - TypeIcon, -} from "lucide-react"; -import { useState } from "react"; - -import { cn } from "../../lib/utils"; -import { serverEnvironment } from "../../state/server"; -import { useAtomCommand } from "../../state/use-atom-command"; -import { Button } from "../ui/button"; -import { DraftInput } from "../ui/draft-input"; -import { Input } from "../ui/input"; -import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; -import { Spinner } from "../ui/spinner"; -import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; -import { - SECRET_PROVIDER_REFERENCE_PLACEHOLDERS, - secretProviderReferenceIssue, - secretProviderSummary, - secretProviderDisplayKind, -} from "./SecretProviderDialog.logic"; -import { SecretInput } from "./SecretInput"; -import { SecretProviderKindIcon } from "./SecretProviderKindIcon"; - -type EnvironmentDraftSource = "plain" | "1password" | "secret-provider"; - -export type SecretProviders = UnifiedSettings["secretProviders"]; - -const PLAIN_SOURCE_OPTION = "plain"; -export const EMPTY_SECRET_PROVIDERS: SecretProviders = {}; -const SECRET_PROVIDER_SOURCE_OPTION_PREFIX = "secret-provider:"; -const ADD_SECRET_PROVIDER_OPTION = "add-secret-provider"; -/** Rows saved before named providers keep their own account; new rows cannot pick it. */ -const ONE_PASSWORD_SOURCE_OPTION_PREFIX = "1password:"; - -function environmentDraftSourceOption( - row: Pick, -): string { - switch (row.source) { - case "plain": - return PLAIN_SOURCE_OPTION; - case "secret-provider": - return `${SECRET_PROVIDER_SOURCE_OPTION_PREFIX}${row.provider}`; - case "1password": - return `${ONE_PASSWORD_SOURCE_OPTION_PREFIX}${row.account}`; - } -} - -type EnvironmentSourceOption = { - readonly value: string; - readonly label: string; - readonly detail?: string; - readonly secret: boolean; - readonly kind?: FlatSecretProviderKind; -}; - -function lookupSecretProvider( - secretProviders: SecretProviders, - id: string, -): StoredSecretProviderConfig | undefined { - return Object.hasOwn(secretProviders, id) - ? secretProviders[SecretProviderId.make(id)] - : undefined; -} - -/** - * Value, then every configured provider. A row also keeps the option it is - * already on: a provider that was removed, or its legacy 1Password account. - */ -function environmentSourceOptions( - row: Pick, - secretProviders: SecretProviders, -): ReadonlyArray { - const options: EnvironmentSourceOption[] = [ - { value: PLAIN_SOURCE_OPTION, label: "Value", secret: false }, - ...Object.entries(secretProviders).map(([id, config]) => { - const kind = secretProviderDisplayKind(config); - return { - value: `${SECRET_PROVIDER_SOURCE_OPTION_PREFIX}${id}`, - label: config.label, - detail: secretProviderSummary(config), - secret: true, - ...(kind === undefined ? {} : { kind }), - }; - }), - ]; - if ( - row.source === "secret-provider" && - lookupSecretProvider(secretProviders, row.provider) === undefined - ) { - options.push({ - value: environmentDraftSourceOption(row), - label: row.provider, - detail: "Not configured in Settings → Secrets", - secret: true, - }); - } - if (row.source === "1password") { - options.push({ - value: environmentDraftSourceOption(row), - label: row.account.length > 0 ? row.account : "1Password", - detail: "1Password account set on this variable", - secret: true, - kind: "1password", - }); - } - return options; -} - -function EnvironmentSourceOptionLabel(props: { readonly option: EnvironmentSourceOption }) { - const Icon = props.option.secret ? KeyRoundIcon : TypeIcon; - return ( - - {props.option.kind !== undefined ? ( - - ) : ( - - )} - {props.option.label} - - ); -} - -export function EnvironmentSourceSelect(props: { - readonly row: Pick; - readonly secretProviders: SecretProviders; - readonly className: string; - readonly ariaLabel: string; - readonly onChange: (patch: Partial>) => void; -}) { - const navigate = useNavigate(); - const sourceOptions = environmentSourceOptions(props.row, props.secretProviders); - const selectedOption = environmentDraftSourceOption(props.row); - const selectedSourceOption = - sourceOptions.find((option) => option.value === selectedOption) ?? sourceOptions[0]!; - return ( - - ); -} - -export function referencePlaceholderFor( - row: Pick, - secretProviders: SecretProviders, -) { - return row.source === "1password" - ? SECRET_PROVIDER_REFERENCE_PLACEHOLDERS["1password"] - : SECRET_PROVIDER_REFERENCE_PLACEHOLDERS[ - secretProviderDisplayKind(lookupSecretProvider(secretProviders, row.provider)) ?? - "1password" - ]; -} - -export type EnvironmentDraftRow = { - readonly id: string; - readonly name: string; - readonly source: EnvironmentDraftSource; - readonly value: string; - readonly reference: string; - readonly account: string; - readonly provider: string; - readonly sensitive: boolean; - readonly valueRedacted?: boolean; - /** The name this row is saved under, so an unfinished edit can fall back to it. */ - readonly savedName?: string; -}; - -const decodeOnePasswordSecretReference = Schema.decodeUnknownResult(OnePasswordSecretReference); -const decodeOnePasswordAccount = Schema.decodeUnknownResult(OnePasswordAccount); -const decodeSecretProviderReference = Schema.decodeUnknownResult(SecretProviderReference); - -/** - * The secret source a draft row describes, or the message explaining why it - * is not one yet. A row may name a provider that is not configured: it saves, - * and resolves once a provider with that name is added again. - */ -export function secretSourceFromDraft( - row: Pick, - secretProviders: SecretProviders, -): Result.Result { - if (row.source === "1password") { - const reference = decodeOnePasswordSecretReference(row.reference); - if (Result.isFailure(reference)) return Result.fail(reference.failure.message); - const account = decodeOnePasswordAccount(row.account); - if (Result.isFailure(account)) return Result.fail(account.failure.message); - return Result.succeed({ - kind: "1password", - reference: reference.success, - account: account.success, - }); - } - return secretProviderSourceFromDraft(row, secretProviders); -} - -function secretProviderSourceFromDraft( - row: Pick, - secretProviders: SecretProviders, -): Result.Result { - if (row.provider.length === 0) return Result.fail("Choose a secret provider."); - const reference = decodeSecretProviderReference(row.reference); - if (Result.isFailure(reference)) return Result.fail(reference.failure.message); - const config = lookupSecretProvider(secretProviders, row.provider); - const kind = secretProviderDisplayKind(config); - const issue = - kind === undefined ? undefined : secretProviderReferenceIssue(kind, reference.success); - if (issue !== undefined) return Result.fail(issue); - return Result.succeed({ - kind: "secret-provider", - provider: SecretProviderId.make(row.provider), - reference: reference.success, - }); -} - -type SecretCheckState = - | { readonly status: "checking" } - | { readonly status: "resolved" } - | { readonly status: "failed"; readonly detail: string }; - -interface RowSecretCheck { - readonly sourceKey: string; - readonly state: SecretCheckState; -} - -export function useSecretSourceChecks(environmentId: EnvironmentId | undefined) { - const canCheck = useAtomValue( - serverEnvironment.checkSecretSource.permissionAtom(environmentId ?? null), - ); - const checkSecretSource = useAtomCommand(serverEnvironment.checkSecretSource); - const [checks, setChecks] = useState>>({}); - - const run = async (rowId: string, source: ProviderSecretSource) => { - if (environmentId === undefined) return; - const sourceKey = JSON.stringify(source); - setChecks((previous) => ({ - ...previous, - [rowId]: { sourceKey, state: { status: "checking" } }, - })); - const response = await checkSecretSource({ environmentId, input: { source } }); - const state: SecretCheckState = - response._tag !== "Success" - ? { status: "failed", detail: "Could not reach the server" } - : response.value._tag === "Resolved" - ? { status: "resolved" } - : { status: "failed", detail: response.value.detail }; - setChecks((previous) => - previous[rowId]?.sourceKey === sourceKey - ? { ...previous, [rowId]: { sourceKey, state } } - : previous, - ); - }; - - return { canCheck, checks, run }; -} - -export function SecretCheckButton(props: { - readonly label: string; - readonly state: SecretCheckState | undefined; - readonly disabledReason: string | undefined; - readonly onCheck: () => void; -}) { - const { state, disabledReason } = props; - const checking = state?.status === "checking"; - const description = - disabledReason ?? - (state === undefined - ? "Check that the secret resolves" - : state.status === "checking" - ? "Checking the secret" - : state.status === "resolved" - ? "Secret resolved" - : state.detail); - return ( - - - - - } - /> - {description} - - ); -} - -/** A secret field being edited: the literal typed so far, or a provider and its reference. */ -export type SecretFieldDraft = - | { readonly kind: "literal"; readonly text: string } - | { readonly kind: "reference"; readonly provider: string; readonly reference: string }; - -export const EMPTY_SECRET_FIELD_DRAFT: SecretFieldDraft = { kind: "literal", text: "" }; - -/** A saved literal arrives redacted, so only a saved reference is shown back for editing. */ -export function secretFieldDraft(value: unknown): SecretFieldDraft { - return isSecretProviderSecretSource(value) - ? { kind: "reference", provider: value.provider, reference: value.reference } - : EMPTY_SECRET_FIELD_DRAFT; -} - -/** - * What saving the draft writes, or why it cannot be saved yet. An empty literal is - * `undefined`: nothing was typed, and the caller decides whether that keeps or clears. - */ -export function secretFieldSubmission( - draft: SecretFieldDraft, - secretProviders: SecretProviders, -): Result.Result { - if (draft.kind === "literal") { - const text = draft.text.trim(); - return Result.succeed(text.length > 0 ? text : undefined); - } - return secretProviderSourceFromDraft( - { provider: draft.provider, reference: draft.reference.trim() }, - secretProviders, - ); -} - -/** A saved credential setting holds a literal or names a secret. */ -export function hasSavedCredential(value: unknown): boolean { - return isSecretProviderSecretSource(value) || (typeof value === "string" && value.length > 0); -} - -type SecretFieldProps = { - readonly id: string; - readonly label: string; - readonly environmentId: EnvironmentId | undefined; - readonly secretProviders: SecretProviders; - /** What is saved; a saved literal is usually redacted. */ - readonly value: SettingsSecretValue | undefined; - /** A literal is saved and redacted. Defaults to any non-empty saved literal. */ - readonly configured?: boolean | undefined; - /** Mask the literal. A plain literal is shown back as saved. */ - readonly sensitive?: boolean | undefined; - readonly placeholder?: string | undefined; - readonly disabled?: boolean | undefined; - readonly describedBy?: string | undefined; - readonly className?: string | undefined; -} & ( - | { - /** Saves a finished literal or reference as soon as it is entered. */ - readonly onCommit: (next: SettingsSecretValue) => void; - readonly draft?: never; - readonly onDraftChange?: never; - } - | { - /** Holds the draft in the caller, for a form that saves it later. */ - readonly draft: SecretFieldDraft; - readonly onDraftChange: (draft: SecretFieldDraft) => void; - readonly onCommit?: never; - } -); - -/** - * A credential input that holds a literal or reads it from a secret provider. The - * source picker is always shown, so the field reads as secret-capable even before a - * provider exists, and a reference can be checked against the server. - */ -export function SecretField(props: SecretFieldProps) { - const { onCommit } = props; - const savedKey = JSON.stringify(props.value ?? ""); - const [pending, setPending] = useState<{ - readonly savedKey: string; - readonly draft: SecretFieldDraft; - } | null>(null); - const draft = - props.draft ?? (pending?.savedKey === savedKey ? pending.draft : secretFieldDraft(props.value)); - const setDraft = (next: SecretFieldDraft) => { - if (props.onDraftChange !== undefined) props.onDraftChange(next); - else setPending({ savedKey, draft: next }); - }; - const secretChecks = useSecretSourceChecks(props.environmentId); - const configured = - props.configured ?? (typeof props.value === "string" && props.value.length > 0); - const savedLiteral = !configured && typeof props.value === "string" ? props.value : ""; - const row = - draft.kind === "literal" - ? { source: "plain" as const, provider: "", account: "" } - : { source: "secret-provider" as const, provider: draft.provider, account: "" }; - const submission = secretFieldSubmission(draft, props.secretProviders); - const source = - Result.isSuccess(submission) && isSecretProviderSecretSource(submission.success) - ? submission.success - : undefined; - const issue = - Result.isFailure(submission) && draft.kind === "reference" && draft.reference.trim() !== "" - ? submission.failure - : undefined; - const sourceKey = source === undefined ? undefined : JSON.stringify(source); - const check = secretChecks.checks[props.id]; - const checkState = check !== undefined && check.sourceKey === sourceKey ? check.state : undefined; - - const commitReference = () => { - if (onCommit !== undefined && source !== undefined && sourceKey !== savedKey) onCommit(source); - }; - const changeSource = (patch: Partial>) => { - if (patch.source === "secret-provider" && patch.provider !== undefined) { - setDraft({ kind: "reference", provider: patch.provider, reference: "" }); - return; - } - setDraft(EMPTY_SECRET_FIELD_DRAFT); - }; - - return ( -
-
- - {draft.kind === "reference" ? ( - <> - setDraft({ ...draft, reference: event.target.value })} - onBlur={commitReference} - onKeyDown={(event) => { - if (event.key !== "Enter" || onCommit === undefined) return; - event.preventDefault(); - commitReference(); - }} - /> - {secretChecks.canCheck && props.environmentId !== undefined ? ( - { - if (source !== undefined) void secretChecks.run(props.id, source); - }} - /> - ) : null} - - ) : props.draft !== undefined ? ( - setDraft({ kind: "literal", text: event.target.value })} - /> - ) : props.sensitive === false ? ( - onCommit?.(next)} - /> - ) : ( - onCommit?.(next)} - /> - )} -
- {issue !== undefined ?

{issue}

: null} -
- ); -} diff --git a/apps/web/src/components/settings/SecretSourceInput.browser.test.tsx b/apps/web/src/components/settings/SecretSourceInput.browser.test.tsx new file mode 100644 index 0000000000000..ff2fe13e12e4f --- /dev/null +++ b/apps/web/src/components/settings/SecretSourceInput.browser.test.tsx @@ -0,0 +1,109 @@ +import { SecretProviderId, SecretProviderKind } from "@t3tools/contracts"; +import { describe, expect, it, vi } from "vite-plus/test"; +import { userEvent } from "vite-plus/test/browser"; + +import { render, type Screen } from "../../../test/render"; +import { SecretSourceInput } from "./SecretSourceInput"; + +const personal = SecretProviderId.make("personal"); +const work = SecretProviderId.make("work"); +const sources = [ + { id: personal, kind: SecretProviderKind.make("one_password"), label: "Personal" }, + { id: work, kind: SecretProviderKind.make("one_password"), label: "Work" }, +]; + +const reference = (screen: Screen) => screen.getByRole("textbox", { name: "API key reference" }); + +async function commitReference(screen: Screen, value: string) { + await userEvent.clear(reference(screen)); + await userEvent.type(reference(screen), value); + await userEvent.keyboard("{Enter}"); +} + +describe("SecretSourceInput", () => { + it("moves a reference to another provider only once one is committed for it", async () => { + const onSecretChange = vi.fn(); + await using screen = await render( + {}} + onSecretChange={onSecretChange} + />, + ); + + await userEvent.click(screen.getByRole("combobox", { name: "API key source" })); + await userEvent.click(screen.getByRole("option", { name: "Work" })); + expect(onSecretChange).not.toHaveBeenCalled(); + await expect.element(reference(screen)).toHaveValue(""); + + await commitReference(screen, "op://Shared/OpenAI/credential"); + expect(onSecretChange).toHaveBeenCalledExactlyOnceWith({ + provider: work, + reference: "op://Shared/OpenAI/credential", + }); + }); + + it("removes the reference when its field is cleared", async () => { + const onSecretChange = vi.fn(); + await using screen = await render( + {}} + onSecretChange={onSecretChange} + />, + ); + + await userEvent.clear(reference(screen)); + await userEvent.keyboard("{Enter}"); + expect(onSecretChange).toHaveBeenCalledExactlyOnceWith(undefined); + }); + + it("rejects a reference 1Password cannot read", async () => { + const onSecretChange = vi.fn(); + await using screen = await render( + {}} + onSecretChange={onSecretChange} + />, + ); + + await commitReference(screen, "Private/OpenAI/credential"); + expect(onSecretChange).not.toHaveBeenCalled(); + await expect.element(reference(screen)).toHaveAttribute("aria-invalid", "true"); + await expect.element(reference(screen)).toHaveValue("Private/OpenAI/credential"); + }); + + it("validates a reference against the kind of the provider it is for", async () => { + const onSecretChange = vi.fn(); + const vault = SecretProviderId.make("vault"); + await using screen = await render( + {}} + onSecretChange={onSecretChange} + />, + ); + + await commitReference(screen, "apps/openai"); + expect(onSecretChange).not.toHaveBeenCalled(); + await expect.element(reference(screen)).toHaveAttribute("aria-invalid", "true"); + + await commitReference(screen, "apps/anthropic#api_key"); + expect(onSecretChange).toHaveBeenCalledExactlyOnceWith({ + provider: vault, + reference: "apps/anthropic#api_key", + }); + }); +}); diff --git a/apps/web/src/components/settings/SecretSourceInput.tsx b/apps/web/src/components/settings/SecretSourceInput.tsx new file mode 100644 index 0000000000000..fbfd709cf1d4f --- /dev/null +++ b/apps/web/src/components/settings/SecretSourceInput.tsx @@ -0,0 +1,284 @@ +import { + SecretReference, + type EnvironmentId, + type SecretProviderConfig, + type SecretProviderId, + type SecretProviderKind, +} from "@t3tools/contracts"; +import { builtInSecretProviderDefinitions } from "@t3tools/secret-provider-definitions/registry"; +import { useAtomValue } from "@effect/atom-react"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { AlertTriangleIcon, CheckIcon, KeyRoundIcon, RefreshCwIcon, TypeIcon } from "lucide-react"; +import { useState, type ComponentProps, type ComponentType, type ReactNode } from "react"; + +import { isSecretsFeatureEnabled } from "../../env"; +import { useEnvironmentSettings } from "../../hooks/useSettings"; +import { serverEnvironment } from "../../state/server"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { Button } from "../ui/button"; +import { DraftInput } from "../ui/draft-input"; +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; +import { Spinner } from "../ui/spinner"; +import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; +import { SecretInput } from "./SecretInput"; +import { classifySecretProvider } from "./SecretProviderDialog.logic"; +import { secretProviderKindIcon } from "./SecretProviderKindIcon"; + +export interface SecretSource { + readonly id: SecretProviderId; + readonly kind: SecretProviderKind; + readonly label: string; +} + +const NO_SECRET_SOURCES: ReadonlyArray = []; +const STORED_VALUE = "__stored__"; +const decodeSecretReference = Schema.decodeUnknownOption(SecretReference); + +/** Secret providers a setting can point at; empty when the environment cannot read references. */ +export function useSecretSources(environmentId: EnvironmentId): ReadonlyArray { + const providers = useEnvironmentSettings(environmentId, (settings) => settings.secretProviders); + const supported = useAtomValue( + serverEnvironment.configValueAtom(environmentId), + (config) => config?.secretReferences === true, + ); + if (!isSecretsFeatureEnabled || !supported) return NO_SECRET_SOURCES; + return (Object.entries(providers) as Array<[SecretProviderId, SecretProviderConfig]>) + .filter(([, config]) => classifySecretProvider(config).kind !== "unsupported") + .map(([id, config]) => ({ id, kind: config.kind, label: config.label })); +} + +function SourceLabel(props: { + readonly icon: ComponentType<{ readonly className?: string }>; + readonly label: string; +}) { + return ( + + + {props.label} + + ); +} + +type SecretCheck = + | { readonly status: "checking" } + | { readonly status: "resolved" } + | { readonly status: "failed"; readonly detail: string }; + +/** Asks the server to resolve the committed reference, so a typo shows here and not at provider start. */ +function SecretCheckButton(props: { + readonly environmentId: EnvironmentId; + readonly secret: SecretReference; + readonly label: string; +}) { + const { environmentId, secret } = props; + const canCheck = useAtomValue(serverEnvironment.checkSecretSource.permissionAtom(environmentId)); + const checkSecretSource = useAtomCommand(serverEnvironment.checkSecretSource); + const secretKey = `${secret.provider}\u0000${secret.reference}`; + const [check, setCheck] = useState<{ + readonly secretKey: string; + readonly state: SecretCheck; + } | null>(null); + const state = check?.secretKey === secretKey ? check.state : undefined; + + const run = async () => { + setCheck({ secretKey, state: { status: "checking" } }); + const response = await checkSecretSource({ environmentId, input: { source: secret } }); + const next: SecretCheck = + response._tag !== "Success" + ? { status: "failed", detail: "Could not reach the server" } + : response.value._tag === "Resolved" + ? { status: "resolved" } + : { status: "failed", detail: response.value.detail }; + setCheck((previous) => + previous?.secretKey === secretKey ? { secretKey, state: next } : previous, + ); + }; + + const description = !canCheck + ? "You cannot check secrets on this environment" + : state === undefined + ? "Check that the secret resolves" + : state.status === "checking" + ? "Checking the secret" + : state.status === "resolved" + ? "Secret resolved" + : state.detail; + return ( + + + + + } + /> + {description} + + ); +} + +type SecretSourceInputProps = Omit, "value" | "children"> & { + readonly value: string | SecretReference; + /** Offers a check of the committed reference on this environment. */ + readonly environmentId?: EnvironmentId | undefined; + readonly sources: ReadonlyArray; + readonly onSecretChange: (secret: SecretReference | undefined) => void; + /** Places the parts in the caller's layout; `source` is null when there is nothing to pick. */ + readonly children?: (parts: { + readonly source: ReactNode; + readonly value: ReactNode; + }) => ReactNode; +}; + +/** A secret entered here, or read from a secret provider when one is configured. */ +export function SecretSourceInput({ + value, + environmentId, + sources, + onSecretChange, + children, + ...rest +}: SecretSourceInputProps) { + // A provider picked here applies once a reference for it is committed. + const [pendingProvider, setPendingProvider] = useState(null); + const [rejectedReference, setRejectedReference] = useState(null); + const secret = typeof value === "string" ? undefined : value; + const input = { ...rest, value: typeof value === "string" ? value : "" }; + const provider = pendingProvider ?? secret?.provider ?? null; + const reference = secret !== undefined && secret.provider === provider ? secret.reference : ""; + const providerSource = sources.find((source) => source.id === provider); + const definition = providerSource && builtInSecretProviderDefinitions.find(providerSource.kind); + const label = input["aria-label"] ?? "Secret"; + const providerLabel = (id: SecretProviderId) => + sources.find((source) => source.id === id)?.label ?? id; + const providerIcon = (id: SecretProviderId) => { + const kind = sources.find((source) => source.id === id)?.kind; + return kind === undefined ? KeyRoundIcon : secretProviderKindIcon(kind); + }; + + const source = + sources.length === 0 && provider === null ? null : ( + + ); + + const valueField = + provider === null ? ( + + ) : ( +
+
+ { + if (next.trim().length === 0) { + setRejectedReference(null); + if (pendingProvider === null && secret !== undefined) onSecretChange(undefined); + return; + } + const decoded = decodeSecretReference({ provider, reference: next }); + if ( + Option.isNone(decoded) || + definition?.isReference(decoded.value.reference) === false + ) { + setRejectedReference(next); + return; + } + setRejectedReference(null); + setPendingProvider(null); + onSecretChange(decoded.value); + }} + autoComplete="off" + placeholder={definition?.referencePlaceholder} + spellCheck={false} + aria-label={`${label} reference`} + aria-invalid={rejectedReference !== null} + /> + {environmentId !== undefined && secret !== undefined && pendingProvider === null ? ( + + ) : null} +
+ {rejectedReference !== null ? ( +

+ {definition?.invalidReferenceMessage ?? "Enter a valid secret reference."} +

+ ) : null} +
+ ); + + if (children !== undefined) return children({ source, value: valueField }); + if (source === null) return valueField; + return ( +
+
{source}
+ {valueField} +
+ ); +} diff --git a/apps/web/src/components/settings/SecretsSettings.tsx b/apps/web/src/components/settings/SecretsSettings.tsx index d869b82e02b14..2905904755c26 100644 --- a/apps/web/src/components/settings/SecretsSettings.tsx +++ b/apps/web/src/components/settings/SecretsSettings.tsx @@ -1,8 +1,4 @@ -import type { - FlatSecretProviderKind, - SecretProviderId, - StoredSecretProviderConfig, -} from "@t3tools/contracts"; +import type { SecretProviderConfig, SecretProviderId } from "@t3tools/contracts"; import { PlusIcon } from "lucide-react"; import { useState } from "react"; @@ -20,43 +16,17 @@ import { } from "../ui/alert-dialog"; import { Button } from "../ui/button"; import { SecretProviderDialog } from "./SecretProviderDialog"; -import { - classifySecretProvider, - secretProviderDisplayKind, - secretProviderSummary, -} from "./SecretProviderDialog.logic"; +import { classifySecretProvider, secretProviderDescription } from "./SecretProviderDialog.logic"; +import { secretProviderKindIcon } from "./SecretProviderKindIcon"; import { classifyProviderEnvironmentAccess, resolveRemoteOperateAccess, } from "./ProviderSettingsPanel.logic"; import { useSettingsScope } from "./SettingsScopeContext"; import { RedactedSensitiveText } from "./RedactedSensitiveText"; -import { SecretProviderKindIcon } from "./SecretProviderKindIcon"; import { searchableSetting } from "./settingsSearch"; import { SettingsPageContainer, SettingsRow, SettingsSection } from "./settingsLayout"; -function secretProviderDescription(provider: StoredSecretProviderConfig): string { - const classified = classifySecretProvider(provider); - if (classified.kind !== "flat") return secretProviderSummary(provider); - const config = classified.config; - switch (config.kind) { - case "1password": - return secretProviderSummary(config); - case "openbao": - return [ - secretProviderSummary(config), - `mount ${config.mount}`, - ...(config.namespace === undefined ? [] : [`namespace ${config.namespace}`]), - ...(config.auth === "env" ? ["BAO_TOKEN"] : config.auth === "cli" ? ["bao CLI login"] : []), - ].join(" · "); - case "aws-secrets-manager": - return [ - secretProviderSummary(config), - ...(config.profile === undefined ? [] : [`profile ${config.profile}`]), - ].join(" · "); - } -} - export function SecretsSettingsPanel() { const { scope, environment } = useSettingsScope(); const connected = environment?.connection.phase === "connected"; @@ -112,9 +82,7 @@ function EnvironmentSecretProviders({ const [dialog, setDialog] = useState< null | { readonly mode: "add" } | { readonly mode: "edit"; readonly id: SecretProviderId } >(null); - const entries = Object.entries(providers) as Array< - [SecretProviderId, StoredSecretProviderConfig] - >; + const entries = Object.entries(providers) as Array<[SecretProviderId, SecretProviderConfig]>; const editingConfig = dialog?.mode === "edit" ? providers[dialog.id] : undefined; const editing = dialog?.mode === "edit" && editingConfig !== undefined @@ -145,12 +113,21 @@ function EnvironmentSecretProviders({ /> ) : ( entries.map(([id, config]) => { - const kind = secretProviderDisplayKind(config); const classified = classifySecretProvider(config); + const KindIcon = + classified.kind === "unsupported" ? null : secretProviderKindIcon(config.kind); + const summary = secretProviderDescription(classified); return ( } + title={ + + {KindIcon ? ( + + ) : null} + {config.label} + + } description={ classified.kind === "one_password" ? ( @@ -163,16 +140,16 @@ function EnvironmentSecretProviders({ className="max-w-full truncate" /> - ) : classified.kind === "unsupported" ? ( - `Unsupported provider (${config.kind})` + ) : summary !== null ? ( + {summary} ) : ( - {secretProviderDescription(config)} + `Unsupported provider (${config.kind})` ) } control={ !readOnly ? (
- {kind !== undefined ? ( + {KindIcon ? (
) : (

- Permissions not checked. Connect to this environment to view this session’s permissions. + {sessionPermissionsUnavailableMessage({ + connected, + failed: session.hasError, + rejected: !session.hasError && !session.isPending && session.data?.authenticated === false, + })}

); } diff --git a/apps/web/src/components/settings/SourceControlHostSettings.tsx b/apps/web/src/components/settings/SourceControlHostSettings.tsx index 045e326f7f2f5..34c04c51104c4 100644 --- a/apps/web/src/components/settings/SourceControlHostSettings.tsx +++ b/apps/web/src/components/settings/SourceControlHostSettings.tsx @@ -26,10 +26,6 @@ export function SourceControlHostSettings({ const updateSettings = useAtomCommand(serverEnvironment.updateSettings, { label: `save ${definition.label} settings`, }); - const secretProviders = useEnvironmentSettings( - environmentId, - (settings) => settings.secretProviders, - ); const [saving, setSaving] = useState(false); const formDefinition = { settingsSchema: definition.settings }; const hasSaved = Object.values(saved ?? {}).some((value) => value !== ""); @@ -52,7 +48,7 @@ export function SourceControlHostSettings({ value={saved ?? {}} idPrefix={`source-control-${definition.kind}-${environmentId}`} variant="settings" - secretSources={{ environmentId, secretProviders }} + environmentId={environmentId} onChange={(next) => { const fields = next ?? {}; const changed = Object.fromEntries( diff --git a/apps/web/src/components/settings/SourceControlTokenSettings.tsx b/apps/web/src/components/settings/SourceControlTokenSettings.tsx index 8037424a92ccb..2ce1c75414b49 100644 --- a/apps/web/src/components/settings/SourceControlTokenSettings.tsx +++ b/apps/web/src/components/settings/SourceControlTokenSettings.tsx @@ -1,5 +1,9 @@ -import type { EnvironmentId, SettingsSecretValue } from "@t3tools/contracts"; -import * as Result from "effect/Result"; +import { + isSecretReference, + type EnvironmentId, + type SecretReference, + type SettingsSecretValue, +} from "@t3tools/contracts"; import { ExternalLinkIcon } from "lucide-react"; import { useState } from "react"; @@ -14,13 +18,10 @@ import { useAtomCommand } from "../../state/use-atom-command"; import { Button, InlineButton } from "../ui/button"; import { Input } from "../ui/input"; import { Label } from "../ui/label"; -import { - hasSavedCredential, - SecretField, - secretFieldDraft, - secretFieldSubmission, - type SecretFieldDraft, -} from "./SecretSourceControls"; +import { SecretSourceInput, useSecretSources } from "./SecretSourceInput"; + +const hasToken = (token: SettingsSecretValue | undefined) => + typeof token === "string" ? token.length > 0 : token !== undefined; export function SourceControlTokenSettings({ environmentId, @@ -36,29 +37,27 @@ export function SourceControlTokenSettings({ const { tokens } = useEnvironmentSettings(environmentId, (settings) => readHostCredentialSettings(settings, definition.kind), ); - const secretProviders = useEnvironmentSettings( - environmentId, - (settings) => settings.secretProviders, - ); const updateSettings = useAtomCommand(serverEnvironment.updateSettings, { label: `save ${definition.label} token`, }); const defaultHost = definition.publicHost ?? ""; const [host, setHost] = useState(defaultHost); + const secretSources = useSecretSources(environmentId); const [pending, setPending] = useState<{ readonly key: string; - readonly draft: SecretFieldDraft; + readonly draft: string | SecretReference; } | null>(null); const [saving, setSaving] = useState(false); const normalizedHost = host.trim().toLowerCase(); const savedToken = tokens[normalizedHost]; - const isSaved = hasSavedCredential(savedToken); + const isSaved = hasToken(savedToken); const savedHosts = Object.entries(tokens) - .filter(([, value]) => hasSavedCredential(value)) + .filter(([, value]) => hasToken(value)) .map(([saved]) => saved); const sourceKey = JSON.stringify([normalizedHost, savedToken ?? ""]); - const draft = pending?.key === sourceKey ? pending.draft : secretFieldDraft(savedToken); - const submission = Result.getOrUndefined(secretFieldSubmission(draft, secretProviders)); + const draft = + pending?.key === sourceKey ? pending.draft : isSecretReference(savedToken) ? savedToken : ""; + const submission = typeof draft === "string" ? draft.trim() : draft; const save = async (target: string, token: SettingsSecretValue) => { setSaving(true); @@ -83,7 +82,7 @@ export function SourceControlTokenSettings({ className="grid gap-3" onSubmit={(event) => { event.preventDefault(); - if (normalizedHost && submission !== undefined) void save(normalizedHost, submission); + if (normalizedHost && submission !== "") void save(normalizedHost, submission); }} > {/* Locked while saving: a successful save clears the draft, which would drop edits made mid-request. */} @@ -124,14 +123,16 @@ export function SourceControlTokenSettings({
- setPending({ key: sourceKey, draft: next })} + onChange={(event) => setPending({ key: sourceKey, draft: event.target.value })} + onSecretChange={(secret) => setPending({ key: sourceKey, draft: secret ?? "" })} />
@@ -155,7 +156,7 @@ export function SourceControlTokenSettings({ diff --git a/apps/web/src/components/settings/ThemeImportDialog.browser.test.tsx b/apps/web/src/components/settings/ThemeImportDialog.browser.test.tsx index 3a6478434c940..2ee0085bbb1c6 100644 --- a/apps/web/src/components/settings/ThemeImportDialog.browser.test.tsx +++ b/apps/web/src/components/settings/ThemeImportDialog.browser.test.tsx @@ -3,6 +3,14 @@ import { describe, expect, it, vi } from "vite-plus/test"; import { page, userEvent } from "vite-plus/test/browser"; import { render } from "../../../test/render"; +import { + CUSTOM_THEMES_STORAGE_KEY, + getCustomThemes, + installCustomTheme, + invalidateCustomThemes, + parseThemeFile, + THEME_FILE_VERSION, +} from "../../themePalette"; import type { OpenVsxThemeExtension } from "../../openVsxThemes"; import { ThemeImportDialog } from "./ThemeImportDialog"; @@ -54,3 +62,53 @@ describe("ThemeImportDialog", () => { ); }); }); + +const theme = (name: string, accent = "#3b82f6") => ({ + version: THEME_FILE_VERSION, + name, + appearance: "dark", + colors: { accent }, +}); + +const themeFile = (name: string, accent?: string) => + new File([JSON.stringify(theme(name, accent))], `${name}.json`, { type: "application/json" }); + +it.each(["Update existing", "Keep both"])( + "keeps batch failures visible after %s", + async (choice) => { + using _themes = { + [Symbol.dispose]() { + window.localStorage.removeItem(CUSTOM_THEMES_STORAGE_KEY); + invalidateCustomThemes(); + }, + }; + installCustomTheme(parseThemeFile(theme("Dusk"))); + const onOpenChange = vi.fn(); + await using _screen = await render( + true} + onImportedMany={() => {}} + />, + ); + const input = document.querySelector('input[type="file"]')!; + await userEvent.upload(input, [ + themeFile("Dawn"), + themeFile("Dusk", "#f97316"), + new File(["{"], "broken.json", { type: "application/json" }), + ]); + + await expect.element(page.getByText("Already installed")).toBeVisible(); + await expect.element(page.getByText(/^broken\.json:/)).toBeVisible(); + expect(getCustomThemes().map((theme) => theme.label)).toContain("Dawn"); + await page.getByRole("button", { name: choice }).click(); + invalidateCustomThemes(); + const label = choice === "Update existing" ? "Dusk" : "Dusk (1)"; + expect(getCustomThemes().find((theme) => theme.label === label)?.colors).toEqual( + parseThemeFile(theme("Dusk", "#f97316")).colors, + ); + await expect.element(page.getByText(/^broken\.json:/)).toBeVisible(); + expect(onOpenChange).not.toHaveBeenCalledWith(false); + }, +); diff --git a/apps/web/src/components/settings/ThemeImportDialog.tsx b/apps/web/src/components/settings/ThemeImportDialog.tsx index 8536498ba5a68..e9d37f45322d1 100644 --- a/apps/web/src/components/settings/ThemeImportDialog.tsx +++ b/apps/web/src/components/settings/ThemeImportDialog.tsx @@ -284,11 +284,10 @@ function ThemeImportBody({ } } if (installed.length > 0) onImportedMany(installed, { updated: false }); - if (failures.length > 0) { - setError(failures.join("; ")); - } else if (conflicting.length > 0) { + setError(failures.length > 0 ? failures.join("; ") : null); + if (conflicting.length > 0) { setConflicts(conflicting); - } else if (installed.length > 0) { + } else if (installed.length > 0 && failures.length === 0) { onClose(); } }; @@ -387,8 +386,8 @@ function ThemeImportBody({ } if (resolved.length > 0) onImportedMany(resolved, { updated: mode === "update" }); setConflicts(null); - if (failures.length > 0) setError(failures.join("; ")); - else onClose(); + if (failures.length > 0) setError([error, ...failures].filter(Boolean).join("; ")); + else if (!error) onClose(); }; const handleSubmit = () => { diff --git a/apps/web/src/components/settings/UsageProviderSettings.tsx b/apps/web/src/components/settings/UsageProviderSettings.tsx index bab7ddd9c52a6..f238850ae04d0 100644 --- a/apps/web/src/components/settings/UsageProviderSettings.tsx +++ b/apps/web/src/components/settings/UsageProviderSettings.tsx @@ -121,6 +121,7 @@ export function UsageProviderSettings({ onOpenChange={setAdding} environmentId={environmentId} environmentLabel={environmentLabel} + sources={sources} /> ) : null} diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 4f0a408156591..0f93cacbf6732 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -923,7 +923,7 @@ export const SETTINGS_SEARCH_ITEMS = [ title: "Open pull requests in T3 Code", to: "/settings/diffs", searchTerms: [ - `pr links ${SOURCE_CONTROL_HOST_LABELS} browser external click default pull request view panel`, + `pr commit links ${SOURCE_CONTROL_HOST_LABELS} browser external click default pull request view panel`, ], }, { diff --git a/apps/web/src/components/sidebar/SidebarChrome.tsx b/apps/web/src/components/sidebar/SidebarChrome.tsx index e233a4144d435..dd41ddd2a0213 100644 --- a/apps/web/src/components/sidebar/SidebarChrome.tsx +++ b/apps/web/src/components/sidebar/SidebarChrome.tsx @@ -1,9 +1,17 @@ -import { ArrowLeftIcon, ChartNoAxesColumnIcon, SettingsIcon } from "lucide-react"; +import { ArrowLeftIcon, ArrowRightIcon, ChartNoAxesColumnIcon, SettingsIcon } from "lucide-react"; import type { ReactNode } from "react"; import { memo } from "react"; import { Link, useLocation, useNavigate } from "@tanstack/react-router"; +import { useAtomValue } from "@effect/atom-react"; import { useEnvironmentIdentificationMode } from "../../hooks/useSettings"; +import { shortcutLabelForCommand } from "../../keybindings"; +import { + navigateHistory, + useCanNavigateHistory, + type HistoryDirection, +} from "../../navigationHistory"; +import { primaryServerKeybindingsAtom } from "../../state/server"; import { cn } from "../../lib/utils"; import { usePullRequestsSupported } from "../../state/environments"; import { T3Wordmark } from "../T3Wordmark"; @@ -14,6 +22,7 @@ import { useEnvironmentStageLabel, } from "../SidebarStageBackdrop"; import { Badge } from "../ui/badge"; +import { Button } from "../ui/button"; import { SidebarFooter, SidebarMenu, @@ -63,7 +72,15 @@ export const SidebarChromeHeader = memo(function SidebarChromeHeader({ />