fix: persist Daytona authentication failures

.changeset/persist-daytona-auth-errors.md

@@ -0,0 +1,6 @@
+---
+'@truefoundry/trueforge': patch
+'@truefoundry/trueforge-core': patch
+---
+
+Persist failed Daytona credentials when sandbox operations return authorization errors.

packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts

@@ -2,7 +2,7 @@ import type { Sandbox, Snapshot } from '@daytona/sdk';
 import { Daytona, DaytonaError } from '@daytona/sdk';
 import { context } from '@opentelemetry/api';
 import { suppressTracing } from '@opentelemetry/core';
-import { randomUUID } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { join } from 'node:path/posix';
 import type { Logger } from 'winston';
 import { extractErrorLogFields } from '../../util/errorLogFields';
@@ -99,6 +99,7 @@ export interface DaytonaSandboxProviderOptions {
   /** Defaults to 1 hour (same as the gateway's max agent execution time). */
   previewUrlExpirySeconds?: number;
   logger: Logger;
+  onError?: ((error: unknown) => Promise<void>) | undefined;
 }
 
 export class DaytonaSandboxProvider implements SandboxProvider {
@@ -118,6 +119,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {
   private readonly apiKey: string;
   private readonly apiUrl: string;
   private readonly logger: Logger;
+  private readonly onError: ((error: unknown) => Promise<void>) | undefined;
   private readonly daytona: Daytona;
   private static readonly cachedSandboxes = new Map<string, { sandbox: Sandbox; defaultTimeoutMs: number }>();
   // De-dupes concurrent recovery attempts on the same sandbox to a single refreshData+start round-trip.
@@ -138,12 +140,21 @@ export class DaytonaSandboxProvider implements SandboxProvider {
     this.natsBridgePort = options.natsBridgePort ?? DEFAULT_SANDBOX_NATS_WS_PORT;
     this.previewUrlExpirySeconds = options.previewUrlExpirySeconds ?? DEFAULT_PREVIEW_URL_EXPIRY_SECONDS;
     this.logger = options.logger.child({ module: 'DaytonaProvider' });
+    this.onError = options.onError;
+  }
+
+  private async reportError(error: unknown): Promise<void> {
+    try {
+      await this.onError?.(error);
+    } catch (reportError) {
+      this.logger.error('Failed to report Daytona error', extractErrorLogFields(reportError));
+    }
   }
 
   private async getOrCreateSandbox(sandboxId?: string): Promise<{ sandbox: Sandbox; defaultTimeoutMs: number }> {
     if (sandboxId) {
       validateSandboxOwnedByTenant({ sandboxId, tenantName: this.tenantName });
-      const cached = DaytonaSandboxProvider.cachedSandboxes.get(sandboxId);
+      const cached = DaytonaSandboxProvider.cachedSandboxes.get(this.sandboxCacheKey(sandboxId));
       if (cached) {
         return cached;
       }
@@ -160,18 +171,30 @@ export class DaytonaSandboxProvider implements SandboxProvider {
         });
 
     const entry = { sandbox, defaultTimeoutMs: this.timeoutMs };
-    DaytonaSandboxProvider.cachedSandboxes.set(sandbox.name, entry);
+    DaytonaSandboxProvider.cachedSandboxes.set(this.sandboxCacheKey(sandbox.name), entry);
     return entry;
   }
 
+  /**
+   * A Sandbox object carries the Daytona client that restored it.  Include the client
+   * identity in the process-wide cache so a settings update with rotated credentials
+   * cannot reuse an object authenticated with the previous key.  Hashing avoids keeping
+   * the raw API key as a Map key or exposing it through diagnostics.
+   */
+  private sandboxCacheKey(sandboxId: string): string {
+    return createHash('sha256')
+      .update(`${this.tenantName}\u0000${this.apiUrl}\u0000${this.apiKey}\u0000${sandboxId}`)
+      .digest('hex');
+  }
+
   // Returns true iff the caller should retry: either we restarted a stopped sandbox, or the cache entry is missing and the retry will rebuild it via the cold path.
-  private static recoverSandboxIfStopped(sandboxId: string): Promise<boolean> {
-    const existing = DaytonaSandboxProvider.inFlightRecoveries.get(sandboxId);
+  private static recoverSandboxIfStopped(cacheKey: string): Promise<boolean> {
+    const existing = DaytonaSandboxProvider.inFlightRecoveries.get(cacheKey);
     if (existing) {
       return existing;
     }
 
-    const cached = DaytonaSandboxProvider.cachedSandboxes.get(sandboxId);
+    const cached = DaytonaSandboxProvider.cachedSandboxes.get(cacheKey);
     // Cache may have been evicted by a concurrent error path; signal retry so getOrCreateSandbox rebuilds via restoreExistingSandbox.
     if (!cached) {
       return Promise.resolve(true);
@@ -186,10 +209,10 @@ export class DaytonaSandboxProvider implements SandboxProvider {
       await cached.sandbox.start();
       return true;
     })().finally(() => {
-      DaytonaSandboxProvider.inFlightRecoveries.delete(sandboxId);
+      DaytonaSandboxProvider.inFlightRecoveries.delete(cacheKey);
     });
 
-    DaytonaSandboxProvider.inFlightRecoveries.set(sandboxId, recovery);
+    DaytonaSandboxProvider.inFlightRecoveries.set(cacheKey, recovery);
     return recovery;
   }
 
@@ -204,7 +227,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {
 
       let recovered: boolean;
       try {
-        recovered = await DaytonaSandboxProvider.recoverSandboxIfStopped(sandboxId);
+        recovered = await DaytonaSandboxProvider.recoverSandboxIfStopped(this.sandboxCacheKey(sandboxId));
       } catch (recoveryError) {
         this.logger.error('Sandbox recovery failed', {
           ...extractErrorLogFields(recoveryError),
@@ -246,11 +269,16 @@ export class DaytonaSandboxProvider implements SandboxProvider {
   }
 
   async createSandbox(): Promise<{ sandboxId: string }> {
-    return context.with(suppressTracing(context.active()), async () => {
-      const { sandbox } = await this.getOrCreateSandbox();
-      this.logger.debug(`Sandbox created: name=${sandbox.name}`);
-      return { sandboxId: sandbox.name };
-    });
+    try {
+      return await context.with(suppressTracing(context.active()), async () => {
+        const { sandbox } = await this.getOrCreateSandbox();
+        this.logger.debug(`Sandbox created: name=${sandbox.name}`);
+        return { sandboxId: sandbox.name };
+      });
+    } catch (error) {
+      await this.reportError(error);
+      throw error;
+    }
   }
 
   /** Resolves undefined when no snapshot carries that name; auth/other failures throw. */
@@ -399,10 +427,11 @@ export class DaytonaSandboxProvider implements SandboxProvider {
           };
         });
       } catch (e: unknown) {
-        DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId);
+        DaytonaSandboxProvider.cachedSandboxes.delete(this.sandboxCacheKey(params.sandboxId));
         if (e instanceof SandboxNotAvailableError) {
           throw e;
         }
+        await this.reportError(e);
         this.logger.error('Sandbox execution error', extractErrorLogFields(e));
         const message = e instanceof Error ? e.message : 'Unknown error';
         return { success: false, error: message };
@@ -438,7 +467,8 @@ export class DaytonaSandboxProvider implements SandboxProvider {
         if (e instanceof DaytonaError && e.statusCode === SANDBOX_NOT_FOUND_STATUS) {
           throw new SandboxFileNotFoundError(params.path);
         }
-        DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId);
+        DaytonaSandboxProvider.cachedSandboxes.delete(this.sandboxCacheKey(params.sandboxId));
+        await this.reportError(e);
         throw e;
       }
     });
@@ -452,7 +482,8 @@ export class DaytonaSandboxProvider implements SandboxProvider {
           await sandbox.fs.uploadFile(params.content, params.remotePath);
         });
       } catch (e: unknown) {
-        DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId);
+        DaytonaSandboxProvider.cachedSandboxes.delete(this.sandboxCacheKey(params.sandboxId));
+        await this.reportError(e);
         throw e;
       }
     });
@@ -468,7 +499,8 @@ export class DaytonaSandboxProvider implements SandboxProvider {
           return signed.url;
         });
       } catch (e: unknown) {
-        DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId);
+        DaytonaSandboxProvider.cachedSandboxes.delete(this.sandboxCacheKey(params.sandboxId));
+        await this.reportError(e);
         this.logger.error('Failed to create signed preview URL', extractErrorLogFields(e));
         throw e;
       }

packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts

@@ -32,6 +32,27 @@ function makeProvider(): DaytonaSandboxProvider {
   });
 }
 
+function makeRuntimeProvider(
+  client: Daytona,
+  onError?: (error: unknown) => Promise<void>,
+  apiKey = 'dtn-test',
+): DaytonaSandboxProvider {
+  return new DaytonaSandboxProvider({
+    client,
+    apiKey,
+    apiUrl: API_URL,
+    tenantName: 'test-tenant',
+    sandboxImage: 'registry.example.com/sandbox:029ea5ff',
+    timeoutMs: 1000,
+    autoStopIntervalInMinutes: 5,
+    autoArchiveIntervalInMinutes: 60,
+    autoDeleteIntervalInMinutes: 7200,
+    fileMaxBytesForDownload: 1024,
+    logger: makeSilentLogger(),
+    onError,
+  });
+}
+
 function mockFetch({ status, body }: { status: number; body: unknown }): jest.SpiedFunction<typeof globalThis.fetch> {
   return jest
     .spyOn(globalThis, 'fetch')
@@ -81,25 +102,152 @@ describe('DaytonaSandboxProvider register-only snapshot create', () => {
 });
 
 describe('DaytonaSandboxProvider exec', () => {
+  it('reports sandbox creation errors before rethrowing them', async () => {
+    const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true });
+    jest.spyOn(client, 'create').mockRejectedValue(new DaytonaError('unauthorized', 401));
+    const onError = jest.fn().mockResolvedValue(undefined);
+    const provider = makeRuntimeProvider(client, onError);
+
+    await expect(provider.createSandbox()).rejects.toMatchObject({ statusCode: 401 });
+    expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 }));
+  });
+
+  it('reports provider errors before converting them to failed exec results', async () => {
+    const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true });
+    jest.spyOn(client, 'get').mockRejectedValue(new DaytonaError('unauthorized', 401));
+    const onError = jest.fn().mockResolvedValue(undefined);
+    const provider = makeRuntimeProvider(client, onError);
+
+    await expect(provider.exec({ sandboxId: 'test-tenant.expired', command: 'true' })).resolves.toMatchObject({
+      success: false,
+    });
+    expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 }));
+  });
+
   it('rethrows SandboxNotAvailableError when the sandbox is gone', async () => {
     const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true });
     jest.spyOn(client, 'get').mockRejectedValue(new DaytonaError('not found', NOT_FOUND_STATUS));
-    const provider = new DaytonaSandboxProvider({
-      client,
-      apiKey: 'dtn-test',
-      apiUrl: API_URL,
-      tenantName: 'test-tenant',
-      sandboxImage: 'registry.example.com/sandbox:029ea5ff',
-      timeoutMs: 1000,
-      autoStopIntervalInMinutes: 5,
-      autoArchiveIntervalInMinutes: 60,
-      autoDeleteIntervalInMinutes: 7200,
-      fileMaxBytesForDownload: 1024,
-      logger: makeSilentLogger(),
-    });
+    const provider = makeRuntimeProvider(client);
 
     await expect(provider.exec({ sandboxId: 'test-tenant.gone', command: 'true' })).rejects.toBeInstanceOf(
       SandboxNotAvailableError,
     );
   });
+
+  it('does not reuse a restored sandbox after the Daytona credentials rotate', async () => {
+    const sandboxId = 'test-tenant.rotated-credentials';
+    const oldClient = new Daytona({ apiKey: 'dtn-old', useDeprecatedPolling: true });
+    const newClient = new Daytona({ apiKey: 'dtn-new', useDeprecatedPolling: true });
+    const oldSandbox = {
+      state: 'started',
+      process: { executeCommand: jest.fn().mockResolvedValue({ exitCode: 0, result: 'old' }) },
+    };
+    const newSandbox = {
+      state: 'started',
+      process: { executeCommand: jest.fn().mockResolvedValue({ exitCode: 0, result: 'new' }) },
+    };
+    jest.spyOn(oldClient, 'get').mockResolvedValue(oldSandbox as never);
+    jest.spyOn(newClient, 'get').mockResolvedValue(newSandbox as never);
+
+    await expect(
+      makeRuntimeProvider(oldClient, undefined, 'dtn-old').exec({ sandboxId, command: 'true' }),
+    ).resolves.toMatchObject({
+      success: true,
+      response: { result: 'old' },
+    });
+    await expect(
+      makeRuntimeProvider(newClient, undefined, 'dtn-new').exec({ sandboxId, command: 'true' }),
+    ).resolves.toMatchObject({
+      success: true,
+      response: { result: 'new' },
+    });
+
+    expect(oldClient.get).toHaveBeenCalledWith(sandboxId);
+    expect(newClient.get).toHaveBeenCalledWith(sandboxId);
+  });
+
+  it.each([
+    [
+      'download',
+      (provider: DaytonaSandboxProvider, sandboxId: string) => provider.downloadFile({ sandboxId, path: '/tmp/file' }),
+    ],
+    [
+      'upload',
+      (provider: DaytonaSandboxProvider, sandboxId: string) =>
+        provider.uploadFile({ sandboxId, remotePath: '/tmp/file', content: Buffer.from('content') }),
+    ],
+    [
+      'preview',
+      (provider: DaytonaSandboxProvider, sandboxId: string) =>
+        (
+          provider as unknown as {
+            getPreviewUrl(params: { sandboxId: string; port: number; expiresInSeconds: number }): Promise<string>;
+          }
+        ).getPreviewUrl({
+          sandboxId,
+          port: 3000,
+          expiresInSeconds: 60,
+        }),
+    ],
+  ])('evicts the credential-scoped cache entry when %s fails', async (_operation, run) => {
+    const sandboxId = 'test-tenant.cached';
+    const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true });
+    const provider = makeRuntimeProvider(client);
+    const internals = DaytonaSandboxProvider as unknown as { cachedSandboxes: Map<string, unknown> };
+    const cacheKey = (provider as unknown as { sandboxCacheKey(id: string): string }).sandboxCacheKey(sandboxId);
+    const failingSandbox = {
+      fs: {
+        getFileDetails: jest.fn().mockResolvedValue({ size: 1, isDir: false }),
+        downloadFile: jest.fn().mockRejectedValue(new Error('download failed')),
+        uploadFile: jest.fn().mockRejectedValue(new Error('upload failed')),
+      },
+      getSignedPreviewUrl: jest.fn().mockRejectedValue(new Error('preview failed')),
+    };
+    internals.cachedSandboxes.set(cacheKey, { sandbox: failingSandbox, defaultTimeoutMs: 1000 });
+
+    await expect(run(provider, sandboxId)).rejects.toThrow();
+
+    expect(internals.cachedSandboxes.has(cacheKey)).toBe(false);
+  });
+
+  it.each([
+    [
+      'download',
+      async (provider: DaytonaSandboxProvider, sandboxId: string) =>
+        provider.downloadFile({ sandboxId, path: '/tmp/output' }),
+    ],
+    [
+      'upload',
+      async (provider: DaytonaSandboxProvider, sandboxId: string) =>
+        provider.uploadFile({ sandboxId, remotePath: '/tmp/output', content: Buffer.from('content') }),
+    ],
+    [
+      'preview',
+      async (provider: DaytonaSandboxProvider, sandboxId: string) =>
+        (
+          provider as unknown as {
+            getPreviewUrl(params: { sandboxId: string; port: number; expiresInSeconds: number }): Promise<string>;
+          }
+        ).getPreviewUrl({ sandboxId, port: 4222, expiresInSeconds: 60 }),
+    ],
+  ])('reports Daytona authentication failures from %s operations', async (_operation, invoke) => {
+    const sandboxId = 'test-tenant.auth-failure';
+    const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true });
+    const unauthorized = new DaytonaError('unauthorized', 401);
+    const sandbox = {
+      state: 'started',
+      fs: {
+        getFileDetails: jest.fn().mockResolvedValue({ size: 1, isDir: false }),
+        downloadFile: jest.fn().mockRejectedValue(unauthorized),
+        uploadFile: jest.fn().mockRejectedValue(unauthorized),
+      },
+      getSignedPreviewUrl: jest.fn().mockRejectedValue(unauthorized),
+    };
+    jest.spyOn(client, 'get').mockResolvedValue(sandbox as never);
+    const onError = jest.fn().mockResolvedValue(undefined);
+    const provider = makeRuntimeProvider(client, onError);
+
+    await expect(invoke(provider, sandboxId)).rejects.toMatchObject({ statusCode: 401 });
+    expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 }));
+  });
 });

packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts

@@ -88,17 +88,23 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore<Trans
     transaction?: Transaction<Database>,
   ): Promise<SandboxProviderRecord | undefined> {
     const db = transaction ?? this.#db;
-    const row = await db
+    const expectedManifest = input.expected_manifest;
+    let query = db
       .updateTable('sandbox_provider')
       .set({
         status: input.status,
         status_reason: input.status_reason,
         build_metadata: input.build_metadata !== null ? json(input.build_metadata) : null,
         updated_at: now(),
       })
-      .where('tenant_id', '=', input.tenant_id)
-      .returningAll()
-      .executeTakeFirst();
+      .where('tenant_id', '=', input.tenant_id);
+    if (expectedManifest !== undefined) {
+      query = query.where('manifest', '=', json(expectedManifest));
+    }
+    if (input.expected_status !== undefined) {
+      query = query.where('status', '=', input.expected_status);
+    }
+    const row = await query.returningAll().executeTakeFirst();
     return row === undefined ? undefined : toRecord(row);
   }
 }