From 28e70a5ad7446a1ff645dd94878c600e38479381 Mon Sep 17 00:00:00 2001 From: Juan del Peral Date: Mon, 18 May 2026 16:24:49 -0300 Subject: [PATCH] chore(infra,web): cap container logs and trim noisy server logs In production the host disk was being filled by container logs because Podman does not rotate logs by default and several server actions were logging a handful of lines per call on hot polling paths. - Add a default json-file logging block (10MB x 3 files) to every service rendered by compose.yml.j2 (traefik, postgres, docker-socket-proxy, reaper, flows, cloudflared, local-dev web). - Add --log-driver k8s-file --log-opt max-size=10m to the production web container started via `podman run` in Ansible. - Add HostConfig.LogConfig to every user workspace container created via dockerode in lib/spawner/docker.ts (this was the largest source on a busy host with many workspaces). - Remove the per-call console.log spam in ensureInstanceRunningAction (polled every 2s during workspace start) and in sendMessageAction. - Drop the verbose direct-IP healthcheck logs in spawner/core.ts. - De-duplicate the `[transformParts] Unknown part type` log so a new unknown OpenCode part type can no longer flood logs once per streamed message. --- apps/web/src/actions/opencode.ts | 25 +------------------ apps/web/src/actions/spawner.ts | 9 ------- apps/web/src/lib/opencode/transform.ts | 14 +++++++++-- .../src/lib/spawner/__tests__/docker.test.ts | 7 ++++++ apps/web/src/lib/spawner/core.ts | 8 ------ apps/web/src/lib/spawner/docker.ts | 10 ++++++++ infra/deploy/ansible/roles/app/tasks/main.yml | 2 ++ .../roles/app/templates/compose.yml.j2 | 13 ++++++++++ 8 files changed, 45 insertions(+), 43 deletions(-) diff --git a/apps/web/src/actions/opencode.ts b/apps/web/src/actions/opencode.ts index 74a0d967..e74ccd70 100644 --- a/apps/web/src/actions/opencode.ts +++ b/apps/web/src/actions/opencode.ts @@ -826,13 +826,6 @@ export async function sendMessageAction( message?: WorkspaceMessage; error?: string; }> { - console.log("[sendMessageAction] Called with:", { - slug, - sessionId, - text: text.substring(0, 50), - model, - }); - // Verify user is authorized const session = await getSession(); if (!session) { @@ -851,11 +844,6 @@ export async function sendMessageAction( !instance.serverPassword || instance.status !== "running" ) { - console.log("[sendMessageAction] Instance unavailable:", { - hasInstance: !!instance, - hasPassword: !!instance?.serverPassword, - status: instance?.status, - }); return { ok: false, error: "instance_unavailable" }; } @@ -865,11 +853,6 @@ export async function sendMessageAction( )}`; const baseUrl = getInstanceUrl(slug); - console.log( - "[sendMessageAction] Sending to:", - `${baseUrl}/session/${sessionId}/message` - ); - const body = { parts: [{ type: "text", text }], model: model @@ -912,14 +895,8 @@ export async function sendMessageAction( } } } - - console.log( - "[sendMessageAction] Extracted text:", - textContent.substring(0, 100) - ); } catch { - // If not valid JSON, maybe it's streaming format (NDJSON) - console.log("[sendMessageAction] JSON parse failed, trying NDJSON"); + // If not valid JSON, fall back to NDJSON parsing. const lines = responseText.split("\n"); for (const line of lines) { diff --git a/apps/web/src/actions/spawner.ts b/apps/web/src/actions/spawner.ts index a57500b1..b469dcb6 100644 --- a/apps/web/src/actions/spawner.ts +++ b/apps/web/src/actions/spawner.ts @@ -85,29 +85,22 @@ export async function ensureInstanceRunningAction(slug: string): Promise<{ status: 'running' | 'starting' | 'error' error?: string }> { - console.log('[ensureInstanceRunning] Starting for slug:', slug) - try { const session = await getSession() if (!session) { - console.log('[ensureInstanceRunning] No session - unauthorized') return { status: 'error', error: 'unauthorized' } } - console.log('[ensureInstanceRunning] Session user:', session.user.slug, 'role:', session.user.role) if (session.user.slug !== slug && session.user.role !== 'ADMIN') { - console.log('[ensureInstanceRunning] Forbidden - slug mismatch') return { status: 'error', error: 'forbidden' } } const kickstartStatus = await getKickstartStatus() if (kickstartStatus !== 'ready') { - console.log('[ensureInstanceRunning] Kickstart setup required') return { status: 'error', error: 'setup_required' } } const instance = await getWorkspaceStatus(slug) - console.log('[ensureInstanceRunning] Current instance status:', instance?.status ?? 'none') if (instance?.status === 'running') { const syncUserId = @@ -146,9 +139,7 @@ export async function ensureInstanceRunningAction(slug: string): Promise<{ return { status: 'starting' } } - console.log('[ensureInstanceRunning] Starting instance...') const result = await startWorkspace(slug, session.user.id) - console.log('[ensureInstanceRunning] Start result:', result) if (!result.ok) { return { status: 'error', error: result.detail ?? result.error } diff --git a/apps/web/src/lib/opencode/transform.ts b/apps/web/src/lib/opencode/transform.ts index ed940322..5d92f88f 100644 --- a/apps/web/src/lib/opencode/transform.ts +++ b/apps/web/src/lib/opencode/transform.ts @@ -7,6 +7,11 @@ import { WORKSPACE_ATTACHMENTS_DIR } from "@/lib/workspace-attachments"; */ const HIDDEN_PART_TYPES = new Set(["snapshot", "compaction"]); +// Track which unknown part types we have already logged so each new kind only +// fires once per process. Without this, a single unrecognized part type would +// log the full object on every streamed message and rapidly fill disk. +const loggedUnknownPartTypes = new Set(); + function resolveFilePartPath(sourcePath: string | undefined, fileUrl: string | undefined): string | undefined { if (sourcePath) { return sourcePath; @@ -238,8 +243,13 @@ export function transformParts(parts: unknown[]): MessagePart[] { } default: { - // Unknown type - preserve as fallback for debugging - console.log("[transformParts] Unknown part type:", partType, part); + // Unknown type - preserve as fallback for debugging. + // Log only once per kind to avoid flooding logs during streaming. + const kind = typeof partType === "string" ? partType : "non-string"; + if (!loggedUnknownPartTypes.has(kind)) { + loggedUnknownPartTypes.add(kind); + console.warn("[transformParts] Unknown part type:", kind); + } const normalizedData = normalizeSerializableValue(part); return { type: "unknown" as const, diff --git a/apps/web/src/lib/spawner/__tests__/docker.test.ts b/apps/web/src/lib/spawner/__tests__/docker.test.ts index b25da7f5..260a4c76 100644 --- a/apps/web/src/lib/spawner/__tests__/docker.test.ts +++ b/apps/web/src/lib/spawner/__tests__/docker.test.ts @@ -156,6 +156,13 @@ describe('docker', () => { '/opt/arche/kb-content:/kb-content', '/opt/arche/users/user-slug:/tmp/arche-user-data:ro', ], + LogConfig: { + Type: 'json-file', + Config: { + 'max-size': '10m', + 'max-file': '3', + }, + }, }, Labels: { 'arche.managed': 'true', diff --git a/apps/web/src/lib/spawner/core.ts b/apps/web/src/lib/spawner/core.ts index 4d53e89c..dbcda017 100644 --- a/apps/web/src/lib/spawner/core.ts +++ b/apps/web/src/lib/spawner/core.ts @@ -283,20 +283,12 @@ async function waitForHealthy(containerId: string, slug: string, password: strin if (directBaseUrl === undefined) { directBaseUrl = await getContainerHealthBaseUrl(containerId) - if (directBaseUrl) { - console.log('[spawner] Using direct container IP for initial healthcheck', { - baseUrl: directBaseUrl, - containerId, - slug, - }) - } } if (directBaseUrl && !directHealthy) { const directHealth = await isInstanceHealthyWithPassword(slug, password, directBaseUrl) if (directHealth.ok) { directHealthy = true - console.log('[spawner] OpenCode responded on direct container IP', { containerId, slug }) } else { lastHealth = directHealth } diff --git a/apps/web/src/lib/spawner/docker.ts b/apps/web/src/lib/spawner/docker.ts index 924972f7..41ff7a6e 100644 --- a/apps/web/src/lib/spawner/docker.ts +++ b/apps/web/src/lib/spawner/docker.ts @@ -213,6 +213,16 @@ export async function createContainer( NetworkMode: getOpencodeNetwork(), RestartPolicy: { Name: "on-failure", MaximumRetryCount: 5 }, Binds: binds, + // Cap per-workspace container logs to avoid filling host disk. + // Podman aliases json-file to k8s-file; max-size is honored, max-file + // is best-effort (some Podman versions truncate instead of rotating). + LogConfig: { + Type: "json-file", + Config: { + "max-size": "10m", + "max-file": "3", + }, + }, }, Labels: { "arche.managed": "true", diff --git a/infra/deploy/ansible/roles/app/tasks/main.yml b/infra/deploy/ansible/roles/app/tasks/main.yml index 4b5ae1f0..8c41703a 100644 --- a/infra/deploy/ansible/roles/app/tasks/main.yml +++ b/infra/deploy/ansible/roles/app/tasks/main.yml @@ -391,6 +391,8 @@ --label arche.role=web \ --label arche.version={{ web_version }} \ --restart on-failure:5 \ + --log-driver k8s-file \ + --log-opt max-size=10m \ {{ web_image }} changed_when: true no_log: true diff --git a/infra/deploy/ansible/roles/app/templates/compose.yml.j2 b/infra/deploy/ansible/roles/app/templates/compose.yml.j2 index 9bf8a560..88f0e75e 100644 --- a/infra/deploy/ansible/roles/app/templates/compose.yml.j2 +++ b/infra/deploy/ansible/roles/app/templates/compose.yml.j2 @@ -5,10 +5,17 @@ name: arche +x-logging: &default-logging + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + services: traefik: image: docker.io/traefik:v3.6.7 restart: "on-failure:5" + logging: *default-logging command: - --providers.docker=true - --providers.docker.exposedbydefault=false @@ -58,6 +65,7 @@ services: cloudflared: image: {{ cloudflared_image }} restart: "on-failure:5" + logging: *default-logging command: - tunnel - --no-autoupdate @@ -73,6 +81,7 @@ services: docker-socket-proxy: image: ghcr.io/tecnativa/docker-socket-proxy:latest restart: "on-failure:5" + logging: *default-logging {% if deploy_mode == 'local-dev' %} user: root security_opt: @@ -98,6 +107,7 @@ services: postgres: image: docker.io/postgres:16 restart: "on-failure:5" + logging: *default-logging environment: POSTGRES_DB: arche POSTGRES_USER: postgres @@ -142,6 +152,7 @@ services: reaper: image: {{ web_image }} restart: "on-failure:5" + logging: *default-logging command: - pnpm - run @@ -164,6 +175,7 @@ services: flows: image: {{ web_image }} restart: "unless-stopped" + logging: *default-logging command: - pnpm - run @@ -199,6 +211,7 @@ services: image: node:24.14.0-bookworm-slim working_dir: /app restart: "unless-stopped" + logging: *default-logging env_file: - {{ env_file_name | default('.env') }} ports: