diff --git a/.github/workflows/nightly-local-regression.yml b/.github/workflows/nightly-local-regression.yml index 2d617802c..45c7e47c4 100644 --- a/.github/workflows/nightly-local-regression.yml +++ b/.github/workflows/nightly-local-regression.yml @@ -242,6 +242,7 @@ jobs: import json import os import pathlib + import re import subprocess import sys @@ -262,7 +263,7 @@ jobs: '-f', 'backend/supabase/docker-compose.yml', '-f', 'backend/supabase/docker-compose.local.yml', '-f', 'backend/supabase/docker-compose.mail.yml', - 'create', '--no-start', + 'create', '--pull=never', 'analytics', 'auth', 'db', 'functions', 'imgproxy', 'kong', 'mail', 'meta', 'realtime', 'rest', 'storage', 'supavisor', 'vector', ] @@ -315,11 +316,198 @@ jobs: ) raise SystemExit(2) PY + - name: Probe Compose service startup + shell: bash + run: | + set -euo pipefail + render_receipt="$RUNNER_TEMP/local-stack-render-preflight.json" + python3 backend/supabase/scripts/local-stack.py render > "$render_receipt" + python3 - "$render_receipt" <<'PY' + import json + import os + import pathlib + import subprocess + import sys + + receipt = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding='utf-8')) + project = receipt.get('project_name') + if receipt.get('schema') != 'local-stack-receipt-v1' or receipt.get('action') != 'render' or receipt.get('ok') is not True: + raise SystemExit(2) + if not isinstance(project, str) or not project.startswith('tzudong-local-'): + raise SystemExit(2) + state = pathlib.Path('backend/supabase/volumes/.local-stack') / project + command_prefix = [ + 'docker', 'compose', + '--project-name', project, + '--env-file', str(state / 'stack.env'), + '-f', 'backend/supabase/docker-compose.yml', + '-f', 'backend/supabase/docker-compose.local.yml', + '-f', 'backend/supabase/docker-compose.mail.yml', + ] + services = ( + 'db', 'analytics', 'auth', 'functions', 'imgproxy', 'kong', 'mail', + 'meta', 'realtime', 'rest', 'storage', 'supavisor', 'vector', + ) + markers = ( + ('port_conflict', ('address already in use', 'port is already allocated')), + ('mount_invalid', ('invalid mount config', 'bind source path does not exist', 'failed to mount local volume')), + ('network_create', ('failed to create network', 'failed to set up container networking', 'network not found')), + ('runtime_create', ('failed to create shim task', 'oci runtime create failed', 'failed to start shim')), + ('exec_invalid', ('exec format error', 'executable file not found')), + ('permission_denied', ('permission denied', 'operation not permitted')), + ) + + def classify(stderr): + normalized = stderr.casefold() + for suffix, values in markers: + if any(value in normalized for value in values): + return suffix + return 'unknown' + + safe_env = { + key: os.environ[key] + for key in ('PATH', 'HOME', 'USER', 'TMPDIR', 'LANG', 'LC_ALL', 'TERM') + if os.environ.get(key) + } + safe_env.setdefault('PATH', '/usr/bin:/bin') + safe_env.setdefault('HOME', str(pathlib.Path.home())) + collective = {'status': 'failed', 'failure_class': 'unknown'} + try: + result = subprocess.run( + [ + *command_prefix, 'up', '-d', '--pull=never', + *services, + ], + capture_output=True, + text=True, + timeout=600, + check=False, + env=safe_env, + ) + if result.returncode == 0: + collective = {'status': 'passed', 'failure_class': 'none'} + else: + collective['failure_class'] = classify(result.stderr) + except FileNotFoundError: + collective['failure_class'] = 'docker_not_found' + except subprocess.TimeoutExpired: + collective['failure_class'] = 'timeout' + results = [] + failed = False + if collective['status'] != 'passed': + subprocess.run( + [*command_prefix, 'down', '--volumes', '--remove-orphans'], + capture_output=True, + text=True, + timeout=120, + check=False, + env=safe_env, + ) + try: + recreate = subprocess.run( + [*command_prefix, 'create', '--pull=never', *services], + capture_output=True, + text=True, + timeout=300, + check=False, + env=safe_env, + ) + if recreate.returncode != 0: + results.append({ + 'service': '__recreate__', + 'status': 'failed', + 'failure_class': classify(recreate.stderr), + }) + failed = True + except FileNotFoundError: + results.append({ + 'service': '__recreate__', + 'status': 'failed', + 'failure_class': 'docker_not_found', + }) + failed = True + except subprocess.TimeoutExpired: + results.append({ + 'service': '__recreate__', + 'status': 'failed', + 'failure_class': 'timeout', + }) + failed = True + for service in services if not failed else (): + record = {'service': service, 'status': 'failed', 'failure_class': 'unknown'} + try: + result = subprocess.run( + [*command_prefix, 'start', service], + capture_output=True, + text=True, + timeout=120, + check=False, + env=safe_env, + ) + if result.returncode == 0: + record = {'service': service, 'status': 'passed', 'failure_class': 'none'} + else: + record['failure_class'] = classify(result.stderr) + except FileNotFoundError: + record['failure_class'] = 'docker_not_found' + except subprocess.TimeoutExpired: + record['failure_class'] = 'timeout' + results.append(record) + if record['status'] != 'passed': + failed = True + break + report = { + 'schema': 'local-compose-start-preflight-v1', + 'project_name': project, + 'collective': collective, + 'services': results, + } + target = pathlib.Path('nightly-artifacts/failure-diagnostics/local-compose-start-preflight.json') + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + json.dumps(report, sort_keys=True, separators=(',', ':')) + '\n', + encoding='utf-8', + ) + subprocess.run( + [*command_prefix, 'down', '--volumes', '--remove-orphans'], + capture_output=True, + text=True, + timeout=120, + check=False, + env=safe_env, + ) + if failed: + print('Compose startup diagnostics recorded; continuing lifecycle acceptance.', file=sys.stderr) + PY - name: Reset disposable local stack + env: + LOCAL_STACK_PRESERVE_FAILURE_STATE: '1' shell: bash run: | set -euo pipefail mkdir -p nightly-artifacts + if [[ -n "${LOCAL_PROJECT:-}" ]]; then + for volume in \ + "${LOCAL_PROJECT}-db-data" \ + "${LOCAL_PROJECT}-db-config" \ + "${LOCAL_PROJECT}-db-init-migrations" \ + "${LOCAL_PROJECT}-db-init-scripts" \ + "${LOCAL_PROJECT}-functions" \ + "${LOCAL_PROJECT}-kong-config" \ + "${LOCAL_PROJECT}-pooler-config" \ + "${LOCAL_PROJECT}-storage-data" \ + "${LOCAL_PROJECT}-vector-config"; do + mapfile -t consumers < <(docker ps -aq --filter "volume=$volume") + if ((${#consumers[@]})); then + docker rm -f "${consumers[@]}" + fi + docker volume rm -f "$volume" || true + if docker volume inspect "$volume"; then + echo "disposable volume remained after scoped cleanup: $volume" >&2 + exit 2 + fi + done + fi python3 backend/supabase/scripts/local-stack.py reset \ > nightly-artifacts/local-stack-reset.json python3 - nightly-artifacts/local-stack-reset.json <<'PY' @@ -358,11 +546,9 @@ jobs: set -euo pipefail state="$LOCAL_STATE" bind=(--project "$LOCAL_PROJECT" --state-dir "$state" --env-file "$state/stack.env") - python3 backend/supabase/scripts/local-migrate.py generate-prerequisite \ - --input backend/supabase/baselines/pre-20260214-public-schema.sql \ - --output backend/supabase/baselines/local/application-prerequisites.sql \ - --manifest-output backend/supabase/baselines/local/APPLICATION_PREREQUISITES.v1.json \ - > "$state/local-prerequisite-generate.json" + python3 backend/supabase/scripts/local-migrate.py verify-prerequisite \ + --input backend/supabase/baselines/local/application-prerequisites.sql \ + > "$state/local-prerequisite-verify.json" python3 backend/supabase/scripts/local-migrate.py manifest \ --output "$state/local-migration-manifest.json" python3 backend/supabase/scripts/local-migrate.py verify \ @@ -406,6 +592,300 @@ jobs: --provenance-file "../../${LOCAL_STATE#"$GITHUB_WORKSPACE/"}/stack.env.provenance.json" \ > nightly-run.log + - name: Capture bounded Compose runtime diagnostics + if: always() + shell: bash + run: | + set +e + mkdir -p nightly-artifacts/failure-diagnostics + python3 - <<'PY' + import json + import os + import pathlib + import re + import subprocess + + project = os.environ.get('LOCAL_PROJECT', '') + records = [] + allowed_states = {'created', 'running', 'paused', 'restarting', 'removing', 'exited', 'dead'} + allowed_health = {'', 'starting', 'healthy', 'unhealthy', 'none'} + markers = ( + ('database_permission', ('permission denied for schema', 'permission denied for table', 'permission denied for relation', 'insufficient privilege')), + ('config_permission', ('permission denied: /etc', 'permission denied opening', 'eacces')), + ('auth_failure', ('password authentication failed', 'authentication failed')), + ('tini_runtime', ('pr_set_child_subreaper', 'subreaper', 'tini')), + ('thread_create', ('failed to create thread', 'pthread_create', 'clone3', 'resource temporarily unavailable')), + ('scheduler_permission', ('setpriority', 'sched_', 'scheduler')), + ('procfs_permission', ('/proc', 'cgroup', 'sysfs')), + ('socket_permission', ('setsockopt', 'bind', 'listen')), + ('signal_permission', ('prctl', 'signal')), + ('namespace_permission', ('user namespace', 'setns', 'namespace')), + ('operation_not_permitted', ('operation not permitted',)), + ('erl_runtime', ('cpu_sup', 'memsup', 'inet_gethost', 'failed to start epmd', 'could not start application')), + ('permission_denied', ('permission denied',)), + ('mount_invalid', ('invalid mount', 'no such file or directory')), + ('runtime_error', ('failed to start', 'failed to create', 'exec format error')), + ('database_connection', ('connection refused', 'econnrefused', 'failed to connect')), + ('database_missing', ('database "_supabase" does not exist', 'database "postgres" does not exist')), + ('boot_error', ('could not start', 'failed to boot', 'uncaught exception')), + ) + + def classify(value): + normalized = str(value or '').casefold() + for suffix, values in markers: + if any(marker in normalized for marker in values): + return suffix + return 'none' if not normalized else 'unknown' + + def signatures(value): + normalized = str(value or '').casefold() + return [ + suffix for suffix, values in markers + if any(marker in normalized for marker in values) + ][:4] + + if project.startswith('tzudong-local-'): + try: + ids = subprocess.check_output( + [ + 'docker', 'ps', '-aq', + '--filter', f'label=com.docker.compose.project={project}', + ], + text=True, + timeout=30, + ).splitlines() + except (OSError, subprocess.SubprocessError): + ids = [] + db_container_id = None + service_payloads = {} + for container_id in ids: + try: + raw = subprocess.check_output( + ['docker', 'inspect', container_id], + text=True, + timeout=30, + ) + payload = json.loads(raw)[0] + except (OSError, subprocess.SubprocessError, ValueError, IndexError, KeyError): + continue + labels = (payload.get('Config') or {}).get('Labels') or {} + service = labels.get('com.docker.compose.service') + if service not in { + 'analytics', 'auth', 'db', 'functions', 'imgproxy', 'kong', + 'mail', 'meta', 'realtime', 'rest', 'storage', 'studio', + 'supavisor', 'vector', + }: + continue + if service == 'db': + db_container_id = container_id + service_payloads[service] = payload + state = payload.get('State') or {} + health = state.get('Health') or {} + health_log = health.get('Log') or [] + try: + logs = subprocess.check_output( + ['docker', 'logs', '--tail', '200', container_id], + text=True, + stderr=subprocess.STDOUT, + timeout=30, + ) + except (OSError, subprocess.SubprocessError): + logs = '' + records.append({ + 'service': service, + 'state': state.get('Status') if state.get('Status') in allowed_states else 'unknown', + 'exit_code': state.get('ExitCode') if isinstance(state.get('ExitCode'), int) and 0 <= state.get('ExitCode') <= 255 else None, + 'state_error_class': classify(state.get('Error')), + 'oom_killed': state.get('OOMKilled') is True, + 'restart_count': payload.get('RestartCount') if isinstance(payload.get('RestartCount'), int) and payload.get('RestartCount') >= 0 else None, + 'log_failure_class': classify(logs), + 'log_signatures': signatures(logs), + 'health': health.get('Status') if health.get('Status') in allowed_health else 'unknown', + 'health_failing_streak': health.get('FailingStreak') if isinstance(health.get('FailingStreak'), int) and health.get('FailingStreak') >= 0 else None, + 'health_log_exit_codes': [ + item.get('ExitCode') + for item in health_log[-5:] + if isinstance(item, dict) and isinstance(item.get('ExitCode'), int) and 0 <= item.get('ExitCode') <= 255 + ], + 'entrypoint_class': ( + 'limits_wrapper' + if (payload.get('Config') or {}).get('Entrypoint') == ['/app/limits.sh'] + else 'image_default' + ), + }) + database_bootstrap = { + 'status': 'unavailable', + 'result': 'unknown', + 'failure_class': 'unknown', + 'exit_code': None, + } + database_presence = { + 'status': 'unavailable', + 'result': 'unknown', + 'exit_code': None, + } + database_init_files = { + 'status': 'unavailable', + 'supabase': 'unknown', + 'logs': 'unknown', + } + runtime_input_checks = {} + runtime_env_presence = {} + for service in ('supavisor',): + payload = service_payloads.get(service) + env_values = (payload.get('Config') or {}).get('Env') if isinstance(payload, dict) else None + env_names = { + item.split('=', 1)[0]: item.split('=', 1)[1] + for item in env_values + if isinstance(item, str) and '=' in item + } if isinstance(env_values, list) else {} + runtime_env_presence[service] = { + key: ( + 'absent' if key not in env_names + else 'empty' if env_names[key] == '' + else 'nonempty' + ) + for key in ('CLUSTER_POSTGRES', 'ERL_AFLAGS', 'ERL_FLAGS', 'RLIMIT_NOFILE', 'CLUSTER_NODES', 'DNS_POLL') + } + for service, destination, path in ( + ('kong', '/home/kong', '/home/kong/temp.yml'), + ('supavisor', '/etc/pooler', '/etc/pooler/pooler.exs'), + ): + payload = service_payloads.get(service) + mounts = payload.get('Mounts') if isinstance(payload, dict) else None + mount = next( + ( + item for item in mounts + if isinstance(item, dict) and item.get('Destination') == destination + ), + None, + ) if isinstance(mounts, list) else None + expected_name = f'{project}-{ "kong-config" if service == "kong" else "pooler-config" }' + runtime_input_checks[service] = { + 'mount': 'passed' if isinstance(mount, dict) and mount.get('Type') == 'volume' and mount.get('Name') == expected_name else 'failed', + 'file': 'unknown', + } + container_id = None + if isinstance(payload, dict): + container_id = payload.get('Id') + if isinstance(container_id, str) and re.fullmatch(r'[0-9a-f]{12,64}', container_id): + try: + probe = subprocess.run( + ['docker', 'exec', container_id, 'test', '-r', path], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + runtime_input_checks[service]['file'] = ( + 'passed' if probe.returncode == 0 + else 'missing' if probe.returncode == 1 + else 'unknown' + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + runtime_input_checks[service]['file'] = 'unknown' + if db_container_id is not None: + try: + presence = subprocess.run( + [ + 'docker', 'exec', db_container_id, + 'psql', '-U', 'postgres', '-d', 'postgres', + '-Atqc', + "select case when exists (select 1 from pg_database where datname = '_supabase') then 'supabase_db_present' else 'supabase_db_missing' end", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + database_presence = { + 'status': 'passed' if presence.returncode == 0 and presence.stdout.strip() in {'supabase_db_present', 'supabase_db_missing'} else 'failed', + 'result': presence.stdout.strip() if presence.stdout.strip() in {'supabase_db_present', 'supabase_db_missing'} else 'unknown', + 'exit_code': presence.returncode if 0 <= presence.returncode <= 255 else None, + } + init_paths = ( + ('supabase', '/docker-entrypoint-initdb.d/migrations/97-_supabase.sql'), + ('logs', '/docker-entrypoint-initdb.d/migrations/99-logs.sql'), + ) + init_results = {} + for key, path in init_paths: + init_probe = subprocess.run( + ['docker', 'exec', db_container_id, 'test', '-r', path], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + init_results[key] = ( + 'present' if init_probe.returncode == 0 + else 'missing' if init_probe.returncode == 1 + else 'unknown' + ) + database_init_files = { + 'status': 'passed' if all(value == 'present' for value in init_results.values()) else 'failed', + **init_results, + } + result = subprocess.run( + [ + 'docker', 'exec', db_container_id, + 'psql', '-U', 'postgres', '-d', '_supabase', + '-Atqc', + "select case when exists (select 1 from pg_namespace where nspname = '_analytics') then 'analytics_schema_present' else 'analytics_schema_missing' end", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + database_bootstrap = { + 'status': 'passed' if result.returncode == 0 and result.stdout.strip() == 'analytics_schema_present' else 'failed', + 'result': result.stdout.strip() if result.stdout.strip() in {'analytics_schema_present', 'analytics_schema_missing'} else 'unknown', + 'failure_class': classify(result.stderr), + 'exit_code': result.returncode if 0 <= result.returncode <= 255 else None, + } + except FileNotFoundError: + database_bootstrap['failure_class'] = 'docker_not_found' + database_presence['status'] = 'unavailable' + database_init_files['status'] = 'unavailable' + except subprocess.TimeoutExpired: + database_bootstrap['failure_class'] = 'timeout' + database_presence['status'] = 'unavailable' + database_init_files['status'] = 'unavailable' + else: + database_bootstrap = { + 'status': 'unavailable', + 'result': 'unknown', + 'failure_class': 'unknown', + 'exit_code': None, + } + database_presence = { + 'status': 'unavailable', + 'result': 'unknown', + 'exit_code': None, + } + database_init_files = { + 'status': 'unavailable', + 'supabase': 'unknown', + 'logs': 'unknown', + } + runtime_input_checks = {} + runtime_env_presence = {} + records.sort(key=lambda item: item['service']) + pathlib.Path('nightly-artifacts/failure-diagnostics/local-compose-runtime-diagnostics.json').write_text( + json.dumps({ + 'schema': 'local-compose-runtime-diagnostics-v1', + 'project_name': project if project.startswith('tzudong-local-') else 'unknown', + 'database_bootstrap': database_bootstrap, + 'database_presence': database_presence, + 'database_init_files': database_init_files, + 'runtime_input_checks': runtime_input_checks, + 'runtime_env_presence': runtime_env_presence, + 'containers': records, + }, sort_keys=True, separators=(',', ':')) + '\n', + encoding='utf-8', + ) + PY + - name: Capture sanitized local receipts if: always() shell: bash diff --git a/apps/web/lib/admin/storyboard/backend-agent.ts b/apps/web/lib/admin/storyboard/backend-agent.ts index 84e5abec4..a5437fff9 100644 --- a/apps/web/lib/admin/storyboard/backend-agent.ts +++ b/apps/web/lib/admin/storyboard/backend-agent.ts @@ -87,6 +87,7 @@ const WINDOWS_PROCESS_TERMINATION_TIMEOUT_MS = 5_000; const WINDOWS_JOB_SUPERVISOR_CLEANUP_GRACE_MS = 5_000; const WINDOWS_JOB_SUPERVISOR_FINAL_CLOSE_TIMEOUT_MS = 5_000; const LINUX_NAMESPACE_TERMINATION_TIMEOUT_MS = 10_000; +const LINUX_NAMESPACE_SUPERVISOR_DRAIN_TIMEOUT_MS = 7_000; const MAX_STORYBOARD_AGENT_TIMEOUT_MS = 600_000; function getRuntimeCwd() { const cwd = Reflect.get(process, "cwd"); @@ -4320,6 +4321,10 @@ function runStoryboardAgentCommand( processControl.platform === "linux" && isNativeProcessControl && !trustedLangGraphFixture; + const lifecycleStreamDrainTimeoutMs = + useLinuxNamespaceSupervisor && processControl.streamDrainTimeoutMs === undefined + ? Math.max(streamDrainTimeoutMs, LINUX_NAMESPACE_SUPERVISOR_DRAIN_TIMEOUT_MS) + : streamDrainTimeoutMs; const linuxSupervisorNonce = useLinuxNamespaceSupervisor ? randomBytes(32).toString("hex") : ""; @@ -4414,7 +4419,7 @@ function runStoryboardAgentCommand( ); stderr = stderrCapture.value; finish(); - }, streamDrainTimeoutMs); + }, lifecycleStreamDrainTimeoutMs); streamWaiters.add(finish); }); const terminateTree = async (awaitWindowsCleanupGrace = false) => { @@ -4647,7 +4652,7 @@ function runStoryboardAgentCommand( "diagnostic stream drain deadline exceeded", exitCode, ); - }, streamDrainTimeoutMs); + }, lifecycleStreamDrainTimeoutMs); } }; diff --git a/apps/web/scripts/run-nightly-regression.mjs b/apps/web/scripts/run-nightly-regression.mjs index 938f832a6..c356ee60b 100644 --- a/apps/web/scripts/run-nightly-regression.mjs +++ b/apps/web/scripts/run-nightly-regression.mjs @@ -1102,7 +1102,11 @@ async function waitForHealth(appProcess, healthUrl, mode, headers = undefined) { } async function runUnitRegression(environment) { - const result = await runCommand('bun', ['run', 'test:unit'], { env: environment }); + const supervisorExecutable = process.env.TZUDONG_NODE24_EXECUTABLE?.trim(); + const unitEnvironment = supervisorExecutable + ? { ...environment, TZUDONG_NODE24_EXECUTABLE: supervisorExecutable } + : environment; + const result = await runCommand('bun', ['run', 'test:unit'], { env: unitEnvironment }); if (result.code !== 0) { throw new Error(`Nightly unit regressions failed with exit code ${result.code}.`); } diff --git a/apps/web/tests-unit/admin-announcements-console-source.test.ts b/apps/web/tests-unit/admin-announcements-console-source.test.ts index eadb49c6c..cd3087b35 100644 --- a/apps/web/tests-unit/admin-announcements-console-source.test.ts +++ b/apps/web/tests-unit/admin-announcements-console-source.test.ts @@ -69,7 +69,9 @@ describe('admin announcements console integration source contract', () => { expect(headerSource).toContain('AnnouncementPanelLoadingFallback'); expect(headerSource).toContain('HeaderAnnouncementPanel ?'); expect(desktopControlPanelSource).toContain('AnnouncementPanelLoadingFallback'); - expect(desktopControlPanelSource).toContain('activeLeftPanelView === "announcement" ?'); + expect(desktopControlPanelSource).toContain( + 'activeLeftPanelView === "announcement" && !isPublicRestrictedMode ?', + ); expect(homeSidePanelsSource).toContain('loading: () => { expect(nightlyRunnerSource).toContain("function main()"); }); + test("passes the verified Node 24 supervisor into the unit lane", () => { + expect(nightlyRunnerSource).toContain( + "const supervisorExecutable = process.env.TZUDONG_NODE24_EXECUTABLE?.trim();", + ); + expect(nightlyRunnerSource).toContain( + "TZUDONG_NODE24_EXECUTABLE: supervisorExecutable", + ); + }); + test("preserves the hosted nightly schedule and bounded diagnostics", () => { expect(hostedWorkflowSource).toContain("cron: '30 18 * * *'"); expect(hostedWorkflowSource).toContain("workflow_dispatch:"); @@ -280,6 +289,7 @@ describe("nightly regression package and source contracts", () => { "user.max_user_namespaces=28633", "python3 backend/supabase/scripts/local-stack.py reset", "python3 backend/supabase/scripts/local-migrate.py apply-prerequisite", + "python3 backend/supabase/scripts/local-migrate.py verify-prerequisite", "python3 backend/supabase/scripts/local-migrate.py apply", "python3 backend/supabase/scripts/local-function-runtime-scan.py smoke", "python3 backend/supabase/scripts/local-migrate.py receipt", @@ -301,12 +311,62 @@ describe("nightly regression package and source contracts", () => { "Probe Compose container creation", "backend/supabase/scripts/local-stack.py render", "local-compose-create-preflight-v1", - "create', '--no-start'", + "create', '--pull=never'", "mount_invalid", "network_create", "runtime_create", + "Probe Compose service startup", + "local-compose-start-preflight-v1", + "collective", + "up', '-d', '--pull=never'", + "docker', 'compose", + "'start', service", + "exec_invalid", + "Capture bounded Compose runtime diagnostics", + "local-compose-runtime-diagnostics-v1", + "health_failing_streak", + "health_log_exit_codes", + "log_failure_class", + "oom_killed", + "restart_count", + "database_bootstrap", + "database_presence", + "database_init_files", + "database_permission", + "config_permission", + "auth_failure", + "tini_runtime", + "thread_create", + "scheduler_permission", + "procfs_permission", + "socket_permission", + "signal_permission", + "namespace_permission", + "operation_not_permitted", + "erl_runtime", + "runtime_input_checks", + "runtime_env_presence", + "RLIMIT_NOFILE", + "nonempty", + "empty", + "entrypoint_class", + "log_signatures", + "supabase_db_present", + "supabase_db_missing", + "analytics_schema_present", + "analytics_schema_missing", "stack.env and credentials excluded", "down --volumes --remove-orphans", + "docker volume rm -f", + "${LOCAL_PROJECT}-db-data", + "${LOCAL_PROJECT}-db-config", + "${LOCAL_PROJECT}-db-init-migrations", + "${LOCAL_PROJECT}-db-init-scripts", + "${LOCAL_PROJECT}-functions", + "${LOCAL_PROJECT}-kong-config", + "${LOCAL_PROJECT}-pooler-config", + "${LOCAL_PROJECT}-storage-data", + "${LOCAL_PROJECT}-vector-config", ]) { expect(localWorkflowSource).toContain(token); } diff --git a/backend/supabase/docker-compose.local.yml b/backend/supabase/docker-compose.local.yml index fe24bec57..e521d3e46 100644 --- a/backend/supabase/docker-compose.local.yml +++ b/backend/supabase/docker-compose.local.yml @@ -45,7 +45,7 @@ services: - "127.0.0.1:${KONG_HTTP_PORT}:8000/tcp" - "127.0.0.1:${KONG_HTTPS_PORT}:8443/tcp" volumes: !override - - "${LOCAL_INPUT_ROOT}/kong.yml:/home/kong/temp.yml:ro,z" + - "local-kong-config:/home/kong:Z" auth: container_name: !reset null @@ -74,7 +74,7 @@ services: functions: container_name: !reset null volumes: !override - - "${LOCAL_INPUT_ROOT}/functions:/home/deno/functions:ro,Z" + - "local-functions:/home/deno/functions:Z" analytics: container_name: !reset null @@ -84,34 +84,62 @@ services: db: container_name: !reset null volumes: !override - - "${LOCAL_INPUT_ROOT}/db-realtime.sql:/docker-entrypoint-initdb.d/migrations/99-realtime.sql:ro,z" - - "${LOCAL_INPUT_ROOT}/db-webhooks.sql:/docker-entrypoint-initdb.d/init-scripts/98-webhooks.sql:ro,z" - - "${LOCAL_INPUT_ROOT}/db-roles.sql:/docker-entrypoint-initdb.d/init-scripts/99-roles.sql:ro,z" - - "${LOCAL_INPUT_ROOT}/db-jwt.sql:/docker-entrypoint-initdb.d/init-scripts/99-jwt.sql:ro,z" + - "local-db-init-migrations:/docker-entrypoint-initdb.d/migrations:Z" + - "local-db-init-scripts:/docker-entrypoint-initdb.d/init-scripts:Z" - "local-db-data:/var/lib/postgresql/data:Z" - - "${LOCAL_INPUT_ROOT}/db-supabase.sql:/docker-entrypoint-initdb.d/migrations/97-_supabase.sql:ro,z" - - "${LOCAL_INPUT_ROOT}/db-logs.sql:/docker-entrypoint-initdb.d/migrations/99-logs.sql:ro,z" - - "${LOCAL_INPUT_ROOT}/db-pooler.sql:/docker-entrypoint-initdb.d/migrations/99-pooler.sql:ro,z" - "local-db-config:/etc/postgresql-custom" vector: container_name: !reset null volumes: !override - - "${LOCAL_INPUT_ROOT}/vector.yml:/etc/vector/vector.yml:ro,z" + - "local-vector-config:/etc/vector:Z" supavisor: container_name: !reset null + # GitHub-hosted Docker denies Tini's subreaper setup; the image's limits + # wrapper preserves bounded startup without requiring that capability. + entrypoint: ["/app/limits.sh"] + environment: !override + PORT: 4000 + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + DATABASE_URL: ecto://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase + SECRET_KEY_BASE: ${SECRET_KEY_BASE} + VAULT_ENC_KEY: ${VAULT_ENC_KEY} + API_JWT_SECRET: ${JWT_SECRET} + METRICS_JWT_SECRET: ${JWT_SECRET} + REGION: local + ERL_AFLAGS: "" + RLIMIT_NOFILE: "" + POOLER_TENANT_ID: ${POOLER_TENANT_ID} + POOLER_DEFAULT_POOL_SIZE: ${POOLER_DEFAULT_POOL_SIZE} + POOLER_MAX_CLIENT_CONN: ${POOLER_MAX_CLIENT_CONN} + POOLER_POOL_MODE: transaction + DB_POOL_SIZE: ${POOLER_DB_POOL_SIZE} ports: !override # POSTGRES_PORT remains the internal 5432 listener; only the host mapping is derived. - "127.0.0.1:${POSTGRES_HOST_PORT}:5432/tcp" - "127.0.0.1:${POOLER_PROXY_PORT_TRANSACTION}:6543/tcp" volumes: !override - - "${LOCAL_INPUT_ROOT}/pooler.exs:/etc/pooler/pooler.exs:ro,z" + - "local-pooler-config:/etc/pooler:Z" volumes: !override local-db-data: name: "${PROJECT_NAME}-db-data" + local-db-init-migrations: + name: "${PROJECT_NAME}-db-init-migrations" + local-db-init-scripts: + name: "${PROJECT_NAME}-db-init-scripts" local-db-config: name: "${PROJECT_NAME}-db-config" + local-functions: + name: "${PROJECT_NAME}-functions" + local-kong-config: + name: "${PROJECT_NAME}-kong-config" + local-pooler-config: + name: "${PROJECT_NAME}-pooler-config" local-storage-data: name: "${PROJECT_NAME}-storage-data" + local-vector-config: + name: "${PROJECT_NAME}-vector-config" diff --git a/backend/supabase/local-inputs/manifest.v1.json b/backend/supabase/local-inputs/manifest.v1.json index 989378e3a..37b7b1056 100644 --- a/backend/supabase/local-inputs/manifest.v1.json +++ b/backend/supabase/local-inputs/manifest.v1.json @@ -133,9 +133,9 @@ "mounts": [ { "service": "kong", - "source": "kong.yml", - "type": "bind", - "destination": "/home/kong/temp.yml" + "source": "local-kong-config", + "type": "volume", + "destination": "/home/kong" }, { "service": "storage", @@ -151,33 +151,15 @@ }, { "service": "functions", - "source": "functions", - "type": "bind", + "source": "local-functions", + "type": "volume", "destination": "/home/deno/functions" }, { "service": "db", - "source": "db-realtime.sql", - "type": "bind", - "destination": "/docker-entrypoint-initdb.d/migrations/99-realtime.sql" - }, - { - "service": "db", - "source": "db-webhooks.sql", - "type": "bind", - "destination": "/docker-entrypoint-initdb.d/init-scripts/98-webhooks.sql" - }, - { - "service": "db", - "source": "db-roles.sql", - "type": "bind", - "destination": "/docker-entrypoint-initdb.d/init-scripts/99-roles.sql" - }, - { - "service": "db", - "source": "db-jwt.sql", - "type": "bind", - "destination": "/docker-entrypoint-initdb.d/init-scripts/99-jwt.sql" + "source": "local-db-init-migrations", + "type": "volume", + "destination": "/docker-entrypoint-initdb.d/migrations" }, { "service": "db", @@ -187,21 +169,9 @@ }, { "service": "db", - "source": "db-supabase.sql", - "type": "bind", - "destination": "/docker-entrypoint-initdb.d/migrations/97-_supabase.sql" - }, - { - "service": "db", - "source": "db-logs.sql", - "type": "bind", - "destination": "/docker-entrypoint-initdb.d/migrations/99-logs.sql" - }, - { - "service": "db", - "source": "db-pooler.sql", - "type": "bind", - "destination": "/docker-entrypoint-initdb.d/migrations/99-pooler.sql" + "source": "local-db-init-scripts", + "type": "volume", + "destination": "/docker-entrypoint-initdb.d/init-scripts" }, { "service": "db", @@ -211,15 +181,15 @@ }, { "service": "vector", - "source": "vector.yml", - "type": "bind", - "destination": "/etc/vector/vector.yml" + "source": "local-vector-config", + "type": "volume", + "destination": "/etc/vector" }, { "service": "supavisor", - "source": "pooler.exs", - "type": "bind", - "destination": "/etc/pooler/pooler.exs" + "source": "local-pooler-config", + "type": "volume", + "destination": "/etc/pooler" } ], "compose_files": [ @@ -229,7 +199,7 @@ }, { "path": "backend/supabase/docker-compose.local.yml", - "sha256": "ba8ad34ed798b306bdfba45c9b0c6410f91ca74b44361afc29b8b531ee555086" + "sha256": "d46ddc6ff5bfb3a268d8e2a537c5c202d7e3d6f055fd67759912ad3e8d7c5c8a" }, { "path": "backend/supabase/docker-compose.mail.yml", diff --git a/backend/supabase/scripts/local-function-runtime-scan.py b/backend/supabase/scripts/local-function-runtime-scan.py index 2994850b1..8be277885 100644 --- a/backend/supabase/scripts/local-function-runtime-scan.py +++ b/backend/supabase/scripts/local-function-runtime-scan.py @@ -721,6 +721,17 @@ def _filtered_environment() -> dict[str, str]: result.setdefault("PATH", "/usr/bin:/bin") result.setdefault("HOME", str(Path.home())) return result + + +def _github_actions_root_owned_socket(path: Path, owner: int) -> bool: + return ( + path == Path("/var/run/docker.sock") + and owner == 0 + and os.environ.get("GITHUB_ACTIONS") == "true" + and os.environ.get("CI") == "true" + ) + + def _assert_local_docker_context(docker: str) -> None: environment = _filtered_environment() try: @@ -764,7 +775,13 @@ def _assert_local_docker_context(docker: str) -> None: info = socket_path.lstat() except OSError as error: raise RuntimeScanError("docker_context") from error - if stat.S_ISLNK(info.st_mode) or not stat.S_ISSOCK(info.st_mode) or info.st_uid != os.getuid(): + owned_by_current_user = info.st_uid == os.getuid() + owned_by_disposable_ci_root = _github_actions_root_owned_socket(socket_path, info.st_uid) + if ( + stat.S_ISLNK(info.st_mode) + or not stat.S_ISSOCK(info.st_mode) + or not (owned_by_current_user or owned_by_disposable_ci_root) + ): raise RuntimeScanError("docker_context") def _validate_endpoint_environment() -> None: if any(os.environ.get(key) for key in _REMOTE_DOCKER_ENV): @@ -911,7 +928,7 @@ def _validate_local_stack_state( return state, values, current, receipt -def _expected_database_mounts(root: Path, state: Path) -> set[tuple[str, str]]: +def _expected_database_mounts(root: Path, state: Path) -> set[tuple[str, str, str]]: input_root = state / "inputs" try: input_info = input_root.lstat() @@ -941,9 +958,28 @@ def _expected_database_mounts(root: Path, state: Path) -> set[tuple[str, str]]: or manifest.get("schema") != "local-stack-input-manifest-v1" or manifest.get("generator_version") != LOCAL_STACK_GENERATOR_VERSION or not isinstance(manifest.get("inputs"), list) + or not isinstance(manifest.get("mounts"), list) ): raise RuntimeScanError("local_input_manifest") - mounts: set[tuple[str, str]] = set() + for mount in manifest.get("mounts", []): + if not isinstance(mount, dict) or mount.get("service") != "db": + continue + source, destination = mount.get("source"), mount.get("destination") + if ( + mount.get("type") != "volume" + or not isinstance(source, str) + or not source.startswith("local-") + or not isinstance(destination, str) + ): + raise RuntimeScanError("local_input_manifest") + mounts: set[tuple[str, str, str]] = set() + project = local_project_name(root) + for mount in manifest["mounts"]: + if not isinstance(mount, dict) or mount.get("service") != "db": + continue + source = mount["source"] + destination = mount["destination"] + mounts.add(("volume", destination, f"{project}-{source.removeprefix('local-')}")) for entry in manifest["inputs"]: if not isinstance(entry, dict) or entry.get("service") != "db": continue @@ -969,7 +1005,6 @@ def _expected_database_mounts(root: Path, state: Path) -> set[tuple[str, str]]: or output_resolved.parent != (state / "inputs").resolve() ): raise RuntimeScanError("local_input_binding") - mounts.add((destination, str(output_resolved))) if not mounts: raise RuntimeScanError("local_input_manifest") return mounts @@ -1105,19 +1140,15 @@ def _validate(self) -> None: raise RuntimeScanError("container_not_repository_compose") if not isinstance(mounts, list): raise RuntimeScanError("container_input_mounts") - actual_mounts: set[tuple[str, str]] = set() + actual_mounts: set[tuple[str, str, str]] = set() for mount in mounts: if ( isinstance(mount, dict) - and mount.get("Type") == "bind" - and mount.get("RW") is False + and mount.get("Type") == "volume" and isinstance(mount.get("Destination"), str) - and isinstance(mount.get("Source"), str) + and isinstance(mount.get("Name"), str) ): - try: - actual_mounts.add((mount["Destination"], str(Path(mount["Source"]).resolve(strict=True)))) - except OSError as error: - raise RuntimeScanError("container_input_mounts") from error + actual_mounts.add(("volume", mount["Destination"], mount["Name"])) if not self._expected_mounts.issubset(actual_mounts): raise RuntimeScanError("container_input_mounts") @@ -1583,7 +1614,7 @@ def _sql_identifier(value: str) -> str: ("public", "prevent_last_admin_role_delete", ""): ("0A000",), ("public", "prevent_last_admin_role_update", ""): ("0A000",), ("public", "prevent_profile_role_client_change", ""): ("0A000",), - ("public", "preview_privacy_incident_transition", "uuid,uuid,public.privacy_incident_status,timestamptz,text,jsonb,uuid"): ("P0001",), + ("public", "preview_privacy_incident_transition", "uuid,uuid,public.privacy_incident_status,timestamptz,text,jsonb,uuid"): ("P0001", "42501"), ("public", "privacy_append_audit_event", "text,uuid,uuid,uuid,uuid,text,text,jsonb,jsonb"): ("42501",), ("public", "set_admin_ai_updated_at", ""): ("0A000",), ("public", "set_admin_restaurant_map_overlays_updated_at", ""): ("0A000",), @@ -1951,9 +1982,9 @@ def main(argv: Sequence[str] | None = None) -> int: client = LocalPsql(args.docker, args.container, args.database, args.timeout) candidates = _candidate_functions(_source_inventory()) runtime = client.query(_runtime_sql(candidates=candidates, smoke=True)) - _validate_runtime(runtime) smoke_status = runtime.get("rpcSmoke", {}).get("status") sys.stdout.buffer.write(canonical_json(runtime) + b"\n") + _validate_runtime(runtime) return 0 if smoke_status == "passed" else 2 if args.command == "apply": sql, metadata = _read_patch(Path(args.patch)) diff --git a/backend/supabase/scripts/local-migrate.py b/backend/supabase/scripts/local-migrate.py index 3ed2ebe48..8cc51d1ed 100644 --- a/backend/supabase/scripts/local-migrate.py +++ b/backend/supabase/scripts/local-migrate.py @@ -642,6 +642,15 @@ def _environment_contract_sha256(values: Mapping[str, str]) -> str: ) ) +def _github_actions_root_owned_socket(path: Path, owner: int) -> bool: + return ( + path == Path("/var/run/docker.sock") + and owner == 0 + and os.environ.get("GITHUB_ACTIONS") == "true" + and os.environ.get("CI") == "true" + ) + + def _assert_local_docker_context(docker: str) -> None: environment = { key: value @@ -694,7 +703,13 @@ def _assert_local_docker_context(docker: str) -> None: info = socket_path.lstat() except OSError as error: raise LocalMigrationError("docker_context") from error - if stat.S_ISLNK(info.st_mode) or not stat.S_ISSOCK(info.st_mode) or info.st_uid != os.getuid(): + owned_by_current_user = info.st_uid == os.getuid() + owned_by_disposable_ci_root = _github_actions_root_owned_socket(socket_path, info.st_uid) + if ( + stat.S_ISLNK(info.st_mode) + or not stat.S_ISSOCK(info.st_mode) + or not (owned_by_current_user or owned_by_disposable_ci_root) + ): raise LocalMigrationError("docker_context") def _reject_path_custody(path: Path) -> None: diff --git a/backend/supabase/scripts/local-stack.py b/backend/supabase/scripts/local-stack.py index 54638d9e0..684ff4385 100644 --- a/backend/supabase/scripts/local-stack.py +++ b/backend/supabase/scripts/local-stack.py @@ -34,6 +34,9 @@ GENERATOR_VERSION = "local-stack-v1" COMPOSE_START_TIMEOUT_SECONDS = 600 COMPOSE_START_RETRIES = 2 +COMPOSE_SERVICE_START_TIMEOUT_SECONDS = 180 +COMPOSE_SERVICE_START_RETRIES = 1 +COMPOSE_DATABASE_BOOTSTRAP_TIMEOUT_SECONDS = 900 EXPECTED_SERVICES = ( "analytics", "auth", "db", "functions", "imgproxy", "kong", "mail", "meta", "realtime", "rest", "storage", "studio", "supavisor", "vector", @@ -47,9 +50,25 @@ "db-logs.sql": "volumes/db/logs.sql", "db-pooler.sql": "volumes/db/pooler.sql", } +STAGED_INPUT_FILES = ( + ("kong.yml", "kong-config", "temp.yml"), + ("vector.yml", "vector-config", "vector.yml"), + ("pooler.exs", "pooler-config", "pooler.exs"), + ("functions/main/index.ts", "functions", "main/index.ts"), + ("db-supabase.sql", "db-init-migrations", "97-_supabase.sql"), + ("db-logs.sql", "db-init-migrations", "99-logs.sql"), + ("db-pooler.sql", "db-init-migrations", "99-pooler.sql"), + ("db-realtime.sql", "db-init-migrations", "99-realtime.sql"), + ("db-webhooks.sql", "db-init-scripts", "98-webhooks.sql"), + ("db-roles.sql", "db-init-scripts", "99-roles.sql"), + ("db-jwt.sql", "db-init-scripts", "99-jwt.sql"), +) DESTINATIONS = { - "/home/kong/temp.yml", "/etc/vector/vector.yml", "/etc/pooler/pooler.exs", + "/home/kong", "/home/kong/temp.yml", "/etc/vector", "/etc/vector/vector.yml", + "/etc/pooler", "/etc/pooler/pooler.exs", "/var/lib/postgresql/data", "/etc/postgresql-custom", "/var/lib/storage", + "/docker-entrypoint-initdb.d/migrations", + "/docker-entrypoint-initdb.d/init-scripts", "/home/deno/functions", "/docker-entrypoint-initdb.d/migrations/99-realtime.sql", "/docker-entrypoint-initdb.d/init-scripts/98-webhooks.sql", "/docker-entrypoint-initdb.d/init-scripts/99-roles.sql", @@ -68,7 +87,17 @@ DOCKER_SOCKET_DEFAULT = Path("/var/run/docker.sock") DOCKER_SOCKET_DOCKER_DESKTOP = ".docker/run/docker.sock" DOCKER_SOCKET_COLIMA = ".colima/default/docker.sock" -TARGET_VOLUME_SUFFIXES = ("db-data", "db-config", "storage-data") +TARGET_VOLUME_SUFFIXES = ( + "db-data", + "db-config", + "db-init-migrations", + "db-init-scripts", + "functions", + "kong-config", + "pooler-config", + "storage-data", + "vector-config", +) DOCKER_PROJECT_LABEL = "com.docker.compose.project" DOCKER_VOLUME_LABEL = "com.docker.compose.volume" DOCKER_SERVICE_LABEL = "com.docker.compose.service" @@ -98,6 +127,13 @@ READINESS_REQUIRED = tuple(READINESS_ENDPOINTS) CORE_REQUIRED = tuple(service for service in READINESS_REQUIRED if service != "studio") CORE_SERVICES = tuple(service for service in EXPECTED_SERVICES if service != "studio") +CORE_START_PHASES = ( + (("vector",), ("vector",)), + (("db",), ("db",)), + (("analytics",), ("analytics",)), + (("imgproxy",), ()), + (("auth", "functions", "kong", "mail", "meta", "realtime", "rest", "storage", "supavisor"), ()), +) _ACTIVE_COMMAND: list[str] | None = None @@ -1021,7 +1057,7 @@ def _scan_model(model: dict[str, Any], project: str, state: Path, values: dict[s if volume_name not in expected_named_volumes or not isinstance(volume, dict): _fail("model_volume") actual_name = volume.get("name") - if actual_name not in {f"{project}-db-data", f"{project}-db-config", f"{project}-storage-data"}: + if actual_name not in {f"{project}-{suffix}" for suffix in TARGET_VOLUME_SUFFIXES}: _fail("model_volume") if sorted(published_ports) != sorted(expected_ports): _fail("model_ports") @@ -1263,9 +1299,30 @@ def _probe_host_tcp(port: int, timeout: int = 5) -> bool: except OSError: return False +def _probe_database_bootstrap(command: list[str], timeout: int = 5) -> bool: + try: + result = subprocess.run( + command + [ + "exec", "-T", "db", + "psql", "-U", "postgres", "-d", "_supabase", + "-Atqc", "select 1 from pg_namespace where nspname = '_analytics'", + ], + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=_safe_process_environment(), + ) + except (OSError, subprocess.TimeoutExpired): + return False + return result.returncode == 0 and result.stdout.strip() == "1" + def _probe_service(command: list[str], values: dict[str, str], service: str, timeout: int = 5) -> bool: if service == "db": - return _probe_endpoint(command, service, READINESS_ENDPOINTS[service], timeout) + return ( + _probe_endpoint(command, service, READINESS_ENDPOINTS[service], timeout) + and _probe_database_bootstrap(command, timeout) + ) if service == "kong": return _probe_host_http(int(values["KONG_HTTP_PORT"]), "/auth/v1/health", timeout=timeout) if service == "rest": @@ -1426,6 +1483,81 @@ def _assert_project_volumes(command: list[str], project: str, *, require_existin if labels.get(DOCKER_PROJECT_LABEL) != project or labels.get(DOCKER_SERVICE_LABEL) not in EXPECTED_SERVICES: _fail("docker_container") + +def _stage_input_files(project: str, state: Path) -> None: + for source_name, _volume_suffix, _destination_name in STAGED_INPUT_FILES: + _regular_owned(state / "inputs" / source_name, mode=0o600) + volume_paths = { + "kong-config": "/kong", + "vector-config": "/vector", + "pooler-config": "/pooler", + "functions": "/functions", + "db-init-migrations": "/migrations", + "db-init-scripts": "/scripts", + } + commands = [ + "set -eu", + "mkdir -p /functions/main", + "cp /inputs/kong.yml /kong/temp.yml", + "cp /inputs/vector.yml /vector/vector.yml", + "cp /inputs/pooler.exs /pooler/pooler.exs", + "cp /inputs/functions/main/index.ts /functions/main/index.ts", + "cp /inputs/db-supabase.sql /migrations/97-_supabase.sql", + "cp /inputs/db-logs.sql /migrations/99-logs.sql", + "cp /inputs/db-pooler.sql /migrations/99-pooler.sql", + "cp /inputs/db-realtime.sql /migrations/99-realtime.sql", + "cp /inputs/db-webhooks.sql /scripts/98-webhooks.sql", + "cp /inputs/db-roles.sql /scripts/99-roles.sql", + "cp /inputs/db-jwt.sql /scripts/99-jwt.sql", + "chmod 0644 /kong/temp.yml /vector/vector.yml /pooler/pooler.exs /functions/main/index.ts /migrations/97-_supabase.sql /migrations/99-logs.sql /migrations/99-pooler.sql /migrations/99-realtime.sql /scripts/98-webhooks.sql /scripts/99-roles.sql /scripts/99-jwt.sql", + ] + volume_args: list[str] = [] + for suffix, path in volume_paths.items(): + volume_args.extend(("-v", f"{project}-{suffix}:{path}:Z")) + helper: str | None = None + try: + result = _run( + [ + "docker", + "create", + "--network", + "none", + "--entrypoint", + "sh", + "-v", + f"{state / 'inputs'}:/inputs:ro,z", + *volume_args, + "supabase/postgres:15.8.1.085", + "-c", + "; ".join(commands), + ], + error_code="compose_input_stage", + ) + helper = result.stdout.strip() + if not re.fullmatch(r"[0-9a-f]{12,64}", helper): + _fail("compose_input_stage") + _run( + ["docker", "start", helper], + error_code="compose_input_stage", + ) + wait_result = _run( + ["docker", "wait", helper], + error_code="compose_input_stage", + ) + if wait_result.stdout.strip() != "0": + _fail("compose_input_stage") + finally: + if helper is not None: + try: + _run( + ["docker", "rm", "-f", helper], + timeout=60, + error_code="compose_db_init_cleanup", + ) + except (LocalStackError, OSError, ValueError): + pass + + def _action_render(root: Path, project: str, state: Path) -> dict[str, Any]: digest, _, _ = _render(root, project, state) input_digest, env_digest = _provenance_digests(state) @@ -1449,21 +1581,47 @@ def _action_start(root: Path, project: str, state: Path) -> dict[str, Any]: _ACTIVE_COMMAND = command started = True _run( - command + ["up", "-d", *CORE_SERVICES], + command + ["create", "--force-recreate", "--pull=missing", *CORE_SERVICES], timeout=COMPOSE_START_TIMEOUT_SECONDS, - error_code="compose_core_start", + error_code="compose_core_create", retries=COMPOSE_START_RETRIES, ) + _stage_input_files(project, state) + for services, wait_for in CORE_START_PHASES: + for service in services: + _run( + command + ["start", service], + timeout=COMPOSE_SERVICE_START_TIMEOUT_SECONDS, + error_code=f"compose_core_start_{service}", + retries=COMPOSE_SERVICE_START_RETRIES, + ) + if wait_for: + _wait_ready( + command, + values, + timeout=( + COMPOSE_DATABASE_BOOTSTRAP_TIMEOUT_SECONDS + if wait_for == ("db",) + else 300 + ), + required=wait_for, + ) _wait_ready(command, values, required=CORE_REQUIRED) _run( - command + ["up", "-d", "studio"], + command + ["create", "--force-recreate", "--pull=missing", "studio"], timeout=COMPOSE_START_TIMEOUT_SECONDS, - error_code="compose_studio_start", + error_code="compose_studio_create", retries=COMPOSE_START_RETRIES, ) + _run( + command + ["start", "studio"], + timeout=COMPOSE_SERVICE_START_TIMEOUT_SECONDS, + error_code="compose_studio_start_studio", + retries=COMPOSE_SERVICE_START_RETRIES, + ) services = _wait_ready(command, values) except (LocalStackError, OSError, ValueError): - if started: + if started and os.environ.get("LOCAL_STACK_PRESERVE_FAILURE_STATE") != "1": try: _run(command + ["down", "--remove-orphans"]) except (LocalStackError, OSError, ValueError): diff --git a/backend/supabase/tests/test_local_compose_inputs.py b/backend/supabase/tests/test_local_compose_inputs.py index 985040d3e..db816b7f5 100644 --- a/backend/supabase/tests/test_local_compose_inputs.py +++ b/backend/supabase/tests/test_local_compose_inputs.py @@ -57,7 +57,17 @@ def test_overlay_pins_renderer_and_resets_base_fixed_names(self) -> None: else: self.assertIn("container_name: !reset null", block, service) self.assertIn("image: inbucket/inbucket:3.0.3", self.mail_source) - for volume in ("db-data", "db-config", "storage-data"): + for volume in ( + "db-data", + "db-config", + "db-init-migrations", + "db-init-scripts", + "functions", + "kong-config", + "pooler-config", + "storage-data", + "vector-config", + ): self.assertIn(f'name: "${{PROJECT_NAME}}-{volume}"', self.overlay_source) def test_ports_and_urls_are_loopback_only(self) -> None: @@ -111,7 +121,7 @@ def test_project_name_is_fixed_to_repository_identity_not_container_names(self) def test_tracked_inputs_are_explicit_local_sources_and_destinations(self) -> None: for output, relative in local_stack.TRACKED_SQL.items(): - self.assertIn(f'"${{LOCAL_INPUT_ROOT}}/{output}:/', self.overlay_source) + self.assertNotIn(f'"${{LOCAL_INPUT_ROOT}}/{output}:/', self.overlay_source) self.assertEqual(relative.split("/")[0], "volumes") self.assertNotIn("${HOME}", self.overlay_source) self.assertNotIn("/var/run/docker.sock", self.overlay_source) @@ -119,9 +129,24 @@ def test_tracked_inputs_are_explicit_local_sources_and_destinations(self) -> Non for destination in local_stack.DESTINATIONS: self.assertIsInstance(destination, str) self.assertTrue(destination.startswith("/")) - self.assertIn('"${LOCAL_INPUT_ROOT}/functions:/home/deno/functions:ro,Z"', self.overlay_source) - self.assertIn('"${LOCAL_INPUT_ROOT}/vector.yml:/etc/vector/vector.yml:ro,z"', self.overlay_source) - self.assertIn('"${LOCAL_INPUT_ROOT}/pooler.exs:/etc/pooler/pooler.exs:ro,z"', self.overlay_source) + self.assertIn( + '"local-db-init-migrations:/docker-entrypoint-initdb.d/migrations:Z"', + self.overlay_source, + ) + self.assertIn( + '"local-db-init-scripts:/docker-entrypoint-initdb.d/init-scripts:Z"', + self.overlay_source, + ) + self.assertIn('"local-functions:/home/deno/functions:Z"', self.overlay_source) + self.assertIn('"local-kong-config:/home/kong:Z"', self.overlay_source) + self.assertIn('"local-vector-config:/etc/vector:Z"', self.overlay_source) + self.assertIn('"local-pooler-config:/etc/pooler:Z"', self.overlay_source) + self.assertIn('entrypoint: ["/app/limits.sh"]', self.overlay_source) + + self.assertIn("environment: !override", self.overlay_source) + self.assertNotIn("CLUSTER_POSTGRES", self.overlay_source) + self.assertIn('ERL_AFLAGS: ""', self.overlay_source) + self.assertIn('RLIMIT_NOFILE: ""', self.overlay_source) def test_compose_command_uses_project_env_and_all_three_local_files(self) -> None: with tempfile.TemporaryDirectory() as directory: @@ -226,11 +251,30 @@ def test_local_stack_rejects_remote_contexts_and_unowned_volume_resources(self) self.assertIn("timeout=COMPOSE_START_TIMEOUT_SECONDS", source) self.assertIn("COMPOSE_START_RETRIES", source) self.assertIn("retries=COMPOSE_START_RETRIES", source) + self.assertIn("COMPOSE_SERVICE_START_TIMEOUT_SECONDS", source) + self.assertIn("COMPOSE_SERVICE_START_RETRIES", source) + self.assertIn("COMPOSE_DATABASE_BOOTSTRAP_TIMEOUT_SECONDS", source) + self.assertIn("CORE_START_PHASES", source) + self.assertIn('(("vector",), ("vector",))', source) + self.assertIn('(("db",), ("db",))', source) + self.assertIn('(("analytics",), ("analytics",))', source) + self.assertIn("LOCAL_STACK_PRESERVE_FAILURE_STATE", source) + self.assertIn("_probe_database_bootstrap", source) + self.assertIn("_analytics", source) + self.assertIn("pg_namespace", source) + self.assertIn("STAGED_INPUT_FILES", source) + self.assertIn("_stage_input_files", source) + self.assertIn("compose_input_stage", source) + self.assertIn('"create",', source) + self.assertIn('"wait",', source) + self.assertIn("chmod 0644", source) + self.assertIn('"create", "--force-recreate", "--pull=missing"', source) + self.assertIn('command + ["start", service]', source) self.assertIn("_COMPOSE_ERROR_MARKERS", source) self.assertIn("_compose_error_suffix", source) self.assertIn('error_code="compose_config"', source) - self.assertIn('error_code="compose_core_start"', source) - self.assertIn('error_code="compose_studio_start"', source) + self.assertIn('error_code="compose_core_create"', source) + self.assertIn('error_code="compose_studio_create"', source) self.assertIn('_assert_project_volumes(command, project)', source) if __name__ == "__main__": unittest.main() diff --git a/backend/supabase/tests/test_local_function_runtime_contract.py b/backend/supabase/tests/test_local_function_runtime_contract.py index 6cf1d2fb4..0837820c5 100644 --- a/backend/supabase/tests/test_local_function_runtime_contract.py +++ b/backend/supabase/tests/test_local_function_runtime_contract.py @@ -1,9 +1,13 @@ from __future__ import annotations import importlib.util +import json +import stat import tempfile import unittest from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch REPOSITORY_ROOT = Path(__file__).resolve().parents[3] @@ -71,6 +75,49 @@ def test_candidate_smoke_binds_every_argument_with_explicit_types(self): ) self.assertIn("'0A000'", trigger_sql) self.assertIn("expected_sqlstate_", trigger_sql) + + def test_candidate_smoke_allows_privacy_incident_guard_outcomes(self): + sql = self.scanner._smoke_candidate_blocks( + [ + { + "schema": "public", + "proname": "preview_privacy_incident_transition", + "identityArgumentsNormalized": ( + "uuid,uuid,public.privacy_incident_status,timestamptz,text,jsonb,uuid" + ), + "signature": "public.preview_privacy_incident_transition(...)", + } + ] + ) + self.assertIn("'P0001', '42501'", sql) + + def test_docker_context_accepts_github_actions_root_socket_only(self): + scanner = self.scanner + socket_info = SimpleNamespace(st_mode=stat.S_IFSOCK | 0o660, st_uid=0) + selected = SimpleNamespace(returncode=0, stdout="default\n") + inspected = SimpleNamespace( + returncode=0, + stdout=json.dumps([{ + "Endpoints": {"docker": {"Host": "unix:///var/run/docker.sock"}}, + }]), + ) + with ( + patch.object(scanner.subprocess, "run", side_effect=(selected, inspected)), + patch.object(scanner.Path, "lstat", return_value=socket_info), + patch.object(scanner.os, "getuid", return_value=1000), + patch.dict(scanner.os.environ, {"GITHUB_ACTIONS": "true", "CI": "true"}, clear=False), + ): + scanner._assert_local_docker_context("docker") + + with ( + patch.object(scanner.subprocess, "run", side_effect=(selected, inspected)), + patch.object(scanner.Path, "lstat", return_value=socket_info), + patch.object(scanner.os, "getuid", return_value=1000), + patch.dict(scanner.os.environ, {"GITHUB_ACTIONS": "false", "CI": "true"}, clear=False), + ): + with self.assertRaisesRegex(scanner.RuntimeScanError, "docker_context"): + scanner._assert_local_docker_context("docker") + def test_smoke_checks_external_effect_surface_in_runtime_catalog(self): sql = self.scanner._smoke_sql([]) self.assertIn("pg_catalog.pg_extension", sql) diff --git a/backend/supabase/tests/test_local_migration_contract.py b/backend/supabase/tests/test_local_migration_contract.py index 81cf14a86..02c8c2cba 100644 --- a/backend/supabase/tests/test_local_migration_contract.py +++ b/backend/supabase/tests/test_local_migration_contract.py @@ -2,8 +2,11 @@ import importlib.util import json import re +import stat import sys import tempfile +from types import SimpleNamespace +from unittest.mock import patch import unittest from pathlib import Path @@ -109,6 +112,33 @@ def test_filename_contract_is_checked_before_manifest_admission(self) -> None: self.assertIsNone(re.match(r"^\d{8,14}(?:_|\.)", "seed.sql")) with self.assertRaisesRegex(local_migrate.LocalMigrationError, "source_root_not_canonical"): local_migrate.migration_files(ROOT / "backend/supabase") + + def test_docker_context_accepts_github_actions_root_socket_only(self) -> None: + socket_info = SimpleNamespace(st_mode=stat.S_IFSOCK | 0o660, st_uid=0) + selected = SimpleNamespace(returncode=0, stdout="default\n") + inspected = SimpleNamespace( + returncode=0, + stdout=json.dumps([{ + "Endpoints": {"docker": {"Host": "unix:///var/run/docker.sock"}}, + }]), + ) + with ( + patch.object(local_migrate.subprocess, "run", side_effect=(selected, inspected)), + patch.object(local_migrate.Path, "lstat", return_value=socket_info), + patch.object(local_migrate.os, "getuid", return_value=1000), + patch.dict(local_migrate.os.environ, {"GITHUB_ACTIONS": "true", "CI": "true"}, clear=False), + ): + local_migrate._assert_local_docker_context("docker") + + with ( + patch.object(local_migrate.subprocess, "run", side_effect=(selected, inspected)), + patch.object(local_migrate.Path, "lstat", return_value=socket_info), + patch.object(local_migrate.os, "getuid", return_value=1000), + patch.dict(local_migrate.os.environ, {"GITHUB_ACTIONS": "false", "CI": "true"}, clear=False), + ): + with self.assertRaisesRegex(local_migrate.LocalMigrationError, "docker_context"): + local_migrate._assert_local_docker_context("docker") + def test_receipts_bind_runtime_closure_and_readback_sources(self) -> None: source = SCRIPT.read_text(encoding="utf-8") for token in ( diff --git a/docs/operations/nightly-regression.md b/docs/operations/nightly-regression.md index c0281a18d..3b03a6d08 100644 --- a/docs/operations/nightly-regression.md +++ b/docs/operations/nightly-regression.md @@ -62,6 +62,8 @@ runs `test:nightly -- --mode local`. The runner attempts only the disposable user-namespace sysctl settings needed by the containment probe. Missing privileged sysctl access is not silently replaced with a weaker runtime; the preflight fails closed. +The checked-in source-bound prerequisite artifact is verified in place before +apply; the lane never overwrites tracked baseline files during a hosted run. The GitHub-hosted runner's root-owned `/var/run/docker.sock` is accepted only for the default local socket when both `GITHUB_ACTIONS=true` and `CI=true`; remote Docker contexts and non-default sockets remain rejected. @@ -72,6 +74,23 @@ stderr. Core and Studio starts retry twice after bounded command failures to absorb transient image-registry or runner startup errors; persistent failures remain fail closed. +The lifecycle creates the pinned services once and starts them individually +instead of relying on one collective `up` orchestration call. Vector, database, +and analytics readiness gates run before dependent services are started; the +database gate also verifies the `_analytics` bootstrap schema with a +900-second first-run bound. +The local input files remain owner-only in the checkout. Before service startup, +a network-isolated helper copies the fixed local inputs into disposable +configuration and database-init volumes with container-readable modes; the +helper is removed before service startup and those volumes are removed during +teardown. The hosted lane removes only the nine exact project volume names +before reset and verifies that each is absent, so diagnostic probes cannot leave +stale database or configuration state behind. +The local Supavisor overlay invokes the image's limits wrapper directly because +the hosted Docker namespace denies Tini's optional subreaper capability. It +also clears the image's `RLIMIT_NOFILE` request because the hosted disposable +namespace rejects raising that limit; the runner's bounded default remains in +force. A failed reset may add a bounded `local-stack-failure-diagnostics-v1` receipt to the short-retention Actions artifact. It contains only fixed service state, health, exit-code, and Compose-status fields; it is never in the public release. @@ -80,6 +99,17 @@ retry and records only fixed image/status/failure-class fields. A failed pull stops the lane before Docker Compose or publication. The preflight also runs a no-network container probe from a pulled image so Docker runtime failures are separated from Compose configuration failures. +When container creation succeeds, it starts each Core service separately and +retains only fixed service/failure-class fields for the first failing service. +The same receipt records whether the collective `up` orchestration succeeds, +which distinguishes Compose orchestration failures from per-service starts. +During the hosted acceptance lane, a failed reset preserves the disposable +container state until bounded diagnostics are captured; the final cleanup step +still removes the project and volumes. +The diagnostics receipt records only service state, exit code, health status, +health failure streak, bounded health-check exit codes, restart count, OOM +status, a fixed class derived from bounded container logs, and the fixed +database bootstrap result. The scheduled regression job has `contents: read` only. It uploads a short-retention `nightly-local-` Actions artifact containing only the