diff --git a/.env.example b/.env.example index 4a472fd..bac4630 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,16 @@ -# Shared secret the SvelteKit BFF uses to call the FastAPI backend. -# openssl rand -hex 32 -BACKEND_SERVICE_TOKEN= +# Generate with: openssl rand -hex 32 +COMPUTE_API_TOKEN= -# Better Auth session secret for the web app. openssl rand -base64 32 +# Generate with: openssl rand -base64 32 BETTER_AUTH_SECRET= -# Public origin of the web app (used by Better Auth + SvelteKit). ORIGIN=http://localhost:3000 -# Optional: comma-separated browser origins allowed to call the API directly. -# Normally empty. The browser talks to the BFF, not FastAPI. +# Leave empty unless a browser calls the API directly. ALLOWED_ORIGINS= + +# Password for the runtime web role. Generate with: openssl rand -hex 32 +APP_DB_PASSWORD= + +# Browser-reachable MinIO endpoint for output download URLs. +MINIO_PUBLIC_ENDPOINT=localhost:9000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c02c1f..705f799 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,6 @@ env: UV_VERSION: "0.11.27" PYTHON_VERSION: "3.14" -# Least-privilege by default. Each job opts into what it needs. permissions: {} jobs: @@ -43,6 +42,8 @@ jobs: runs-on: ubuntu-latest env: TSDHN_MODEL_DIR: ${{ github.workspace }}/model + # Coverage.py cannot trace Numba's compiled kernels. + NUMBA_DISABLE_JIT: "1" strategy: fail-fast: false matrix: @@ -57,13 +58,10 @@ jobs: enable-cache: true cache-dependency-glob: "uv.lock" - # libgmt is needed by pygmt (a runtime dep of `tsdhn`). - # Install the system lib and create the libgmt.so symlink that - # pygmt expects. The symlink step is idempotent. - - name: Install GMT (libgmt) for pygmt + - name: Install GMT (libgmt) and Ghostscript for pygmt run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends gmt libgmt-dev + sudo apt-get install -y --no-install-recommends gmt libgmt-dev ghostscript SO=$(ls /usr/lib/**/libgmt.so.* 2>/dev/null | head -n1) if [ -z "$SO" ]; then echo "::error::libgmt.so.* not found after apt install" @@ -75,7 +73,10 @@ jobs: - run: uv sync --all-packages --group dev - name: Run tests with coverage - run: uv run pytest -n auto --maxfail=1 -q --cov=packages --cov-report= --cov-fail-under=0 + run: | + uv run pytest -n auto --maxfail=1 -q -m "not integration" \ + --cov=packages/tsdhn/tsdhn --cov=packages/api/api \ + --cov-report= --cov-fail-under=0 - name: Name coverage data if: always() @@ -94,14 +95,19 @@ jobs: if-no-files-found: ignore coverage: - name: Coverage gate + name: Diff coverage gate runs-on: ubuntu-latest - needs: test + needs: [test, api-integration] if: always() permissions: contents: read steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Fetch master for the diff base + run: git fetch origin master:refs/remotes/origin/master - uses: astral-sh/setup-uv@6a191366842ac1502ba6c07e9b5acd5c2d9d8db3 # 8.3.2 with: @@ -119,13 +125,22 @@ jobs: pattern: coverage-data-* merge-multiple: true - - name: Combine coverage and enforce baseline + - name: Combine coverage and report the total run: | uv run coverage combine uv run coverage html --skip-covered --skip-empty - uv run coverage report --format=markdown --fail-under=0 >> "$GITHUB_STEP_SUMMARY" + uv run coverage xml + uv run coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" uv run coverage report + - name: Enforce diff coverage on new/changed lines + run: | + uv run diff-cover coverage.xml \ + --compare-branch=origin/master \ + --markdown-report diff-coverage.md \ + --fail-under=80 + cat diff-coverage.md >> "$GITHUB_STEP_SUMMARY" + - name: Upload HTML coverage report if: failure() uses: actions/upload-artifact@v7 @@ -150,6 +165,103 @@ jobs: - run: bun run fmt:check - run: bun --filter web check + web-test: + name: Web tests (diff coverage gate) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Fetch master for the diff base + run: git fetch origin master:refs/remotes/origin/master + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.14" + + - uses: astral-sh/setup-uv@6a191366842ac1502ba6c07e9b5acd5c2d9d8db3 # 8.3.2 + with: + version: ${{ env.UV_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} + enable-cache: true + cache-dependency-glob: "uv.lock" + + - run: bun install --frozen-lockfile + - run: uv sync --group dev --no-install-project + + - run: bun --filter web test:coverage + + - name: Enforce diff coverage on new/changed lines + run: | + # lcov paths are relative to apps/web; diff-cover runs at the root. + sed -i 's|^SF:|SF:apps/web/|' apps/web/coverage/lcov.info + uv run diff-cover apps/web/coverage/lcov.info \ + --compare-branch=origin/master \ + --markdown-report diff-coverage-web.md \ + --fail-under=80 + cat diff-coverage-web.md >> "$GITHUB_STEP_SUMMARY" + + api-integration: + name: Persistence integration + runs-on: ubuntu-latest + permissions: + contents: read + services: + postgres: + image: postgres:18 + env: + POSTGRES_USER: tsdhn + POSTGRES_PASSWORD: tsdhn + POSTGRES_DB: tsdhn + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v7 + + - uses: astral-sh/setup-uv@6a191366842ac1502ba6c07e9b5acd5c2d9d8db3 # 8.3.2 + with: + version: ${{ env.UV_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} + enable-cache: true + cache-dependency-glob: "uv.lock" + + - run: uv sync --all-packages --group dev + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.14" + + - run: bun install --frozen-lockfile + + - name: Run disposable database integration tests + run: bash scripts/integration.sh --coverage + env: + COMPUTE_DATABASE_URL: postgresql://tsdhn:tsdhn@localhost:5432/tsdhn + + - name: Name coverage data + if: always() + run: | + if [ -f .coverage ]; then + mv .coverage .coverage.integration + fi + + - name: Upload coverage data + if: always() + uses: actions/upload-artifact@v7 + with: + name: coverage-data-integration + path: .coverage.integration + include-hidden-files: true + if-no-files-found: ignore + contract: name: API client contract (no drift) runs-on: ubuntu-latest @@ -165,7 +277,6 @@ jobs: enable-cache: true cache-dependency-glob: "uv.lock" - # api.main imports pygmt through tsdhn, which needs libgmt at import time. - name: Install GMT (libgmt) for pygmt run: | sudo apt-get update diff --git a/.github/workflows/crash-recovery.yml b/.github/workflows/crash-recovery.yml new file mode 100644 index 0000000..e15984a --- /dev/null +++ b/.github/workflows/crash-recovery.yml @@ -0,0 +1,83 @@ +name: Crash-recovery e2e + +# Run the crash-recovery scenarios against the real Compose stack. +on: + workflow_dispatch: + +concurrency: + group: crash-recovery-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: {} + +jobs: + crash-recovery: + name: Crash-recovery e2e (real toolchain) + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + env: + COMPUTE_API_TOKEN: compose-e2e-token + APP_DB_PASSWORD: compose-e2e-app-db-password + BETTER_AUTH_SECRET: compose-e2e-better-auth-secret + ORIGIN: http://localhost:3000 + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + MINIO_BUCKET: tsdhn-results + steps: + - uses: actions/checkout@v7 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Build toolchain image (cached) + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: deploy/toolchain.Dockerfile + tags: localhost/tsdhn-toolchain:crash-recovery-ci + load: true + cache-from: type=gha,scope=toolchain + cache-to: type=gha,mode=max,scope=toolchain + + - name: Build api image (cached) + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: deploy/api.Dockerfile + build-args: TOOLCHAIN_IMAGE=localhost/tsdhn-toolchain:crash-recovery-ci + tags: localhost/tsdhn-api:crash-recovery-ci + load: true + cache-from: type=gha,scope=api + cache-to: type=gha,mode=max,scope=api + + - name: Start compute stack + env: + API_IMAGE: localhost/tsdhn-api:crash-recovery-ci + run: docker compose up -d --no-build postgres minio compute-migrate api worker + + - name: Run crash-recovery e2e scenarios + env: + API_IMAGE: localhost/tsdhn-api:crash-recovery-ci + run: bash scripts/e2e/crash_recovery_e2e.sh + + - name: Capture compose diagnostics + if: always() + run: | + mkdir -p e2e-artifacts + docker compose ps -a > e2e-artifacts/compose-ps.txt || true + docker compose logs --no-color > e2e-artifacts/compose.log || true + cp -f ./*.json e2e-artifacts/ 2>/dev/null || true + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@v7 + with: + name: crash-recovery-diagnostics + path: e2e-artifacts + if-no-files-found: ignore + + - name: Stop stack + if: always() + run: docker compose down -v --remove-orphans diff --git a/.github/workflows/golden.yml b/.github/workflows/golden.yml new file mode 100644 index 0000000..03318d6 --- /dev/null +++ b/.github/workflows/golden.yml @@ -0,0 +1,55 @@ +name: Golden pipeline regression + +# Run the full pipeline regression against the real scientific toolchain. +on: + workflow_dispatch: + +concurrency: + group: golden-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: {} + +jobs: + golden: + name: Golden pipeline (real toolchain) + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Build toolchain image (cached) + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: deploy/toolchain.Dockerfile + tags: localhost/tsdhn-toolchain:golden-ci + load: true + cache-from: type=gha,scope=toolchain + cache-to: type=gha,mode=max,scope=toolchain + + - name: Build api image (cached) + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: deploy/api.Dockerfile + build-args: TOOLCHAIN_IMAGE=localhost/tsdhn-toolchain:golden-ci + tags: localhost/tsdhn-api:golden-ci + load: true + cache-from: type=gha,scope=api + cache-to: type=gha,mode=max,scope=api + + - name: Run golden suite against the real toolchain + run: | + docker run --rm \ + localhost/tsdhn-api:golden-ci \ + sh -c ' + set -eu + uv sync --frozen --group dev --all-packages + uv run pytest -m golden -v packages/tsdhn/tests/test_pipeline_golden.py + ' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6eeed58..2fdca08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -232,8 +232,9 @@ jobs: env: API_IMAGE: ghcr.io/${{ github.repository_owner }}/tsdhn-api:candidate-${{ github.run_id }}-${{ github.run_attempt }} TOOLCHAIN_IMAGE: ghcr.io/${{ github.repository_owner }}/tsdhn-toolchain:candidate-${{ github.run_id }}-${{ github.run_attempt }} - APP_JOB_ID: 4cfe522f-7e7d-46e0-96ca-7b98743fb9f5 - BACKEND_SERVICE_TOKEN: compose-e2e-token + SIMULATION_ID: 4cfe522f-7e7d-46e0-96ca-7b98743fb9f5 + COMPUTE_API_TOKEN: compose-e2e-token + APP_DB_PASSWORD: compose-e2e-app-db-password BETTER_AUTH_SECRET: compose-e2e-better-auth-secret ORIGIN: http://localhost:3000 MINIO_ACCESS_KEY: minioadmin @@ -260,11 +261,11 @@ jobs: - name: Validate compose config run: docker compose config --quiet - - name: Start backend stack from candidate images + - name: Start compute stack from candidate images run: docker compose up -d --no-build postgres minio compute-migrate api worker - - name: Run backend smoke test - run: bash scripts/e2e/backend_stack_smoke.sh + - name: Run compute smoke test + run: bash scripts/e2e/compute_stack_smoke.sh - name: Capture compose diagnostics if: always() diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index ac41069..9358b90 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -41,8 +41,8 @@ jobs: with: category: "/language:${{ matrix.language }}" - pip-audit: - name: pip-audit + audit: + name: Dependency audit (runtime) runs-on: ubuntu-latest permissions: contents: read @@ -55,26 +55,24 @@ jobs: enable-cache: true cache-dependency-glob: "uv.lock" - - name: Export locked requirements - # --no-emit-workspace drops the api/cli workspace - # members so pip-audit doesn't see them as editable installs - # (which fail --strict even with --skip-editable). - # --locked fails the job if uv.lock is stale. - # https://docs.astral.sh/uv/reference/cli/#uv-export - run: | - set -euxo pipefail - uv export \ - --locked \ - --no-hashes \ - --all-packages \ - --no-emit-workspace \ - --output-file=requirements.lock.txt + - run: uv audit --locked --preview-features audit-command --no-dev --no-group build - - name: pip-audit - # uvx runs pip-audit in an isolated env so it doesn't sync - # the workspace (which would re-introduce editable installs). - # https://pypi.org/project/pip-audit/ - run: uvx pip-audit --strict --requirement requirements.lock.txt + audit-toolchain: + name: Dependency audit (toolchain, advisory) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + + - uses: astral-sh/setup-uv@6a191366842ac1502ba6c07e9b5acd5c2d9d8db3 # 8.3.2 + with: + version: ${{ env.UV_VERSION }} + enable-cache: true + cache-dependency-glob: "uv.lock" + + - run: uv audit --locked --preview-features audit-command + continue-on-error: true gitleaks: name: Gitleaks (secrets) diff --git a/.gitignore b/.gitignore index 967266c..64c64ef 100644 --- a/.gitignore +++ b/.gitignore @@ -13,10 +13,13 @@ node_modules/ # coverage reports .coverage .coverage.* +coverage +coverage.xml coverage.json htmlcov/ # runtime outputs +.data data /jobs/ hypo.dat diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..2c00558 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,193 @@ +# Architecture + +This document explains what each component owns and how a simulation moves +through the system. Component READMEs explain how to work on each component. +[`DEPLOY.md`](./DEPLOY.md) explains how to run the services. + +## System + +```text +Browser + | + v +SvelteKit web app + |-- users, sessions, and simulations in public + |-- reads current jobs from compute.jobs + | + v +FastAPI compute service + |-- compute.jobs and the Procrastinate queue + |-- worker -> tsdhn engine -> MinIO +``` + +The browser talks to the web app. The web app checks the session and handles +the researcher-facing flow. Its server code calls the compute API at +`COMPUTE_API_URL` with `COMPUTE_API_TOKEN`. The browser never receives the +token and never calls the compute API directly. + +The web app and compute service may use the same PostgreSQL server. Separate +schemas and database roles keep their writes independent. + +## Responsibilities + +### Web app + +The web app owns: + +- users, sessions, accounts, and verification records; +- `simulation_id`, user ID, submitted parameters, and creation time; +- failures that happen while submitting a simulation to the compute service; +- authentication, ownership checks, and user-facing responses; +- joining the simulation with current compute state for display. + +The web app does not store compute progress, queue state, an internal compute +job ID, a compute-service selector, or output storage keys. + +### Compute service + +The compute service owns: + +- the API used by the web server; +- `compute.jobs` and the Procrastinate queue tables; +- the internal compute job ID; +- job progress, retry state, and worker heartbeats; +- simulation work directories and checkpoints; +- output metadata and uploads to MinIO. + +The compute service receives a `simulation_id` from the web app. It does not +know about web sessions, users, or passwords. + +### Simulation engine + +The `tsdhn` package owns the scientific calculation, pipeline order, run +directory, checkpoints, and output files. It has no dependency on web users, +PostgreSQL jobs, queues, or MinIO. + +The engine preserves several numerical and file-format rules from the original +MATLAB and Fortran programs. Those rules are documented with the engine and +beside the code that implements them. Legacy behavior is compatibility +evidence; it is not treated as scientific validation without a source. + +### Output storage + +The worker may use local disk while a simulation is running. That work +directory contains intermediate files and checkpoints needed for recovery. + +MinIO stores completed output files. The compute database stores their names, +media types, filenames, and private object keys. Public API responses omit the +object keys. + +## Database ownership + +The database owner runs migrations and owns all schemas and tables. Runtime +services use restricted roles. + +The web app creates and writes only its tables in `public`. Its simulation +table contains: + +```text +id +user_id +params +submission_error +created_at +``` + +The compute service creates and writes `compute.jobs` and the Procrastinate +queue tables. `compute.jobs.simulation_id` links a compute job to the web +simulation. The value is unique because repeating a submission must return the +same compute job. + +The web runtime role can read and write the web tables and read +`compute.jobs`. It cannot change compute jobs, read queue tables, or run schema +changes. The table definition in `apps/web/src/lib/server/db/compute.ts` is +used only for reads and is excluded from web migrations. + +## Identifiers + +| Name | Owner | Purpose | +| --- | --- | --- | +| `simulation_id` | web app | Public simulation ID, used again when a submission is retried | +| internal job ID | compute service | Database and queue ID; never returned to the web app | +| output object key | compute service | Private MinIO location | + +The public page is `/simulations/{simulation_id}`. The web app creates +`simulation_id` before calling the compute API and uses the same value for +every retry. + +Repeating a request with the same `simulation_id` and input returns the +existing compute job. Reusing the ID with different input is rejected. A +researcher can therefore retry after a lost response without starting the +same simulation twice. + +## Displayed state + +The compute row is the current record once it exists. Before then, the web app +derives a short submission state from its own row: + +```text +compute row exists use the compute job status +no compute row, submission error submission_failed +no compute row, no error submitting +``` + +A stale submission error never overrides an existing compute row. A successful +retry clears the saved submission error. + +Compute jobs use these states: + +```text +queued waiting for a worker +running executing the simulation +completed output metadata and files were stored +failed the simulation cannot continue +``` + +## Submit a simulation + +1. The web app authenticates the user and validates the form. +2. It creates the simulation ID and saves the user input. +3. It sends the ID and input to the compute API. +4. The compute service creates the job and queue task in one database + transaction, or returns the existing job for the same ID and input. +5. The browser opens the simulation page. + +If the request fails before a compute row can be read, the web app saves a +submission error. The researcher can retry with the same `simulation_id`. + +## Run a simulation + +1. A worker claims the queue task. +2. It marks the compute job as running. +3. It runs the engine in `TSDHN_JOBS_DIR/{simulation_id}`. +4. Progress callbacks update the compute job and notify listeners. +5. The worker uploads completed output files and their metadata to MinIO. +6. It marks the job as completed only after storage succeeds. + +## Show progress + +The web app checks ownership, then reads the simulation and its compute row. +For live progress, it relays the compute API event stream to the browser. The +compute service sends the current state, listens for PostgreSQL notifications, +and closes the stream when the job finishes or the configured stream lifetime +ends. + +## Download an output file + +1. The browser asks the web app for an output name. +2. The web app checks the session, simulation ownership, and available names. +3. The compute API creates a short-lived MinIO URL. +4. The browser downloads the file directly from MinIO. + +The web app and compute API do not relay output bytes. + +## Failure and recovery + +The compute service retries temporary failures in PostgreSQL, MinIO, or other +services. Invalid input, missing model files, and failed scientific steps do +not become more likely to succeed when repeated, so they fail the job. + +The engine records enough state to continue valid completed work after a +retry. The compute service keeps the job in `compute.jobs` and reports the +final failure when retries are exhausted. Deployment settings determine retry +limits, worker recovery, storage, and cleanup. diff --git a/apps/web/.env.example b/apps/web/.env.example index 00b6d03..5050b66 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -1,7 +1,6 @@ -# Drizzle / libSQL. Self-hosted libsql-server (sqld) in prod: -# DATABASE_URL=http://libsql:8080 -# Local file for quick dev: -DATABASE_URL=file:local.db +# PostgreSQL URL for the web app. +# The Compose stack creates this role before starting the app. +DATABASE_URL=postgresql://tsdhn_app:change-me@localhost:5432/tsdhn ORIGIN="http://localhost:5173" @@ -9,6 +8,6 @@ ORIGIN="http://localhost:5173" # https://www.better-auth.com/docs/installation BETTER_AUTH_SECRET="" -# TSDHN backend (FastAPI). The BFF calls it server-to-server with the token. -BACKEND_URL="http://localhost:8000" -BACKEND_SERVICE_TOKEN="" +# FastAPI compute service. Only server code sends this token. +COMPUTE_API_URL="http://localhost:8000" +COMPUTE_API_TOKEN="" diff --git a/apps/web/drizzle.config.ts b/apps/web/drizzle.config.ts index e3a4d95..9264734 100644 --- a/apps/web/drizzle.config.ts +++ b/apps/web/drizzle.config.ts @@ -4,7 +4,8 @@ if (!process.env.DATABASE_URL) throw new Error("DATABASE_URL is not set"); export default defineConfig({ schema: "./src/lib/server/db/schema.ts", - dialect: "sqlite", + out: "./drizzle", + dialect: "postgresql", dbCredentials: { url: process.env.DATABASE_URL }, verbose: true, strict: true, diff --git a/apps/web/drizzle/0000_blue_tana_nile.sql b/apps/web/drizzle/0000_blue_tana_nile.sql deleted file mode 100644 index f4bd759..0000000 --- a/apps/web/drizzle/0000_blue_tana_nile.sql +++ /dev/null @@ -1,81 +0,0 @@ -CREATE TABLE `simulation` ( - `id` text PRIMARY KEY NOT NULL, - `user_id` text NOT NULL, - `params` text NOT NULL, - `status` text DEFAULT 'pending_dispatch' NOT NULL, - `compute_backend` text, - `compute_job_id` text, - `result_bucket` text, - `result_key` text, - `details` text, - `step` text, - `step_index` integer, - `total_steps` integer, - `calculation` text, - `travel_times` text, - `artifacts_available` integer DEFAULT false NOT NULL, - `error` text, - `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, - `updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, - `dispatched_at` integer, - `finished_at` integer, - FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE INDEX `simulation_userId_idx` ON `simulation` (`user_id`);--> statement-breakpoint -CREATE INDEX `simulation_user_createdAt_idx` ON `simulation` (`user_id`,`created_at`);--> statement-breakpoint -CREATE INDEX `simulation_status_idx` ON `simulation` (`status`);--> statement-breakpoint -CREATE INDEX `simulation_compute_idx` ON `simulation` (`compute_backend`,`compute_job_id`);--> statement-breakpoint -CREATE TABLE `account` ( - `id` text PRIMARY KEY NOT NULL, - `account_id` text NOT NULL, - `provider_id` text NOT NULL, - `user_id` text NOT NULL, - `access_token` text, - `refresh_token` text, - `id_token` text, - `access_token_expires_at` integer, - `refresh_token_expires_at` integer, - `scope` text, - `password` text, - `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, - `updated_at` integer NOT NULL, - FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE INDEX `account_userId_idx` ON `account` (`user_id`);--> statement-breakpoint -CREATE TABLE `session` ( - `id` text PRIMARY KEY NOT NULL, - `expires_at` integer NOT NULL, - `token` text NOT NULL, - `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, - `updated_at` integer NOT NULL, - `ip_address` text, - `user_agent` text, - `user_id` text NOT NULL, - FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE UNIQUE INDEX `session_token_unique` ON `session` (`token`);--> statement-breakpoint -CREATE INDEX `session_userId_idx` ON `session` (`user_id`);--> statement-breakpoint -CREATE TABLE `user` ( - `id` text PRIMARY KEY NOT NULL, - `name` text NOT NULL, - `email` text NOT NULL, - `email_verified` integer DEFAULT false NOT NULL, - `image` text, - `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, - `updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `user_email_unique` ON `user` (`email`);--> statement-breakpoint -CREATE TABLE `verification` ( - `id` text PRIMARY KEY NOT NULL, - `identifier` text NOT NULL, - `value` text NOT NULL, - `expires_at` integer NOT NULL, - `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, - `updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL -); ---> statement-breakpoint -CREATE INDEX `verification_identifier_idx` ON `verification` (`identifier`); \ No newline at end of file diff --git a/apps/web/drizzle/0000_fair_eternals.sql b/apps/web/drizzle/0000_fair_eternals.sql new file mode 100644 index 0000000..428cf15 --- /dev/null +++ b/apps/web/drizzle/0000_fair_eternals.sql @@ -0,0 +1,63 @@ +CREATE TABLE "simulation" ( + "id" uuid PRIMARY KEY NOT NULL, + "user_id" text NOT NULL, + "params" jsonb NOT NULL, + "submission_error" text, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "account" ( + "id" text PRIMARY KEY NOT NULL, + "account_id" text NOT NULL, + "provider_id" text NOT NULL, + "user_id" text NOT NULL, + "access_token" text, + "refresh_token" text, + "id_token" text, + "access_token_expires_at" timestamp, + "refresh_token_expires_at" timestamp, + "scope" text, + "password" text, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp NOT NULL +); +--> statement-breakpoint +CREATE TABLE "session" ( + "id" text PRIMARY KEY NOT NULL, + "expires_at" timestamp NOT NULL, + "token" text NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp NOT NULL, + "ip_address" text, + "user_agent" text, + "user_id" text NOT NULL, + CONSTRAINT "session_token_unique" UNIQUE("token") +); +--> statement-breakpoint +CREATE TABLE "user" ( + "id" text PRIMARY KEY NOT NULL, + "name" text NOT NULL, + "email" text NOT NULL, + "email_verified" boolean DEFAULT false NOT NULL, + "image" text, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "user_email_unique" UNIQUE("email") +); +--> statement-breakpoint +CREATE TABLE "verification" ( + "id" text PRIMARY KEY NOT NULL, + "identifier" text NOT NULL, + "value" text NOT NULL, + "expires_at" timestamp NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "simulation" ADD CONSTRAINT "simulation_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "account" ADD CONSTRAINT "account_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "session" ADD CONSTRAINT "session_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "simulation_user_created_at_idx" ON "simulation" USING btree ("user_id","created_at" DESC NULLS LAST);--> statement-breakpoint +CREATE INDEX "account_userId_idx" ON "account" USING btree ("user_id");--> statement-breakpoint +CREATE INDEX "session_userId_idx" ON "session" USING btree ("user_id");--> statement-breakpoint +CREATE INDEX "verification_identifier_idx" ON "verification" USING btree ("identifier"); diff --git a/apps/web/drizzle/meta/0000_snapshot.json b/apps/web/drizzle/meta/0000_snapshot.json index 396d51c..b5c072c 100644 --- a/apps/web/drizzle/meta/0000_snapshot.json +++ b/apps/web/drizzle/meta/0000_snapshot.json @@ -1,177 +1,66 @@ { - "version": "6", - "dialect": "sqlite", - "id": "eab180e2-5562-41ec-b5a9-53f6c6c2da69", + "id": "5eb232ea-77a2-4e8d-8739-41f05d5fab84", "prevId": "00000000-0000-0000-0000-000000000000", + "version": "7", + "dialect": "postgresql", "tables": { - "simulation": { + "public.simulation": { "name": "simulation", + "schema": "", "columns": { "id": { "name": "id", - "type": "text", + "type": "uuid", "primaryKey": true, - "notNull": true, - "autoincrement": false + "notNull": true }, "user_id": { "name": "user_id", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "params": { "name": "params", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending_dispatch'" - }, - "compute_backend": { - "name": "compute_backend", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "compute_job_id": { - "name": "compute_job_id", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "result_bucket": { - "name": "result_bucket", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "result_key": { - "name": "result_key", - "type": "text", + "type": "jsonb", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": true }, - "details": { - "name": "details", + "submission_error": { + "name": "submission_error", "type": "text", "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "step": { - "name": "step", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "step_index": { - "name": "step_index", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "total_steps": { - "name": "total_steps", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "calculation": { - "name": "calculation", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "travel_times": { - "name": "travel_times", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "artifacts_available": { - "name": "artifacts_available", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "error": { - "name": "error", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "created_at": { "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(cast(unixepoch('subsecond') * 1000 as integer))" - }, - "updated_at": { - "name": "updated_at", - "type": "integer", + "type": "timestamp with time zone", "primaryKey": false, "notNull": true, - "autoincrement": false, - "default": "(cast(unixepoch('subsecond') * 1000 as integer))" - }, - "dispatched_at": { - "name": "dispatched_at", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "finished_at": { - "name": "finished_at", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false + "default": "now()" } }, "indexes": { - "simulation_userId_idx": { - "name": "simulation_userId_idx", - "columns": ["user_id"], - "isUnique": false - }, - "simulation_user_createdAt_idx": { - "name": "simulation_user_createdAt_idx", - "columns": ["user_id", "created_at"], - "isUnique": false - }, - "simulation_status_idx": { - "name": "simulation_status_idx", - "columns": ["status"], - "isUnique": false - }, - "simulation_compute_idx": { - "name": "simulation_compute_idx", - "columns": ["compute_backend", "compute_job_id"], - "isUnique": false + "simulation_user_created_at_idx": { + "name": "simulation_user_created_at_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -187,109 +76,109 @@ }, "compositePrimaryKeys": {}, "uniqueConstraints": {}, - "checkConstraints": {} + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, - "account": { + "public.account": { "name": "account", + "schema": "", "columns": { "id": { "name": "id", "type": "text", "primaryKey": true, - "notNull": true, - "autoincrement": false + "notNull": true }, "account_id": { "name": "account_id", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "provider_id": { "name": "provider_id", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "user_id": { "name": "user_id", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "access_token": { "name": "access_token", "type": "text", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "refresh_token": { "name": "refresh_token", "type": "text", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "id_token": { "name": "id_token", "type": "text", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "access_token_expires_at": { "name": "access_token_expires_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "refresh_token_expires_at": { "name": "refresh_token_expires_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "scope": { "name": "scope", "type": "text", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "password": { "name": "password", "type": "text", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "created_at": { "name": "created_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, "notNull": true, - "autoincrement": false, - "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + "default": "now()" }, "updated_at": { "name": "updated_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true } }, "indexes": { "account_userId_idx": { "name": "account_userId_idx", - "columns": ["user_id"], - "isUnique": false + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -305,79 +194,79 @@ }, "compositePrimaryKeys": {}, "uniqueConstraints": {}, - "checkConstraints": {} + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, - "session": { + "public.session": { "name": "session", + "schema": "", "columns": { "id": { "name": "id", "type": "text", "primaryKey": true, - "notNull": true, - "autoincrement": false + "notNull": true }, "expires_at": { "name": "expires_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "token": { "name": "token", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "created_at": { "name": "created_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, "notNull": true, - "autoincrement": false, - "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + "default": "now()" }, "updated_at": { "name": "updated_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "ip_address": { "name": "ip_address", "type": "text", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "user_agent": { "name": "user_agent", "type": "text", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "user_id": { "name": "user_id", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true } }, "indexes": { - "session_token_unique": { - "name": "session_token_unique", - "columns": ["token"], - "isUnique": true - }, "session_userId_idx": { "name": "session_userId_idx", - "columns": ["user_id"], - "isUnique": false + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -392,146 +281,158 @@ } }, "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, - "user": { + "public.user": { "name": "user", + "schema": "", "columns": { "id": { "name": "id", "type": "text", "primaryKey": true, - "notNull": true, - "autoincrement": false + "notNull": true }, "name": { "name": "name", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "email": { "name": "email", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "email_verified": { "name": "email_verified", - "type": "integer", + "type": "boolean", "primaryKey": false, "notNull": true, - "autoincrement": false, "default": false }, "image": { "name": "image", "type": "text", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "created_at": { "name": "created_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, "notNull": true, - "autoincrement": false, - "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + "default": "now()" }, "updated_at": { "name": "updated_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, "notNull": true, - "autoincrement": false, - "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + "default": "now()" } }, - "indexes": { + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { "user_email_unique": { "name": "user_email_unique", - "columns": ["email"], - "isUnique": true + "nullsNotDistinct": false, + "columns": ["email"] } }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, - "verification": { + "public.verification": { "name": "verification", + "schema": "", "columns": { "id": { "name": "id", "type": "text", "primaryKey": true, - "notNull": true, - "autoincrement": false + "notNull": true }, "identifier": { "name": "identifier", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "value": { "name": "value", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "expires_at": { "name": "expires_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "created_at": { "name": "created_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, "notNull": true, - "autoincrement": false, - "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + "default": "now()" }, "updated_at": { "name": "updated_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, "notNull": true, - "autoincrement": false, - "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + "default": "now()" } }, "indexes": { "verification_identifier_idx": { "name": "verification_identifier_idx", - "columns": ["identifier"], - "isUnique": false + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": {}, "compositePrimaryKeys": {}, "uniqueConstraints": {}, - "checkConstraints": {} + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false } }, - "views": {}, "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, "_meta": { + "columns": {}, "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} + "tables": {} } } diff --git a/apps/web/drizzle/meta/_journal.json b/apps/web/drizzle/meta/_journal.json index d60481b..b38ea2a 100644 --- a/apps/web/drizzle/meta/_journal.json +++ b/apps/web/drizzle/meta/_journal.json @@ -1,12 +1,12 @@ { "version": "7", - "dialect": "sqlite", + "dialect": "postgresql", "entries": [ { "idx": 0, - "version": "6", - "when": 1783444160559, - "tag": "0000_blue_tana_nile", + "version": "7", + "when": 1788069469697, + "tag": "0000_fair_eternals", "breakpoints": true } ] diff --git a/apps/web/package.json b/apps/web/package.json index a31a8ce..f0bcc7d 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -10,6 +10,10 @@ "prepare": "svelte-kit sync || echo ''", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", + "test": "vitest run --exclude 'src/**/*.integration.test.ts'", + "test:integration": "vitest run src/lib/server/simulations.integration.test.ts", + "test:watch": "vitest --exclude 'src/**/*.integration.test.ts'", + "test:coverage": "vitest run --exclude 'src/**/*.integration.test.ts' --coverage", "db:push": "drizzle-kit push", "db:generate": "drizzle-kit generate", "db:migrate": "drizzle-kit migrate", @@ -27,7 +31,6 @@ }, "devDependencies": { "@better-auth/drizzle-adapter": "~1.6.23", - "@libsql/client": "^0.17.4", "@sveltejs/adapter-auto": "^7.0.1", "@sveltejs/adapter-node": "^5.5.7", "@sveltejs/kit": "^2.69.1", @@ -37,15 +40,18 @@ "@tailwindcss/vite": "^4.3.2", "@types/geojson": "^7946.0.16", "@types/node": "^26.1.0", + "@vitest/coverage-v8": "^4.1.11", "auth": "~1.6.23", "better-auth": "~1.6.23", "drizzle-kit": "^0.31.10", "drizzle-orm": "^0.45.2", "mdsvex": "^0.12.7", + "postgres": "^3.4.7", "svelte": "^5.56.4", "svelte-check": "^4.7.2", "tailwindcss": "^4.3.2", "typescript": "^6.0.3", - "vite": "^8.1.3" + "vite": "^8.1.3", + "vitest": "^4.1.11" } } diff --git a/apps/web/readme.md b/apps/web/readme.md index 305b477..761f3a5 100644 --- a/apps/web/readme.md +++ b/apps/web/readme.md @@ -1,19 +1,14 @@ # TSDHN web -`apps/web` is the SvelteKit application for authenticated simulation requests, -calculation previews, progress display, and artifact status. +`apps/web` is the SvelteKit web app. It provides authentication, simulation +history, the input form, progress pages, and output downloads. -The app uses a server-side web backend pattern: +The browser talks to SvelteKit. Server code calls the FastAPI compute service +at `COMPUTE_API_URL` with `COMPUTE_API_TOKEN`. See +[`ARCHITECTURE.md`](../../ARCHITECTURE.md) for how the parts share data and +derive the state shown to researchers. -- Browser requests go to SvelteKit routes. -- SvelteKit server code calls FastAPI with `BACKEND_SERVICE_TOKEN`. -- The service token stays in server-side code. -- SQLite/libSQL stores data managed by the web app, such as users and submitted simulation - records. -- FastAPI manages live simulation progress, worker state, and MinIO artifact - pointers. - -## Commands +## Development From the repository root: @@ -22,70 +17,60 @@ bun install bun --filter web dev bun --filter web check bun --filter web build +bun --filter web test ``` -The root package also exposes: +`bun --filter web test` is the fast Vitest suite. It does not need PostgreSQL, +the compute service, or MinIO. It tests route decisions and server behavior +without a database. PostgreSQL queries are tested by: ```sh -mise run web-dev -mise run web-check -mise run web-build +mise run test-integration ``` -## Environment - -Create `apps/web/.env` from [`apps/web/.env.example`](./.env.example). - -| Variable | Example from `.env.example` | Purpose | -| ----------------------- | --------------------------- | ----------------------------------------------- | -| `DATABASE_URL` | `file:local.db` | Drizzle/libSQL connection string | -| `ORIGIN` | `http://localhost:5173` | Public origin used by SvelteKit and Better Auth | -| `BETTER_AUTH_SECRET` | empty | Better Auth session secret | -| `BACKEND_URL` | `http://localhost:8000` | FastAPI backend base URL | -| `BACKEND_SERVICE_TOKEN` | empty | Bearer token sent only by SvelteKit server code | - -`BETTER_AUTH_SECRET` and `BACKEND_SERVICE_TOKEN` must be non-empty outside -local-only development. +That task starts PostgreSQL, creates a temporary database, and runs the web +integration tests. It does not use the normal development database. ## Database -The app uses Drizzle with SQLite/libSQL. The schema lives in -[`src/lib/server/db/schema.ts`](./src/lib/server/db/schema.ts), and Better Auth -tables are generated into [`src/lib/server/db/auth.schema.ts`](./src/lib/server/db/auth.schema.ts). - -```sh -bun --filter web db:generate -bun --filter web db:migrate -bun --filter web db:push -bun --filter web db:studio -``` - -Regenerate the Better Auth schema with: +Web tables are declared in `src/lib/server/db/schema.ts`. Better Auth tables +are generated into `src/lib/server/db/auth.schema.ts`: ```sh bun --filter web auth:schema ``` -## Backend integration - -Typed backend calls use `@tsdhn/api-client` from -[`libs/api-client`](../../libs/api-client/readme.md). The wrapper in -[`src/lib/server/api.ts`](./src/lib/server/api.ts) attaches the bearer token and -uses SvelteKit's request-aware `fetch` for server-side calls. - -Current server-side backend calls include: +`src/lib/server/db/compute.ts` describes the columns read from `compute.jobs`. +It exists so Drizzle can join current compute state to a simulation. It is not +part of the web migration schema. -- `POST /api/v1/calculations` from [`src/routes/api/calculations/+server.ts`](./src/routes/api/calculations/+server.ts) -- `POST /api/v1/jobs` from [`src/lib/server/dispatch.ts`](./src/lib/server/dispatch.ts) -- `GET /api/v1/jobs/{id}` from app-owned simulation pages and dashboard status sync -- `GET /api/v1/jobs/{id}/events` proxied by [`src/routes/(app)/simulations/[id]/events/+server.ts`](./src/routes/%28app%29/simulations/%5Bid%5D/events/+server.ts) - -## Docker - -The root Compose file can run the web app with the backend stack: +Run web migrations with a database administrator connection: ```sh -docker compose --profile web up +bun --filter web db:generate +bun --filter web db:migrate ``` -The web image is built from [`deploy/web.Dockerfile`](../../deploy/web.Dockerfile). +The running app uses the restricted role created by the compute migration. See +[`DEPLOY.md`](../../DEPLOY.md) for the complete startup order and environment. + +## Server modules + +- `src/lib/server/compute-api.ts` builds the typed compute API client from + `COMPUTE_API_URL` and `COMPUTE_API_TOKEN`. Both values stay on the server. +- `src/lib/server/submit-simulation.ts` submits a simulation. It records an + error if the compute service rejects the request and clears that error after + a successful retry. +- `src/lib/server/simulation-repository.ts` contains simulation queries and + accepts the database connection to use. +- `src/hooks.server.ts` creates the production repository and puts it in + `event.locals`. +- `src/lib/server/simulation-details.ts` joins the simulation record with its + current compute status for pages. +- `src/lib/server/outputs.ts` checks requested output names against the files + available for the simulation. +- The progress and output route handlers check the session and simulation + ownership before calling the compute API. + +For local schema work, `db:push` and `db:studio` are available. Use migrations +for a deployed database. diff --git a/apps/web/src/app.d.ts b/apps/web/src/app.d.ts index 6ebc6a8..00b6a65 100644 --- a/apps/web/src/app.d.ts +++ b/apps/web/src/app.d.ts @@ -1,10 +1,13 @@ import type { User, Session } from "better-auth"; +import type { SimulationRepository } from "$lib/server/simulation-repository"; + declare global { namespace App { interface Locals { user?: User; session?: Session; + simulationRepository: SimulationRepository; } } } diff --git a/apps/web/src/hooks.server.ts b/apps/web/src/hooks.server.ts index 940acb4..b1508aa 100644 --- a/apps/web/src/hooks.server.ts +++ b/apps/web/src/hooks.server.ts @@ -1,9 +1,15 @@ import type { Handle } from "@sveltejs/kit"; import { building } from "$app/environment"; import { auth } from "$lib/server/auth"; +import { db } from "$lib/server/db"; +import { createSimulationRepository } from "$lib/server/simulation-repository"; import { svelteKitHandler } from "better-auth/svelte-kit"; +const simulationRepository = createSimulationRepository(db); + const handleBetterAuth: Handle = async ({ event, resolve }) => { + event.locals.simulationRepository = simulationRepository; + const session = await auth.api.getSession({ headers: event.request.headers }); if (session) { diff --git a/apps/web/src/lib/components/Map.svelte b/apps/web/src/lib/components/Map.svelte index 4326ee5..1423694 100644 --- a/apps/web/src/lib/components/Map.svelte +++ b/apps/web/src/lib/components/Map.svelte @@ -76,12 +76,10 @@ return () => map?.remove(); }); - // Form edits can update coordinates without a map click. $effect(() => { marker?.setLngLat([lon, lat]); }); - // Fault geometry arrives after the backend preview completes. $effect(() => { const data = faultData(); if (ready) { diff --git a/apps/web/src/lib/schema/earthquake.test.ts b/apps/web/src/lib/schema/earthquake.test.ts new file mode 100644 index 0000000..4a13ae6 --- /dev/null +++ b/apps/web/src/lib/schema/earthquake.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; + +import { earthquakeSchema, toEarthquakeInput } from "./earthquake"; + +describe("earthquake input", () => { + it("converts the form timestamp to UTC day and HHMM fields", () => { + const input = toEarthquakeInput({ + magnitude: 8.0, + depth: 12, + latitude: -20.5, + longitude: -70.5, + datetime: "2026-08-30T23:45:00-05:00", + }); + + expect(input).toEqual({ + Mw: 8.0, + h: 12, + lat0: -20.5, + lon0: -70.5, + dia: "31", + hhmm: "0445", + }); + }); + + it("rejects values outside the form's physical input limits", () => { + const result = earthquakeSchema.safeParse({ + magnitude: 6.4, + depth: -1, + latitude: -91, + longitude: 181, + datetime: "", + }); + + expect(result.success).toBe(false); + }); +}); diff --git a/apps/web/src/lib/schema/earthquake.ts b/apps/web/src/lib/schema/earthquake.ts index ca71746..4ea112e 100644 --- a/apps/web/src/lib/schema/earthquake.ts +++ b/apps/web/src/lib/schema/earthquake.ts @@ -19,7 +19,7 @@ export interface EarthquakeInput { hhmm: string; } -/** The backend expects event day and time in UTC fields. */ +/** The compute API expects event day and time in UTC fields. */ export function toEarthquakeInput(values: EarthquakeForm): EarthquakeInput { const when = new Date(values.datetime); const dia = String(when.getUTCDate()).padStart(2, "0"); diff --git a/apps/web/src/lib/server/api.ts b/apps/web/src/lib/server/api.ts deleted file mode 100644 index 298ef55..0000000 --- a/apps/web/src/lib/server/api.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { env } from "$env/dynamic/private"; -import { createTsdhnClient, type TsdhnClient } from "@tsdhn/api-client"; - -export function backend(fetch: typeof globalThis.fetch): TsdhnClient { - if (!env.BACKEND_URL) throw new Error("BACKEND_URL is not set"); - if (!env.BACKEND_SERVICE_TOKEN) throw new Error("BACKEND_SERVICE_TOKEN is not set"); - return createTsdhnClient({ - baseUrl: env.BACKEND_URL, - serviceToken: env.BACKEND_SERVICE_TOKEN, - fetch, - }); -} - -export function backendRaw(): { url: string; headers: Record } { - if (!env.BACKEND_URL) throw new Error("BACKEND_URL is not set"); - if (!env.BACKEND_SERVICE_TOKEN) throw new Error("BACKEND_SERVICE_TOKEN is not set"); - return { - url: env.BACKEND_URL.replace(/\/$/, ""), - headers: { Authorization: `Bearer ${env.BACKEND_SERVICE_TOKEN}` }, - }; -} diff --git a/apps/web/src/lib/server/auth.ts b/apps/web/src/lib/server/auth.ts index da3b685..0433493 100644 --- a/apps/web/src/lib/server/auth.ts +++ b/apps/web/src/lib/server/auth.ts @@ -8,7 +8,7 @@ import { db } from "$lib/server/db"; export const auth = betterAuth({ baseURL: env.ORIGIN, secret: env.BETTER_AUTH_SECRET, - database: drizzleAdapter(db, { provider: "sqlite" }), + database: drizzleAdapter(db, { provider: "pg" }), emailAndPassword: { enabled: true }, plugins: [sveltekitCookies(getRequestEvent)], }); diff --git a/apps/web/src/lib/server/calculation-route.test.ts b/apps/web/src/lib/server/calculation-route.test.ts new file mode 100644 index 0000000..d7ed220 --- /dev/null +++ b/apps/web/src/lib/server/calculation-route.test.ts @@ -0,0 +1,65 @@ +import { isHttpError } from "@sveltejs/kit"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + computeClient: vi.fn(), +})); + +vi.mock("$lib/server/compute-api", () => ({ computeClient: mocks.computeClient })); + +import { POST } from "../../routes/api/calculations/+server"; + +function context(overrides: Record = {}) { + return { + request: new Request("https://web.example/api/calculations", { + method: "POST", + body: JSON.stringify({ Mw: 8.0 }), + }), + locals: { user: { id: "user-1" } }, + fetch: vi.fn(), + ...overrides, + }; +} + +async function expectHttpError(action: unknown, status: number) { + try { + await action; + throw new Error("expected handler to throw"); + } catch (caught) { + expect(isHttpError(caught)).toBe(true); + if (isHttpError(caught)) expect(caught.status).toBe(status); + } +} + +describe("calculation preview route", () => { + beforeEach(() => { + vi.resetAllMocks(); + }); + + it("requires a signed-in user before contacting the compute service", async () => { + await expectHttpError(POST(context({ locals: { user: null } }) as never), 401); + expect(mocks.computeClient).not.toHaveBeenCalled(); + }); + + it("forwards the form body and returns the compute preview", async () => { + const client = { POST: vi.fn() }; + const preview = { calculation: { length: 1 }, travel_times: { arrival_times: {} } }; + client.POST.mockResolvedValue({ data: preview, error: undefined }); + mocks.computeClient.mockReturnValue(client); + + const response = await POST(context() as never); + + expect(await response.json()).toEqual(preview); + expect(client.POST).toHaveBeenCalledWith("/api/v1/calculations", { + body: { Mw: 8.0 }, + }); + }); + + it("returns a controlled gateway error when the compute preview fails", async () => { + const client = { POST: vi.fn() }; + client.POST.mockResolvedValue({ data: undefined, error: { detail: "failed" } }); + mocks.computeClient.mockReturnValue(client); + + await expectHttpError(POST(context() as never), 502); + }); +}); diff --git a/apps/web/src/lib/server/compute-api.ts b/apps/web/src/lib/server/compute-api.ts new file mode 100644 index 0000000..424145a --- /dev/null +++ b/apps/web/src/lib/server/compute-api.ts @@ -0,0 +1,21 @@ +import { env } from "$env/dynamic/private"; +import { createTsdhnClient, type TsdhnClient } from "@tsdhn/api-client"; + +export function computeClient(fetch: typeof globalThis.fetch): TsdhnClient { + if (!env.COMPUTE_API_URL) throw new Error("COMPUTE_API_URL is not set"); + if (!env.COMPUTE_API_TOKEN) throw new Error("COMPUTE_API_TOKEN is not set"); + return createTsdhnClient({ + baseUrl: env.COMPUTE_API_URL, + computeApiToken: env.COMPUTE_API_TOKEN, + fetch, + }); +} + +export function computeRequestConfig(): { url: string; headers: Record } { + if (!env.COMPUTE_API_URL) throw new Error("COMPUTE_API_URL is not set"); + if (!env.COMPUTE_API_TOKEN) throw new Error("COMPUTE_API_TOKEN is not set"); + return { + url: env.COMPUTE_API_URL.replace(/\/$/, ""), + headers: { Authorization: `Bearer ${env.COMPUTE_API_TOKEN}` }, + }; +} diff --git a/apps/web/src/lib/server/db/auth.schema.ts b/apps/web/src/lib/server/db/auth.schema.ts index 8dd30b2..413a524 100644 --- a/apps/web/src/lib/server/db/auth.schema.ts +++ b/apps/web/src/lib/server/db/auth.schema.ts @@ -1,31 +1,27 @@ -import { relations, sql } from "drizzle-orm"; -import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core"; +import { relations } from "drizzle-orm"; +import { pgTable, text, timestamp, boolean, index } from "drizzle-orm/pg-core"; -export const user = sqliteTable("user", { +export const user = pgTable("user", { id: text("id").primaryKey(), name: text("name").notNull(), email: text("email").notNull().unique(), - emailVerified: integer("email_verified", { mode: "boolean" }).default(false).notNull(), + emailVerified: boolean("email_verified").default(false).notNull(), image: text("image"), - createdAt: integer("created_at", { mode: "timestamp_ms" }) - .default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`) - .notNull(), - updatedAt: integer("updated_at", { mode: "timestamp_ms" }) - .default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`) + createdAt: timestamp("created_at").defaultNow().notNull(), + updatedAt: timestamp("updated_at") + .defaultNow() .$onUpdate(() => /* @__PURE__ */ new Date()) .notNull(), }); -export const session = sqliteTable( +export const session = pgTable( "session", { id: text("id").primaryKey(), - expiresAt: integer("expires_at", { mode: "timestamp_ms" }).notNull(), + expiresAt: timestamp("expires_at").notNull(), token: text("token").notNull().unique(), - createdAt: integer("created_at", { mode: "timestamp_ms" }) - .default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`) - .notNull(), - updatedAt: integer("updated_at", { mode: "timestamp_ms" }) + createdAt: timestamp("created_at").defaultNow().notNull(), + updatedAt: timestamp("updated_at") .$onUpdate(() => /* @__PURE__ */ new Date()) .notNull(), ipAddress: text("ip_address"), @@ -37,7 +33,7 @@ export const session = sqliteTable( (table) => [index("session_userId_idx").on(table.userId)], ); -export const account = sqliteTable( +export const account = pgTable( "account", { id: text("id").primaryKey(), @@ -49,36 +45,28 @@ export const account = sqliteTable( accessToken: text("access_token"), refreshToken: text("refresh_token"), idToken: text("id_token"), - accessTokenExpiresAt: integer("access_token_expires_at", { - mode: "timestamp_ms", - }), - refreshTokenExpiresAt: integer("refresh_token_expires_at", { - mode: "timestamp_ms", - }), + accessTokenExpiresAt: timestamp("access_token_expires_at"), + refreshTokenExpiresAt: timestamp("refresh_token_expires_at"), scope: text("scope"), password: text("password"), - createdAt: integer("created_at", { mode: "timestamp_ms" }) - .default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`) - .notNull(), - updatedAt: integer("updated_at", { mode: "timestamp_ms" }) + createdAt: timestamp("created_at").defaultNow().notNull(), + updatedAt: timestamp("updated_at") .$onUpdate(() => /* @__PURE__ */ new Date()) .notNull(), }, (table) => [index("account_userId_idx").on(table.userId)], ); -export const verification = sqliteTable( +export const verification = pgTable( "verification", { id: text("id").primaryKey(), identifier: text("identifier").notNull(), value: text("value").notNull(), - expiresAt: integer("expires_at", { mode: "timestamp_ms" }).notNull(), - createdAt: integer("created_at", { mode: "timestamp_ms" }) - .default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`) - .notNull(), - updatedAt: integer("updated_at", { mode: "timestamp_ms" }) - .default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`) + expiresAt: timestamp("expires_at").notNull(), + createdAt: timestamp("created_at").defaultNow().notNull(), + updatedAt: timestamp("updated_at") + .defaultNow() .$onUpdate(() => /* @__PURE__ */ new Date()) .notNull(), }, diff --git a/apps/web/src/lib/server/db/compute.ts b/apps/web/src/lib/server/db/compute.ts new file mode 100644 index 0000000..31e1d15 --- /dev/null +++ b/apps/web/src/lib/server/db/compute.ts @@ -0,0 +1,28 @@ +import { integer, jsonb, pgSchema, text, timestamp, uuid } from "drizzle-orm/pg-core"; + +const computeSchema = pgSchema("compute"); + +export type StoredOutput = { + name: string; + key: string; + filename: string; + content_type: string; +}; + +export const computeJob = computeSchema.table("jobs", { + id: uuid("id").primaryKey(), + simulationId: uuid("simulation_id").notNull().unique(), + status: text("status").notNull(), + details: text("details"), + step: text("step"), + stepIndex: integer("step_index"), + totalSteps: integer("total_steps"), + calculation: jsonb("calculation"), + travelTimes: jsonb("travel_times"), + outputs: jsonb("outputs").$type(), + error: text("error"), + startedAt: timestamp("started_at", { withTimezone: true }), + finishedAt: timestamp("finished_at", { withTimezone: true }), +}); + +export type ComputeJob = typeof computeJob.$inferSelect; diff --git a/apps/web/src/lib/server/db/index.ts b/apps/web/src/lib/server/db/index.ts index 84cc8a2..8586469 100644 --- a/apps/web/src/lib/server/db/index.ts +++ b/apps/web/src/lib/server/db/index.ts @@ -1,10 +1,11 @@ -import { drizzle } from "drizzle-orm/libsql"; -import { createClient } from "@libsql/client"; -import * as schema from "./schema"; import { env } from "$env/dynamic/private"; +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; + +import * as schema from "./schema"; if (!env.DATABASE_URL) throw new Error("DATABASE_URL is not set"); -const client = createClient({ url: env.DATABASE_URL }); +const client = postgres(env.DATABASE_URL, { max: 10 }); export const db = drizzle(client, { schema }); diff --git a/apps/web/src/lib/server/db/schema.ts b/apps/web/src/lib/server/db/schema.ts index 7d1de8d..a5b2475 100644 --- a/apps/web/src/lib/server/db/schema.ts +++ b/apps/web/src/lib/server/db/schema.ts @@ -1,53 +1,22 @@ import { relations, sql } from "drizzle-orm"; -import { index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { index, jsonb, pgTable, text, timestamp, uuid } from "drizzle-orm/pg-core"; import { user } from "./auth.schema"; -/** - * A tsunami simulation submitted through the web app. - * - * `id` is the app_job_id generated by the web app before dispatch. It is the - * stable public id used in URLs and user history. - */ -export const simulation = sqliteTable( +export const simulation = pgTable( "simulation", { - id: text("id").primaryKey(), + id: uuid("id").primaryKey(), userId: text("user_id") .notNull() .references(() => user.id, { onDelete: "cascade" }), - params: text("params", { mode: "json" }).notNull(), - status: text("status").notNull().default("pending_dispatch"), - computeBackend: text("compute_backend"), - computeJobId: text("compute_job_id"), - resultBucket: text("result_bucket"), - resultKey: text("result_key"), - details: text("details"), - step: text("step"), - stepIndex: integer("step_index"), - totalSteps: integer("total_steps"), - calculation: text("calculation", { mode: "json" }), - travelTimes: text("travel_times", { mode: "json" }), - artifactsAvailable: integer("artifacts_available", { mode: "boolean" }) - .notNull() - .default(false), - error: text("error"), - createdAt: integer("created_at", { mode: "timestamp_ms" }) - .default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`) - .notNull(), - updatedAt: integer("updated_at", { mode: "timestamp_ms" }) - .default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`) - .$onUpdate(() => /* @__PURE__ */ new Date()) + params: jsonb("params").notNull(), + submissionError: text("submission_error"), + createdAt: timestamp("created_at", { withTimezone: true }) + .default(sql`now()`) .notNull(), - dispatchedAt: integer("dispatched_at", { mode: "timestamp_ms" }), - finishedAt: integer("finished_at", { mode: "timestamp_ms" }), }, - (table) => [ - index("simulation_userId_idx").on(table.userId), - index("simulation_user_createdAt_idx").on(table.userId, table.createdAt), - index("simulation_status_idx").on(table.status), - index("simulation_compute_idx").on(table.computeBackend, table.computeJobId), - ], + (table) => [index("simulation_user_created_at_idx").on(table.userId, table.createdAt.desc())], ); export const simulationRelations = relations(simulation, ({ one }) => ({ diff --git a/apps/web/src/lib/server/dispatch.ts b/apps/web/src/lib/server/dispatch.ts deleted file mode 100644 index 919c2af..0000000 --- a/apps/web/src/lib/server/dispatch.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { TsdhnClient } from "@tsdhn/api-client"; - -import type { EarthquakeInput } from "$lib/schema/earthquake"; -import type { Simulation } from "$lib/server/db/schema"; -import { markDispatchAccepted, markDispatchFailed } from "$lib/server/simulations"; - -const DEFAULT_COMPUTE_BACKEND = "default"; - -function errorMessage(error: unknown): string { - if (error instanceof Error && error.message) return error.message; - return "No se pudo iniciar la simulación en el backend."; -} - -export async function dispatchSimulation( - sim: Pick, - client: TsdhnClient, - computeBackend = DEFAULT_COMPUTE_BACKEND, -): Promise<{ ok: true; computeJobId: string } | { ok: false; error: string }> { - try { - const { data, error } = await client.POST("/api/v1/jobs", { - body: { - app_job_id: sim.id, - input: sim.params as EarthquakeInput, - }, - }); - - if (error || !data) { - const message = - typeof error === "object" && error && "detail" in error - ? String(error.detail) - : "No se pudo iniciar la simulación en el backend."; - await markDispatchFailed(sim.id, message); - return { ok: false, error: message }; - } - - await markDispatchAccepted(sim.id, computeBackend, data.compute_job_id); - return { ok: true, computeJobId: data.compute_job_id }; - } catch (error) { - const message = errorMessage(error); - await markDispatchFailed(sim.id, message); - return { ok: false, error: message }; - } -} diff --git a/apps/web/src/lib/server/events-route.test.ts b/apps/web/src/lib/server/events-route.test.ts new file mode 100644 index 0000000..80f9a30 --- /dev/null +++ b/apps/web/src/lib/server/events-route.test.ts @@ -0,0 +1,90 @@ +import { isHttpError } from "@sveltejs/kit"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + computeRequestConfig: vi.fn(), + getSimulation: vi.fn(), +})); + +vi.mock("$lib/server/compute-api", () => ({ + computeRequestConfig: mocks.computeRequestConfig, +})); +import { GET } from "../../routes/(app)/simulations/[id]/events/+server"; + +function context(overrides: Record = {}) { + return { + params: { id: "sim-1" }, + locals: { + user: { id: "user-1" }, + simulationRepository: { getSimulation: mocks.getSimulation }, + }, + fetch: vi.fn(), + ...overrides, + }; +} + +async function expectHttpError(action: unknown, status: number) { + try { + await action; + throw new Error("expected handler to throw"); + } catch (caught) { + expect(isHttpError(caught)).toBe(true); + if (isHttpError(caught)) expect(caught.status).toBe(status); + } +} + +describe("simulation events route", () => { + beforeEach(() => { + vi.resetAllMocks(); + mocks.computeRequestConfig.mockReturnValue({ + url: "https://compute.example", + headers: { authorization: "Bearer token" }, + }); + }); + + it("rejects a simulation that has not reached the compute service", async () => { + mocks.getSimulation.mockResolvedValue({ id: "sim-1", status: "submitting" }); + + await expectHttpError(GET(context() as never), 409); + expect(mocks.computeRequestConfig).not.toHaveBeenCalled(); + }); + + it("returns a controlled gateway error when the upstream stream is unavailable", async () => { + mocks.getSimulation.mockResolvedValue({ id: "sim-1", status: "running" }); + const fetch = vi.fn().mockResolvedValue(new Response(null, { status: 503 })); + + await expectHttpError(GET(context({ fetch }) as never), 502); + }); + + it("requires an authenticated user before querying the simulation", async () => { + await expectHttpError(GET(context({ locals: { user: null } }) as never), 401); + + expect(mocks.getSimulation).not.toHaveBeenCalled(); + }); + + it("returns not found when the user cannot access the simulation", async () => { + mocks.getSimulation.mockResolvedValue(undefined); + + await expectHttpError(GET(context() as never), 404); + expect(mocks.computeRequestConfig).not.toHaveBeenCalled(); + }); + + it("passes through a healthy upstream event stream with safe response headers", async () => { + mocks.getSimulation.mockResolvedValue({ id: "sim-1", status: "running" }); + const body = new ReadableStream({ + start(controller) { + controller.close(); + }, + }); + const fetch = vi.fn().mockResolvedValue(new Response(body, { status: 200 })); + + const response = await GET(context({ fetch }) as never); + + expect(response.headers.get("content-type")).toBe("text/event-stream"); + expect(response.headers.get("cache-control")).toBe("no-cache"); + expect(response.body).toBe(body); + expect(fetch).toHaveBeenCalledWith("https://compute.example/api/v1/jobs/sim-1/events", { + headers: { authorization: "Bearer token" }, + }); + }); +}); diff --git a/apps/web/src/lib/server/output-route.test.ts b/apps/web/src/lib/server/output-route.test.ts new file mode 100644 index 0000000..178ee45 --- /dev/null +++ b/apps/web/src/lib/server/output-route.test.ts @@ -0,0 +1,94 @@ +import { isHttpError } from "@sveltejs/kit"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + computeRequestConfig: vi.fn(), + getSimulation: vi.fn(), +})); + +vi.mock("$lib/server/compute-api", () => ({ + computeRequestConfig: mocks.computeRequestConfig, +})); +import { GET } from "../../routes/(app)/simulations/[id]/outputs/[name]/+server"; + +const SIM = { id: "sim-1", outputs: ["max_height_map"] }; + +function context(overrides: Record = {}) { + return { + params: { id: "sim-1", name: "max_height_map" }, + locals: { + user: { id: "user-1" }, + simulationRepository: { getSimulation: mocks.getSimulation }, + }, + fetch: vi.fn(), + ...overrides, + }; +} + +async function expectHttpError(action: unknown, status: number) { + try { + await action; + throw new Error("expected handler to throw"); + } catch (caught) { + expect(isHttpError(caught)).toBe(true); + if (isHttpError(caught)) expect(caught.status).toBe(status); + } +} + +describe("output download route", () => { + beforeEach(() => { + vi.resetAllMocks(); + mocks.getSimulation.mockResolvedValue(SIM); + mocks.computeRequestConfig.mockReturnValue({ + url: "https://compute.example", + headers: { authorization: "Bearer token" }, + }); + }); + + it("requires an authenticated user before querying the simulation", async () => { + await expectHttpError(GET(context({ locals: { user: null } }) as never), 401); + + expect(mocks.getSimulation).not.toHaveBeenCalled(); + }); + + it("returns not found without contacting compute for an unknown simulation", async () => { + mocks.getSimulation.mockResolvedValue(undefined); + + await expectHttpError(GET(context() as never), 404); + + expect(mocks.computeRequestConfig).not.toHaveBeenCalled(); + }); + + it("returns a controlled gateway error when compute does not redirect", async () => { + const fetch = vi.fn().mockResolvedValue(new Response(null, { status: 500 })); + const requestContext = context({ fetch }); + + await expectHttpError(GET(requestContext as never), 502); + }); + + it("redirects the browser after ownership and output checks", async () => { + const fetch = vi.fn().mockResolvedValue( + new Response(null, { + status: 307, + headers: { location: "https://minio.example/result.pdf" }, + }), + ); + const requestContext = context({ fetch }); + + try { + await GET(requestContext as never); + throw new Error("expected redirect"); + } catch (redirect) { + expect(redirect).toMatchObject({ + status: 302, + location: "https://minio.example/result.pdf", + }); + } + + expect(mocks.getSimulation).toHaveBeenCalledWith("user-1", "sim-1"); + expect(fetch).toHaveBeenCalledWith( + "https://compute.example/api/v1/jobs/sim-1/outputs/max_height_map", + { headers: { authorization: "Bearer token" }, redirect: "manual" }, + ); + }); +}); diff --git a/apps/web/src/lib/server/outputs.test.ts b/apps/web/src/lib/server/outputs.test.ts new file mode 100644 index 0000000..d812b4a --- /dev/null +++ b/apps/web/src/lib/server/outputs.test.ts @@ -0,0 +1,50 @@ +import { isHttpError } from "@sveltejs/kit"; +import { describe, expect, it } from "vitest"; + +import { assertOutputAccessible } from "./outputs"; +import type { SimulationDetails } from "./simulation-details"; + +const SIM: SimulationDetails = { + id: "sim-1", + userId: "user-1", + params: null, + createdAt: new Date(), + submissionError: null, + status: "completed", + details: null, + step: null, + stepIndex: null, + totalSteps: null, + calculation: null, + travelTimes: null, + error: null, + outputs: ["max_height_map"], + startedAt: null, + finishedAt: null, +}; + +describe("assertOutputAccessible", () => { + it("allows a name present in the simulation outputs", () => { + expect(() => assertOutputAccessible(SIM, "max_height_map")).not.toThrow(); + }); + + it("rejects with 404 when the simulation does not exist", () => { + try { + assertOutputAccessible(undefined, "max_height_map"); + throw new Error("expected assertOutputAccessible to throw"); + } catch (err) { + expect(isHttpError(err)).toBe(true); + if (isHttpError(err)) expect(err.status).toBe(404); + } + }); + + it("rejects with 404 when the name is not on the simulation", () => { + try { + assertOutputAccessible(SIM, "not_a_real_output"); + throw new Error("expected assertOutputAccessible to throw"); + } catch (err) { + expect(isHttpError(err)).toBe(true); + if (isHttpError(err)) expect(err.status).toBe(404); + } + }); +}); diff --git a/apps/web/src/lib/server/outputs.ts b/apps/web/src/lib/server/outputs.ts new file mode 100644 index 0000000..81d8bea --- /dev/null +++ b/apps/web/src/lib/server/outputs.ts @@ -0,0 +1,13 @@ +import { error } from "@sveltejs/kit"; + +import type { SimulationDetails } from "$lib/server/simulation-details"; + +export function assertOutputAccessible( + sim: SimulationDetails | undefined, + name: string, +): asserts sim is SimulationDetails { + if (!sim) error(404); + if (!sim.outputs.includes(name)) { + error(404, "Este resultado no está disponible."); + } +} diff --git a/apps/web/src/lib/server/page-actions.test.ts b/apps/web/src/lib/server/page-actions.test.ts new file mode 100644 index 0000000..18238e1 --- /dev/null +++ b/apps/web/src/lib/server/page-actions.test.ts @@ -0,0 +1,187 @@ +import { isHttpError, isRedirect } from "@sveltejs/kit"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + computeClient: vi.fn(), + submitSimulation: vi.fn(), + createSimulation: vi.fn(), + getSimulation: vi.fn(), + message: vi.fn(), + superValidate: vi.fn(), + toEarthquakeInput: vi.fn(), + zod4: vi.fn(), +})); + +vi.mock("$lib/server/compute-api", () => ({ computeClient: mocks.computeClient })); +vi.mock("$lib/server/submit-simulation", () => ({ + submitSimulation: mocks.submitSimulation, +})); +vi.mock("$lib/schema/earthquake", () => ({ + defaultEarthquake: {}, + earthquakeSchema: {}, + toEarthquakeInput: mocks.toEarthquakeInput, +})); +vi.mock("sveltekit-superforms", () => ({ + message: mocks.message, + superValidate: mocks.superValidate, +})); +vi.mock("sveltekit-superforms/adapters", () => ({ zod4: mocks.zod4 })); + +import { actions as newActions } from "../../routes/(app)/new/+page.server"; +import { actions as simulationActions } from "../../routes/(app)/simulations/[id]/+page.server"; + +const SIMULATION_ID = "11111111-1111-4111-8111-111111111111"; +const INPUT = { + Mw: 8.0, + h: 12, + lat0: -20.5, + lon0: -70.5, + dia: "30", + hhmm: "0445", +}; +const FORM = { valid: true, data: { magnitude: 8.0 } }; +const SIM = { + id: "sim-1", + params: INPUT, + status: "submission_failed", +}; + +function repository() { + return { + createSimulation: mocks.createSimulation, + getSimulation: mocks.getSimulation, + }; +} + +function newContext(overrides: Record = {}) { + return { + request: new Request("https://web.example/new", { method: "POST" }), + locals: { user: { id: "user-1" }, simulationRepository: repository() }, + fetch: vi.fn(), + ...overrides, + }; +} + +function retryContext(overrides: Record = {}) { + return { + params: { id: "sim-1" }, + locals: { user: { id: "user-1" }, simulationRepository: repository() }, + fetch: vi.fn(), + ...overrides, + }; +} + +async function expectRedirect(action: unknown, location: string) { + try { + await action; + throw new Error("expected action to redirect"); + } catch (caught) { + expect(isRedirect(caught)).toBe(true); + if (isRedirect(caught)) { + expect(caught.status).toBe(303); + expect(caught.location).toBe(location); + } + } +} + +async function expectHttpError(action: unknown, status: number) { + try { + await action; + throw new Error("expected action to throw"); + } catch (caught) { + expect(isHttpError(caught)).toBe(true); + if (isHttpError(caught)) expect(caught.status).toBe(status); + } +} + +describe("new simulation action", () => { + beforeEach(() => { + vi.resetAllMocks(); + vi.stubGlobal("crypto", { randomUUID: () => SIMULATION_ID }); + mocks.zod4.mockReturnValue("adapter"); + mocks.toEarthquakeInput.mockReturnValue(INPUT); + mocks.superValidate.mockResolvedValue(FORM); + mocks.createSimulation.mockResolvedValue(undefined); + mocks.computeClient.mockReturnValue({}); + mocks.submitSimulation.mockResolvedValue({ ok: true }); + mocks.message.mockImplementation((form, text, options) => ({ form, text, options })); + }); + + it("persists and submits valid input before redirecting", async () => { + await expectRedirect( + newActions.default(newContext() as never), + `/simulations/${SIMULATION_ID}`, + ); + + expect(mocks.createSimulation).toHaveBeenCalledWith({ + id: SIMULATION_ID, + userId: "user-1", + params: INPUT, + }); + expect(mocks.submitSimulation).toHaveBeenCalledWith( + { id: SIMULATION_ID, params: INPUT }, + expect.anything(), + expect.objectContaining({ createSimulation: mocks.createSimulation }), + ); + }); + + it("does not create a simulation for invalid form data", async () => { + mocks.superValidate.mockResolvedValue({ valid: false, errors: { magnitude: "bad" } }); + + const result = await newActions.default(newContext() as never); + + expect(result).toMatchObject({ status: 400 }); + expect(mocks.createSimulation).not.toHaveBeenCalled(); + expect(mocks.submitSimulation).not.toHaveBeenCalled(); + }); + + it("returns a form error when the compute service rejects valid input", async () => { + mocks.submitSimulation.mockResolvedValue({ ok: false, error: "queue unavailable" }); + + const result = await newActions.default(newContext() as never); + + expect(result).toEqual({ + form: FORM, + text: "No se pudo iniciar la simulación.", + options: { status: 502 }, + }); + }); +}); + +describe("simulation retry action", () => { + beforeEach(() => { + vi.resetAllMocks(); + mocks.getSimulation.mockResolvedValue(SIM); + mocks.computeClient.mockReturnValue({}); + mocks.submitSimulation.mockResolvedValue({ ok: true }); + }); + + it("resubmits a simulation whose earlier submission failed", async () => { + await expectRedirect(simulationActions.retry(retryContext() as never), "/simulations/sim-1"); + + expect(mocks.getSimulation).toHaveBeenCalledWith("user-1", "sim-1"); + expect(mocks.submitSimulation).toHaveBeenCalledWith( + SIM, + expect.anything(), + expect.objectContaining({ getSimulation: mocks.getSimulation }), + ); + }); + + it("does not contact compute for a non-retryable simulation", async () => { + mocks.getSimulation.mockResolvedValue({ ...SIM, status: "running" }); + + const result = await simulationActions.retry(retryContext() as never); + + expect(result).toMatchObject({ status: 400 }); + expect(mocks.computeClient).not.toHaveBeenCalled(); + expect(mocks.submitSimulation).not.toHaveBeenCalled(); + }); + + it("rejects an unauthenticated retry", async () => { + await expectHttpError( + simulationActions.retry(retryContext({ locals: { user: null } }) as never), + 401, + ); + expect(mocks.getSimulation).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/lib/server/simulation-details.test.ts b/apps/web/src/lib/server/simulation-details.test.ts new file mode 100644 index 0000000..56ead40 --- /dev/null +++ b/apps/web/src/lib/server/simulation-details.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; + +import { toSimulationDetails } from "./simulation-details"; + +const BASE = { + id: "sim-1", + userId: "user-1", + params: { Mw: 8.0 }, + createdAt: new Date("2026-01-01T00:00:00Z"), + submissionError: null, + computeStatus: null, + details: null, + step: null, + stepIndex: null, + totalSteps: null, + calculation: null, + travelTimes: null, + computeError: null, + outputs: null, + startedAt: null, + finishedAt: null, +}; + +describe("toSimulationDetails", () => { + it("reports submitting before a compute row exists", () => { + const details = toSimulationDetails(BASE); + + expect(details.status).toBe("submitting"); + }); + + it("reports submission_failed when submission failed before a compute row exists", () => { + const details = toSimulationDetails({ + ...BASE, + submissionError: "compute service unreachable", + }); + + expect(details.status).toBe("submission_failed"); + expect(details.error).toBe("compute service unreachable"); + }); + + it("prefers the live compute status once a compute row exists", () => { + const details = toSimulationDetails({ + ...BASE, + submissionError: "stale submission error", + computeStatus: "running", + }); + + expect(details.status).toBe("running"); + }); + + it("prefers the compute error over an earlier submission error", () => { + const details = toSimulationDetails({ + ...BASE, + submissionError: "stale submission error", + computeStatus: "failed", + computeError: "worker crashed", + }); + + expect(details.error).toBe("worker crashed"); + }); + + it("uses the submission error when there is no compute row", () => { + const details = toSimulationDetails({ + ...BASE, + submissionError: "submission failed", + }); + + expect(details.error).toBe("submission failed"); + }); + + it("maps stored outputs to names used by download links", () => { + const details = toSimulationDetails({ + ...BASE, + outputs: [ + { name: "max_height_map", contentType: "application/pdf" }, + { name: "mareogram", contentType: "image/svg+xml" }, + ], + }); + + expect(details.outputs).toEqual(["max_height_map", "mareogram"]); + }); + + it("defaults outputs to an empty list when the compute row has none", () => { + const details = toSimulationDetails({ ...BASE, outputs: null }); + + expect(details.outputs).toEqual([]); + }); +}); diff --git a/apps/web/src/lib/server/simulation-details.ts b/apps/web/src/lib/server/simulation-details.ts new file mode 100644 index 0000000..7ed28d8 --- /dev/null +++ b/apps/web/src/lib/server/simulation-details.ts @@ -0,0 +1,64 @@ +import type { StoredOutput } from "$lib/server/db/compute"; + +export type SimulationDetails = { + id: string; + userId: string; + params: unknown; + createdAt: Date; + submissionError: string | null; + status: string; + details: string | null; + step: string | null; + stepIndex: number | null; + totalSteps: number | null; + calculation: unknown; + travelTimes: unknown; + error: string | null; + outputs: string[]; + startedAt: Date | null; + finishedAt: Date | null; +}; + +type SimulationDetailsRow = { + id: unknown; + userId: unknown; + params: unknown; + createdAt: unknown; + submissionError: unknown; + computeStatus: unknown; + details: unknown; + step: unknown; + stepIndex: unknown; + totalSteps: unknown; + calculation: unknown; + travelTimes: unknown; + computeError: unknown; + outputs: unknown; + startedAt: unknown; + finishedAt: unknown; +}; + +export function toSimulationDetails(row: SimulationDetailsRow): SimulationDetails { + const computeStatus = row.computeStatus as string | null; + const submissionError = row.submissionError as string | null; + const outputs = (row.outputs as StoredOutput[] | null) ?? []; + + return { + id: row.id as string, + userId: row.userId as string, + params: row.params, + createdAt: row.createdAt as Date, + submissionError, + status: computeStatus ?? (submissionError ? "submission_failed" : "submitting"), + details: row.details as string | null, + step: row.step as string | null, + stepIndex: row.stepIndex as number | null, + totalSteps: row.totalSteps as number | null, + calculation: row.calculation, + travelTimes: row.travelTimes, + error: (row.computeError as string | null) ?? submissionError, + outputs: outputs.map((output) => output.name), + startedAt: row.startedAt as Date | null, + finishedAt: row.finishedAt as Date | null, + }; +} diff --git a/apps/web/src/lib/server/simulation-repository.ts b/apps/web/src/lib/server/simulation-repository.ts new file mode 100644 index 0000000..a99c418 --- /dev/null +++ b/apps/web/src/lib/server/simulation-repository.ts @@ -0,0 +1,69 @@ +import { and, desc, eq } from "drizzle-orm"; +import type { PostgresJsDatabase } from "drizzle-orm/postgres-js"; + +import { computeJob } from "$lib/server/db/compute"; +import * as schema from "$lib/server/db/schema"; +import { type NewSimulation, simulation } from "$lib/server/db/schema"; +import { type SimulationDetails, toSimulationDetails } from "$lib/server/simulation-details"; + +export type SimulationDatabase = PostgresJsDatabase; + +export type SimulationRepository = ReturnType; + +const simulationDetailsSelection = { + id: simulation.id, + userId: simulation.userId, + params: simulation.params, + createdAt: simulation.createdAt, + submissionError: simulation.submissionError, + computeStatus: computeJob.status, + details: computeJob.details, + step: computeJob.step, + stepIndex: computeJob.stepIndex, + totalSteps: computeJob.totalSteps, + calculation: computeJob.calculation, + travelTimes: computeJob.travelTimes, + computeError: computeJob.error, + outputs: computeJob.outputs, + startedAt: computeJob.startedAt, + finishedAt: computeJob.finishedAt, +}; + +export function createSimulationRepository(database: SimulationDatabase) { + return { + async createSimulation(data: NewSimulation): Promise { + await database.insert(simulation).values(data); + }, + + async clearSubmissionFailure(id: string): Promise { + await database.update(simulation).set({ submissionError: null }).where(eq(simulation.id, id)); + }, + + async recordSubmissionFailure(id: string, error: string): Promise { + await database + .update(simulation) + .set({ submissionError: error }) + .where(eq(simulation.id, id)); + }, + + async listSimulations(userId: string): Promise { + const rows = await database + .select(simulationDetailsSelection) + .from(simulation) + .leftJoin(computeJob, eq(computeJob.simulationId, simulation.id)) + .where(eq(simulation.userId, userId)) + .orderBy(desc(simulation.createdAt)); + return rows.map(toSimulationDetails); + }, + + async getSimulation(userId: string, id: string): Promise { + const rows = await database + .select(simulationDetailsSelection) + .from(simulation) + .leftJoin(computeJob, eq(computeJob.simulationId, simulation.id)) + .where(and(eq(simulation.id, id), eq(simulation.userId, userId))) + .limit(1); + return rows[0] ? toSimulationDetails(rows[0]) : undefined; + }, + }; +} diff --git a/apps/web/src/lib/server/simulations.integration.test.ts b/apps/web/src/lib/server/simulations.integration.test.ts new file mode 100644 index 0000000..775e425 --- /dev/null +++ b/apps/web/src/lib/server/simulations.integration.test.ts @@ -0,0 +1,90 @@ +import { randomUUID } from "node:crypto"; + +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import * as schema from "./db/schema"; +import { createSimulationRepository } from "./simulation-repository"; + +const databaseUrl = process.env.DATABASE_URL; + +describe("simulation repository", () => { + let client: ReturnType; + let repository: ReturnType; + + beforeAll(() => { + if (!databaseUrl) { + throw new Error("DATABASE_URL is not set; run `mise run test-integration`"); + } + client = postgres(databaseUrl, { max: 2 }); + repository = createSimulationRepository(drizzle(client, { schema })); + }); + + afterAll(async () => { + await client.end(); + }); + + it("keeps simulation reads scoped to the owning user", async () => { + const ownerId = randomUUID(); + const otherUserId = randomUUID(); + const ownerSimulationId = randomUUID(); + const otherSimulationId = randomUUID(); + + await client` + INSERT INTO "user" ("id", "name", "email") + VALUES (${ownerId}, 'Owner', ${`${ownerId}@example.test`}), + (${otherUserId}, 'Other', ${`${otherUserId}@example.test`}) + `; + + await repository.createSimulation({ + id: ownerSimulationId, + userId: ownerId, + params: { Mw: 8.0 }, + }); + await repository.createSimulation({ + id: otherSimulationId, + userId: otherUserId, + params: { Mw: 7.5 }, + }); + + const ownerDetails = await repository.getSimulation(ownerId, ownerSimulationId); + + expect(ownerDetails).toMatchObject({ + id: ownerSimulationId, + userId: ownerId, + status: "submitting", + outputs: [], + }); + await expect(repository.getSimulation(otherUserId, ownerSimulationId)).resolves.toBeUndefined(); + await expect(repository.getSimulation(ownerId, otherSimulationId)).resolves.toBeUndefined(); + await expect(repository.listSimulations(ownerId)).resolves.toHaveLength(1); + }); + + it("records and clears submission failures", async () => { + const userId = randomUUID(); + const simulationId = randomUUID(); + + await client` + INSERT INTO "user" ("id", "name", "email") + VALUES (${userId}, 'Submitter', ${`${userId}@example.test`}) + `; + await repository.createSimulation({ + id: simulationId, + userId, + params: { Mw: 8.0 }, + }); + + await repository.recordSubmissionFailure(simulationId, "queue unavailable"); + await expect(repository.getSimulation(userId, simulationId)).resolves.toMatchObject({ + status: "submission_failed", + submissionError: "queue unavailable", + }); + + await repository.clearSubmissionFailure(simulationId); + await expect(repository.getSimulation(userId, simulationId)).resolves.toMatchObject({ + status: "submitting", + submissionError: null, + }); + }); +}); diff --git a/apps/web/src/lib/server/simulations.ts b/apps/web/src/lib/server/simulations.ts deleted file mode 100644 index 2c953d9..0000000 --- a/apps/web/src/lib/server/simulations.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { and, desc, eq } from "drizzle-orm"; - -import { db } from "$lib/server/db"; -import { type NewSimulation, type Simulation, simulation } from "$lib/server/db/schema"; - -export function createSimulation(data: NewSimulation): Promise { - return db.insert(simulation).values(data); -} - -export function markDispatchAccepted( - id: string, - computeBackend: string, - computeJobId: string, -): Promise { - return db - .update(simulation) - .set({ - status: "queued", - computeBackend, - computeJobId, - error: null, - artifactsAvailable: false, - dispatchedAt: new Date(), - finishedAt: null, - }) - .where(eq(simulation.id, id)); -} - -export function markDispatchFailed(id: string, error: string): Promise { - return db - .update(simulation) - .set({ - status: "dispatch_failed", - error, - finishedAt: new Date(), - }) - .where(eq(simulation.id, id)); -} - -export function listSimulations(userId: string): Promise { - return db - .select() - .from(simulation) - .where(eq(simulation.userId, userId)) - .orderBy(desc(simulation.createdAt)); -} - -export async function getSimulation(userId: string, id: string): Promise { - const rows = await db - .select() - .from(simulation) - .where(and(eq(simulation.id, id), eq(simulation.userId, userId))) - .limit(1); - return rows[0]; -} - -export function syncStatus( - id: string, - status: string, - artifactsAvailable: boolean, - snapshot?: { - details?: string | null; - step?: string | null; - step_index?: number | null; - total_steps?: number | null; - calculation?: unknown; - travel_times?: unknown; - result_bucket?: string | null; - result_key?: string | null; - error?: string | null; - finished_at?: string | null; - }, -): Promise { - const terminal = status === "completed" || status === "failed" || status === "cancelled"; - const finishedAt = terminal - ? snapshot?.finished_at - ? new Date(snapshot.finished_at) - : new Date() - : null; - return db - .update(simulation) - .set({ - status, - artifactsAvailable, - details: snapshot?.details ?? null, - step: snapshot?.step ?? null, - stepIndex: snapshot?.step_index ?? null, - totalSteps: snapshot?.total_steps ?? null, - calculation: snapshot?.calculation ?? null, - travelTimes: snapshot?.travel_times ?? null, - resultBucket: snapshot?.result_bucket ?? null, - resultKey: snapshot?.result_key ?? null, - error: snapshot?.error ?? null, - finishedAt, - }) - .where(eq(simulation.id, id)); -} diff --git a/apps/web/src/lib/server/submit-simulation.test.ts b/apps/web/src/lib/server/submit-simulation.test.ts new file mode 100644 index 0000000..d11953f --- /dev/null +++ b/apps/web/src/lib/server/submit-simulation.test.ts @@ -0,0 +1,88 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + clearSubmissionFailure: vi.fn(), + recordSubmissionFailure: vi.fn(), +})); + +import { submitSimulation } from "./submit-simulation"; + +const SIM = { + id: "sim-1", + params: { Mw: 8.0 }, + submissionError: "earlier failure", +}; + +function client() { + return { POST: vi.fn() }; +} + +function repository() { + return { + clearSubmissionFailure: mocks.clearSubmissionFailure, + recordSubmissionFailure: mocks.recordSubmissionFailure, + }; +} + +describe("submitSimulation", () => { + beforeEach(() => { + vi.resetAllMocks(); + mocks.clearSubmissionFailure.mockResolvedValue(undefined); + mocks.recordSubmissionFailure.mockResolvedValue(undefined); + }); + + it("submits with the simulation id and clears an earlier failure", async () => { + const api = client(); + api.POST.mockResolvedValue({ data: { simulation_id: "sim-1" }, error: undefined }); + + await expect(submitSimulation(SIM, api as never, repository())).resolves.toEqual({ ok: true }); + + expect(api.POST).toHaveBeenCalledWith("/api/v1/jobs", { + body: { simulation_id: "sim-1", input: SIM.params }, + }); + expect(mocks.clearSubmissionFailure).toHaveBeenCalledWith("sim-1"); + expect(mocks.recordSubmissionFailure).not.toHaveBeenCalled(); + }); + + it("does not update a new simulation after successful submission", async () => { + const api = client(); + api.POST.mockResolvedValue({ data: { simulation_id: "sim-1" }, error: undefined }); + + await expect( + submitSimulation({ id: SIM.id, params: SIM.params }, api as never, repository()), + ).resolves.toEqual({ ok: true }); + + expect(mocks.clearSubmissionFailure).not.toHaveBeenCalled(); + expect(mocks.recordSubmissionFailure).not.toHaveBeenCalled(); + }); + + it("records the compute API detail when submission is rejected", async () => { + const api = client(); + api.POST.mockResolvedValue({ + data: undefined, + error: { detail: "same simulation id has different input" }, + }); + + await expect(submitSimulation(SIM, api as never, repository())).resolves.toEqual({ + ok: false, + error: "same simulation id has different input", + }); + + expect(mocks.recordSubmissionFailure).toHaveBeenCalledWith( + "sim-1", + "same simulation id has different input", + ); + }); + + it("records transport failures for the researcher", async () => { + const api = client(); + api.POST.mockRejectedValue(new Error("compute unavailable")); + + await expect(submitSimulation(SIM, api as never, repository())).resolves.toEqual({ + ok: false, + error: "compute unavailable", + }); + + expect(mocks.recordSubmissionFailure).toHaveBeenCalledWith("sim-1", "compute unavailable"); + }); +}); diff --git a/apps/web/src/lib/server/submit-simulation.ts b/apps/web/src/lib/server/submit-simulation.ts new file mode 100644 index 0000000..43b1047 --- /dev/null +++ b/apps/web/src/lib/server/submit-simulation.ts @@ -0,0 +1,48 @@ +import type { TsdhnClient } from "@tsdhn/api-client"; + +import type { EarthquakeInput } from "$lib/schema/earthquake"; +import type { SimulationRepository } from "$lib/server/simulation-repository"; + +type SubmissionRepository = Pick< + SimulationRepository, + "clearSubmissionFailure" | "recordSubmissionFailure" +>; + +function errorMessage(error: unknown): string { + if (error instanceof Error && error.message) return error.message; + return "No se pudo enviar la simulación al servicio de cálculo."; +} + +export async function submitSimulation( + sim: { id: string; params: unknown; submissionError?: string | null }, + client: TsdhnClient, + repository: SubmissionRepository, +): Promise<{ ok: true } | { ok: false; error: string }> { + let response: Awaited>; + + try { + response = await client.POST("/api/v1/jobs", { + body: { + simulation_id: sim.id, + input: sim.params as EarthquakeInput, + }, + }); + } catch (error) { + const message = errorMessage(error); + await repository.recordSubmissionFailure(sim.id, message); + return { ok: false, error: message }; + } + + const { data, error } = response; + if (error || !data) { + const message = + typeof error === "object" && error && "detail" in error + ? String(error.detail) + : "No se pudo enviar la simulación al servicio de cálculo."; + await repository.recordSubmissionFailure(sim.id, message); + return { ok: false, error: message }; + } + + if (sim.submissionError) await repository.clearSubmissionFailure(sim.id); + return { ok: true }; +} diff --git a/apps/web/src/routes/(app)/dashboard/+page.server.ts b/apps/web/src/routes/(app)/dashboard/+page.server.ts index b5f73fc..831d06a 100644 --- a/apps/web/src/routes/(app)/dashboard/+page.server.ts +++ b/apps/web/src/routes/(app)/dashboard/+page.server.ts @@ -1,32 +1,10 @@ import { error } from "@sveltejs/kit"; -import { backend } from "$lib/server/api"; -import { listSimulations, syncStatus } from "$lib/server/simulations"; - import type { PageServerLoad } from "./$types"; -export const load: PageServerLoad = async ({ locals, fetch }) => { +export const load: PageServerLoad = async ({ locals }) => { const user = locals.user; if (!user) error(401); - const simulations = await listSimulations(user.id); - const client = backend(fetch); - - await Promise.all( - simulations - .filter((s) => s.computeJobId && (s.status === "queued" || s.status === "running")) - .map(async (s) => { - const { data } = await client.GET("/api/v1/jobs/{app_job_id}", { - params: { path: { app_job_id: s.id } }, - }); - if (data) { - await syncStatus(s.id, data.status, data.artifacts_available, data); - s.status = data.status; - s.artifactsAvailable = data.artifacts_available; - s.error = data.error ?? null; - } - }), - ); - - return { simulations }; + return { simulations: await locals.simulationRepository.listSimulations(user.id) }; }; diff --git a/apps/web/src/routes/(app)/dashboard/+page.svelte b/apps/web/src/routes/(app)/dashboard/+page.svelte index 46ae92c..2c2753c 100644 --- a/apps/web/src/routes/(app)/dashboard/+page.svelte +++ b/apps/web/src/routes/(app)/dashboard/+page.svelte @@ -5,8 +5,8 @@ let { data } = $props(); const STATUS: Record = { - pending_dispatch: { label: "Preparando", class: "bg-neutral-100 text-neutral-600" }, - dispatch_failed: { label: "No enviada", class: "bg-red-100 text-red-700" }, + submitting: { label: "Enviando", class: "bg-neutral-100 text-neutral-600" }, + submission_failed: { label: "No enviada", class: "bg-red-100 text-red-700" }, queued: { label: "En cola", class: "bg-neutral-100 text-neutral-600" }, running: { label: "Ejecutándose", class: "bg-brand-100 text-brand-700" }, completed: { label: "Completada", class: "bg-green-100 text-green-700" }, diff --git a/apps/web/src/routes/(app)/new/+page.server.ts b/apps/web/src/routes/(app)/new/+page.server.ts index 1cb5738..96e104f 100644 --- a/apps/web/src/routes/(app)/new/+page.server.ts +++ b/apps/web/src/routes/(app)/new/+page.server.ts @@ -3,9 +3,8 @@ import { message, superValidate } from "sveltekit-superforms"; import { zod4 } from "sveltekit-superforms/adapters"; import { defaultEarthquake, earthquakeSchema, toEarthquakeInput } from "$lib/schema/earthquake"; -import { backend } from "$lib/server/api"; -import { dispatchSimulation } from "$lib/server/dispatch"; -import { createSimulation } from "$lib/server/simulations"; +import { computeClient } from "$lib/server/compute-api"; +import { submitSimulation } from "$lib/server/submit-simulation"; import type { Actions, PageServerLoad } from "./$types"; @@ -22,21 +21,24 @@ export const actions: Actions = { if (!form.valid) return fail(400, { form }); const input = toEarthquakeInput(form.data); - const appJobId = crypto.randomUUID(); + const simulationId = crypto.randomUUID(); - await createSimulation({ - id: appJobId, + await locals.simulationRepository.createSimulation({ + id: simulationId, userId: user.id, params: input, - status: "pending_dispatch", }); - const client = backend(fetch); - const dispatch = await dispatchSimulation({ id: appJobId, params: input }, client); - if (!dispatch.ok) { + const client = computeClient(fetch); + const submission = await submitSimulation( + { id: simulationId, params: input }, + client, + locals.simulationRepository, + ); + if (!submission.ok) { return message(form, "No se pudo iniciar la simulación.", { status: 502 }); } - redirect(303, `/simulations/${appJobId}`); + redirect(303, `/simulations/${simulationId}`); }, }; diff --git a/apps/web/src/routes/(app)/simulations/[id]/+page.server.ts b/apps/web/src/routes/(app)/simulations/[id]/+page.server.ts index 8d795a3..1e62967 100644 --- a/apps/web/src/routes/(app)/simulations/[id]/+page.server.ts +++ b/apps/web/src/routes/(app)/simulations/[id]/+page.server.ts @@ -1,42 +1,20 @@ import { error, fail, redirect } from "@sveltejs/kit"; -import { backend } from "$lib/server/api"; -import { dispatchSimulation } from "$lib/server/dispatch"; -import { getSimulation, syncStatus } from "$lib/server/simulations"; +import { computeClient } from "$lib/server/compute-api"; +import { submitSimulation } from "$lib/server/submit-simulation"; import type { Actions, PageServerLoad } from "./$types"; -const TERMINAL = new Set(["completed", "failed", "dispatch_failed", "cancelled"]); -const RETRYABLE = new Set(["pending_dispatch", "dispatch_failed"]); +const RETRYABLE = new Set(["submitting", "submission_failed"]); -export const load: PageServerLoad = async ({ params, locals, fetch }) => { +export const load: PageServerLoad = async ({ params, locals }) => { const user = locals.user; if (!user) error(401); - const sim = await getSimulation(user.id, params.id); + const sim = await locals.simulationRepository.getSimulation(user.id, params.id); if (!sim) error(404, "Simulación no encontrada"); - if (TERMINAL.has(sim.status)) return { sim, status: null }; - if (!sim.computeJobId) return { sim, status: null }; - - const client = backend(fetch); - const { data: status } = await client.GET("/api/v1/jobs/{app_job_id}", { - params: { path: { app_job_id: sim.id } }, - }); - if (status) { - await syncStatus(sim.id, status.status, status.artifacts_available, status); - sim.status = status.status; - sim.artifactsAvailable = status.artifacts_available; - sim.details = status.details ?? null; - sim.step = status.step ?? null; - sim.stepIndex = status.step_index ?? null; - sim.totalSteps = status.total_steps ?? null; - sim.calculation = status.calculation ?? null; - sim.travelTimes = status.travel_times ?? null; - sim.error = status.error ?? null; - } - - return { sim, status: status ?? null }; + return { sim }; }; export const actions: Actions = { @@ -44,16 +22,16 @@ export const actions: Actions = { const user = locals.user; if (!user) error(401); - const sim = await getSimulation(user.id, params.id); + const sim = await locals.simulationRepository.getSimulation(user.id, params.id); if (!sim) error(404, "Simulación no encontrada"); if (!RETRYABLE.has(sim.status)) { return fail(400, { retryError: "Esta simulación ya no se puede reenviar." }); } - const client = backend(fetch); - const dispatch = await dispatchSimulation(sim, client, sim.computeBackend ?? undefined); - if (!dispatch.ok) return fail(502, { retryError: dispatch.error }); + const client = computeClient(fetch); + const submission = await submitSimulation(sim, client, locals.simulationRepository); + if (!submission.ok) return fail(502, { retryError: submission.error }); redirect(303, `/simulations/${sim.id}`); }, diff --git a/apps/web/src/routes/(app)/simulations/[id]/+page.svelte b/apps/web/src/routes/(app)/simulations/[id]/+page.svelte index 144016c..f604e9f 100644 --- a/apps/web/src/routes/(app)/simulations/[id]/+page.svelte +++ b/apps/web/src/routes/(app)/simulations/[id]/+page.svelte @@ -9,37 +9,56 @@ import Button from "$lib/components/ui/Button.svelte"; let { data, form } = $props(); - type JobStatusResponse = components["schemas"]["JobStatusResponse"]; + type Calculation = components["schemas"]["CalculationResponse"]; + type JobStatusResponse = { + simulation_id: string; + status: string; + details: string | null; + step: string | null; + step_index: number | null; + total_steps: number | null; + calculation: Calculation | null; + travel_times: unknown; + error: string | null; + finished_at: string | null; + outputs: string[]; + }; const params = $derived(data.sim.params as EarthquakeInput); const simulationId = $derived(data.sim.id); function statusFromSnapshot(): JobStatusResponse { - return ( - data.status ?? { - app_job_id: data.sim.id, - compute_job_id: data.sim.computeJobId ?? "", - status: data.sim.status, - details: data.sim.details, - step: data.sim.step, - step_index: data.sim.stepIndex, - total_steps: data.sim.totalSteps, - calculation: data.sim.calculation as JobStatusResponse["calculation"], - travel_times: data.sim.travelTimes as JobStatusResponse["travel_times"], - result_bucket: data.sim.resultBucket, - result_key: data.sim.resultKey, - error: data.sim.error, - finished_at: data.sim.finishedAt?.toISOString() ?? null, - artifacts_available: data.sim.artifactsAvailable, - } - ); + return { + simulation_id: data.sim.id, + status: data.sim.status, + details: data.sim.details, + step: data.sim.step, + step_index: data.sim.stepIndex, + total_steps: data.sim.totalSteps, + calculation: data.sim.calculation as Calculation | null, + travel_times: data.sim.travelTimes, + error: data.sim.error, + finished_at: data.sim.finishedAt?.toISOString() ?? null, + outputs: data.sim.outputs, + }; } + const OUTPUT_LABEL: Record = { + max_height_map: "Mapa de altura máxima (PDF)", + arrival_time_map: "Mapa de tiempos de arribo (PDF)", + mareogram: "Mareograma (SVG)", + calculation: "Parámetros de la fuente (JSON)", + travel_times_json: "Tiempos de arribo (JSON)", + travel_times_csv: "Tiempos de arribo (CSV)", + input: "Parámetros de entrada (JSON)", + runtime: "Entorno de ejecución (JSON)", + }; + function isTerminal(status: string): boolean { return ( status === "completed" || status === "failed" || - status === "dispatch_failed" || + status === "submission_failed" || status === "cancelled" ); } @@ -47,7 +66,7 @@ let live = $state(statusFromSnapshot()); const terminal = $derived(isTerminal(live.status)); - const retryable = $derived(live.status === "pending_dispatch" || live.status === "dispatch_failed"); + const retryable = $derived(live.status === "submitting" || live.status === "submission_failed"); const percent = $derived( live.step_index !== null && live.step_index !== undefined && live.total_steps ? Math.round((live.step_index / live.total_steps) * 100) @@ -59,8 +78,8 @@ ); const STATUS_LABEL: Record = { - pending_dispatch: "Preparando", - dispatch_failed: "No enviada", + submitting: "Enviando", + submission_failed: "No enviada", queued: "En cola", running: "Ejecutándose", completed: "Completada", @@ -73,7 +92,7 @@ const initial = statusFromSnapshot(); live = initial; - if (isTerminal(initial.status) || !data.sim.computeJobId) return; + if (isTerminal(initial.status) || retryable) return; const es = new EventSource(`/simulations/${id}/events`); es.onmessage = (event) => { @@ -85,7 +104,7 @@ void invalidateAll(); } } catch { - /* ignore malformed frames */ + // Keepalive frames are not JSON status updates. } }; es.onerror = () => es.close(); @@ -130,7 +149,7 @@ {/if} - {#if live.status === "failed" || live.status === "dispatch_failed"} + {#if live.status === "failed" || live.status === "submission_failed"} {live.error ?? "Error desconocido."} @@ -155,10 +174,22 @@ {/if} - {#if live.status === "completed" && live.artifacts_available} - - Los resultados estructurados y mapas fueron generados por el backend. - + {#if live.status === "completed" && live.outputs.length > 0} +
+

Resultados

+ +
{/if} diff --git a/apps/web/src/routes/(app)/simulations/[id]/events/+server.ts b/apps/web/src/routes/(app)/simulations/[id]/events/+server.ts index a754ddb..bff5f10 100644 --- a/apps/web/src/routes/(app)/simulations/[id]/events/+server.ts +++ b/apps/web/src/routes/(app)/simulations/[id]/events/+server.ts @@ -1,18 +1,18 @@ import { error } from "@sveltejs/kit"; -import { backendRaw } from "$lib/server/api"; -import { getSimulation } from "$lib/server/simulations"; +import { computeRequestConfig } from "$lib/server/compute-api"; import type { RequestHandler } from "./$types"; -/** Proxy backend progress only after the simulation owner is verified. */ export const GET: RequestHandler = async ({ params, locals, fetch }) => { if (!locals.user) error(401); - const sim = await getSimulation(locals.user.id, params.id); + const sim = await locals.simulationRepository.getSimulation(locals.user.id, params.id); if (!sim) error(404); - if (!sim.computeJobId) error(409, "La simulación aún no fue aceptada por el backend."); + if (sim.status === "submitting" || sim.status === "submission_failed") { + error(409, "La simulación aún no fue aceptada por el servicio de cálculo."); + } - const { url, headers } = backendRaw(); + const { url, headers } = computeRequestConfig(); const upstream = await fetch(`${url}/api/v1/jobs/${encodeURIComponent(sim.id)}/events`, { headers, }); diff --git a/apps/web/src/routes/(app)/simulations/[id]/outputs/[name]/+server.ts b/apps/web/src/routes/(app)/simulations/[id]/outputs/[name]/+server.ts new file mode 100644 index 0000000..0be8f40 --- /dev/null +++ b/apps/web/src/routes/(app)/simulations/[id]/outputs/[name]/+server.ts @@ -0,0 +1,26 @@ +import { error, redirect } from "@sveltejs/kit"; + +import { computeRequestConfig } from "$lib/server/compute-api"; +import { assertOutputAccessible } from "$lib/server/outputs"; + +import type { RequestHandler } from "./$types"; + +export const GET: RequestHandler = async ({ params, locals, fetch }) => { + if (!locals.user) error(401); + + const sim = await locals.simulationRepository.getSimulation(locals.user.id, params.id); + assertOutputAccessible(sim, params.name); + + const { url, headers } = computeRequestConfig(); + const upstream = await fetch( + `${url}/api/v1/jobs/${encodeURIComponent(sim.id)}/outputs/${encodeURIComponent(params.name)}`, + { headers, redirect: "manual" }, + ); + + const location = upstream.headers.get("location"); + if (upstream.status !== 307 || !location) { + error(502, "No se pudo preparar la descarga."); + } + + redirect(302, location); +}; diff --git a/apps/web/src/routes/+page.svelte b/apps/web/src/routes/+page.svelte index c393c99..b6cdfe3 100644 --- a/apps/web/src/routes/+page.svelte +++ b/apps/web/src/routes/+page.svelte @@ -1,2 +1 @@ -

Redirigiendo…

diff --git a/apps/web/src/routes/api/calculations/+server.ts b/apps/web/src/routes/api/calculations/+server.ts index b053600..89e7252 100644 --- a/apps/web/src/routes/api/calculations/+server.ts +++ b/apps/web/src/routes/api/calculations/+server.ts @@ -1,6 +1,6 @@ import { error, json } from "@sveltejs/kit"; -import { backend } from "$lib/server/api"; +import { computeClient } from "$lib/server/compute-api"; import type { RequestHandler } from "./$types"; @@ -8,7 +8,7 @@ export const POST: RequestHandler = async ({ request, locals, fetch }) => { if (!locals.user) error(401); const body = await request.json(); - const client = backend(fetch); + const client = computeClient(fetch); const { data, error: apiError } = await client.POST("/api/v1/calculations", { body }); if (apiError || !data) error(502, "No se pudo calcular la vista previa."); diff --git a/apps/web/src/routes/app-route-loads.test.ts b/apps/web/src/routes/app-route-loads.test.ts new file mode 100644 index 0000000..161e7d7 --- /dev/null +++ b/apps/web/src/routes/app-route-loads.test.ts @@ -0,0 +1,81 @@ +import { isHttpError } from "@sveltejs/kit"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + computeClient: vi.fn(), + submitSimulation: vi.fn(), + listSimulations: vi.fn(), + getSimulation: vi.fn(), +})); + +vi.mock("$lib/server/compute-api", () => ({ computeClient: mocks.computeClient })); +vi.mock("$lib/server/submit-simulation", () => ({ + submitSimulation: mocks.submitSimulation, +})); + +import { load as dashboardLoad } from "./(app)/dashboard/+page.server"; +import { load as simulationLoad } from "./(app)/simulations/[id]/+page.server"; + +async function expectHttpError(action: unknown, status: number) { + try { + await action; + throw new Error("expected load to throw"); + } catch (caught) { + expect(isHttpError(caught)).toBe(true); + if (isHttpError(caught)) expect(caught.status).toBe(status); + } +} + +function context(overrides: Record = {}) { + return { + params: { id: "sim-1" }, + locals: { + user: { id: "user-1" }, + simulationRepository: { + listSimulations: mocks.listSimulations, + getSimulation: mocks.getSimulation, + }, + }, + ...overrides, + }; +} + +describe("dashboard load", () => { + beforeEach(() => vi.resetAllMocks()); + + it("requires authentication before reading simulations", async () => { + await expectHttpError(dashboardLoad({ locals: { user: null } } as never), 401); + + expect(mocks.listSimulations).not.toHaveBeenCalled(); + }); + + it("returns only the signed-in user's simulations", async () => { + const simulations = [{ id: "sim-1" }]; + mocks.listSimulations.mockResolvedValue(simulations); + + const result = await dashboardLoad(context() as never); + + expect(result).toEqual({ simulations }); + expect(mocks.listSimulations).toHaveBeenCalledWith("user-1"); + }); +}); + +describe("simulation detail load", () => { + beforeEach(() => vi.resetAllMocks()); + + it("returns the owned simulation", async () => { + const simulation = { id: "sim-1", status: "running" }; + mocks.getSimulation.mockResolvedValue(simulation); + + const result = await simulationLoad(context() as never); + + expect(result).toEqual({ sim: simulation }); + expect(mocks.getSimulation).toHaveBeenCalledWith("user-1", "sim-1"); + }); + + it("returns not found when the simulation is not owned by the user", async () => { + mocks.getSimulation.mockResolvedValue(undefined); + + await expectHttpError(simulationLoad(context() as never), 404); + }); +}); diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index af609a3..3d7b4f5 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -12,7 +12,7 @@ export default defineConfig({ tailwindcss(), sveltekit({ compilerOptions: { - // Project components use runes while dependencies keep their own Svelte mode. + // Enable runes for project files, not for dependencies. runes: ({ filename }) => filename.split(/[/\\]/).includes("node_modules") ? undefined : true, }, diff --git a/apps/web/vitest.config.ts b/apps/web/vitest.config.ts new file mode 100644 index 0000000..d809172 --- /dev/null +++ b/apps/web/vitest.config.ts @@ -0,0 +1,28 @@ +import path from "node:path"; + +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + resolve: { + alias: { + $lib: path.resolve(__dirname, "src/lib"), + }, + }, + test: { + environment: "node", + include: ["src/**/*.test.ts"], + coverage: { + provider: "v8", + reporter: ["text", "lcov"], + include: ["src/**/*.ts"], + exclude: [ + "src/**/*.test.ts", + "src/lib/server/db/auth.schema.ts", + // Database-backed routes are covered by integration tests. + "src/lib/server/db/index.ts", + "src/hooks.server.ts", + "src/lib/server/simulation-repository.ts", + ], + }, + }, +}); diff --git a/bun.lock b/bun.lock index aa13672..294ad22 100644 --- a/bun.lock +++ b/bun.lock @@ -24,7 +24,6 @@ }, "devDependencies": { "@better-auth/drizzle-adapter": "~1.6.23", - "@libsql/client": "^0.17.4", "@sveltejs/adapter-auto": "^7.0.1", "@sveltejs/adapter-node": "^5.5.7", "@sveltejs/kit": "^2.69.1", @@ -34,16 +33,19 @@ "@tailwindcss/vite": "^4.3.2", "@types/geojson": "^7946.0.16", "@types/node": "^26.1.0", + "@vitest/coverage-v8": "^4.1.11", "auth": "~1.6.23", "better-auth": "~1.6.23", "drizzle-kit": "^0.31.10", "drizzle-orm": "^0.45.2", "mdsvex": "^0.12.7", + "postgres": "^3.4.7", "svelte": "^5.56.4", "svelte-check": "^4.7.2", "tailwindcss": "^4.3.2", "typescript": "^6.0.3", "vite": "^8.1.3", + "vitest": "^4.1.11", }, }, "libs/api-client": { @@ -127,6 +129,8 @@ "@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], + "@bcoe/v8-coverage": ["@bcoe/v8-coverage@1.0.2", "", {}, "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA=="], + "@better-auth/core": ["@better-auth/core@1.6.23", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.39.0", "@standard-schema/spec": "^1.1.0", "zod": "^4.3.6" }, "peerDependencies": { "@better-auth/utils": "0.4.2", "@better-fetch/fetch": "1.3.1", "@cloudflare/workers-types": ">=4", "@opentelemetry/api": "^1.9.0", "better-call": "1.3.7", "jose": "^6.1.0", "kysely": "^0.28.5 || ^0.29.0", "nanostores": "^1.0.1" }, "optionalPeers": ["@cloudflare/workers-types", "@opentelemetry/api"] }, "sha512-beEhOs0uVeOxYOZKUfIEBd/nQV2Bd4/6wyLxZ0OFkn6CMTK2Vi+hXuZLnyPBeB6RdHpebEoJWiHqwHxBIxgPDQ=="], "@better-auth/drizzle-adapter": ["@better-auth/drizzle-adapter@1.6.23", "", { "peerDependencies": { "@better-auth/core": "^1.6.23", "@better-auth/utils": "0.4.2", "drizzle-orm": "^0.45.2" }, "optionalPeers": ["drizzle-orm"] }, "sha512-2+/PTVfIP9E7iz6af8TB3lhnowHUj9ljC66kECmHaFEdUqPgzHoWux9epotKwO7XDg2ui4ttWQ8CMeNFLvQeKQ=="], @@ -545,8 +549,12 @@ "@tybys/wasm-util": ["@tybys/wasm-util@0.10.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg=="], + "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], + "@types/cookie": ["@types/cookie@0.6.0", "", {}, "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA=="], + "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], + "@types/estree": ["@types/estree@1.0.9", "", {}, "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg=="], "@types/geojson": ["@types/geojson@7946.0.16", "", {}, "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg=="], @@ -577,6 +585,22 @@ "@vinejs/vine": ["@vinejs/vine@3.0.1", "", { "dependencies": { "@poppinss/macroable": "^1.0.4", "@types/validator": "^13.12.2", "@vinejs/compiler": "^3.0.0", "camelcase": "^8.0.0", "dayjs": "^1.11.13", "dlv": "^1.1.3", "normalize-url": "^8.0.1", "validator": "^13.12.0" } }, "sha512-ZtvYkYpZOYdvbws3uaOAvTFuvFXoQGAtmzeiXu+XSMGxi5GVsODpoI9Xu9TplEMuD/5fmAtBbKb9cQHkWkLXDQ=="], + "@vitest/coverage-v8": ["@vitest/coverage-v8@4.1.11", "", { "dependencies": { "@bcoe/v8-coverage": "^1.0.2", "@vitest/utils": "4.1.11", "ast-v8-to-istanbul": "^1.0.0", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", "istanbul-reports": "^3.2.0", "magicast": "^0.5.2", "obug": "^2.1.1", "std-env": "^4.0.0-rc.1", "tinyrainbow": "^3.1.0" }, "peerDependencies": { "@vitest/browser": "4.1.11", "vitest": "4.1.11" }, "optionalPeers": ["@vitest/browser"] }, "sha512-8MVGEFnJIcdGjcbfKmeq8z0pZHH0JlVtoVZH9Q/qwUp6wyFnEJUBMrw9DCaj+ra3vShGmhavjalMIhPNxZAUcw=="], + + "@vitest/expect": ["@vitest/expect@4.1.11", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.1.11", "@vitest/utils": "4.1.11", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" } }, "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw=="], + + "@vitest/mocker": ["@vitest/mocker@4.1.11", "", { "dependencies": { "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["msw", "vite"] }, "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ=="], + + "@vitest/pretty-format": ["@vitest/pretty-format@4.1.11", "", { "dependencies": { "tinyrainbow": "^3.1.0" } }, "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw=="], + + "@vitest/runner": ["@vitest/runner@4.1.11", "", { "dependencies": { "@vitest/utils": "4.1.11", "pathe": "^2.0.3" } }, "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw=="], + + "@vitest/snapshot": ["@vitest/snapshot@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" } }, "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog=="], + + "@vitest/spy": ["@vitest/spy@4.1.11", "", {}, "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA=="], + + "@vitest/utils": ["@vitest/utils@4.1.11", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ=="], + "acorn": ["acorn@8.17.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg=="], "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], @@ -591,6 +615,10 @@ "arktype": ["arktype@2.2.3", "", { "dependencies": { "@ark/schema": "0.56.2", "@ark/util": "0.56.2", "arkregex": "0.0.8" } }, "sha512-7W+0RLTUNJiBFIIZXwOQxSR8Z273IAd6IvqBeG9+gHnQKFsIx2C0iOtGTmMrPnlX4qLXyc5+ll7A0BIj9WrbTg=="], + "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], + + "ast-v8-to-istanbul": ["ast-v8-to-istanbul@1.0.5", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.31", "estree-walker": "^3.0.3", "js-tokens": "^10.0.0" } }, "sha512-UPAgKJFSEGMWSDr3LX4tqnAb4f7KGT8O40Tyx8wbYmmZ/yn58lNCm8h3svs3eXgiGd5AXxz8NDOvXWvicq+rJA=="], + "auth": ["auth@1.6.23", "", { "dependencies": { "@babel/core": "^7.29.0", "@babel/preset-react": "^7.28.5", "@babel/preset-typescript": "^7.28.5", "@better-auth/core": "1.6.23", "@better-auth/telemetry": "1.6.23", "@better-auth/utils": "0.4.2", "@clack/prompts": "^0.11.0", "@mrleebo/prisma-ast": "^0.13.1", "better-auth": "1.6.23", "c12": "^3.3.3", "chalk": "^5.6.2", "commander": "^12.1.0", "dotenv": "^17.3.1", "get-tsconfig": "^4.13.6", "open": "^10.2.0", "prettier": "^3.8.1", "prompts": "^2.4.2", "semver": "^7.7.4", "yocto-spinner": "^0.2.3", "zod": "^4.3.6" }, "bin": { "auth": "./dist/index.mjs", "better-auth": "./dist/index.mjs" } }, "sha512-HJHK9nCD5/7ZCfy7k+0lPY3miNDAt9erDOkWPn6y2U+FRkdjVS74sZGkZEfhmK7jaBVJ2d9B8AvY1C5/thHLSA=="], "axobject-query": ["axobject-query@4.1.0", "", {}, "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ=="], @@ -629,6 +657,8 @@ "caniuse-lite": ["caniuse-lite@1.0.30001799", "", {}, "sha512-hG1bReV+OUU+MOqK4t/ZWI0tZOyz3rqS9XuhOUz1cIcbwBKjOyJEJuw9ER5JuNyqxNk8u/JUVbGibBOL1yrjFw=="], + "chai": ["chai@6.2.2", "", {}, "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg=="], + "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="], "change-case": ["change-case@5.4.4", "", {}, "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w=="], @@ -679,7 +709,7 @@ "destr": ["destr@2.0.5", "", {}, "sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA=="], - "detect-libc": ["detect-libc@2.0.2", "", {}, "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw=="], + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], "devalue": ["devalue@5.8.1", "", {}, "sha512-4CXDYRBGqN+57wVJkuXBYmpAVUSg3L6JAQa/DFqm238G73E1wuyc/JhGQJzN7vUf/CMphYau2zXbfWzDR5aTEw=="], @@ -703,6 +733,8 @@ "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], + "es-module-lexer": ["es-module-lexer@2.3.2", "", {}, "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw=="], + "esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], @@ -715,6 +747,8 @@ "expand-template": ["expand-template@2.0.3", "", {}, "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg=="], + "expect-type": ["expect-type@1.4.0", "", {}, "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA=="], + "exsolve": ["exsolve@1.1.0", "", {}, "sha512-D+42+T12DdIlJM3uepa55qGiL3sYdLBOxIl2ifQCzCHz4c7eiolaHsi3BIqEr7JxBzxv2pYZQX9kw16ziMcEmw=="], "fast-check": ["fast-check@3.23.2", "", { "dependencies": { "pure-rand": "^6.1.0" } }, "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A=="], @@ -745,8 +779,12 @@ "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], + "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], + "hasown": ["hasown@2.0.4", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A=="], + "html-escaper": ["html-escaper@2.0.2", "", {}, "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg=="], + "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], @@ -771,6 +809,12 @@ "is-wsl": ["is-wsl@3.1.1", "", { "dependencies": { "is-inside-container": "^1.0.0" } }, "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw=="], + "istanbul-lib-coverage": ["istanbul-lib-coverage@3.2.2", "", {}, "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg=="], + + "istanbul-lib-report": ["istanbul-lib-report@3.0.1", "", { "dependencies": { "istanbul-lib-coverage": "^3.0.0", "make-dir": "^4.0.0", "supports-color": "^7.1.0" } }, "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw=="], + + "istanbul-reports": ["istanbul-reports@3.2.0", "", { "dependencies": { "html-escaper": "^2.0.0", "istanbul-lib-report": "^3.0.0" } }, "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA=="], + "jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="], "joi": ["joi@17.13.4", "", { "dependencies": { "@hapi/hoek": "^9.3.0", "@hapi/topo": "^5.1.0", "@sideway/address": "^4.1.5", "@sideway/formula": "^3.0.1", "@sideway/pinpoint": "^2.0.0" } }, "sha512-1RuuER6kmt8K8I3nIWvPZKi5RQCb568ZPyY4Pwjlua+yo+63ZTmIwxLZH0heBmiKN4uxjvCiarDrjaeH84xicQ=="], @@ -781,7 +825,7 @@ "js-levenshtein": ["js-levenshtein@1.1.6", "", {}, "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g=="], - "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + "js-tokens": ["js-tokens@10.0.0", "", {}, "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q=="], "js-yaml": ["js-yaml@4.2.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw=="], @@ -841,6 +885,10 @@ "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], + "magicast": ["magicast@0.5.4", "", { "dependencies": { "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7", "source-map-js": "^1.2.1" } }, "sha512-llBEhWm1SacoRwgHUoQJYtwp4PBLF4faQi5TCpIGyGs9n4y5+juI0tDgyKIfpqxckRHaHzouUEph3THklWh03w=="], + + "make-dir": ["make-dir@4.0.0", "", { "dependencies": { "semver": "^7.5.3" } }, "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw=="], + "maplibre-gl": ["maplibre-gl@5.24.0", "", { "dependencies": { "@mapbox/jsonlint-lines-primitives": "^2.0.2", "@mapbox/point-geometry": "^1.1.0", "@mapbox/tiny-sdf": "^2.1.0", "@mapbox/unitbezier": "^0.0.1", "@mapbox/vector-tile": "^2.0.4", "@mapbox/whoots-js": "^3.1.0", "@maplibre/geojson-vt": "^6.1.0", "@maplibre/maplibre-gl-style-spec": "^24.8.1", "@maplibre/mlt": "^1.1.8", "@maplibre/vt-pbf": "^4.3.0", "@types/geojson": "^7946.0.16", "earcut": "^3.0.2", "gl-matrix": "^3.4.4", "kdbush": "^4.0.2", "murmurhash-js": "^1.0.0", "pbf": "^4.0.1", "potpack": "^2.1.0", "quickselect": "^3.0.0", "tinyqueue": "^3.0.0" } }, "sha512-ALyFxgtd5R+65UqZ/++lOqwWcC0SNho9c27fYSyLmG7AfnAul2o46F05aDJGPbFU57wos9dgcIySHs0Xe6ia3A=="], "mdsvex": ["mdsvex@0.12.7", "", { "dependencies": { "@types/mdast": "^4.0.4", "@types/unist": "^2.0.3", "prism-svelte": "^0.4.7", "prismjs": "^1.17.1", "unist-util-visit": "^2.0.1", "vfile-message": "^2.0.4" }, "peerDependencies": { "svelte": "^3.56.0 || ^4.0.0 || ^5.0.0-next.120" } }, "sha512-gx4bReLCUvq+MPErHXYeyX+TEq1hsS2KfiZtEOMNTcbibSouFy8AHc5h04KbGCl+g5tLuo4/lbgRVYRnc7bJZw=="], @@ -933,6 +981,8 @@ "postcss-selector-parser": ["postcss-selector-parser@6.0.10", "", { "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" } }, "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w=="], + "postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="], + "postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="], "postgres-bytea": ["postgres-bytea@1.0.1", "", {}, "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ=="], @@ -1001,6 +1051,8 @@ "set-cookie-parser": ["set-cookie-parser@3.1.1", "", {}, "sha512-vM9SUhjsUYs6UeJUmygc5Ofm5eQGe85riob5ju6XCgFGJI5PLV4nrDAQpQjd+LkFBpAkADn5BQQpZ9EUNkyLuA=="], + "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], + "simple-concat": ["simple-concat@1.0.1", "", {}, "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q=="], "simple-get": ["simple-get@4.0.1", "", { "dependencies": { "decompress-response": "^6.0.0", "once": "^1.3.1", "simple-concat": "^1.0.0" } }, "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA=="], @@ -1017,6 +1069,10 @@ "split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="], + "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], + + "std-env": ["std-env@4.2.0", "", {}, "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw=="], + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], "strip-json-comments": ["strip-json-comments@2.0.1", "", {}, "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ=="], @@ -1051,12 +1107,18 @@ "tiny-case": ["tiny-case@1.0.3", "", {}, "sha512-Eet/eeMhkO6TX8mnUteS9zgPbUMQa4I6Kkp5ORiBD5476/m+PIRiumP5tmh5ioJpH7k51Kehawy2UDfsnxxY8Q=="], + "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], + + "tinyexec": ["tinyexec@1.3.0", "", {}, "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ=="], + "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], "tinypool": ["tinypool@2.1.0", "", {}, "sha512-Pugqs6M0m7Lv1I7FtxN4aoyToKg1C4tu+/381vH35y8oENM/Ai7f7C4StcoK4/+BSw9ebcS8jRiVrORFKCALLw=="], "tinyqueue": ["tinyqueue@3.0.0", "", {}, "sha512-gRa9gwYU3ECmQYv3lslts5hxuIa90veaEcxDYuu3QGOIAEM2mOZkVHp48ANJuu1CURtRdHKUBY5Lm1tHV+sD4g=="], + "tinyrainbow": ["tinyrainbow@3.1.1", "", {}, "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw=="], + "toposort": ["toposort@2.0.2", "", {}, "sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg=="], "totalist": ["totalist@3.0.1", "", {}, "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ=="], @@ -1103,8 +1165,12 @@ "vitefu": ["vitefu@1.1.3", "", { "peerDependencies": { "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["vite"] }, "sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg=="], + "vitest": ["vitest@4.1.11", "", { "dependencies": { "@vitest/expect": "4.1.11", "@vitest/mocker": "4.1.11", "@vitest/pretty-format": "4.1.11", "@vitest/runner": "4.1.11", "@vitest/snapshot": "4.1.11", "@vitest/spy": "4.1.11", "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", "obug": "^2.1.1", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^4.0.0-rc.1", "tinybench": "^2.9.0", "tinyexec": "^1.0.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.1.0", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.1.11", "@vitest/browser-preview": "4.1.11", "@vitest/browser-webdriverio": "4.1.11", "@vitest/coverage-istanbul": "4.1.11", "@vitest/coverage-v8": "4.1.11", "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@opentelemetry/api", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/coverage-istanbul", "@vitest/coverage-v8", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "./vitest.mjs" } }, "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw=="], + "web": ["web@workspace:apps/web"], + "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], "ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="], @@ -1131,6 +1197,8 @@ "zod-v3-to-json-schema": ["zod-v3-to-json-schema@4.0.0", "", { "peerDependencies": { "zod": "^3.25 || ^4.0.14" } }, "sha512-KixLrhX/uPmRFnDgsZrzrk4x5SSJA+PmaE5adbfID9+3KPJcdxqRobaHU397EfWBqfQircrjKqvEqZ/mW5QH6w=="], + "@babel/code-frame/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + "@babel/core/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], "@babel/helper-compilation-targets/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], @@ -1161,13 +1229,17 @@ "@types/ws/@types/node": ["@types/node@24.13.2", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA=="], + "@vitest/mocker/estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], + + "ast-v8-to-istanbul/estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], + "c12/chokidar": ["chokidar@5.0.0", "", { "dependencies": { "readdirp": "^5.0.0" } }, "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw=="], "formsnap/svelte-toolbelt": ["svelte-toolbelt@0.5.0", "", { "dependencies": { "clsx": "^2.1.1", "style-to-object": "^1.0.8" }, "peerDependencies": { "svelte": "^5.0.0-next.126" } }, "sha512-t3tenZcnfQoIeRuQf/jBU7bvTeT3TGkcEE+1EUr5orp0lR7NEpprflpuie3x9Dn0W9nOKqs3HwKGJeeN5Ok1sQ=="], - "lightningcss/detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + "istanbul-lib-report/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], - "prebuild-install/detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + "libsql/detect-libc": ["detect-libc@2.0.2", "", {}, "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw=="], "prompts/kleur": ["kleur@3.0.3", "", {}, "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="], diff --git a/deploy/api.Dockerfile b/deploy/api.Dockerfile index 15d8ac6..b6ae915 100644 --- a/deploy/api.Dockerfile +++ b/deploy/api.Dockerfile @@ -1,8 +1,5 @@ -# FastAPI and the Procrastinate worker run in this image for the self-hosted backend. -# -# Builds on the TSDHN toolchain base, which provides the scientific runtime: -# GMT, Intel Fortran, and ttt_client. The base ships Python 3.12 for -# system tooling; the app uses uv-managed Python 3.14. +# Compute API and worker image. +# The toolchain base provides GMT, Intel Fortran, and ttt_client. # # Build context is the repo root: docker build -f deploy/api.Dockerfile . ARG TOOLCHAIN_IMAGE=ghcr.io/totallynotdavid/tsdhn-toolchain:master @@ -18,35 +15,37 @@ COPY --from=ghcr.io/astral-sh/uv:0.11.27 /uv /uvx /usr/local/bin/ WORKDIR /app -# Dependencies are installed before source changes invalidate the build cache. +# Install dependencies before copying the remaining source. COPY pyproject.toml uv.lock ./ COPY packages ./packages RUN uv python install 3.14 \ && uv sync --frozen --no-dev --package tsdhn-api # TSDHN_MODEL_DIR points at this copied model tree at runtime. +# The pipeline uses Python plus GMT and ttt_client. The compiled Fortran +# binaries are used by parity tests, not by normal simulation runs. COPY model ./model -RUN mkdir -p /app/tools \ - && ifx -parallel /app/model/fault_plane.f90 -o /app/tools/fault_plane \ - && ifx -parallel /app/model/def_oka.f -o /app/tools/deform \ - && ifx -parallel -qopenmp /app/model/tsunami1.for -o /app/tools/tsunami \ - && command -v gmt \ +RUN command -v gmt \ && command -v gs \ - && command -v ttt_client \ - && test -x /app/tools/fault_plane \ - && test -x /app/tools/deform \ - && test -x /app/tools/tsunami + && command -v ttt_client + +# Ghostscript is restricted to its allowed paths in the container. Keep its +# session files and simulation workspaces under /var/tmp. +RUN mkdir -p /var/tmp/jobs \ + && chown -R appuser:appuser /app /var/tmp/jobs ENV APP_HOST=0.0.0.0 \ APP_PORT=8000 \ - COMPUTE_DATABASE_URL=postgresql://tsdhn:tsdhn@postgres:5432/tsdhn_compute \ + COMPUTE_DATABASE_URL=postgresql://tsdhn:tsdhn@postgres:5432/tsdhn \ MINIO_ENDPOINT=minio:9000 \ MINIO_ACCESS_KEY=minioadmin \ MINIO_SECRET_KEY=minioadmin \ MINIO_BUCKET=tsdhn-results \ TSDHN_MODEL_DIR=/app/model \ - TSDHN_TOOLS_DIR=/app/tools \ - TSDHN_JOBS_DIR=/app/jobs + TSDHN_JOBS_DIR=/var/tmp/jobs \ + HOME=/var/tmp + +USER appuser EXPOSE 8000 CMD ["uv", "run", "--no-dev", "tsdhn-api"] diff --git a/deploy/web.Dockerfile b/deploy/web.Dockerfile index b6f2d04..fc8abae 100644 --- a/deploy/web.Dockerfile +++ b/deploy/web.Dockerfile @@ -1,14 +1,12 @@ -# SvelteKit runs with the Node adapter for the self-hosted web target. -# -# Edge deployments use adapter-auto and point BACKEND_URL/DATABASE_URL at the -# self-hosted services. This image is for `docker compose --profile web up`. +# SvelteKit image for the self-hosted web target. +# Edge deployments use adapter-auto and these runtime variables. # # Build context is the repo root: docker build -f deploy/web.Dockerfile . FROM oven/bun:1.3.14 WORKDIR /app -# Dependencies are installed before source changes invalidate the build cache. +# Install dependencies before copying the remaining source. COPY package.json bun.lock ./ COPY apps/web/package.json apps/web/package.json COPY libs/api-client/package.json libs/api-client/package.json @@ -17,11 +15,12 @@ RUN bun install --frozen-lockfile COPY libs ./libs COPY apps/web ./apps/web ENV ADAPTER=node -RUN DATABASE_URL=file:/tmp/tsdhn-build.db \ +# The build only needs syntactically valid runtime variables. +RUN DATABASE_URL=postgresql://build:build@127.0.0.1:5432/build \ ORIGIN=http://localhost:3000 \ BETTER_AUTH_SECRET=build-time-placeholder-not-for-runtime \ - BACKEND_URL=http://127.0.0.1:8000 \ - BACKEND_SERVICE_TOKEN=build-time-placeholder \ + COMPUTE_API_URL=http://127.0.0.1:8000 \ + COMPUTE_API_TOKEN=build-time-placeholder \ bun --filter web build WORKDIR /app/apps/web @@ -29,5 +28,5 @@ ENV HOST=0.0.0.0 \ PORT=3000 EXPOSE 3000 -# adapter-node emits build/index.js. node_modules stays available for externalized deps. +# The Node adapter builds this entry point. CMD ["bun", "./build/index.js"] diff --git a/docker-compose.yml b/docker-compose.yml index dfb407e..81d2f61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,24 +1,15 @@ -# Self-hosted backend stack for TSDHN. -# -# docker compose up -d # build/run postgres + minio + libsql + api + worker -# docker compose --profile web up # also build/run the SvelteKit app (adapter-node) -# API_IMAGE=ghcr.io/org/tsdhn-api:tag docker compose up --no-build api worker -# -# The backend is self-hosted because it needs the Fortran/GMT/TTT toolchain. -# Edge web deployments should point BACKEND_URL and DATABASE_URL at these services. - services: postgres: image: postgres:18-alpine restart: unless-stopped environment: - POSTGRES_DB: tsdhn_compute + POSTGRES_DB: tsdhn POSTGRES_USER: tsdhn POSTGRES_PASSWORD: tsdhn volumes: - postgres-data:/var/lib/postgresql healthcheck: - test: ["CMD-SHELL", "pg_isready -U tsdhn -d tsdhn_compute"] + test: ["CMD-SHELL", "pg_isready -U tsdhn -d tsdhn"] interval: 10s timeout: 3s retries: 5 @@ -41,16 +32,6 @@ services: timeout: 5s retries: 5 - libsql: - image: ghcr.io/tursodatabase/libsql-server:v0.24.32 - restart: unless-stopped - environment: - SQLD_NODE: primary - ports: - - "8080:8080" # libSQL HTTP (Drizzle / @libsql/client connect here) - volumes: - - libsql-data:/var/lib/sqld - compute-migrate: image: ${API_IMAGE:-tsdhn-api:local} build: @@ -62,16 +43,14 @@ services: postgres: condition: service_healthy environment: - COMPUTE_DATABASE_URL: postgresql://tsdhn:tsdhn@postgres:5432/tsdhn_compute - MINIO_ENDPOINT: minio:9000 - MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin} - MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin} - MINIO_BUCKET: ${MINIO_BUCKET:-tsdhn-results} + COMPUTE_DATABASE_URL: postgresql://tsdhn:tsdhn@postgres:5432/tsdhn + APP_DB_ROLE: ${APP_DB_ROLE:-tsdhn_app} + APP_DB_PASSWORD: ${APP_DB_PASSWORD:?set APP_DB_PASSWORD in .env} command: [ "sh", "-lc", - "uv run --no-dev procrastinate --app=api.core.procrastinate_app.app schema --apply && uv run --no-dev tsdhn-compute-migrate", + "uv run --no-dev tsdhn-compute-migrate && uv run --no-dev tsdhn-procrastinate-migrate", ] restart: "no" @@ -89,12 +68,13 @@ services: minio: condition: service_healthy environment: - COMPUTE_DATABASE_URL: postgresql://tsdhn:tsdhn@postgres:5432/tsdhn_compute + COMPUTE_DATABASE_URL: postgresql://tsdhn:tsdhn@postgres:5432/tsdhn MINIO_ENDPOINT: minio:9000 + MINIO_PUBLIC_ENDPOINT: ${MINIO_PUBLIC_ENDPOINT:-localhost:9000} MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin} MINIO_BUCKET: ${MINIO_BUCKET:-tsdhn-results} - BACKEND_SERVICE_TOKEN: ${BACKEND_SERVICE_TOKEN:?set BACKEND_SERVICE_TOKEN in .env} + COMPUTE_API_TOKEN: ${COMPUTE_API_TOKEN:?set COMPUTE_API_TOKEN in .env} ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-} ports: - "8000:8000" @@ -113,17 +93,40 @@ services: minio: condition: service_healthy environment: - COMPUTE_DATABASE_URL: postgresql://tsdhn:tsdhn@postgres:5432/tsdhn_compute + COMPUTE_DATABASE_URL: postgresql://tsdhn:tsdhn@postgres:5432/tsdhn MINIO_ENDPOINT: minio:9000 MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin} MINIO_BUCKET: ${MINIO_BUCKET:-tsdhn-results} TSDHN_MODEL_DIR: /app/model - TSDHN_TOOLS_DIR: /app/tools - TSDHN_JOBS_DIR: /app/jobs + TSDHN_JOBS_DIR: /var/tmp/jobs command: ["uv", "run", "--no-dev", "tsdhn-worker"] volumes: - - jobs-data:/app/jobs + - jobs-data:/var/tmp/jobs + + web-migrate: + image: ${WEB_IMAGE:-tsdhn-web:local} + profiles: ["web"] + depends_on: + compute-migrate: + condition: service_completed_successfully + environment: + DATABASE_URL: postgresql://tsdhn:tsdhn@postgres:5432/tsdhn + command: ["bun", "run", "db:migrate"] + restart: "no" + + web-grants: + image: ${API_IMAGE:-tsdhn-api:local} + profiles: ["web"] + depends_on: + web-migrate: + condition: service_completed_successfully + environment: + COMPUTE_DATABASE_URL: postgresql://tsdhn:tsdhn@postgres:5432/tsdhn + APP_DB_ROLE: ${APP_DB_ROLE:-tsdhn_app} + APP_DB_PASSWORD: ${APP_DB_PASSWORD:?set APP_DB_PASSWORD in .env} + command: ["uv", "run", "--no-dev", "tsdhn-web-grants"] + restart: "no" web: image: ${WEB_IMAGE:-tsdhn-web:local} @@ -133,12 +136,14 @@ services: dockerfile: deploy/web.Dockerfile restart: unless-stopped depends_on: - - api - - libsql + web-grants: + condition: service_completed_successfully + api: + condition: service_started environment: - BACKEND_URL: http://api:8000 - BACKEND_SERVICE_TOKEN: ${BACKEND_SERVICE_TOKEN:?set BACKEND_SERVICE_TOKEN in .env} - DATABASE_URL: http://libsql:8080 + COMPUTE_API_URL: http://api:8000 + COMPUTE_API_TOKEN: ${COMPUTE_API_TOKEN:?set COMPUTE_API_TOKEN in .env} + DATABASE_URL: postgresql://${APP_DB_ROLE:-tsdhn_app}:${APP_DB_PASSWORD:?set APP_DB_PASSWORD in .env}@postgres:5432/tsdhn BETTER_AUTH_SECRET: ${BETTER_AUTH_SECRET:?set BETTER_AUTH_SECRET in .env} ORIGIN: ${ORIGIN:-http://localhost:3000} ports: @@ -147,5 +152,4 @@ services: volumes: postgres-data: minio-data: - libsql-data: jobs-data: diff --git a/libs/api-client/openapi.json b/libs/api-client/openapi.json index b784ed3..3987f28 100644 --- a/libs/api-client/openapi.json +++ b/libs/api-client/openapi.json @@ -165,21 +165,21 @@ } } }, - "/api/v1/jobs/{app_job_id}": { + "/api/v1/jobs/{simulation_id}": { "get": { "tags": [ "jobs" ], "summary": "Get Job", - "operationId": "get_job_api_v1_jobs__app_job_id__get", + "operationId": "get_job_api_v1_jobs__simulation_id__get", "parameters": [ { - "name": "app_job_id", + "name": "simulation_id", "in": "path", "required": true, "schema": { "type": "string", - "title": "App Job Id" + "title": "Simulation Id" } }, { @@ -223,21 +223,140 @@ } } }, - "/api/v1/jobs/{app_job_id}/events": { + "/api/v1/jobs/{simulation_id}/outputs": { + "get": { + "tags": [ + "jobs" + ], + "summary": "List Outputs", + "operationId": "list_outputs_api_v1_jobs__simulation_id__outputs_get", + "parameters": [ + { + "name": "simulation_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Simulation Id" + } + }, + { + "name": "authorization", + "in": "header", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OutputList" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/jobs/{simulation_id}/outputs/{name}": { + "get": { + "tags": [ + "jobs" + ], + "summary": "Get Output", + "operationId": "get_output_api_v1_jobs__simulation_id__outputs__name__get", + "parameters": [ + { + "name": "simulation_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Simulation Id" + } + }, + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Name" + } + }, + { + "name": "authorization", + "in": "header", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "responses": { + "307": { + "description": "Successful Response" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/api/v1/jobs/{simulation_id}/events": { "get": { "tags": [ "jobs" ], "summary": "Job Events", - "operationId": "job_events_api_v1_jobs__app_job_id__events_get", + "description": "Stream job state changes from the job's Postgres notification channel.", + "operationId": "job_events_api_v1_jobs__simulation_id__events_get", "parameters": [ { - "name": "app_job_id", + "name": "simulation_id", "in": "path", "required": true, "schema": { "type": "string", - "title": "App Job Id" + "title": "Simulation Id" } }, { @@ -466,55 +585,28 @@ }, "JobCreated": { "properties": { - "app_job_id": { - "type": "string", - "title": "App Job Id" - }, - "compute_job_id": { + "simulation_id": { "type": "string", - "title": "Compute Job Id" + "title": "Simulation Id" }, "status": { "type": "string", "title": "Status" - }, - "result_bucket": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Result Bucket" - }, - "result_key": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Result Key" } }, "type": "object", "required": [ - "app_job_id", - "compute_job_id", + "simulation_id", "status" ], "title": "JobCreated" }, "JobRequest": { "properties": { - "app_job_id": { + "simulation_id": { "type": "string", "format": "uuid", - "title": "App Job Id" + "title": "Simulation Id" }, "input": { "$ref": "#/components/schemas/EarthquakeInput" @@ -522,20 +614,16 @@ }, "type": "object", "required": [ - "app_job_id", + "simulation_id", "input" ], "title": "JobRequest" }, "JobStatusResponse": { "properties": { - "app_job_id": { - "type": "string", - "title": "App Job Id" - }, - "compute_job_id": { + "simulation_id": { "type": "string", - "title": "Compute Job Id" + "title": "Simulation Id" }, "status": { "type": "string", @@ -605,28 +693,6 @@ } ] }, - "result_bucket": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Result Bucket" - }, - "result_key": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Result Key" - }, "error": { "anyOf": [ { @@ -671,20 +737,66 @@ ], "title": "Finished At" }, - "artifacts_available": { - "type": "boolean", - "title": "Artifacts Available", - "default": false + "outputs": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Outputs", + "default": [] } }, "type": "object", "required": [ - "app_job_id", - "compute_job_id", + "simulation_id", "status" ], "title": "JobStatusResponse" }, + "OutputList": { + "properties": { + "simulation_id": { + "type": "string", + "title": "Simulation Id" + }, + "outputs": { + "items": { + "$ref": "#/components/schemas/StoredOutput" + }, + "type": "array", + "title": "Outputs" + } + }, + "type": "object", + "required": [ + "simulation_id", + "outputs" + ], + "title": "OutputList" + }, + "StoredOutput": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "filename": { + "type": "string", + "title": "Filename" + }, + "content_type": { + "type": "string", + "title": "Content Type" + } + }, + "type": "object", + "required": [ + "name", + "filename", + "content_type" + ], + "title": "StoredOutput" + }, "TsunamiTravelResponse": { "properties": { "arrival_times": { diff --git a/libs/api-client/readme.md b/libs/api-client/readme.md index 80d92ab..4eae66a 100644 --- a/libs/api-client/readme.md +++ b/libs/api-client/readme.md @@ -1,16 +1,18 @@ # @tsdhn/api-client -`@tsdhn/api-client` is the TypeScript client package generated from the FastAPI -OpenAPI contract. It is consumed by the SvelteKit server in -[`apps/web`](../../apps/web/readme.md). +`@tsdhn/api-client` is generated from the compute API's OpenAPI definition. +The SvelteKit server is its only consumer. + +The client sends `COMPUTE_API_TOKEN` with every authenticated request. Use it +only in server code; importing it into browser code could expose the token. ## Files -| File | Purpose | -| ------------------------------------------------------ | -------------------------------------------------------- | -| [`openapi.json`](./openapi.json) | Exported FastAPI OpenAPI schema | -| [`src/generated/schema.ts`](./src/generated/schema.ts) | TypeScript types generated by `openapi-typescript` | -| [`src/index.ts`](./src/index.ts) | Small `openapi-fetch` wrapper with service-token headers | +| File | Purpose | +| ------------------------- | ------------------------------------------------------------------ | +| `openapi.json` | OpenAPI definition exported from FastAPI | +| `src/generated/schema.ts` | Generated TypeScript paths and schemas | +| `src/index.ts` | Small `openapi-fetch` wrapper that adds compute API authentication | ## Usage @@ -18,29 +20,22 @@ OpenAPI contract. It is consumed by the SvelteKit server in import { createTsdhnClient } from "@tsdhn/api-client"; const client = createTsdhnClient({ - baseUrl: process.env.BACKEND_URL!, - serviceToken: process.env.BACKEND_SERVICE_TOKEN!, + baseUrl: process.env.COMPUTE_API_URL!, + computeApiToken: process.env.COMPUTE_API_TOKEN!, }); ``` -The service token must stay on the server. Do not use this client directly in -browser code. - -## Regeneration +## Regenerate -From the repository root: +After changing a FastAPI route or schema: ```sh -bun run gen:client +mise run gen-client ``` -The script at [`scripts/gen-client.ts`](../../scripts/gen-client.ts): - -1. Runs [`scripts/export_openapi.py`](../../scripts/export_openapi.py) through - `uv`. -2. Writes `libs/api-client/openapi.json`. -3. Runs `openapi-typescript`. -4. Writes `libs/api-client/src/generated/schema.ts`. +The command exports the FastAPI schema to `openapi.json`, then generates +`src/generated/schema.ts`. Do not edit either generated output by hand. -Run this after changing FastAPI routes, Pydantic response models, or request -schemas. +Route behavior belongs to the compute API and is visible in its OpenAPI UI. +This package owns only the generated TypeScript representation and the +server-only client wrapper. diff --git a/libs/api-client/src/generated/schema.ts b/libs/api-client/src/generated/schema.ts index 994e818..e970ad4 100644 --- a/libs/api-client/src/generated/schema.ts +++ b/libs/api-client/src/generated/schema.ts @@ -72,7 +72,7 @@ export interface paths { patch?: never; trace?: never; }; - "/api/v1/jobs/{app_job_id}": { + "/api/v1/jobs/{simulation_id}": { parameters: { query?: never; header?: never; @@ -80,7 +80,7 @@ export interface paths { cookie?: never; }; /** Get Job */ - get: operations["get_job_api_v1_jobs__app_job_id__get"]; + get: operations["get_job_api_v1_jobs__simulation_id__get"]; put?: never; post?: never; delete?: never; @@ -89,15 +89,52 @@ export interface paths { patch?: never; trace?: never; }; - "/api/v1/jobs/{app_job_id}/events": { + "/api/v1/jobs/{simulation_id}/outputs": { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - /** Job Events */ - get: operations["job_events_api_v1_jobs__app_job_id__events_get"]; + /** List Outputs */ + get: operations["list_outputs_api_v1_jobs__simulation_id__outputs_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/jobs/{simulation_id}/outputs/{name}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Output */ + get: operations["get_output_api_v1_jobs__simulation_id__outputs__name__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/jobs/{simulation_id}/events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Job Events + * @description Stream job state changes from the job's Postgres notification channel. + */ + get: operations["job_events_api_v1_jobs__simulation_id__events_get"]; put?: never; post?: never; delete?: never; @@ -186,32 +223,24 @@ export interface components { }; /** JobCreated */ JobCreated: { - /** App Job Id */ - app_job_id: string; - /** Compute Job Id */ - compute_job_id: string; + /** Simulation Id */ + simulation_id: string; /** Status */ status: string; - /** Result Bucket */ - result_bucket?: string | null; - /** Result Key */ - result_key?: string | null; }; /** JobRequest */ JobRequest: { /** - * App Job Id + * Simulation Id * Format: uuid */ - app_job_id: string; + simulation_id: string; input: components["schemas"]["EarthquakeInput"]; }; /** JobStatusResponse */ JobStatusResponse: { - /** App Job Id */ - app_job_id: string; - /** Compute Job Id */ - compute_job_id: string; + /** Simulation Id */ + simulation_id: string; /** Status */ status: string; /** Details */ @@ -224,10 +253,6 @@ export interface components { total_steps?: number | null; calculation?: components["schemas"]["CalculationResponse"] | null; travel_times?: components["schemas"]["TsunamiTravelResponse"] | null; - /** Result Bucket */ - result_bucket?: string | null; - /** Result Key */ - result_key?: string | null; /** Error */ error?: string | null; /** Created At */ @@ -237,10 +262,26 @@ export interface components { /** Finished At */ finished_at?: string | null; /** - * Artifacts Available - * @default false + * Outputs + * @default [] */ - artifacts_available: boolean; + outputs: string[]; + }; + /** OutputList */ + OutputList: { + /** Simulation Id */ + simulation_id: string; + /** Outputs */ + outputs: components["schemas"]["StoredOutput"][]; + }; + /** StoredOutput */ + StoredOutput: { + /** Name */ + name: string; + /** Filename */ + filename: string; + /** Content Type */ + content_type: string; }; /** TsunamiTravelResponse */ TsunamiTravelResponse: { @@ -396,14 +437,14 @@ export interface operations { }; }; }; - get_job_api_v1_jobs__app_job_id__get: { + get_job_api_v1_jobs__simulation_id__get: { parameters: { query?: never; header?: { authorization?: string | null; }; path: { - app_job_id: string; + simulation_id: string; }; cookie?: never; }; @@ -429,14 +470,79 @@ export interface operations { }; }; }; - job_events_api_v1_jobs__app_job_id__events_get: { + list_outputs_api_v1_jobs__simulation_id__outputs_get: { + parameters: { + query?: never; + header?: { + authorization?: string | null; + }; + path: { + simulation_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["OutputList"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_output_api_v1_jobs__simulation_id__outputs__name__get: { + parameters: { + query?: never; + header?: { + authorization?: string | null; + }; + path: { + simulation_id: string; + name: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 307: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + job_events_api_v1_jobs__simulation_id__events_get: { parameters: { query?: never; header?: { authorization?: string | null; }; path: { - app_job_id: string; + simulation_id: string; }; cookie?: never; }; diff --git a/libs/api-client/src/index.ts b/libs/api-client/src/index.ts index a4d6d8f..53e195a 100644 --- a/libs/api-client/src/index.ts +++ b/libs/api-client/src/index.ts @@ -1,23 +1,20 @@ import createClient, { type Client } from "openapi-fetch"; import type { paths } from "./generated/schema"; -export type { paths }; export type * from "./generated/schema"; export interface TsdhnClientOptions { baseUrl: string; - serviceToken: string; + computeApiToken: string; fetch?: typeof globalThis.fetch; } -/** - * Server-only client. It attaches the backend service token to every request. - */ +/** Server-only client for authenticated compute API requests. */ export function createTsdhnClient(options: TsdhnClientOptions): Client { return createClient({ baseUrl: options.baseUrl, fetch: options.fetch, - headers: { Authorization: `Bearer ${options.serviceToken}` }, + headers: { Authorization: `Bearer ${options.computeApiToken}` }, }); } diff --git a/mise.toml b/mise.toml index 1f02809..71b1efb 100644 --- a/mise.toml +++ b/mise.toml @@ -2,8 +2,8 @@ python = "3.14.6" uv = "0.11.29" bun = "1.3.14" +postgres = "18.6" gitleaks = "8.30.1" -"libsql-server" = { version = "0.24.32", version_prefix = "libsql-server-v" } [tasks] install = "uv sync --all-packages --group dev --group build" @@ -16,23 +16,49 @@ lint = [ fix = ["uv run ruff check --fix --unsafe-fixes .", "uv run ruff format ."] -test = "uv run pytest -n auto" +# The fast suite is service-free. The integration task owns PostgreSQL. +test = "uv run pytest -n auto -m 'not integration'" +test-integration = "bash scripts/integration.sh" +db-migrate = "mise run db:migrate" +# Golden tests need GMT and ttt_client. +test-golden = "uv run pytest -m golden" + +# Parity tests use frozen MATLAB fixtures and optional Fortran tools. +test-parity = "uv run pytest packages/tsdhn/tests/parity packages/tsdhn-parity/tests" + +# Capture MATLAB checkpoints manually. See scripts/capture_matlab_fixtures.py. +parity-capture-matlab = "uv run python scripts/capture_matlab_fixtures.py" + +# Disable Numba JIT so coverage.py can trace the tsunami kernels. coverage = [ - "uv run coverage run -m pytest", - "uv run coverage report --fail-under=70", + "NUMBA_DISABLE_JIT=1 uv run coverage run -m pytest", + "uv run coverage report", "uv run coverage json", + "uv run coverage xml", +] + +diff-coverage = [ + "mise run db:start", + "mise run coverage", + "uv run coverage run --append -m pytest -m integration packages/api/tests", + "uv run coverage xml", + "uv run diff-cover coverage.xml --compare-branch=origin/master --fail-under=90", ] security = [ "uv run bandit -r packages -lll --skip B101", - # pip-audit --strict does not support editable workspace packages. - # Export only external dependencies and run pip-audit in an isolated env. - "uv export --locked --no-hashes --all-packages --no-emit-workspace --output-file=requirements.lock.txt", - "uvx pip-audit --strict --requirement requirements.lock.txt", + "mise run audit", + "mise run audit-toolchain", "gitleaks detect --source . --no-banner", ] +# Audit production dependencies. +audit = "uv audit --locked --preview-features audit-command --no-dev --no-group build" + +# Audit development dependencies. +audit-toolchain = "uv audit --locked --preview-features audit-command" + build = [ "uv build --package tsdhn --out-dir dist/tsdhn", "uv build --package tsdhn-api --out-dir dist/api", @@ -45,13 +71,20 @@ web-install = "bun install" web-dev = "bun --filter web dev" web-build = "bun --filter web build" web-check = "bun --filter web check" +web-test = "bun --filter web test" lint-js = "bun run lint" fmt-js = "bun run fmt" fmt-check-js = "bun run fmt:check" gen-client = "bun run gen:client" -# Self-hosted libSQL database (sqld). Data persists under ./data/libsql. -db = "sqld --http-listen-addr 127.0.0.1:8080 --db-path ./data/libsql" +diff-coverage-js = [ + "bun --filter web test:coverage", + "sed -i.bak 's|^SF:|SF:apps/web/|' apps/web/coverage/lcov.info", + "uv run diff-cover apps/web/coverage/lcov.info --compare-branch=origin/master --fail-under=90", +] + +# PostgreSQL is managed by mise. Compose runs the service stack. +db = "mise run db:start" dev-up = "podman compose up -d" dev-web = "podman compose --profile web up" @@ -69,6 +102,60 @@ doctor = [ "gitleaks version", ] -# mise ignores missing env files so setup.sh is optional until runtime. +# setup.sh creates this file when the scientific toolchain is installed. [env] _.file = ".tsdhn/env" + +[tasks."db:init"] +description = "Initialize the project-local PostgreSQL cluster" +run = """ +data_dir="$PWD/.data/postgres" + +if [ ! -f "$data_dir/PG_VERSION" ]; then + mkdir -p "$data_dir" + mise x postgres -- initdb -D "$data_dir" -U tsdhn \ + --auth-host=trust --auth-local=trust --no-instructions +fi +""" + +[tasks."db:start"] +description = "Start the project-local PostgreSQL cluster" +depends = ["db:init"] +run = """ +data_dir="$PWD/.data/postgres" + +if ! mise x postgres -- pg_isready -h 127.0.0.1 -p 5432 -U tsdhn -q; then + mise x postgres -- pg_ctl -D "$data_dir" \ + -l "$data_dir/server.log" \ + -o "-h 127.0.0.1 -p 5432 -c fsync=off -c synchronous_commit=off -c full_page_writes=off" \ + -w start +fi + +if ! mise x postgres -- psql -h 127.0.0.1 -p 5432 -U tsdhn -d tsdhn -qAtc "SELECT 1" >/dev/null 2>&1; then + mise x postgres -- createdb -h 127.0.0.1 -p 5432 -U tsdhn tsdhn +fi +""" + +[tasks."db:stop"] +description = "Stop the project-local PostgreSQL cluster" +run = """ +data_dir="$PWD/.data/postgres" +mise x postgres -- pg_ctl -D "$data_dir" stop -m fast -w 2>/dev/null || true +""" + +[tasks."db:reset"] +description = "Delete the project-local PostgreSQL cluster" +run = """ +mise run db:stop +rm -rf "$PWD/.data/postgres" +""" + +[tasks."db:migrate"] +description = "Apply compute, queue, and web migrations to local PostgreSQL" +depends = ["db:start"] +run = [ + "uv run tsdhn-compute-migrate", + "uv run tsdhn-procrastinate-migrate", + "DATABASE_URL=\"${COMPUTE_DATABASE_URL:-postgresql://tsdhn:tsdhn@127.0.0.1:5432/tsdhn}\" bun --filter web db:migrate", + "uv run tsdhn-web-grants", +] diff --git a/model/Makefile b/model/Makefile index 9c8b9aa..a0e87df 100644 --- a/model/Makefile +++ b/model/Makefile @@ -1,8 +1,8 @@ VER=ifort com=-parallel -deform: deform.for - $(VER) -O deform.for -o deform $(com) +deform: def_oka.f + $(VER) -O def_oka.f -o deform $(com) espejo: espejo.f $(VER) -O espejo.f -o espejo $(com) fault_plane: fault_plane.f90 diff --git a/model/readme.md b/model/readme.md new file mode 100644 index 0000000..3c6328c --- /dev/null +++ b/model/readme.md @@ -0,0 +1,68 @@ +# Model and older reference files + +This directory contains model inputs plus MATLAB and Fortran programs used to +understand or compare the Python implementation. Not every source file is part +of the active pipeline. + +The Python engine uses installed copies of the required model files. The +sources here also support development and legacy comparison tests. + +## Active older references + +| File | Role | +| --- | --- | +| `fault_plane.f90` | Active reference for fault placement, mechanism selection, source dimensions, slip, and intermediate files | +| `def_oka.f` | Active deformation comparison; Okada-based vertical displacement for one fault segment | +| `tsunami1.for` | Active reference for the linear shallow-water propagation solver | + +`scripts/setup.sh` and this directory's `Makefile` compile `def_oka.f` as the +`deform` executable. `packages/tsdhn/tsdhn/deform.py` ports this program. + +## Historical and exploratory sources + +| File | Status | +| --- | --- | +| `deform.for` | Older Mansinha-Smylie deformation program. It is not the reference for the Python deformation stage. | +| `fault_plane_n.m` | Exploratory multiple-subfault MATLAB path with spherical geometry. It is not used by the current single-fault pipeline. | +| `mareografo_a.m` | Historical manual helper. Its source-editing instructions are not current run instructions. | + +Do not build `deform.for` and use its output to judge compatibility with +`deform.py`. The two programs implement different deformation models. + +## Model inputs + +| Path | Use | +| --- | --- | +| `bathy/grid_a.grd` | Full propagation bathymetry grid | +| `bathy/xa.dat`, `bathy/ya.dat` | Grid axes used to place the fault and deformation window | +| `mecfoc.dat` | Candidate focal mechanisms used when strike and dip are absent from public input | +| `tidal.dat` | Virtual-gauge indices for propagation output | +| `puertos.txt` | Ports used by approximate arrival-time calculations | +| `pacifico.mat`, `maper1.mat` | Bathymetry and coastline data used by source calculations | +| `ttt_mundo/` | Inputs consumed by `ttt_client` and GMT arrival-time reporting | + +Files such as `pfalla.inp`, `xyo.dat`, `meca.dat`, `deform_a.grd`, and +`zfolder/*` are also checked in as captured examples. During a run they are +generated intermediate or output files, not immutable model inputs. + +## Coordinate and format warnings + +The model grid uses longitudes in the `0..360` frame. Public earthquake inputs +normally use `-180..180`. The active fault-plane stage converts between them. + +Intermediate files preserve one-based Fortran indices and fixed-width numeric +records. Their exact layouts are described in +[`packages/tsdhn/docs/pipeline.md`](../packages/tsdhn/docs/pipeline.md). + +## What comparison establishes + +Compiled Fortran output can show that Python preserved the selected old +program's behavior. It does not establish that the equations, constants, or +empirical corrections are scientifically valid. See +[`packages/tsdhn/docs/testing.md`](../packages/tsdhn/docs/testing.md) before +using comparison output as research evidence. + +Keep an older source until its active behavior, file formats, and remaining +provenance have been documented. Original MATLAB and Fortran files are frozen; +explain corrections in researcher documentation rather than editing those +sources. diff --git a/packages/api/api/core/db.py b/packages/api/api/core/db.py new file mode 100644 index 0000000..072cbd0 --- /dev/null +++ b/packages/api/api/core/db.py @@ -0,0 +1,86 @@ +"""Database connections for API reads, workers, and job notifications.""" + +import threading +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from api.core.errors import TransientInfraError +from api.core.settings import ( + COMPUTE_DATABASE_URL, + DB_POOL_MAX_SIZE, + DB_POOL_MIN_SIZE, +) + +__all__ = [ + "CONNECT_TIMEOUT", + "JobRow", + "close_pool", + "connect", + "get_pool", + "notify_channel", + "pooled", +] + +JobRow = dict[str, Any] +CONNECT_TIMEOUT = 2 + +_pool: ConnectionPool[psycopg.Connection[JobRow]] | None = None +_pool_lock = threading.Lock() + + +def _new_pool() -> ConnectionPool[psycopg.Connection[JobRow]]: + return ConnectionPool( + COMPUTE_DATABASE_URL, + min_size=DB_POOL_MIN_SIZE, + max_size=DB_POOL_MAX_SIZE, + kwargs={"row_factory": dict_row, "connect_timeout": CONNECT_TIMEOUT}, + open=False, + ) + + +def get_pool() -> ConnectionPool[psycopg.Connection[JobRow]]: + """Return the process-wide pool, creating it on first use.""" + global _pool + with _pool_lock: + if _pool is None or _pool.closed: + _pool = _new_pool() + _pool.open(wait=False) + return _pool + + +def close_pool() -> None: + global _pool + with _pool_lock: + if _pool is not None and not _pool.closed: + _pool.close() + _pool = None + + +@contextmanager +def pooled() -> Iterator[psycopg.Connection[JobRow]]: + """Provide a short-lived connection for API queries.""" + with get_pool().connection() as conn: + yield conn + + +def connect() -> psycopg.Connection[JobRow]: + """Open a dedicated worker connection.""" + try: + return psycopg.connect( + COMPUTE_DATABASE_URL, + connect_timeout=CONNECT_TIMEOUT, + row_factory=dict_row, + ) + except psycopg.OperationalError as e: + raise TransientInfraError("database connection failed") from e + + +def notify_channel(simulation_id: uuid.UUID) -> str: + """Return the SQL-safe notification channel for one job.""" + return f"tsdhn_job_{simulation_id.hex}" diff --git a/packages/api/api/core/errors.py b/packages/api/api/core/errors.py new file mode 100644 index 0000000..4d648bb --- /dev/null +++ b/packages/api/api/core/errors.py @@ -0,0 +1,5 @@ +__all__ = ["TransientInfraError"] + + +class TransientInfraError(Exception): + """An infrastructure error that is safe to retry.""" diff --git a/packages/api/api/core/jobs.py b/packages/api/api/core/jobs.py deleted file mode 100644 index 527c213..0000000 --- a/packages/api/api/core/jobs.py +++ /dev/null @@ -1,399 +0,0 @@ -import logging -import shutil -import uuid -from datetime import datetime -from typing import Any, cast - -import psycopg -from psycopg.rows import dict_row -from psycopg.types.json import Jsonb - -from api.core.procrastinate_app import app -from api.core.settings import ( - COMPUTE_DATABASE_URL, - JOBS_DIR, - PROCRASTINATE_QUEUE, -) -from api.core.storage import artifact_store, iso -from tsdhn.domain import EarthquakeInput, JobStatus -from tsdhn.engine import SimulationResult, run_simulation -from tsdhn.utils.file_utils import sanitize_for_log - -__all__ = [ - "ComputeJobs", - "JobStatus", - "compute_jobs", - "install_compute_schema", - "run_simulation_task", -] - -logger = logging.getLogger(__name__) - -JobRow = dict[str, Any] -DB_CONNECT_TIMEOUT_SECONDS = 2 - - -COMPUTE_JOBS_SCHEMA = """ -CREATE TABLE IF NOT EXISTS compute_jobs ( - id uuid PRIMARY KEY, - external_id uuid UNIQUE NOT NULL, - status text NOT NULL, - input_params jsonb NOT NULL, - details text, - step text, - step_index integer, - total_steps integer, - calculation jsonb, - travel_times jsonb, - result_bucket text, - result_key text, - error text, - created_at timestamptz NOT NULL DEFAULT now(), - updated_at timestamptz NOT NULL DEFAULT now(), - started_at timestamptz, - finished_at timestamptz -); - -CREATE INDEX IF NOT EXISTS compute_jobs_status_idx - ON compute_jobs(status); - -CREATE INDEX IF NOT EXISTS compute_jobs_created_at_idx - ON compute_jobs(created_at DESC); -""" - - -SELECT_BY_EXTERNAL_ID = """ -SELECT id, external_id, status, input_params, details, step, - step_index, total_steps, calculation, travel_times, result_bucket, - result_key, error, created_at, updated_at, started_at, finished_at -FROM compute_jobs -WHERE external_id = %s -""" - - -SELECT_BY_ID = """ -SELECT id, external_id, status, input_params, details, step, - step_index, total_steps, calculation, travel_times, result_bucket, - result_key, error, created_at, updated_at, started_at, finished_at -FROM compute_jobs -WHERE id = %s -""" - - -INSERT_JOB_SQL = """ -INSERT INTO compute_jobs ( - id, external_id, status, input_params, details -) -VALUES (%s, %s, %s, %s, %s) -ON CONFLICT (external_id) DO NOTHING -RETURNING id, external_id, status, input_params, details, step, - step_index, total_steps, calculation, travel_times, result_bucket, - result_key, error, created_at, updated_at, started_at, finished_at -""" - - -def install_compute_schema() -> None: - with psycopg.connect( - COMPUTE_DATABASE_URL, connect_timeout=DB_CONNECT_TIMEOUT_SECONDS - ) as conn: - conn.execute(COMPUTE_JOBS_SCHEMA) - conn.commit() - - -def _model_dump(data: EarthquakeInput) -> dict[str, Any]: - dumped: object = data.model_dump(mode="json") - if not isinstance(dumped, dict): - raise TypeError("EarthquakeInput did not dump to a JSON object") - return cast(dict[str, Any], dumped) - - -def _as_uuid(value: str) -> uuid.UUID: - return uuid.UUID(value, version=4) - - -def _fetch_by_external_id( - conn: psycopg.Connection[JobRow], external_id: uuid.UUID -) -> JobRow | None: - row = conn.execute(SELECT_BY_EXTERNAL_ID, [external_id]).fetchone() - return row - - -def _fetch_by_id(conn: psycopg.Connection[JobRow], job_id: uuid.UUID) -> JobRow | None: - row = conn.execute(SELECT_BY_ID, [job_id]).fetchone() - return row - - -def _status_from_row(row: JobRow) -> dict[str, Any]: - status = str(row["status"]) - return { - "compute_job_id": str(row["id"]), - "status": status, - "details": row["details"], - "step": row["step"], - "step_index": row["step_index"], - "total_steps": row["total_steps"], - "calculation": row["calculation"], - "travel_times": row["travel_times"], - "result_bucket": row["result_bucket"], - "result_key": row["result_key"], - "error": row["error"], - "created_at": iso(row["created_at"]), - "started_at": iso(row["started_at"]), - "finished_at": iso(row["finished_at"]), - "artifacts_available": status == JobStatus.COMPLETED.value - and row["result_key"] is not None, - } - - -def _progress_update( - conn: psycopg.Connection[JobRow], - compute_job_id: uuid.UUID, - *, - details: str, - values: dict[str, Any] | None = None, -) -> None: - values = values or {} - conn.execute( - """ - UPDATE compute_jobs - SET status = %s, - details = %s, - step = COALESCE(%s, step), - step_index = COALESCE(%s, step_index), - total_steps = COALESCE(%s, total_steps), - calculation = COALESCE(%s, calculation), - travel_times = COALESCE(%s, travel_times), - updated_at = now() - WHERE id = %s - """, - [ - JobStatus.RUNNING.value, - details, - values.get("step"), - values.get("step_index"), - values.get("total_steps"), - Jsonb(values["calculation"]) if "calculation" in values else None, - Jsonb(values["travel_times"]) if "travel_times" in values else None, - compute_job_id, - ], - ) - conn.commit() - - -class ComputeJobs: - def create_or_get_job( - self, - *, - data: EarthquakeInput, - external_id: str, - ) -> dict[str, Any]: - app_job_id = _as_uuid(external_id) - input_params = _model_dump(data) - - with ( - psycopg.connect( - COMPUTE_DATABASE_URL, - connect_timeout=DB_CONNECT_TIMEOUT_SECONDS, - row_factory=dict_row, - ) as conn, - conn.transaction(), - ): - compute_job_id = uuid.uuid4() - inserted = conn.execute( - INSERT_JOB_SQL, - [ - compute_job_id, - app_job_id, - JobStatus.QUEUED.value, - Jsonb(input_params), - "Queued for simulation worker", - ], - ).fetchone() - - if inserted is None: - existing = _fetch_by_external_id(conn, app_job_id) - if existing is None: - raise RuntimeError("Compute job was not persisted") - if existing["input_params"] != input_params: - raise ValueError("Job id already exists with different input") - return _status_from_row(existing) - - run_simulation_task.configure( - connection=conn, - queue=PROCRASTINATE_QUEUE, - queueing_lock=f"simulation:{app_job_id}", - ).defer(compute_job_id=str(compute_job_id)) - - return _status_from_row(inserted) - - def get_job_status(self, app_job_id: str) -> dict[str, Any]: - try: - external_id = _as_uuid(app_job_id) - with psycopg.connect( - COMPUTE_DATABASE_URL, - connect_timeout=DB_CONNECT_TIMEOUT_SECONDS, - row_factory=dict_row, - ) as conn: - row = _fetch_by_external_id(conn, external_id) - except Exception as e: - logger.error( - "Job lookup failed for %s: %s", - sanitize_for_log(app_job_id), - e, - ) - raise ValueError("Invalid or unknown job ID") from e - - if row is None: - raise ValueError("Invalid or unknown job ID") - return _status_from_row(row) - - def is_database_connected(self) -> bool: - try: - with psycopg.connect( - COMPUTE_DATABASE_URL, connect_timeout=DB_CONNECT_TIMEOUT_SECONDS - ) as conn: - conn.execute("SELECT 1") - return True - except Exception: - return False - - def is_storage_connected(self) -> bool: - return artifact_store.is_connected() - - -@app.task( - name="api.run_simulation", - queue=PROCRASTINATE_QUEUE, - retry=False, -) -def run_simulation_task(compute_job_id: str) -> None: - job_uuid = _as_uuid(compute_job_id) - - with psycopg.connect( - COMPUTE_DATABASE_URL, - connect_timeout=DB_CONNECT_TIMEOUT_SECONDS, - row_factory=dict_row, - ) as conn: - row = _fetch_by_id(conn, job_uuid) - if row is None: - raise RuntimeError(f"Unknown compute job {compute_job_id}") - - app_job_id = str(row["external_id"]) - work_dir = JOBS_DIR / app_job_id - data = EarthquakeInput(**row["input_params"]) - - conn.execute( - """ - UPDATE compute_jobs - SET status = %s, - details = %s, - started_at = COALESCE(started_at, now()), - updated_at = now() - WHERE id = %s - """, - [JobStatus.RUNNING.value, "Simulation worker started", job_uuid], - ) - conn.commit() - row = _fetch_by_id(conn, job_uuid) - if row is None: - raise RuntimeError(f"Unknown compute job {compute_job_id}") - - def on_progress(message: str, details: dict[str, Any]) -> None: - _progress_update(conn, job_uuid, details=message, values=details) - - try: - result = run_simulation( - data, - work_dir, - on_progress=on_progress, - ) - _complete_job(conn, row, result) - shutil.rmtree(work_dir, ignore_errors=True) - except Exception as e: - logger.exception("Pipeline failed for job %s", compute_job_id) - conn.execute( - """ - UPDATE compute_jobs - SET status = %s, - details = %s, - error = %s, - finished_at = now(), - updated_at = now() - WHERE id = %s - """, - [ - JobStatus.FAILED.value, - "Pipeline failed - check error logs", - f"{type(e).__name__}: {e!s}", - job_uuid, - ], - ) - conn.commit() - raise - - -def _complete_job( - conn: psycopg.Connection[JobRow], - row: JobRow, - result: SimulationResult, -) -> None: - now = datetime.now().astimezone() - app_job_id = str(row["external_id"]) - compute_job_id = str(row["id"]) - calculation = result.calculation.model_dump(mode="json") - travel_times = result.travel_times.model_dump(mode="json") - - metadata = { - "app_job_id": app_job_id, - "compute_job_id": compute_job_id, - "status": JobStatus.COMPLETED.value, - "created_at": iso(row["created_at"]), - "started_at": iso(row["started_at"]), - "finished_at": now.isoformat(), - "calculation": calculation, - "travel_times": travel_times, - "artifacts": [ - { - "name": artifact.name, - "key": f"simulations/{app_job_id}/artifacts/{artifact.path.name}", - "content_type": artifact.content_type, - } - for artifact in result.bundle.artifacts - ], - } - bucket, metadata_key = artifact_store.upload_simulation_result( - app_job_id=app_job_id, - compute_job_id=compute_job_id, - bundle=result.bundle, - metadata=metadata, - ) - - conn.execute( - """ - UPDATE compute_jobs - SET status = %s, - details = %s, - calculation = %s, - travel_times = %s, - result_bucket = %s, - result_key = %s, - error = NULL, - finished_at = %s, - updated_at = now() - WHERE id = %s - """, - [ - JobStatus.COMPLETED.value, - "Simulation completed successfully", - Jsonb(calculation), - Jsonb(travel_times), - bucket, - metadata_key, - now, - row["id"], - ], - ) - conn.commit() - - -compute_jobs = ComputeJobs() diff --git a/packages/api/api/core/procrastinate_app.py b/packages/api/api/core/procrastinate_app.py index a9db70d..94171c3 100644 --- a/packages/api/api/core/procrastinate_app.py +++ b/packages/api/api/core/procrastinate_app.py @@ -1,10 +1,13 @@ import procrastinate -from api.core.settings import COMPUTE_DATABASE_URL +from api.core.settings import COMPUTE_DATABASE_URL, PROCRASTINATE_SEARCH_PATH __all__ = ["app"] app = procrastinate.App( - connector=procrastinate.PsycopgConnector(conninfo=COMPUTE_DATABASE_URL), - import_paths=("api.core.jobs",), + connector=procrastinate.PsycopgConnector( + conninfo=COMPUTE_DATABASE_URL, + kwargs={"options": f"-c search_path={PROCRASTINATE_SEARCH_PATH}"}, + ), + import_paths=("api.core.tasks",), ) diff --git a/packages/api/api/core/repository.py b/packages/api/api/core/repository.py new file mode 100644 index 0000000..fd571f8 --- /dev/null +++ b/packages/api/api/core/repository.py @@ -0,0 +1,389 @@ +"""Read and write the compute service's job state.""" + +import logging +import uuid +from datetime import datetime +from typing import Any, cast + +import psycopg +from psycopg.types.json import Jsonb + +from api.core.db import JobRow, connect, notify_channel, pooled +from api.core.storage import iso, output_store +from tsdhn.domain import EarthquakeInput, JobStatus +from tsdhn.engine import SimulationResult +from tsdhn.utils.file_utils import sanitize_for_log + +__all__ = [ + "StoredOutput", + "complete_job", + "create_or_get_job", + "fetch_by_id", + "get_current_step", + "get_job_status", + "get_outputs", + "is_database_connected", + "list_abandoned_work_dirs", + "mark_started", + "record_failure", + "record_progress", +] + +logger = logging.getLogger(__name__) + +StoredOutput = dict[str, str] + +# These details are safe to show to researchers. +FAILED_JOB_DETAILS = "Pipeline failed - check error logs" + +_COLUMNS = """ + id, simulation_id, status, input_params, details, step, step_index, + total_steps, calculation, travel_times, outputs, error, created_at, + updated_at, started_at, finished_at +""" + + +def _sql(template: str) -> str: + return template.format(columns=_COLUMNS) + + +SELECT_BY_SIMULATION_ID = _sql( + "SELECT {columns} FROM compute.jobs WHERE simulation_id = %s" +) +SELECT_BY_ID = _sql("SELECT {columns} FROM compute.jobs WHERE id = %s") + +INSERT_JOB_SQL = _sql( + """ + INSERT INTO compute.jobs (id, simulation_id, status, input_params, details) + VALUES (%s, %s, %s, %s, %s) + ON CONFLICT (simulation_id) DO NOTHING + RETURNING {columns} + """ +) + + +def as_uuid(value: str) -> uuid.UUID: + return uuid.UUID(value, version=4) + + +def _model_dump(data: EarthquakeInput) -> dict[str, Any]: + dumped: object = data.model_dump(mode="json") + if not isinstance(dumped, dict): + raise TypeError("EarthquakeInput did not dump to a JSON object") + return cast(dict[str, Any], dumped) + + +def _notify(conn: psycopg.Connection[JobRow], simulation_id: uuid.UUID) -> None: + """PostgreSQL delivers the notification when the update commits.""" + conn.execute(f"NOTIFY {notify_channel(simulation_id)}") + + +def status_from_row(row: JobRow) -> dict[str, Any]: + status = str(row["status"]) + outputs = row["outputs"] or [] + return { + "status": status, + "details": row["details"], + "step": row["step"], + "step_index": row["step_index"], + "total_steps": row["total_steps"], + "calculation": row["calculation"], + "travel_times": row["travel_times"], + "error": row["error"], + "created_at": iso(row["created_at"]), + "started_at": iso(row["started_at"]), + "finished_at": iso(row["finished_at"]), + "outputs": [output["name"] for output in outputs], + } + + +def _public_error(e: Exception, step: str | None) -> str: + """The user-facing error stored in compute.jobs.error. + + Raw exception messages embed server filesystem paths (jobs dir, model + locations), so only the exception class and failed step are exposed; + the full traceback stays in the worker logs. + """ + if step: + return f"Simulation failed at step '{step}' ({type(e).__name__})" + return f"Simulation failed ({type(e).__name__})" + + +def fetch_by_id(conn: psycopg.Connection[JobRow], job_id: uuid.UUID) -> JobRow | None: + return conn.execute(SELECT_BY_ID, [job_id]).fetchone() + + +def create_or_get_job( + *, data: EarthquakeInput, simulation_id: str, defer: Any +) -> dict[str, Any]: + """Insert a job and enqueue it, atomically. + + `defer` receives the open connection and new compute job id. Keeping it as + an argument leaves this module independent of the queue implementation. + The insert and enqueue commit together, so a job row always has a queue + entry. + """ + simulation_uuid = as_uuid(simulation_id) + input_params = _model_dump(data) + + with pooled() as conn, conn.transaction(): + compute_job_id = uuid.uuid4() + inserted = conn.execute( + INSERT_JOB_SQL, + [ + compute_job_id, + simulation_uuid, + JobStatus.QUEUED.value, + Jsonb(input_params), + "Queued for simulation worker", + ], + ).fetchone() + + if inserted is None: + existing = conn.execute( + SELECT_BY_SIMULATION_ID, [simulation_uuid] + ).fetchone() + if existing is None: + raise RuntimeError("Compute job was not persisted") + if existing["input_params"] != input_params: + raise ValueError("Job id already exists with different input") + return status_from_row(existing) + + defer(conn, compute_job_id) + return status_from_row(inserted) + + +def get_job_status(simulation_id: str) -> dict[str, Any]: + try: + simulation_uuid = as_uuid(simulation_id) + with pooled() as conn: + row = conn.execute(SELECT_BY_SIMULATION_ID, [simulation_uuid]).fetchone() + except Exception as e: + logger.error("Job lookup failed for %s: %s", sanitize_for_log(simulation_id), e) + raise ValueError("Invalid or unknown job ID") from e + + if row is None: + raise ValueError("Invalid or unknown job ID") + return status_from_row(row) + + +def get_outputs(simulation_id: str) -> list[StoredOutput]: + """Return the outputs recorded with a completed job.""" + simulation_uuid = as_uuid(simulation_id) + with pooled() as conn: + row = conn.execute( + "SELECT status, outputs FROM compute.jobs WHERE simulation_id = %s", + [simulation_uuid], + ).fetchone() + if row is None: + raise ValueError("Invalid or unknown job ID") + if row["status"] != JobStatus.COMPLETED.value: + return [] + return cast(list[StoredOutput], row["outputs"] or []) + + +def get_current_step( + conn: psycopg.Connection[JobRow], job_uuid: uuid.UUID +) -> str | None: + row = conn.execute( + "SELECT step FROM compute.jobs WHERE id = %s", [job_uuid] + ).fetchone() + return str(row["step"]) if row and row["step"] else None + + +def list_abandoned_work_dirs(cutoff: datetime) -> list[str]: + """Return failed job workspaces older than the retention cutoff.""" + with pooled() as conn: + rows = conn.execute( + """ + SELECT simulation_id FROM compute.jobs + WHERE status = %s AND finished_at IS NOT NULL AND finished_at < %s + """, + [JobStatus.FAILED.value, cutoff], + ).fetchall() + return [str(row["simulation_id"]) for row in rows] + + +def is_database_connected() -> bool: + try: + with pooled() as conn: + conn.execute("SELECT 1") + return True + except Exception: + return False + + +def mark_started( + conn: psycopg.Connection[JobRow], job_uuid: uuid.UUID, simulation_id: uuid.UUID +) -> None: + conn.execute( + """ + UPDATE compute.jobs + SET status = %s, details = %s, + started_at = COALESCE(started_at, now()), updated_at = now() + WHERE id = %s + """, + [JobStatus.RUNNING.value, "Simulation worker started", job_uuid], + ) + _notify(conn, simulation_id) + conn.commit() + + +def record_progress( + conn: psycopg.Connection[JobRow], + job_uuid: uuid.UUID, + simulation_id: uuid.UUID, + message: str, + details: dict[str, Any], +) -> None: + conn.execute( + """ + UPDATE compute.jobs + SET status = %s, + details = %s, + step = COALESCE(%s, step), + step_index = COALESCE(%s, step_index), + total_steps = COALESCE(%s, total_steps), + calculation = COALESCE(%s, calculation), + travel_times = COALESCE(%s, travel_times), + updated_at = now() + WHERE id = %s + """, + [ + JobStatus.RUNNING.value, + message, + details.get("step"), + details.get("step_index"), + details.get("total_steps"), + Jsonb(details["calculation"]) if "calculation" in details else None, + Jsonb(details["travel_times"]) if "travel_times" in details else None, + job_uuid, + ], + ) + _notify(conn, simulation_id) + conn.commit() + + +def record_failure( + conn: psycopg.Connection[JobRow], + job_uuid: uuid.UUID, + simulation_id: uuid.UUID, + exc: Exception, + *, + step: str | None, + will_retry: bool, +) -> None: + """Record a failed run. + + When the exception is about to be retried this only updates `details` + and leaves the status RUNNING, so an in-flight retry does not look + terminal to anyone watching. + """ + logger.exception("Simulation failed for compute job %s", job_uuid) + if will_retry: + conn.execute( + "UPDATE compute.jobs SET details = %s, updated_at = now() WHERE id = %s", + [f"Retrying after transient error ({type(exc).__name__})", job_uuid], + ) + else: + conn.execute( + """ + UPDATE compute.jobs + SET status = %s, details = %s, error = %s, + finished_at = now(), updated_at = now() + WHERE id = %s + """, + [ + JobStatus.FAILED.value, + FAILED_JOB_DETAILS, + _public_error(exc, step), + job_uuid, + ], + ) + _notify(conn, simulation_id) + conn.commit() + + +def fail_job( + conn: psycopg.Connection[JobRow], + job_uuid: uuid.UUID, + simulation_id: uuid.UUID, + error: str, +) -> None: + conn.execute( + """ + UPDATE compute.jobs + SET status = %s, details = %s, error = %s, + finished_at = now(), updated_at = now() + WHERE id = %s + """, + [JobStatus.FAILED.value, FAILED_JOB_DETAILS, error, job_uuid], + ) + _notify(conn, simulation_id) + conn.commit() + + +def complete_job( + conn: psycopg.Connection[JobRow], row: JobRow, result: SimulationResult +) -> None: + """Upload the result and commit its manifest with the terminal state.""" + now = datetime.now().astimezone() + simulation_id = str(row["simulation_id"]) + compute_job_id = str(row["id"]) + calculation = result.calculation.model_dump(mode="json") + travel_times = result.travel_times.model_dump(mode="json") + + outputs: list[StoredOutput] = [ + { + "name": output.name, + "key": f"simulations/{simulation_id}/outputs/{output.path.name}", + "filename": output.path.name, + "content_type": output.content_type, + } + for output in result.outputs.files + ] + + metadata = { + "simulation_id": simulation_id, + "compute_job_id": compute_job_id, + "status": JobStatus.COMPLETED.value, + "created_at": iso(row["created_at"]), + "started_at": iso(row["started_at"]), + "finished_at": now.isoformat(), + "calculation": calculation, + "travel_times": travel_times, + "outputs": outputs, + } + bucket, metadata_key = output_store.upload_simulation_result( + simulation_id=simulation_id, + compute_job_id=compute_job_id, + outputs=result.outputs, + metadata=metadata, + ) + + conn.execute( + """ + UPDATE compute.jobs + SET status = %s, details = %s, calculation = %s, travel_times = %s, + outputs = %s, result_bucket = %s, result_key = %s, error = NULL, + finished_at = %s, updated_at = now() + WHERE id = %s + """, + [ + JobStatus.COMPLETED.value, + "Simulation completed successfully", + Jsonb(calculation), + Jsonb(travel_times), + Jsonb(outputs), + bucket, + metadata_key, + now, + row["id"], + ], + ) + _notify(conn, row["simulation_id"]) + conn.commit() + + +def open_worker_connection() -> psycopg.Connection[JobRow]: + return connect() diff --git a/packages/api/api/core/schema.py b/packages/api/api/core/schema.py new file mode 100644 index 0000000..f793576 --- /dev/null +++ b/packages/api/api/core/schema.py @@ -0,0 +1,71 @@ +"""DDL for compute state.""" + +__all__ = ["COMPUTE_SCHEMA_SQL"] + +COMPUTE_SCHEMA_SQL = """ +CREATE SCHEMA IF NOT EXISTS compute; + +CREATE TABLE IF NOT EXISTS compute.jobs ( + id uuid PRIMARY KEY, + simulation_id uuid UNIQUE NOT NULL, + status text NOT NULL, + input_params jsonb NOT NULL, + details text, + step text, + step_index integer, + total_steps integer, + calculation jsonb, + travel_times jsonb, + outputs jsonb NOT NULL DEFAULT '[]'::jsonb, + result_bucket text, + result_key text, + error text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + started_at timestamptz, + finished_at timestamptz +); + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'compute' AND table_name = 'jobs' + AND column_name = 'external_id' + ) AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'compute' AND table_name = 'jobs' + AND column_name = 'simulation_id' + ) THEN + ALTER TABLE compute.jobs RENAME COLUMN external_id TO simulation_id; + END IF; + + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'compute' AND table_name = 'jobs' + AND column_name = 'artifacts' + ) AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'compute' AND table_name = 'jobs' + AND column_name = 'outputs' + ) THEN + ALTER TABLE compute.jobs RENAME COLUMN artifacts TO outputs; + END IF; + + IF EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'compute.jobs'::regclass + AND conname = 'jobs_external_id_key' + ) AND NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'compute.jobs'::regclass + AND conname = 'jobs_simulation_id_key' + ) THEN + ALTER TABLE compute.jobs + RENAME CONSTRAINT jobs_external_id_key TO jobs_simulation_id_key; + END IF; +END $$; + +CREATE INDEX IF NOT EXISTS jobs_status_idx ON compute.jobs(status); +CREATE INDEX IF NOT EXISTS jobs_created_at_idx ON compute.jobs(created_at DESC); +""" diff --git a/packages/api/api/core/settings.py b/packages/api/api/core/settings.py index a786f34..8e2b991 100644 --- a/packages/api/api/core/settings.py +++ b/packages/api/api/core/settings.py @@ -2,27 +2,50 @@ from pathlib import Path __all__ = [ + "APP_DB_PASSWORD", + "APP_DB_ROLE", "COMPUTE_DATABASE_URL", + "DB_POOL_MAX_SIZE", + "DB_POOL_MIN_SIZE", "JOBS_DIR", + "LOG_LEVEL", "MINIO_ACCESS_KEY", "MINIO_BUCKET", "MINIO_ENDPOINT", + "MINIO_PUBLIC_ENDPOINT", "MINIO_SECRET_KEY", "MINIO_SECURE", + "NUMBA_THREADS", + "OUTPUT_URL_TTL", "PROCRASTINATE_QUEUE", - "REPORT_DOWNLOAD_MAX_BYTES", + "PROCRASTINATE_SCHEMA", + "PROCRASTINATE_SEARCH_PATH", + "SSE_MAX_DURATION", ] COMPUTE_DATABASE_URL = os.environ.get( "COMPUTE_DATABASE_URL", - "postgresql://tsdhn:tsdhn@localhost:5432/tsdhn_compute", + "postgresql://tsdhn:tsdhn@localhost:5432/tsdhn", ) PROCRASTINATE_QUEUE = os.environ.get("PROCRASTINATE_QUEUE", "simulations") +PROCRASTINATE_SCHEMA = "compute" +PROCRASTINATE_SEARCH_PATH = f"{PROCRASTINATE_SCHEMA},public" JOBS_DIR: Path = Path(os.environ.get("TSDHN_JOBS_DIR", "jobs")).resolve() +LOG_LEVEL = os.environ.get("TSDHN_LOG_LEVEL", "INFO").upper() + +# API reads use a pool; worker runs use dedicated connections. +DB_POOL_MIN_SIZE = int(os.environ.get("DB_POOL_MIN_SIZE", "1")) +DB_POOL_MAX_SIZE = int(os.environ.get("DB_POOL_MAX_SIZE", "10")) + +APP_DB_ROLE = os.environ.get("APP_DB_ROLE", "tsdhn_app") +APP_DB_PASSWORD = os.environ.get("APP_DB_PASSWORD", "") + MINIO_ENDPOINT = os.environ.get("MINIO_ENDPOINT", "localhost:9000") +# Public endpoint differs from API endpoint for browser downloads. +MINIO_PUBLIC_ENDPOINT = os.environ.get("MINIO_PUBLIC_ENDPOINT", MINIO_ENDPOINT) MINIO_ACCESS_KEY = os.environ.get("MINIO_ACCESS_KEY", "minioadmin") MINIO_SECRET_KEY = os.environ.get("MINIO_SECRET_KEY", "minioadmin") MINIO_BUCKET = os.environ.get("MINIO_BUCKET", "tsdhn-results") @@ -31,6 +54,14 @@ "true", "yes", } -REPORT_DOWNLOAD_MAX_BYTES = int( - os.environ.get("REPORT_DOWNLOAD_MAX_BYTES", str(50 * 1024 * 1024)) + +OUTPUT_URL_TTL = int(os.environ.get("OUTPUT_URL_TTL_SECONDS", str(15 * 60))) + +SSE_MAX_DURATION = int(os.environ.get("SSE_MAX_DURATION_SECONDS", str(30 * 60))) + +# Unset means Numba uses the CPUs visible to the process. +NUMBA_THREADS: int | None = ( + int(os.environ["TSDHN_NUMBA_THREADS"]) + if os.environ.get("TSDHN_NUMBA_THREADS") + else None ) diff --git a/packages/api/api/core/storage.py b/packages/api/api/core/storage.py index a67ca20..b190f45 100644 --- a/packages/api/api/core/storage.py +++ b/packages/api/api/core/storage.py @@ -1,33 +1,28 @@ import json -from collections.abc import Iterator -from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from io import BytesIO from typing import Any import urllib3 from minio import Minio +from minio.error import MinioException +from api.core.errors import TransientInfraError from api.core.settings import ( MINIO_ACCESS_KEY, MINIO_BUCKET, MINIO_ENDPOINT, + MINIO_PUBLIC_ENDPOINT, MINIO_SECRET_KEY, MINIO_SECURE, + OUTPUT_URL_TTL, ) -from tsdhn.engine import ArtifactBundle +from tsdhn.engine import SimulationOutputs -__all__ = ["ArtifactStore", "StoredObjectInfo", "artifact_store"] +__all__ = ["OutputStore", "output_store"] -@dataclass(frozen=True) -class StoredObjectInfo: - object_name: str - size: int - content_type: str | None - - -class ArtifactStore: +class OutputStore: def __init__(self) -> None: self.bucket = MINIO_BUCKET self._client = Minio( @@ -40,6 +35,13 @@ def __init__(self) -> None: retries=False, ), ) + # Browser downloads may use a different endpoint from MinIO uploads. + self._public_client = Minio( + MINIO_PUBLIC_ENDPOINT, + access_key=MINIO_ACCESS_KEY, + secret_key=MINIO_SECRET_KEY, + secure=MINIO_SECURE, + ) def is_connected(self) -> bool: try: @@ -55,69 +57,65 @@ def ensure_bucket(self) -> None: def upload_simulation_result( self, *, - app_job_id: str, + simulation_id: str, compute_job_id: str, - bundle: ArtifactBundle, + outputs: SimulationOutputs, metadata: dict[str, Any], ) -> tuple[str, str]: - self.ensure_bucket() - - prefix = f"simulations/{app_job_id}" + prefix = f"simulations/{simulation_id}" metadata_key = f"{prefix}/metadata.json" - for artifact in bundle.artifacts: - self._client.fput_object( + try: + self.ensure_bucket() + + for output in outputs.files: + self._client.fput_object( + bucket_name=self.bucket, + object_name=f"{prefix}/outputs/{output.path.name}", + file_path=str(output.path), + content_type=output.content_type, + metadata={ + "simulation-id": simulation_id, + "compute-job-id": compute_job_id, + "output-name": output.name, + }, + ) + + payload = json.dumps( + metadata, ensure_ascii=True, separators=(",", ":") + ).encode("utf-8") + self._client.put_object( bucket_name=self.bucket, - object_name=f"{prefix}/artifacts/{artifact.path.name}", - file_path=str(artifact.path), - content_type=artifact.content_type, - metadata={ - "app-job-id": app_job_id, - "compute-job-id": compute_job_id, - "artifact-name": artifact.name, - }, + object_name=metadata_key, + data=BytesIO(payload), + length=len(payload), + content_type="application/json", ) - - payload = json.dumps(metadata, ensure_ascii=True, separators=(",", ":")).encode( - "utf-8" - ) - self._client.put_object( - bucket_name=self.bucket, - object_name=metadata_key, - data=BytesIO(payload), - length=len(payload), - content_type="application/json", - ) + except (MinioException, urllib3.exceptions.HTTPError, OSError) as e: + # Storage outages are transient. Let the task retry the upload. + raise TransientInfraError("output upload failed") from e return self.bucket, metadata_key - def stat_object(self, object_name: str) -> StoredObjectInfo: - result = self._client.stat_object( - bucket_name=self.bucket, - object_name=object_name, - ) - return StoredObjectInfo( - object_name=object_name, - size=int(result.size or 0), - content_type=result.content_type, - ) - - def stream_object( - self, object_name: str, *, chunk_size: int = 1024 * 1024 - ) -> Iterator[bytes]: - response = self._client.get_object( - bucket_name=self.bucket, - object_name=object_name, - ) + def presigned_url(self, object_name: str, *, filename: str) -> str: + """Return a short-lived URL for downloading an output file.""" try: - yield from response.stream(chunk_size) - finally: - response.close() - response.release_conn() + return self._public_client.presigned_get_object( + bucket_name=self.bucket, + object_name=object_name, + expires=timedelta(seconds=OUTPUT_URL_TTL), + response_headers={ + "response-content-disposition": ( + f'attachment; filename="{filename}"' + ) + }, + ) + except (MinioException, urllib3.exceptions.HTTPError, OSError) as e: + raise TransientInfraError("presigning output URL failed") from e def iso(value: datetime | None) -> str | None: return value.isoformat() if value is not None else None -artifact_store = ArtifactStore() +output_store = OutputStore() diff --git a/packages/api/api/core/tasks.py b/packages/api/api/core/tasks.py new file mode 100644 index 0000000..8e6d11d --- /dev/null +++ b/packages/api/api/core/tasks.py @@ -0,0 +1,194 @@ +"""Queue tasks for simulation runs and worker maintenance.""" + +import logging +import shutil +import uuid +from datetime import datetime, timedelta +from typing import Any, Literal + +import psycopg +from procrastinate import JobContext, RetryStrategy +from procrastinate import exceptions as procrastinate_exceptions +from procrastinate.jobs import Job as ProcrastinateJob +from procrastinate.jobs import Status as ProcrastinateStatus + +from api.core import repository +from api.core.db import JobRow, connect, pooled +from api.core.errors import TransientInfraError +from api.core.procrastinate_app import app +from api.core.settings import JOBS_DIR, PROCRASTINATE_QUEUE +from tsdhn.domain import EarthquakeInput, JobStatus +from tsdhn.engine import run_simulation +from tsdhn.utils.file_utils import sanitize_for_log + +__all__ = [ + "enqueue_simulation", + "reap_action", + "reap_stalled_jobs_task", + "run_simulation_task", + "sweep_abandoned_work_dirs_task", +] + +type ReapAction = Literal["retry", "exhausted"] + +logger = logging.getLogger(__name__) + +# Retry only infrastructure failures. Domain and pipeline errors are terminal. +MAX_ATTEMPTS = 3 +TRANSIENT_RETRY = RetryStrategy( + max_attempts=MAX_ATTEMPTS, + exponential_wait=15, + retry_exceptions=(TransientInfraError,), +) + +# This is a heartbeat timeout, not a simulation runtime limit. +STALLED_HEARTBEAT_SECONDS = 90 + +# Delay requeued work so several jobs do not start at once after a worker crash. +CRASH_REQUEUE_DELAY_SECONDS = 30 + +CRASH_BUDGET_EXHAUSTED_ERROR = ( + "Simulation worker stopped responding mid-run (retry budget exhausted)" +) + +# Keep failed workspaces for local inspection and manual recovery. +WORK_DIR_TTL = timedelta(hours=24) + + +def enqueue_simulation( + conn: psycopg.Connection[JobRow], compute_job_id: uuid.UUID +) -> None: + """Queue the task on `conn` so it commits with the job row.""" + run_simulation_task.configure( + connection=conn, + queue=PROCRASTINATE_QUEUE, + queueing_lock=f"simulation:{compute_job_id}", + lock=f"compute-job:{compute_job_id}", + ).defer(compute_job_id=str(compute_job_id)) + + +@app.task( + name="api.run_simulation", + queue=PROCRASTINATE_QUEUE, + pass_context=True, + retry=TRANSIENT_RETRY, +) +def run_simulation_task(context: JobContext, compute_job_id: str) -> None: + """Run one simulation end to end, streaming progress into compute.jobs.""" + job_uuid = repository.as_uuid(compute_job_id) + + # A simulation keeps its connection for the duration of the run. + with connect() as conn: + row = repository.fetch_by_id(conn, job_uuid) + if row is None: + raise RuntimeError(f"Unknown compute job {compute_job_id}") + + simulation_id: uuid.UUID = row["simulation_id"] + work_dir = JOBS_DIR / str(simulation_id) + data = EarthquakeInput(**row["input_params"]) + + repository.mark_started(conn, job_uuid, simulation_id) + + def on_progress(message: str, details: dict[str, Any]) -> None: + repository.record_progress(conn, job_uuid, simulation_id, message, details) + + try: + result = run_simulation( + data, + work_dir, + # Keep outputs and checkpoints when a retry has a work directory. + resume=work_dir.exists(), + on_progress=on_progress, + ) + repository.complete_job(conn, row, result) + except Exception as e: + will_retry = ( + isinstance(e, TransientInfraError) + and context.job.attempts < MAX_ATTEMPTS + ) + repository.record_failure( + conn, + job_uuid, + simulation_id, + e, + step=repository.get_current_step(conn, job_uuid), + will_retry=will_retry, + ) + raise + else: + shutil.rmtree(work_dir, ignore_errors=True) + + +def reap_action(job: ProcrastinateJob) -> ReapAction: + return "retry" if job.attempts < MAX_ATTEMPTS else "exhausted" + + +@app.periodic(cron="*/2 * * * *") +@app.task(name="api.reap_stalled_jobs", queue=PROCRASTINATE_QUEUE) +async def reap_stalled_jobs_task(timestamp: int) -> None: + stalled = list( + await app.job_manager.get_stalled_jobs( + seconds_since_heartbeat=STALLED_HEARTBEAT_SECONDS + ) + ) + if not stalled: + return + + retry_at = datetime.now().astimezone() + timedelta( + seconds=CRASH_REQUEUE_DELAY_SECONDS + ) + + for job in stalled: + if job.id is None: # pragma: no cover - persisted jobs always have an id + continue + compute_job_id = str(job.task_kwargs.get("compute_job_id", "")) + if not compute_job_id: + continue + + if reap_action(job) == "retry": + logger.warning( + "Requeuing stalled job %s (compute job %s): heartbeat went stale", + job.id, + sanitize_for_log(compute_job_id), + ) + try: + await app.job_manager.retry_job_by_id_async( + job_id=job.id, retry_at=retry_at + ) + except procrastinate_exceptions.ConnectorException: + logger.info("Stalled job %s resolved before requeue", job.id) + continue + + logger.warning( + "Retry budget exhausted for compute job %s: marking FAILED", + sanitize_for_log(compute_job_id), + ) + _fail_exhausted(compute_job_id) + try: + await app.job_manager.finish_job_by_id_async( + job_id=job.id, status=ProcrastinateStatus.FAILED, delete_job=False + ) + except procrastinate_exceptions.ConnectorException: + logger.info("Stalled job %s was already resolved", job.id) + + +def _fail_exhausted(compute_job_id: str) -> None: + job_uuid = repository.as_uuid(compute_job_id) + with pooled() as conn: + row = repository.fetch_by_id(conn, job_uuid) + if row is None or row["status"] in { + JobStatus.COMPLETED.value, + JobStatus.FAILED.value, + }: + return + repository.fail_job( + conn, job_uuid, row["simulation_id"], CRASH_BUDGET_EXHAUSTED_ERROR + ) + + +@app.periodic(cron="0 * * * *") +@app.task(name="api.sweep_abandoned_work_dirs", queue=PROCRASTINATE_QUEUE) +def sweep_abandoned_work_dirs_task(timestamp: int) -> None: + cutoff = datetime.now().astimezone() - WORK_DIR_TTL + for simulation_id in repository.list_abandoned_work_dirs(cutoff): + shutil.rmtree(JOBS_DIR / simulation_id, ignore_errors=True) diff --git a/packages/api/api/main.py b/packages/api/api/main.py index 78cf093..ba431de 100644 --- a/packages/api/api/main.py +++ b/packages/api/api/main.py @@ -8,12 +8,14 @@ from fastapi.middleware.cors import CORSMiddleware from api import __version__ +from api.core.db import close_pool, get_pool +from api.core.settings import LOG_LEVEL from api.routes import get_calculator, ops_router, router +# Send logs to stdout so container runtimes can collect them. logging.basicConfig( - filename=os.environ.get("TSDHN_API_LOG", "tsunami_api.log"), - level=logging.DEBUG, - format="%(asctime)s - %(levelname)s - %(message)s", + level=LOG_LEVEL, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) logger = logging.getLogger(__name__) @@ -21,8 +23,13 @@ @asynccontextmanager async def lifespan(_app: FastAPI) -> AsyncIterator[None]: get_calculator() + # Open the pool without waiting for Postgres. + get_pool() logger.info("TSDHN API ready") - yield + try: + yield + finally: + close_pool() def create_app() -> FastAPI: @@ -52,12 +59,11 @@ def create_app() -> FastAPI: def start_app() -> None: - # Containers set APP_HOST=0.0.0.0. Local runs stay on loopback by default. uvicorn.run( app, host=os.environ.get("APP_HOST", "127.0.0.1"), port=int(os.environ.get("APP_PORT", "8000")), - log_level="info", + log_level=LOG_LEVEL.lower(), ) diff --git a/packages/api/api/migrate.py b/packages/api/api/migrate.py index e514778..3315e97 100644 --- a/packages/api/api/migrate.py +++ b/packages/api/api/migrate.py @@ -1,9 +1,142 @@ -from api.core.jobs import install_compute_schema +"""Create the compute schema and provision the web database role. + +Run this before applying the Procrastinate and web schemas. The command uses +the database owner for migrations; the web role is a runtime-only role. +""" + +import logging + +import psycopg +from psycopg import sql + +from api.core.schema import COMPUTE_SCHEMA_SQL +from api.core.settings import ( + APP_DB_PASSWORD, + APP_DB_ROLE, + COMPUTE_DATABASE_URL, +) + +logger = logging.getLogger(__name__) + + +def install_compute_schema(conn: psycopg.Connection[tuple[str, ...]]) -> None: + conn.execute(COMPUTE_SCHEMA_SQL) + + +def transfer_web_ownership( + conn: psycopg.Connection[tuple[str, ...]], + web_role: str, + migration_role: sql.Identifier, +) -> None: + """Move objects from the pre-runtime-role design to the owner role.""" + objects = conn.execute( + """ + SELECT n.nspname, c.relname, c.relkind + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname IN ('public', 'drizzle') + AND c.relkind IN ('r', 'p', 'S') + AND pg_get_userbyid(c.relowner) = %s + """, + [web_role], + ).fetchall() + for schema_name, name, kind in objects: + object_type = "SEQUENCE" if kind == "S" else "TABLE" + conn.execute( + sql.SQL( + "ALTER {object_type} {schema_name}.{name} OWNER TO {migration_role}" + ).format( + object_type=sql.SQL(object_type), + schema_name=sql.Identifier(schema_name), + name=sql.Identifier(name), + migration_role=migration_role, + ) + ) + + drizzle_owner = conn.execute( + """ + SELECT 1 + FROM pg_namespace + WHERE nspname = 'drizzle' AND nspowner = %s::regrole + """, + [web_role], + ).fetchone() + if drizzle_owner: + conn.execute( + sql.SQL("ALTER SCHEMA drizzle OWNER TO {migration_role}").format( + migration_role=migration_role + ) + ) + conn.execute( + sql.SQL("REVOKE ALL PRIVILEGES ON SCHEMA drizzle FROM {web_role}").format( + web_role=sql.Identifier(web_role) + ) + ) + conn.execute("REVOKE CREATE ON SCHEMA drizzle FROM PUBLIC") + + +def provision_web_role(conn: psycopg.Connection[tuple[str, ...]]) -> None: + """Create the runtime web role with only its required data privileges.""" + if not APP_DB_PASSWORD: + raise RuntimeError( + "APP_DB_PASSWORD must be set: it is the password for the " + f"web app's database role ({APP_DB_ROLE})." + ) + + role = sql.Identifier(APP_DB_ROLE) + database = sql.Identifier(conn.info.dbname) + migration_role = sql.Identifier(conn.info.user) + exists = conn.execute( + "SELECT 1 FROM pg_roles WHERE rolname = %s", [APP_DB_ROLE] + ).fetchone() + + action = sql.SQL("ALTER ROLE") if exists else sql.SQL("CREATE ROLE") + conn.execute( + sql.SQL("{action} {role} LOGIN PASSWORD {password}").format( + action=action, role=role, password=sql.Literal(APP_DB_PASSWORD) + ) + ) + transfer_web_ownership(conn, APP_DB_ROLE, migration_role) + + # The web role is deliberately not a migration role. Re-running + # provisioning repairs these security boundaries if it was over-granted. + for statement in ( + sql.SQL("REVOKE CREATE ON DATABASE {database} FROM {role}"), + sql.SQL("REVOKE CREATE ON SCHEMA public FROM PUBLIC"), + sql.SQL("REVOKE CREATE ON SCHEMA public FROM {role}"), + sql.SQL("REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM {role}"), + sql.SQL("REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM {role}"), + sql.SQL("REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA compute FROM {role}"), + sql.SQL("REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA compute FROM {role}"), + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {migration_role} IN SCHEMA public " + "REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM {role}" + ), + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {migration_role} IN SCHEMA public " + "REVOKE USAGE, SELECT, UPDATE ON SEQUENCES FROM {role}" + ), + sql.SQL("REVOKE ALL PRIVILEGES ON compute.jobs FROM {role}"), + sql.SQL("GRANT CONNECT ON DATABASE {database} TO {role}"), + sql.SQL("GRANT USAGE ON SCHEMA public TO {role}"), + sql.SQL("GRANT USAGE ON SCHEMA compute TO {role}"), + sql.SQL("GRANT SELECT ON compute.jobs TO {role}"), + ): + conn.execute( + statement.format( + role=role, database=database, migration_role=migration_role + ) + ) def main() -> None: - install_compute_schema() + logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") + with psycopg.connect(COMPUTE_DATABASE_URL, connect_timeout=5) as conn: + install_compute_schema(conn) + provision_web_role(conn) + conn.commit() + logger.info("compute schema applied; role %s provisioned", APP_DB_ROLE) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover main() diff --git a/packages/api/api/queue_migrate.py b/packages/api/api/queue_migrate.py new file mode 100644 index 0000000..d7f3527 --- /dev/null +++ b/packages/api/api/queue_migrate.py @@ -0,0 +1,192 @@ +"""Apply the vendor queue schema once and make the operation repeatable.""" + +from __future__ import annotations + +import logging + +import procrastinate +import psycopg +from psycopg import sql + +from api.core.settings import ( + COMPUTE_DATABASE_URL, + PROCRASTINATE_SCHEMA, + PROCRASTINATE_SEARCH_PATH, +) + +logger = logging.getLogger(__name__) + + +def queue_schema_state( + conn: psycopg.Connection[tuple[str, ...]], schema: str = PROCRASTINATE_SCHEMA +) -> tuple[bool, ...]: + """Return whether the required Procrastinate types and tables exist.""" + result = conn.execute( + """ + SELECT + to_regtype(%s) IS NOT NULL, + to_regtype(%s) IS NOT NULL, + to_regtype(%s) IS NOT NULL, + to_regclass(%s) IS NOT NULL, + to_regclass(%s) IS NOT NULL, + to_regclass(%s) IS NOT NULL, + to_regclass(%s) IS NOT NULL + """, + [ + f"{schema}.procrastinate_job_status", + f"{schema}.procrastinate_job_event_type", + f"{schema}.procrastinate_job_to_defer_v1", + f"{schema}.procrastinate_workers", + f"{schema}.procrastinate_jobs", + f"{schema}.procrastinate_periodic_defers", + f"{schema}.procrastinate_events", + ], + ).fetchone() + if result is None: + raise RuntimeError("failed to inspect the Procrastinate schema") + return tuple(bool(value) for value in result) + + +def move_legacy_schema(conn: psycopg.Connection[tuple[str, ...]]) -> None: + """Move a pre-compute-schema Procrastinate install into `compute`.""" + for type_name in ( + "procrastinate_job_status", + "procrastinate_job_event_type", + "procrastinate_job_to_defer_v1", + ): + conn.execute( + sql.SQL("ALTER TYPE public.{name} SET SCHEMA compute").format( + name=sql.Identifier(type_name) + ) + ) + + objects = conn.execute( + """ + SELECT c.relname, c.relkind + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' + AND c.relname LIKE 'procrastinate_%' + AND c.relkind IN ('r', 'p') + """ + ).fetchall() + for name, _kind in objects: + conn.execute( + sql.SQL("ALTER TABLE public.{name} SET SCHEMA compute").format( + name=sql.Identifier(name) + ) + ) + + sequences = conn.execute( + """ + SELECT sequence.relname, table_.relname, column_.attname + FROM pg_class AS sequence + JOIN pg_namespace AS sequence_schema + ON sequence_schema.oid = sequence.relnamespace + JOIN pg_depend AS dependency + ON dependency.classid = 'pg_class'::regclass + AND dependency.objid = sequence.oid + AND dependency.deptype = 'a' + JOIN pg_class AS table_ ON table_.oid = dependency.refobjid + JOIN pg_namespace AS table_schema + ON table_schema.oid = table_.relnamespace + JOIN pg_attribute AS column_ + ON column_.attrelid = table_.oid + AND column_.attnum = dependency.refobjsubid + WHERE sequence_schema.nspname = 'public' + AND sequence.relkind = 'S' + AND table_schema.nspname = 'compute' + """ + ).fetchall() + for sequence_name, table_name, column_name in sequences: + sequence_identifier = sql.Identifier(sequence_name) + conn.execute( + sql.SQL("ALTER SEQUENCE public.{name} OWNED BY NONE").format( + name=sequence_identifier + ) + ) + conn.execute( + sql.SQL("ALTER SEQUENCE public.{name} SET SCHEMA compute").format( + name=sequence_identifier + ) + ) + conn.execute( + sql.SQL( + "ALTER SEQUENCE compute.{sequence_name} " + "OWNED BY compute.{table_name}.{column_name}" + ).format( + sequence_name=sequence_identifier, + table_name=sql.Identifier(table_name), + column_name=sql.Identifier(column_name), + ) + ) + + conn.execute( + """ + DO $$ + DECLARE function_record record; + BEGIN + FOR function_record IN + SELECT p.proname, pg_get_function_identity_arguments(p.oid) AS arguments + FROM pg_proc AS p + JOIN pg_namespace AS n ON n.oid = p.pronamespace + WHERE n.nspname = 'public' AND p.proname LIKE 'procrastinate_%' + LOOP + EXECUTE format( + 'ALTER FUNCTION public.%I(%s) SET SCHEMA compute', + function_record.proname, + function_record.arguments + ); + END LOOP; + END $$; + """ + ) + + +def apply_schema(conninfo: str) -> None: + """Apply Procrastinate's schema in `compute`, rejecting partial installs.""" + with psycopg.connect(conninfo) as conn: + state = queue_schema_state(conn) + legacy_state = queue_schema_state(conn, "public") + + if all(state): + logger.info("Procrastinate schema already applied") + return + if all(legacy_state): + with psycopg.connect(conninfo) as conn: + move_legacy_schema(conn) + if not all(queue_schema_state(conn)): + raise RuntimeError("failed to move the Procrastinate schema to compute") + conn.commit() + logger.info("Procrastinate schema moved from public to compute") + return + if any(state): + raise RuntimeError( + "partial Procrastinate schema in compute; repair it before retrying" + ) + if any(legacy_state): + raise RuntimeError( + "partial Procrastinate schema in public; repair it before retrying" + ) + + connector = procrastinate.PsycopgConnector( + conninfo=conninfo, + kwargs={"options": f"-c search_path={PROCRASTINATE_SEARCH_PATH}"}, + ) + sync_connector = connector.get_sync_connector() + sync_connector.open() + try: + schema_manager = procrastinate.App(connector=connector).schema_manager + schema_manager.apply_schema() + finally: + sync_connector.close() + logger.info("Procrastinate schema applied") + + +def main() -> None: # pragma: no cover + logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") + apply_schema(COMPUTE_DATABASE_URL) + + +if __name__ == "__main__": # pragma: no cover + main() diff --git a/packages/api/api/routes.py b/packages/api/api/routes.py index d470ea9..9db99b5 100644 --- a/packages/api/api/routes.py +++ b/packages/api/api/routes.py @@ -1,41 +1,44 @@ -""" -Two routers: `ops_router` is unauthenticated (health/version, for liveness -probes); `router` carries the service-token dependency on every data route. -""" - import json import logging import tempfile from collections.abc import AsyncIterator -from datetime import datetime +from datetime import UTC, datetime from functools import lru_cache, partial from pathlib import Path +from typing import Any import anyio +import psycopg from fastapi import APIRouter, Depends, HTTPException, status -from fastapi.responses import StreamingResponse +from fastapi.responses import RedirectResponse, StreamingResponse from api import __version__ -from api.core.jobs import ( - JobStatus, - compute_jobs, -) +from api.core import repository +from api.core.db import CONNECT_TIMEOUT, notify_channel +from api.core.settings import COMPUTE_DATABASE_URL, SSE_MAX_DURATION +from api.core.storage import output_store +from api.core.tasks import enqueue_simulation from api.schemas import ( CalculationPreview, HealthStatus, JobCreated, JobRequest, JobStatusResponse, + OutputList, + StoredOutput, VersionInfo, ) -from api.security import require_service_token +from api.security import require_compute_api_token from tsdhn.calculator import TsunamiCalculator -from tsdhn.domain import EarthquakeInput +from tsdhn.domain import EarthquakeInput, JobStatus logger = logging.getLogger(__name__) _TERMINAL = {JobStatus.COMPLETED.value, JobStatus.FAILED.value} +# Proxies may close an idle SSE connection without a keepalive. +_KEEPALIVE_SECONDS = 20.0 + @lru_cache(maxsize=1) def get_calculator() -> TsunamiCalculator: @@ -46,7 +49,7 @@ def get_calculator() -> TsunamiCalculator: ops_router = APIRouter(prefix="/api/v1", tags=["ops"]) router = APIRouter( prefix="/api/v1", - dependencies=[Depends(require_service_token)], + dependencies=[Depends(require_compute_api_token)], tags=["jobs"], ) @@ -54,14 +57,12 @@ def get_calculator() -> TsunamiCalculator: @ops_router.get("/health", response_model=HealthStatus) async def health() -> HealthStatus: database_connected = await anyio.to_thread.run_sync( - compute_jobs.is_database_connected - ) - storage_connected = await anyio.to_thread.run_sync( - compute_jobs.is_storage_connected + repository.is_database_connected ) + storage_connected = await anyio.to_thread.run_sync(output_store.is_connected) return HealthStatus( status="healthy" if database_connected and storage_connected else "degraded", - timestamp=datetime.now().isoformat(), + timestamp=datetime.now(UTC).isoformat(), database_connected=database_connected, storage_connected=storage_connected, ) @@ -92,13 +93,14 @@ def compute() -> CalculationPreview: status_code=status.HTTP_201_CREATED, ) async def create_job(req: JobRequest) -> JobCreated: - app_job_id = str(req.app_job_id) + simulation_id = str(req.simulation_id) try: job_status = await anyio.to_thread.run_sync( partial( - compute_jobs.create_or_get_job, + repository.create_or_get_job, data=req.input, - external_id=app_job_id, + simulation_id=simulation_id, + defer=enqueue_simulation, ) ) except ValueError as e: @@ -112,34 +114,125 @@ async def create_job(req: JobRequest) -> JobCreated: detail="Failed to start simulation pipeline", ) from e - return JobCreated(app_job_id=app_job_id, **job_status) + return JobCreated( + simulation_id=simulation_id, + status=job_status["status"], + ) + + +@router.get("/jobs/{simulation_id}", response_model=JobStatusResponse) +async def get_job(simulation_id: str) -> JobStatusResponse: + job_status = await _job_status(simulation_id) + return JobStatusResponse(simulation_id=simulation_id, **job_status) -@router.get("/jobs/{app_job_id}", response_model=JobStatusResponse) -async def get_job(app_job_id: str) -> JobStatusResponse: +@router.get("/jobs/{simulation_id}/outputs", response_model=OutputList) +async def list_outputs(simulation_id: str) -> OutputList: try: - job_status = await anyio.to_thread.run_sync( - compute_jobs.get_job_status, app_job_id - ) + outputs = await anyio.to_thread.run_sync(repository.get_outputs, simulation_id) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) from e - return JobStatusResponse(app_job_id=app_job_id, **job_status) + return OutputList( + simulation_id=simulation_id, + outputs=[ + StoredOutput( + name=output["name"], + filename=output["filename"], + content_type=output["content_type"], + ) + for output in outputs + ], + ) + + +@router.get( + "/jobs/{simulation_id}/outputs/{name}", + status_code=status.HTTP_307_TEMPORARY_REDIRECT, + response_class=RedirectResponse, +) +async def get_output(simulation_id: str, name: str) -> RedirectResponse: + try: + outputs = await anyio.to_thread.run_sync(repository.get_outputs, simulation_id) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) from e + + match = next((output for output in outputs if output["name"] == name), None) + if match is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"No output '{name}' for this job", + ) + + url = await anyio.to_thread.run_sync( + partial(output_store.presigned_url, match["key"], filename=match["filename"]) + ) + return RedirectResponse(url=url, status_code=status.HTTP_307_TEMPORARY_REDIRECT) -@router.get("/jobs/{app_job_id}/events") -async def job_events(app_job_id: str) -> StreamingResponse: +@router.get("/jobs/{simulation_id}/events") +async def job_events(simulation_id: str) -> StreamingResponse: + """Stream job state changes from the job's Postgres notification channel.""" + try: + simulation_uuid = repository.as_uuid(simulation_id) + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Invalid or unknown job ID", + ) from e + channel = notify_channel(simulation_uuid) + async def stream() -> AsyncIterator[str]: - while True: - try: - job_status = await anyio.to_thread.run_sync( - compute_jobs.get_job_status, app_job_id - ) - except ValueError: - yield f"event: error\ndata: {json.dumps({'error': 'unknown job'})}\n\n" - return - yield f"data: {json.dumps({'app_job_id': app_job_id, **job_status})}\n\n" - if job_status["status"] in _TERMINAL: - return - await anyio.sleep(2) - - return StreamingResponse(stream(), media_type="text/event-stream") + try: + job_status = await _job_status(simulation_id) + except HTTPException: + yield _sse_error("unknown job") + return + + yield _sse_data(simulation_id, job_status) + if job_status["status"] in _TERMINAL: + return + + deadline = anyio.current_time() + SSE_MAX_DURATION + aconn = await psycopg.AsyncConnection.connect( + COMPUTE_DATABASE_URL, autocommit=True, connect_timeout=CONNECT_TIMEOUT + ) + try: + await aconn.execute(f"LISTEN {channel}") + last = job_status + while anyio.current_time() < deadline: + notified = False + async for _ in aconn.notifies(timeout=_KEEPALIVE_SECONDS, stop_after=1): + notified = True + + # Read on every tick to cover notifications that race the wait. + job_status = await _job_status(simulation_id) + if job_status != last: + last = job_status + yield _sse_data(simulation_id, job_status) + if job_status["status"] in _TERMINAL: + return + elif not notified: + yield ": keepalive\n\n" + finally: + await aconn.close() + + return StreamingResponse( + stream(), + media_type="text/event-stream", + headers={"cache-control": "no-cache", "x-accel-buffering": "no"}, + ) + + +async def _job_status(simulation_id: str) -> dict[str, Any]: + try: + return await anyio.to_thread.run_sync(repository.get_job_status, simulation_id) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) from e + + +def _sse_data(simulation_id: str, job_status: dict[str, Any]) -> str: + return f"data: {json.dumps({'simulation_id': simulation_id, **job_status})}\n\n" + + +def _sse_error(message: str) -> str: + return f"event: error\ndata: {json.dumps({'error': message})}\n\n" diff --git a/packages/api/api/schemas.py b/packages/api/api/schemas.py index 3166dd5..167a54e 100644 --- a/packages/api/api/schemas.py +++ b/packages/api/api/schemas.py @@ -10,6 +10,8 @@ "JobCreated", "JobRequest", "JobStatusResponse", + "OutputList", + "StoredOutput", "VersionInfo", ] @@ -22,21 +24,17 @@ class CalculationPreview(BaseModel): class JobRequest(BaseModel): - app_job_id: UUID + simulation_id: UUID input: EarthquakeInput class JobCreated(BaseModel): - app_job_id: str - compute_job_id: str + simulation_id: str status: str - result_bucket: str | None = None - result_key: str | None = None class JobStatusResponse(BaseModel): - app_job_id: str - compute_job_id: str + simulation_id: str status: str details: str | None = None step: str | None = None @@ -44,13 +42,23 @@ class JobStatusResponse(BaseModel): total_steps: int | None = None calculation: CalculationResponse | None = None travel_times: TsunamiTravelResponse | None = None - result_bucket: str | None = None - result_key: str | None = None error: str | None = None created_at: str | None = None started_at: str | None = None finished_at: str | None = None - artifacts_available: bool = False + # Output names are public. Storage keys remain inside the compute service. + outputs: list[str] = [] + + +class StoredOutput(BaseModel): + name: str + filename: str + content_type: str + + +class OutputList(BaseModel): + simulation_id: str + outputs: list[StoredOutput] class HealthStatus(BaseModel): diff --git a/packages/api/api/security.py b/packages/api/api/security.py index e261dc7..e4d72c5 100644 --- a/packages/api/api/security.py +++ b/packages/api/api/security.py @@ -1,7 +1,4 @@ -""" -The browser never calls this API directly; the SvelteKit BFF does, server to -server, presenting a shared secret. Every data route depends on this check. -""" +"""Authentication for requests from the SvelteKit server to the compute API.""" import os import secrets @@ -9,12 +6,14 @@ from fastapi import Header, HTTPException, status -def require_service_token(authorization: str | None = Header(default=None)) -> None: - expected = os.environ.get("BACKEND_SERVICE_TOKEN") +def require_compute_api_token( + authorization: str | None = Header(default=None), +) -> None: + expected = os.environ.get("COMPUTE_API_TOKEN") if not expected: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Service token not configured", + detail="Compute API token not configured", ) token = "" @@ -24,5 +23,5 @@ def require_service_token(authorization: str | None = Header(default=None)) -> N if not token or not secrets.compare_digest(token, expected): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid or missing service token", + detail="Invalid or missing compute API token", ) diff --git a/packages/api/api/web_grants.py b/packages/api/api/web_grants.py new file mode 100644 index 0000000..cbf1c68 --- /dev/null +++ b/packages/api/api/web_grants.py @@ -0,0 +1,47 @@ +"""Grant the runtime web role access to web-owned public tables.""" + +from __future__ import annotations + +import logging + +import psycopg +from psycopg import sql + +from api.core.settings import APP_DB_ROLE, COMPUTE_DATABASE_URL + +logger = logging.getLogger(__name__) + + +def grant_web_tables(conn: psycopg.Connection[tuple[str, ...]]) -> None: + role = sql.Identifier(APP_DB_ROLE) + migration_role = sql.Identifier(conn.info.user) + for statement in ( + sql.SQL( + "GRANT SELECT, INSERT, UPDATE, DELETE " + "ON ALL TABLES IN SCHEMA public TO {role}" + ), + sql.SQL( + "GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO {role}" + ), + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {migration_role} " + "IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO {role}" + ), + sql.SQL( + "ALTER DEFAULT PRIVILEGES FOR ROLE {migration_role} " + "IN SCHEMA public GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO {role}" + ), + ): + conn.execute(statement.format(role=role, migration_role=migration_role)) + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") + with psycopg.connect(COMPUTE_DATABASE_URL) as conn: + grant_web_tables(conn) + conn.commit() + logger.info("web table privileges granted to %s", APP_DB_ROLE) + + +if __name__ == "__main__": + main() diff --git a/packages/api/api/worker.py b/packages/api/api/worker.py index 47076b7..7d82150 100644 --- a/packages/api/api/worker.py +++ b/packages/api/api/worker.py @@ -1,20 +1,36 @@ import logging +import numba + +from api.core.db import close_pool from api.core.procrastinate_app import app -from api.core.settings import PROCRASTINATE_QUEUE +from api.core.settings import LOG_LEVEL, NUMBA_THREADS, PROCRASTINATE_QUEUE logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + level=LOG_LEVEL, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) +logger = logging.getLogger(__name__) + +def main() -> None: # pragma: no cover + if NUMBA_THREADS is not None: + # Numba lacks type stubs; suppress type checking. + numba.set_num_threads(NUMBA_THREADS) # type: ignore[no-untyped-call] + logger.info( + "numba parallel-region thread count capped to %d (TSDHN_NUMBA_THREADS)", + NUMBA_THREADS, + ) -def main() -> None: app.open() try: app.run_worker(queues=[PROCRASTINATE_QUEUE]) finally: app.close() + # The periodic tasks use the read pool; close it so its worker + # threads are joined before interpreter shutdown. + close_pool() -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover main() diff --git a/packages/api/pyproject.toml b/packages/api/pyproject.toml index e28c7aa..b0c9817 100644 --- a/packages/api/pyproject.toml +++ b/packages/api/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "uvicorn==0.51.0", "pydantic==2.13.4", "anyio==4.14.1", - "psycopg[binary]>=3.3.4", + "psycopg[binary,pool]>=3.3.4", "minio>=7.2.20", "procrastinate>=3.9.0", ] @@ -23,6 +23,8 @@ dependencies = [ tsdhn-api = "api.main:start_app" tsdhn-worker = "api.worker:main" tsdhn-compute-migrate = "api.migrate:main" +tsdhn-procrastinate-migrate = "api.queue_migrate:main" +tsdhn-web-grants = "api.web_grants:main" [build-system] requires = ["uv_build==0.11.28"] diff --git a/packages/api/readme.md b/packages/api/readme.md index 0027066..278840c 100644 --- a/packages/api/readme.md +++ b/packages/api/readme.md @@ -1,136 +1,84 @@ # tsdhn-api -`tsdhn-api` serves the TSDHN simulation API. It stores job state in Postgres, -runs long simulations through Procrastinate, writes completed artifacts to -MinIO, and uses the shared `tsdhn` engine. +`tsdhn-api` accepts requests from the web server, records job state in +PostgreSQL, queues work with Procrastinate, runs the shared `tsdhn` engine, +and stores output files in MinIO. -## Entry Points +The browser does not call this service. The web app calls it with +`COMPUTE_API_TOKEN`. See [`ARCHITECTURE.md`](../../ARCHITECTURE.md) for service +responsibilities and [`DEPLOY.md`](../../DEPLOY.md) for configuration, +migrations, storage, and worker resources. + +## Commands + +Run the service and worker from the repository root: ```sh uv run tsdhn-api uv run tsdhn-worker -uv run tsdhn-compute-migrate ``` -`tsdhn-api` starts the FastAPI application from [`api/main.py`](./api/main.py). -`tsdhn-worker` runs a Procrastinate worker from -[`api/worker.py`](./api/worker.py). `tsdhn-compute-migrate` creates the -application-owned `compute_jobs` table; run it after applying the Procrastinate -schema. - -The API documentation UI is served at: +Create the compute and queue tables before starting the API or worker: -```txt -http://localhost:8000/api-docs +```sh +uv run tsdhn-compute-migrate +uv run tsdhn-procrastinate-migrate ``` -## Environment - -| Variable | Used by | Default | Purpose | -| --- | --- | --- | --- | -| `APP_HOST` | API | `127.0.0.1` | Uvicorn bind host | -| `APP_PORT` | API | `8000` | Uvicorn bind port | -| `ALLOWED_ORIGINS` | API | empty | Comma-separated browser origins for direct CORS access | -| `BACKEND_SERVICE_TOKEN` | API | none | Required bearer token for simulation routes | -| `COMPUTE_DATABASE_URL` | API, worker, migration | `postgresql://tsdhn:tsdhn@localhost:5432/tsdhn_compute` | Worker Postgres connection | -| `PROCRASTINATE_QUEUE` | API, worker | `simulations` | Procrastinate queue name | -| `MINIO_ENDPOINT` | worker, API health | `localhost:9000` | MinIO or S3-compatible endpoint | -| `MINIO_ACCESS_KEY` | worker, API health | `minioadmin` | MinIO access key | -| `MINIO_SECRET_KEY` | worker, API health | `minioadmin` | MinIO secret key | -| `MINIO_BUCKET` | worker, API health | `tsdhn-results` | Bucket for artifacts and metadata | -| `MINIO_SECURE` | worker, API health | `false` | Use HTTPS for MinIO client connections | -| `TSDHN_API_LOG` | API | `tsunami_api.log` | API log file path | -| `TSDHN_MODEL_DIR` | API, worker | none | Model asset directory loaded by `tsdhn` | -| `TSDHN_TOOLS_DIR` | worker | none | Directory containing prebuilt model executables | -| `TSDHN_JOBS_DIR` | worker | `jobs` | Temporary per-job simulation workspace root | - -The API startup path loads `TsunamiCalculator`, so local API runs need -`TSDHN_MODEL_DIR`. The worker needs both model assets and tool executables for -full simulations. - -## Authentication - -Health and version routes are unauthenticated: - -- `GET /api/v1/health` -- `GET /api/v1/version` - -Simulation routes require: - -```txt -Authorization: Bearer $BACKEND_SERVICE_TOKEN -``` +For a complete deployment, including web migrations and grants, follow +[`DEPLOY.md`](../../DEPLOY.md). For local PostgreSQL, `mise run db-migrate` +applies every database change. `mise run test-integration` creates disposable +databases and runs the database-backed tests. -The SvelteKit app calls these routes server-to-server. Browser code calls the -web app, not FastAPI directly. +## API definition -## Routes +The OpenAPI UI at is the current list of +routes, inputs, and responses. Health and version checks are public. Simulation +and calculation routes require `COMPUTE_API_TOKEN`. -| Method | Path | Auth | Purpose | -| --- | --- | --- | --- | -| `GET` | `/api/v1/health` | No | API liveness, Postgres readiness, and MinIO readiness | -| `GET` | `/api/v1/version` | No | Package name and version | -| `POST` | `/api/v1/calculations` | Yes | Source-parameter and travel-time preview | -| `POST` | `/api/v1/jobs` | Yes | Enqueue a full simulation using the web app's `app_job_id` idempotency key | -| `GET` | `/api/v1/jobs/{app_job_id}` | Yes | Read queue status and completed metadata | -| `GET` | `/api/v1/jobs/{app_job_id}/events` | Yes | Server-sent progress stream | +The API accepts `simulation_id` as the web app's identifier. Repeating a +submission with the same ID and input returns the existing job. Output +responses expose output names and filenames, while storage keys remain private. +See [`ARCHITECTURE.md`](../../ARCHITECTURE.md) for the full submission and +download flows. -## Request examples +## Package map -Health check: +- `api/routes.py` defines health, calculation, submission, progress, and output + routes. +- `api/schemas.py` defines public request and response models. +- `api/security.py` checks `COMPUTE_API_TOKEN`. +- `api/core/repository.py` reads and updates compute jobs. +- `api/core/tasks.py` runs queued simulations and records progress. +- `api/core/storage.py` uploads output files and creates download URLs. +- `api/core/procrastinate_app.py` defines the queue and scheduled cleanup work. +- `api/migrate.py`, `api/queue_migrate.py`, and `api/web_grants.py` apply the + database changes described in `DEPLOY.md`. +- `api/worker.py` starts a worker for the configured queue. -```sh -curl -s http://localhost:8000/api/v1/health -``` +## Tests -Calculation preview: +Run the fast API tests with: ```sh -curl -s http://localhost:8000/api/v1/calculations \ - -H "Authorization: Bearer $BACKEND_SERVICE_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "Mw": 9.0, - "h": 12.0, - "lat0": -20.5, - "lon0": -70.5, - "dia": "23", - "hhmm": "0000" - }' +uv run --package tsdhn-api pytest packages/api/tests +uv run --package tsdhn-api pytest packages/api/tests/test_api.py::test_version ``` -Enqueue a simulation: +The integration tests use PostgreSQL and disposable databases: ```sh -curl -s http://localhost:8000/api/v1/jobs \ - -H "Authorization: Bearer $BACKEND_SERVICE_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "app_job_id": "4cfe522f-7e7d-46e0-96ca-7b98743fb9f5", - "input": { - "Mw": 9.0, - "h": 12.0, - "lat0": -20.5, - "lon0": -70.5, - "dia": "23", - "hhmm": "0000" - } - }' +mise run test-integration ``` -## Development +## Generated client -From the repository root: +After changing `api/routes.py` or `api/schemas.py`, regenerate the TypeScript +client: ```sh -uv run --package tsdhn-api pytest packages/api/tests -uv run --package tsdhn-api procrastinate --app=api.core.procrastinate_app.app schema --apply -uv run --package tsdhn-api tsdhn-compute-migrate -uv run python scripts/export_openapi.py +mise run gen-client ``` -Regenerate the TypeScript API client after route or schema changes: - -```sh -bun run gen:client -``` +The generation command exports OpenAPI from the running application code. Do +not edit the exported schema or generated TypeScript by hand. diff --git a/packages/api/tests/conftest.py b/packages/api/tests/conftest.py new file mode 100644 index 0000000..afc1440 --- /dev/null +++ b/packages/api/tests/conftest.py @@ -0,0 +1,35 @@ +"""Shared fixtures for API database tests.""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator + +import psycopg +import pytest + +from api.core.schema import COMPUTE_SCHEMA_SQL +from scripts.database import create_database, drop_database + +LOCAL_DATABASE_URL = "postgresql://tsdhn:tsdhn@127.0.0.1:5432/tsdhn" + + +@pytest.fixture +def isolated_database() -> Iterator[str]: + """Yield a fresh compute database and remove it after the test.""" + database_name = f"tsdhn_test_{uuid.uuid4().hex}" + try: + target = create_database(LOCAL_DATABASE_URL, database_name) + except psycopg.OperationalError as error: + pytest.exit( + f"PostgreSQL is not reachable at {LOCAL_DATABASE_URL} ({error}). " + "Run `mise run db:start` or `mise run test-integration` first.", + returncode=1, + ) + + try: + with psycopg.connect(target.database_url) as connection: + connection.execute(COMPUTE_SCHEMA_SQL) + yield target.database_url + finally: + drop_database(LOCAL_DATABASE_URL, database_name) diff --git a/packages/api/tests/test_api.py b/packages/api/tests/test_api.py index 8ad7d91..895b4d9 100644 --- a/packages/api/tests/test_api.py +++ b/packages/api/tests/test_api.py @@ -1,19 +1,20 @@ -""" -The synchronous `/calculations` route exercises the real `TsunamiCalculator` -against the bundled model data; worker execution and MinIO uploads are covered -by the docker-compose end-to-end check, not here. -""" - -from collections.abc import Iterator +import json +from collections.abc import AsyncIterator, Iterator from pathlib import Path +from typing import Any, cast import pytest +from fastapi import HTTPException from fastapi.testclient import TestClient -import api.routes as routes +from api import routes +from api.core import repository +from api.core.storage import output_store from api.main import app -TOKEN = "test-service-token" +routes_module: Any = routes + +TOKEN = "test-compute-api-token" SAMPLE = { "Mw": 8.0, "h": 10.0, @@ -22,12 +23,12 @@ "hhmm": "0000", "dia": "23", } -EXTERNAL_ID = "4cfe522f-7e7d-46e0-96ca-7b98743fb9f5" +SIMULATION_ID = "4cfe522f-7e7d-46e0-96ca-7b98743fb9f5" @pytest.fixture(autouse=True) def _service_token(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("BACKEND_SERVICE_TOKEN", TOKEN) + monkeypatch.setenv("COMPUTE_API_TOKEN", TOKEN) monkeypatch.setenv("TSDHN_MODEL_DIR", str(Path("model").resolve())) @@ -44,14 +45,8 @@ def _auth() -> dict[str, str]: def test_health_is_unauthenticated( client: TestClient, monkeypatch: pytest.MonkeyPatch ) -> None: - class ComputeJobsStub: - def is_database_connected(self) -> bool: - return True - - def is_storage_connected(self) -> bool: - return True - - monkeypatch.setattr(routes, "compute_jobs", ComputeJobsStub()) + monkeypatch.setattr(repository, "is_database_connected", lambda: True) + monkeypatch.setattr(output_store, "is_connected", lambda: True) response = client.get("/api/v1/health") assert response.status_code == 200 @@ -64,12 +59,62 @@ def is_storage_connected(self) -> bool: } +def test_health_reports_degraded_when_a_dependency_is_down( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(repository, "is_database_connected", lambda: True) + monkeypatch.setattr(output_store, "is_connected", lambda: False) + + assert client.get("/api/v1/health").json()["status"] == "degraded" + + def test_version(client: TestClient) -> None: response = client.get("/api/v1/version") assert response.status_code == 200 assert response.json()["name"] == "tsdhn-api" +def test_get_job_returns_repository_status( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + repository, + "get_job_status", + lambda _simulation_id: { + "status": "running", + "details": "Processing tsunami", + "step": "tsunami", + "step_index": 3, + "total_steps": 8, + "outputs": [], + }, + ) + + response = client.get(f"/api/v1/jobs/{SIMULATION_ID}", headers=_auth()) + + assert response.status_code == 200 + assert response.json()["simulation_id"] == SIMULATION_ID + assert response.json()["status"] == "running" + assert response.json()["step_index"] == 3 + + +def test_get_job_maps_unknown_job_to_not_found( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def unknown(_simulation_id: str) -> dict[str, Any]: + raise ValueError("unknown") + + monkeypatch.setattr( + repository, + "get_job_status", + unknown, + ) + + response = client.get(f"/api/v1/jobs/{SIMULATION_ID}", headers=_auth()) + + assert response.status_code == 404 + + def test_calculations_rejects_missing_token(client: TestClient) -> None: response = client.post("/api/v1/calculations", json=SAMPLE) assert response.status_code == 401 @@ -88,49 +133,281 @@ def test_calculations_returns_preview(client: TestClient) -> None: def test_jobs_rejects_missing_token(client: TestClient) -> None: response = client.post( "/api/v1/jobs", - json={"app_job_id": EXTERNAL_ID, "input": SAMPLE}, + json={"simulation_id": SIMULATION_ID, "input": SAMPLE}, ) assert response.status_code == 401 -def test_jobs_use_app_job_id_as_idempotency_key( +def test_jobs_use_simulation_id_to_reuse_an_existing_job( client: TestClient, monkeypatch: pytest.MonkeyPatch ) -> None: - calls: list[dict[str, object]] = [] - - class ComputeJobsStub: - def create_or_get_job(self, **kwargs: object) -> dict[str, object]: - calls.append(kwargs) - return { - "compute_job_id": "compute-job-123", - "status": "queued", - "result_bucket": None, - "result_key": None, - } + calls: list[dict[str, Any]] = [] - monkeypatch.setattr(routes, "compute_jobs", ComputeJobsStub()) + def create_or_get_job(**kwargs: Any) -> dict[str, Any]: + calls.append(kwargs) + return { + "status": "queued", + } + + monkeypatch.setattr(repository, "create_or_get_job", create_or_get_job) response = client.post( "/api/v1/jobs", headers=_auth(), - json={"app_job_id": EXTERNAL_ID, "input": SAMPLE}, + json={"simulation_id": SIMULATION_ID, "input": SAMPLE}, ) assert response.status_code == 201 assert response.json() == { - "app_job_id": EXTERNAL_ID, - "compute_job_id": "compute-job-123", + "simulation_id": SIMULATION_ID, "status": "queued", - "result_bucket": None, - "result_key": None, } - assert calls[0]["external_id"] == EXTERNAL_ID + assert calls[0]["simulation_id"] == SIMULATION_ID + + +def test_outputs_require_a_token(client: TestClient) -> None: + assert client.get(f"/api/v1/jobs/{SIMULATION_ID}/outputs").status_code == 401 + + +def test_outputs_list_is_empty_until_the_job_completes( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(repository, "get_outputs", lambda _id: []) + + response = client.get(f"/api/v1/jobs/{SIMULATION_ID}/outputs", headers=_auth()) + assert response.status_code == 200 + assert response.json() == {"simulation_id": SIMULATION_ID, "outputs": []} + + +def test_outputs_list_names_what_the_job_produced( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + repository, + "get_outputs", + lambda _id: [ + { + "name": "max_height_map", + "key": f"simulations/{SIMULATION_ID}/outputs/maxola.pdf", + "filename": "maxola.pdf", + "content_type": "application/pdf", + } + ], + ) + + response = client.get(f"/api/v1/jobs/{SIMULATION_ID}/outputs", headers=_auth()) + assert response.status_code == 200 + assert response.json()["outputs"] == [ + { + "name": "max_height_map", + "filename": "maxola.pdf", + "content_type": "application/pdf", + } + ] + # Storage object keys must not reach the client. + assert "key" not in response.text + + +def test_output_download_redirects_to_a_presigned_url( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + repository, + "get_outputs", + lambda _id: [ + { + "name": "max_height_map", + "key": f"simulations/{SIMULATION_ID}/outputs/maxola.pdf", + "filename": "maxola.pdf", + "content_type": "application/pdf", + } + ], + ) + monkeypatch.setattr( + output_store, + "presigned_url", + lambda key, *, filename: f"https://minio.example/{key}?sig=abc", + ) + + response = client.get( + f"/api/v1/jobs/{SIMULATION_ID}/outputs/max_height_map", + headers=_auth(), + follow_redirects=False, + ) + assert response.status_code == 307 + assert response.headers["location"].startswith("https://minio.example/simulations/") + + +def test_unknown_output_name_is_404( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(repository, "get_outputs", lambda _id: []) + + response = client.get( + f"/api/v1/jobs/{SIMULATION_ID}/outputs/nope", + headers=_auth(), + follow_redirects=False, + ) + assert response.status_code == 404 + + +def test_output_download_maps_an_unknown_job_to_not_found( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def unknown(_simulation_id: str) -> list[dict[str, str]]: + raise ValueError("unknown") + + monkeypatch.setattr(repository, "get_outputs", unknown) + + response = client.get( + f"/api/v1/jobs/{SIMULATION_ID}/outputs/max_height_map", + headers=_auth(), + follow_redirects=False, + ) + + assert response.status_code == 404 + + +def test_job_events_maps_an_invalid_job_id_to_not_found(client: TestClient) -> None: + response = client.get( + "/api/v1/jobs/not-a-uuid/events", + headers=_auth(), + ) + + assert response.status_code == 404 + + +class _FakeAsyncConnection: + def __init__(self, *, notify: bool) -> None: + self.notify = notify + self.executed: list[str] = [] + self.closed = False + + async def execute(self, statement: str) -> None: + self.executed.append(statement) + + async def notifies( + self, *, timeout: float, stop_after: int + ) -> AsyncIterator[object]: + if self.notify: + yield object() + + async def close(self) -> None: + self.closed = True + + +@pytest.mark.asyncio +async def test_job_events_emits_a_terminal_snapshot_without_opening_a_listener( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def status(_simulation_id: str) -> dict[str, Any]: + return {"status": "completed", "outputs": ["result"]} + + monkeypatch.setattr(routes, "_job_status", status) + + response = await routes.job_events(SIMULATION_ID) + chunks = [chunk async for chunk in response.body_iterator] + + assert len(chunks) == 1 + assert json.loads(cast(str, chunks[0])[len("data: ") : -2]) == { + "simulation_id": SIMULATION_ID, + "status": "completed", + "outputs": ["result"], + } + + +@pytest.mark.asyncio +async def test_job_events_emits_an_sse_error_for_an_unknown_job( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def unknown(_simulation_id: str) -> dict[str, Any]: + raise HTTPException(status_code=404) + + monkeypatch.setattr(routes, "_job_status", unknown) + + response = await routes.job_events(SIMULATION_ID) + chunks = [chunk async for chunk in response.body_iterator] + + assert chunks == ['event: error\ndata: {"error": "unknown job"}\n\n'] + + +@pytest.mark.asyncio +async def test_job_events_reads_a_changed_snapshot_and_closes_the_listener( + monkeypatch: pytest.MonkeyPatch, +) -> None: + snapshots = iter( + [ + {"status": "running", "step": "tsunami"}, + {"status": "completed", "step": "tsunami"}, + ] + ) + connection = _FakeAsyncConnection(notify=True) + + async def status(_simulation_id: str) -> dict[str, Any]: + return next(snapshots) + + async def connect(*args: Any, **kwargs: Any) -> _FakeAsyncConnection: + return connection + + monkeypatch.setattr(routes, "_job_status", status) + monkeypatch.setattr(routes_module.psycopg.AsyncConnection, "connect", connect) + monkeypatch.setattr(routes_module.anyio, "current_time", lambda: 0.0) + + response = await routes.job_events(SIMULATION_ID) + chunks = [chunk async for chunk in response.body_iterator] + + assert len(chunks) == 2 + assert '"status": "completed"' in chunks[1] + assert connection.executed == ["LISTEN tsdhn_job_4cfe522f7e7d46e096ca7b98743fb9f5"] + assert connection.closed + + +@pytest.mark.asyncio +async def test_job_events_sends_keepalive_when_state_does_not_change( + monkeypatch: pytest.MonkeyPatch, +) -> None: + snapshot = {"status": "running", "step": "tsunami"} + connection = _FakeAsyncConnection(notify=False) + snapshots = iter([snapshot, snapshot]) + clock = iter([0.0, 0.0, float("inf")]) + + async def status(_simulation_id: str) -> dict[str, Any]: + return next(snapshots) + + async def connect(*args: Any, **kwargs: Any) -> _FakeAsyncConnection: + return connection + + monkeypatch.setattr(routes, "_job_status", status) + monkeypatch.setattr(routes_module.psycopg.AsyncConnection, "connect", connect) + monkeypatch.setattr(routes_module.anyio, "current_time", lambda: next(clock)) + + response = await routes.job_events(SIMULATION_ID) + chunks = [chunk async for chunk in response.body_iterator] + + assert chunks[-1] == ": keepalive\n\n" + assert connection.closed def test_legacy_simulations_endpoint_is_removed(client: TestClient) -> None: response = client.post( "/api/v1/simulations", headers=_auth(), - json={"app_job_id": EXTERNAL_ID, "input": SAMPLE}, + json={"simulation_id": SIMULATION_ID, "input": SAMPLE}, ) assert response.status_code == 404 + + +def test_public_error_does_not_leak_exception_message() -> None: + leaky = FileNotFoundError( + "[Errno 2] No such file or directory: " + "'/home/dubu/picv-2025/jobs/f5171f2a/tsunami'" + ) + + with_step = repository._public_error(leaky, "tsunami") + without_step = repository._public_error(leaky, None) + + assert with_step == "Simulation failed at step 'tsunami' (FileNotFoundError)" + assert without_step == "Simulation failed (FileNotFoundError)" + for message in (with_step, without_step): + assert "/home/" not in message + assert "jobs" not in message diff --git a/packages/api/tests/test_db.py b/packages/api/tests/test_db.py new file mode 100644 index 0000000..957fc54 --- /dev/null +++ b/packages/api/tests/test_db.py @@ -0,0 +1,44 @@ +"""Connection boundary behavior for the compute service.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +import psycopg +import pytest + +from api.core import db +from api.core.errors import TransientInfraError + +db_module: Any = db + + +class _Pool: + def __init__(self) -> None: + self.connection_value = object() + + @contextmanager + def connection(self) -> Iterator[object]: + yield self.connection_value + + +def test_pooled_yields_a_connection_from_the_process_pool( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pool = _Pool() + monkeypatch.setattr(db_module, "get_pool", lambda: pool) + + with db.pooled() as connection: + assert connection is pool.connection_value + + +def test_connect_classifies_a_database_outage_as_transient( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fail(*args: Any, **kwargs: Any) -> object: + raise psycopg.OperationalError("database unavailable") + + monkeypatch.setattr(db_module.psycopg, "connect", fail) + + with pytest.raises(TransientInfraError, match="database connection failed"): + db.connect() diff --git a/packages/api/tests/test_jobs.py b/packages/api/tests/test_jobs.py new file mode 100644 index 0000000..9a2dab1 --- /dev/null +++ b/packages/api/tests/test_jobs.py @@ -0,0 +1,63 @@ +"""Database-free tests for retry, failure, and notification behavior.""" + +import uuid + +from procrastinate.jobs import Job as ProcrastinateJob + +from api.core.db import notify_channel +from api.core.errors import TransientInfraError +from api.core.tasks import MAX_ATTEMPTS, TRANSIENT_RETRY, reap_action + + +def _job(attempts: int, job_id: int | None = None) -> ProcrastinateJob: + return ProcrastinateJob( + id=job_id, + queue="simulations", + lock=None, + queueing_lock=None, + task_name="api.run_simulation", + task_kwargs={"compute_job_id": "11111111-1111-4111-8111-111111111111"}, + attempts=attempts, + ) + + +def test_transient_retry_retries_transient_infra_error_within_budget() -> None: + decision = TRANSIENT_RETRY.get_retry_decision( + exception=TransientInfraError("db down"), job=_job(attempts=0) + ) + assert decision is not None + + +def test_transient_retry_gives_up_once_attempts_exhausted() -> None: + decision = TRANSIENT_RETRY.get_retry_decision( + exception=TransientInfraError("db down"), + job=_job(attempts=MAX_ATTEMPTS), + ) + assert decision is None + + +def test_transient_retry_does_not_retry_other_exceptions() -> None: + decision = TRANSIENT_RETRY.get_retry_decision( + exception=RuntimeError("bad epicenter"), job=_job(attempts=0) + ) + assert decision is None + + +def test_notify_channel_is_a_bare_identifier_per_job() -> None: + simulation_id = uuid.UUID("4cfe522f-7e7d-46e0-96ca-7b98743fb9f5") + channel = notify_channel(simulation_id) + + assert channel == "tsdhn_job_4cfe522f7e7d46e096ca7b98743fb9f5" + # No hyphens or quoting needed: it goes straight into LISTEN/NOTIFY. + assert channel.replace("_", "").isalnum() + assert notify_channel(uuid.uuid4()) != channel + + +def test_reap_action_retries_a_stalled_job_within_budget() -> None: + assert reap_action(_job(attempts=0)) == "retry" + assert reap_action(_job(attempts=MAX_ATTEMPTS - 1)) == "retry" + + +def test_reap_action_gives_up_once_the_budget_is_spent() -> None: + assert reap_action(_job(attempts=MAX_ATTEMPTS)) == "exhausted" + assert reap_action(_job(attempts=MAX_ATTEMPTS + 1)) == "exhausted" diff --git a/packages/api/tests/test_migrations_integration.py b/packages/api/tests/test_migrations_integration.py new file mode 100644 index 0000000..8870bdd --- /dev/null +++ b/packages/api/tests/test_migrations_integration.py @@ -0,0 +1,204 @@ +"""Database-role behavior for the compute migration.""" + +import uuid +from collections.abc import Iterator + +import psycopg +import pytest +from psycopg import sql + +from api import migrate, queue_migrate, web_grants +from api.core.schema import COMPUTE_SCHEMA_SQL + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def temporary_web_role( + isolated_database: str, monkeypatch: pytest.MonkeyPatch +) -> Iterator[str]: + role = f"tsdhn_test_{uuid.uuid4().hex[:16]}" + monkeypatch.setattr(migrate, "APP_DB_ROLE", role) + monkeypatch.setattr(migrate, "APP_DB_PASSWORD", "test-password") + monkeypatch.setattr(web_grants, "APP_DB_ROLE", role) + + with psycopg.connect(isolated_database) as conn: + conn.execute(COMPUTE_SCHEMA_SQL) + migrate.provision_web_role(conn) + conn.commit() + + yield role + + with psycopg.connect(isolated_database) as conn: + role_identifier = sql.Identifier(role) + conn.execute(sql.SQL("DROP OWNED BY {}").format(role_identifier)) + conn.execute(sql.SQL("DROP ROLE IF EXISTS {}").format(role_identifier)) + conn.commit() + + +def test_provision_web_role_grants_only_the_runtime_boundary( + isolated_database: str, temporary_web_role: str +) -> None: + with psycopg.connect(isolated_database) as conn: + privileges = conn.execute( + """ + SELECT + has_database_privilege(%s, current_database(), 'CONNECT'), + has_database_privilege(%s, current_database(), 'CREATE'), + has_schema_privilege(%s, 'public', 'CREATE'), + has_schema_privilege(%s, 'compute', 'USAGE'), + has_table_privilege(%s, 'compute.jobs', 'SELECT'), + has_table_privilege(%s, 'compute.jobs', 'INSERT') + """, + [temporary_web_role] * 6, + ).fetchone() + + assert privileges == (True, False, False, True, True, False) + + +def test_provision_web_role_repairs_compute_write_privileges( + isolated_database: str, temporary_web_role: str +) -> None: + with psycopg.connect(isolated_database) as conn: + role = sql.Identifier(temporary_web_role) + conn.execute( + sql.SQL("GRANT INSERT, UPDATE, DELETE ON compute.jobs TO {} ").format(role) + ) + conn.commit() + + migrate.provision_web_role(conn) + conn.commit() + + privileges = conn.execute( + """ + SELECT + has_table_privilege(%s, 'compute.jobs', 'INSERT'), + has_table_privilege(%s, 'compute.jobs', 'UPDATE'), + has_table_privilege(%s, 'compute.jobs', 'DELETE') + """, + [temporary_web_role] * 3, + ).fetchone() + + assert privileges == (False, False, False) + + +def test_provision_web_role_is_idempotent( + isolated_database: str, temporary_web_role: str +) -> None: + with psycopg.connect(isolated_database) as conn: + migrate.provision_web_role(conn) + conn.commit() + exists = conn.execute( + "SELECT 1 FROM pg_roles WHERE rolname = %s", [temporary_web_role] + ).fetchone() + + assert exists is not None + + +def test_compute_migration_renames_old_job_columns(isolated_database: str) -> None: + with psycopg.connect(isolated_database) as conn: + conn.execute( + "ALTER TABLE compute.jobs RENAME COLUMN simulation_id TO external_id" + ) + conn.execute("ALTER TABLE compute.jobs RENAME COLUMN outputs TO artifacts") + migrate.install_compute_schema(conn) + columns = { + row[0] + for row in conn.execute( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'compute' AND table_name = 'jobs' + """ + ).fetchall() + } + + assert "simulation_id" in columns + assert "outputs" in columns + assert "external_id" not in columns + assert "artifacts" not in columns + + +def test_web_role_can_use_future_tables_but_cannot_run_ddl( + isolated_database: str, temporary_web_role: str +) -> None: + with psycopg.connect(isolated_database) as conn: + conn.execute( + "CREATE TABLE public.permission_probe " + "(id integer PRIMARY KEY, value text NOT NULL)" + ) + web_grants.grant_web_tables(conn) + conn.commit() + + with psycopg.connect( + isolated_database, user=temporary_web_role, password="test-password" + ) as conn: + conn.execute("INSERT INTO public.permission_probe VALUES (1, 'before')") + conn.execute("UPDATE public.permission_probe SET value = 'after' WHERE id = 1") + value = conn.execute( + "SELECT value FROM public.permission_probe WHERE id = 1" + ).fetchone() + conn.execute("DELETE FROM public.permission_probe WHERE id = 1") + conn.commit() + + with pytest.raises(psycopg.errors.InsufficientPrivilege): + conn.execute("CREATE TABLE public.forbidden (id integer)") + + assert value == ("after",) + + +def test_provision_web_role_transfers_legacy_table_ownership( + isolated_database: str, temporary_web_role: str +) -> None: + with psycopg.connect(isolated_database) as conn: + migration_user = conn.info.user + role = sql.Identifier(temporary_web_role) + conn.execute("CREATE TABLE public.legacy_probe (id integer PRIMARY KEY)") + conn.execute( + sql.SQL("ALTER TABLE public.legacy_probe OWNER TO {} ").format(role) + ) + conn.commit() + + migrate.provision_web_role(conn) + conn.commit() + + owner = conn.execute( + "SELECT pg_get_userbyid(c.relowner) FROM pg_class AS c " + "JOIN pg_namespace AS n ON n.oid = c.relnamespace " + "WHERE n.nspname = 'public' AND c.relname = 'legacy_probe'" + ).fetchone() + + with ( + psycopg.connect( + isolated_database, user=temporary_web_role, password="test-password" + ) as conn, + pytest.raises(psycopg.errors.InsufficientPrivilege), + ): + conn.execute("ALTER TABLE public.legacy_probe ADD COLUMN forbidden text") + + assert owner == (migration_user,) + + +def test_procrastinate_schema_migration_is_repeatable(isolated_database: str) -> None: + queue_migrate.apply_schema(isolated_database) + queue_migrate.apply_schema(isolated_database) + + with psycopg.connect(isolated_database) as conn: + assert all(queue_migrate.queue_schema_state(conn)) + + +def test_web_role_cannot_read_compute_queue_tables( + isolated_database: str, temporary_web_role: str +) -> None: + with psycopg.connect(isolated_database) as conn: + web_grants.grant_web_tables(conn) + conn.commit() + queue_migrate.apply_schema(isolated_database) + + with ( + psycopg.connect( + isolated_database, user=temporary_web_role, password="test-password" + ) as conn, + pytest.raises(psycopg.errors.InsufficientPrivilege), + ): + conn.execute("SELECT 1 FROM compute.procrastinate_jobs") diff --git a/packages/api/tests/test_repository.py b/packages/api/tests/test_repository.py new file mode 100644 index 0000000..8c08753 --- /dev/null +++ b/packages/api/tests/test_repository.py @@ -0,0 +1,79 @@ +import uuid +from datetime import UTC, datetime +from typing import Any + +from api.core.repository import status_from_row +from tsdhn.domain import JobStatus + +CREATED = datetime(2026, 8, 30, 12, 0, tzinfo=UTC) + + +def _row(**overrides: Any) -> dict[str, Any]: + row: dict[str, Any] = { + "id": uuid.uuid4(), + "simulation_id": uuid.uuid4(), + "status": JobStatus.QUEUED.value, + "input_params": {}, + "details": "Queued for simulation worker", + "step": None, + "step_index": None, + "total_steps": None, + "calculation": None, + "travel_times": None, + "outputs": [], + "error": None, + "created_at": CREATED, + "updated_at": CREATED, + "started_at": None, + "finished_at": None, + } + return row | overrides + + +def test_status_exposes_output_names_only() -> None: + status = status_from_row( + _row( + status=JobStatus.COMPLETED.value, + outputs=[ + { + "name": "max_height_map", + "key": "simulations/abc/outputs/maxola.pdf", + "filename": "maxola.pdf", + "content_type": "application/pdf", + } + ], + ) + ) + + assert status["outputs"] == ["max_height_map"] + assert "simulations/abc" not in str(status) + + +def test_status_has_no_outputs_before_completion() -> None: + assert status_from_row(_row())["outputs"] == [] + + +def test_status_handles_a_null_outputs_column() -> None: + assert status_from_row(_row(outputs=None))["outputs"] == [] + + +def test_status_serializes_timestamps_as_iso_strings() -> None: + status = status_from_row(_row(started_at=CREATED)) + + assert status["created_at"] == CREATED.isoformat() + assert status["started_at"] == CREATED.isoformat() + assert status["finished_at"] is None + + +def test_status_carries_step_progress() -> None: + status = status_from_row( + _row( + status=JobStatus.RUNNING.value, step="tsunami", step_index=3, total_steps=8 + ) + ) + + assert (status["step"], status["step_index"], status["total_steps"]) == ( + "tsunami", + 3, + 8, + ) diff --git a/packages/api/tests/test_repository_integration.py b/packages/api/tests/test_repository_integration.py new file mode 100644 index 0000000..6a8e863 --- /dev/null +++ b/packages/api/tests/test_repository_integration.py @@ -0,0 +1,335 @@ +"""Compute repository behavior against PostgreSQL.""" + +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import psycopg +import pytest +from psycopg.rows import dict_row + +from api.core import repository +from api.core.errors import TransientInfraError +from api.core.storage import output_store +from tsdhn.domain import CalculationResponse, EarthquakeInput, TsunamiTravelResponse +from tsdhn.engine import OutputFile, SimulationOutputs, SimulationResult +from tsdhn.runtime import RuntimeContext + +pytestmark = pytest.mark.integration + +INPUT = EarthquakeInput(Mw=8.0, h=10.0, lat0=-20.5, lon0=-70.5, hhmm="0000", dia="23") + + +@pytest.fixture +def database( + isolated_database: str, + monkeypatch: pytest.MonkeyPatch, +) -> str: + @contextmanager + def pooled() -> Iterator[psycopg.Connection[dict[str, Any]]]: + with psycopg.connect(isolated_database, row_factory=dict_row) as conn: + yield conn + + # Keep the production query path while pointing it at the isolated database. + monkeypatch.setattr(repository, "pooled", pooled) + return isolated_database + + +def _create_job( + database: str, + *, + data: EarthquakeInput = INPUT, +) -> tuple[str, uuid.UUID]: + simulation_id = uuid.uuid4() + deferred: list[uuid.UUID] = [] + + repository.create_or_get_job( + data=data, + simulation_id=str(simulation_id), + defer=lambda _conn, compute_job_id: deferred.append(compute_job_id), + ) + + assert len(deferred) == 1 + return str(simulation_id), deferred[0] + + +def test_create_or_get_job_is_idempotent_and_rejects_changed_input( + database: str, +) -> None: + simulation_id, compute_job_id = _create_job(database) + + second_defer: list[uuid.UUID] = [] + same = repository.create_or_get_job( + data=INPUT, + simulation_id=simulation_id, + defer=lambda _conn, job_id: second_defer.append(job_id), + ) + + assert same["status"] == "queued" + assert second_defer == [] + + changed = INPUT.model_copy(update={"Mw": 8.1}) + with pytest.raises(ValueError, match="different input"): + repository.create_or_get_job( + data=changed, + simulation_id=simulation_id, + defer=lambda *_args: None, + ) + + persisted = repository.get_job_status(simulation_id) + assert persisted["status"] == "queued" + with psycopg.connect(database) as conn: + persisted_id = conn.execute( + "SELECT id FROM compute.jobs WHERE simulation_id = %s", + [simulation_id], + ).fetchone() + assert persisted_id == (compute_job_id,) + + +def test_create_or_get_job_rolls_back_when_enqueue_fails( + database: str, +) -> None: + simulation_id = str(uuid.uuid4()) + + def fail_to_enqueue(_conn: Any, _compute_job_id: uuid.UUID) -> None: + raise RuntimeError("queue unavailable") + + with pytest.raises(RuntimeError, match="queue unavailable"): + repository.create_or_get_job( + data=INPUT, + simulation_id=simulation_id, + defer=fail_to_enqueue, + ) + + with pytest.raises(ValueError, match="Invalid or unknown job ID"): + repository.get_job_status(simulation_id) + + +def test_concurrent_identical_submissions_create_one_job( + database: str, +) -> None: + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + + simulation_id = str(uuid.uuid4()) + start = Barrier(2) + deferred: list[uuid.UUID] = [] + + def submit() -> dict[str, Any]: + start.wait() + return repository.create_or_get_job( + data=INPUT, + simulation_id=simulation_id, + defer=lambda _conn, compute_job_id: deferred.append(compute_job_id), + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = list(executor.map(lambda _index: submit(), range(2))) + + assert {result["status"] for result in results} == {"queued"} + assert len(deferred) == 1 + with psycopg.connect(database) as conn: + count = conn.execute( + "SELECT count(*) FROM compute.jobs WHERE simulation_id = %s", + [simulation_id], + ).fetchone() + assert count == (1,) + + +def test_job_lookup_rejects_invalid_and_unknown_simulation_ids( + database: str, +) -> None: + with pytest.raises(ValueError, match="Invalid or unknown job ID"): + repository.get_job_status("not-a-uuid") + with pytest.raises(ValueError, match="Invalid or unknown job ID"): + repository.get_job_status(str(uuid.uuid4())) + + +def test_outputs_are_empty_before_completion_and_unknown_jobs_are_rejected( + database: str, +) -> None: + simulation_id, _compute_job_id = _create_job(database) + + assert repository.get_outputs(simulation_id) == [] + with pytest.raises(ValueError, match="Invalid or unknown job ID"): + repository.get_outputs(str(uuid.uuid4())) + + +def test_job_state_updates_are_persisted_as_client_visible_behavior( + database: str, +) -> None: + database_url = database + simulation_id_text, compute_job_id = _create_job(database) + simulation_id = uuid.UUID(simulation_id_text) + + with psycopg.connect(database_url, row_factory=dict_row) as conn: + repository.mark_started(conn, compute_job_id, simulation_id) + repository.record_progress( + conn, + compute_job_id, + simulation_id, + "Processing tsunami", + {"step": "tsunami", "step_index": 3, "total_steps": 8}, + ) + assert repository.get_current_step(conn, compute_job_id) == "tsunami" + + running = repository.get_job_status(simulation_id_text) + assert running["status"] == "running" + assert running["details"] == "Processing tsunami" + assert (running["step"], running["step_index"], running["total_steps"]) == ( + "tsunami", + 3, + 8, + ) + + with psycopg.connect(database_url, row_factory=dict_row) as conn: + repository.fail_job(conn, compute_job_id, simulation_id, "worker vanished") + + failed = repository.get_job_status(simulation_id_text) + assert failed["status"] == "failed" + assert failed["error"] == "worker vanished" + assert failed["finished_at"] is not None + + +def test_list_abandoned_work_dirs_returns_only_old_failed_jobs( + database: str, +) -> None: + database_url = database + simulation_id_text, compute_job_id = _create_job(database) + simulation_id = uuid.UUID(simulation_id_text) + fresh_simulation_id_text, fresh_job_id = _create_job(database) + fresh_simulation_id = uuid.UUID(fresh_simulation_id_text) + cutoff = datetime.now(UTC) - timedelta(hours=24) + + with psycopg.connect(database_url, row_factory=dict_row) as conn: + repository.fail_job(conn, compute_job_id, simulation_id, "worker vanished") + repository.fail_job(conn, fresh_job_id, fresh_simulation_id, "worker vanished") + conn.execute( + "UPDATE compute.jobs SET finished_at = %s WHERE id = %s", + [cutoff - timedelta(minutes=1), compute_job_id], + ) + + abandoned = repository.list_abandoned_work_dirs(cutoff) + assert simulation_id_text in abandoned + assert fresh_simulation_id_text not in abandoned + + +def test_record_failure_keeps_running_for_a_transient_retry( + database: str, +) -> None: + database_url = database + simulation_id_text, compute_job_id = _create_job(database) + simulation_id = uuid.UUID(simulation_id_text) + + with psycopg.connect(database_url, row_factory=dict_row) as conn: + repository.mark_started(conn, compute_job_id, simulation_id) + repository.record_failure( + conn, + compute_job_id, + simulation_id, + TransientInfraError("minio down"), + step="maxola", + will_retry=True, + ) + + status = repository.get_job_status(simulation_id_text) + assert status["status"] == "running" + assert status["details"] == "Retrying after transient error (TransientInfraError)" + assert status["finished_at"] is None + + +def test_record_failure_persists_a_sanitized_terminal_error( + database: str, +) -> None: + database_url = database + simulation_id_text, compute_job_id = _create_job(database) + simulation_id = uuid.UUID(simulation_id_text) + + with psycopg.connect(database_url, row_factory=dict_row) as conn: + repository.record_failure( + conn, + compute_job_id, + simulation_id, + FileNotFoundError("/private/jobs/secret/tsunami"), + step="tsunami", + will_retry=False, + ) + + status = repository.get_job_status(simulation_id_text) + assert status["status"] == "failed" + assert status["details"] == "Pipeline failed - check error logs" + assert status["error"] == "Simulation failed at step 'tsunami' (FileNotFoundError)" + assert "/private/jobs" not in status["error"] + + +def test_complete_job_persists_the_uploaded_manifest_and_result( + database: str, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + database_url = database + simulation_id_text, compute_job_id = _create_job(database) + output_path = tmp_path / "calculation.json" + output_path.write_text("{}", encoding="utf-8") + + calculation = CalculationResponse( + length=1.0, + width=2.0, + dislocation=3.0, + seismic_moment=4.0, + tsunami_warning="none", + distance_to_coast=5.0, + azimuth=6.0, + dip=7.0, + epicenter_location="0.00/0.00", + rectangle_parameters={}, + rectangle_corners=[], + ) + travel_times = TsunamiTravelResponse( + arrival_times={"PORT": "01:00"}, + distances={"PORT": 100.0}, + epicenter_info={"lat": "0.0"}, + ) + outputs = SimulationOutputs( + root=tmp_path, + files=(OutputFile("calculation", output_path, "application/json"),), + ) + result = SimulationResult( + calculation=calculation, + travel_times=travel_times, + runtime=RuntimeContext( + model_dir=tmp_path, model_version="test", capabilities={} + ), + outputs=outputs, + ) + uploaded: dict[str, Any] = {} + + def fake_upload(**kwargs: Any) -> tuple[str, str]: + uploaded.update(kwargs) + return "results", "simulations/result/metadata.json" + + monkeypatch.setattr(output_store, "upload_simulation_result", fake_upload) + + with psycopg.connect(database_url, row_factory=dict_row) as conn: + row = repository.fetch_by_id(conn, compute_job_id) + assert row is not None + repository.complete_job(conn, row, result) + + status = repository.get_job_status(simulation_id_text) + assert status["status"] == "completed" + assert status["calculation"] == calculation.model_dump(mode="json") + assert status["travel_times"] == travel_times.model_dump(mode="json") + assert status["outputs"] == ["calculation"] + assert uploaded["simulation_id"] == simulation_id_text + + assert repository.get_outputs(simulation_id_text) == [ + { + "name": "calculation", + "key": f"simulations/{simulation_id_text}/outputs/calculation.json", + "filename": "calculation.json", + "content_type": "application/json", + } + ] diff --git a/packages/api/tests/test_storage.py b/packages/api/tests/test_storage.py new file mode 100644 index 0000000..0338c3c --- /dev/null +++ b/packages/api/tests/test_storage.py @@ -0,0 +1,131 @@ +from pathlib import Path +from typing import Any + +import pytest +from minio.error import MinioException + +from api.core.errors import TransientInfraError +from api.core.storage import OutputStore +from tsdhn.engine import OutputFile, SimulationOutputs + + +class _RaisingMinioClient: + def bucket_exists(self, bucket_name: str) -> bool: + return True + + def fput_object(self, **kwargs: object) -> None: + raise MinioException("simulated MinIO outage") + + +class _RaisingPresignClient: + def presigned_get_object(self, **kwargs: object) -> str: + raise MinioException("simulated MinIO outage") + + +class _RecordingMinioClient: + def __init__(self) -> None: + self.created_bucket = False + self.uploads: list[dict[str, Any]] = [] + self.metadata_uploads: list[dict[str, Any]] = [] + self.presign: dict[str, Any] | None = None + + def bucket_exists(self, bucket_name: str) -> bool: + return self.created_bucket + + def make_bucket(self, bucket_name: str) -> None: + self.created_bucket = True + + def fput_object(self, **kwargs: Any) -> None: + self.uploads.append(kwargs) + + def put_object(self, **kwargs: Any) -> None: + payload = kwargs["data"].read() + self.metadata_uploads.append({**kwargs, "payload": payload}) + + def presigned_get_object(self, **kwargs: Any) -> str: + self.presign = kwargs + return "https://minio.example/signed" + + +def test_upload_simulation_result_wraps_minio_failure(tmp_path: Path) -> None: + store = OutputStore.__new__(OutputStore) + store.bucket = "tsdhn-results" + store._client = _RaisingMinioClient() # type: ignore[assignment] + + output_path = tmp_path / "maxola.pdf" + output_path.write_bytes(b"%PDF-1.4\n") + outputs = SimulationOutputs( + root=tmp_path, + files=(OutputFile("max_height_map", output_path, "application/pdf"),), + ) + + with pytest.raises(TransientInfraError): + store.upload_simulation_result( + simulation_id="job-1", + compute_job_id="compute-1", + outputs=outputs, + metadata={}, + ) + + +def test_presigned_url_wraps_minio_failure() -> None: + store = OutputStore.__new__(OutputStore) + store.bucket = "tsdhn-results" + store._public_client = _RaisingPresignClient() # type: ignore[assignment] + + with pytest.raises(TransientInfraError, match="presigning output URL failed"): + store.presigned_url("simulations/job-1/result.pdf", filename="result.pdf") + + +def test_upload_simulation_result_creates_bucket_and_persists_manifest( + tmp_path: Path, +) -> None: + store = OutputStore.__new__(OutputStore) + store.bucket = "tsdhn-results" + client = _RecordingMinioClient() + store._client = client # type: ignore[assignment] + + output_path = tmp_path / "maxola.pdf" + output_path.write_bytes(b"pdf") + outputs = SimulationOutputs( + root=tmp_path, + files=(OutputFile("max_height_map", output_path, "application/pdf"),), + ) + + bucket, metadata_key = store.upload_simulation_result( + simulation_id="job-1", + compute_job_id="compute-1", + outputs=outputs, + metadata={"status": "completed"}, + ) + + assert (bucket, metadata_key) == ( + "tsdhn-results", + "simulations/job-1/metadata.json", + ) + assert client.created_bucket + assert client.uploads[0]["object_name"] == ("simulations/job-1/outputs/maxola.pdf") + assert client.uploads[0]["metadata"] == { + "simulation-id": "job-1", + "compute-job-id": "compute-1", + "output-name": "max_height_map", + } + assert client.metadata_uploads[0]["payload"] == b'{"status":"completed"}' + assert client.metadata_uploads[0]["content_type"] == "application/json" + + +def test_presigned_url_uses_public_storage_and_download_filename() -> None: + store = OutputStore.__new__(OutputStore) + store.bucket = "tsdhn-results" + client = _RecordingMinioClient() + store._public_client = client # type: ignore[assignment] + + url = store.presigned_url("simulations/job-1/result.pdf", filename="result.pdf") + + assert url == "https://minio.example/signed" + assert client.presign is not None + assert client.presign["bucket_name"] == "tsdhn-results" + assert client.presign["object_name"] == "simulations/job-1/result.pdf" + assert client.presign["response_headers"] == { + "response-content-disposition": 'attachment; filename="result.pdf"' + } diff --git a/packages/api/tests/test_tasks.py b/packages/api/tests/test_tasks.py new file mode 100644 index 0000000..057dbf7 --- /dev/null +++ b/packages/api/tests/test_tasks.py @@ -0,0 +1,439 @@ +"""Worker lifecycle behavior at the repository and engine boundaries.""" + +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +from procrastinate import exceptions as procrastinate_exceptions +from procrastinate.jobs import Job as ProcrastinateJob + +from api.core import tasks +from api.core.errors import TransientInfraError +from api.core.tasks import MAX_ATTEMPTS +from tsdhn.domain import EarthquakeInput + +tasks_module: Any = tasks + +INPUT = EarthquakeInput(Mw=8.0, h=10.0, lat0=-20.5, lon0=-70.5, hhmm="0000", dia="23") + + +class _Connection: + pass + + +class _TaskContext: + def __init__(self, attempts: int) -> None: + self.job = SimpleNamespace(attempts=attempts) + + +def _row(job_id: uuid.UUID, simulation_id: uuid.UUID) -> dict[str, Any]: + return { + "id": job_id, + "simulation_id": simulation_id, + "input_params": INPUT.model_dump(mode="json"), + } + + +@pytest.fixture +def worker_job( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> tuple[uuid.UUID, uuid.UUID, Path, _Connection]: + job_id = uuid.uuid4() + simulation_id = uuid.uuid4() + work_dir = tmp_path / "jobs" / str(simulation_id) + work_dir.mkdir(parents=True) + connection = _Connection() + + @contextmanager + def connect() -> Iterator[_Connection]: + yield connection + + monkeypatch.setattr(tasks_module, "connect", connect) + monkeypatch.setattr(tasks_module, "JOBS_DIR", tmp_path / "jobs") + monkeypatch.setattr( + tasks_module.repository, + "fetch_by_id", + lambda _conn, _id: _row(job_id, simulation_id), + ) + return job_id, simulation_id, work_dir, connection + + +def test_run_simulation_task_completes_and_removes_the_workspace( + worker_job: tuple[uuid.UUID, uuid.UUID, Path, _Connection], + monkeypatch: pytest.MonkeyPatch, +) -> None: + job_id, simulation_id, work_dir, connection = worker_job + events: list[tuple[str, Any]] = [] + result = object() + + monkeypatch.setattr( + tasks_module.repository, + "mark_started", + lambda conn, current_id, current_external: events.append( + ("started", (conn, current_id, current_external)) + ), + ) + monkeypatch.setattr( + tasks_module.repository, + "record_progress", + lambda conn, current_id, current_external, message, details: events.append( + ("progress", (conn, current_id, current_external, message, details)) + ), + ) + monkeypatch.setattr( + tasks_module.repository, + "complete_job", + lambda conn, row, completed: events.append(("completed", (conn, completed))), + ) + + def run_simulation( + data: EarthquakeInput, + current_work_dir: Path, + *, + resume: bool, + on_progress: Any, + ) -> object: + events.append(("run", (data, current_work_dir, resume))) + on_progress("Processing tsunami", {"step": "tsunami"}) + return result + + monkeypatch.setattr(tasks_module, "run_simulation", run_simulation) + + tasks_module.run_simulation_task.func(_TaskContext(attempts=0), str(job_id)) + + assert events == [ + ("started", (connection, job_id, simulation_id)), + ("run", (INPUT, work_dir, True)), + ( + "progress", + ( + connection, + job_id, + simulation_id, + "Processing tsunami", + {"step": "tsunami"}, + ), + ), + ("completed", (connection, result)), + ] + assert not work_dir.exists() + + +@pytest.mark.parametrize(("attempts", "will_retry"), [(0, True), (MAX_ATTEMPTS, False)]) +def test_run_simulation_task_records_failure_and_preserves_workspace( + worker_job: tuple[uuid.UUID, uuid.UUID, Path, _Connection], + monkeypatch: pytest.MonkeyPatch, + attempts: int, + will_retry: bool, +) -> None: + job_id, simulation_id, work_dir, connection = worker_job + failures: list[dict[str, Any]] = [] + + monkeypatch.setattr(tasks_module.repository, "mark_started", lambda *_args: None) + monkeypatch.setattr( + tasks_module.repository, "get_current_step", lambda _conn, _job_id: "tsunami" + ) + + def record_failure( + conn: Any, + current_id: Any, + current_external: Any, + exc: Exception, + *, + step: str | None, + will_retry: bool, + ) -> None: + failures.append( + { + "conn": conn, + "job_id": current_id, + "simulation_id": current_external, + "exception": exc, + "step": step, + "will_retry": will_retry, + } + ) + + monkeypatch.setattr( + tasks_module.repository, + "record_failure", + record_failure, + ) + + def fail(*_args: Any, **_kwargs: Any) -> object: + raise TransientInfraError("storage unavailable") + + monkeypatch.setattr(tasks_module, "run_simulation", fail) + + with pytest.raises(TransientInfraError, match="storage unavailable"): + tasks_module.run_simulation_task.func(_TaskContext(attempts), str(job_id)) + + assert len(failures) == 1 + assert failures[0]["conn"] is connection + assert failures[0]["job_id"] == job_id + assert failures[0]["simulation_id"] == simulation_id + assert isinstance(failures[0]["exception"], TransientInfraError) + assert failures[0]["step"] == "tsunami" + assert failures[0]["will_retry"] is will_retry + assert work_dir.exists() + + +def test_run_simulation_task_rejects_an_unknown_job( + monkeypatch: pytest.MonkeyPatch, +) -> None: + connection = _Connection() + + @contextmanager + def connect() -> Iterator[_Connection]: + yield connection + + monkeypatch.setattr(tasks_module, "connect", connect) + monkeypatch.setattr(tasks_module.repository, "fetch_by_id", lambda *_args: None) + job_id = uuid.uuid4() + + with pytest.raises(RuntimeError, match="Unknown compute job"): + tasks_module.run_simulation_task.func(_TaskContext(0), str(job_id)) + + +def test_enqueue_simulation_configures_and_defers_the_worker_job( + monkeypatch: pytest.MonkeyPatch, +) -> None: + connection = _Connection() + compute_job_id = uuid.uuid4() + configured: list[dict[str, Any]] = [] + deferred: list[dict[str, Any]] = [] + + class _ConfiguredTask: + def configure(self, **kwargs: Any) -> Any: + configured.append(kwargs) + return self + + def defer(self, **kwargs: Any) -> None: + deferred.append(kwargs) + + monkeypatch.setattr(tasks_module, "run_simulation_task", _ConfiguredTask()) + + tasks_module.enqueue_simulation(connection, compute_job_id) + + assert configured == [ + { + "connection": connection, + "queue": tasks_module.PROCRASTINATE_QUEUE, + "queueing_lock": f"simulation:{compute_job_id}", + "lock": f"compute-job:{compute_job_id}", + } + ] + assert deferred == [{"compute_job_id": str(compute_job_id)}] + + +class _JobManager: + def __init__(self, jobs: list[ProcrastinateJob]) -> None: + self.jobs = jobs + self.retried: list[int] = [] + self.finished: list[int] = [] + + async def get_stalled_jobs( + self, *, seconds_since_heartbeat: int + ) -> list[ProcrastinateJob]: + assert seconds_since_heartbeat == tasks_module.STALLED_HEARTBEAT_SECONDS + return self.jobs + + async def retry_job_by_id_async(self, *, job_id: int, retry_at: Any) -> None: + self.retried.append(job_id) + + async def finish_job_by_id_async( + self, *, job_id: int, status: Any, delete_job: bool + ) -> None: + assert status.value == "failed" + assert delete_job is False + self.finished.append(job_id) + + +def _queue_job(job_id: int | None, attempts: int) -> ProcrastinateJob: + return ProcrastinateJob( + id=job_id, + queue="simulations", + lock=None, + queueing_lock=None, + task_name="api.run_simulation", + task_kwargs={"compute_job_id": str(uuid.uuid4())}, + attempts=attempts, + ) + + +@pytest.mark.asyncio +async def test_reap_stalled_jobs_retries_or_finishes_by_attempt_budget( + monkeypatch: pytest.MonkeyPatch, +) -> None: + retry_job = _queue_job(1, 0) + exhausted_job = _queue_job(2, MAX_ATTEMPTS) + manager = _JobManager([retry_job, exhausted_job]) + exhausted_ids: list[str] = [] + + monkeypatch.setattr(tasks_module.app, "job_manager", manager) + monkeypatch.setattr( + tasks_module, + "_fail_exhausted", + lambda compute_job_id: exhausted_ids.append(compute_job_id), + ) + + await tasks_module.reap_stalled_jobs_task.func(0) + + assert manager.retried == [1] + assert manager.finished == [2] + assert exhausted_ids == [exhausted_job.task_kwargs["compute_job_id"]] + + +@pytest.mark.asyncio +async def test_reap_stalled_jobs_returns_when_the_queue_is_empty( + monkeypatch: pytest.MonkeyPatch, +) -> None: + manager = _JobManager([]) + monkeypatch.setattr(tasks_module.app, "job_manager", manager) + + await tasks_module.reap_stalled_jobs_task.func(0) + + assert manager.retried == [] + assert manager.finished == [] + + +@pytest.mark.asyncio +async def test_reap_stalled_jobs_ignores_malformed_queue_entries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + missing_id = _queue_job(None, 0) + missing_compute_id = ProcrastinateJob( + id=3, + queue="simulations", + lock=None, + queueing_lock=None, + task_name="api.run_simulation", + task_kwargs={}, + attempts=0, + ) + manager = _JobManager([missing_id, missing_compute_id]) + monkeypatch.setattr(tasks_module.app, "job_manager", manager) + + await tasks_module.reap_stalled_jobs_task.func(0) + + assert manager.retried == [] + assert manager.finished == [] + + +@pytest.mark.asyncio +async def test_reap_stalled_jobs_tolerates_queue_connector_races( + monkeypatch: pytest.MonkeyPatch, +) -> None: + retry_job = _queue_job(1, 0) + exhausted_job = _queue_job(2, MAX_ATTEMPTS) + manager = _JobManager([retry_job, exhausted_job]) + retry_attempts: list[int] = [] + finish_attempts: list[int] = [] + exhausted_ids: list[str] = [] + + async def retry_failure(*, job_id: int, retry_at: Any) -> None: + retry_attempts.append(job_id) + raise procrastinate_exceptions.ConnectorException("job already resolved") + + async def finish_failure(*, job_id: int, status: Any, delete_job: bool) -> None: + finish_attempts.append(job_id) + raise procrastinate_exceptions.ConnectorException("job already resolved") + + monkeypatch.setattr(tasks_module.app, "job_manager", manager) + monkeypatch.setattr(tasks_module, "_fail_exhausted", exhausted_ids.append) + monkeypatch.setattr(manager, "retry_job_by_id_async", retry_failure) + monkeypatch.setattr(manager, "finish_job_by_id_async", finish_failure) + + await tasks_module.reap_stalled_jobs_task.func(0) + + assert retry_attempts == [1] + assert exhausted_ids == [exhausted_job.task_kwargs["compute_job_id"]] + assert finish_attempts == [2] + + +def test_sweep_abandoned_work_dirs_removes_only_selected_workspaces( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + old_id = str(uuid.uuid4()) + old_dir = tmp_path / old_id + old_dir.mkdir() + unrelated = tmp_path / "keep" + unrelated.mkdir() + monkeypatch.setattr( + tasks_module.repository, "list_abandoned_work_dirs", lambda _cutoff: [old_id] + ) + monkeypatch.setattr(tasks_module, "JOBS_DIR", tmp_path) + + tasks_module.sweep_abandoned_work_dirs_task.func(0) + + assert not old_dir.exists() + assert unrelated.exists() + + +@pytest.mark.parametrize("status", ["completed", "failed"]) +def test_fail_exhausted_does_not_overwrite_terminal_jobs( + monkeypatch: pytest.MonkeyPatch, status: str +) -> None: + connection = _Connection() + + @contextmanager + def pooled() -> Iterator[_Connection]: + yield connection + + monkeypatch.setattr(tasks_module, "pooled", pooled) + fetched: list[tuple[Any, ...]] = [] + + def fetch(*args: Any) -> dict[str, Any]: + fetched.append(args) + return {"status": status, "simulation_id": uuid.uuid4()} + + monkeypatch.setattr(tasks_module.repository, "fetch_by_id", fetch) + monkeypatch.setattr( + tasks_module.repository, + "fail_job", + lambda *_args: pytest.fail("terminal job must not be overwritten"), + ) + + tasks_module._fail_exhausted(str(uuid.uuid4())) + + assert len(fetched) == 1 + + +def test_fail_exhausted_marks_an_active_job_failed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + connection = _Connection() + simulation_id = uuid.uuid4() + failures: list[tuple[Any, ...]] = [] + + @contextmanager + def pooled() -> Iterator[_Connection]: + yield connection + + monkeypatch.setattr(tasks_module, "pooled", pooled) + monkeypatch.setattr( + tasks_module.repository, + "fetch_by_id", + lambda *_args: {"status": "running", "simulation_id": simulation_id}, + ) + monkeypatch.setattr( + tasks_module.repository, + "fail_job", + lambda *args: failures.append(args), + ) + + job_id = uuid.uuid4() + tasks_module._fail_exhausted(str(job_id)) + + assert failures == [ + ( + connection, + job_id, + simulation_id, + tasks_module.CRASH_BUDGET_EXHAUSTED_ERROR, + ) + ] diff --git a/packages/tsdhn-parity/pyproject.toml b/packages/tsdhn-parity/pyproject.toml new file mode 100644 index 0000000..ff9e98e --- /dev/null +++ b/packages/tsdhn-parity/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "tsdhn-parity" +version = "0.0.1" +description = "Checkpoint-trace parity testing between legacy MATLAB/Fortran units and their tsdhn Python ports" +requires-python = ">=3.14" +readme = "readme.md" +license = { text = "MIT" } +authors = [ + { name = "David Duran", email = "dadch1404@gmail.com" }, +] +dependencies = [ + "tsdhn", + "numpy==2.4.6", + "pytest==9.1.1", +] + +[build-system] +requires = ["uv_build==0.11.28"] +build-backend = "uv_build" + +[tool.uv.build-backend] +module-name = "tsdhn_parity" +module-root = "" diff --git a/packages/tsdhn-parity/readme.md b/packages/tsdhn-parity/readme.md new file mode 100644 index 0000000..56c22a5 --- /dev/null +++ b/packages/tsdhn-parity/readme.md @@ -0,0 +1,78 @@ +# tsdhn-parity + +`tsdhn-parity` compares the Python engine with saved MATLAB runs and compiled +Fortran output. It is test support, not production code. + +These comparisons protect compatibility. Agreement means that Python matches +the selected older program within an explicit tolerance. It does not prove +that either result is physically correct. An older program can contain an +error, an empirical calibration, or an assumption that applies only to its +original research setting. + +## Run it + +```sh +mise run test-parity +``` + +Saved MATLAB runs and Python cases run without MATLAB. Fortran cases run only +when `TSDHN_TOOLS_DIR` points to the compiled older programs. Missing optional +programs cause those cases to skip. + +The active deformation executable must be compiled from `model/def_oka.f`. +`model/deform.for` is an older Mansinha-Smylie implementation and does not test +the Okada-based Python port. + +## Layout + +- `tsdhn_parity/cases.py`: exhaustive and pairwise input generation. +- `tsdhn_parity/trace.py`: checkpoint and trace types. +- `tsdhn_parity/compare.py`: checkpoint comparison and tolerances. +- `tsdhn_parity/adapters/`: runners for Python, Fortran, and saved traces. +- `tsdhn_parity/pytest_plugin.py`: pytest integration. +- `packages/tsdhn/tests/parity//`: comparison cases, readers, + tolerances, and saved data. + +Each checkpoint isolates a named intermediate result. Readers for old +fixed-width files must use the declared field width because whitespace parsing +can hide a malformed record. Tolerances account for float32 rounding and file +quantization; they should not be widened without identifying the difference. + +The tsunami comparison gives the same fault-plane and deformation result to +both propagation solvers. This is deliberate: it isolates propagation. It +does not compare the full fault-plane, deformation, and propagation chain as +independent runs. + +## Refresh saved MATLAB runs + +Use the capture command when saved data needs to change: + +```sh +uv run python scripts/capture_matlab_fixtures.py fault_plane +uv run python scripts/capture_matlab_fixtures.py fault_plane --case alaska_1964 +``` + +The command needs a MathWorks container and license. It writes committed +`.npz` files. Pytest reads those files and does not start MATLAB. + +Before replacing a saved run, record the input case, MATLAB source, checkpoint +names, and reason for the change. Do not replace expected data only because a +comparison failed. + +## Interpreting a failure + +A comparison failure can come from: + +- a real change in equations or update order; +- float32 evaluation order; +- one-based versus zero-based indices; +- longitude conversion; +- fixed-width output rounding; +- a different old executable than the one the Python stage ports; +- a stale or incorrectly captured MATLAB run. + +First compare the earliest failing checkpoint. Later checkpoints often differ +only because an earlier stage changed. See the engine's +[`testing.md`](../tsdhn/docs/testing.md) and +[`legacy.md`](../tsdhn/docs/legacy.md) guides for the limits of comparison +tests and the active source map. diff --git a/packages/tsdhn-parity/tests/test_adapters_fortran.py b/packages/tsdhn-parity/tests/test_adapters_fortran.py new file mode 100644 index 0000000..9517729 --- /dev/null +++ b/packages/tsdhn-parity/tests/test_adapters_fortran.py @@ -0,0 +1,67 @@ +from pathlib import Path + +import numpy as np +import pytest + +from tsdhn.utils.file_utils import make_executable +from tsdhn_parity.adapters.fortran import FortranBinaryAdapter, read_fixed_width_grid +from tsdhn_parity.cases import Case + +# Exercise the adapter's copy, execute, and read-back contract. +FIXTURE_PROGRAM = """#!/bin/sh +value=$(cat input.txt) +echo $((value * 2)) > output.txt +""" + + +@pytest.fixture +def tools_dir(tmp_path: Path) -> Path: + directory = tmp_path / "tools" + directory.mkdir() + program = directory / "double" + program.write_text(FIXTURE_PROGRAM) + make_executable(program) + return directory + + +def _write_input(case: Case, working_dir: Path) -> None: + (working_dir / "input.txt").write_text(str(case.params["value"])) + + +def test_runs_binary_and_reads_declared_checkpoints_in_order(tools_dir: Path) -> None: + adapter = FortranBinaryAdapter( + executable="double", + checkpoints=("output.txt",), + prepare=_write_input, + tools_dir=tools_dir, + ) + + trace = adapter.run(Case(id="value_21", params={"value": 21})) + + assert trace.case_id == "value_21" + assert [checkpoint.name for checkpoint in trace.checkpoints] == ["output.txt"] + assert np.asarray(trace.checkpoints[0].value).item() == pytest.approx(42.0) + + +def test_skips_when_executable_is_missing_from_tools_dir(tmp_path: Path) -> None: + empty_tools_dir = tmp_path / "empty" + empty_tools_dir.mkdir() + adapter = FortranBinaryAdapter( + executable="double", + checkpoints=("output.txt",), + prepare=_write_input, + tools_dir=empty_tools_dir, + ) + + with pytest.raises(pytest.skip.Exception): + adapter.run(Case(id="value_1", params={"value": 1})) + + +def test_read_fixed_width_grid_handles_touching_columns(tmp_path: Path) -> None: + # Fixed-width output can place adjacent fields without a separator. + path = tmp_path / "grid.dat" + path.write_text("123.4-56.7\n 0.0 1.2\n") + + grid = read_fixed_width_grid(5)(path) + + assert grid.tolist() == [[123.4, -56.7], [0.0, 1.2]] diff --git a/packages/tsdhn-parity/tests/test_cases.py b/packages/tsdhn-parity/tests/test_cases.py new file mode 100644 index 0000000..7bbf61a --- /dev/null +++ b/packages/tsdhn-parity/tests/test_cases.py @@ -0,0 +1,50 @@ +from collections.abc import Mapping, Sequence + +from tsdhn_parity.cases import Case, ParamValue, exhaustive, pairwise + + +def test_exhaustive_is_full_cartesian_product() -> None: + cases = exhaustive(a=[1, 2], b=["x", "y", "z"]) + + assert len(cases) == 6 + assert all(isinstance(case, Case) for case in cases) + seen = {(case.params["a"], case.params["b"]) for case in cases} + assert seen == {(1, "x"), (1, "y"), (1, "z"), (2, "x"), (2, "y"), (2, "z")} + + +def test_exhaustive_case_ids_are_unique() -> None: + cases = exhaustive(a=[1, 2], b=[1, 2]) + + assert len({case.id for case in cases}) == len(cases) + + +def test_pairwise_covers_every_two_parameter_combination() -> None: + value_lists: dict[str, Sequence[ParamValue]] = { + "a": [1, 2], + "b": ["x", "y"], + "c": [True, False], + } + + cases = pairwise(**value_lists) + + assert len(cases) < len(exhaustive(**value_lists)) + assert _every_pair_covered(cases, value_lists) + + +def test_pairwise_with_fewer_than_two_params_matches_exhaustive() -> None: + assert pairwise(a=[1, 2, 3]) == exhaustive(a=[1, 2, 3]) + + +def _every_pair_covered( + cases: list[Case], value_lists: Mapping[str, Sequence[ParamValue]] +) -> bool: + names = list(value_lists) + for i in range(len(names)): + for j in range(i + 1, len(names)): + required = { + (a, b) for a in value_lists[names[i]] for b in value_lists[names[j]] + } + covered = {(case.params[names[i]], case.params[names[j]]) for case in cases} + if not required <= covered: + return False + return True diff --git a/packages/tsdhn-parity/tests/test_compare.py b/packages/tsdhn-parity/tests/test_compare.py new file mode 100644 index 0000000..274443e --- /dev/null +++ b/packages/tsdhn-parity/tests/test_compare.py @@ -0,0 +1,85 @@ +from tsdhn_parity.compare import Tolerance, compare +from tsdhn_parity.trace import Checkpoint, Trace + + +def _trace(case_id: str, **checkpoints: object) -> Trace: + return Trace( + case_id=case_id, + checkpoints=tuple( + Checkpoint.of(name, value) for name, value in checkpoints.items() + ), + ) + + +def test_matching_traces_compare_ok() -> None: + legacy = _trace("c1", step_a=[1.0, 2.0], step_b=3.0) + python = _trace("c1", step_a=[1.0, 2.0], step_b=3.0) + + result = compare(legacy, python) + + assert result.ok + assert result.first_divergence is None + + +def test_diverging_checkpoint_is_reported() -> None: + legacy = _trace("c1", step_a=[1.0, 2.0], step_b=3.0) + python = _trace("c1", step_a=[1.0, 2.0], step_b=30.0) + + result = compare(legacy, python) + + assert not result.ok + divergence = result.first_divergence + assert divergence is not None + assert divergence.name == "step_b" + assert divergence.status == "diverge" + assert divergence.max_abs_diff == 27.0 + + +def test_earliest_divergence_wins_when_multiple_checkpoints_diverge() -> None: + legacy = _trace("c1", step_a=1.0, step_b=1.0) + python = _trace("c1", step_a=9.0, step_b=9.0) + + result = compare(legacy, python) + + divergence = result.first_divergence + assert divergence is not None + assert divergence.name == "step_a" + + +def test_missing_checkpoint_on_either_side_is_reported() -> None: + legacy = _trace("c1", only_legacy=1.0) + python = _trace("c1", only_python=1.0) + + result = compare(legacy, python) + + statuses = {diff.name: diff.status for diff in result.diffs} + assert statuses["only_legacy"] == "missing_in_python" + assert statuses["only_python"] == "missing_in_legacy" + + +def test_shape_mismatch_is_a_divergence_not_a_crash() -> None: + legacy = _trace("c1", grid=[1.0, 2.0, 3.0]) + python = _trace("c1", grid=[1.0, 2.0]) + + result = compare(legacy, python) + + divergence = result.first_divergence + assert divergence is not None + assert divergence.status == "diverge" + assert divergence.max_abs_diff is None + + +def test_per_checkpoint_tolerance_overrides_default() -> None: + legacy = _trace("c1", noisy=1.0) + python = _trace("c1", noisy=1.2) + + tight = compare(legacy, python, default=Tolerance(rtol=1e-6)) + loose = compare( + legacy, + python, + tolerances={"noisy": Tolerance(rtol=0.5)}, + default=Tolerance(rtol=1e-6), + ) + + assert not tight.ok + assert loose.ok diff --git a/packages/tsdhn-parity/tsdhn_parity/__init__.py b/packages/tsdhn-parity/tsdhn_parity/__init__.py new file mode 100644 index 0000000..c9ca044 --- /dev/null +++ b/packages/tsdhn-parity/tsdhn_parity/__init__.py @@ -0,0 +1,47 @@ +from tsdhn_parity.adapters import ( + FortranBinaryAdapter, + FrozenFixtureAdapter, + LegacyAdapter, + PythonAdapter, + PythonCallableAdapter, + read_checkpoint_text, + read_fixed_width_grid, + write_fixture, +) +from tsdhn_parity.cases import Case, ParamValue, exhaustive, pairwise +from tsdhn_parity.compare import ( + DEFAULT_TOLERANCE, + CheckpointDiff, + ComparisonResult, + Tolerance, + compare, +) +from tsdhn_parity.pytest_plugin import UnitSpec, assert_parity, parity_cases +from tsdhn_parity.trace import Checkpoint, CheckpointRecorder, OnCheckpoint, Trace + +__all__ = [ + "DEFAULT_TOLERANCE", + "Case", + "Checkpoint", + "CheckpointDiff", + "CheckpointRecorder", + "ComparisonResult", + "FortranBinaryAdapter", + "FrozenFixtureAdapter", + "LegacyAdapter", + "OnCheckpoint", + "ParamValue", + "PythonAdapter", + "PythonCallableAdapter", + "Tolerance", + "Trace", + "UnitSpec", + "assert_parity", + "compare", + "exhaustive", + "pairwise", + "parity_cases", + "read_checkpoint_text", + "read_fixed_width_grid", + "write_fixture", +] diff --git a/packages/tsdhn-parity/tsdhn_parity/adapters/__init__.py b/packages/tsdhn-parity/tsdhn_parity/adapters/__init__.py new file mode 100644 index 0000000..074f461 --- /dev/null +++ b/packages/tsdhn-parity/tsdhn_parity/adapters/__init__.py @@ -0,0 +1,32 @@ +from typing import Protocol + +from tsdhn_parity.adapters.fortran import ( + FortranBinaryAdapter, + read_checkpoint_text, + read_fixed_width_grid, +) +from tsdhn_parity.adapters.frozen_fixture import FrozenFixtureAdapter, write_fixture +from tsdhn_parity.adapters.python_callable import PythonCallableAdapter +from tsdhn_parity.cases import Case +from tsdhn_parity.trace import Trace + +__all__ = [ + "FortranBinaryAdapter", + "FrozenFixtureAdapter", + "LegacyAdapter", + "PythonAdapter", + "PythonCallableAdapter", + "read_checkpoint_text", + "read_fixed_width_grid", + "write_fixture", +] + + +class LegacyAdapter(Protocol): + def run(self, case: Case) -> Trace: + """Replay the reference implementation and record its checkpoints.""" + + +class PythonAdapter(Protocol): + def run(self, case: Case) -> Trace: + """Run the ported implementation and record its checkpoints.""" diff --git a/packages/tsdhn-parity/tsdhn_parity/adapters/fortran.py b/packages/tsdhn-parity/tsdhn_parity/adapters/fortran.py new file mode 100644 index 0000000..f5569bd --- /dev/null +++ b/packages/tsdhn-parity/tsdhn_parity/adapters/fortran.py @@ -0,0 +1,93 @@ +import os +import shutil +import subprocess +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pytest + +from tsdhn_parity.cases import Case +from tsdhn_parity.trace import Checkpoint, Trace + +SKIP_REASON = ( + "requires the real compiled toolchain (TSDHN_TOOLS_DIR); see mise run test-parity" +) + + +def read_checkpoint_text(path: Path) -> np.ndarray: + return np.loadtxt(path) + + +def read_fixed_width_grid(width: int) -> Callable[[Path], np.ndarray]: + """Read fixed-width Fortran output using the format's column width.""" + + def read(path: Path) -> np.ndarray: + return np.array( + [ + [float(line[i : i + width]) for i in range(0, len(line), width)] + for line in path.read_text().splitlines() + ] + ) + + return read + + +def run_tool(executable: Path, working_dir: Path, args: tuple[str, ...] = ()) -> None: + """Copy a legacy executable into its work directory and run it there. + + Legacy executables resolve paths relative to their working directory. + """ + target = working_dir / executable.name + shutil.copy2(executable, target) + target.chmod(target.stat().st_mode | 0o111) + subprocess.run( + [f"./{executable.name}", *args], + cwd=working_dir, + check=True, + ) + + +@dataclass(frozen=True) +class FortranBinaryAdapter: + """Run a legacy executable and read its declared checkpoints. + + The unit-specific `prepare` function writes the input files. + """ + + executable: str + checkpoints: tuple[str, ...] + prepare: Callable[[Case, Path], None] + args: tuple[str, ...] = () + tools_dir: Path | None = None + read_checkpoint: Callable[[Path], np.ndarray] = read_checkpoint_text + + def run(self, case: Case) -> Trace: + binary = _resolve_executable(self.tools_dir, self.executable) + if binary is None: + pytest.skip(SKIP_REASON) + + with tempfile.TemporaryDirectory(prefix="tsdhn-parity-") as raw_dir: + working_dir = Path(raw_dir) + self.prepare(case, working_dir) + run_tool(binary, working_dir, self.args) + checkpoints = tuple( + Checkpoint.of(name, self.read_checkpoint(working_dir / name)) + for name in self.checkpoints + ) + return Trace(case_id=case.id, checkpoints=checkpoints) + + +def _resolve_executable(explicit: Path | None, executable: str) -> Path | None: + tools_dir = explicit or _env_tools_dir() + if tools_dir is None: + return None + candidate = tools_dir / executable + return candidate if candidate.is_file() else None + + +def _env_tools_dir() -> Path | None: + value = os.environ.get("TSDHN_TOOLS_DIR") + return Path(value).resolve() if value else None diff --git a/packages/tsdhn-parity/tsdhn_parity/adapters/frozen_fixture.py b/packages/tsdhn-parity/tsdhn_parity/adapters/frozen_fixture.py new file mode 100644 index 0000000..b49ce76 --- /dev/null +++ b/packages/tsdhn-parity/tsdhn_parity/adapters/frozen_fixture.py @@ -0,0 +1,42 @@ +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from tsdhn_parity.cases import Case +from tsdhn_parity.trace import Checkpoint, Trace + +_ORDER_KEY = "__order__" + + +@dataclass(frozen=True) +class FrozenFixtureAdapter: + """Load a captured MATLAB trace from an `.npz` fixture. + + The fixture stores checkpoint order explicitly. + """ + + fixtures_dir: Path + + def run(self, case: Case) -> Trace: + path = self.fixtures_dir / f"{case.id}.npz" + if not path.is_file(): + raise FileNotFoundError( + f"No captured fixture for case '{case.id}' at {path}. " + "Run scripts/capture_matlab_fixtures.py to generate it." + ) + with np.load(path) as data: + order = list(data[_ORDER_KEY]) + checkpoints = tuple(Checkpoint.of(name, data[name]) for name in order) + return Trace(case_id=case.id, checkpoints=checkpoints) + + +def write_fixture(path: Path, trace: Trace) -> None: + """Write a trace as a compressed NumPy fixture.""" + order = np.array([checkpoint.name for checkpoint in trace.checkpoints]) + arrays = {checkpoint.name: checkpoint.value for checkpoint in trace.checkpoints} + arrays[_ORDER_KEY] = order + path.parent.mkdir(parents=True, exist_ok=True) + # mypy can't prove `arrays` excludes the "allow_pickle" keyword savez_compressed + # also accepts, so it checks **arrays against that bool-typed parameter too. + np.savez_compressed(path, **arrays) # type: ignore[arg-type] diff --git a/packages/tsdhn-parity/tsdhn_parity/adapters/python_callable.py b/packages/tsdhn-parity/tsdhn_parity/adapters/python_callable.py new file mode 100644 index 0000000..5cf28d1 --- /dev/null +++ b/packages/tsdhn-parity/tsdhn_parity/adapters/python_callable.py @@ -0,0 +1,23 @@ +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + +from tsdhn_parity.cases import Case +from tsdhn_parity.trace import CheckpointRecorder, Trace + + +@dataclass(frozen=True) +class PythonCallableAdapter: + """Run a Python implementation and record its checkpoints and result.""" + + fn: Callable[..., Any] + map_params: Callable[[Mapping[str, Any]], Mapping[str, Any]] | None = None + result_checkpoint: str | None = "result" + + def run(self, case: Case) -> Trace: + recorder = CheckpointRecorder() + kwargs = self.map_params(case.params) if self.map_params else case.params + result = self.fn(**kwargs, on_checkpoint=recorder) + if self.result_checkpoint is not None: + recorder(self.result_checkpoint, result) + return recorder.trace(case.id) diff --git a/packages/tsdhn-parity/tsdhn_parity/cases.py b/packages/tsdhn-parity/tsdhn_parity/cases.py new file mode 100644 index 0000000..2c64178 --- /dev/null +++ b/packages/tsdhn-parity/tsdhn_parity/cases.py @@ -0,0 +1,97 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from itertools import combinations, product + +type ParamValue = float | int | str | bool +type PairKey = tuple[int, ParamValue, int, ParamValue] + + +@dataclass(frozen=True) +class Case: + id: str + params: Mapping[str, ParamValue] + + +def exhaustive(**value_lists: Sequence[ParamValue]) -> list[Case]: + """Return every combination of the named parameter values.""" + names = tuple(value_lists) + return [ + _case_from_values(names, values) + for values in product(*(value_lists[name] for name in names)) + ] + + +def pairwise(**value_lists: Sequence[ParamValue]) -> list[Case]: + """Return cases that cover every pair of parameter values.""" + names = tuple(value_lists) + if len(names) < 2: + return exhaustive(**value_lists) + + values = {name: tuple(value_lists[name]) for name in names} + uncovered = _all_pairs(names, values) + covering_cases: list[tuple[ParamValue, ...]] = [] + while uncovered: + seed = next(iter(uncovered)) + combo = _greedy_combination(names, values, uncovered, seed) + covering_cases.append(combo) + uncovered -= _pairs_in(names, combo) + + return [_case_from_values(names, combo) for combo in covering_cases] + + +def _case_from_values(names: tuple[str, ...], values: tuple[ParamValue, ...]) -> Case: + params = dict(zip(names, values, strict=True)) + case_id = "_".join(f"{name}={params[name]}" for name in names) + return Case(id=case_id, params=params) + + +def _all_pairs( + names: tuple[str, ...], values: Mapping[str, tuple[ParamValue, ...]] +) -> set[PairKey]: + return { + (i, a, j, b) + for i, j in combinations(range(len(names)), 2) + for a in values[names[i]] + for b in values[names[j]] + } + + +def _pairs_in(names: tuple[str, ...], combo: tuple[ParamValue, ...]) -> set[PairKey]: + return {(i, combo[i], j, combo[j]) for i, j in combinations(range(len(names)), 2)} + + +def _greedy_combination( + names: tuple[str, ...], + values: Mapping[str, tuple[ParamValue, ...]], + uncovered: set[PairKey], + seed: PairKey, +) -> tuple[ParamValue, ...]: + """Build one case that covers `seed` and as many other pairs as possible.""" + seed_i, seed_a, seed_j, seed_b = seed + assigned: dict[int, ParamValue] = {seed_i: seed_a, seed_j: seed_b} + + for index, name in enumerate(names): + if index not in assigned: + assigned[index] = _best_value(values[name], index, assigned, uncovered) + + return tuple(assigned[index] for index in range(len(names))) + + +def _best_value( + candidates: tuple[ParamValue, ...], + index: int, + assigned: Mapping[int, ParamValue], + uncovered: set[PairKey], +) -> ParamValue: + def newly_covered(candidate: ParamValue) -> int: + return sum( + 1 + for other_index, other_value in assigned.items() + if _pair_key(other_index, other_value, index, candidate) in uncovered + ) + + return max(candidates, key=newly_covered) + + +def _pair_key(i: int, a: ParamValue, j: int, b: ParamValue) -> PairKey: + return (i, a, j, b) if i < j else (j, b, i, a) diff --git a/packages/tsdhn-parity/tsdhn_parity/compare.py b/packages/tsdhn-parity/tsdhn_parity/compare.py new file mode 100644 index 0000000..0a0053e --- /dev/null +++ b/packages/tsdhn-parity/tsdhn_parity/compare.py @@ -0,0 +1,109 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Literal + +import numpy as np + +from tsdhn_parity.trace import Checkpoint, Trace + +type CheckpointStatus = Literal[ + "match", "diverge", "missing_in_legacy", "missing_in_python" +] + + +@dataclass(frozen=True) +class Tolerance: + rtol: float = 1e-4 + atol: float = 1e-9 + + +DEFAULT_TOLERANCE = Tolerance() + + +@dataclass(frozen=True) +class CheckpointDiff: + name: str + status: CheckpointStatus + max_abs_diff: float | None = None + max_rel_diff: float | None = None + + @property + def ok(self) -> bool: + return self.status == "match" + + +@dataclass(frozen=True) +class ComparisonResult: + case_id: str + diffs: tuple[CheckpointDiff, ...] + + @property + def first_divergence(self) -> CheckpointDiff | None: + return next((diff for diff in self.diffs if not diff.ok), None) + + @property + def ok(self) -> bool: + return all(diff.ok for diff in self.diffs) + + +def compare( + legacy: Trace, + python: Trace, + tolerances: Mapping[str, Tolerance] | None = None, + default: Tolerance = DEFAULT_TOLERANCE, +) -> ComparisonResult: + """Compare checkpoints by name and keep their first-seen order.""" + tolerances = tolerances or {} + legacy_by_name = legacy.by_name() + python_by_name = python.by_name() + + diffs = tuple( + _diff_checkpoint( + name, + legacy_by_name.get(name), + python_by_name.get(name), + tolerances.get(name, default), + ) + for name in _ordered_names(legacy.checkpoints, python.checkpoints) + ) + return ComparisonResult(case_id=legacy.case_id, diffs=diffs) + + +def _ordered_names( + legacy_checkpoints: tuple[Checkpoint, ...], + python_checkpoints: tuple[Checkpoint, ...], +) -> list[str]: + seen: dict[str, None] = {} + for checkpoint in (*legacy_checkpoints, *python_checkpoints): + seen.setdefault(checkpoint.name, None) + return list(seen) + + +def _diff_checkpoint( + name: str, + legacy: Checkpoint | None, + python: Checkpoint | None, + tolerance: Tolerance, +) -> CheckpointDiff: + if legacy is None: + return CheckpointDiff(name=name, status="missing_in_legacy") + if python is None: + return CheckpointDiff(name=name, status="missing_in_python") + if legacy.value.shape != python.value.shape: + # A shape mismatch cannot use an element-wise comparison. + return CheckpointDiff(name=name, status="diverge") + + legacy_values = legacy.value.astype(float) + python_values = python.value.astype(float) + abs_diff = np.abs(legacy_values - python_values) + denominator = np.maximum(np.abs(legacy_values), tolerance.atol) + + matches = np.allclose( + legacy_values, python_values, rtol=tolerance.rtol, atol=tolerance.atol + ) + return CheckpointDiff( + name=name, + status="match" if matches else "diverge", + max_abs_diff=float(np.max(abs_diff)), + max_rel_diff=float(np.max(abs_diff / denominator)), + ) diff --git a/packages/tsdhn-parity/tsdhn_parity/pytest_plugin.py b/packages/tsdhn-parity/tsdhn_parity/pytest_plugin.py new file mode 100644 index 0000000..6e5decf --- /dev/null +++ b/packages/tsdhn-parity/tsdhn_parity/pytest_plugin.py @@ -0,0 +1,60 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +import pytest + +from tsdhn_parity.adapters import LegacyAdapter, PythonAdapter +from tsdhn_parity.cases import Case +from tsdhn_parity.compare import CheckpointDiff, ComparisonResult, Tolerance, compare + + +@dataclass(frozen=True) +class UnitSpec: + id: str + cases: Sequence[Case] + legacy: LegacyAdapter + python: PythonAdapter + tolerances: Mapping[str, Tolerance] = field(default_factory=dict) + default_tolerance: Tolerance = field(default_factory=Tolerance) + + +def parity_cases(spec: UnitSpec) -> list[Any]: + """Return pytest parameters named by case ID.""" + return [pytest.param(case, id=case.id) for case in spec.cases] + + +def assert_parity(spec: UnitSpec, case: Case) -> ComparisonResult: + """Run both adapters and fail at the first divergent checkpoint.""" + legacy_trace = spec.legacy.run(case) + python_trace = spec.python.run(case) + result = compare( + legacy_trace, python_trace, spec.tolerances, spec.default_tolerance + ) + divergence = result.first_divergence + if divergence is not None: + raise AssertionError(_failure_message(spec, result, divergence)) + return result + + +def _failure_message( + spec: UnitSpec, result: ComparisonResult, divergence: CheckpointDiff +) -> str: + lines = [ + f"parity check failed for unit '{spec.id}', case '{result.case_id}'", + f"first divergence at checkpoint '{divergence.name}': {divergence.status}", + ] + if divergence.max_abs_diff is not None: + lines.append(f" max_abs_diff = {divergence.max_abs_diff:.6g}") + if divergence.max_rel_diff is not None: + lines.append(f" max_rel_diff = {divergence.max_rel_diff:.6g}") + + later_diverging = [ + diff.name + for diff in result.diffs + if not diff.ok and diff.name != divergence.name + ] + if later_diverging: + lines.append(f" also diverges later at: {', '.join(later_diverging)}") + + return "\n".join(lines) diff --git a/packages/tsdhn-parity/tsdhn_parity/trace.py b/packages/tsdhn-parity/tsdhn_parity/trace.py new file mode 100644 index 0000000..5ca94cf --- /dev/null +++ b/packages/tsdhn-parity/tsdhn_parity/trace.py @@ -0,0 +1,39 @@ +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import numpy as np + +type OnCheckpoint = Callable[[str, Any], None] + + +@dataclass(frozen=True) +class Checkpoint: + name: str + value: np.ndarray + + @staticmethod + def of(name: str, value: Any) -> Checkpoint: + return Checkpoint(name=name, value=np.asarray(value)) + + +@dataclass(frozen=True) +class Trace: + case_id: str + checkpoints: tuple[Checkpoint, ...] # Checkpoint order defines the trace. + + def by_name(self) -> dict[str, Checkpoint]: + return {checkpoint.name: checkpoint for checkpoint in self.checkpoints} + + +class CheckpointRecorder: + """Collect named checkpoints emitted by a Python adapter.""" + + def __init__(self) -> None: + self._checkpoints: list[Checkpoint] = [] + + def __call__(self, name: str, value: Any) -> None: + self._checkpoints.append(Checkpoint.of(name, value)) + + def trace(self, case_id: str) -> Trace: + return Trace(case_id=case_id, checkpoints=tuple(self._checkpoints)) diff --git a/packages/tsdhn/docs/legacy.md b/packages/tsdhn/docs/legacy.md new file mode 100644 index 0000000..1f0e1f7 --- /dev/null +++ b/packages/tsdhn/docs/legacy.md @@ -0,0 +1,94 @@ +# Older MATLAB and Fortran references + +The older programs are being phased out. They remain useful for identifying +file formats, numerical ordering, and behavior that the Python port intended +to preserve. Agreement with them is compatibility evidence, not scientific +validation. + +## Active reference map + +| Python code | Older reference | Status | +| --- | --- | --- | +| `calculator.py` source relations | formulas also present in `fault_plane.f90` | Active compatibility basis for source dimensions and slip | +| `fault_plane.py` | `model/fault_plane.f90` | Active Fortran comparison source | +| `deform.py` | `model/def_oka.f` | Active deformation comparison source; Okada-based | +| `tsunami.py` | `model/tsunami1.for` | Active propagation comparison source | +| MATLAB trace comparisons | saved runs produced by capture scripts | Compatibility evidence for selected intermediate values | + +The supported setup script and `model/Makefile` build `def_oka.f` as the +`deform` executable. `model/deform.for` is a different, older implementation +based on Mansinha and Smylie and is not the reference for `deform.py`. + +## Historical and exploratory files + +### `packages/tsdhn/tests/original.m` + +This is a historical logging script, not the definition of the current +calculation. It differs from the active path in rigidity, dip handling, +degree-distance conversion, and longitude treatment. Do not combine its +constants with values from `fault_plane.f90` or the Python port. + +### `model/fault_plane_n.m` + +This MATLAB program explores multiple subfaults and spherical geometry. The +current pipeline uses the single-fault geometry in `fault_plane.f90` and does +not use these spherical corrections. Treat it as a separate research path. + +### `model/mareografo_a.m` + +This is a historical manual helper. Instructions embedded in it for editing a +specific line in the Fortran solver are not current pipeline instructions. + +### `model/deform.for` + +This is the older Mansinha-Smylie deformation program. It has different +equations and its own small-value handling. Building or comparing it as if it +were `def_oka.f` can produce a false conclusion about the Python port. + +## Compatibility preserved by the Python port + +The Python stages intentionally preserve several old-program details: + +- mixed longitude frames between public inputs and model grids; +- one-based grid indices in intermediate files; +- integer truncation before fault-window snapping; +- a fixed rake of 90 degrees in the active fault-plane path; +- float32 calculation in deformation and propagation; +- branch behavior near singular points in the Okada-based calculation; +- fixed grid and time-step constants; +- continuity, boundary, and momentum update order; +- fixed-width output fields and decimal quantization; +- gauge and maximum-height sampling intervals. + +These details are compatibility requirements only while matching the active +older reference is a project goal. A scientifically justified replacement may +change them, but the change should be explicit and tested at the behavior +boundary. + +## Deliberate Python differences + +The Python propagation stage keeps the active grid-A calculation but does not +write older movie frames or travel-time arrays that the current pipeline does +not consume. It swaps array references instead of copying full time levels. +The boundary routine is passed the previous momentum arrays explicitly so this +optimization preserves the old update order. + +The Python solver adds resumable checkpoints. Checkpoints are an operational +feature and are not part of the Fortran result. Resume tests require a resumed +run to produce the same fixed-width outputs as an uninterrupted Python run. + +## Removing an older source + +Before removing a MATLAB or Fortran file, verify that: + +1. No comparison test compiles or invokes it. +2. Its file formats and numerical rules are documented in the researcher + guides or focused tests. +3. Any saved trace identifies how it was produced. +4. The active Python behavior has an independent test where practical. +5. The removal does not erase the only known provenance for a constant or + empirical correction. + +If a source is retained only for history, label it as historical in +[`model/readme.md`](../../../model/readme.md). Do not let a generic build target +make it look active. diff --git a/packages/tsdhn/docs/pipeline.md b/packages/tsdhn/docs/pipeline.md new file mode 100644 index 0000000..f95ce47 --- /dev/null +++ b/packages/tsdhn/docs/pipeline.md @@ -0,0 +1,133 @@ +# Simulation pipeline + +This guide follows data through one run. It is intended for researchers who +need to replace a stage, inspect an intermediate result, or change a numerical +assumption without confusing raw solver data with report data. + +## Before the stages + +The engine validates the selected model directory and prepares a run +directory. Model inputs are linked or copied into that directory. Generated +files from an earlier run are not treated as model inputs. + +Source-parameter calculation writes `hypo.dat` before the declared processing +stages begin. Port arrival-time estimates are also calculated outside the +propagation stages. See [`science.md`](science.md) for their assumptions. + +## Stage map + +| Stage | Reads | Writes | Important rules | +| --- | --- | --- | --- | +| `fault_plane` | `hypo.dat`, `mecfoc.dat`, `bathy/xa.dat`, `bathy/ya.dat` | `pfalla.inp`, `xyo.dat`, `meca.dat` | Converts longitude, selects a nearby mechanism, snaps indices, preserves fixed file layouts | +| `deform` | `pfalla.inp`, `xyo.dat` | `deform_a.grd` | Okada-based float32 compatibility calculation for one segment | +| `tsunami` | `bathy/grid_a.grd`, `deform_a.grd`, `xyo.dat`, `tidal.dat` | `zfolder/green.dat`, `zfolder/zmax_a.grd` | Linear shallow-water update with fixed order, sampling, and resumable state | +| `maxola` | `zfolder/zmax_a.grd`, `meca.dat`, station configuration | `maxola.pdf` | Rescales the raw grid to a 12 m display maximum before plotting | +| `ttt_max` | `zfolder/green.dat` | `zfolder/green_rev.dat`, `ttt_max.dat`, `mareograma.svg` | Applies empirical station factors before summaries and plots | +| `ttt_inverso` | `meca.dat`, `ttt_mundo/cortado.i2` | `ttt_mundo/ttt.b` | Calls `ttt_client`; GMT rewrites the grid header for later contouring | +| `point_ttt` | `ttt_mundo/ttt.b` and map inputs | `ttt_mundo/ttt.pdf` | Builds the GMT arrival-time map | +| `copy_ttt_pdf` | `ttt_mundo/ttt.pdf` | `ttt.pdf` | Copies the report to the run root | + +The first five stages run in the run root. The travel-time map stages run in +`ttt_mundo`. + +## Fault-plane files + +### `hypo.dat` + +Five lines: origin time, longitude, latitude, hypocentral depth in km, and +moment magnitude. Longitude is written in the public `-180..180` frame. + +### `pfalla.inp` + +Nine whitespace-separated fields: + +```text +I0 J0 D0 L0 W0 strike dip rake top_edge_depth +``` + +`I0` and `J0` are one-based indices in the full bathymetry grid. Slip, length, +width, and depth are in meters. Angles are in degrees. + +### `xyo.dat` + +The first four fields are one-based inclusive bounds: + +```text +IDS IDE JDS JDE +``` + +The current writer also appends the full grid dimensions. The deformation and +tsunami readers consume only the first four values. + +### `meca.dat` + +A fixed-width mechanism record used by the map and travel-time stages. Its +longitude is in the `0..360` frame. Changing its layout affects readers that +expect the old ten-field record. + +## Deformation file + +`deform_a.grd` contains the initial vertical sea-surface displacement inside +the `xyo.dat` window. Each row follows the first grid axis and contains +nine-character fields with three decimal places. The tsunami stage inserts +this smaller array into the full solver grid. + +This file is quantized. Use the in-memory deformation array when studying +differences smaller than its output precision. + +## Tsunami files + +`green.dat` has one row per sampling time. The first seven-character field is +minutes from the origin; the following fields are elevations at virtual-gauge +indices from `tidal.dat`. + +`zmax_a.grd` has the full `2461 x 2056` grid. Each value is the greatest +positive elevation observed at a sampling time. It is not a maximum over every +3-second solver step. + +The checkpoint file under `zfolder` is temporary run state, not a scientific +output. It is removed after successful completion. Resume accepts it only when +its version and array layout match the current solver. + +## Raw values and reports + +| File | Raw solver values? | Transformation | +| --- | --- | --- | +| `deform_a.grd` | Numerical stage output, but fixed-width quantized | Three decimal places | +| `green.dat` | Sampled solver values, but fixed-width quantized | Sampled every 60 s; three decimal places | +| `zmax_a.grd` | Sampled solver maximum, but fixed-width quantized | Updated every 60 s; three decimal places | +| `green_rev.dat` | No | Station-specific empirical amplitude factors | +| `ttt_max.dat` | No | Summary of corrected station values | +| `mareograma.svg` | No | Corrected station values and selected plot scale | +| `maxola.pdf` | No | Grid rescaled so its displayed maximum is 12 m | + +## Completion markers and resume + +The engine records a stage as complete only after every declared output +exists. On resume, a stage is skipped only when its marker and outputs remain +valid. Removing or renaming an output invalidates that stage. + +The tsunami checkpoint is more detailed than a stage marker. It preserves the +time levels and sampled state needed to continue inside the propagation loop. +Any change to the saved arrays, update order, sampling meaning, or buffer-swap +position requires a pipeline-version change. + +## Where to make a change + +| Research change | Primary location | Required follow-up | +| --- | --- | --- | +| Rupture scaling or rigidity | `calculator.py`, `fault_plane.py` | Document source; update focused values and fault-plane comparisons | +| Mechanism selection | `calculator.py`, `fault_plane.py` | Define longitude and distance rule; test wrap behavior | +| Fault window or depth | `fault_plane.py` | Check `pfalla.inp`, `xyo.dat`, deformation shape, and old-program comparison | +| Deformation equations or singular handling | `deform.py` | Add independent cases; run deformation comparison and golden tests | +| Grid, time step, boundary, or wet-cell rule | `tsunami.py` | Update checkpoint version; run small behavior tests and full comparison | +| Gauge sampling | `tsunami.py` | Update checkpoint meaning and explain whether maxima are sampled | +| Maximum-height visualization | `render/maxola.py` | Keep raw and display values distinct; test grid orientation and scale | +| Station amplitude correction | `render/ttt_max.py` | Record provenance and update report tests | +| Approximate port arrival time | `calculator.py` | Record calibration source and validate intended geographic domain | +| `ttt_client` processing | `render/ttt_inverso.py` | Test command, fixed epicenter format, and GMT grid compatibility | + +When a change deliberately breaks compatibility, preserve a small description +of the old behavior in [`legacy.md`](legacy.md) and record why the new behavior +is preferred. Saved expected values should change only after that decision is +documented. diff --git a/packages/tsdhn/docs/science.md b/packages/tsdhn/docs/science.md new file mode 100644 index 0000000..cd1216f --- /dev/null +++ b/packages/tsdhn/docs/science.md @@ -0,0 +1,246 @@ +# Scientific and numerical assumptions + +This guide describes what the current code computes. It separates rules with +named scientific references from rules inherited from older programs. A rule +that matches old code is not necessarily scientifically validated. + +## Input parameters + +| Field | Meaning | Unit or format | +| --- | --- | --- | +| `Mw` | Moment magnitude | dimensionless | +| `h` | Hypocentral depth | km | +| `lat0` | Epicenter latitude | decimal degrees, north positive | +| `lon0` | Epicenter longitude | decimal degrees, east positive; western longitudes are negative | +| `hhmm` | Origin time | four digits, UTC hour and minute | +| `dia` | Day used in arrival-time text | two-character value | + +The simulation models one rectangular fault. Strike and dip are copied from +the nearest record in `mecfoc.dat` because they are not public input fields. +Rake is fixed at 90 degrees in the fault-plane stage. + +## Source dimensions and slip + +`calculator.py` and `fault_plane.py` use these magnitude relations: + +```text +L = 10^(0.55 Mw - 2.19) km +W = 10^(0.31 Mw - 0.63) km +``` + +The active Fortran source identifies them as Papazachos et al. (2004). The +repository does not contain the paper or a full citation, so this guide records +the attribution without claiming that the relation is appropriate for every +tectonic setting. + +Moment and average slip are calculated as: + +```text +M0 = 10^(1.5 Mw + 9.1) N m +D = M0 / (mu L W) m +mu = 4.0e10 N/m^2 +``` + +The moment relation is labeled Hanks and Kanamori in the historical MATLAB +script. The rigidity value is inherited from the active `fault_plane.f90`. +The older `packages/tsdhn/tests/original.m` uses `4.5e10 N/m^2`; that script is +historical and is not the active reference. + +## Coordinate frames + +Two longitude frames exist because the model files and public inputs use +different conventions: + +- Public inputs and map calculations use `-180..180` degrees. +- The bathymetry axis and active Fortran fault-plane program use `0..360` + degrees. + +The fault-plane stage adds 360 to a negative epicenter longitude before +searching `mecfoc.dat` and the bathymetry grid. The calculator preview instead +converts mechanism records into the public input frame. These searches can +choose different records near the longitude wrap. That difference is +currently preserved for compatibility. + +Fault rectangle offsets use two inherited approximations: + +- `111.0 km` per degree when placing the fault origin and computing its depth. +- `60 * 1853 m` per degree when constructing the displayed rectangle corners. + +The second value treats one degree as 60 nautical miles and one nautical mile +as 1853 m. These are compatibility rules, not a general geodesic model. The +corner list starts at the fault origin, follows the rectangle, and repeats the +first point to close the polygon. + +## Fault-plane stage + +The fault-plane stage follows `model/fault_plane.f90`: + +1. Read origin time, longitude, latitude, depth, and magnitude from + `hypo.dat`. +2. Calculate rupture length, width, moment, and average slip. +3. Select strike and dip from the nearest `mecfoc.dat` record in the `0..360` + longitude frame. +4. Place the fault origin and snap it to the bathymetry axes. +5. Recompute the depth of the fault's upper edge from the continuous origin. +6. Write the deformation inputs and the mechanism record used by maps. + +The grid window is deliberately unusual. Its geographic bounds are converted +to integers before the nearest bathymetry cell is selected. Magnitudes above +8 use a window multiplier of 1.4; other magnitudes use 2.8. The source of +these multipliers is not documented in the repository. + +If the calculated upper-edge depth is negative, the stage substitutes 5000 m. +This is inherited behavior. It should not be described as a physical depth +correction without a source. + +## Deformation stage + +`deform.py` is a compatibility port of `model/def_oka.f`. That Fortran source +identifies its formulation as Okada (1985). The repository does not include a +full citation or an independent derivation. + +The stage computes vertical displacement for one rectangular segment. Inputs +are grid indices, slip, length, width, strike, dip, rake, and top-edge depth. +Lengths and depths are in meters; angles are in degrees. The grid spacing is +`7412.9951096 m` on both axes. + +Compatibility depends on details that would normally be implementation +choices: + +- calculations use float32 arrays to match Fortran `REAL*4`; +- pi is calculated through float32 `asin`; +- strike values of 0 or 360 degrees receive a 0.001 degree offset; +- singular branches use an epsilon of `1e-8`; +- values with absolute magnitude at least 20 m are replaced with zero; +- `deform_a.grd` uses fixed-width fields with three decimal places. + +The 20 m rule comes directly from `def_oka.f`. Its scientific basis is not +known. Treat it as an inherited outlier guard, not a physical correction. + +## Tsunami propagation stage + +`tsunami.py` ports the active calculation in `model/tsunami1.for`. The Fortran +header describes a linear shallow-water model on a spherical grid. The Python +port preserves the active grid-A calculation and omits old outputs that the +current pipeline does not consume. + +| Quantity | Current value | +| --- | ---: | +| Grid shape | 2461 by 2056 cells | +| Angular spacing | 240 arcseconds, or 1/15 degree | +| Time step | 3 s | +| Number of steps | 33,602 | +| Gauge sampling interval | 20 steps, or 60 s | +| Number of gauges | 17 | +| Earth radius | `6.37e6 m` | +| Gravity | `9.8 m/s^2` | +| Southern latitude origin | `-76.006 degrees` | +| Flush threshold | `1e-5` | + +Bathymetry uses the sign convention inherited from `grid_a.grd`: positive +values are water depths and negative values are land. Positive water depths +below 10 m are raised to 10 m before integration. Zero is not raised. + +The solver uses staggered discharge depths. `HM` averages adjacent cells along +the first grid axis; `HN` averages along the second. The final row or column +keeps the original bathymetry value. Latitude factors are advanced by repeated +float32 addition because recomputing them from an index changes accumulated +rounding relative to the Fortran program. + +Each time step runs in this order: + +1. Apply the continuity equation to produce the next elevation. +2. Apply open-boundary radiation using the previous momentum arrays. +3. Update both momentum components from the new elevation. +4. Sample gauges and maximum positive elevation at the sampling interval. +5. Swap the current and next buffers. + +The first Fortran row and column are outside the interior continuity update. +Momentum is written only when both cells beside a discharge point are wet. +Small elevations and momenta are set to exactly zero. Dry-cell boundary points +are left unchanged by the boundary routine. + +Boundary passes have a fixed order. The two latitude edges are written first, +then the two longitude edges. A later edge pass owns a shared corner. Changing +the pass order changes results. + +`zfolder/green.dat` contains sampled time in minutes followed by 17 sampled +elevations. `zfolder/zmax_a.grd` contains the maximum positive elevation seen +at the same sampling times. It is not updated at every solver step. Both files +use fixed-width fields with three decimal places. + +### Checkpoints + +The Python solver saves both elevation buffers, both buffers for each momentum +component, the sampled maximum grid, gauge rows, and the last completed step. +Saved arrays must have the current shape and float32 dtype. A checkpoint is +rejected when its pipeline version, shape, gauge count, or step index is not +valid. + +The checkpoint is written after buffer swaps, so it represents a fully +completed step. Changing the meaning or order of saved state requires a +pipeline-version change. A successful run removes the checkpoint after writing +the final fixed-width outputs. + +## Arrival-time estimates + +The calculator's port arrival times are separate from the propagation solver. +It first computes spherical distance, then applies inherited rules: + +- distances of at least 750 km use `distance / 790 + 0.2` hours; +- epicenters outside latitude `-19..0` use `distance / 700` hours; +- other paths sample bathymetry at 101 points and integrate `1/sqrt(g h)` with + Simpson's rule; +- the integrated result is multiplied by 0.5; +- integrated times above 3 hours are replaced by `distance / 733 + 0.25`; +- integrated times between 1.4 and 3 hours are replaced by + `distance / 690 + 0.2`. + +The path construction also uses `110 km` per degree. The repository does not +identify the source of the thresholds, speeds, offsets, multiplier, or path +approximation. These are inherited arrival-time calibration rules. Do not +present them as a validated travel-time model until their provenance and +intended domain are established. + +`ttt_client` produces the separate arrival-time grid used for `ttt.pdf`. + +## Report transformations + +### Maximum-height map + +`zfolder/zmax_a.grd` is the raw sampled solver grid. `maxola` reshapes and +flips that grid, then rescales every finite value so the largest displayed +value is 12 m and rounds values to two decimals. The resulting map is +pixel-registered for GMT. + +The 12 m maximum is a display transformation. Values read from `maxola.pdf` +must not be treated as raw solver amplitudes. Quantitative work should start +from `zfolder/zmax_a.grd` and state whether the sampling interval matters. + +### Station reports + +`ttt_max` multiplies each virtual-gauge series by a station-specific factor of +the form `(numerator / denominator)^0.25`. It then writes corrected series, +first-positive-sample indices, peak values, and the mareogram. + +The repository does not explain the numerator, denominator, or fourth-root +rule. These factors are empirical report corrections applied after the solver. +They are not part of the shallow-water calculation, and this documentation +does not claim that they represent bathymetry, observations, or a validated +physical calibration. + +## Changes that need special care + +Before changing numerical behavior, record: + +- the source or research reason for the change; +- units and longitude frame; +- array axis and one-based or zero-based indexing assumptions; +- float dtype and evaluation order; +- fixed-width field size and decimal precision; +- whether compatibility with the older program is intended; +- whether checkpoint state or meaning changes. + +Add an independent focused test where practical. Run legacy comparisons when +preserving compatibility, and inspect spatial golden outputs when changing the +full pipeline. Do not update saved values merely to make a changed test pass. diff --git a/packages/tsdhn/docs/testing.md b/packages/tsdhn/docs/testing.md new file mode 100644 index 0000000..304422f --- /dev/null +++ b/packages/tsdhn/docs/testing.md @@ -0,0 +1,107 @@ +# Testing numerical and research changes + +The test groups answer different questions. No single group establishes that +the model is scientifically correct. + +## Fast behavior tests + +`mise run test` runs the normal Python suite without starting services. The +numerical tests use small arrays and focused files to check behavior such as: + +- source-parameter units and known values; +- longitude conversion and grid-window truncation; +- singular branches and float32 deformation output; +- wet and dry cell rules; +- continuity, boundary, and momentum update behavior; +- fixed-width file formats; +- checkpoint rejection and resume equivalence; +- report grid orientation and display scaling. + +The strongest focused tests use expected values derived independently from the +production implementation. Examples include hand-computed shallow-water cells, +sentinel values that expose unexpected writes, and an independent fixed-width +reader. A test that copies the production loop or formula can preserve the +same bug and should be avoided unless no smaller observable behavior exists. + +Run one test with: + +```sh +uv run pytest packages/tsdhn/tests/test_tsunami.py::test_mass_step_hand_computed +``` + +## Golden pipeline tests + +`mise run test-golden` runs the Python pipeline with real GMT and +`ttt_client`. It checks complete output sets, fixed-width file fingerprints, +and selected spatial values. + +Golden tests answer: did the Python pipeline output change for this saved +scenario? They do not establish whether the old or new result is physically +correct. + +Aggregate statistics can miss a transposed or rearranged grid, so the suite +also checks values at known coordinates. A deliberate numerical change should +inspect spatial output, not only update means and maxima. + +## Legacy comparison tests + +`mise run test-parity` compares Python checkpoints with saved MATLAB runs or +compiled Fortran output. These tests establish whether Python reproduces the +selected older implementation within the stated tolerance. + +They do not prove that the older implementation is correct. They can preserve +old mistakes, inherited calibrations, or a limited geographic assumption. + +The tsunami comparison builds the fault-plane and deformation input once and +gives that same initial condition to both propagation solvers. This isolates +the shallow-water solver. It does not independently compare the complete +fault-plane-to-tsunami chain. + +Tolerances must account for float32 rounding and fixed-width output +quantization. They should still be narrow enough to catch a meaningful change. +Document why a tolerance exists; do not widen it only because a comparison +started failing. + +## Saved MATLAB data + +Saved MATLAB arrays are captured evidence, not self-explaining constants. The +comparison package README describes how to refresh them. A refresh should +record: + +- the scenario and input values; +- the MATLAB source and command used; +- the checkpoint names; +- why a new capture is needed; +- whether the expected scientific behavior changed. + +Do not refresh saved data to hide an unexplained difference. + +## Choosing tests for a change + +| Change | Minimum evidence | +| --- | --- | +| Refactor with no intended numerical change | Focused behavior tests and relevant legacy comparison | +| File-format change | Independent reader/writer test and downstream stage test | +| Float dtype or evaluation-order change | Focused edge cases, legacy comparison, and golden spatial inspection | +| New scientific relation | Unit test from an independent worked example plus source citation | +| Boundary or wet-cell rule | Hand-computed small-grid tests and propagation comparison | +| Checkpoint layout or meaning | Interrupted/resumed equivalence and pipeline-version rejection | +| Display-only transformation | Raw-value preservation test and explicit report transformation test | +| Empirical correction | Provenance, intended domain, direct behavior tests, and report regression test | + +## Reviewing a numerical test + +Ask: + +1. What observable behavior does this test protect? +2. Is the expected result independent of the implementation? +3. Are units, coordinate frame, dtype, and indexing explicit? +4. Does a tolerance represent known rounding or merely make the test pass? +5. Is the scenario scientifically relevant, or only combinatorial? +6. Could both Python and the older program share the same error? +7. If expected data changed, is the research reason recorded? + +Pairwise input generation improves combinations but does not establish +scientific coverage of earthquake scenarios. Add named research cases when a +tectonic setting, magnitude range, depth, coastline relation, or numerical +edge case matters. diff --git a/packages/tsdhn/pyproject.toml b/packages/tsdhn/pyproject.toml index 6f7bf78..d6e5dbc 100644 --- a/packages/tsdhn/pyproject.toml +++ b/packages/tsdhn/pyproject.toml @@ -11,7 +11,8 @@ authors = [ dependencies = [ "pydantic==2.13.4", "scipy==1.18.0", - "numpy==2.5.1", + "numpy==2.4.6", + "numba==0.66.0", "pygmt==0.19.0", "pyyaml==6.0.3", "typer==0.26.8", diff --git a/packages/tsdhn/readme.md b/packages/tsdhn/readme.md index a147fc7..5cc5bbc 100644 --- a/packages/tsdhn/readme.md +++ b/packages/tsdhn/readme.md @@ -1,8 +1,110 @@ # tsdhn -`tsdhn` contains the CLI for researchers and the shared simulation engine. The -API, worker, and CLI should use this same engine. +`tsdhn` is the simulation engine and researcher CLI. The CLI, API, and worker +run the same Python pipeline. -The public interface is the `tsdhn` command. Model datasets are versioned -release assets and are resolved by the runtime code, not by deployment-specific -code. +The Python implementation is replacing a collection of MATLAB and Fortran +programs. The engine preserves legacy behavior to demonstrate successful +porting, but compatibility does not establish scientific validity. + +## Start here + +Install the Python workspace and model files: + +```sh +mise run install +uv run tsdhn assets install +uv run tsdhn doctor +``` + +Calculate source parameters without running the propagation model: + +```sh +uv run tsdhn calc --mw 8.0 --lat -20.5 --lon -70.5 +``` + +Run the full pipeline: + +```sh +uv run tsdhn run --mw 8.0 --lat -20.5 --lon -70.5 +``` + +Use `--model-version` to select installed model files, `--model-dir` to use a +specific model directory, and `--work-dir` to keep a run in a known directory. + +## Researcher guides + +- [`docs/science.md`](docs/science.md) explains units, coordinates, equations, + numerical assumptions, and empirical corrections. +- [`docs/pipeline.md`](docs/pipeline.md) explains each stage, its files, + checkpoints, and where to make changes. +- [`docs/legacy.md`](docs/legacy.md) maps the active Python stages to the older + MATLAB and Fortran programs. +- [`docs/testing.md`](docs/testing.md) explains unit, golden, and legacy + comparison tests and the limits of each test type. +- [`model/readme.md`](../../model/readme.md) inventories the checked-in model + files and identifies active and historical references. + +Read these guides before changing a formula, array layout, coordinate +conversion, fixed-width file, or pipeline checkpoint. + +## Model files and external programs + +Model files resolve in this order: + +1. `--model-dir`. +2. `TSDHN_MODEL_DIR`. +3. The installed model version for the package. + +`tsdhn assets install` downloads a versioned archive from the TSDHN GitHub +release repository. The default model directory is +`$XDG_DATA_HOME/tsdhn/models`, or `$HOME/.local/share/tsdhn/models` when +`XDG_DATA_HOME` is not set. Set `TSDHN_DATA_HOME` to use another location. + +The report stages call GMT and `ttt_client`. The older Fortran programs are +needed only for comparison tests. + +## Pipeline summary + +```text +fault_plane -> deform -> tsunami -> maxola -> ttt_max +ttt_inverso -> point_ttt -> copy_ttt_pdf +``` + +Each stage declares its output files. The engine writes a completion marker +after those files exist. A resumed run skips a stage only when its marker and +outputs are still valid. The tsunami stage also saves its numerical state so a +worker restart can continue the propagation loop. + +The main researcher outputs are: + +| File | Meaning | +| --- | --- | +| `calculation.json` | Source parameters and fault rectangle | +| `travel_times.json` | Port arrival times and distances | +| `zfolder/green.dat` | Raw sampled solver elevation at virtual gauges | +| `zfolder/zmax_a.grd` | Raw sampled maximum positive solver elevation | +| `maxola.pdf` | Display map made from a rescaled copy of `zmax_a.grd` | +| `ttt.pdf` | Arrival-time map produced with `ttt_client` and GMT | +| `mareograma.svg` | Selected station series after empirical scaling | + +`maxola.pdf` and `mareograma.svg` do not show untouched solver values. See the +science guide before using them for quantitative analysis. + +## Code map + +- `tsdhn/calculator.py`: source parameters and approximate port arrival times. +- `tsdhn/fault_plane.py`: fault placement and stage input files. +- `tsdhn/deform.py`: Okada-based vertical displacement compatibility port. +- `tsdhn/tsunami.py`: linear shallow-water compatibility solver. +- `tsdhn/pipeline/`: stage definitions and order. +- `tsdhn/render/`: maps, station summaries, and report transformations. +- `tsdhn/engine.py`: run setup, stage execution, resume markers, and output + collection. +- `tsdhn/runtime.py`: model validation and external-program checks. +- `tsdhn/assets.py`: versioned model installation. +- `tsdhn/cli/`: researcher commands. + +Service deployment is documented in [`DEPLOY.md`](../../DEPLOY.md). System +responsibilities and request flow are documented in +[`ARCHITECTURE.md`](../../ARCHITECTURE.md). diff --git a/packages/tsdhn/tests/parity/__init__.py b/packages/tsdhn/tests/parity/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/tsdhn/tests/parity/conftest.py b/packages/tsdhn/tests/parity/conftest.py new file mode 100644 index 0000000..1a80b1e --- /dev/null +++ b/packages/tsdhn/tests/parity/conftest.py @@ -0,0 +1,4 @@ +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[4] +MODEL_DIR = REPO_ROOT / "model" diff --git a/packages/tsdhn/tests/parity/deform/__init__.py b/packages/tsdhn/tests/parity/deform/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/tsdhn/tests/parity/deform/spec.py b/packages/tsdhn/tests/parity/deform/spec.py new file mode 100644 index 0000000..834e41f --- /dev/null +++ b/packages/tsdhn/tests/parity/deform/spec.py @@ -0,0 +1,104 @@ +"""Compare the Python Okada port with the compiled Fortran output. + +Both sides write F9.3 grids, so the comparison reads the formatted files. +The explicit tolerance accounts for float32 rounding and the 0.001 output +quantization. +""" + +import tempfile +from pathlib import Path +from typing import Any + +from tsdhn.deform import clip_anomalous_values, compute_deform_grid, write_deform_grid +from tsdhn_parity import ( + Case, + FortranBinaryAdapter, + OnCheckpoint, + PythonCallableAdapter, + Tolerance, + UnitSpec, + read_fixed_width_grid, +) + +_ALASKA_1964_WINDOW = { + "IDS": 1032, + "IDE": 1249, + "JDS": 1872, + "JDE": 2056, + "IA": 2461, + "JA": 2056, +} + +CASES = [ + Case( + id="alaska_1964", + params={ + "I0": 1180, + "J0": 1988, + "D0": 11.965752308065987, # slip (m), using the legacy rigidity + "L0": 575439.9373371573, # length (m) + "W0": 144543.97707459278, # width (m) + "TH": 247.0, # strike / azimuth (deg) + "DL": 18.0, # dip (deg) + "RD": 90.0, # rake (degrees) + "HH": 5000.0, # depth to top of fault (m) + **_ALASKA_1964_WINDOW, + }, + ), + Case( + id="alaska_1964_rd45", + params={ + "I0": 1180, + "J0": 1988, + "D0": 11.965752308065987, + "L0": 575439.9373371573, + "W0": 144543.97707459278, + "TH": 247.0, + "DL": 18.0, + "RD": 45.0, # covers a different strike-slip/dip-slip split + "HH": 5000.0, + **_ALASKA_1964_WINDOW, + }, + ), +] + +# The deformation step ignores the grid dimensions after the requested window. +_UNUSED_PARAMS = ("IA", "JA") +_READ_GRID = read_fixed_width_grid(9) # Legacy deformation grids use F9.3 fields. + + +def _prepare(case: Case, working_dir: Path) -> None: + p = case.params + (working_dir / "pfalla.inp").write_text( + f"{p['I0']} {p['J0']} {p['D0']} {p['L0']} {p['W0']} " + f"{p['TH']} {p['DL']} {p['RD']} {p['HH']}\n" + ) + (working_dir / "xyo.dat").write_text( + f"{p['IDS']} {p['IDE']} {p['JDS']} {p['JDE']} {p['IA']} {p['JA']}\n" + ) + + +LEGACY = FortranBinaryAdapter( + executable="deform", + checkpoints=("deform_a.grd",), + prepare=_prepare, + read_checkpoint=_READ_GRID, +) + + +def _run(on_checkpoint: OnCheckpoint, **kwargs: Any) -> None: + params = {k: v for k, v in kwargs.items() if k not in _UNUSED_PARAMS} + grid = clip_anomalous_values(compute_deform_grid(**params)) + with tempfile.TemporaryDirectory(prefix="tsdhn-parity-deform-") as raw_dir: + path = Path(raw_dir) / "deform_a.grd" + write_deform_grid(path, grid) + on_checkpoint("deform_a.grd", _READ_GRID(path)) + + +SPEC = UnitSpec( + id="deform", + cases=CASES, + legacy=LEGACY, + python=PythonCallableAdapter(fn=_run, result_checkpoint=None), + tolerances={"deform_a.grd": Tolerance(atol=0.02)}, +) diff --git a/packages/tsdhn/tests/parity/deform/test_deform_parity.py b/packages/tsdhn/tests/parity/deform/test_deform_parity.py new file mode 100644 index 0000000..e977512 --- /dev/null +++ b/packages/tsdhn/tests/parity/deform/test_deform_parity.py @@ -0,0 +1,10 @@ +import pytest + +from tsdhn_parity import Case, assert_parity, parity_cases + +from .spec import SPEC + + +@pytest.mark.parametrize("case", parity_cases(SPEC)) +def test_deform_matches_binary(case: Case) -> None: + assert_parity(SPEC, case) diff --git a/packages/tsdhn/tests/parity/fault_plane/__init__.py b/packages/tsdhn/tests/parity/fault_plane/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/tsdhn/tests/parity/fault_plane/fixtures/alaska_1964.npz b/packages/tsdhn/tests/parity/fault_plane/fixtures/alaska_1964.npz new file mode 100644 index 0000000..e6e5467 Binary files /dev/null and b/packages/tsdhn/tests/parity/fault_plane/fixtures/alaska_1964.npz differ diff --git a/packages/tsdhn/tests/parity/fault_plane/spec.py b/packages/tsdhn/tests/parity/fault_plane/spec.py new file mode 100644 index 0000000..52891db --- /dev/null +++ b/packages/tsdhn/tests/parity/fault_plane/spec.py @@ -0,0 +1,52 @@ +"""Compare rectangle geometry with the captured MATLAB trace. + +The binary parity spec covers the grid lookup conversion and full output +files. This spec compares only the geometry intermediates shared with MATLAB. +""" + +from pathlib import Path +from typing import Any + +from tsdhn.calculator import calculate_rectangle_parameters +from tsdhn_parity import ( + Case, + FrozenFixtureAdapter, + OnCheckpoint, + PythonCallableAdapter, + UnitSpec, +) + +FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures" + +# These parameters match calculate_rectangle_parameters's signature. +CASES = [ + Case( + id="alaska_1964", + params={ + "L": 575.439937, + "W": 144.543977, + "lon0": -156.0, + "lat0": 56.0, + "azimuth": 247.0, + "dip": 18.0, + }, + ), +] + +GEOMETRY_CHECKPOINTS = ("L1", "W1", "beta", "alfa", "h1", "a1", "b1") + + +def _geometry_only(on_checkpoint: OnCheckpoint, **kwargs: Any) -> None: + def filtered(name: str, value: Any) -> None: + if name in GEOMETRY_CHECKPOINTS: + on_checkpoint(name, value) + + calculate_rectangle_parameters(on_checkpoint=filtered, **kwargs) + + +SPEC = UnitSpec( + id="fault_plane", + cases=CASES, + legacy=FrozenFixtureAdapter(fixtures_dir=FIXTURES_DIR), + python=PythonCallableAdapter(fn=_geometry_only, result_checkpoint=None), +) diff --git a/packages/tsdhn/tests/parity/fault_plane/spec_binary.py b/packages/tsdhn/tests/parity/fault_plane/spec_binary.py new file mode 100644 index 0000000..a9a6702 --- /dev/null +++ b/packages/tsdhn/tests/parity/fault_plane/spec_binary.py @@ -0,0 +1,123 @@ +"""Compare fault-plane calculations with the compiled Fortran output. + +The executable is the reference for dislocation and for the full +`pfalla.inp`, `xyo.dat`, and `meca.dat` step outputs. The MATLAB spec covers +only the geometry intermediates that share the same convention. +""" + +import tempfile +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import numpy as np + +from tsdhn.calculator import TsunamiCalculator +from tsdhn.domain import EarthquakeInput +from tsdhn.fault_plane import run_fault_plane +from tsdhn.utils.file_utils import prepare_simulation_workspace +from tsdhn_parity import ( + Case, + FortranBinaryAdapter, + OnCheckpoint, + PythonCallableAdapter, + UnitSpec, +) + +from ..conftest import MODEL_DIR + +CASES = [ + Case( + id="alaska_1964", + params={ + "Mw": 9.0, + "h": 12.0, + "lat0": 56.0, + "lon0": -156.0, + "hhmm": "0000", + "dia": "23", + }, + ), +] + + +def _write_hypo_dat(working_dir: Path, params: Mapping[str, Any]) -> None: + (working_dir / "hypo.dat").write_text( + "\n".join( + [ + str(params["hhmm"]), + f"{params['lon0']:.2f}", + f"{params['lat0']:.2f}", + f"{params['h']:.0f}", + f"{params['Mw']:.1f}", + ] + ) + ) + + +def _prepare(case: Case, working_dir: Path) -> None: + prepare_simulation_workspace(MODEL_DIR, working_dir) + _write_hypo_dat(working_dir, case.params) + + +def _read_slip(path: Path) -> np.ndarray: + # List-directed Fortran output may wrap. Read whitespace-separated tokens. + tokens = path.read_text().split() + return np.asarray(float(tokens[2])) + + +def _run(on_checkpoint: OnCheckpoint, **kwargs: Any) -> None: + calculator = TsunamiCalculator(MODEL_DIR) + response = calculator.calculate_earthquake_parameters(EarthquakeInput(**kwargs)) + on_checkpoint("pfalla.inp", np.asarray(response.dislocation)) + + +SPEC = UnitSpec( + id="fault_plane_binary", + cases=CASES, + legacy=FortranBinaryAdapter( + executable="fault_plane", + checkpoints=("pfalla.inp",), + prepare=_prepare, + read_checkpoint=_read_slip, + ), + python=PythonCallableAdapter(fn=_run, result_checkpoint=None), +) + + +_STEP_CHECKPOINTS = ("pfalla.inp", "xyo.dat", "meca.dat") + + +def _read_fault_plane_checkpoint(path: Path) -> np.ndarray: + # List-directed Fortran output may wrap. Read whitespace-separated tokens. + tokens = path.read_text().split() + if path.name == "pfalla.inp": + return np.asarray([float(t) for t in tokens[:9]]) + if path.name == "xyo.dat": + return np.asarray([float(t) for t in tokens[:4]]) + if path.name == "meca.dat": + return np.asarray([float(t) for t in tokens[:7]]) + raise ValueError(f"no checkpoint reader for {path.name}") + + +def _run_step(on_checkpoint: OnCheckpoint, **kwargs: Any) -> None: + with tempfile.TemporaryDirectory(prefix="tsdhn-parity-") as raw_dir: + working_dir = Path(raw_dir) + prepare_simulation_workspace(MODEL_DIR, working_dir) + _write_hypo_dat(working_dir, kwargs) + run_fault_plane(working_dir) + for name in _STEP_CHECKPOINTS: + on_checkpoint(name, _read_fault_plane_checkpoint(working_dir / name)) + + +STEP_SPEC = UnitSpec( + id="fault_plane_step", + cases=CASES, + legacy=FortranBinaryAdapter( + executable="fault_plane", + checkpoints=_STEP_CHECKPOINTS, + prepare=_prepare, + read_checkpoint=_read_fault_plane_checkpoint, + ), + python=PythonCallableAdapter(fn=_run_step, result_checkpoint=None), +) diff --git a/packages/tsdhn/tests/parity/fault_plane/test_fault_plane_binary_parity.py b/packages/tsdhn/tests/parity/fault_plane/test_fault_plane_binary_parity.py new file mode 100644 index 0000000..ef0363f --- /dev/null +++ b/packages/tsdhn/tests/parity/fault_plane/test_fault_plane_binary_parity.py @@ -0,0 +1,10 @@ +import pytest + +from tsdhn_parity import Case, assert_parity, parity_cases + +from .spec_binary import SPEC + + +@pytest.mark.parametrize("case", parity_cases(SPEC)) +def test_fault_plane_dislocation_matches_binary(case: Case) -> None: + assert_parity(SPEC, case) diff --git a/packages/tsdhn/tests/parity/fault_plane/test_fault_plane_parity.py b/packages/tsdhn/tests/parity/fault_plane/test_fault_plane_parity.py new file mode 100644 index 0000000..6db7700 --- /dev/null +++ b/packages/tsdhn/tests/parity/fault_plane/test_fault_plane_parity.py @@ -0,0 +1,10 @@ +import pytest + +from tsdhn_parity import Case, assert_parity, parity_cases + +from .spec import SPEC + + +@pytest.mark.parametrize("case", parity_cases(SPEC)) +def test_fault_plane_matches_matlab(case: Case) -> None: + assert_parity(SPEC, case) diff --git a/packages/tsdhn/tests/parity/fault_plane/test_fault_plane_step_parity.py b/packages/tsdhn/tests/parity/fault_plane/test_fault_plane_step_parity.py new file mode 100644 index 0000000..6cabb21 --- /dev/null +++ b/packages/tsdhn/tests/parity/fault_plane/test_fault_plane_step_parity.py @@ -0,0 +1,10 @@ +import pytest + +from tsdhn_parity import Case, assert_parity, parity_cases + +from .spec_binary import STEP_SPEC + + +@pytest.mark.parametrize("case", parity_cases(STEP_SPEC)) +def test_fault_plane_step_matches_binary(case: Case) -> None: + assert_parity(STEP_SPEC, case) diff --git a/packages/tsdhn/tests/parity/tsunami/__init__.py b/packages/tsdhn/tests/parity/tsunami/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/tsdhn/tests/parity/tsunami/spec.py b/packages/tsdhn/tests/parity/tsunami/spec.py new file mode 100644 index 0000000..bb1cf23 --- /dev/null +++ b/packages/tsdhn/tests/parity/tsunami/spec.py @@ -0,0 +1,135 @@ +"""Compare the Python shallow-water solver with the compiled Fortran output. + +This test runs the full integration and is marked for explicit parity runs. +The output files use fixed-width fields, so the readers below use their +declared column widths. Tolerances account for output quantization and +float32 rounding during the integration. +""" + +import shutil +import tempfile +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import numpy as np + +from tsdhn.deform import run_deform +from tsdhn.fault_plane import run_fault_plane +from tsdhn.tsunami import run_tsunami +from tsdhn.utils.file_utils import prepare_simulation_workspace +from tsdhn_parity import ( + Case, + FortranBinaryAdapter, + OnCheckpoint, + PythonCallableAdapter, + Tolerance, + UnitSpec, + read_fixed_width_grid, +) + +from ..conftest import MODEL_DIR + +CASES = [ + Case( + id="alaska_1964", + params={ + "Mw": 9.0, + "h": 12.0, + "lat0": 56.0, + "lon0": -156.0, + "hhmm": "0000", + "dia": "23", + }, + ), +] + +# The output files use seven-character and eight-character numeric fields. +_READERS: dict[str, Callable[[Path], np.ndarray]] = { + "green.dat": read_fixed_width_grid(7), + "zmax_a.grd": read_fixed_width_grid(8), +} + +_CHECKPOINTS = ("zfolder/green.dat", "zfolder/zmax_a.grd") + +# Build each scenario's initial condition once and reuse it for both adapters. +_chained: dict[str, Path] = {} + + +def _read_checkpoint(path: Path) -> np.ndarray: + return _READERS[path.name](path) + + +def _write_hypo_dat(working_dir: Path, params: dict[str, Any]) -> None: + (working_dir / "hypo.dat").write_text( + "\n".join( + [ + str(params["hhmm"]), + f"{params['lon0']:.2f}", + f"{params['lat0']:.2f}", + f"{params['h']:.0f}", + f"{params['Mw']:.1f}", + ] + ) + ) + + +def _params_key(params: dict[str, Any]) -> str: + return repr(sorted(params.items())) + + +def _chained_inputs(params: dict[str, Any]) -> Path: + """Return the shared fault-plane and deformation inputs for a case.""" + key = _params_key(params) + if key not in _chained: + workspace = Path(tempfile.mkdtemp(prefix="tsdhn-parity-tsunami-chain-")) + prepare_simulation_workspace(MODEL_DIR, workspace) + _write_hypo_dat(workspace, params) + run_fault_plane(workspace) + run_deform(workspace) + _chained[key] = workspace + return _chained[key] + + +def _seed_tsunami_workspace(params: dict[str, Any], working_dir: Path) -> None: + chained = _chained_inputs(params) + (working_dir / "bathy").mkdir(parents=True, exist_ok=True) + (working_dir / "zfolder").mkdir(exist_ok=True) + shutil.copy2( + MODEL_DIR / "bathy" / "grid_a.grd", working_dir / "bathy" / "grid_a.grd" + ) + shutil.copy2(MODEL_DIR / "tidal.dat", working_dir / "tidal.dat") + shutil.copy2(chained / "deform_a.grd", working_dir / "deform_a.grd") + shutil.copy2(chained / "xyo.dat", working_dir / "xyo.dat") + + +def _prepare(case: Case, working_dir: Path) -> None: + _seed_tsunami_workspace(dict(case.params), working_dir) + + +def _run(on_checkpoint: OnCheckpoint, **kwargs: Any) -> None: + with tempfile.TemporaryDirectory(prefix="tsdhn-parity-tsunami-") as raw_dir: + working_dir = Path(raw_dir) + _seed_tsunami_workspace(kwargs, working_dir) + run_tsunami(working_dir) + for name in _CHECKPOINTS: + on_checkpoint(name, _read_checkpoint(working_dir / name)) + + +LEGACY = FortranBinaryAdapter( + executable="tsunami", + checkpoints=_CHECKPOINTS, + prepare=_prepare, + read_checkpoint=_read_checkpoint, +) + +SPEC = UnitSpec( + id="tsunami", + cases=CASES, + legacy=LEGACY, + python=PythonCallableAdapter(fn=_run, result_checkpoint=None), + tolerances={ + "zfolder/green.dat": Tolerance(atol=0.002), + "zfolder/zmax_a.grd": Tolerance(atol=0.02), + }, +) diff --git a/packages/tsdhn/tests/parity/tsunami/test_tsunami_parity.py b/packages/tsdhn/tests/parity/tsunami/test_tsunami_parity.py new file mode 100644 index 0000000..c5385d6 --- /dev/null +++ b/packages/tsdhn/tests/parity/tsunami/test_tsunami_parity.py @@ -0,0 +1,10 @@ +import pytest + +from tsdhn_parity import Case, assert_parity, parity_cases + +from .spec import SPEC + + +@pytest.mark.parametrize("case", parity_cases(SPEC)) +def test_tsunami_matches_binary(case: Case) -> None: + assert_parity(SPEC, case) diff --git a/packages/tsdhn/tests/test_assets.py b/packages/tsdhn/tests/test_assets.py new file mode 100644 index 0000000..d23d277 --- /dev/null +++ b/packages/tsdhn/tests/test_assets.py @@ -0,0 +1,255 @@ +import hashlib +import io +import json +import shutil +import tarfile +from pathlib import Path + +import pytest + +from tsdhn.assets import ( + ModelStore, + _default_data_root, + _download, + _extract_model_archive, +) + +# Keep the archive fixture independent from the validator's constants. +MODEL_DIRS = ("bathy", "ttt_mundo") +MODEL_FILES = ( + "pacifico.mat", + "maper1.mat", + "mecfoc.dat", + "puertos.txt", + "tidal.dat", + "bathy/grid_a.grd", + "bathy/xa.dat", + "bathy/ya.dat", + "ttt_mundo/cortado.i2", +) + + +def _build_model_dir(root: Path) -> Path: + model_dir = root / "model" + for dirname in MODEL_DIRS: + (model_dir / dirname).mkdir(parents=True) + for relative_name in MODEL_FILES: + path = model_dir / relative_name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"{relative_name}\n", encoding="utf-8") + return model_dir + + +def _build_archive( + tmp_path: Path, *, nested_root: str | None = "tsdhn-model-1.0.0" +) -> Path: + model_dir = _build_model_dir(tmp_path / "src") + archive = tmp_path / "archive.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + arcname = nested_root or "." + tar.add(model_dir, arcname=arcname if nested_root else ".") + return archive + + +class _FakeResponse(io.BytesIO): + def __enter__(self) -> _FakeResponse: + return self + + def __exit__(self, *exc_info: object) -> None: + self.close() + + +def test_download_rejects_non_https_urls(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="HTTPS"): + _download("http://example.com/model.tar.gz", tmp_path / "out.tar.gz") + + +def test_download_writes_the_response_body( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + payload = b"archive-bytes" + + def fake_urlopen(url: str, timeout: int) -> _FakeResponse: + assert url == "https://example.com/model.tar.gz" + return _FakeResponse(payload) + + monkeypatch.setattr("tsdhn.assets.urllib.request.urlopen", fake_urlopen) + destination = tmp_path / "out.tar.gz" + + _download("https://example.com/model.tar.gz", destination) + + assert destination.read_bytes() == payload + + +def test_extract_model_archive_finds_a_single_nested_root(tmp_path: Path) -> None: + archive = _build_archive(tmp_path, nested_root="tsdhn-model-1.0.0") + target = tmp_path / "installed" + + _extract_model_archive(archive, target) + + for dirname in MODEL_DIRS: + assert (target / dirname).is_dir() + + +def test_extract_model_archive_uses_manifest_when_present(tmp_path: Path) -> None: + model_dir = _build_model_dir(tmp_path / "src") + manifest = {"model_root": "payload/model-data"} + nested = tmp_path / "src" / "manifest_root" / "payload" / "model-data" + nested.parent.mkdir(parents=True) + shutil.copytree(model_dir, nested) + (tmp_path / "src" / "manifest_root" / "manifest.json").write_text( + json.dumps(manifest), encoding="utf-8" + ) + + archive = tmp_path / "archive.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + tar.add(tmp_path / "src" / "manifest_root", arcname=".") + + target = tmp_path / "installed" + _extract_model_archive(archive, target) + + for dirname in MODEL_DIRS: + assert (target / dirname).is_dir() + + +def test_extract_model_archive_raises_when_no_valid_root_exists(tmp_path: Path) -> None: + archive = tmp_path / "archive.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + empty = tmp_path / "empty.txt" + empty.write_text("nothing here") + tar.add(empty, arcname="empty.txt") + + with pytest.raises(RuntimeError, match="valid TSDHN model dataset"): + _extract_model_archive(archive, tmp_path / "installed") + + +def test_extract_model_archive_cannot_write_outside_the_staging_directory( + tmp_path: Path, +) -> None: + archive = tmp_path / "archive.tar.gz" + payload = b"should not escape" + with tarfile.open(archive, "w:gz") as tar: + member = tarfile.TarInfo("../outside.txt") + member.size = len(payload) + tar.addfile(member, io.BytesIO(payload)) + + with pytest.raises(tarfile.OutsideDestinationError): + _extract_model_archive(archive, tmp_path / "installed") + + assert not (tmp_path / "outside.txt").exists() + + +def test_model_store_install_downloads_verifies_and_extracts( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + archive = _build_archive(tmp_path) + archive_bytes = archive.read_bytes() + digest = hashlib.sha256(archive_bytes).hexdigest() + + def fake_download(url: str, destination: Path) -> None: + destination.write_bytes(archive_bytes) + + monkeypatch.setattr("tsdhn.assets._download", fake_download) + store = ModelStore(root=tmp_path / "data") + + dataset = store.install("1.0.0", sha256=digest) + + assert dataset.version == "1.0.0" + assert dataset.managed + for dirname in MODEL_DIRS: + assert (dataset.path / dirname).is_dir() + + +def test_model_store_install_rejects_a_checksum_mismatch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + archive = _build_archive(tmp_path) + archive_bytes = archive.read_bytes() + + monkeypatch.setattr( + "tsdhn.assets._download", + lambda url, destination: destination.write_bytes(archive_bytes), + ) + store = ModelStore(root=tmp_path / "data") + + with pytest.raises(RuntimeError, match="checksum mismatch"): + store.install("1.0.0", sha256="0" * 64) + + +def test_model_store_install_is_idempotent_without_force( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + store = ModelStore(root=tmp_path / "data") + installed = _build_model_dir(tmp_path / "data" / "models") + target = store.dataset_dir("1.0.0") + installed.rename(target) + + def fail_download(url: str, destination: Path) -> None: + raise AssertionError("should not download when already installed") + + monkeypatch.setattr("tsdhn.assets._download", fail_download) + dataset = store.install("1.0.0", url="https://example.com/never-used.tar.gz") + + assert dataset.path == target + + +def test_model_store_status_reports_missing_entries(tmp_path: Path) -> None: + store = ModelStore(root=tmp_path / "data") + + status = store.status("1.0.0") + + assert status["installed"] is False + assert status["missing"] == [ + *(f"{dirname}/" for dirname in MODEL_DIRS), + *MODEL_FILES, + ] + + +def test_model_store_status_reports_a_valid_installed_dataset(tmp_path: Path) -> None: + store = ModelStore(root=tmp_path / "data") + source = _build_model_dir(tmp_path / "source") + target = store.dataset_dir("1.0.0") + target.parent.mkdir(parents=True) + source.rename(target) + + assert store.status("1.0.0") == { + "version": "1.0.0", + "path": str(target), + "installed": True, + "missing": [], + } + + +def test_model_store_resolve_installed_returns_none_when_absent(tmp_path: Path) -> None: + store = ModelStore(root=tmp_path / "data") + + assert store.resolve_installed("1.0.0") is None + + +def test_model_store_release_asset_url_matches_the_github_release_convention() -> None: + store = ModelStore(root=Path("unused"), repository="acme/tsdhn") + + url = store.release_asset_url("1.2.3") + + assert url == ( + "https://github.com/acme/tsdhn/releases/download/v1.2.3/" + "tsdhn-model-v1.2.3.tar.gz" + ) + + +def test_default_data_root_prefers_tsdhn_data_home( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("TSDHN_DATA_HOME", str(tmp_path / "explicit")) + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "xdg")) + + assert _default_data_root() == tmp_path / "explicit" + + +def test_default_data_root_falls_back_to_xdg_data_home( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.delenv("TSDHN_DATA_HOME", raising=False) + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "xdg")) + + assert _default_data_root() == tmp_path / "xdg" / "tsdhn" diff --git a/packages/tsdhn/tests/test_cli.py b/packages/tsdhn/tests/test_cli.py new file mode 100644 index 0000000..ef6b4ee --- /dev/null +++ b/packages/tsdhn/tests/test_cli.py @@ -0,0 +1,84 @@ +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +import tsdhn.cli.main as cli_module +from tsdhn.domain import CalculationResponse, TsunamiTravelResponse +from tsdhn.runtime import RuntimeContext + +RUNNER = CliRunner() + +CALCULATION = CalculationResponse( + length=10.0, + width=20.0, + dislocation=1.5, + seismic_moment=2.0e18, + tsunami_warning="warning", + distance_to_coast=30.0, + azimuth=40.0, + dip=50.0, + epicenter_location="mar", + rectangle_parameters={}, + rectangle_corners=[], +) + +TRAVEL_TIMES = TsunamiTravelResponse( + arrival_times={"Callao": "12:36 05Aug"}, + distances={"Callao": 19.7}, + epicenter_info={}, +) + + +class _Calculator: + def __init__(self, _model_dir: Path) -> None: + pass + + def calculate_earthquake_parameters( + self, _data: object, _output_dir: Path + ) -> CalculationResponse: + return CALCULATION + + def calculate_tsunami_travel_times(self, _data: object) -> TsunamiTravelResponse: + return TRAVEL_TIMES + + +def test_calc_command_prints_source_and_arrival_results( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + runtime = RuntimeContext(model_dir=tmp_path, model_version="test", capabilities={}) + monkeypatch.setattr( + RuntimeContext, + "resolve", + classmethod(lambda cls, **kwargs: runtime), + ) + monkeypatch.setattr(cli_module, "TsunamiCalculator", _Calculator) + + result = RUNNER.invoke(cli_module.app, ["calc", "--model-dir", str(tmp_path)]) + + assert result.exit_code == 0, result.output + assert "Source parameters" in result.output + assert "Rupture length (km)" in result.output + assert "Callao" in result.output + assert "12:36 05Aug" in result.output + + +def test_run_command_reports_a_failed_simulation_and_keeps_the_run_path( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + work_dir = tmp_path / "failed-run" + + def fail(*_args: object, **_kwargs: object) -> object: + raise RuntimeError("model step failed") + + monkeypatch.setattr(cli_module, "run_simulation", fail) + + result = RUNNER.invoke( + cli_module.app, + ["run", "--work-dir", str(work_dir)], + ) + + assert result.exit_code == 1 + assert "Simulation failed" in result.output + assert "Inspect run directory" in result.output + assert str(work_dir) in result.output diff --git a/packages/tsdhn/tests/test_deform.py b/packages/tsdhn/tests/test_deform.py new file mode 100644 index 0000000..6216746 --- /dev/null +++ b/packages/tsdhn/tests/test_deform.py @@ -0,0 +1,149 @@ +from pathlib import Path + +import numpy as np +import pytest + +import tsdhn.deform as deform_module +from tsdhn.deform import ( + _udip_gz, + _ustrike_fz, + clip_anomalous_values, + compute_deform_grid, + write_deform_grid, +) + + +def test_clip_anomalous_values_uses_greater_equal_threshold() -> None: + # Values at or beyond 20 m are outliers, not values subject to a small + # value floor. Keep the examples away from the float32 boundary. + grid = np.array([19.9, 20.1, -19.9, -20.1, 0.5], dtype=np.float32) + result = clip_anomalous_values(grid) + np.testing.assert_allclose(result, [19.9, 0.0, -19.9, 0.0, 0.5], atol=1e-4) + + +def test_ustrike_fz_and_udip_gz_finite_away_from_fault() -> None: + q = np.array([1234.5], dtype=np.float32) + cs = np.float32(0.9) + sn = np.float32(0.3) + xi = np.array([500.0], dtype=np.float32) + et = np.array([300.0], dtype=np.float32) + eps = np.float32(1.0e-8) + rmu = np.float32(0.5) + + fz = _ustrike_fz(eps, rmu, q, cs, sn, xi, et) + gz = _udip_gz(eps, rmu, q, cs, sn, xi, et) + assert np.all(np.isfinite(fz)) + assert np.all(np.isfinite(gz)) + + +def test_udip_gz_handles_q_near_zero() -> None: + q = np.array([0.0], dtype=np.float32) + cs = np.float32(0.9) + sn = np.float32(0.3) + xi = np.array([100.0], dtype=np.float32) + et = np.array([200.0], dtype=np.float32) + eps = np.float32(1.0e-8) + rmu = np.float32(0.5) + + gz = _udip_gz(eps, rmu, q, cs, sn, xi, et) + assert np.all(np.isfinite(gz)) + + +def test_ustrike_fz_handles_ret_singular() -> None: + # Exercise RET=R+ET=0, including the RDH branch. + q = np.array([0.0], dtype=np.float32) + cs = np.float32(0.9) + sn = np.float32(0.3) + xi = np.array([0.0], dtype=np.float32) + et = np.array([-5.0], dtype=np.float32) + eps = np.float32(1.0e-8) + rmu = np.float32(0.5) + + fz = _ustrike_fz(eps, rmu, q, cs, sn, xi, et) + assert np.all(np.isfinite(fz)) + + +def _read_fixed_width_9(path: Path) -> np.ndarray: + # Independent reimplementation (not tsdhn_parity's reader) so this + # round-trip test doesn't share a bug with the code it's checking. + return np.array( + [ + [float(line[i : i + 9]) for i in range(0, len(line), 9)] + for line in path.read_text().splitlines() + ] + ) + + +def test_write_deform_grid_round_trips_fixed_width_9(tmp_path: Path) -> None: + grid = np.array([[1.5, -0.001, 123.457], [0.02, -12.3, 0.0]], dtype=np.float32) + path = tmp_path / "deform_a.grd" + write_deform_grid(path, grid) + read_back = _read_fixed_width_9(path) + assert read_back.shape == grid.shape + np.testing.assert_allclose(read_back, grid, atol=5e-4) + + +def test_compute_deform_grid_alaska_1964_shape() -> None: + grid = compute_deform_grid( + I0=1180, + J0=1988, + D0=11.965752308065987, + L0=575439.9373371573, + W0=144543.97707459278, + TH=247.0, + DL=18.0, + RD=90.0, + HH=5000.0, + IDS=1032, + IDE=1249, + JDS=1872, + JDE=2056, + ) + expected_rows = 1249 - 1032 + 1 + expected_cols = 2056 - 1872 + 1 + assert grid.shape == (expected_rows, expected_cols) + assert np.all(np.isfinite(grid)) + assert grid.dtype == np.float32 + + +def test_compute_deform_grid_accepts_a_zero_strike() -> None: + grid = compute_deform_grid( + I0=1180, + J0=1988, + D0=11.965752308065987, + L0=575439.9373371573, + W0=144543.97707459278, + TH=0.0, + DL=18.0, + RD=90.0, + HH=5000.0, + IDS=1032, + IDE=1034, + JDS=1872, + JDE=1874, + ) + + assert grid.shape == (3, 3) + assert np.all(np.isfinite(grid)) + + +def test_run_deform_reads_workspace_inputs_and_clips_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / "pfalla.inp").write_text( + "1180 1987 11.96 575439.9 144543.9 247 8 90 1941.7\n", + encoding="utf-8", + ) + (tmp_path / "xyo.dat").write_text("1 2 1 2\n", encoding="utf-8") + monkeypatch.setattr( + deform_module, + "compute_deform_grid", + lambda **kwargs: np.array([[19.5, 20.0], [-20.0, -19.5]], dtype=np.float32), + ) + + deform_module.run_deform(tmp_path) + + np.testing.assert_allclose( + np.loadtxt(tmp_path / "deform_a.grd"), + [[19.5, 0.0], [0.0, -19.5]], + ) diff --git a/packages/tsdhn/tests/test_engine.py b/packages/tsdhn/tests/test_engine.py new file mode 100644 index 0000000..bfed0e8 --- /dev/null +++ b/packages/tsdhn/tests/test_engine.py @@ -0,0 +1,198 @@ +"""Pipeline sequencing and output collection behavior.""" + +from pathlib import Path +from typing import Any + +import pytest + +from tsdhn.domain import CalculationResponse, EarthquakeInput, TsunamiTravelResponse +from tsdhn.engine import ( + SimulationEngine, + SimulationRequest, + step_directory, + write_simulation_outputs, +) +from tsdhn.pipeline.types import ProcessingStep +from tsdhn.runtime import CapabilityStatus, RuntimeContext + +INPUT = EarthquakeInput(Mw=8.0, h=10.0, lat0=-10.0, lon0=280.0, hhmm="0000", dia="00") + +CALCULATION = CalculationResponse( + length=1.0, + width=1.0, + dislocation=1.0, + seismic_moment=1.0, + tsunami_warning="none", + distance_to_coast=1.0, + azimuth=1.0, + dip=1.0, + epicenter_location="0.00/0.00", + rectangle_parameters={}, + rectangle_corners=[], +) + +TRAVEL_TIMES = TsunamiTravelResponse( + arrival_times={"PORT": "01:00"}, + distances={"PORT": 100.0}, + epicenter_info={"lat": "0.0"}, +) + + +class _FakeCalculator: + def __init__(self, model_dir: Path) -> None: + self.model_dir = model_dir + + def calculate_earthquake_parameters( + self, data: EarthquakeInput, work_dir: Path + ) -> CalculationResponse: + return CALCULATION + + def calculate_tsunami_travel_times( + self, data: EarthquakeInput + ) -> TsunamiTravelResponse: + return TRAVEL_TIMES + + +def _write_marker(name: str) -> ProcessingStep: + def runner(working_dir: Path) -> None: + (working_dir / name).write_text("ok\n") + + return ProcessingStep(name=name, outputs=(name,), runner=runner) + + +@pytest.fixture +def fake_engine( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> tuple[SimulationEngine, Path]: + model_dir = tmp_path / "model" + model_dir.mkdir() + + def fake_resolve( + cls: type[RuntimeContext], + model_dir: Path | None = None, + *, + model_version: str | None = None, + ) -> RuntimeContext: + return RuntimeContext( + model_dir=model_dir or (tmp_path / "model"), + model_version="test", + capabilities={"gmt": CapabilityStatus(name="gmt", available=True)}, + ) + + monkeypatch.setattr(RuntimeContext, "resolve", classmethod(fake_resolve)) + monkeypatch.setattr("tsdhn.engine.TsunamiCalculator", _FakeCalculator) + monkeypatch.setattr("tsdhn.engine.ensure_executables", lambda names: None) + monkeypatch.setattr( + "tsdhn.engine.prepare_simulation_workspace", lambda *a, **k: None + ) + + steps = (_write_marker("a.txt"), _write_marker("b.txt")) + return SimulationEngine(steps=steps), tmp_path + + +def test_run_executes_steps_in_order_and_reports_progress( + fake_engine: tuple[SimulationEngine, Path], +) -> None: + engine, tmp_path = fake_engine + work_dir = tmp_path / "work" + work_dir.mkdir() + events: list[str] = [] + + def on_progress(message: str, details: dict[str, Any]) -> None: + events.append(message) + + request = SimulationRequest(input=INPUT, work_dir=work_dir) + result = engine.run(request, on_progress=on_progress) + + assert (work_dir / "a.txt").is_file() + assert (work_dir / "b.txt").is_file() + assert result.calculation == CALCULATION + assert result.travel_times == TRAVEL_TIMES + assert events == [ + "Running earthquake calculations", + "Earthquake calculations complete", + "Calculating tsunami travel times", + "Tsunami calculations complete", + "Processing a.txt", + "Processing b.txt", + "Simulation completed successfully", + ] + + +def test_run_skips_completed_steps_on_resume( + fake_engine: tuple[SimulationEngine, Path], +) -> None: + engine, tmp_path = fake_engine + work_dir = tmp_path / "work" + work_dir.mkdir() + + # Establish completed markers before the resumed run. + engine.run(SimulationRequest(input=INPUT, work_dir=work_dir)) + + events: list[str] = [] + request = SimulationRequest(input=INPUT, work_dir=work_dir, resume=True) + engine.run(request, on_progress=lambda message, details: events.append(message)) + + assert events == [ + "Running earthquake calculations", + "Earthquake calculations complete", + "Calculating tsunami travel times", + "Tsunami calculations complete", + "Skipping completed step a.txt", + "Skipping completed step b.txt", + "Simulation completed successfully", + ] + + +def test_step_directory_uses_working_dir_override(tmp_path: Path) -> None: + plain = ProcessingStep(name="plain", outputs=(), runner=lambda wd: None) + nested = ProcessingStep( + name="nested", outputs=(), runner=lambda wd: None, working_dir="sub" + ) + + assert step_directory(tmp_path, plain) == tmp_path + assert step_directory(tmp_path, nested) == tmp_path / "sub" + + +def test_write_simulation_outputs_always_includes_json_and_csv(tmp_path: Path) -> None: + runtime = RuntimeContext(model_dir=tmp_path, model_version="test", capabilities={}) + request = SimulationRequest(input=INPUT, work_dir=tmp_path) + + outputs = write_simulation_outputs( + request=request, + calculation=CALCULATION, + travel_times=TRAVEL_TIMES, + runtime=runtime, + ) + + names = {output.name for output in outputs.files} + assert names == { + "input", + "runtime", + "calculation", + "travel_times_json", + "travel_times_csv", + } + assert (tmp_path / "travel_times.csv").read_text().splitlines()[ + 1 + ] == "PORT,01:00,100.0" + + +def test_write_simulation_outputs_includes_render_outputs_only_when_present( + tmp_path: Path, +) -> None: + runtime = RuntimeContext(model_dir=tmp_path, model_version="test", capabilities={}) + request = SimulationRequest(input=INPUT, work_dir=tmp_path) + (tmp_path / "maxola.pdf").write_bytes(b"%PDF-1.4") + + outputs = write_simulation_outputs( + request=request, + calculation=CALCULATION, + travel_times=TRAVEL_TIMES, + runtime=runtime, + ) + + by_name = outputs.by_name() + assert "max_height_map" in by_name + assert "arrival_time_map" not in by_name + assert "mareogram" not in by_name diff --git a/packages/tsdhn/tests/test_external.py b/packages/tsdhn/tests/test_external.py new file mode 100644 index 0000000..6e90d06 --- /dev/null +++ b/packages/tsdhn/tests/test_external.py @@ -0,0 +1,31 @@ +import shutil +from pathlib import Path + +import pytest + +from tsdhn.external import ensure_executables, resolve + + +def test_resolve_returns_the_path_when_found(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(shutil, "which", lambda name: f"/usr/bin/{name}") + resolve.cache_clear() + + assert resolve("gmt") == Path("/usr/bin/gmt") + + +def test_resolve_raises_when_not_on_path(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(shutil, "which", lambda name: None) + resolve.cache_clear() + + with pytest.raises(RuntimeError, match="not found on PATH"): + resolve("missing_tool") + + +def test_ensure_executables_raises_on_the_first_missing_tool( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(shutil, "which", lambda name: None) + resolve.cache_clear() + + with pytest.raises(RuntimeError, match="missing_tool"): + ensure_executables(("missing_tool",)) diff --git a/packages/tsdhn/tests/test_fault_plane.py b/packages/tsdhn/tests/test_fault_plane.py new file mode 100644 index 0000000..4c710b3 --- /dev/null +++ b/packages/tsdhn/tests/test_fault_plane.py @@ -0,0 +1,136 @@ +from pathlib import Path + +import numpy as np +import pytest + +from tsdhn.fault_plane import ( + _grid_window, + _nearest_mechanism, + _recompute_depth, + _to_0_360, + _write_meca_dat, + _write_pfalla_inp, + _write_xyo_dat, + run_fault_plane, +) +from tsdhn.utils.file_utils import prepare_simulation_workspace + +MODEL_DIR = Path(__file__).resolve().parents[3] / "model" + + +def test_to_0_360_shifts_only_negative_longitudes() -> None: + assert _to_0_360(-156.0) == pytest.approx(204.0) + assert _to_0_360(156.0) == pytest.approx(156.0) + assert _to_0_360(0.0) == pytest.approx(0.0) + + +def test_grid_window_truncates_target_before_snapping() -> None: + # The legacy integer window truncates the target before snapping it to a + # grid cell. A fine grid makes truncation distinguishable from rounding. + xa = np.array([9.0, 10.0, 11.0, 12.0]) + ya = np.array([9.0, 10.0, 11.0, 12.0]) + ids, ide, jds, jde = _grid_window(xa, ya, xep=10.9, yep=10.9, l_km=0.0, mw=9.0) + assert (ids, ide, jds, jde) == (2, 2, 2, 2) + + +def test_nearest_mechanism_matches_real_mecfoc_alaska_1964() -> None: + mecfoc = np.loadtxt(MODEL_DIR / "mecfoc.dat") + az, dip = _nearest_mechanism(mecfoc, xep=204.0, yep=56.0) + assert az == pytest.approx(247.0) + assert dip == pytest.approx(8.0) + + +def test_recompute_depth_clamps_negative_to_5000() -> None: + h_m = _recompute_depth( + lon0=-156.0, lat0=56.0, xo=-153.36, yo=56.42, zep_km=0.1, az=247.0, dip=8.0 + ) + assert h_m == pytest.approx(5000.0) + + +def test_write_pfalla_inp_is_nine_whitespace_tokens(tmp_path: Path) -> None: + path = tmp_path / "pfalla.inp" + _write_pfalla_inp( + path, 1180, 1987, 11.96, 575439.9, 144543.9, 247.0, 8.0, 90.0, 1941.7 + ) + tokens = path.read_text().split() + assert len(tokens) == 9 + assert int(tokens[0]) == 1180 + assert int(tokens[1]) == 1987 + + +def test_write_xyo_dat_includes_trailing_ia_ja_padding(tmp_path: Path) -> None: + path = tmp_path / "xyo.dat" + _write_xyo_dat(path, 1021, 1246, 1861, 2056) + tokens = path.read_text().split() + assert tokens == ["1021", "1246", "1861", "2056", "2461", "2056"] + + +def test_write_meca_dat_matches_real_captured_format(tmp_path: Path) -> None: + path = tmp_path / "meca.dat" + _write_meca_dat(path, 204.0, 56.0, 12.0, 247.0, 8.0, 9.0, "0000") + real = (MODEL_DIR / "meca.dat").read_text().strip() + assert path.read_text().strip() == real + + +def test_run_fault_plane_matches_real_captured_alaska_1964(tmp_path: Path) -> None: + prepare_simulation_workspace(MODEL_DIR, tmp_path) + (tmp_path / "hypo.dat").write_text( + "\n".join(["0000", "-156.00", "56.00", "12", "9.0"]) + ) + + run_fault_plane(tmp_path) + + ( + real_i0, + real_j0, + real_slip, + real_l, + real_w, + real_az, + real_dip, + real_rake, + real_h, + ) = (float(t) for t in (MODEL_DIR / "pfalla.inp").read_text().split()) + ( + mine_i0, + mine_j0, + mine_slip, + mine_l, + mine_w, + mine_az, + mine_dip, + mine_rake, + mine_h, + ) = (float(t) for t in (tmp_path / "pfalla.inp").read_text().split()) + assert mine_i0 == real_i0 + assert mine_j0 == real_j0 + assert mine_az == real_az + assert mine_dip == real_dip + assert mine_rake == real_rake + np.testing.assert_allclose( + [mine_slip, mine_l, mine_w, mine_h], + [real_slip, real_l, real_w, real_h], + rtol=2e-4, + ) + + assert (tmp_path / "xyo.dat").read_text().split() == ( + MODEL_DIR / "xyo.dat" + ).read_text().split() + assert (tmp_path / "meca.dat").read_text().strip() == ( + MODEL_DIR / "meca.dat" + ).read_text().strip() + + +def test_run_fault_plane_rejects_an_epicenter_outside_the_grid( + tmp_path: Path, +) -> None: + prepare_simulation_workspace(MODEL_DIR, tmp_path) + (tmp_path / "hypo.dat").write_text( + "\n".join(["0000", "-156.00", "56.00", "12", "9.0"]) + ) + xa = tmp_path / "bathy" / "xa.dat" + xa.unlink() + xa.write_text("300.0\n301.0\n") + + with pytest.raises(RuntimeError, match="outside the computational grid"): + run_fault_plane(tmp_path) diff --git a/packages/tsdhn/tests/test_maxola.py b/packages/tsdhn/tests/test_maxola.py new file mode 100644 index 0000000..c3d3c68 --- /dev/null +++ b/packages/tsdhn/tests/test_maxola.py @@ -0,0 +1,162 @@ +"""Behavior of the GMT-backed maximum-height plot.""" + +import logging +from pathlib import Path +from typing import Any + +import numpy as np +import pygmt +import pytest +import xarray as xr + +from tsdhn.render.maxola import ( + GridConfig, + StyleConfig, + TidalStation, + add_tidal_stations, + cleanup_files, + create_cpt_files, + create_grid_dataarray, + generate_maxola_plot, +) + +MECA_LINE = "210.25 -9.50 10 20 30 40 7.5 210 -9 event\n" + + +def _psconvert_works(tmp_path_factory: pytest.TempPathFactory) -> bool: + """Return whether GMT and Ghostscript can render a PDF here.""" + probe_dir = tmp_path_factory.mktemp("psconvert-probe") + try: + fig = pygmt.Figure() + fig.basemap(region=[0, 1, 0, 1], projection="X1c", frame=False) + fig.savefig(str(probe_dir / "probe.pdf")) + except pygmt.exceptions.GMTCLibError: + return False + return True + + +def test_create_cpt_files_writes_the_legacy_wave_height_convention( + tmp_path: Path, +) -> None: + depth_cpt, hgt_cpt = create_cpt_files(tmp_path) + + assert depth_cpt.is_file() + assert hgt_cpt.is_file() + tail = hgt_cpt.read_text().splitlines()[-3:] + assert tail == ["B 0 0 255", "F 255 0 0", "N 255 255 255"] + + +def test_add_tidal_stations_skips_when_none_are_active( + caplog: pytest.LogCaptureFixture, +) -> None: + fig = pygmt.Figure() + fig.basemap(region=[190, 290, -60, 60], projection="M10c", frame=True) + inactive = TidalStation(lon=210.0, lat=-10.0, code="X", name="x", active=False) + + with caplog.at_level(logging.WARNING): + add_tidal_stations(fig, [inactive], StyleConfig()) + + assert "No active tidal stations" in caplog.text + + +class _RecordingFigure: + """Record the renderer's semantic drawing operations, not GMT internals.""" + + def __init__(self) -> None: + self.calls: list[tuple[str, dict[str, Any]]] = [] + + def _record(self, name: str, **kwargs: Any) -> None: + self.calls.append((name, kwargs)) + + def shift_origin(self, **kwargs: Any) -> None: + self._record("shift_origin", **kwargs) + + def grdimage(self, **kwargs: Any) -> None: + self._record("grdimage", **kwargs) + + def coast(self, **kwargs: Any) -> None: + self._record("coast", **kwargs) + + def plot(self, **kwargs: Any) -> None: + self._record("plot", **kwargs) + + def text(self, **kwargs: Any) -> None: + self._record("text", **kwargs) + + def meca(self, **kwargs: Any) -> None: + self._record("meca", **kwargs) + + def savefig(self, path: str) -> None: + self._record("savefig", path=path) + Path(path).write_bytes(b"rendered plot") + + +def test_generate_maxola_plot_requests_the_complete_plot( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The plot must contain the map, context, stations, mechanism, and legend.""" + figure = _RecordingFigure() + monkeypatch.setattr(pygmt, "Figure", lambda: figure) + monkeypatch.setattr( + "tsdhn.render.maxola.process_grid", lambda work_dir, grid_config: _tiny_grid() + ) + (tmp_path / "meca.dat").write_text(MECA_LINE, encoding="utf-8") + + generate_maxola_plot(tmp_path) + + names = [name for name, _ in figure.calls] + assert names[0] == "shift_origin" + assert names[-1] == "savefig" + assert {"grdimage", "coast", "plot", "meca"} <= set(names) + text_values = [kwargs["text"] for name, kwargs in figure.calls if name == "text"] + assert {"TALA", "CALL", "MATA", "+", "PACIFIC OCEAN"} <= set(text_values) + + +def test_cleanup_files_removes_existing_and_ignores_missing(tmp_path: Path) -> None: + present = tmp_path / "present.txt" + present.write_text("x") + missing = tmp_path / "missing.txt" + + cleanup_files([present, missing]) + + assert not present.exists() + + +def _tiny_grid() -> xr.DataArray: + config = GridConfig(ncols=4, nrows=3, dx=111.1994) + data = np.linspace(0, 1, 12, dtype=np.float32).reshape(3, 4) + return create_grid_dataarray(data, config) + + +def test_generate_maxola_plot_produces_a_pdf( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + tmp_path_factory: pytest.TempPathFactory, +) -> None: + if not _psconvert_works(tmp_path_factory): + pytest.skip("this machine's Ghostscript cannot run GMT's psconvert") + monkeypatch.setattr( + "tsdhn.render.maxola.process_grid", lambda work_dir, grid_config: _tiny_grid() + ) + (tmp_path / "meca.dat").write_text(MECA_LINE, encoding="utf-8") + + generate_maxola_plot(tmp_path) + + output = tmp_path / "maxola.pdf" + assert output.is_file() + assert output.stat().st_size > 0 + + +def test_generate_maxola_plot_cleans_up_cpt_files_even_on_failure( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + def boom(work_dir: Path, grid_config: GridConfig) -> xr.DataArray: + raise RuntimeError("grid processing error") + + monkeypatch.setattr("tsdhn.render.maxola.process_grid", boom) + + with pytest.raises(RuntimeError, match="grid processing error"): + generate_maxola_plot(tmp_path) + + assert not (tmp_path / "depth.cpt").exists() + assert not (tmp_path / "hgt.cpt").exists() diff --git a/packages/tsdhn/tests/test_numerical_values.py b/packages/tsdhn/tests/test_numerical_values.py index 7390f5f..6c5b520 100644 --- a/packages/tsdhn/tests/test_numerical_values.py +++ b/packages/tsdhn/tests/test_numerical_values.py @@ -19,34 +19,37 @@ def input_data() -> EarthquakeInput: ) +# Compatibility values for the checked-in Alaska model data. They are not an +# independent scientific validation of the source relations. expected_basic = { "length": 575.439937, # L (km) "width": 144.543977, # W (km) "seismic_moment": 3.981072e22, # M0 (N*m) - "dislocation": 10.636224, # D (m) + "dislocation": 11.965752, # D (m) "azimuth": 247.0, # Focal mechanism strike (deg) - "dip": 18.0, # Dip (deg) + "dip": 8.0, # Dip (deg) "distance_to_coast": 10439.472791, # (km) } +# These values depend on the legacy rigidity constant and longitude frame. expected_rect_params = { "L1": 575439.937337, # in m - "W1": 137469.491288, # in m - "beta": 13.435833, # in degrees + "W1": 143137.284952, # in m + "beta": 13.968498, # in degrees "alfa": -23.0, # in degrees - "h1": 591632.472501, # in m - "a1": -49.150481, # in km - "b1": 291.704432, # in km - "xo": -153.348142, # longitude - "yo": 56.446823, # latitude + "h1": 592975.044532, # in m + "a1": -46.541865, # in km + "b1": 292.811724, # in km + "xo": -153.362057, # longitude + "yo": 56.419296, # latitude } expected_corners = [ - (-153.348142, 56.446823), - (-158.112445, 54.424496), - (-158.595568, 55.562662), - (-153.831264, 57.584989), - (-153.348142, 56.446823), + (-153.362057, 56.419296), + (-158.126360, 54.396969), + (-158.629402, 55.582062), + (-153.865098, 57.604388), + (-153.362057, 56.419296), ] @@ -88,12 +91,6 @@ def test_rectangle_corners( ) -def test_focal_mechanism(calc_result: CalculationResponse) -> None: - # The nearest-mechanism lookup should stay stable for this epicenter. - assert calc_result.azimuth == pytest.approx(expected_basic["azimuth"], rel=1e-6) - assert calc_result.dip == pytest.approx(expected_basic["dip"], rel=1e-6) - - def test_port_line_parsing_uses_semantic_name() -> None: port = parse_port_line(" -77.1667 -12.06888 % Callao C") @@ -103,6 +100,14 @@ def test_port_line_parsing_uses_semantic_name() -> None: assert port.lat == pytest.approx(-12.06888) +@pytest.mark.parametrize( + "line", + ["", "not coordinates", "-77.0"], +) +def test_port_line_parsing_skips_malformed_lines(line: str) -> None: + assert parse_port_line(line) is None + + def test_tsunami_travel_times_are_keyed_by_port_name( calculator: TsunamiCalculator, ) -> None: @@ -111,6 +116,8 @@ def test_tsunami_travel_times_are_keyed_by_port_name( ) assert "Callao" in travel.arrival_times + assert travel.arrival_times["Callao"].startswith("12:36 05") + assert travel.distances["Callao"] == pytest.approx(19.6797, rel=1e-4) assert not any(name.startswith("-") for name in travel.arrival_times) diff --git a/packages/tsdhn/tests/test_pipeline_golden.py b/packages/tsdhn/tests/test_pipeline_golden.py new file mode 100644 index 0000000..074596c --- /dev/null +++ b/packages/tsdhn/tests/test_pipeline_golden.py @@ -0,0 +1,199 @@ +import shutil +import statistics +import subprocess +import tempfile +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from tsdhn.domain import EarthquakeInput +from tsdhn.engine import SimulationResult, run_simulation + +pytestmark = pytest.mark.golden + +MODEL_DIR = Path(__file__).resolve().parents[3] / "model" +SCENARIO = EarthquakeInput( + Mw=9.0, h=12.0, lat0=56.0, lon0=-156.0, hhmm="0000", dia="23" +) + +# The toolchain produces small floating-point differences between runs. This +# tolerance leaves room for that noise while still catching science changes. +REL_TOL = 1e-4 + +# The production pipeline needs GMT and ttt_client. It does not need legacy +# binaries. +SKIP_REASON = "requires gmt + ttt_client on PATH; see mise run test-golden" + + +def _toolchain_available() -> bool: + return shutil.which("gmt") is not None and shutil.which("ttt_client") is not None + + +@pytest.fixture(scope="module") +def golden_result() -> Iterator[SimulationResult]: + if not _toolchain_available(): + pytest.skip(SKIP_REASON) + + # GMT uses Ghostscript for PDF output. The default temporary directory is + # allowed by the container's Ghostscript policy. + work_root = Path(tempfile.mkdtemp(prefix="tsdhn-golden-")) + try: + yield run_simulation(SCENARIO, work_root, model_dir=MODEL_DIR) + finally: + shutil.rmtree(work_root, ignore_errors=True) + + +def _fingerprint_text(path: Path) -> dict[str, float | int]: + text = path.read_text() + values = [float(x) for x in text.split()] + return { + "line_count": len(text.splitlines()), + "value_count": len(values), + "min": min(values), + "max": max(values), + "mean": statistics.fmean(values), + } + + +def test_simulation_produces_expected_outputs( + golden_result: SimulationResult, +) -> None: + names = {output.name for output in golden_result.outputs.files} + assert names == { + "input", + "runtime", + "calculation", + "travel_times_json", + "travel_times_csv", + "max_height_map", + "arrival_time_map", + "mareogram", + } + + +def test_pfalla_inp_fingerprint(golden_result: SimulationResult) -> None: + fp = _fingerprint_text(golden_result.outputs.root / "pfalla.inp") + # The Python port writes one line. The Fortran reader accepts + # whitespace-separated fields. + assert fp["line_count"] == 1 + assert fp["value_count"] == 9 + assert fp["min"] == pytest.approx(8.0, rel=REL_TOL) + assert fp["max"] == pytest.approx(575440.2, rel=REL_TOL) + assert fp["mean"] == pytest.approx(80605.53208444444, rel=REL_TOL) + + +def test_green_dat_fingerprint(golden_result: SimulationResult) -> None: + fp = _fingerprint_text(golden_result.outputs.root / "zfolder" / "green.dat") + assert fp["line_count"] == 1681 + assert fp["value_count"] == 30258 + assert fp["min"] == pytest.approx(-0.22, rel=REL_TOL) + assert fp["max"] == pytest.approx(1680.0, rel=REL_TOL) + assert fp["mean"] == pytest.approx(46.66686595280587, rel=REL_TOL) + + +def test_zmax_a_grd_fingerprint(golden_result: SimulationResult) -> None: + fp = _fingerprint_text(golden_result.outputs.root / "zfolder" / "zmax_a.grd") + assert fp["line_count"] == 2461 + assert fp["value_count"] == 5059816 + assert fp["min"] == pytest.approx(0.0, abs=1e-9) + assert fp["max"] == pytest.approx(14.441, rel=REL_TOL) + assert fp["mean"] == pytest.approx(0.03851133361371243, rel=REL_TOL) + + +def test_zmax_a_grd_preserves_spatial_samples(golden_result: SimulationResult) -> None: + """Aggregate statistics cannot detect a spatially rearranged grid.""" + path = golden_result.outputs.root / "zfolder" / "zmax_a.grd" + samples = {} + wanted = {(1000, 2), (1000, 1000), (1232, 2), (1232, 2000), (2000, 1000)} + with path.open() as stream: + for row_number, line in enumerate(stream, start=1): + if row_number not in {row for row, _column in wanted}: + continue + values = line.split() + for row, column in wanted: + if row == row_number: + samples[(row, column)] = float(values[column - 1]) + + assert samples == pytest.approx( + { + (1000, 2): 0.027, + (1000, 1000): 0.038, + (1232, 2): 0.038, + (1232, 2000): 0.220, + (2000, 1000): 0.064, + }, + rel=REL_TOL, + ) + + +def test_green_rev_dat_fingerprint(golden_result: SimulationResult) -> None: + fp = _fingerprint_text(golden_result.outputs.root / "zfolder" / "green_rev.dat") + assert fp["line_count"] == 1681 + assert fp["value_count"] == 6724 + assert fp["min"] == pytest.approx(-0.28353207, rel=REL_TOL) + assert fp["max"] == pytest.approx(28.0, rel=REL_TOL) + assert fp["mean"] == pytest.approx(3.4998662468186943, rel=REL_TOL) + + +def test_ttt_max_dat_fingerprint(golden_result: SimulationResult) -> None: + fp = _fingerprint_text(golden_result.outputs.root / "ttt_max.dat") + assert fp["line_count"] == 17 + assert fp["value_count"] == 34 + assert fp["min"] == pytest.approx(0.06, rel=REL_TOL) + assert fp["max"] == pytest.approx(954.0, rel=REL_TOL) + assert fp["mean"] == pytest.approx(444.84117647058827, rel=REL_TOL) + + +def test_ttt_b_fingerprint(golden_result: SimulationResult) -> None: + ttt_mundo_dir = golden_result.outputs.root / "ttt_mundo" + result = subprocess.run( + [shutil.which("gmt") or "gmt", "grdinfo", "-C", "-L1", "-L2", "ttt.b=bf"], + cwd=ttt_mundo_dir, + capture_output=True, + text=True, + check=True, + ) + # grdinfo -C always prefixes the row with the grid name/path. + fields = result.stdout.strip().split("\t")[1:] + ( + x_min, + x_max, + y_min, + y_max, + z_min, + z_max, + x_inc, + y_inc, + n_columns, + n_rows, + median, + mad, + mean, + stdev, + rms, + registration, + n_bands, + ) = fields + + assert (float(x_min), float(x_max)) == pytest.approx((120.0, 300.0)) + assert (float(y_min), float(y_max)) == pytest.approx((-80.0, 89.0)) + assert (float(x_inc), float(y_inc)) == pytest.approx( + (0.0666666666667, 0.0666666666667) + ) + assert int(n_columns) == 2701 + assert int(n_rows) == 2536 + assert int(registration) == 0 + assert int(n_bands) == 1 + + assert float(z_min) == pytest.approx(0.0, abs=1e-9) + assert float(z_max) == pytest.approx(56.8421516418, rel=REL_TOL) + assert float(median) == pytest.approx(11.3055953979, rel=REL_TOL) + assert float(mad) == pytest.approx(6.00670206013, rel=REL_TOL) + assert float(mean) == pytest.approx(11.9470979091, rel=REL_TOL) + assert float(stdev) == pytest.approx(6.21503040015, rel=REL_TOL) + assert float(rms) == pytest.approx(13.466987178, rel=REL_TOL) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/packages/tsdhn/tests/test_processing.py b/packages/tsdhn/tests/test_processing.py new file mode 100644 index 0000000..cc76c14 --- /dev/null +++ b/packages/tsdhn/tests/test_processing.py @@ -0,0 +1,92 @@ +from pathlib import Path + +import pytest + +from tsdhn.pipeline.types import ProcessingStep +from tsdhn.utils.file_utils import atomic_write +from tsdhn.utils.processing import is_step_complete, process_step + + +def _write_output(working_dir: Path) -> None: + (working_dir / "out.txt").write_text("done\n") + + +def _make_step(name: str = "toy") -> ProcessingStep: + return ProcessingStep(name=name, outputs=("out.txt",), runner=_write_output) + + +def test_is_step_complete_false_before_running(tmp_path: Path) -> None: + assert not is_step_complete(_make_step(), tmp_path) + + +def test_process_step_marks_step_complete(tmp_path: Path) -> None: + step = _make_step() + process_step(step, tmp_path) + + assert (tmp_path / "out.txt").is_file() + assert is_step_complete(step, tmp_path) + + +def test_process_step_raises_when_a_declared_output_is_missing( + tmp_path: Path, +) -> None: + """Declared outputs must exist before a completion marker is written.""" + step = ProcessingStep( + name="toy", + outputs=("out.txt", "never_written.txt"), + runner=_write_output, + ) + + with pytest.raises(FileNotFoundError, match=r"never_written\.txt"): + process_step(step, tmp_path) + + assert not is_step_complete(step, tmp_path) + + +def test_is_step_complete_false_if_output_deleted_after_marker(tmp_path: Path) -> None: + step = _make_step() + process_step(step, tmp_path) + (tmp_path / "out.txt").unlink() + + assert not is_step_complete(step, tmp_path) + + +def test_is_step_complete_false_on_pipeline_version_bump( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + step = _make_step() + process_step(step, tmp_path) + assert is_step_complete(step, tmp_path) + + import tsdhn.utils.processing as processing_module + + monkeypatch.setattr(processing_module, "PIPELINE_VERSION", 999) + + assert not is_step_complete(step, tmp_path) + + +def test_is_step_complete_false_for_different_step_outputs(tmp_path: Path) -> None: + step = _make_step() + process_step(step, tmp_path) + + changed_step = ProcessingStep( + name=step.name, + outputs=("out.txt", "extra.txt"), + runner=step.runner, + ) + assert not is_step_complete(changed_step, tmp_path) + + +def test_atomic_write_removes_a_partial_file_on_failure(tmp_path: Path) -> None: + path = tmp_path / "result.txt" + temporary = path.with_name("result.txt.tmp") + + with ( + pytest.raises(RuntimeError, match="interrupted"), + atomic_write(path) as temporary_path, + ): + temporary_path.write_text("partial", encoding="utf-8") + raise RuntimeError("interrupted") + + assert not path.exists() + assert not temporary.exists() diff --git a/packages/tsdhn/tests/test_runtime_contract.py b/packages/tsdhn/tests/test_runtime_contract.py index cf8d428..d8b385c 100644 --- a/packages/tsdhn/tests/test_runtime_contract.py +++ b/packages/tsdhn/tests/test_runtime_contract.py @@ -1,16 +1,17 @@ import subprocess from pathlib import Path +from typing import Self import numpy as np +import pygmt import pytest from pygmt.enums import GridRegistration, GridType -from tsdhn.pipeline.types import ProcessingStep, ToolRunner +from tsdhn.pipeline.types import ProcessingStep from tsdhn.render import ttt_inverso from tsdhn.render.maxola import GridConfig, load_stations, process_grid from tsdhn.runtime import ( REQUIRED_MODEL_FILES, - REQUIRED_TOOL_EXECUTABLES, RuntimeContext, ) from tsdhn.utils.file_utils import ( @@ -33,84 +34,42 @@ def _create_model_dir(root: Path) -> Path: return model_dir -def _create_tools_dir(root: Path) -> Path: - tools_dir = root / "tools" - tools_dir.mkdir() - for executable in REQUIRED_TOOL_EXECUTABLES: - path = tools_dir / executable - path.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") - path.chmod(0o755) - return tools_dir - - def test_runtime_context_reports_missing_managed_model( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.delenv("TSDHN_MODEL_DIR", raising=False) - monkeypatch.delenv("TSDHN_TOOLS_DIR", raising=False) monkeypatch.setenv("TSDHN_DATA_HOME", str(tmp_path / "missing")) with pytest.raises(RuntimeError, match="tsdhn assets install"): - RuntimeContext.resolve(require_tools=False) + RuntimeContext.resolve() -def test_runtime_paths_resolve_explicit_paths(tmp_path: Path) -> None: +def test_runtime_resolves_an_explicit_model_dir(tmp_path: Path) -> None: model_dir = _create_model_dir(tmp_path) - tools_dir = _create_tools_dir(tmp_path) - runtime = RuntimeContext.resolve(model_dir=model_dir, tools_dir=tools_dir) + runtime = RuntimeContext.resolve(model_dir=model_dir) assert runtime.model_dir == model_dir.resolve() - assert runtime.tools_dir == tools_dir.resolve() -def test_runtime_paths_can_resolve_model_only_when_no_tools_are_needed( +def test_runtime_resolves_the_model_dir_from_the_environment( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: model_dir = _create_model_dir(tmp_path) - monkeypatch.delenv("TSDHN_TOOLS_DIR", raising=False) + monkeypatch.setenv("TSDHN_MODEL_DIR", str(model_dir)) - runtime = RuntimeContext.resolve(model_dir=model_dir, require_tools=False) + assert RuntimeContext.resolve().model_dir == model_dir.resolve() - assert runtime.model_dir == model_dir.resolve() - assert runtime.tools_dir is None - -def test_runtime_paths_validate_only_required_tools(tmp_path: Path) -> None: +def test_runtime_reports_external_tool_capabilities(tmp_path: Path) -> None: + """Report GMT and ttt_client when they are resolved from PATH.""" model_dir = _create_model_dir(tmp_path) - tools_dir = tmp_path / "tools" - tools_dir.mkdir() - executable = tools_dir / "fault_plane" - executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") - executable.chmod(0o755) - - runtime = RuntimeContext.resolve( - model_dir=model_dir, - tools_dir=tools_dir, - required_tools=("fault_plane",), - ) - assert runtime.tools_dir == tools_dir.resolve() - - -def test_runtime_paths_validate_only_required_tools_from_environment( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - model_dir = _create_model_dir(tmp_path) - tools_dir = tmp_path / "tools" - tools_dir.mkdir() - executable = tools_dir / "fault_plane" - executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") - executable.chmod(0o755) - monkeypatch.setenv("TSDHN_TOOLS_DIR", str(tools_dir)) - - runtime = RuntimeContext.resolve( - model_dir=model_dir, - required_tools=("fault_plane",), - ) + runtime = RuntimeContext.resolve(model_dir=model_dir) - assert runtime.tools_dir == tools_dir.resolve() + assert set(runtime.capabilities) == {"gmt", "ttt_client"} + assert not hasattr(runtime, "tools_dir") def test_prepare_simulation_workspace_links_only_required_inputs( @@ -208,6 +167,7 @@ def test_ttt_inverso_uses_shared_meca_spec_for_epicenter( encoding="utf-8", ) commands: list[tuple[list[str], Path]] = [] + module_calls: list[tuple[str, list[str]]] = [] def fake_resolve(executable: str) -> Path: return Path("/tools") / executable @@ -223,8 +183,19 @@ def fake_run( commands.append((args, cwd)) return subprocess.CompletedProcess(args, 0) + class FakeSession: + def __enter__(self) -> Self: + return self + + def __exit__(self, *exc_info: object) -> None: + return None + + def call_module(self, module: str, args: list[str]) -> None: + module_calls.append((module, args)) + monkeypatch.setattr(ttt_inverso, "resolve", fake_resolve) monkeypatch.setattr("tsdhn.render.ttt_inverso.subprocess.run", fake_run) + monkeypatch.setattr(ttt_inverso, "Session", FakeSession) ttt_inverso.ttt_inverso_python(working_dir) @@ -239,19 +210,9 @@ def fake_run( ], working_dir, ), - ( - [ - "/tools/gmt", - "grdmath", - "ttt.b=bf", - "1.0", - "MUL", - "=", - "ttt.b=bf", - ], - working_dir, - ), ] + grid_arg = f"{working_dir / 'ttt.b'}=bf" + assert module_calls == [("grdmath", [grid_arg, "1.0", "MUL", "=", grid_arg])] def test_ttt_inverso_keeps_full_meca_dat_validation(tmp_path: Path) -> None: @@ -263,25 +224,49 @@ def test_ttt_inverso_keeps_full_meca_dat_validation(tmp_path: Path) -> None: ttt_inverso.ttt_inverso_python(working_dir) -def test_process_step_uses_prebuilt_executable(tmp_path: Path) -> None: - tools_dir = tmp_path / "tools" +def test_ttt_inverso_surfaces_a_grdmath_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + working_dir = tmp_path / "ttt" + working_dir.mkdir() + (tmp_path / "meca.dat").write_text( + "210.25 -9.50 10 20 30 40 7.5 210 -9 event\n", + encoding="utf-8", + ) + + class BrokenSession: + def __enter__(self) -> BrokenSession: + return self + + def __exit__(self, *exc_info: object) -> None: + return None + + def call_module(self, module: str, args: list[str]) -> None: + raise pygmt.exceptions.GMTError("grdmath failed") + + monkeypatch.setattr(ttt_inverso, "resolve", lambda name: Path("/tools") / name) + monkeypatch.setattr( + "tsdhn.render.ttt_inverso.subprocess.run", + lambda *args, **kwargs: subprocess.CompletedProcess(args, 0), + ) + monkeypatch.setattr(ttt_inverso, "Session", BrokenSession) + + with pytest.raises(pygmt.exceptions.GMTError, match="grdmath failed"): + ttt_inverso.ttt_inverso_python(working_dir) + + +def test_process_step_runs_a_python_step_and_validates_its_outputs( + tmp_path: Path, +) -> None: work_dir = tmp_path / "work" - tools_dir.mkdir() work_dir.mkdir() - executable = tools_dir / "hello" - executable.write_text("#!/bin/sh\necho ok > ran.txt\n", encoding="utf-8") - executable.chmod(0o755) + def write_result(working_dir: Path) -> None: + (working_dir / "ran.txt").write_text("ok\n", encoding="utf-8") process_step( - ProcessingStep( - name="hello", - outputs=("ran.txt",), - runner=ToolRunner("hello"), - file_checks=(("ran.txt", "prebuilt executable did not run"),), - ), + ProcessingStep(name="hello", outputs=("ran.txt",), runner=write_result), work_dir, - tools_dir, ) assert (work_dir / "ran.txt").read_text(encoding="utf-8").strip() == "ok" diff --git a/packages/tsdhn/tests/test_tsunami.py b/packages/tsdhn/tests/test_tsunami.py new file mode 100644 index 0000000..c23632a --- /dev/null +++ b/packages/tsdhn/tests/test_tsunami.py @@ -0,0 +1,413 @@ +import math +from pathlib import Path + +import numpy as np +import pytest + +import tsdhn.tsunami as tsunami_module +from tsdhn.tsunami import ( + _bout, + _hmn, + _mass, + _mmnt, + _prelim, + _read_deform_a, + _read_grid_a, + _read_tidal_dat, + _read_xyo_dat, + _write_green_dat, + _write_zmax_a, + run_tsunami, +) + +MODEL_DIR = Path(__file__).resolve().parents[3] / "model" + + +def test_hmn_staggered_averages_and_edges() -> None: + h = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) + hm, hn = _hmn(h) + np.testing.assert_array_equal( + hm, np.array([[2.0, 3.0], [4.0, 5.0], [5.0, 6.0]], dtype=np.float32) + ) + np.testing.assert_array_equal( + hn, np.array([[1.5, 2.0], [3.5, 4.0], [5.5, 6.0]], dtype=np.float32) + ) + + +def test_prelim_factors_match_formulas() -> None: + hm = np.full((3, 4), 1000.0, dtype=np.float32) + hn = np.full((3, 4), 2000.0, dtype=np.float32) + rx, cj, xx, yy = _prelim(hm, hn) + + delta = 240.0 / 3600.0 + da = math.pi * delta / 180.0 + rz0 = math.radians(-76.006) + assert rx[0] == pytest.approx(3.0 / (6.37e6 * math.cos(rz0) * da), rel=1e-5) + assert cj[0] == pytest.approx(math.cos(rz0 + da / 2.0), rel=1e-5) + # A later latitude catches changes to the repeated float32 angle step. + assert rx[3] == pytest.approx( + 3.0 / (6.37e6 * math.cos(rz0 + 3 * da) * da), rel=1e-5 + ) + + np.testing.assert_allclose(xx, rx[np.newaxis, :4] * np.float32(9.8) * hm, rtol=1e-6) + np.testing.assert_allclose(yy, 3.0 * 9.8 * hn / (6.37e6 * da), rtol=1e-5) + + +def test_mass_step_hand_computed() -> None: + # These values are calculated directly from the continuity equation. + h = np.full((3, 3), 100.0, dtype=np.float32) + h[2, 2] = -5.0 + rx = np.full(3, 0.5, dtype=np.float32) + cj = np.ones(3, dtype=np.float32) + z1 = np.zeros((3, 3), dtype=np.float32) + m1 = np.zeros((3, 3), dtype=np.float32) + n1 = np.zeros((3, 3), dtype=np.float32) + m1[1, 1] = 1.0 + m1[2, 1] = 2.0 + z2 = np.full((3, 3), 9.0, dtype=np.float32) + _mass(z1, z2, m1, n1, h, rx, cj) + + assert z2[1, 1] == np.float32(-0.5) + assert z2[2, 1] == np.float32(-0.5) + assert z2[1, 2] == np.float32(0.0) + assert z2[2, 2] == np.float32(0.0) + # Sentinels show that the first Fortran row and column were not written. + assert np.all(z2[0, :] == np.float32(9.0)) + assert np.all(z2[:, 0] == np.float32(9.0)) + + +def test_mass_flushes_small_values_to_zero() -> None: + h = np.full((3, 3), 100.0, dtype=np.float32) + rx = np.full(3, 0.5, dtype=np.float32) + cj = np.ones(3, dtype=np.float32) + z1 = np.zeros((3, 3), dtype=np.float32) + m1 = np.zeros((3, 3), dtype=np.float32) + n1 = np.zeros((3, 3), dtype=np.float32) + m1[1, 1] = np.float32(1.6e-5) + z2 = np.empty((3, 3), dtype=np.float32) + _mass(z1, z2, m1, n1, h, rx, cj) + assert z2[1, 1] == np.float32(0.0) + + +def test_mmnt_wet_pair_condition_and_flush() -> None: + h = np.full((3, 3), 100.0, dtype=np.float32) + h[2, 2] = -5.0 + xx = np.full((3, 3), 0.25, dtype=np.float32) + yy = np.full((3, 3), 0.125, dtype=np.float32) + z2 = np.zeros((3, 3), dtype=np.float32) + z2[2, 1] = 2.0 + z2[1, 2] = 4.0 + m1 = np.zeros((3, 3), dtype=np.float32) + n1 = np.zeros((3, 3), dtype=np.float32) + m2 = np.full((3, 3), 9.0, dtype=np.float32) + n2 = np.full((3, 3), 9.0, dtype=np.float32) + _mmnt(z2, m1, m2, n1, n2, h, xx, yy) + + assert m2[1, 1] == np.float32(-0.5) + assert m2[1, 2] == np.float32(0.0) + assert n2[1, 1] == np.float32(-0.5) + assert n2[2, 1] == np.float32(0.0) + # Sentinels protect the two staggered outer edges from unexpected writes. + assert np.all(m2[2, :] == np.float32(9.0)) + assert np.all(n2[:, 2] == np.float32(9.0)) + + +def test_bout_radiation_sign_and_corner_order() -> None: + # Outgoing flow radiates positive elevation and incoming flow flips the + # sign. At corners, the second edge pass wins. + ia = ja = 4 + h = np.full((ia, ja), 100.0, dtype=np.float32) + m_prev = np.zeros((ia, ja), dtype=np.float32) + n_prev = np.zeros((ia, ja), dtype=np.float32) + n_prev[1, 1] = 7.0 + n_prev[2, 1] = -7.0 + z2 = np.zeros((ia, ja), dtype=np.float32) + _bout(z2, m_prev, n_prev, h) + + cc = math.sqrt(9.8 * 100.0) + assert z2[2, 1] == pytest.approx(7.0 / cc, rel=1e-6) + # The second edge pass overwrites the corner with its independent value. + assert z2[1, 1] == pytest.approx(3.5 / cc, rel=1e-6) + + +def test_bout_skips_dry_cells() -> None: + ia = ja = 4 + h = np.full((ia, ja), 100.0, dtype=np.float32) + h[2, 1] = -50.0 + h[1, 2] = -50.0 + m_prev = np.zeros((ia, ja), dtype=np.float32) + n_prev = np.zeros((ia, ja), dtype=np.float32) + z2 = np.full((ia, ja), 0.25, dtype=np.float32) + _bout(z2, m_prev, n_prev, h) + assert z2[2, 1] == np.float32(0.25) + assert z2[1, 2] == np.float32(0.25) + + +def test_read_tidal_dat_real_file() -> None: + ip, jp = _read_tidal_dat(MODEL_DIR / "tidal.dat") + assert ip.shape == (17,) + assert (ip[0], jp[0]) == (2272, 1088) + assert (ip[-1], jp[-1]) == (2425, 864) + + +def test_read_xyo_dat_real_file() -> None: + # The captured file includes grid dimensions after the four consumed fields. + assert _read_xyo_dat(MODEL_DIR / "xyo.dat") == (1021, 1246, 1861, 2056) + + +def test_read_deform_a_real_file() -> None: + grid = _read_deform_a(MODEL_DIR / "deform_a.grd", 1021, 1246, 1861, 2056) + assert grid.shape == (226, 196) + assert grid.dtype == np.float32 + assert np.all(np.isfinite(grid)) + + +def test_write_green_dat_fixed_width_format(tmp_path: Path) -> None: + path = tmp_path / "green.dat" + _write_green_dat( + path, + [ + (0.0, np.array([0.123, -0.5], dtype=np.float32)), + (1680.0, np.array([12.345, 0.0], dtype=np.float32)), + ], + ) + lines = path.read_text().splitlines() + assert lines[0] == " 0.0 0.123 -0.500" + assert lines[1] == " 1680.0 12.345 0.000" + + +def test_write_zmax_a_fixed_width_format(tmp_path: Path) -> None: + path = tmp_path / "zmax_a.grd" + _write_zmax_a(path, np.array([[0.0, 14.441], [-0.5, 1.0]], dtype=np.float32)) + lines = path.read_text().splitlines() + assert lines[0] == " 0.000 14.441" + assert lines[1] == " -0.500 1.000" + + +def test_read_grid_a_applies_shallow_water_floor( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(tsunami_module, "IA", 2) + monkeypatch.setattr(tsunami_module, "JA", 2) + path = tmp_path / "grid_a.grd" + path.write_text("5.0 -3.0\n0.0 20.0\n") + np.testing.assert_array_equal( + _read_grid_a(path), + np.array([[10.0, -3.0], [0.0, 20.0]], dtype=np.float32), + ) + + +def test_run_tsunami_mini_end_to_end( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(tsunami_module, "IA", 6) + monkeypatch.setattr(tsunami_module, "JA", 5) + monkeypatch.setattr(tsunami_module, "KE", 4) + monkeypatch.setattr(tsunami_module, "KD", 2) + monkeypatch.setattr(tsunami_module, "NG", 2) + + (tmp_path / "bathy").mkdir() + rows = [] + for _ in range(6): + rows.append(" ".join("1000.0" for _ in range(5))) + (tmp_path / "bathy" / "grid_a.grd").write_text("\n".join(rows) + "\n") + (tmp_path / "tidal.dat").write_text("1 3 3\n2 4 4\n") + (tmp_path / "xyo.dat").write_text("2 3 2 3\n") + (tmp_path / "deform_a.grd").write_text("1.0 1.0\n1.0 1.0\n") + + run_tsunami(tmp_path) + + green = (tmp_path / "zfolder" / "green.dat").read_text().splitlines() + assert green == [ + " 0.0 1.000 0.000", + " 0.1 0.837 0.000", + ] + + zmax = np.loadtxt(tmp_path / "zfolder" / "zmax_a.grd", dtype=np.float32) + np.testing.assert_allclose( + zmax, + np.array( + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.007, 0.292, 0.003, 0.0], + [0.0, 0.076, 1.0, 0.005, 0.0], + [0.0, 0.003, 0.078, 0.0, 0.0], + [0.0, 0.0, 0.001, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + dtype=np.float32, + ), + atol=0.001, + ) + + +def _write_toy_grid_inputs(work_dir: Path) -> None: + (work_dir / "bathy").mkdir() + rows = [" ".join("1000.0" for _ in range(5)) for _ in range(6)] + (work_dir / "bathy" / "grid_a.grd").write_text("\n".join(rows) + "\n") + (work_dir / "tidal.dat").write_text("1 3 3\n2 4 4\n") + (work_dir / "xyo.dat").write_text("2 3 2 3\n") + (work_dir / "deform_a.grd").write_text("1.0 1.0\n1.0 1.0\n") + + +def test_run_tsunami_resumes_from_checkpoint_after_interruption( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # A crash after a checkpoint must resume with the same outputs as a fresh + # run. + monkeypatch.setattr(tsunami_module, "IA", 6) + monkeypatch.setattr(tsunami_module, "JA", 5) + monkeypatch.setattr(tsunami_module, "KE", 8) + monkeypatch.setattr(tsunami_module, "KD", 2) + monkeypatch.setattr(tsunami_module, "NG", 2) + monkeypatch.setattr(tsunami_module, "_CHECKPOINT_INTERVAL", 2) + + reference_dir = tmp_path / "reference" + reference_dir.mkdir() + _write_toy_grid_inputs(reference_dir) + run_tsunami(reference_dir) + reference_green = (reference_dir / "zfolder" / "green.dat").read_text() + reference_zmax = np.loadtxt( + reference_dir / "zfolder" / "zmax_a.grd", dtype=np.float32 + ) + + resumed_dir = tmp_path / "resumed" + resumed_dir.mkdir() + _write_toy_grid_inputs(resumed_dir) + + real_mmnt = tsunami_module._mmnt + call_count = 0 + + def flaky_mmnt( + z2: np.ndarray, + m1: np.ndarray, + m2: np.ndarray, + n1: np.ndarray, + n2: np.ndarray, + h: np.ndarray, + xx: np.ndarray, + yy: np.ndarray, + ) -> None: + nonlocal call_count + call_count += 1 + if call_count == 4: + raise RuntimeError("simulated worker crash") + real_mmnt(z2, m1, m2, n1, n2, h, xx, yy) + + monkeypatch.setattr(tsunami_module, "_mmnt", flaky_mmnt) + with pytest.raises(RuntimeError, match="simulated worker crash"): + run_tsunami(resumed_dir) + + checkpoint_path = resumed_dir / "zfolder" / "_checkpoint.npz" + assert checkpoint_path.is_file() + + monkeypatch.setattr(tsunami_module, "_mmnt", real_mmnt) + run_tsunami(resumed_dir) + + assert not checkpoint_path.exists() + resumed_green = (resumed_dir / "zfolder" / "green.dat").read_text() + resumed_zmax = np.loadtxt(resumed_dir / "zfolder" / "zmax_a.grd", dtype=np.float32) + + assert resumed_green == reference_green + np.testing.assert_array_equal(resumed_zmax, reference_zmax) + + +def test_read_checkpoint_rejects_stale_pipeline_version( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + zeros = np.zeros((6, 5), dtype=np.float32) + checkpoint_path = tmp_path / "checkpoint.npz" + + monkeypatch.setattr(tsunami_module, "PIPELINE_VERSION", 999) + tsunami_module._write_checkpoint( + checkpoint_path, 5, zeros, zeros, zeros, zeros, zeros, zeros, zeros, [] + ) + + monkeypatch.setattr(tsunami_module, "PIPELINE_VERSION", 1) + result = tsunami_module._read_checkpoint(checkpoint_path, ia=6, ja=5, n_gauges=2) + + assert result is None + + +def test_read_grid_a_rejects_the_wrong_shape( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(tsunami_module, "IA", 2) + monkeypatch.setattr(tsunami_module, "JA", 2) + path = tmp_path / "grid_a.grd" + path.write_text("5.0 -3.0 1.0\n") + + with pytest.raises(ValueError, match=r"Unexpected grid_a\.grd shape"): + _read_grid_a(path) + + +def test_read_deform_a_rejects_a_window_size_mismatch(tmp_path: Path) -> None: + path = tmp_path / "deform_a.grd" + path.write_text("1.0 1.0 1.0\n") + + with pytest.raises(ValueError, match=r"Unexpected deform_a\.grd size"): + _read_deform_a(path, ids=1, ide=2, jds=1, jde=2) + + +def test_read_checkpoint_rejects_an_array_shape_mismatch(tmp_path: Path) -> None: + wrong_shape = np.zeros((3, 3), dtype=np.float32) + checkpoint_path = tmp_path / "checkpoint.npz" + tsunami_module._write_checkpoint( + checkpoint_path, + 5, + wrong_shape, + wrong_shape, + wrong_shape, + wrong_shape, + wrong_shape, + wrong_shape, + wrong_shape, + [], + ) + + result = tsunami_module._read_checkpoint(checkpoint_path, ia=6, ja=5, n_gauges=2) + + assert result is None + + +def test_read_checkpoint_rejects_a_gauge_count_mismatch(tmp_path: Path) -> None: + zeros = np.zeros((6, 5), dtype=np.float32) + checkpoint_path = tmp_path / "checkpoint.npz" + gauges = np.zeros(6, dtype=np.float32) + tsunami_module._write_checkpoint( + checkpoint_path, + 5, + zeros, + zeros, + zeros, + zeros, + zeros, + zeros, + zeros, + [(0.0, gauges)], + ) + + result = tsunami_module._read_checkpoint(checkpoint_path, ia=6, ja=5, n_gauges=2) + + assert result is None + + +def test_read_checkpoint_rejects_a_step_index_out_of_range(tmp_path: Path) -> None: + zeros = np.zeros((6, 5), dtype=np.float32) + checkpoint_path = tmp_path / "checkpoint.npz" + tsunami_module._write_checkpoint( + checkpoint_path, + tsunami_module.KE + 1, + zeros, + zeros, + zeros, + zeros, + zeros, + zeros, + zeros, + [], + ) + + result = tsunami_module._read_checkpoint(checkpoint_path, ia=6, ja=5, n_gauges=2) + + assert result is None diff --git a/packages/tsdhn/tests/test_utils.py b/packages/tsdhn/tests/test_utils.py index 1e02932..883d4a8 100644 --- a/packages/tsdhn/tests/test_utils.py +++ b/packages/tsdhn/tests/test_utils.py @@ -12,18 +12,16 @@ def test_calculate_distance_to_coast() -> None: coast_points = np.array([[-70.0, -20.0], [-71.0, -21.0]]) distance = calculate_distance_to_coast(coast_points, -70.5, -20.5) - assert isinstance(distance, float) - assert distance > 0 + assert distance == pytest.approx(78.6266867) def test_format_arrival_time() -> None: formatted_time = format_arrival_time(14.5, "15") - assert isinstance(formatted_time, str) - assert ":" in formatted_time + assert formatted_time.startswith("14:30 15") # Arrival formatting rolls into the next day after 24 hours. rollover_time = format_arrival_time(25.5, "15") - assert rollover_time.split()[0].startswith("01") + assert rollover_time.startswith("01:30 16") @pytest.mark.parametrize( diff --git a/packages/tsdhn/tsdhn/__init__.py b/packages/tsdhn/tsdhn/__init__.py index 604cefa..24f7fcd 100644 --- a/packages/tsdhn/tsdhn/__init__.py +++ b/packages/tsdhn/tsdhn/__init__.py @@ -5,9 +5,9 @@ TsunamiTravelResponse, ) from tsdhn.engine import ( - Artifact, - ArtifactBundle, + OutputFile, SimulationEngine, + SimulationOutputs, SimulationRequest, SimulationResult, run_simulation, @@ -15,13 +15,13 @@ from tsdhn.runtime import RuntimeContext __all__ = [ - "Artifact", - "ArtifactBundle", "CalculationResponse", "EarthquakeInput", "JobStatus", + "OutputFile", "RuntimeContext", "SimulationEngine", + "SimulationOutputs", "SimulationRequest", "SimulationResult", "TsunamiTravelResponse", diff --git a/packages/tsdhn/tsdhn/calculator.py b/packages/tsdhn/tsdhn/calculator.py index 8f8eea9..adf9d37 100644 --- a/packages/tsdhn/tsdhn/calculator.py +++ b/packages/tsdhn/tsdhn/calculator.py @@ -1,7 +1,10 @@ +"""Source parameters and approximate port arrival times.""" + import logging +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path -from typing import cast +from typing import Any, cast import numpy as np from scipy.interpolate import RegularGridInterpolator @@ -26,6 +29,27 @@ # Legacy corner formulas use 60 nautical miles per degree. NM_CONVERSION = 60 * 1853 +# Rigidity (N/m^2). Matches model/fault_plane.f90. +RIGIDITY_N_PER_M2 = 4.0e10 + + +def rupture_dimensions(Mw: float) -> tuple[float, float]: + """Return Papazachos et al. (2004) rupture length and width in km.""" + L = 10 ** (0.55 * Mw - 2.19) + W = 10 ** (0.31 * Mw - 0.63) + return L, W + + +def average_slip(Mw: float, L: float, W: float) -> tuple[float, float]: + """Hanks and Kanamori moment magnitude relation, solved for slip. + + L, W in km. Returns (M0, D): seismic moment (N*m), average dislocation + (m). + """ + M0 = 10 ** (1.5 * Mw + 9.1) + D = M0 / (RIGIDITY_N_PER_M2 * (L * 1000) * (W * 1000)) + return M0, D + @dataclass(frozen=True) class Port: @@ -55,12 +79,83 @@ def parse_port_line(line: str) -> Port | None: return Port(name=name, lon=lon, lat=lat) +def calculate_rectangle_parameters( + L: float, + W: float, + lon0: float, + lat0: float, + azimuth: float, + dip: float, + on_checkpoint: Callable[[str, Any], None] | None = None, +) -> tuple[dict[str, float], list[dict[str, float]]]: + """Return the MATLAB-compatible fault-plane parameters and corners. + + The optional `on_checkpoint` callback receives each named intermediate in + calculation order. + """ + + def checkpoint(name: str, value: Any) -> None: + if on_checkpoint is not None: + on_checkpoint(name, value) + + L1 = L * 1000 + checkpoint("L1", L1) + W1 = W * 1000 * np.cos(np.deg2rad(dip)) + checkpoint("W1", W1) + beta = np.degrees(np.arctan(W1 / L1)) + checkpoint("beta", beta) + alfa = azimuth - 270 + checkpoint("alfa", alfa) + h1 = np.hypot(L1, W1) + checkpoint("h1", h1) + + # Keep the names used by the legacy report files. + params = { + "L1": L1, + "W1": W1, + "beta": beta, + "alfa": alfa, + "h1": h1, + "a1": 0.5 * h1 * np.sin(np.deg2rad(alfa + beta)) / 1000, + "b1": 0.5 * h1 * np.cos(np.deg2rad(alfa + beta)) / 1000, + # Use the longitude frame expected by the legacy grid. + "xo": lon0 + (0.5 * h1 * np.cos(np.deg2rad(alfa + beta)) / 1000) / 111.0, + "yo": lat0 - (0.5 * h1 * np.sin(np.deg2rad(alfa + beta)) / 1000) / 111.0, + } + checkpoint("a1", params["a1"]) + checkpoint("b1", params["b1"]) + checkpoint("xo", params["xo"]) + checkpoint("yo", params["yo"]) + + # Legacy geometry expresses fault-plane offsets in degree-space nautical miles. + angles = -np.radians([(azimuth - 90), azimuth]) + + r = np.array([L1, W1]) / NM_CONVERSION + + sx = ( + r[0] * np.cos(angles[0]) * np.array([0, 1, 1, 0, 0]) + + r[1] * np.cos(angles[1]) * np.array([0, 0, 1, 1, 0]) + ) + params["xo"] + + sy = ( + r[0] * np.sin(angles[0]) * np.array([0, 1, 1, 0, 0]) + + r[1] * np.sin(angles[1]) * np.array([0, 0, 1, 1, 0]) + ) + params["yo"] + + corners = [{"lon": x, "lat": y} for x, y in zip(sx, sy, strict=False)] + checkpoint( + "corners", np.array([[corner["lon"], corner["lat"]] for corner in corners]) + ) + + return params, corners + + class TsunamiCalculator: def __init__(self, model_dir: Path | None = None) -> None: self.model_dir = ( validate_model_dir(model_dir.resolve()) if model_dir is not None - else RuntimeContext.resolve(require_tools=False).model_dir + else RuntimeContext.resolve().model_dir ) self.xa: np.ndarray self.ya: np.ndarray @@ -86,7 +181,7 @@ def _load_geographic_data(self) -> None: self.ya = pacifico["ya"].flatten() self.bathymetry = pacifico["A"] - # Bathymetry uses 0..360 longitudes. Calculations use -180..180. + # Model bathymetry uses 0..360 longitudes; inputs use -180..180. self.vlon = self.xa - 360 self.vlat = self.ya if self.vlat[0] > self.vlat[-1]: @@ -111,7 +206,7 @@ def _load_static_files(self) -> None: mech_path = self.model_dir / "mecfoc.dat" self.mechanism_data = np.loadtxt(mech_path) - # CMT data uses eastern positive degrees. Lookup uses western negatives. + # CMT data uses eastern-positive longitudes; lookup uses western negatives. self.mechanism_data[:, 0] = np.where( self.mechanism_data[:, 0] > 0, self.mechanism_data[:, 0] - 360, @@ -130,21 +225,12 @@ def calculate_earthquake_parameters( ) -> CalculationResponse: """Calculate source parameters and write hypo.dat for the simulation.""" try: - # Papazachos et al. (2004) magnitude scaling relations. - L = 10 ** (0.55 * data.Mw - 2.19) # Rupture length (km) - W = 10 ** (0.31 * data.Mw - 0.63) # Rupture width (km) - - # Hanks and Kanamori moment magnitude relation. - M0 = 10 ** (1.5 * data.Mw + 9.1) # Seismic moment (N*m) - - # Seismic moment definition solved for average slip. - u = 4.5e10 # Rigidity (N/m^2) - D = M0 / (u * (L * 1000) * (W * 1000)) # Dislocation (m) + L, W = rupture_dimensions(data.Mw) # km + M0, D = average_slip(data.Mw, L, W) # N*m, m azimuth, dip = self._get_focal_mechanism(data.lon0, data.lat0) - # Fault plane geometry - rect_params, rect_corners = self._calculate_rectangle_parameters( + rect_params, rect_corners = calculate_rectangle_parameters( L, W, data.lon0, data.lat0, azimuth, dip ) @@ -158,10 +244,8 @@ def calculate_earthquake_parameters( if self.bathy_interpolator is None: raise RuntimeError("Bathymetry interpolator not loaded") - # Get bathymetry at epicenter h0 = self.bathy_interpolator((data.lat0, data.lon0)) - # Determine location and warning location = determine_epicenter_location(h0, distance_to_coast) warning = determine_tsunami_warning(data.Mw, data.h, h0, distance_to_coast) @@ -247,50 +331,10 @@ def _get_focal_mechanism(self, lon0: float, lat0: float) -> tuple[float, float]: + (self.mechanism_data[:, 1] - lat0) ** 2 ) closest_idx = np.argmin(distances) - # The MATLAB model fixes dip at 18 degrees after selecting strike. - return self.mechanism_data[closest_idx, 2], 18.0 - - def _calculate_rectangle_parameters( - self, L: float, W: float, lon0: float, lat0: float, azimuth: float, dip: float - ) -> tuple[dict[str, float], list[dict[str, float]]]: - """Return the MATLAB-compatible fault-plane parameters and corners.""" - L1 = L * 1000 # Meters. - W1 = W * 1000 * np.cos(np.deg2rad(dip)) - beta = np.degrees(np.arctan(W1 / L1)) - alfa = azimuth - 270 - h1 = np.hypot(L1, W1) - - # Report keys preserve the legacy MATLAB parameter names. - params = { - "L1": L1, - "W1": W1, - "beta": beta, - "alfa": alfa, - "h1": h1, - "a1": 0.5 * h1 * np.sin(np.deg2rad(alfa + beta)) / 1000, # in km - "b1": 0.5 * h1 * np.cos(np.deg2rad(alfa + beta)) / 1000, # in km - "xo": lon0 + (0.5 * h1 * np.cos(np.deg2rad(alfa + beta)) / 1000) / 110, - "yo": lat0 - (0.5 * h1 * np.sin(np.deg2rad(alfa + beta)) / 1000) / 110, - } - - # Legacy geometry expresses fault-plane offsets in degree-space nautical miles. - angles = -np.radians([(azimuth - 90), azimuth]) - - r = np.array([L1, W1]) / NM_CONVERSION - - sx = ( - r[0] * np.cos(angles[0]) * np.array([0, 1, 1, 0, 0]) - + r[1] * np.cos(angles[1]) * np.array([0, 0, 1, 1, 0]) - ) + params["xo"] - - sy = ( - r[0] * np.sin(angles[0]) * np.array([0, 1, 1, 0, 0]) - + r[1] * np.sin(angles[1]) * np.array([0, 0, 1, 1, 0]) - ) + params["yo"] - - corners = [{"lon": x, "lat": y} for x, y in zip(sx, sy, strict=False)] - - return params, corners + return ( + self.mechanism_data[closest_idx, 2], + self.mechanism_data[closest_idx, 3], + ) def _calculate_travel_time( self, lon0: float, lat0: float, port_lon: float, port_lat: float, time0: float @@ -300,7 +344,6 @@ def _calculate_travel_time( t2 = np.pi / 2 - np.radians(port_lat) f2 = np.radians(port_lon) - # Spherical law of cosines. cos_alpha = np.sin(t1) * np.sin(t2) * np.cos(f1 - f2) + np.cos(t1) * np.cos(t2) alpha = np.arccos(np.clip(cos_alpha, -1, 1)) distance = TSUNAMI_MODEL_EARTH_RADIUS_KM * alpha @@ -317,7 +360,7 @@ def _calculate_travel_time( indices = np.arange(n_points + 1)[:, None] - # Bathymetry interpolation expects (lat, lon), not path (lon, lat). + # The interpolator takes (lat, lon), while path points are (lon, lat). path_points = np.array([lon0, lat0]) + indices * delta * vu bath_points = path_points[:, [1, 0]] @@ -327,7 +370,6 @@ def _calculate_travel_time( h = np.abs(self.bathy_interpolator(bath_points)) v = np.sqrt(STANDARD_GRAVITY_M_PER_S2 * h) * 3.6 # Velocity in km/h - # Simpson's rule integration. delta_dist = (alpha / n_points) * TSUNAMI_MODEL_EARTH_RADIUS_KM y = 1 / v integral = ( @@ -338,7 +380,7 @@ def _calculate_travel_time( travel_time = 0.5 * integral - # Empirical corrections match the legacy arrival-time calibration. + # These inherited calibration rules have no source in the repository. if travel_time > 3.0: travel_time = distance / 733 + 0.25 elif 1.4 < travel_time < 3.0: diff --git a/packages/tsdhn/tsdhn/cli/main.py b/packages/tsdhn/tsdhn/cli/main.py index dd559fd..6db9722 100644 --- a/packages/tsdhn/tsdhn/cli/main.py +++ b/packages/tsdhn/tsdhn/cli/main.py @@ -107,7 +107,6 @@ def doctor( runtime = RuntimeContext.resolve( model_dir=model_dir, model_version=model_version, - require_tools=False, ) model_status = "available" model_detail = str(runtime.model_dir) @@ -147,7 +146,6 @@ def calc( runtime = RuntimeContext.resolve( model_dir=model_dir, model_version=model_version, - require_tools=False, ) calculator = TsunamiCalculator(runtime.model_dir) with tempfile.TemporaryDirectory() as tmp: @@ -223,13 +221,13 @@ def on_progress(message: str, details: dict[str, object]) -> None: except Exception as e: progress.stop() console.print(f"[red]Simulation failed:[/red] {e}") - console.print(f"Inspect run directory: [bold]{work_dir}[/bold]") + print(f"Inspect run directory: {work_dir}") raise typer.Exit(code=1) from e console.print(_calculation_table(result.calculation.model_dump())) console.print("[green]Simulation complete.[/green]") - for artifact in result.bundle.artifacts: - console.print(f"{artifact.name}: [bold]{artifact.path}[/bold]") + for output in result.outputs.files: + console.print(f"{output.name}: [bold]{output.path}[/bold]") def main() -> None: diff --git a/packages/tsdhn/tsdhn/deform.py b/packages/tsdhn/tsdhn/deform.py new file mode 100644 index 0000000..9f71905 --- /dev/null +++ b/packages/tsdhn/tsdhn/deform.py @@ -0,0 +1,250 @@ +"""Okada-based vertical-displacement port of `model/def_oka.f`.""" + +import logging +from pathlib import Path + +import numpy as np + +from tsdhn.utils.file_utils import atomic_write + +logger = logging.getLogger(__name__) + +# The legacy grid uses the same spacing on both axes. +_DX = np.float32(7412.9951096) +# These velocities feed the dimensionless rigidity term. +_VP = np.float32(4.82e3) +_VS = np.float32(2.78e3) +_RMU = _VS * _VS / (_VP * _VP - _VS * _VS) +_EPS = np.float32(1.0e-8) +# Compute pi through float32 asin to match the legacy arithmetic. +_PI = np.float32(2.0) * np.arcsin(np.float32(1.0)) +_DEG2RAD = _PI / np.float32(180.0) + + +def _ustrike_fz( + eps: np.floating, + rmu: np.floating, + q: np.ndarray, + cs: np.floating, + sn: np.floating, + xi: np.ndarray, + et: np.ndarray, +) -> np.ndarray: + """Compute the vertical displacement from the strike component.""" + dh = et * sn - q * cs + r = np.sqrt(xi**2 + et**2 + q**2) + ret = r + et + rdh = r + dh + + ret_regular = np.abs(ret) >= eps + cs_regular = np.abs(cs) >= eps + rdh_regular = np.abs(rdh) >= eps + + with np.errstate(divide="ignore", invalid="ignore"): + xi4_reg_csreg = rmu * (np.log(r + dh) - sn * np.log(r + et)) / cs + xi4_reg_cssing = -rmu * q / (r + dh) + xi4_reg = np.where(cs_regular, xi4_reg_csreg, xi4_reg_cssing) + + xi4_sing_csreg_rdhreg = rmu * (np.log(r + dh) + sn * np.log(r - et)) / cs + xi4_sing_csreg_rdhsing = rmu * (-np.log(r - dh) + sn * np.log(r - et)) / cs + xi4_sing_csreg = np.where( + rdh_regular, xi4_sing_csreg_rdhreg, xi4_sing_csreg_rdhsing + ) + xi4_sing_cssing_rdhreg = -rmu * q / (r + dh) + xi4_sing_cssing_rdhsing = np.zeros_like(r) + xi4_sing_cssing = np.where( + rdh_regular, xi4_sing_cssing_rdhreg, xi4_sing_cssing_rdhsing + ) + xi4_sing = np.where(cs_regular, xi4_sing_csreg, xi4_sing_cssing) + + xi4 = np.where(ret_regular, xi4_reg, xi4_sing) + + suz1 = np.where(ret_regular, dh * q / (r * (r + et)), 0.0) + suz2 = np.where(ret_regular, q * sn / (r + et), 0.0) + suz3 = xi4 * sn + fz = suz1 + suz2 + suz3 + + return np.asarray(fz, dtype=np.float32) + + +def _udip_gz( + eps: np.floating, + rmu: np.floating, + q: np.ndarray, + cs: np.floating, + sn: np.floating, + xi: np.ndarray, + et: np.ndarray, +) -> np.ndarray: + """Compute the vertical displacement from the dip component.""" + dh = et * sn - q * cs + r = np.sqrt(xi**2 + et**2 + q**2) + ret = r + et + xx = np.sqrt(xi**2 + q**2) + + ret_regular = np.abs(ret) >= eps + cs_regular = np.abs(cs) >= eps + xi_regular = np.abs(xi) >= eps + r_plus_xi_regular = np.abs(r + xi) >= eps + q_regular = np.abs(q) >= eps + + with np.errstate(divide="ignore", invalid="ignore"): + xi5in = (et * (xx + q * cs) + xx * (r + xx) * sn) / (xi * (r + xx) * cs) + xi5_cs_regular_generic = rmu * 2.0 * np.arctan(xi5in) / cs + + xi5_reg_csreg = np.where(xi_regular, xi5_cs_regular_generic, 0.0) + xi5_reg_cssing = np.where(xi_regular, -rmu * xi * sn / (r + dh), 0.0) + xi5_reg = np.where(cs_regular, xi5_reg_csreg, xi5_reg_cssing) + + xi5_sing_csreg = np.where(xi_regular, xi5_cs_regular_generic, 0.0) + xi5_sing_cssing = np.where(xi_regular, -rmu * xi * sn / (r + dh), 0.0) + xi5_sing = np.where(cs_regular, xi5_sing_csreg, xi5_sing_cssing) + + xi5 = np.where(ret_regular, xi5_reg, xi5_sing) + + uz1 = np.where(r_plus_xi_regular, dh * q / (r * (r + xi)), 0.0) + uz2 = np.where(q_regular, sn * np.arctan(xi * et / (q * r)), 0.0) + uz3 = -xi5 * sn * cs + gz = uz1 + uz2 + uz3 + + return np.asarray(gz, dtype=np.float32) + + +def compute_deform_grid( + I0: int, + J0: int, + D0: float, + L0: float, + W0: float, + TH: float, + DL: float, + RD: float, + HH: float, + IDS: int, + IDE: int, + JDS: int, + JDE: int, +) -> np.ndarray: + """Compute vertical displacement over the requested grid window.""" + ia = IDE - IDS + 1 + ja = JDE - JDS + 1 + i0 = I0 - IDS + 1 + j0 = J0 - JDS + 1 + + st = np.float32(TH) + # The reference offsets these exact strikes to avoid a singular transform. + if st == 0.0 or st == 360.0: + st = st + np.float32(0.001) + di = np.float32(DL) + sl = np.float32(RD) + d0 = np.float32(D0) + l0 = np.float32(L0) + w0 = np.float32(W0) + hh = np.float32(HH) + + str_ = (np.float32(90.0) - st) * _DEG2RAD + dir_ = di * _DEG2RAD + slr = sl * _DEG2RAD + + cs = np.cos(dir_) + sn = np.sin(dir_) + de = hh + w0 * sn + + dst = d0 * np.cos(slr) + ddp = d0 * np.sin(slr) + + cos_str = np.cos(str_) + sin_str = np.sin(str_) + tan_str = np.tan(str_) + + ii = np.arange(ia, dtype=np.float32).reshape(-1, 1) + jj = np.arange(ja, dtype=np.float32).reshape(1, -1) + x0 = (ii - i0) * _DX + y0 = (jj - j0) * _DX + x = x0 / cos_str + (y0 - x0 * tan_str) * sin_str + y = y0 * cos_str - x0 * sin_str + w0 * cs + + p = y * cs + de * sn + q = y * sn - de * cs + + with np.errstate(divide="ignore", invalid="ignore"): + fz1 = _ustrike_fz(_EPS, _RMU, q, cs, sn, x, p) + fz2 = _ustrike_fz(_EPS, _RMU, q, cs, sn, x, p - w0) + fz3 = _ustrike_fz(_EPS, _RMU, q, cs, sn, x - l0, p) + fz4 = _ustrike_fz(_EPS, _RMU, q, cs, sn, x - l0, p - w0) + + gz1 = _udip_gz(_EPS, _RMU, q, cs, sn, x, p) + gz2 = _udip_gz(_EPS, _RMU, q, cs, sn, x, p - w0) + gz3 = _udip_gz(_EPS, _RMU, q, cs, sn, x - l0, p) + gz4 = _udip_gz(_EPS, _RMU, q, cs, sn, x - l0, p - w0) + + uzst = -(fz1 - fz2 - fz3 + fz4) * dst / (2.0 * _PI) + uzdp = -(gz1 - gz2 - gz3 + gz4) * ddp / (2.0 * _PI) + return np.asarray(uzst + uzdp, dtype=np.float32) + + +def clip_anomalous_values(grid: np.ndarray, threshold: float = 20.0) -> np.ndarray: + """Apply the inherited `def_oka.f` outlier guard.""" + anomalous = np.abs(grid) >= threshold + if np.any(anomalous): + logger.warning( + "clip_anomalous_values: %d cell(s) exceeded |Z|>=%.1f, zeroed", + int(np.count_nonzero(anomalous)), + threshold, + ) + return np.asarray(np.where(anomalous, np.float32(0.0), grid), dtype=np.float32) + + +def write_deform_grid(path: Path, grid: np.ndarray) -> None: + """Write the fixed-width grid consumed by the legacy tsunami step.""" + with atomic_write(path) as tmp_path: + np.savetxt(tmp_path, grid, fmt="%9.3f", delimiter="") + + +def _parse_pfalla_inp( + path: Path, +) -> tuple[int, int, float, float, float, float, float, float, float]: + # List-directed Fortran output may wrap a record, so parse tokens globally. + tokens = path.read_text().split() + i0, j0, d0, l0, w0, th, dl, rd, hh = tokens[:9] + return ( + int(float(i0)), + int(float(j0)), + float(d0), + float(l0), + float(w0), + float(th), + float(dl), + float(rd), + float(hh), + ) + + +def _parse_xyo_dat(path: Path) -> tuple[int, int, int, int]: + # The legacy reader consumes four tokens; trailing grid dimensions are + # padding written by the fault-plane step. + ids, ide, jds, jde = path.read_text().split()[:4] + return int(float(ids)), int(float(ide)), int(float(jds)), int(float(jde)) + + +def run_deform(working_dir: Path) -> None: + """Read fault-plane files and write the deformation grid.""" + I0, J0, D0, L0, W0, TH, DL, RD, HH = _parse_pfalla_inp(working_dir / "pfalla.inp") + IDS, IDE, JDS, JDE = _parse_xyo_dat(working_dir / "xyo.dat") + grid = compute_deform_grid( + I0=I0, + J0=J0, + D0=D0, + L0=L0, + W0=W0, + TH=TH, + DL=DL, + RD=RD, + HH=HH, + IDS=IDS, + IDE=IDE, + JDS=JDS, + JDE=JDE, + ) + grid = clip_anomalous_values(grid) + write_deform_grid(working_dir / "deform_a.grd", grid) diff --git a/packages/tsdhn/tsdhn/engine.py b/packages/tsdhn/tsdhn/engine.py index fb18c55..634c43d 100644 --- a/packages/tsdhn/tsdhn/engine.py +++ b/packages/tsdhn/tsdhn/engine.py @@ -8,21 +8,29 @@ from tsdhn.calculator import TsunamiCalculator from tsdhn.domain import CalculationResponse, EarthquakeInput, TsunamiTravelResponse from tsdhn.external import ensure_executables -from tsdhn.pipeline.types import ProcessingStep, ToolRunner +from tsdhn.pipeline.types import ProcessingStep from tsdhn.runtime import RuntimeContext from tsdhn.utils.file_utils import prepare_simulation_workspace -from tsdhn.utils.processing import process_step +from tsdhn.utils.processing import is_step_complete, process_step __all__ = [ - "Artifact", - "ArtifactBundle", + "OutputFile", "ProgressCallback", "SimulationEngine", + "SimulationOutputs", "SimulationRequest", "SimulationResult", "run_simulation", + "step_directory", + "write_simulation_outputs", ] + +def step_directory(work_dir: Path, step: ProcessingStep) -> Path: + """Where `step` runs: the job work_dir, or a subdirectory of it.""" + return work_dir / step.working_dir if step.working_dir else work_dir + + ProgressCallback = Callable[[str, dict[str, Any]], None] @@ -35,24 +43,24 @@ class SimulationRequest: input: EarthquakeInput work_dir: Path model_dir: Path | None = None - tools_dir: Path | None = None model_version: str | None = None + resume: bool = False @dataclass(frozen=True) -class Artifact: +class OutputFile: name: str path: Path content_type: str @dataclass(frozen=True) -class ArtifactBundle: +class SimulationOutputs: root: Path - artifacts: tuple[Artifact, ...] + files: tuple[OutputFile, ...] - def by_name(self) -> dict[str, Artifact]: - return {artifact.name: artifact for artifact in self.artifacts} + def by_name(self) -> dict[str, OutputFile]: + return {output.name: output for output in self.files} @dataclass(frozen=True) @@ -60,7 +68,7 @@ class SimulationResult: calculation: CalculationResponse travel_times: TsunamiTravelResponse runtime: RuntimeContext - bundle: ArtifactBundle + outputs: SimulationOutputs class SimulationEngine: @@ -77,20 +85,14 @@ def run( *, on_progress: ProgressCallback = _noop, ) -> SimulationResult: - required_tools = tuple( - step.runner.executable - for step in self.steps - if isinstance(step.runner, ToolRunner) - ) runtime = RuntimeContext.resolve( model_dir=request.model_dir, - tools_dir=request.tools_dir, model_version=request.model_version, - require_tools=bool(required_tools), - required_tools=required_tools, ) calculator = TsunamiCalculator(runtime.model_dir) - prepare_simulation_workspace(runtime.model_dir, request.work_dir) + prepare_simulation_workspace( + runtime.model_dir, request.work_dir, resume=request.resume + ) on_progress("Running earthquake calculations", {}) calculation = calculator.calculate_earthquake_parameters( @@ -117,19 +119,27 @@ def run( ensure_executables(system_executables) total_steps = len(self.steps) for index, step in enumerate(self.steps, start=1): + step_dir = step_directory(request.work_dir, step) + step_dir.mkdir(parents=True, exist_ok=True) + + if request.resume and is_step_complete(step, step_dir): + on_progress( + f"Skipping completed step {step.name}", + { + "step": step.name, + "step_index": index, + "total_steps": total_steps, + }, + ) + continue + on_progress( f"Processing {step.name}", {"step": step.name, "step_index": index, "total_steps": total_steps}, ) - step_dir = ( - request.work_dir / step.working_dir - if step.working_dir - else request.work_dir - ) - step_dir.mkdir(parents=True, exist_ok=True) - process_step(step, step_dir, runtime.tools_dir) + process_step(step, step_dir) - bundle = write_artifact_bundle( + outputs = write_simulation_outputs( request=request, calculation=calculation, travel_times=travel_times, @@ -137,13 +147,13 @@ def run( ) on_progress( "Simulation completed successfully", - {"artifacts": [a.name for a in bundle.artifacts]}, + {"outputs": [output.name for output in outputs.files]}, ) return SimulationResult( calculation=calculation, travel_times=travel_times, runtime=runtime, - bundle=bundle, + outputs=outputs, ) @@ -152,27 +162,27 @@ def run_simulation( work_dir: Path, *, model_dir: Path | None = None, - tools_dir: Path | None = None, model_version: str | None = None, + resume: bool = False, on_progress: ProgressCallback = _noop, ) -> SimulationResult: request = SimulationRequest( input=data, work_dir=work_dir, model_dir=model_dir, - tools_dir=tools_dir, model_version=model_version, + resume=resume, ) return SimulationEngine().run(request, on_progress=on_progress) -def write_artifact_bundle( +def write_simulation_outputs( *, request: SimulationRequest, calculation: CalculationResponse, travel_times: TsunamiTravelResponse, runtime: RuntimeContext, -) -> ArtifactBundle: +) -> SimulationOutputs: root = request.work_dir _write_json(root / "input.json", request.input.model_dump(mode="json")) _write_json(root / "calculation.json", calculation.model_dump(mode="json")) @@ -183,7 +193,6 @@ def write_artifact_bundle( { "model_dir": str(runtime.model_dir), "model_version": runtime.model_version, - "tools_dir": str(runtime.tools_dir) if runtime.tools_dir else None, "capabilities": { name: { "available": status.available, @@ -196,12 +205,12 @@ def write_artifact_bundle( }, ) - artifacts = [ - Artifact("input", root / "input.json", "application/json"), - Artifact("runtime", root / "runtime.json", "application/json"), - Artifact("calculation", root / "calculation.json", "application/json"), - Artifact("travel_times_json", root / "travel_times.json", "application/json"), - Artifact("travel_times_csv", root / "travel_times.csv", "text/csv"), + outputs = [ + OutputFile("input", root / "input.json", "application/json"), + OutputFile("runtime", root / "runtime.json", "application/json"), + OutputFile("calculation", root / "calculation.json", "application/json"), + OutputFile("travel_times_json", root / "travel_times.json", "application/json"), + OutputFile("travel_times_csv", root / "travel_times.csv", "text/csv"), ] for name, relative_path, content_type in ( ("max_height_map", "maxola.pdf", "application/pdf"), @@ -210,9 +219,9 @@ def write_artifact_bundle( ): path = root / relative_path if path.is_file(): - artifacts.append(Artifact(name, path, content_type)) + outputs.append(OutputFile(name, path, content_type)) - return ArtifactBundle(root=root, artifacts=tuple(artifacts)) + return SimulationOutputs(root=root, files=tuple(outputs)) def _write_json(path: Path, data: object) -> None: diff --git a/packages/tsdhn/tsdhn/fault_plane.py b/packages/tsdhn/tsdhn/fault_plane.py new file mode 100644 index 0000000..750b664 --- /dev/null +++ b/packages/tsdhn/tsdhn/fault_plane.py @@ -0,0 +1,178 @@ +"""Fault placement and file-format port of `model/fault_plane.f90`.""" + +import math +from pathlib import Path + +import numpy as np + +from tsdhn.calculator import ( + average_slip, + calculate_rectangle_parameters, + rupture_dimensions, +) +from tsdhn.utils.file_utils import atomic_write + +_RAKE = 90.0 +_IA = 2461 +_JA = 2056 + + +def _to_0_360(lon: float) -> float: + """Convert a negative longitude to the legacy 0..360 convention.""" + return lon + 360.0 if lon < 0 else lon + + +def _read_hypo_dat(path: Path) -> tuple[str, float, float, float, float]: + """Read the five fields consumed by the legacy fault-plane step.""" + lines = path.read_text().splitlines() + hhmm = lines[0].strip() + lon0 = float(lines[1]) + lat0 = float(lines[2]) + zep_km = float(lines[3]) + mw = float(lines[4]) + return hhmm, lon0, lat0, zep_km, mw + + +def _nearest_mechanism( + mecfoc: np.ndarray, xep: float, yep: float +) -> tuple[float, float]: + """Choose the nearest mechanism in the legacy 0..360 longitude frame. + + This lookup intentionally uses a different longitude frame from the + calculator's preview lookup. The two results can differ near the wrap. + """ + lon = np.where(mecfoc[:, 0] < 0, mecfoc[:, 0] + 360.0, mecfoc[:, 0]) + lat = mecfoc[:, 1] + dist = np.sqrt((lon - xep) ** 2 + (lat - yep) ** 2) + pos = int(np.argmin(dist)) + return float(mecfoc[pos, 2]), float(mecfoc[pos, 3]) + + +def _grid_snap( + xa: np.ndarray, ya: np.ndarray, xo_grid: float, yo: float +) -> tuple[int, int]: + """Return 1-based indices for the nearest bathymetry cells.""" + i0 = int(np.argmin(np.abs(xa - xo_grid))) + 1 + j0 = int(np.argmin(np.abs(ya - yo))) + 1 + return i0, j0 + + +def _grid_window( + xa: np.ndarray, ya: np.ndarray, xep: float, yep: float, l_km: float, mw: float +) -> tuple[int, int, int, int]: + """Return the deformation window in full-grid indices.""" + cte = 1.4 if mw > 8.0 else 2.8 + off = cte * l_km / 111.0 + # The Fortran assignment truncates each geographic bound before snapping. + ids = int(np.argmin(np.abs(xa - int(xep - off)))) + 1 + ide = int(np.argmin(np.abs(xa - int(xep + off)))) + 1 + jds = int(np.argmin(np.abs(ya - int(yep - off)))) + 1 + jde = int(np.argmin(np.abs(ya - int(yep + off)))) + 1 + return ids, ide, jds, jde + + +def _recompute_depth( + lon0: float, lat0: float, xo: float, yo: float, zep_km: float, az: float, dip: float +) -> float: + """Compute the fault's upper-edge depth in meters.""" + delta_x = (lon0 - xo) * 111.0 + delta_y = (lat0 - yo) * 111.0 + h = zep_km - ( + delta_x * math.cos(math.radians(-az)) + delta_y * math.sin(math.radians(-az)) + ) * math.tan(math.radians(dip)) + h_m = h * 1000.0 + if h_m < 0: + # The active Fortran reference substitutes 5000 m for a negative depth. + h_m = 5000.0 + return h_m + + +def _write_pfalla_inp( + path: Path, + i0: int, + j0: int, + d0_m: float, + l0_m: float, + w0_m: float, + az: float, + dip: float, + rake: float, + h_m: float, +) -> None: + """Write the whitespace-separated fault geometry in meters.""" + with atomic_write(path) as tmp_path: + tmp_path.write_text(f"{i0} {j0} {d0_m} {l0_m} {w0_m} {az} {dip} {rake} {h_m}\n") + + +def _write_xyo_dat( + path: Path, ids: int, ide: int, jds: int, jde: int, ia: int = _IA, ja: int = _JA +) -> None: + """Write the grid window plus the legacy trailing grid dimensions. + + The tsunami reader consumes the first four tokens. The final two remain + as padding because legacy producers still write them. + """ + with atomic_write(path) as tmp_path: + tmp_path.write_text(f"{ids} {ide} {jds} {jde} {ia} {ja}\n") + + +def _write_meca_dat( + path: Path, + xep: float, + lat0: float, + zep_km: float, + az: float, + dip: float, + mw: float, + hhmm: str, + rake: float = _RAKE, +) -> None: + """Write the legacy fixed-width `meca.dat` record.""" + fields = "".join(f"{v:7.2f}" for v in (xep, lat0, zep_km, az, dip, rake, mw)) + with atomic_write(path) as tmp_path: + tmp_path.write_text(f"{fields} 0 0 {hhmm}\n") + + +def run_fault_plane(working_dir: Path) -> None: + """Read the workspace inputs and write the fault-plane files.""" + hhmm, lon0, lat0, zep_km, mw = _read_hypo_dat(working_dir / "hypo.dat") + xep = _to_0_360(lon0) + + l_km, w_km = rupture_dimensions(mw) + _, dislocation_m = average_slip(mw, l_km, w_km) + + mecfoc = np.loadtxt(working_dir / "mecfoc.dat") + az, dip = _nearest_mechanism(mecfoc, xep, lat0) + + rect_params, _ = calculate_rectangle_parameters(l_km, w_km, lon0, lat0, az, dip) + xo, yo = float(rect_params["xo"]), float(rect_params["yo"]) + + xa = np.loadtxt(working_dir / "bathy/xa.dat") + ya = np.loadtxt(working_dir / "bathy/ya.dat") + + if xep < xa[0]: + raise RuntimeError( + f"Epicenter is outside the computational grid (xep={xep} < xa[0]={xa[0]})" + ) + + xo_grid = _to_0_360(xo) + i0, j0 = _grid_snap(xa, ya, xo_grid, yo) + h_m = _recompute_depth(lon0, lat0, xo, yo, zep_km, az, dip) + + _write_pfalla_inp( + working_dir / "pfalla.inp", + i0, + j0, + dislocation_m, + l_km * 1000.0, + w_km * 1000.0, + az, + dip, + _RAKE, + h_m, + ) + + ids, ide, jds, jde = _grid_window(xa, ya, xep, lat0, l_km, mw) + _write_xyo_dat(working_dir / "xyo.dat", ids, ide, jds, jde) + + _write_meca_dat(working_dir / "meca.dat", xep, lat0, zep_km, az, dip, mw, hhmm) diff --git a/packages/tsdhn/tsdhn/pipeline/__init__.py b/packages/tsdhn/tsdhn/pipeline/__init__.py index 88c3072..f4307dd 100644 --- a/packages/tsdhn/tsdhn/pipeline/__init__.py +++ b/packages/tsdhn/tsdhn/pipeline/__init__.py @@ -1,17 +1,6 @@ -from tsdhn.pipeline.types import ( - FileCheck, - ProcessingStep, - PythonRunner, - StepFunction, - StepRunner, - ToolRunner, -) +from tsdhn.pipeline.types import ProcessingStep, StepFunction __all__ = [ - "FileCheck", "ProcessingStep", - "PythonRunner", "StepFunction", - "StepRunner", - "ToolRunner", ] diff --git a/packages/tsdhn/tsdhn/pipeline/registry.py b/packages/tsdhn/tsdhn/pipeline/registry.py index cf98343..dca2588 100644 --- a/packages/tsdhn/tsdhn/pipeline/registry.py +++ b/packages/tsdhn/tsdhn/pipeline/registry.py @@ -1,14 +1,17 @@ -from tsdhn.pipeline.types import ProcessingStep, PythonRunner, ToolRunner +# pragma: no cover - This is configuration data, not executable logic. +from tsdhn.deform import run_deform +from tsdhn.fault_plane import run_fault_plane +from tsdhn.pipeline.types import ProcessingStep from tsdhn.render.copy import copy_ttt_pdf from tsdhn.render.maxola import generate_maxola_plot from tsdhn.render.point_ttt import generate_ttt_map from tsdhn.render.ttt_inverso import ttt_inverso_python from tsdhn.render.ttt_max import process_tsunami_data +from tsdhn.tsunami import run_tsunami __all__ = [ "DEFAULT_PIPELINE", "PROCESSING_PIPELINE", - "REPORT_PIPELINE", "TTT_MUNDO_PIPELINE", ] @@ -16,70 +19,53 @@ PROCESSING_PIPELINE: tuple[ProcessingStep, ...] = ( ProcessingStep( name="fault_plane", - outputs=("pfalla.inp",), - runner=ToolRunner("fault_plane"), - file_checks=(("pfalla.inp", "Input file for deform not generated"),), + outputs=("pfalla.inp", "xyo.dat", "meca.dat"), + runner=run_fault_plane, ), ProcessingStep( name="deform", - outputs=("deform",), - runner=ToolRunner("deform"), - file_checks=(("deform", "Deform executable missing"),), + outputs=("deform_a.grd",), + runner=run_deform, ), ProcessingStep( name="tsunami", outputs=("zfolder/green.dat", "zfolder/zmax_a.grd"), - runner=ToolRunner("tsunami"), - file_checks=( - ("zfolder/green.dat", "Green data file missing"), - ("zfolder/zmax_a.grd", "Zmax grid file missing"), - ), + runner=run_tsunami, ), ProcessingStep( name="maxola", outputs=("maxola.pdf",), - runner=PythonRunner(generate_maxola_plot), + runner=generate_maxola_plot, required_system_executables=("gmt",), - file_checks=(("maxola.pdf", "Maxola output missing"),), ), ProcessingStep( name="ttt_max", outputs=("zfolder/green_rev.dat", "ttt_max.dat", "mareograma.svg"), - runner=PythonRunner(process_tsunami_data), - file_checks=( - ("zfolder/green_rev.dat", "Scaled wave height data output missing"), - ("ttt_max.dat", "TTT Max data output missing"), - ("mareograma.svg", "Mareogram plot missing"), - ), + runner=process_tsunami_data, ), ) TTT_MUNDO_PIPELINE: tuple[ProcessingStep, ...] = ( ProcessingStep( name="ttt_inverso", - outputs=("ttt_mundo/ttt.b",), - runner=PythonRunner(ttt_inverso_python), + outputs=("ttt.b",), + runner=ttt_inverso_python, required_system_executables=("gmt", "ttt_client"), working_dir="ttt_mundo", - file_checks=(("ttt.b", "ttt_client output missing"),), ), ProcessingStep( name="point_ttt", - outputs=("ttt_mundo/ttt.pdf",), - runner=PythonRunner(generate_ttt_map), + outputs=("ttt.pdf",), + runner=generate_ttt_map, required_system_executables=("gmt",), working_dir="ttt_mundo", - file_checks=(("ttt.pdf", "ttt.pdf not generated"),), ), ProcessingStep( name="copy_ttt_pdf", - outputs=("ttt.pdf",), - runner=PythonRunner(copy_ttt_pdf), + outputs=("../ttt.pdf",), + runner=copy_ttt_pdf, working_dir="ttt_mundo", - file_checks=(("../ttt.pdf", "ttt.pdf not copied to parent directory"),), ), ) -REPORT_PIPELINE: tuple[ProcessingStep, ...] = () - DEFAULT_PIPELINE: tuple[ProcessingStep, ...] = PROCESSING_PIPELINE + TTT_MUNDO_PIPELINE diff --git a/packages/tsdhn/tsdhn/pipeline/types.py b/packages/tsdhn/tsdhn/pipeline/types.py index e2cb1eb..a68ddf2 100644 --- a/packages/tsdhn/tsdhn/pipeline/types.py +++ b/packages/tsdhn/tsdhn/pipeline/types.py @@ -1,57 +1,24 @@ -import shutil -import subprocess from collections.abc import Callable from dataclasses import dataclass from pathlib import Path -from tsdhn.utils.file_utils import make_executable - type StepFunction = Callable[[Path], None] -type FileCheck = tuple[str, str] @dataclass(frozen=True) -class PythonRunner: - fn: StepFunction - - def run(self, working_dir: Path, tools_dir: Path | None) -> None: - self.fn(working_dir) - - -@dataclass(frozen=True) -class ToolRunner: - executable: str - args: tuple[str, ...] = () - - def run(self, working_dir: Path, tools_dir: Path | None) -> None: - if tools_dir is None: - raise RuntimeError( - f"Tool step '{self.executable}' requires TSDHN_TOOLS_DIR with " - "prebuilt executables." - ) - - source = tools_dir / self.executable - if not source.is_file(): - raise FileNotFoundError(f"Required model executable missing: {source}") - - target = working_dir / self.executable - shutil.copy2(source, target) - make_executable(target) - subprocess.run( - [f"./{self.executable}", *self.args], - cwd=working_dir, - check=True, - ) - - -type StepRunner = PythonRunner | ToolRunner +class ProcessingStep: + """One unit of the simulation pipeline. + `outputs` are paths relative to the step's own working directory (the + job work_dir, or work_dir/`working_dir` when set). They are checked after + the step and included in the completion marker used by resumed runs. + """ -@dataclass(frozen=True) -class ProcessingStep: name: str outputs: tuple[str, ...] - runner: StepRunner + runner: StepFunction required_system_executables: tuple[str, ...] = () - file_checks: tuple[FileCheck, ...] = () working_dir: str | None = None + + def run(self, working_dir: Path) -> None: + self.runner(working_dir) diff --git a/packages/tsdhn/tsdhn/pipeline_version.py b/packages/tsdhn/tsdhn/pipeline_version.py new file mode 100644 index 0000000..696ccd5 --- /dev/null +++ b/packages/tsdhn/tsdhn/pipeline_version.py @@ -0,0 +1,6 @@ +"""Version for resumable pipeline outputs and tsunami checkpoints. + +Bump it when simulation logic changes the on-disk output. +""" + +PIPELINE_VERSION = 1 diff --git a/packages/tsdhn/tsdhn/render/maxola.py b/packages/tsdhn/tsdhn/render/maxola.py index afd793f..467b98a 100644 --- a/packages/tsdhn/tsdhn/render/maxola.py +++ b/packages/tsdhn/tsdhn/render/maxola.py @@ -76,13 +76,12 @@ def create_cpt_files(work_dir: Path) -> tuple[Path, Path]: hgt_cpt = work_dir / "hgt.cpt" try: - # GMT writes CPT files through a temporary path, then the pipeline owns them. with GMTTempFile() as temp_cpt: pygmt.makecpt(cmap="globe", output=temp_cpt.name) shutil.move(temp_cpt.name, depth_cpt) with GMTTempFile() as temp_cpt: - # The polar CPT preserves the legacy blue-to-red wave-height convention. + # Preserve the legacy blue-to-red wave-height scale. pygmt.makecpt( cmap="polar", series="-0.5/0.5/0.01", @@ -91,7 +90,6 @@ def create_cpt_files(work_dir: Path) -> tuple[Path, Path]: ) shutil.move(temp_cpt.name, hgt_cpt) - # GMT uses B, F, and N rows for below-range, above-range, and NaN colors. with open(hgt_cpt, "a") as f: f.write("B 0 0 255\nF 255 0 0\nN 255 255 255\n") @@ -135,6 +133,7 @@ def reshape_model_grid(values: np.ndarray, grid_config: GridConfig) -> np.ndarra def normalize_max_height_grid(max_height_grid: np.ndarray) -> np.ndarray: + """Rescale solver values for display with a 12 m maximum.""" finite_values = max_height_grid[np.isfinite(max_height_grid)] if finite_values.size == 0: return np.zeros_like(max_height_grid, dtype=np.float32) @@ -282,7 +281,7 @@ def generate_maxola_plot(work_dir: Path) -> None: fig = pygmt.Figure() fig.shift_origin(xshift="4.2c", yshift="10.0c") - # Azimuthal projection centered on the Pacific basin. + # Center the projection on the Pacific basin. fig.grdimage( grid=max_height_grid, cmap=hgt_cpt, diff --git a/packages/tsdhn/tsdhn/render/point_ttt.py b/packages/tsdhn/tsdhn/render/point_ttt.py index 4fc49dc..c479dc6 100644 --- a/packages/tsdhn/tsdhn/render/point_ttt.py +++ b/packages/tsdhn/tsdhn/render/point_ttt.py @@ -14,7 +14,7 @@ def generate_ttt_map(working_dir: Path) -> None: projection = "M16c" frame_parameters = ["WsNe", "xa20f10", "ya20f10"] - # GMT binary grid suffixes: cortado.i2 is signed short, ttt.b is binary float. + # These suffixes tell GMT how to decode the legacy binary grids. grd_filename = "cortado.i2" grd_params = "=bs" tttb_filename = "ttt.b" @@ -59,7 +59,7 @@ def generate_ttt_map(working_dir: Path) -> None: shorelines="0.5,30", ) - # Travel-time isolines are annotated in hours to match the legacy TTT map. + # The legacy map labels travel-time isolines in hours. fig.grdcontour( grid=f"{tttb_file}{tttb_params}", region=region, diff --git a/packages/tsdhn/tsdhn/render/ttt_inverso.py b/packages/tsdhn/tsdhn/render/ttt_inverso.py index 0ef407f..632643e 100644 --- a/packages/tsdhn/tsdhn/render/ttt_inverso.py +++ b/packages/tsdhn/tsdhn/render/ttt_inverso.py @@ -2,6 +2,9 @@ import subprocess from pathlib import Path +import pygmt +from pygmt.clib import Session + from tsdhn.external import resolve from tsdhn.render.meca import read_meca_spec @@ -52,15 +55,13 @@ def ttt_inverso_python(working_dir: Path) -> None: logger.exception("ttt_client execution failed: %s", e) raise + # PyGMT does not expose grdmath. The no-op rewrite makes the ttt_client + # grid header readable by the later GMT contour step. + grid_arg = f"{working_dir / 'ttt.b'}=bf" try: - subprocess.run( - [str(resolve("gmt")), "grdmath", "ttt.b=bf", "1.0", "MUL", "=", "ttt.b=bf"], - cwd=working_dir, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) + with Session() as session: + session.call_module("grdmath", [grid_arg, "1.0", "MUL", "=", grid_arg]) logger.info("grdmath executed successfully.") - except subprocess.CalledProcessError as e: + except pygmt.exceptions.GMTError as e: logger.exception("grdmath execution failed: %s", e) raise diff --git a/packages/tsdhn/tsdhn/render/ttt_max.py b/packages/tsdhn/tsdhn/render/ttt_max.py index cc9fb1d..56a929e 100644 --- a/packages/tsdhn/tsdhn/render/ttt_max.py +++ b/packages/tsdhn/tsdhn/render/ttt_max.py @@ -60,7 +60,7 @@ def process_tsunami_data(working_dir: Path) -> None: for i, name in enumerate(station_names): station_data[name] = data[:, i + 1] - # Green's relation adjusts synthetic amplitudes to each station depth. + # Adjust synthetic amplitudes for each station's depth. scaling_factors = { "cruz": (12.9 / 14.0) ** (0.25), "tala": (183.1 / 20.0) ** (0.25), @@ -99,7 +99,7 @@ def process_tsunami_data(working_dir: Path) -> None: logger.error(f"Error writing green_rev.dat: {e}") raise OSError(f"Failed to write green_rev.dat: {e}") from e - # ttt_max.dat stores first positive sample index and peak amplitude. + # The report needs the first positive sample and peak amplitude. first_nonzero = {} max_values = {} for name in station_names: @@ -123,7 +123,7 @@ def process_tsunami_data(working_dir: Path) -> None: maxmax = max(max_values.values()) logger.info(f"Maximum wave height: {maxmax}") - # The legacy report expects a scale bucket that contains the largest wave. + # Choose a scale that contains the largest wave. if maxmax <= 0.1: plot_mareograma(scale=0.1, tick_type="small") elif maxmax <= 0.2: @@ -189,8 +189,10 @@ def y_to_px(value: float, top: float) -> float: ".series{fill:none;stroke:#1d4ed8;stroke-width:1.5}" ) elements = [ - '', + ( + '' + ), '', style, ] diff --git a/packages/tsdhn/tsdhn/runtime.py b/packages/tsdhn/tsdhn/runtime.py index ea914ae..40da20a 100644 --- a/packages/tsdhn/tsdhn/runtime.py +++ b/packages/tsdhn/tsdhn/runtime.py @@ -8,12 +8,12 @@ __all__ = [ "REQUIRED_MODEL_DIRS", "REQUIRED_MODEL_FILES", - "REQUIRED_TOOL_EXECUTABLES", + "SYSTEM_EXECUTABLES", "CapabilityStatus", "RuntimeContext", - "tools_dir_from_env", + "check_capabilities", + "resolve_model_dir", "validate_model_dir", - "validate_tools_dir", ] REQUIRED_MODEL_DIRS: tuple[str, ...] = ("bathy", "ttt_mundo") @@ -28,7 +28,9 @@ "bathy/ya.dat", "ttt_mundo/cortado.i2", ) -REQUIRED_TOOL_EXECUTABLES: tuple[str, ...] = ("fault_plane", "deform", "tsunami") + +# Report steps resolve these tools from PATH. +SYSTEM_EXECUTABLES: tuple[str, ...] = ("gmt", "ttt_client") @dataclass(frozen=True) @@ -43,7 +45,6 @@ class CapabilityStatus: @dataclass(frozen=True) class RuntimeContext: model_dir: Path - tools_dir: Path | None model_version: str capabilities: dict[str, CapabilityStatus] @@ -51,31 +52,17 @@ class RuntimeContext: def resolve( cls, model_dir: Path | None = None, - tools_dir: Path | None = None, *, - require_tools: bool = True, - required_tools: tuple[str, ...] | None = None, model_version: str | None = None, ) -> RuntimeContext: resolved_model_dir, resolved_model_version = resolve_model_dir( model_dir=model_dir, model_version=model_version, ) - resolved_tools_dir = None - if require_tools: - tools_path = ( - tools_dir.resolve() if tools_dir is not None else _tools_dir_env_path() - ) - resolved_tools_dir = validate_tools_dir( - tools_path, - required_tools=required_tools, - ) - capabilities = check_capabilities(required_tools=required_tools or ()) return cls( model_dir=resolved_model_dir, - tools_dir=resolved_tools_dir, model_version=resolved_model_version, - capabilities=capabilities, + capabilities=check_capabilities(), ) @@ -102,19 +89,6 @@ def resolve_model_dir( ) -def tools_dir_from_env() -> Path: - return validate_tools_dir(_tools_dir_env_path()) - - -def _tools_dir_env_path() -> Path: - value = os.environ.get("TSDHN_TOOLS_DIR") - if not value: - raise RuntimeError( - "TSDHN_TOOLS_DIR must be set to the prebuilt model executable directory." - ) - return Path(value).resolve() - - def validate_model_dir(path: Path) -> Path: missing_dirs = [ dirname for dirname in REQUIRED_MODEL_DIRS if not (path / dirname).is_dir() @@ -130,26 +104,11 @@ def validate_model_dir(path: Path) -> Path: return path -def validate_tools_dir( - path: Path, *, required_tools: tuple[str, ...] | None = None -) -> Path: - required = required_tools or REQUIRED_TOOL_EXECUTABLES - missing = [ - executable for executable in required if not (path / executable).is_file() - ] - if missing: - raise RuntimeError( - f"Invalid TSDHN tools directory '{path}'. Missing: {', '.join(missing)}" - ) - return path - - def check_capabilities( *, - required_tools: tuple[str, ...] = (), + executables: tuple[str, ...] = SYSTEM_EXECUTABLES, ) -> dict[str, CapabilityStatus]: - names = tuple(dict.fromkeys((*required_tools, "gmt", "ttt_client"))) - return {name: _check_executable(name) for name in names} + return {name: _check_executable(name) for name in executables} def _check_executable(name: str) -> CapabilityStatus: diff --git a/packages/tsdhn/tsdhn/tsunami.py b/packages/tsdhn/tsdhn/tsunami.py new file mode 100644 index 0000000..e73f0a5 --- /dev/null +++ b/packages/tsdhn/tsdhn/tsunami.py @@ -0,0 +1,399 @@ +"""Linear shallow-water propagation port of `model/tsunami1.for`.""" + +import logging +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import numba +import numpy as np + +from tsdhn.pipeline_version import PIPELINE_VERSION +from tsdhn.utils.file_utils import atomic_write + +logger = logging.getLogger(__name__) + +# Numba lacks type stubs for prange; keep this alias typed for mypy. +prange = cast("Callable[[int, int], range]", numba.prange) + + +def _jit[F: Callable[..., None]](fn: F) -> F: + # Keep IEEE float32 evaluation order. Parallel writes are independent. + return cast("F", numba.njit(cache=True, parallel=True)(fn)) + + +IA = 2461 +JA = 2056 +_DELTA = np.float32(240.0) / np.float32(3600.0) +_DT = np.float32(3.0) +KE = 33602 +KD = 20 +NG = 17 +_RT = np.float32(6.37e6) +_BLATA = np.float32(-76.006) +# Compute pi through float32 atan to match legacy Fortran arithmetic (like deform.py). +_PI = np.float32(4.0) * np.arctan(np.float32(1.0)) +_DA = _PI * _DELTA / np.float32(180.0) +_GG = np.float32(9.8) +_FLUSH = np.float32(1.0e-5) + + +def _read_xyo_dat(path: Path) -> tuple[int, int, int, int]: + # Parse the first four tokens; trailing values are padding from fault-plane. + ids, ide, jds, jde = path.read_text().split()[:4] + return int(float(ids)), int(float(ide)), int(float(jds)), int(float(jde)) + + +def _read_tidal_dat(path: Path) -> tuple[np.ndarray, np.ndarray]: + # The solver uses gauge indices. Gauge names are metadata only. + ip = np.empty(NG, dtype=np.int64) + jp = np.empty(NG, dtype=np.int64) + lines = [line for line in path.read_text().splitlines() if line.strip()] + for gauge, line in enumerate(lines[:NG]): + _name, i_text, j_text = line.split()[:3] + ip[gauge] = int(i_text) + jp[gauge] = int(j_text) + return ip, jp + + +def _read_grid_a(path: Path) -> np.ndarray: + # Preserve the legacy shallow-water floor for positive depths below 10 m. + grid = np.loadtxt(path, dtype=np.float32) + if grid.shape != (IA, JA): + raise ValueError(f"Unexpected grid_a.grd shape: {grid.shape}") + grid[(grid > 0.0) & (grid < 10.0)] = np.float32(10.0) + return grid + + +def _read_deform_a(path: Path, ids: int, ide: int, jds: int, jde: int) -> np.ndarray: + # The file contains the initial elevation for the requested grid window. + values = np.array(path.read_text().split(), dtype=np.float32) + rows = ide - ids + 1 + cols = jde - jds + 1 + if values.size != rows * cols: + raise ValueError( + f"Unexpected deform_a.grd size: {values.size} values, " + f"expected {rows}x{cols} for window {ids}:{ide},{jds}:{jde}" + ) + return values.reshape(rows, cols) + + +def _hmn(h: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Return depths at the staggered discharge nodes. + + HM averages along I (last row keeps H itself), HN along J (last column + keeps H itself). + """ + hm = np.empty_like(h) + hn = np.empty_like(h) + half = np.float32(0.5) + hm[:-1, :] = half * (h[:-1, :] + h[1:, :]) + hm[-1, :] = h[-1, :] + hn[:, :-1] = half * (h[:, :-1] + h[:, 1:]) + hn[:, -1] = h[:, -1] + return hm, hn + + +def _prelim( + hm: np.ndarray, hn: np.ndarray +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Build latitude and momentum factors for the solver. + + RZ and RN advance by repeated float32 addition of DA. The scalar walk + preserves the accumulated rounding of the reference solver. + """ + ja = hm.shape[1] + rx = np.empty(ja, dtype=np.float32) + cj = np.empty(ja, dtype=np.float32) + rz = _BLATA * _PI / np.float32(180.0) + rn = rz + _DA / np.float32(2.0) + for j in range(ja): + rx[j] = _DT / (_RT * np.cos(rz) * _DA) + cj[j] = np.cos(rn) + rz += _DA + rn += _DA + xx = (rx * _GG)[np.newaxis, :] * hm + yy = _DT * _GG * hn / (_RT * _DA) + return rx, cj, xx, yy + + +@_jit +def _mass( + z1: np.ndarray, + z2: np.ndarray, + m1: np.ndarray, + n1: np.ndarray, + h: np.ndarray, + rx: np.ndarray, + cj: np.ndarray, +) -> None: + """Apply continuity to wet interior cells and flush small elevations.""" + ia, ja = h.shape + zero = np.float32(0.0) + for i in prange(1, ia): + for j in range(1, ja): + if h[i, j] > zero: + z = ( + z1[i, j] + - rx[j] * (m1[i, j] - m1[i - 1, j]) + - rx[j] * (n1[i, j] * cj[j] - n1[i, j - 1] * cj[j - 1]) + ) + if abs(z) < _FLUSH: + z = zero + z2[i, j] = z + else: + z2[i, j] = zero + + +@_jit +def _bout( + z2: np.ndarray, + m_prev: np.ndarray, + n_prev: np.ndarray, + h: np.ndarray, +) -> None: + """Apply open-boundary radiation on the four grid edges. + + The edge passes run in a fixed order, so the second pass owns a corner. + The buffer-swap port passes previous-step momentum explicitly. + """ + ia, ja = h.shape + half = np.float32(0.5) + zero = np.float32(0.0) + for pass_index in range(2): + j = 1 if pass_index == 0 else ja - 1 + for i in prange(1, ia - 1): + if h[i, j] < zero: + continue + cc = np.sqrt(_GG * h[i, j]) + uu = half * abs(m_prev[i, j] + m_prev[i - 1, j]) + n_edge = n_prev[i, j] if pass_index == 0 else n_prev[i, j - 1] + uu = np.sqrt(uu * uu + n_edge * n_edge) + zz = uu / cc + if (pass_index == 0 and n_edge > zero) or ( + pass_index == 1 and n_edge < zero + ): + zz = -zz + z2[i, j] = zz + for pass_index in range(2): + i = 1 if pass_index == 0 else ia - 1 + for j in prange(1, ja - 1): + if h[i, j] < zero: + continue + cc = np.sqrt(_GG * h[i, j]) + uu = half * abs(n_prev[i, j] + n_prev[i, j - 1]) + m_edge = m_prev[i, j] if pass_index == 0 else m_prev[i - 1, j] + uu = np.sqrt(uu * uu + m_edge * m_edge) + zz = uu / cc + if (pass_index == 0 and m_edge > zero) or ( + pass_index == 1 and m_edge < zero + ): + zz = -zz + z2[i, j] = zz + + +@_jit +def _mmnt( + z2: np.ndarray, + m1: np.ndarray, + m2: np.ndarray, + n1: np.ndarray, + n2: np.ndarray, + h: np.ndarray, + xx: np.ndarray, + yy: np.ndarray, +) -> None: + """Update momentum from the new elevation across wet cells.""" + ia, ja = h.shape + zero = np.float32(0.0) + for i in prange(1, ia - 1): + for j in range(1, ja): + if h[i, j] > zero and h[i + 1, j] > zero: + m = m1[i, j] - xx[i, j] * (z2[i + 1, j] - z2[i, j]) + if abs(m) < _FLUSH: + m = zero + m2[i, j] = m + else: + m2[i, j] = zero + for i in prange(1, ia): + for j in range(1, ja - 1): + if h[i, j] > zero and h[i, j + 1] > zero: + n = n1[i, j] - yy[i, j] * (z2[i, j + 1] - z2[i, j]) + if abs(n) < _FLUSH: + n = zero + n2[i, j] = n + else: + n2[i, j] = zero + + +def _write_green_dat(path: Path, rows: list[tuple[float, np.ndarray]]) -> None: + # Reports read the legacy fixed-width gauge format. + with atomic_write(path) as tmp_path, tmp_path.open("w") as handle: + for minutes, gauges in rows: + handle.write( + f"{minutes:7.1f}" + "".join(f"{z:7.3f}" for z in gauges) + "\n" + ) + + +def _write_zmax_a(path: Path, zmxa: np.ndarray) -> None: + # Reports read the legacy fixed-width maximum-height grid. + with atomic_write(path) as tmp_path: + np.savetxt(tmp_path, zmxa, fmt="%8.3f", delimiter="") + + +# Checkpoints contain every time level needed to resume after a worker restart. +_CHECKPOINT_INTERVAL = 2000 + + +def _checkpoint_path(working_dir: Path) -> Path: + return working_dir / "zfolder" / "_checkpoint.npz" + + +def _write_checkpoint( + path: Path, + k: int, + z1: np.ndarray, + z2: np.ndarray, + m1: np.ndarray, + m2: np.ndarray, + n1: np.ndarray, + n2: np.ndarray, + zmxa: np.ndarray, + gauge_rows: list[tuple[float, np.ndarray]], +) -> None: + gauge_minutes = np.array([minutes for minutes, _ in gauge_rows], dtype=np.float64) + gauge_values = ( + np.stack([gauges for _, gauges in gauge_rows]) + if gauge_rows + else np.empty((0, 0), dtype=np.float32) + ) + with atomic_write(path) as tmp_path, tmp_path.open("wb") as handle: + np.savez( + handle, + pipeline_version=np.int64(PIPELINE_VERSION), + k=np.int64(k), + z1=z1, + z2=z2, + m1=m1, + m2=m2, + n1=n1, + n2=n2, + zmxa=zmxa, + gauge_minutes=gauge_minutes, + gauge_values=gauge_values, + ) + + +type _TsunamiState = tuple[ + int, # k, the last fully-completed step + np.ndarray, # z1 + np.ndarray, # z2 + np.ndarray, # m1 + np.ndarray, # m2 + np.ndarray, # n1 + np.ndarray, # n2 + np.ndarray, # zmxa + list[tuple[float, np.ndarray]], # gauge_rows +] + + +def _read_checkpoint( + path: Path, *, ia: int, ja: int, n_gauges: int +) -> _TsunamiState | None: + if not path.is_file(): + return None + try: + with np.load(path) as data: + if int(data["pipeline_version"]) != PIPELINE_VERSION: + raise ValueError("checkpoint predates a pipeline version bump") + k = int(data["k"]) + z1, z2 = data["z1"], data["z2"] + m1, m2 = data["m1"], data["m2"] + n1, n2 = data["n1"], data["n2"] + zmxa = data["zmxa"] + gauge_minutes = data["gauge_minutes"] + gauge_values = data["gauge_values"] + + for array in (z1, z2, m1, m2, n1, n2, zmxa): + if array.shape != (ia, ja) or array.dtype != np.float32: + raise ValueError("checkpoint array shape/dtype mismatch") + if gauge_values.shape[0] and gauge_values.shape[1] != n_gauges: + raise ValueError("checkpoint gauge count mismatch") + if not (0 <= k <= KE): + raise ValueError("checkpoint step index out of range") + except Exception: + logger.warning( + "tsunami checkpoint at %s is unusable, starting from step 0", path + ) + return None + + gauge_rows = list(zip(gauge_minutes.tolist(), gauge_values, strict=True)) + return k, z1, z2, m1, m2, n1, n2, zmxa, gauge_rows + + +def run_tsunami(working_dir: Path) -> None: + """Run the tsunami step and write the gauge and maximum-height grids.""" + zfolder = working_dir / "zfolder" + zfolder.mkdir(exist_ok=True) + + ids, ide, jds, jde = _read_xyo_dat(working_dir / "xyo.dat") + ip, jp = _read_tidal_dat(working_dir / "tidal.dat") + h = _read_grid_a(working_dir / "bathy" / "grid_a.grd") + + hm, hn = _hmn(h) + rx, cj, xx, yy = _prelim(hm, hn) + + checkpoint_path = _checkpoint_path(working_dir) + checkpoint = _read_checkpoint(checkpoint_path, ia=IA, ja=JA, n_gauges=len(ip)) + + if checkpoint is not None: + start_k, z1, z2, m1, m2, n1, n2, zmxa, gauge_rows = checkpoint + logger.info("resuming tsunami run from step %d of %d", start_k, KE) + else: + start_k = 0 + # The first Fortran row and column remain outside the interior update. + z1 = np.zeros((IA, JA), dtype=np.float32) + z2 = np.zeros((IA, JA), dtype=np.float32) + m1 = np.zeros((IA, JA), dtype=np.float32) + m2 = np.zeros((IA, JA), dtype=np.float32) + n1 = np.zeros((IA, JA), dtype=np.float32) + n2 = np.zeros((IA, JA), dtype=np.float32) + zmxa = np.zeros((IA, JA), dtype=np.float32) + + # Deformation occupies only the one-based window written by fault_plane. + z1[ids - 1 : ide, jds - 1 : jde] = _read_deform_a( + working_dir / "deform_a.grd", ids, ide, jds, jde + ) + + gauge_rows = [] + + gauge_i = ip - 1 + gauge_j = jp - 1 + + for k in range(start_k + 1, KE + 1): + kk = k - 1 + if k % 1000 == 0: + logger.info("tsunami step %d of %d", k, KE) + + _mass(z1, z2, m1, n1, h, rx, cj) + _bout(z2, m1, n1, h) + _mmnt(z2, m1, m2, n1, n2, h, xx, yy) + + # The maximum grid is sampled with gauges, not at every solver step. + if kk % KD == 0: + gauge_rows.append((kk * float(_DT) / 60.0, z2[gauge_i, gauge_j].copy())) + np.maximum(zmxa, z2, out=zmxa) + + # Boundary radiation has already read the previous momentum buffers. + z1, z2 = z2, z1 + m1, m2 = m2, m1 + n1, n2 = n2, n1 + + if k % _CHECKPOINT_INTERVAL == 0: + _write_checkpoint( + checkpoint_path, k, z1, z2, m1, m2, n1, n2, zmxa, gauge_rows + ) + + _write_green_dat(zfolder / "green.dat", gauge_rows) + _write_zmax_a(zfolder / "zmax_a.grd", zmxa) + checkpoint_path.unlink(missing_ok=True) diff --git a/packages/tsdhn/tsdhn/utils/file_utils.py b/packages/tsdhn/tsdhn/utils/file_utils.py index 3ca6b43..e1579e0 100644 --- a/packages/tsdhn/tsdhn/utils/file_utils.py +++ b/packages/tsdhn/tsdhn/utils/file_utils.py @@ -1,6 +1,12 @@ +import contextlib +import os import shutil -from collections.abc import Sequence +from collections.abc import Iterator from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from tsdhn.pipeline.types import ProcessingStep WORKSPACE_DIRS: tuple[str, ...] = ("bathy", "ttt_mundo", "zfolder") WORKSPACE_INPUTS: tuple[str, ...] = ( @@ -17,34 +23,55 @@ def make_executable(file_path: Path) -> None: file_path.chmod(file_path.stat().st_mode | 0o111) -def validate_files(cwd: Path, checks: Sequence[tuple[str, str]]) -> None: - missing = [] - for filename, msg in checks: - if not (cwd / filename).exists(): - missing.append(f"{filename}: {msg}") - if missing: - raise FileNotFoundError("\n".join(missing)) +@contextlib.contextmanager +def atomic_write(path: Path) -> Iterator[Path]: + """Yield a sibling temp path; rename it onto `path` only on clean exit. + + A killed writer must not leave a truncated file that passes the existence + check used by resume and skip logic. `os.replace` is atomic on POSIX, so + readers see the old file, no file, or the complete new file. + """ + tmp_path = path.parent / f"{path.name}.tmp" + try: + yield tmp_path + except BaseException: + tmp_path.unlink(missing_ok=True) + raise + else: + os.replace(tmp_path, path) -def prepare_simulation_workspace(model_dir: Path, work_dir: Path) -> None: - """Create the per-run filesystem expected by the legacy pipeline. +def validate_outputs(working_dir: Path, step: ProcessingStep) -> None: + """Raise if any of `step`'s declared outputs is absent from disk. - Only immutable inputs required by the active pipeline are linked/copied into - the workspace. Generated files are intentionally absent at startup. + Paths are resolved relative to `working_dir`, which is the step's own + directory (see ProcessingStep.outputs). """ - if work_dir.exists(): + missing = [name for name in step.outputs if not (working_dir / name).exists()] + if missing: + raise FileNotFoundError( + f"Step '{step.name}' did not produce: {', '.join(missing)}" + ) + + +def prepare_simulation_workspace( + model_dir: Path, work_dir: Path, *, resume: bool = False +) -> None: + """Create a clean simulation workspace, unless `resume` is true.""" + if work_dir.exists() and not resume: shutil.rmtree(work_dir) - work_dir.mkdir(parents=True) + work_dir.mkdir(parents=True, exist_ok=True) for dirname in WORKSPACE_DIRS: - (work_dir / dirname).mkdir() + (work_dir / dirname).mkdir(exist_ok=True) for relative_name in WORKSPACE_INPUTS: source = model_dir / relative_name if not source.is_file(): raise FileNotFoundError(f"Required model input missing: {source}") destination = work_dir / relative_name - _link_or_copy(source, destination) + if not destination.exists(): + _link_or_copy(source, destination) def _link_or_copy(source: Path, destination: Path) -> None: diff --git a/packages/tsdhn/tsdhn/utils/geo.py b/packages/tsdhn/tsdhn/utils/geo.py index 332449d..f2d7d77 100644 --- a/packages/tsdhn/tsdhn/utils/geo.py +++ b/packages/tsdhn/tsdhn/utils/geo.py @@ -44,14 +44,12 @@ def determine_tsunami_warning(Mw: float, h: float, h0: float, dist_min: float) - if h0 > 0: if dist_min < 50: return LAND_NEAR_COAST_WARNING - if dist_min > 50: - return LAND_NO_TSUNAMI_WARNING - elif h0 <= 0: - if h > 60 or Mw < 7.0: - return SEA_NO_TSUNAMI_WARNING - for threshold, message in SEA_TSUNAMI_TIERS: - if Mw >= threshold: - return message + return LAND_NO_TSUNAMI_WARNING + if h > 60 or Mw < 7.0: + return SEA_NO_TSUNAMI_WARNING + for threshold, message in SEA_TSUNAMI_TIERS: + if Mw >= threshold: + return message return NO_TSUNAMI_WARNING diff --git a/packages/tsdhn/tsdhn/utils/processing.py b/packages/tsdhn/tsdhn/utils/processing.py index 8f4a8fe..bddc1fc 100644 --- a/packages/tsdhn/tsdhn/utils/processing.py +++ b/packages/tsdhn/tsdhn/utils/processing.py @@ -1,13 +1,36 @@ +import hashlib from pathlib import Path from tsdhn.pipeline.types import ProcessingStep -from tsdhn.utils.file_utils import validate_files +from tsdhn.pipeline_version import PIPELINE_VERSION +from tsdhn.utils.file_utils import atomic_write, validate_outputs -__all__ = ["process_step"] +__all__ = ["is_step_complete", "process_step"] -def process_step( - step: ProcessingStep, working_dir: Path, tools_dir: Path | None -) -> None: - step.runner.run(working_dir, tools_dir) - validate_files(working_dir, step.file_checks) +def _marker_path(working_dir: Path, step: ProcessingStep) -> Path: + return working_dir / f".{step.name}.complete" + + +def _fingerprint(step: ProcessingStep) -> str: + payload = f"{PIPELINE_VERSION}:{step.name}:{','.join(step.outputs)}" + return hashlib.sha256(payload.encode()).hexdigest()[:16] + + +def is_step_complete(step: ProcessingStep, working_dir: Path) -> bool: + """Return whether a resumed run can skip `step`.""" + marker = _marker_path(working_dir, step) + if not marker.is_file() or marker.read_text().strip() != _fingerprint(step): + return False + try: + validate_outputs(working_dir, step) + except FileNotFoundError: + return False + return True + + +def process_step(step: ProcessingStep, working_dir: Path) -> None: + step.run(working_dir) + validate_outputs(working_dir, step) + with atomic_write(_marker_path(working_dir, step)) as tmp_path: + tmp_path.write_text(_fingerprint(step)) diff --git a/pyproject.toml b/pyproject.toml index 570aefe..220694c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,11 +10,12 @@ authors = [ ] [tool.uv.workspace] -members = ["packages/tsdhn", "packages/api"] +members = ["packages/tsdhn", "packages/api", "packages/tsdhn-parity"] [tool.uv.sources] tsdhn = { workspace = true } tsdhn-api = { workspace = true } +tsdhn-parity = { workspace = true } [dependency-groups] dev = [ @@ -27,6 +28,7 @@ dev = [ "pytest-cov==7.1.0", "httpx2==2.5.0", "coverage==7.15.0", + "diff-cover==10.5.1", "bandit==1.9.4", "pip-audit==2.10.1", ] @@ -51,14 +53,16 @@ extend-exclude = ["__pycache__", "dist", "build"] select = ["E", "F", "W", "I", "B", "UP", "S", "SIM", "RUF"] [tool.ruff.lint.isort] -known-first-party = ["tsdhn", "api"] +known-first-party = ["tsdhn", "api", "tsdhn_parity"] [tool.ruff.lint.per-file-ignores] "**/tests/**/*.py" = ["S101", "S105", "S106"] # asserts and dummy tokens are fine in tests # S603 is suppressed: every subprocess call uses arguments from # validated floats, local Path objects, or frozen dataclass fields. -"packages/tsdhn/tsdhn/pipeline/types.py" = ["S603"] +"packages/tsdhn-parity/tsdhn_parity/adapters/fortran.py" = ["S603"] "packages/tsdhn/tsdhn/render/ttt_inverso.py" = ["S603"] +"packages/tsdhn/tests/test_pipeline_golden.py" = ["S603"] +"scripts/capture_matlab_fixtures.py" = ["S603"] [tool.ruff.format] quote-style = "double" @@ -74,11 +78,14 @@ disallow_untyped_defs = true [tool.coverage.run] branch = true -source = ["packages"] +# tsdhn-parity is a comparison harness against legacy MATLAB/Fortran +# reference outputs, not shipped product code (see its readme). Whether +# the harness itself is well-tested is a different question from whether +# the product is, so it is not measured here. +source = ["packages/tsdhn/tsdhn", "packages/api/api"] omit = ["**/tests/*", "**/__init__.py"] [tool.coverage.report] -fail_under = 50 exclude_lines = [ "pragma: no cover", "if TYPE_CHECKING:", @@ -87,5 +94,10 @@ exclude_lines = [ ] [tool.pytest.ini_options] -testpaths = ["packages/tsdhn/tests", "packages/api/tests"] +testpaths = ["packages/tsdhn/tests", "packages/api/tests", "packages/tsdhn-parity/tests"] +pythonpath = ["."] addopts = "-ra" +markers = [ + "golden: requires the real compiled toolchain; see mise run test-golden", + "integration: uses disposable databases on the mise-managed PostgreSQL cluster; see mise run test-integration", +] diff --git a/readme.md b/readme.md index e8823fe..bc45748 100644 --- a/readme.md +++ b/readme.md @@ -1,171 +1,88 @@ # TSDHN - -
+TSDHN runs tsunami simulations from earthquake source parameters. -[![CI](https://github.com/totallynotdavid/tsdhn/actions/workflows/ci.yml/badge.svg?branch=master&event=push)](https://github.com/totallynotdavid/tsdhn/actions/workflows/ci.yml) -[![Security](https://github.com/totallynotdavid/tsdhn/actions/workflows/security.yml/badge.svg)](https://github.com/totallynotdavid/tsdhn/actions/workflows/security.yml) -[![OpenSSF Scorecard](https://img.shields.io/ossf-scorecard/github.com/totallynotdavid/tsdhn?label=scorecard)](https://scorecard.dev/viewer/?uri=github.com/totallynotdavid/tsdhn) +The repository contains the Python simulation engine and researcher CLI, the +FastAPI compute service and worker, the SvelteKit web app, the generated +TypeScript client, and tools that compare the Python results with the older +MATLAB and Fortran programs. -
- +## Get started -TSDHN runs tsunami simulations from earthquake source -parameters. The repository contains one shared `tsdhn` simulation engine and -CLI for researchers, a FastAPI service, a Procrastinate worker, a SvelteKit web -app, and the generated TypeScript API client used by the web server. +Install the pinned tools and project dependencies: -## Documentation - -| Area | Docs | Covers | -| --- | --- | --- | -| Python engine + CLI | [`packages/tsdhn`](./packages/tsdhn/readme.md) | Calculations, runtime paths, model assets, pipeline execution | -| API service | [`packages/api`](./packages/api/readme.md) | FastAPI routes, service-token auth, worker entry point | -| Web app | [`apps/web`](./apps/web/readme.md) | SvelteKit app, auth, database, and server-side backend configuration | -| API client | [`libs/api-client`](./libs/api-client/readme.md) | OpenAPI schema and generated TypeScript types | - -> [!TIP] -> Start with the component README for the package or app you are changing. The root -> README gives orientation and shared commands; component READMEs carry the -> exact usage details for their own layer. - -## Architecture - -```mermaid -flowchart LR - Browser[Browser] --> Web[SvelteKit web app] - Web -->|server-side Bearer token| API[FastAPI /api/v1] - API -->|create job + defer task| PG[(Compute Postgres)] - Worker[Procrastinate worker] -->|claim task + update status| PG - Worker --> Engine[tsdhn engine] - CLI[tsdhn CLI] --> Engine - Engine --> Model[versioned model assets] - Engine --> Tools[Fortran, GMT, TTT] - Worker --> MinIO[(MinIO artifacts)] - Worker --> Jobs[(temporary jobs directory)] - Web --> DB[(SQLite/libSQL)] +```sh +mise install +mise run install +mise run web-install ``` -The browser talks to the SvelteKit app. The SvelteKit server calls the FastAPI -backend with `BACKEND_SERVICE_TOKEN`; that token is never sent to browser code. -Long simulations run in the Procrastinate worker, which calls the shared -`tsdhn` engine, updates `compute_jobs` in Postgres, and writes artifacts and -metadata to MinIO. - -## Quick start - -The backend stack is self-hosted because the simulation runtime needs the -Fortran/GMT/TTT toolchain. Compose uses the images and Dockerfiles under -[`deploy/`](./deploy/). [Podman](https://podman.io/) is the default local -engine; `docker compose` works identically and is what CI uses. - -Install `podman` plus the `docker-compose-plugin` package, then enable the -rootless socket once per machine: +Run the researcher CLI without starting the services: ```sh -systemctl --user enable --now podman.socket +uv run tsdhn assets install +uv run tsdhn doctor +uv run tsdhn calc --mw 8.0 --lat -20.5 --lon -70.5 ``` +Create `.env`, set `COMPUTE_API_TOKEN`, `BETTER_AUTH_SECRET`, and +`APP_DB_PASSWORD`, then run the self-hosted stack: + ```sh cp .env.example .env mise run dev-up -``` - -Set `BACKEND_SERVICE_TOKEN` and `BETTER_AUTH_SECRET` in `.env` before running -the web profile: - -```sh mise run dev-web ``` -Rootless containers stop when you log out. Run `loginctl enable-linger -$(whoami)` once if you want the stack to survive logout or reboot. - -For local development, install the pinned tools with -[mise](https://mise.jdx.dev/getting-started.html), then install the Python -workspace: - -```sh -mise install -mise run install -mise run test -``` - -The repository uses [uv](https://docs.astral.sh/uv/) for Python packages and -[Bun](https://bun.sh/docs) for the web workspace. Windows users run the -scientific backend under WSL 2; Microsoft documents the setup in the -[WSL install guide](https://learn.microsoft.com/windows/wsl/install). +The web app is at . The API is at +. -## Common commands +## Commands | Command | Purpose | | --- | --- | -| `mise run install` | Install all Python workspace packages with dev and build groups | -| `mise run test` | Run the Python test suite with `pytest -n auto` | -| `mise run lint` | Run Ruff and mypy for Python packages | -| `mise run api` | Start the FastAPI service with `tsdhn-api` | -| `mise run worker` | Start the Procrastinate worker with `tsdhn-worker` | -| `mise run web-dev` | Start the SvelteKit dev server | -| `mise run gen-client` | Export FastAPI OpenAPI JSON and regenerate TypeScript types | -| `mise run dev-up` | Run Postgres, MinIO, libSQL, API, and worker (Podman) | -| `mise run dev-web` | Run the backend stack plus the SvelteKit web app | -| `mise run dev-down` | Stop the local stack | -| `mise run dev-logs` | Follow logs for the local stack | - -## Workspace - -```txt -picv-2025/ -├── apps/ -│ └── web/ # SvelteKit app and server-side web routes -├── deploy/ # Dockerfiles for toolchain, API, and web images -├── libs/ -│ └── api-client/ # Generated TypeScript client from FastAPI OpenAPI -├── model/ # TSDHN model assets and legacy Fortran sources -├── packages/ -│ ├── api/ # FastAPI compute service and Procrastinate worker -│ └── tsdhn/ # Shared engine, runtime, assets, and CLI -├── scripts/ -│ ├── export_openapi.py # FastAPI schema export -│ └── gen-client.ts # OpenAPI TypeScript generation -├── docker-compose.yml -├── mise.toml # Tool versions and repo tasks -├── package.json # Bun workspaces for apps/* and libs/* -├── pyproject.toml # uv workspace for packages/* -└── uv.lock -``` - -## Runtime notes - -`tsdhn` validates model and tool paths before running simulations. -Non-container backend runs need: - -- `TSDHN_MODEL_DIR` pointing at the model asset directory. -- `TSDHN_TOOLS_DIR` pointing at prebuilt `fault_plane`, `deform`, and `tsunami` - executables when command pipeline steps are active. -- `TSDHN_JOBS_DIR` for temporary simulation workspaces when running the API worker. +| `mise run install` | Install Python dependencies | +| `mise run web-install` | Install the Bun workspace | +| `mise run dev-up` | Start the API, worker, Postgres, and MinIO | +| `mise run dev-web` | Start the web profile and web app | +| `mise run db-migrate` | Apply local database migrations | +| `mise run test` | Run the fast Python test suite | +| `mise run test-integration` | Run disposable PostgreSQL tests | +| `mise run web-test` | Run the fast web test suite | +| `mise run lint-all` | Run Python and JavaScript checks | +| `mise run gen-client` | Regenerate the OpenAPI client | +| `mise run test-golden` | Run the real pipeline regression | +| `mise run test-parity` | Compare Python output with the older programs | + +The fast test suites do not need services. `mise run test-integration` starts +the project-local PostgreSQL cluster, creates disposable databases, runs the +database-backed tests, and removes those databases when it exits. -The API container image sets these paths to `/app/model`, `/app/tools`, and -`/app/jobs`. - -
-Scientific runtime dependencies for non-container backend runs - -The containerized path is the maintained setup for the backend runtime. -Non-container runs must provide the same external tools: - -- Intel Fortran compiler (`ifx`) from - [Intel oneAPI Fortran Essentials](https://www.intel.com/content/www/us/en/docs/oneapi/installation-guide-linux/latest/overview.html) -- [Generic Mapping Tools](https://docs.generic-mapping-tools.org/latest/) -- [Ghostscript](https://www.ghostscript.com/) (`gs`), required by PyGMT when - finalizing plots -- [TTT SDK](https://www.geoware-online.com/tsunami.html), including - `ttt_client` -- `ps2eps` and `csh` - -
- -## License +## Documentation -This project is licensed under the terms declared in -[`pyproject.toml`](./pyproject.toml). +| Document | Use it for | +| --- | --- | +| [`ARCHITECTURE.md`](./ARCHITECTURE.md) | Responsibilities, database ownership, identifiers, displayed state, and request flows | +| [`DEPLOY.md`](./DEPLOY.md) | Compose deployment, configuration, and operations | +| [`packages/tsdhn`](./packages/tsdhn/readme.md) | Engine, CLI, model files, and simulation outputs | +| [`packages/api`](./packages/api/readme.md) | Compute API and worker development | +| [`apps/web`](./apps/web/readme.md) | SvelteKit development and server modules | +| [`libs/api-client`](./libs/api-client/readme.md) | Generated client and regeneration | +| [`packages/tsdhn-parity`](./packages/tsdhn-parity/readme.md) | Comparing Python results with MATLAB and Fortran results | + +Start with the architecture document when changing a boundary. Start with a +component README when changing code inside one component. + +## Repository layout + +```text +apps/web/ SvelteKit web app +deploy/ Container images +libs/api-client/ Generated TypeScript client +packages/tsdhn/ Simulation engine and researcher CLI +packages/api/ FastAPI service and worker +packages/tsdhn-parity/ Legacy-output comparison tools +scripts/ Setup, generation, database, and end-to-end tasks +docker-compose.yml Self-hosted service stack +mise.toml Pinned tools and project tasks +``` diff --git a/scripts/capture_matlab_fixtures.py b/scripts/capture_matlab_fixtures.py new file mode 100644 index 0000000..6a4df8a --- /dev/null +++ b/scripts/capture_matlab_fixtures.py @@ -0,0 +1,160 @@ +"""Capture MATLAB checkpoints as committed NumPy fixtures. + +Run this script by hand. Pytest reads the resulting fixture through +`FrozenFixtureAdapter` and does not start MATLAB. + +Usage: + uv run python scripts/capture_matlab_fixtures.py fault_plane + uv run python scripts/capture_matlab_fixtures.py fault_plane --case alaska_1964 + uv run python scripts/capture_matlab_fixtures.py fault_plane --runtime podman + +Each parity unit provides `spec.py` and `capture.m` under +`packages/tsdhn/tests/parity//`. The script imports `CASES` from +`spec.py`, runs `capture.m`, and converts `checkpoints.mat` to one `.npz` file +per case. Use `--from-mat` to convert a file from a manual MATLAB run. +""" + +from __future__ import annotations + +import argparse +import importlib +import json +import subprocess +import sys +import tempfile +from pathlib import Path +from types import ModuleType + +from scipy.io import loadmat + +from tsdhn_parity.adapters.frozen_fixture import write_fixture +from tsdhn_parity.cases import Case +from tsdhn_parity.trace import Checkpoint, Trace + +REPO_ROOT = Path(__file__).resolve().parent.parent +TESTS_DIR = REPO_ROOT / "packages" / "tsdhn" / "tests" +PARITY_DIR = TESTS_DIR / "parity" + +MATLAB_IMAGE = "mathworks/matlab:r2026a" +MATLAB_CONFIG_VOLUME = "matlab-config" +MATLAB_CONFIG_MOUNT = "/home/matlab/.matlab/MATLAB_R2026a" + + +def main() -> None: + args = _parse_args() + unit_dir = PARITY_DIR / args.unit + fixtures_dir = unit_dir / "fixtures" + fixtures_dir.mkdir(exist_ok=True) + + if args.from_mat is not None: + if args.case is None: + raise SystemExit("--from-mat requires --case (one .mat is one case)") + _write_fixture_from_mat(fixtures_dir, args.case, args.from_mat) + return + + driver = unit_dir / "capture.m" + if not driver.is_file(): + raise SystemExit( + f"{driver} does not exist. Nothing to capture for '{args.unit}'. " + "Add a capture.m next to spec.py first; see this script's docstring." + ) + + for case in _select_cases(unit_dir, args.case): + print(f"[{args.unit}] capturing case '{case.id}' ...") + checkpoints_mat = _run_matlab(driver, case, runtime=args.runtime) + _write_fixture_from_mat(fixtures_dir, case.id, checkpoints_mat) + + +def _write_fixture_from_mat( + fixtures_dir: Path, case_id: str, checkpoints_mat: Path +) -> None: + trace = _trace_from_mat(case_id, checkpoints_mat) + out = fixtures_dir / f"{case_id}.npz" + write_fixture(out, trace) + print(f"wrote {out}") + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "unit", help="unit directory name under packages/tsdhn/tests/parity/" + ) + parser.add_argument( + "--case", help="capture only this case id (default: every case in spec.py)" + ) + parser.add_argument("--runtime", default="docker", choices=["docker", "podman"]) + parser.add_argument( + "--from-mat", + type=Path, + default=None, + help=( + "skip MATLAB and convert this existing checkpoints.mat instead " + "(requires --case)" + ), + ) + return parser.parse_args() + + +def _select_cases(unit_dir: Path, only_case_id: str | None) -> list[Case]: + spec_module = _import_unit_spec(unit_dir.name) + cases: list[Case] = list(spec_module.CASES) # type: ignore[attr-defined] + if only_case_id is None: + return cases + matching = [case for case in cases if case.id == only_case_id] + if not matching: + raise SystemExit(f"no case '{only_case_id}' in {unit_dir / 'spec.py'}") + return matching + + +def _import_unit_spec(unit: str) -> ModuleType: + # Import through the parity package so relative imports work as they do + # under pytest. + sys.path.insert(0, str(TESTS_DIR)) + try: + return importlib.import_module(f"parity.{unit}.spec") + finally: + sys.path.remove(str(TESTS_DIR)) + + +def _run_matlab(driver: Path, case: Case, *, runtime: str) -> Path: + run_dir = Path(tempfile.mkdtemp(prefix="tsdhn-parity-matlab-")) + (run_dir / "capture.m").write_text(driver.read_text()) + (run_dir / "case_params.json").write_text(json.dumps(dict(case.params))) + + subprocess.run( + [ + runtime, + "run", + "--rm", + "--volume", + f"{run_dir}:/work", + "--volume", + f"{MATLAB_CONFIG_VOLUME}:{MATLAB_CONFIG_MOUNT}", + "--workdir", + "/work", + MATLAB_IMAGE, + "-batch", + "run('capture.m')", + ], + check=True, + ) + + checkpoints_mat = run_dir / "checkpoints.mat" + if not checkpoints_mat.is_file(): + raise RuntimeError( + f"capture.m did not produce {checkpoints_mat} for case '{case.id}'" + ) + return checkpoints_mat + + +def _trace_from_mat(case_id: str, checkpoints_mat: Path) -> Trace: + data = loadmat(checkpoints_mat) + names = [ + name for name in data if not (name.startswith("__") and name.endswith("__")) + ] + checkpoints = tuple(Checkpoint.of(name, data[name]) for name in names) + return Trace(case_id=case_id, checkpoints=checkpoints) + + +if __name__ == "__main__": + main() diff --git a/scripts/database.py b/scripts/database.py new file mode 100644 index 0000000..e705df5 --- /dev/null +++ b/scripts/database.py @@ -0,0 +1,127 @@ +"""Create and remove disposable PostgreSQL databases for test tasks.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from urllib.parse import SplitResult, quote, urlsplit, urlunsplit + +import psycopg +from psycopg import sql + + +@dataclass(frozen=True) +class DatabaseTarget: + """An administrative connection and a connection to one database.""" + + name: str + maintenance_url: str + database_url: str + + +def _netloc(parts: SplitResult, user: str | None, password: str | None) -> str: + hostname = parts.hostname + if hostname is None: + raise ValueError("database URL must include a hostname") + if ":" in hostname and not hostname.startswith("["): + hostname = f"[{hostname}]" + + port = parts.port + host = f"{hostname}:{port}" if port is not None else hostname + if user is None and password is None: + return parts.netloc + + credentials = quote(user or "", safe="") + if password is not None: + credentials += f":{quote(password, safe='')}" + return f"{credentials}@{host}" + + +def database_target( + base_url: str, + name: str, + *, + user: str | None = None, + password: str | None = None, +) -> DatabaseTarget: + """Build maintenance and target URLs without interpolating SQL values.""" + parts = urlsplit(base_url) + maintenance_url = urlunsplit(parts._replace(path="/postgres")) + database_url = urlunsplit( + parts._replace( + netloc=_netloc(parts, user, password), + path=f"/{name}", + ) + ) + return DatabaseTarget(name, maintenance_url, database_url) + + +def create_database(base_url: str, name: str) -> DatabaseTarget: + """Create a database from a privileged maintenance connection.""" + target = database_target(base_url, name) + with psycopg.connect(target.maintenance_url, autocommit=True) as connection: + connection.execute( + sql.SQL("CREATE DATABASE {}").format(sql.Identifier(target.name)) + ) + return target + + +def drop_database(base_url: str, name: str) -> None: + """Drop a disposable database, terminating any test connections first.""" + target = database_target(base_url, name) + with psycopg.connect(target.maintenance_url, autocommit=True) as connection: + connection.execute( + sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format( + sql.Identifier(target.name) + ) + ) + + +def drop_role(base_url: str, role: str) -> None: + """Remove a role created for an isolated integration run.""" + target = database_target(base_url, role) + with psycopg.connect(target.maintenance_url, autocommit=True) as connection: + identifier = sql.Identifier(role) + connection.execute(sql.SQL("DROP OWNED BY {}").format(identifier)) + connection.execute(sql.SQL("DROP ROLE IF EXISTS {}").format(identifier)) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + create = subparsers.add_parser("create") + create.add_argument("--base-url", required=True) + create.add_argument("--name", required=True) + + url = subparsers.add_parser("url") + url.add_argument("--base-url", required=True) + url.add_argument("--name", required=True) + url.add_argument("--user", required=True) + url.add_argument("--password", required=True) + + drop = subparsers.add_parser("drop") + drop.add_argument("--base-url", required=True) + drop.add_argument("--name", required=True) + drop.add_argument("--role") + + args = parser.parse_args() + if args.command == "create": + print(create_database(args.base_url, args.name).database_url) + elif args.command == "url": + print( + database_target( + args.base_url, + args.name, + user=args.user, + password=args.password, + ).database_url + ) + else: + drop_database(args.base_url, args.name) + if args.role: + drop_role(args.base_url, args.role) + + +if __name__ == "__main__": + main() diff --git a/scripts/e2e/backend_stack_smoke.sh b/scripts/e2e/compute_stack_smoke.sh similarity index 51% rename from scripts/e2e/backend_stack_smoke.sh rename to scripts/e2e/compute_stack_smoke.sh index ca0f979..bf5d7ce 100755 --- a/scripts/e2e/backend_stack_smoke.sh +++ b/scripts/e2e/compute_stack_smoke.sh @@ -1,16 +1,16 @@ #!/usr/bin/env bash set -euo pipefail -: "${APP_JOB_ID:=4cfe522f-7e7d-46e0-96ca-7b98743fb9f5}" -: "${BACKEND_SERVICE_TOKEN:=compose-e2e-token}" +: "${SIMULATION_ID:=4cfe522f-7e7d-46e0-96ca-7b98743fb9f5}" +: "${COMPUTE_API_TOKEN:=compose-e2e-token}" : "${MINIO_ACCESS_KEY:=minioadmin}" : "${MINIO_SECRET_KEY:=minioadmin}" : "${MINIO_BUCKET:=tsdhn-results}" -: "${API_BASE_URL:=http://localhost:8000}" +: "${COMPUTE_API_URL:=http://localhost:8000}" wait_for_api() { for _ in {1..60}; do - if curl -fsS "$API_BASE_URL/api/v1/version"; then + if curl -fsS "$COMPUTE_API_URL/api/v1/version"; then return 0 fi docker compose ps @@ -27,12 +27,12 @@ create_results_bucket() { docker compose exec -T minio mc mb --ignore-existing "local/$MINIO_BUCKET" } -submit_idempotent_job() { - local payload first second first_compute_job_id second_compute_job_id conflicting status +submit_idempotent_simulation() { + local payload first second conflicting status payload="$( - jq -cn --arg app_job_id "$APP_JOB_ID" '{ - app_job_id: $app_job_id, + jq -cn --arg simulation_id "$SIMULATION_ID" '{ + simulation_id: $simulation_id, input: { Mw: 8.0, h: 10.0, @@ -45,8 +45,8 @@ submit_idempotent_job() { )" first="$( - curl -fsS -X POST "$API_BASE_URL/api/v1/jobs" \ - -H "Authorization: Bearer $BACKEND_SERVICE_TOKEN" \ + curl -fsS -X POST "$COMPUTE_API_URL/api/v1/jobs" \ + -H "Authorization: Bearer $COMPUTE_API_TOKEN" \ -H "Content-Type: application/json" \ -d "$payload" )" @@ -54,21 +54,22 @@ submit_idempotent_job() { echo "$first" > first-job.json second="$( - curl -fsS -X POST "$API_BASE_URL/api/v1/jobs" \ - -H "Authorization: Bearer $BACKEND_SERVICE_TOKEN" \ + curl -fsS -X POST "$COMPUTE_API_URL/api/v1/jobs" \ + -H "Authorization: Bearer $COMPUTE_API_TOKEN" \ -H "Content-Type: application/json" \ -d "$payload" )" echo "$second" | jq . echo "$second" > second-job.json - first_compute_job_id="$(jq -r .compute_job_id first-job.json)" - second_compute_job_id="$(jq -r .compute_job_id second-job.json)" - test "$first_compute_job_id" = "$second_compute_job_id" + jq -e --arg simulation_id "$SIMULATION_ID" \ + '.simulation_id == $simulation_id' first-job.json + jq -e --arg simulation_id "$SIMULATION_ID" \ + '.simulation_id == $simulation_id' second-job.json conflicting="$( - jq -cn --arg app_job_id "$APP_JOB_ID" '{ - app_job_id: $app_job_id, + jq -cn --arg simulation_id "$SIMULATION_ID" '{ + simulation_id: $simulation_id, input: { Mw: 8.1, h: 10.0, @@ -81,8 +82,8 @@ submit_idempotent_job() { )" status="$( curl -sS -o conflict-response.json -w "%{http_code}" \ - -X POST "$API_BASE_URL/api/v1/jobs" \ - -H "Authorization: Bearer $BACKEND_SERVICE_TOKEN" \ + -X POST "$COMPUTE_API_URL/api/v1/jobs" \ + -H "Authorization: Bearer $COMPUTE_API_TOKEN" \ -H "Content-Type: application/json" \ -d "$conflicting" )" @@ -94,16 +95,17 @@ assert_queued_job() { local compute_count queue_count compute_count="$( - docker compose exec -T postgres psql -U tsdhn -d tsdhn_compute -tAc \ - "SELECT count(*) FROM compute_jobs WHERE external_id = '$APP_JOB_ID'::uuid" + docker compose exec -T postgres psql -U tsdhn -d tsdhn -tAc \ + "SELECT count(*) FROM compute.jobs WHERE simulation_id = '$SIMULATION_ID'::uuid" )" test "$compute_count" = "1" - docker compose exec -T postgres psql -U tsdhn -d tsdhn_compute -c \ + docker compose exec -T postgres psql -U tsdhn -d tsdhn -c \ "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'procrastinate_%' ORDER BY tablename" + # Idempotent submissions must not create a second queue row. queue_count="$( - docker compose exec -T postgres psql -U tsdhn -d tsdhn_compute -tAc \ + docker compose exec -T postgres psql -U tsdhn -d tsdhn -tAc \ "SELECT count(*) FROM procrastinate_jobs WHERE task_name = 'api.run_simulation' AND queue_name = 'simulations'" )" test "$queue_count" = "1" @@ -117,8 +119,8 @@ poll_job_to_completion() { while [ "$SECONDS" -lt "$deadline" ]; do body="$( - curl -fsS "$API_BASE_URL/api/v1/jobs/$APP_JOB_ID" \ - -H "Authorization: Bearer $BACKEND_SERVICE_TOKEN" + curl -fsS "$COMPUTE_API_URL/api/v1/jobs/$SIMULATION_ID" \ + -H "Authorization: Bearer $COMPUTE_API_TOKEN" )" echo "$body" | jq -c . | tee -a status-history.jsonl @@ -146,31 +148,63 @@ assert_completed_outputs() { local persisted jq . final-status.json - jq -e --arg key "simulations/$APP_JOB_ID/metadata.json" ' + jq -e ' .status == "completed" - and .artifacts_available == true - and .result_bucket == env.MINIO_BUCKET - and .result_key == $key + and (.outputs | length) > 0 + and (.outputs | index("max_height_map")) != null and (.step_index | type == "number") and (.total_steps | type == "number") and (.calculation | type == "object") and (.travel_times | type == "object") ' final-status.json - docker compose exec -T minio mc stat "local/$MINIO_BUCKET/simulations/$APP_JOB_ID/metadata.json" - docker compose exec -T minio mc stat "local/$MINIO_BUCKET/simulations/$APP_JOB_ID/artifacts/calculation.json" - docker compose exec -T minio mc stat "local/$MINIO_BUCKET/simulations/$APP_JOB_ID/artifacts/travel_times.csv" + docker compose exec -T minio mc stat "local/$MINIO_BUCKET/simulations/$SIMULATION_ID/metadata.json" + docker compose exec -T minio mc stat "local/$MINIO_BUCKET/simulations/$SIMULATION_ID/outputs/calculation.json" + docker compose exec -T minio mc stat "local/$MINIO_BUCKET/simulations/$SIMULATION_ID/outputs/travel_times.csv" persisted="$( - docker compose exec -T postgres psql -U tsdhn -d tsdhn_compute -tAc \ - "SELECT status || '|' || result_key FROM compute_jobs WHERE external_id = '$APP_JOB_ID'::uuid" + docker compose exec -T postgres psql -U tsdhn -d tsdhn -tAc \ + "SELECT status || '|' || result_key FROM compute.jobs WHERE simulation_id = '$SIMULATION_ID'::uuid" )" - test "$persisted" = "completed|simulations/$APP_JOB_ID/metadata.json" + test "$persisted" = "completed|simulations/$SIMULATION_ID/metadata.json" +} + +assert_output_is_downloadable() { + local listed location bytes + + listed="$( + curl -fsS "$COMPUTE_API_URL/api/v1/jobs/$SIMULATION_ID/outputs" \ + -H "Authorization: Bearer $COMPUTE_API_TOKEN" + )" + echo "$listed" | jq . + jq -e '.outputs | map(.name) | index("max_height_map") != null' <<< "$listed" + + location="$( + curl -fsS -o /dev/null -w '%{redirect_url}' \ + "$COMPUTE_API_URL/api/v1/jobs/$SIMULATION_ID/outputs/max_height_map" \ + -H "Authorization: Bearer $COMPUTE_API_TOKEN" + )" + test -n "$location" + + # The URL must be usable from the browser-facing endpoint. + bytes="$(curl -fsS "$location" | head -c 4)" + test "$bytes" = "%PDF" +} + +assert_unauthenticated_output_is_refused() { + local status + status="$( + curl -sS -o /dev/null -w '%{http_code}' \ + "$COMPUTE_API_URL/api/v1/jobs/$SIMULATION_ID/outputs/max_height_map" + )" + test "$status" = "401" } wait_for_api create_results_bucket -submit_idempotent_job +submit_idempotent_simulation assert_queued_job poll_job_to_completion assert_completed_outputs +assert_output_is_downloadable +assert_unauthenticated_output_is_refused diff --git a/scripts/e2e/crash_recovery_e2e.sh b/scripts/e2e/crash_recovery_e2e.sh new file mode 100755 index 0000000..d2f81d7 --- /dev/null +++ b/scripts/e2e/crash_recovery_e2e.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# Exercise worker recovery against a running Compose stack. +set -euo pipefail + +: "${CRASH_SIMULATION_ID:=b6e1f5a0-2c1b-4a7c-9c7a-3f7b6a5d9e11}" +: "${TRANSIENT_SIMULATION_ID:=c1a2b3c4-5d6e-4f70-8a9b-0c1d2e3f4a5b}" +: "${COMPUTE_API_TOKEN:=compose-e2e-token}" +: "${MINIO_ACCESS_KEY:=minioadmin}" +: "${MINIO_SECRET_KEY:=minioadmin}" +: "${MINIO_BUCKET:=tsdhn-results}" +: "${COMPUTE_API_URL:=http://localhost:8000}" +# The crash must happen after the resumable step writes a checkpoint. +: "${TSUNAMI_CHECKPOINT_WAIT_SECONDS:=120}" + +psql_c() { + docker compose exec -T postgres psql -U tsdhn -d tsdhn -tAc "$1" +} + +wait_for_api() { + for _ in {1..60}; do + if curl -fsS "$COMPUTE_API_URL/api/v1/version"; then + return 0 + fi + docker compose ps + sleep 5 + done + echo "::error::API did not become reachable" + return 1 +} + +create_results_bucket() { + docker compose exec -T minio \ + mc alias set local http://127.0.0.1:9000 "$MINIO_ACCESS_KEY" "$MINIO_SECRET_KEY" + docker compose exec -T minio mc mb --ignore-existing "local/$MINIO_BUCKET" +} + +submit_simulation() { + local simulation_id="$1" payload response + payload="$( + jq -cn --arg simulation_id "$simulation_id" '{ + simulation_id: $simulation_id, + input: { Mw: 8.0, h: 10.0, lat0: -20.5, lon0: -70.5, hhmm: "0000", dia: "23" } + }' + )" + response="$( + curl -fsS -X POST "$COMPUTE_API_URL/api/v1/jobs" \ + -H "Authorization: Bearer $COMPUTE_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d "$payload" + )" + echo "$response" | jq . +} + +job_status() { + local simulation_id="$1" + curl -fsS "$COMPUTE_API_URL/api/v1/jobs/$simulation_id" \ + -H "Authorization: Bearer $COMPUTE_API_TOKEN" +} + +wait_for_step() { + local simulation_id="$1" target_step="$2" deadline_seconds="$3" deadline body step + deadline=$((SECONDS + deadline_seconds)) + while [ "$SECONDS" -lt "$deadline" ]; do + body="$(job_status "$simulation_id")" + step="$(echo "$body" | jq -r .step)" + if [ "$step" = "$target_step" ]; then + echo "$body" | jq . + return 0 + fi + if [ "$(echo "$body" | jq -r .status)" = "failed" ]; then + echo "$body" | jq . + echo "::error::Job failed before reaching step '$target_step'" + return 1 + fi + sleep 5 + done + echo "::error::Timed out waiting for step '$target_step'" + return 1 +} + +poll_job_to_completion() { + local simulation_id="$1" deadline_seconds="$2" deadline body status + deadline=$((SECONDS + deadline_seconds)) + while [ "$SECONDS" -lt "$deadline" ]; do + body="$(job_status "$simulation_id")" + echo "$body" | jq -c . + status="$(echo "$body" | jq -r .status)" + case "$status" in + completed) + echo "$body" > "final-status-${simulation_id}.json" + return 0 + ;; + failed) + echo "$body" > "final-status-${simulation_id}.json" + echo "::error::Job ended FAILED instead of completing" + return 1 + ;; + esac + sleep 10 + done + echo "::error::Job did not complete before timeout" + return 1 +} + +scenario_crash_and_requeue() { + echo "--- Scenario 1: kill worker mid-tsunami, expect auto-requeue + checkpoint resume ---" + + submit_simulation "$CRASH_SIMULATION_ID" + wait_for_step "$CRASH_SIMULATION_ID" "tsunami" 600 + echo "In tsunami step; waiting ${TSUNAMI_CHECKPOINT_WAIT_SECONDS}s for checkpoints to accumulate" + sleep "$TSUNAMI_CHECKPOINT_WAIT_SECONDS" + + # Requeueing increments the existing queue row's attempt counter. + local queue_job_id before_attempts + queue_job_id="$( + psql_c "SELECT id FROM procrastinate_jobs + WHERE task_name = 'api.run_simulation' + AND task_kwargs->>'compute_job_id' = + (SELECT id::text FROM compute.jobs + WHERE simulation_id = '$CRASH_SIMULATION_ID'::uuid)" + )" + test -n "$queue_job_id" + before_attempts="$(psql_c "SELECT attempts FROM procrastinate_jobs WHERE id = $queue_job_id")" + + echo "Killing worker (SIGKILL) to simulate a crash" + docker compose kill -s SIGKILL worker + + echo "Waiting up to 240s for the reaper to detect the stale job and requeue it" + local deadline requeued=0 + deadline=$((SECONDS + 240)) + while [ "$SECONDS" -lt "$deadline" ]; do + local attempts status + attempts="$(psql_c "SELECT attempts FROM procrastinate_jobs WHERE id = $queue_job_id")" + status="$(job_status "$CRASH_SIMULATION_ID" | jq -r .status)" + if [ "$status" = "failed" ]; then + echo "::error::Job was marked FAILED instead of being requeued" + return 1 + fi + if [ "${attempts:-0}" -gt "${before_attempts:-0}" ]; then + requeued=1 + break + fi + sleep 10 + done + test "$requeued" = "1" || { + echo "::error::Job was never requeued after the worker crash" + return 1 + } + echo "Confirmed: attempts incremented on the same queue job, not marked FAILED" + + poll_job_to_completion "$CRASH_SIMULATION_ID" 1800 + + docker compose logs worker --no-color | grep -q "resuming tsunami run from step" || { + echo "::error::Worker log has no evidence of resuming from a checkpoint" + return 1 + } + echo "Confirmed: worker log shows checkpoint resume" + + docker compose exec -T minio mc stat "local/$MINIO_BUCKET/simulations/$CRASH_SIMULATION_ID/metadata.json" +} + +scenario_ttl_sweep() { + echo "--- Scenario 2: TTL sweep removes a terminally-FAILED job's work_dir ---" + + local sweep_job_id work_dir + sweep_job_id="$(cat /proc/sys/kernel/random/uuid)" + work_dir="/var/tmp/jobs/${sweep_job_id}" + + docker compose exec -T worker sh -c "mkdir -p '$work_dir' && touch '$work_dir/marker'" + + psql_c " + INSERT INTO compute.jobs (id, simulation_id, status, input_params, details, finished_at) + VALUES (gen_random_uuid(), '${sweep_job_id}'::uuid, 'failed', '{}'::jsonb, + 'synthetic e2e row', now() - interval '25 hours') + " > /dev/null + + docker compose exec -T worker uv run --no-dev python -c " +from api.core.tasks import sweep_abandoned_work_dirs_task +sweep_abandoned_work_dirs_task(0) +" + + if docker compose exec -T worker test -e "$work_dir"; then + echo "::error::sweep_abandoned_work_dirs_task did not remove $work_dir" + return 1 + fi + echo "Confirmed: expired FAILED job's work_dir was swept" +} + +scenario_transient_retry() { + echo "--- Scenario 3: MinIO outage during finalize triggers TRANSIENT_RETRY, not a failure ---" + + submit_simulation "$TRANSIENT_SIMULATION_ID" + wait_for_step "$TRANSIENT_SIMULATION_ID" "copy_ttt_pdf" 1800 + echo "Reached the last pipeline step; stopping MinIO to force finalize's upload to fail" + docker compose stop minio + + # Give the queue time to record the transient retry. + sleep 30 + local details + details="$(job_status "$TRANSIENT_SIMULATION_ID" | jq -r .details)" + echo "$details" + case "$details" in + *"Retrying after transient error"*) ;; + *) + echo "::error::Expected an in-flight TRANSIENT_RETRY, got: $details" + docker compose start minio + return 1 + ;; + esac + echo "Confirmed: TRANSIENT_RETRY recorded a retry instead of failing the job" + + docker compose start minio + poll_job_to_completion "$TRANSIENT_SIMULATION_ID" 300 +} + +wait_for_api +create_results_bucket +scenario_crash_and_requeue +scenario_ttl_sweep +scenario_transient_retry diff --git a/scripts/export_openapi.py b/scripts/export_openapi.py index 2e23b62..23e458f 100644 --- a/scripts/export_openapi.py +++ b/scripts/export_openapi.py @@ -1,8 +1,4 @@ -""" -Print the FastAPI OpenAPI schema to stdout without starting a server. -Used by `scripts/gen-client.ts` to regenerate the typed TS client hermetically -(no running backend or compute worker). -""" +"""Print the FastAPI OpenAPI schema without starting a server.""" import json diff --git a/scripts/gen-client.ts b/scripts/gen-client.ts index 716d6af..cbe84df 100644 --- a/scripts/gen-client.ts +++ b/scripts/gen-client.ts @@ -1,7 +1,4 @@ #!/usr/bin/env bun -/** - * CI fails when generated OpenAPI types drift from the FastAPI contract. - */ import { $ } from 'bun'; const SCHEMA_JSON = 'libs/api-client/openapi.json'; diff --git a/scripts/integration.sh b/scripts/integration.sh new file mode 100755 index 0000000..7537ca3 --- /dev/null +++ b/scripts/integration.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +# These tests own this project-local cluster and never accept a production URL. +base_url="postgresql://tsdhn:tsdhn@127.0.0.1:5432/tsdhn" +database_name="tsdhn_integration_$(date +%s)_$$" +app_role="${database_name}_role" +app_password="tsdhn-web-test-password" + +case "${1:-}" in + "") coverage=0 ;; + --coverage) coverage=1 ;; + *) echo "usage: $0 [--coverage]" >&2; exit 2 ;; +esac + +# Skip database startup if COMPUTE_DATABASE_URL is set (e.g., by CI with service container) +if [ -z "${COMPUTE_DATABASE_URL:-}" ]; then + mise run db:start +fi + +admin_url="$( + uv run python -m scripts.database create \ + --base-url "$base_url" \ + --name "$database_name" +)" +app_url="$( + uv run python -m scripts.database url \ + --base-url "$admin_url" \ + --name "$database_name" \ + --user "$app_role" \ + --password "$app_password" +)" + +cleanup() { + uv run python -m scripts.database drop \ + --base-url "$base_url" \ + --name "$database_name" \ + --role "$app_role" >/dev/null +} +trap cleanup EXIT + +COMPUTE_DATABASE_URL="$admin_url" \ +APP_DB_ROLE="$app_role" \ +APP_DB_PASSWORD="$app_password" \ +uv run tsdhn-compute-migrate + +COMPUTE_DATABASE_URL="$admin_url" \ +uv run tsdhn-procrastinate-migrate + +# Schema changes run with the database-owner connection. The app role below +# is intentionally limited to runtime DML and compute-state reads. +DATABASE_URL="$admin_url" bun --filter web db:migrate + +COMPUTE_DATABASE_URL="$admin_url" \ +APP_DB_ROLE="$app_role" \ +uv run tsdhn-web-grants + +if [[ "$coverage" == "1" ]]; then + uv run coverage run -m pytest -m integration packages/api/tests +else + uv run pytest -m integration packages/api/tests +fi + +DATABASE_URL="$app_url" \ + bun --filter web test:integration diff --git a/scripts/setup.sh b/scripts/setup.sh index b26196e..e6c2186 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -38,7 +38,10 @@ Options: --skip-gmt-source Do not build GMT from source if apt GMT is too old. --skip-ttt Do not install ttt_client. --skip-ifx Do not install Intel Fortran Essentials. - --skip-tools Do not compile fault_plane/deform/tsunami. + --skip-tools Do not compile fault_plane/deform/tsunami. These + legacy binaries are used only by the parity suite + (mise run test-parity); the pipeline itself is pure + Python and does not need them. --tools-dir DIR Directory for compiled model executables. -h, --help Show this help. @@ -169,6 +172,7 @@ APT_PACKAGES=( libcurl4 libfftw3-dev libgdal-dev + libgmt-dev liblapack-dev libnetcdf-dev libpcre3 diff --git a/uv.lock b/uv.lock index 00952b1..73ab0f7 100644 --- a/uv.lock +++ b/uv.lock @@ -15,6 +15,7 @@ members = [ "picv-2025", "tsdhn", "tsdhn-api", + "tsdhn-parity", ] [[package]] @@ -259,6 +260,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/17/1073b53b68c9b5ca6914adf5f8bf55aacc2d3be102418c90700160ea8605/cffi-2.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210", size = 186405, upload-time = "2026-07-06T21:34:24.857Z" }, ] +[[package]] +name = "chardet" +version = "7.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/cd61c567092a6cec796144510a68aff158ebfc1df82950a45bae65f28413/chardet-7.6.0.tar.gz", hash = "sha256:93d9df6089ded42ed1fe9f57e272c0b74bd0464d45c0c7d50f09f26f31105c3c", size = 914462, upload-time = "2026-08-14T20:36:59.305Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/78/e14991c9487277ed7d006fff58f3084ac792ed94cced081788abb95df70c/chardet-7.6.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c6061adf247ab5dda173b67010e13904c6071717660c7c8077fb50aca362b264", size = 1087678, upload-time = "2026-08-14T20:36:43.2Z" }, + { url = "https://files.pythonhosted.org/packages/7a/40/0f95e04cb1820e0a582cd6d86bbf26be8302a94ccf330f8ba5f69735389d/chardet-7.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fc1e1571321baf8927582fe34363ad7f02279f11c8c2839c14b4c76894148db6", size = 1065310, upload-time = "2026-08-14T20:36:44.489Z" }, + { url = "https://files.pythonhosted.org/packages/3b/a1/04404dcc9d6e1253b02583e562d2c7ddca50a7e91f460d62eff7e1d07c92/chardet-7.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8900f6c7cf6b015b17a51767cc6144689059ba1cdceaa383d29eb037ac28579e", size = 1485028, upload-time = "2026-08-14T20:36:45.797Z" }, + { url = "https://files.pythonhosted.org/packages/f4/88/360064c4c7d9d0664561dae03b74c871d2f5332b329f5c99f1c997fb869a/chardet-7.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cedbc584789eb2edfde20fd03669972a833ce6019e60014ae613f9bfc440e8e3", size = 1510242, upload-time = "2026-08-14T20:36:47.206Z" }, + { url = "https://files.pythonhosted.org/packages/43/4c/302869fa1c69a5a41e4e78782680b12552ffd4286c3ae3c839b7b8df53d4/chardet-7.6.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5dc835e40e0e09c2c3eab43731a8b5127834f42786dda09ba2f4b699ccd527a", size = 1456456, upload-time = "2026-08-14T20:36:48.668Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a9/ff4fef15ed25fc3f945a3b981ae0f43c8559b3fbedb40267e59e583d105b/chardet-7.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:0f304de7041afaec0195ad6464937cd112392002e9d72ed15d55f20a9abd3a13", size = 1159103, upload-time = "2026-08-14T20:36:50.107Z" }, + { url = "https://files.pythonhosted.org/packages/31/44/94e6f89485ce6c630e8fec6d388e3fa747e58b7246b0cc8c5c532ba98650/chardet-7.6.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a4f0a368ad04d5def08bdfaa17c7e15e71552f93923dc2aa9b2f7d9dee02fbb6", size = 1120302, upload-time = "2026-08-14T20:36:51.484Z" }, + { url = "https://files.pythonhosted.org/packages/e0/cd/8e68ba12f13aaf4ee82d54ef8a1f83d62942e7a6bc1e579dd2d530a3e26e/chardet-7.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:75d6c3a4d2046d49e83d2d2206eb073a1f390743e856d90c1bbc19949b26acf4", size = 1093157, upload-time = "2026-08-14T20:36:52.987Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f6/6a35342b9efa69dcceab0ea8966571c6442a59c336bf460f0a2af95ed234/chardet-7.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:459e2b1c98f9a86a4698112aa42dffa802bbbff883c1ff144071f87224125862", size = 1521213, upload-time = "2026-08-14T20:36:54.812Z" }, + { url = "https://files.pythonhosted.org/packages/df/8c/8f09bfabbaedae45caa72b996e96d5e3f6436dfddc55c1311ffa2f6bd7d2/chardet-7.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bdb6f03107b7ace3f44e0edd91aa24456ee558787df265cc19daf45785b31c7", size = 1528427, upload-time = "2026-08-14T20:36:56.224Z" }, + { url = "https://files.pythonhosted.org/packages/0c/3d/2540627b193112e08b8045f3cad9295570866208555aa6625b63cc69c6cd/chardet-7.6.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:0c44a32da32cc8b23d6b20d98ace15ec7600950e4955d1bf5ab1f849b0187fdb", size = 1087374, upload-time = "2026-08-14T21:04:14.994Z" }, + { url = "https://files.pythonhosted.org/packages/fe/de/dac4f550cde73c4732b4916e72ec489ae36933fbbd1697e4122d3ae2e9dd/chardet-7.6.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7bbc8a9652c7f859c593847f220c1d264f25749369abb1a267b404ee8cceb209", size = 1064688, upload-time = "2026-08-14T21:04:16.806Z" }, + { url = "https://files.pythonhosted.org/packages/06/45/f4f3f288496797d2cf1d1e9036f920ed2b4c79787b21c42a3314f4a6d532/chardet-7.6.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83512a475a2f3886166aa0bca1bbb39343a4eb3186dd5532127d6f2591d09118", size = 1486320, upload-time = "2026-08-14T21:04:18.535Z" }, + { url = "https://files.pythonhosted.org/packages/c0/81/3ca30c16e6c6015b737fca22d9342957cc617d8a65329d3e963ad754a31b/chardet-7.6.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:271ab71ec1be61dbbce0436de0848895c03eae051c379e937a39573d9ce403d9", size = 1515374, upload-time = "2026-08-14T21:04:20.724Z" }, + { url = "https://files.pythonhosted.org/packages/6e/98/163a75b8b3372a6b6503f5f5a4154a222ac135c7a706b5d176f8e4b237a2/chardet-7.6.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da86fc1b40ff5996fbb5e4c2d2dca770eac2c893cef157dacc050b8b4d929846", size = 1469504, upload-time = "2026-08-14T21:04:22.352Z" }, + { url = "https://files.pythonhosted.org/packages/d7/4f/3ad1b1bd27ae7f0d3d13cf711366525cd781a9e067950d8a09aa280916cb/chardet-7.6.0-cp315-cp315-win_amd64.whl", hash = "sha256:b73f277c1ac09c4f8076c4214b816c7aa78a0a2f0cb7156742f4303f856bedc3", size = 1158551, upload-time = "2026-08-14T21:04:24.259Z" }, + { url = "https://files.pythonhosted.org/packages/8d/92/22a609c68c5123ebdccd8fe1a80e423609012ce20166de19a2ed3955b2f7/chardet-7.6.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:61238d5945b36af9a2ad13494f8969b7deb3c3b4abe223e54670c064e73f5328", size = 1119186, upload-time = "2026-08-14T21:04:25.761Z" }, + { url = "https://files.pythonhosted.org/packages/29/35/75a90143e4200e197f4f6cf5895d379e22bc785d0c7711120df18fa5347c/chardet-7.6.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:f2ec3c78cc6b54bf8e091ec4ee885473078b5d7ef18ab1b01c86ae1e98bf88f7", size = 1092596, upload-time = "2026-08-14T21:04:27.884Z" }, + { url = "https://files.pythonhosted.org/packages/d1/2c/b6d5f47d878c46e04ae6fcb58e0969467925bc0819b333c682e0c41bbc5c/chardet-7.6.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:167d7ba3ee08b654e36d7b43ebd9a36606c9a12e2fabdb361757a095ca3b7e3d", size = 1516644, upload-time = "2026-08-14T21:04:29.624Z" }, + { url = "https://files.pythonhosted.org/packages/25/d2/2bc3f1066c6f27bc3fa3a98dfd8a2a9c1e969a9d6bdc1edcd35278791fc4/chardet-7.6.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f14f46ef1977e41ce1f4814ca6984cea7f8b6baf8cbc6626ef7bf3d13cf7ea13", size = 1533221, upload-time = "2026-08-14T21:04:31.39Z" }, + { url = "https://files.pythonhosted.org/packages/cf/6e/5a0b348fa4cd7847567a28c6e697ccf58391960bfd13a6e7473ee23ca2f2/chardet-7.6.0-py3-none-any.whl", hash = "sha256:4076d795897ce45239825956a1334e134322ecc4bfe84dbb12acd5390de0fbc1", size = 680279, upload-time = "2026-08-14T20:36:57.763Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.9" @@ -390,6 +420,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "diff-cover" +version = "10.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "chardet" }, + { name = "jinja2" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/74/b71a61b2610b5866aa29a9388bc1b58738e49600345dbe0598676b436717/diff_cover-10.5.1.tar.gz", hash = "sha256:f1c9417c62111e40a81c482c692c5cdd131ff93317baa993044cb291912ebba3", size = 113297, upload-time = "2026-08-16T18:09:19.466Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/da/ac0ad341c7377e82f7d6dcb18461029579da224c3923f73ff6537d8591cf/diff_cover-10.5.1-py3-none-any.whl", hash = "sha256:67ce07494e605b5d7d7b04aeec2cf524950ed0c12f026dd176883cf33c74fba0", size = 62924, upload-time = "2026-08-16T18:09:18.387Z" }, +] + [[package]] name = "execnet" version = "2.1.2" @@ -479,6 +524,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "librt" version = "0.13.0" @@ -526,6 +583,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/40/791891d4c0c4dab4c5e187c17261cedc26285fd41541577f900470a45a4d/license_expression-30.4.4-py3-none-any.whl", hash = "sha256:421788fdcadb41f049d2dc934ce666626265aeccefddd25e162a26f23bcbf8a4", size = 120615, upload-time = "2025-07-22T11:13:31.217Z" }, ] +[[package]] +name = "llvmlite" +version = "0.48.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/a0/acc8ffcd5bdc63df0097e22c719bfcd61b604358343089313a8aebbb24ab/llvmlite-0.48.0.tar.gz", hash = "sha256:543b19f9ef8f3c7c60d1468191e4ee1b1537bf9f8a3d56f64c0ddd98de92edd2", size = 184016, upload-time = "2026-07-02T20:20:05.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/8e/8170f2e0c217f88069c333d85bb976e536b332aecfcce606ddbdb249385f/llvmlite-0.48.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:321f1ac39b462603f0b589751aecf2d237d056f6d005749c1752b6f23ec3f074", size = 40480650, upload-time = "2026-07-01T18:42:07.935Z" }, + { url = "https://files.pythonhosted.org/packages/d6/e1/05b50692b647cac3c18200ac485b04f342f00ed173c9cc46767274469a15/llvmlite-0.48.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:05f0103c8f2f96a37441337e3643863c01b8e83e530aff38960dcb383c54a065", size = 59890115, upload-time = "2026-07-01T18:42:17.805Z" }, + { url = "https://files.pythonhosted.org/packages/f7/c3/470b8c4ff9ae2db2f9cf5c3e73de76ed908a32788ae9eb5602d43e6a476b/llvmlite-0.48.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37d66fae72802175b0bfe1ea06e624b51e2d7aee6c3c34bbd09739b8f88e8e0b", size = 58343457, upload-time = "2026-07-01T18:42:13.217Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2d/6a5171fb7236ac0895e1a02ccba3735bf291e8597239aa6421894d3c0ba8/llvmlite-0.48.0-cp314-cp314-win_amd64.whl", hash = "sha256:966dcab0a598e2bd8fb5f2cc082cf7b07bae564fc485a3a8692393caf986facf", size = 42986372, upload-time = "2026-07-01T18:42:21.483Z" }, + { url = "https://files.pythonhosted.org/packages/94/e3/7a93e09c9f94e637ca90209ceef0334a9a1d45b0bdb7c92ff922d25d6187/llvmlite-0.48.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7a5c413317050a1d67c34708bde97707f9b2257ef1017f7532d21fe7d9a9ff30", size = 40480654, upload-time = "2026-07-01T18:42:25.076Z" }, + { url = "https://files.pythonhosted.org/packages/27/98/a29133b4728671a175f7d616fab8b1c6e1d8c269d1523581d3160697bfb1/llvmlite-0.48.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d9f952dff6c350c529997423d4fa43abae9722a884ac7ffafc37a3af676e7db", size = 59890119, upload-time = "2026-07-01T18:42:33.88Z" }, + { url = "https://files.pythonhosted.org/packages/1a/cf/7aac11a1f1c7ec54b60c7f6814e87561fb6b55b2f290455d7941eb113420/llvmlite-0.48.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:054aa7d46595935565f276cf0c1659b4f10929c996dd4a606875fae26fba2a23", size = 58343460, upload-time = "2026-07-01T18:42:29.545Z" }, + { url = "https://files.pythonhosted.org/packages/db/41/b96f440c7df5ebba07872cad4e30fbc3560387755b1ea0b629adb76d5ca8/llvmlite-0.48.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d0b3c61aac83b42fb48cc96bffbf57c81b82b2aa92276b7ed6420c814629a99a", size = 42986383, upload-time = "2026-07-01T18:42:37.544Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -538,6 +611,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, ] +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -633,33 +736,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "numba" +version = "0.66.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llvmlite" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/a0/570e3dc53e5602b49108f62a13e529f1eec8bfc7ef37d49c825924dcf546/numba-0.66.0.tar.gz", hash = "sha256:b900e63a0e26c05ea9a6d5a3a5a0a177cb64c5011887bf43edb8c3ed2c38d363", size = 2806181, upload-time = "2026-07-01T23:12:46.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/7a/7e0e73550eb4e41ede6e72fb5371f4539537a4d770a3b73fa9b61aea0622/numba-0.66.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:46ae5f2b19e2af3c33c2df100306a90ea2f981c8158b0390f8bf6c20eee7357e", size = 2727296, upload-time = "2026-07-01T23:12:32.39Z" }, + { url = "https://files.pythonhosted.org/packages/0f/26/885774c006de6620ed3d10f45d8e20fe0b8e6aad6d573211a2cbc8b3e528/numba-0.66.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e2b101f23b8b63978d334574d2039f27f0dccfe1d891756f33a2e2f3e4c88cf4", size = 3842720, upload-time = "2026-07-01T23:12:33.938Z" }, + { url = "https://files.pythonhosted.org/packages/93/99/edebf7de890b73973d839dd971cf73734adfb81ffa1b4504f84b9059c3e5/numba-0.66.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63b943eb2c9ba371908ce2cd6dfc643db51fc40f7966993376a1701bc922f537", size = 3543537, upload-time = "2026-07-01T23:12:35.566Z" }, + { url = "https://files.pythonhosted.org/packages/66/c5/b46ad28ac3681d035ea21365c5e052149062e1a0a9affd0563d2760ea6ff/numba-0.66.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd57790acd20f6a468e0ad333ef6b82355e309a92310fb7dff80e919f01a21a9", size = 2799250, upload-time = "2026-07-01T23:12:37.154Z" }, + { url = "https://files.pythonhosted.org/packages/10/6f/5e77a7397a37dd16f57a7b72e7e470db5227b68e3639df0d13a8e674883d/numba-0.66.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:db7735d15ea17a283d6485b9fa3504769f78fd86e5146638ad5e8da57c031b9e", size = 2730342, upload-time = "2026-07-01T23:12:38.758Z" }, + { url = "https://files.pythonhosted.org/packages/39/fd/e9c9680a3813f3d781c20e5d53c1074801b787d4feecca0472fdd7c05ce1/numba-0.66.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:651a2b53298340956db26ecbe7ab043106b50a40c2807e66d617a4917245a4ab", size = 3878695, upload-time = "2026-07-01T23:12:40.302Z" }, + { url = "https://files.pythonhosted.org/packages/61/3a/9b363287b85fcd4537ea3878793822878b2ac1008a78159d2096fea628de/numba-0.66.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c1144ba1720ea59ad79f4f488ed54d149b2613b357f7e445678b7d0739c70e9", size = 3596323, upload-time = "2026-07-01T23:12:42.805Z" }, + { url = "https://files.pythonhosted.org/packages/4c/f2/dca53d50b8f2289dd01954ace9da261e0487d5b74b188b4304e4ecc3492c/numba-0.66.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d426178fb991a85714c43112a8ea7b9d9579ea856ad8dcdb9c1c3941903ba5be", size = 2804772, upload-time = "2026-07-01T23:12:44.399Z" }, +] + [[package]] name = "numpy" -version = "2.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" }, - { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" }, - { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" }, - { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" }, - { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" }, - { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" }, - { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" }, - { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" }, - { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" }, - { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" }, - { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" }, - { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" }, - { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" }, - { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" }, - { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" }, - { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" }, - { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" }, - { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" }, - { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" }, - { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" }, - { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, + { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, + { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, + { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, + { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, + { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, + { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, + { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, ] [[package]] @@ -730,6 +853,7 @@ build = [ dev = [ { name = "bandit" }, { name = "coverage" }, + { name = "diff-cover" }, { name = "httpx2" }, { name = "mypy" }, { name = "pip-audit" }, @@ -748,6 +872,7 @@ build = [{ name = "uv-build", specifier = "==0.11.28" }] dev = [ { name = "bandit", specifier = "==1.9.4" }, { name = "coverage", specifier = "==7.15.0" }, + { name = "diff-cover", specifier = "==10.5.1" }, { name = "httpx2", specifier = "==2.5.0" }, { name = "mypy", specifier = "==2.2.0" }, { name = "pip-audit", specifier = "==2.10.1" }, @@ -761,11 +886,11 @@ dev = [ [[package]] name = "pip" -version = "26.1.2" +version = "26.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/91/47e7d486260f618783899587af63ccf7980fb60245c3e63dd4571c6b57ad/pip-26.1.2.tar.gz", hash = "sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605", size = 1840799, upload-time = "2026-05-31T17:33:58.56Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/15/4500e320e6b101ec3b719ae85b697d9940b6cda672bc555bd6016fc60c6f/pip-26.2.1.tar.gz", hash = "sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f", size = 1848877, upload-time = "2026-08-04T22:51:14.148Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/95/6b5cb3461ea5673ba0995989746db58eb18b91b54dbf331e72f569540946/pip-26.1.2-py3-none-any.whl", hash = "sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab", size = 1813144, upload-time = "2026-05-31T17:33:56.772Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl", hash = "sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e", size = 1816632, upload-time = "2026-08-04T22:51:12.472Z" }, ] [[package]] @@ -1304,6 +1429,7 @@ name = "tsdhn" version = "0.0.1" source = { editable = "packages/tsdhn" } dependencies = [ + { name = "numba" }, { name = "numpy" }, { name = "pydantic" }, { name = "pygmt" }, @@ -1316,7 +1442,8 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "numpy", specifier = "==2.5.1" }, + { name = "numba", specifier = "==0.66.0" }, + { name = "numpy", specifier = "==2.4.6" }, { name = "pydantic", specifier = "==2.13.4" }, { name = "pygmt", specifier = "==0.19.0" }, { name = "pyyaml", specifier = "==6.0.3" }, @@ -1335,7 +1462,7 @@ dependencies = [ { name = "fastapi" }, { name = "minio" }, { name = "procrastinate" }, - { name = "psycopg", extra = ["binary"] }, + { name = "psycopg", extra = ["binary", "pool"] }, { name = "pydantic" }, { name = "tsdhn" }, { name = "uvicorn" }, @@ -1347,12 +1474,29 @@ requires-dist = [ { name = "fastapi", specifier = "==0.139.0" }, { name = "minio", specifier = ">=7.2.20" }, { name = "procrastinate", specifier = ">=3.9.0" }, - { name = "psycopg", extras = ["binary"], specifier = ">=3.3.4" }, + { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.3.4" }, { name = "pydantic", specifier = "==2.13.4" }, { name = "tsdhn", editable = "packages/tsdhn" }, { name = "uvicorn", specifier = "==0.51.0" }, ] +[[package]] +name = "tsdhn-parity" +version = "0.0.1" +source = { editable = "packages/tsdhn-parity" } +dependencies = [ + { name = "numpy" }, + { name = "pytest" }, + { name = "tsdhn" }, +] + +[package.metadata] +requires-dist = [ + { name = "numpy", specifier = "==2.4.6" }, + { name = "pytest", specifier = "==9.1.1" }, + { name = "tsdhn", editable = "packages/tsdhn" }, +] + [[package]] name = "typer" version = "0.26.8"