diff --git a/.agents/skills b/.agents/skills new file mode 120000 index 000000000..454b8427c --- /dev/null +++ b/.agents/skills @@ -0,0 +1 @@ +../.claude/skills \ No newline at end of file diff --git a/.claude/skills/.gitkeep b/.claude/skills/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/.codex/config.toml b/.codex/config.toml new file mode 100644 index 000000000..90013ed35 --- /dev/null +++ b/.codex/config.toml @@ -0,0 +1,6 @@ +[features] +memories = false + +[memories] +generate_memories = false +use_memories = false diff --git a/.github/workflows/test-web-backend-release.yml b/.github/workflows/test-web-backend-release.yml new file mode 100644 index 000000000..24200e0c7 --- /dev/null +++ b/.github/workflows/test-web-backend-release.yml @@ -0,0 +1,70 @@ +name: Web backend release dependencies + +on: + push: + branches: [master] + pull_request: + branches: [master] + +permissions: + contents: read + +jobs: + release-backend: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # Every test runs under a hard per-test limit, so one slow or stuck test + # fails by name instead of the whole job being cancelled. The XRF + # extraction tests run full detector calibrations and get their own job. + - suite: main + select: not xrf_slow + per_test_seconds: 120 + minutes: 25 + - suite: xrf + select: xrf_slow + per_test_seconds: 180 + minutes: 40 + name: release-backend (${{ matrix.suite }}) + timeout-minutes: ${{ matrix.minutes }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check deployment and watcher regressions + shell: bash + run: | + bash tests/test_deploy_xraylarch_web.sh + bash tests/test_deploy_release_dependencies.sh + bash tests/test_watcher.sh + bash tests/test_watcher_bootstrap.sh + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Build the release backend and verify dependencies in a clean environment + shell: bash + env: + SELECT: ${{ matrix.select }} + PER_TEST_SECONDS: ${{ matrix.per_test_seconds }} + run: | + set -Eeuo pipefail + python -m venv "$GITHUB_WORKSPACE/backend/.venv" + XRAYLARCH_WEB_TEST_MODE=1 source scripts/deploy-xraylarch-web.sh + REQUESTED_SHA="$(git rev-parse HEAD)" + validate_sha "$REQUESTED_SHA" + SAFE_PATH="$(dirname "$(command -v python)"):/usr/local/bin:/usr/bin:/bin" + DATA_ROOT="$RUNNER_TEMP/xraylarch-release-data" + CANDIDATE_DATA="$RUNNER_TEMP/xraylarch-release-candidate" + CLEAN_HOME="$CANDIDATE_DATA/home" + mkdir -p "$DATA_ROOT" "$CLEAN_HOME" "$CANDIDATE_DATA/runtime/pycache" + install_release_backend "$GITHUB_WORKSPACE" + verify_release_backend "$GITHUB_WORKSPACE" + # The per-test limit is a test-runner plugin only, added after the + # release environment has been verified, so it never reaches a release. + "$GITHUB_WORKSPACE/backend/.venv/bin/python" -m pip install --quiet pytest-timeout==2.3.1 + cd "$GITHUB_WORKSPACE/backend" + run_clean "$GITHUB_WORKSPACE/backend/.venv/bin/python" -m pytest -p no:cacheprovider -q \ + -m "$SELECT" --timeout="$PER_TEST_SECONDS" --durations=25 \ + --basetemp "$RUNNER_TEMP/xraylarch-full-backend-tests" tests diff --git a/.github/workflows/test-web-frontend.yml b/.github/workflows/test-web-frontend.yml new file mode 100644 index 000000000..907d5e91b --- /dev/null +++ b/.github/workflows/test-web-frontend.yml @@ -0,0 +1,66 @@ +name: Web frontend + +on: + push: + branches: [master] + pull_request: + branches: [master] + +permissions: + contents: read + +concurrency: + group: web-frontend-${{ github.ref }} + cancel-in-progress: true + +jobs: + frontend: + runs-on: ubuntu-latest + timeout-minutes: 35 + env: + NEXT_TELEMETRY_DISABLED: '1' + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version-file: frontend/.nvmrc + cache: npm + cache-dependency-path: frontend/package-lock.json + - run: npm ci + - run: npm run typecheck + - run: npm test + - run: npm run build + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install the release backend for browser acceptance + working-directory: . + shell: bash + run: | + set -Eeuo pipefail + python -m venv "$GITHUB_WORKSPACE/backend/.venv" + XRAYLARCH_WEB_TEST_MODE=1 source scripts/deploy-xraylarch-web.sh + REQUESTED_SHA="$(git rev-parse HEAD)" + validate_sha "$REQUESTED_SHA" + SAFE_PATH="$(dirname "$(command -v python)"):/usr/local/bin:/usr/bin:/bin" + DATA_ROOT="$RUNNER_TEMP/xraylarch-frontend-data" + CANDIDATE_DATA="$RUNNER_TEMP/xraylarch-frontend-candidate" + CLEAN_HOME="$CANDIDATE_DATA/home" + mkdir -p "$DATA_ROOT" "$CLEAN_HOME" "$CANDIDATE_DATA/runtime/pycache" + install_release_backend "$GITHUB_WORKSPACE" + - run: npx playwright install --with-deps chromium + - name: Measured spectra, saved EXAFS fits, and mounted integration lifecycle + run: npm run test:e2e -- --config=playwright.ci.config.ts + - name: Upload browser failure evidence + if: failure() + uses: actions/upload-artifact@v4 + with: + name: web-frontend-browser-failures + path: frontend/test-results + if-no-files-found: ignore + retention-days: 7 diff --git a/.gitignore b/.gitignore index bef821831..c092f7eac 100755 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,31 @@ Thumbs.db *iframe_figures .vscode/ .empty/ + +backend/.venv/ +backend/data/ +backend/.pytest_cache/ +backend/*.egg-info/ +frontend/node_modules/ +frontend/.next/ +frontend/.next-e2e/ +frontend/.next-e2e-mounted/ +frontend/.next-verify/ +frontend/.next-artemis/ +frontend/.next-artemis-check/ +frontend/.next-wavelet-resize-e2e/ +frontend/.next-*/ +!backend/tests/fixtures/demeter-athena-json.prj +!backend/tests/fixtures/athena-detector-probe.prj +!backend/tests/fixtures/demeter-mee-LaCoO3.prj +frontend/next-env.d.ts +frontend/*.tsbuildinfo +frontend/playwright-report/ +frontend/test-results/ .eggs/ .coverage *code-workspace +demo/data/ + +# Real-data outputs, numbers and screenshots: never committed. +demo/private/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..9fc2f353a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,319 @@ +# Shared project memory + +Read [CLAUDE.md](CLAUDE.md) for the shared memory bridge. Run the exact session +bootstrap `memoryctl load` command before project mutation, and publish durable +notes through `memoryctl` transactions. The API instructions below remain in force. + +# Driving this app without a browser + +Athena Web processes X-ray absorption spectra. This file is for an agent that has to +operate it through its HTTP API rather than through the UI — no browser, no screenshots. + +If you are here to *change the code*, read `README.md` and `CONTEXT.md` instead; +`CONTEXT.md` is the domain glossary and the two of us should argue in the same words. + +## Start the stack + +``` +PYTHONPATH=backend backend/.venv/bin/python -m uvicorn xraylarch_web.main:app \ + --port 8006 +cd frontend && npm run dev -- --port 3004 # only if you need the UI +``` + +The API is the whole instrument: `http://127.0.0.1:8006/api/athena`. The frontend at +`3004` proxies the same routes under `/api/backend/...` behind an allowlist. Talk to +`8006` directly unless you are specifically testing the proxy. + +## Always pass a view + +`GET /api/athena/projects/{id}` returns the project *with every array in it*: about +240,000 tokens for the bundled five-group example, near a million for twenty groups. +`POST .../command` answers with the whole project too, and each preview with the curves +it wants drawn. All three take `?view=summary`. + +| request | cost | what you get | +|---|---|---| +| `?view=summary` | ~740 tokens | every group's identity, state, E0, edge step, range, and what it was derived from | +| `POST .../command?view=summary` | ~1,000 tokens | the same summary, plus `last_operation` | +| `POST .../merge/preview?view=summary` | ~900 tokens | the preview with each curve replaced by `<611 numbers, 8786.2 .. 11352.9>` | +| `?view=parameters` | ~2,150 tokens | each group's recipe, requested against effective | +| `.../groups/{gid}/digest` | ~720 tokens | one spectrum characterised in numbers | +| `.../compare?groups=a,b,c` | ~850 tokens for five | each group against the first: shift, XANES and chi(k) differences, shared range, duplicates | +| `.../transcript` | ~120 tokens each | what has already been tried here, failures included | +| `GET /api/artemis/capabilities` | ~1,100 tokens | the fit and FEFF bodies, with their defaults | +| `GET /api/artemis/feff/jobs/{job}?view=summary` | ~170 tokens | a FEFF job's status and its paths, without the files | +| `POST /api/artemis/.../fit?view=summary` | ~800 tokens | one fit's values, per-path distances, `concerns` | +| *(no view)* | ~239,000 tokens | everything, arrays included | + +## The loop + +1. `GET /api/athena/capabilities`: every action in one line each, `undo` and `redo` + among them. +2. `GET /api/athena/capabilities/{action}`: one action in full, with its options' + types, bounds and defaults, its group selection, its preview endpoint, its traps. +3. `GET /api/athena/projects/{id}?view=summary`: what is there now, and the `version`. +4. Preview, if the action has a preview twin, with `?view=summary`. It saves nothing. +5. `POST /api/athena/projects/{id}/command?view=summary` with + `{version, action, group_ids, options}`. +6. Read the reply. A 200 is not proof: check `processing_error`, `warnings`, and on a + created group, `derived`. A `parameters` reply carries `last_operation.applied`, the + value you asked for beside the value Larch used, per group and key. + +A stale `version` is a 409 whose `recovery` names the version to resend. A body that +fails validation is a 422 whose `message` says what each field wanted. Nearly every +mutation is undoable with `action: "undo"`. + +`GET .../transcript` lists the commands issued against the project, rejected ones with +their errors, and previews marked `preview: true`. It returns the last twenty; +`?since=` returns those after a seq you have read. Read it before retrying +something that was refused. If a command times out, do not resend it blind: the retry's +version is stale and comes back as a 409. Send an `Idempotency-Key` header instead, and +a retry under a used key is answered from the record, marked +`last_operation.idempotent_replay`. `POST .../import` takes the same header: a +retry under a used key imports nothing and returns the current project, with the +groups the first import made in `idempotent_replay.group_ids`. + +## Or use the CLI + +`larchctl --help`, with the alias below. It fetches the version, resolves groups by +label (in `group_ids` and in `standard_id`, `reference_id` and +`background_standard_id`, which take ids over HTTP), elides arrays, and prints tables. + +``` +alias larchctl="PYTHONPATH=backend backend/.venv/bin/python -m xraylarch_web.larchctl" +export LARCHCTL_PROJECT=$(larchctl new --name "Cu series" | head -1) +larchctl do example +larchctl summary +larchctl digest "10 K" +larchctl compare "10 K" "50 K" "300 K" +larchctl describe # every action +larchctl describe merge # one action +larchctl do parameters "10 K" -o kmax=12 -o kweight=3 +larchctl do merge "10 K" "50 K" -o method=demeter-larch --preview +larchctl do rebin "10 K" --key retry-1 +larchctl export "10 K" --space k --out chi.csv +larchctl log --since 4 +``` + +- `--json`, before or after the subcommand, prints JSON. Arrays stay elided unless you + add `--arrays`. +- `-o key=value` reads the value as JSON, then as a bare string: `-o kmax=18`, + `-o reference_id=null` and `-o exclude_short_data=false` arrive typed, and + `-o window=hanning` needs no quotes. +- A label matches a whole group label first, then a whole part between its `·`s + ("300 K" is "Cu foil · 300 K", not a merge whose label lists 300 K), then any + unambiguous part. +- After a command it prints `+` for a group created, `-` for one removed, and `~` for + each field moved on a group it kept, such as `energy_shift 0.000->-2.959`. + +## Reading numbers correctly + +**Requested is not effective.** You ask for `kmax`; Larch clips it to the data and uses +something else. `?view=parameters` and the digest report both side by side, and the CLI +prints `auto->25.019` when nothing was asked for and `18.000` when a value was honoured. +Compare against the effective value. + +**XRF-made groups use edge-step units.** Their steps and fitted/window step ratio are about one by construction; recover raw yield scales from `source.extraction.raw_edge_steps`. + +**Ranges are on the shifted axis.** An energy range already has `energy_shift` folded +in, and `energy_shift` is reported beside it. + +**|chi(R)| peaks are not bond lengths.** The digest applies no phase correction, so a +peak sits roughly 0.2–0.5 Å below the true distance. Peaks are for recognising +structure; fit for distances (below). + +**Choosing a kmax.** A wide gap between `kmax` and `available_kmax` means the transform +runs into data a plot would show as noise. The digest's `signal_to_noise` bins rms chi(k) +in 2 Å⁻¹ windows against a floor measured over the whole k support, so it does not move +when the transform range does; a window near 1 is noise. The ratios compare windows +within one group, not groups with each other, since each group has its own floor. +`noise.epsilon_k` and `noise.recommended_kmax` are Larch's, measured over the current +transform range: the first moves with that range, and the second runs pessimistic. +When `noise.recommended_kmax_applicable` is false, the `parameters` command would refuse +that kmax as it stands; `noise.warnings` says whether it lies past the measured k or below +the saved kmin. + +**To ask whether groups are comparable, compare them.** `GET .../compare?groups=a,b,...` +measures each group after the first against the first, reading and never applying: + +- `energy_shift`: the shift align would fit now, relative to the group's current one. + It works on linked groups that align refuses to move. +- `e0_difference` and `edge_step_ratio`. +- `xanes_max_difference`: the largest |norm difference| from E0 −20 to +50 eV. +- `common_range`: the energy support the two share. +- `chi_amplitude`: rms of k-weighted chi(k) against the first group, per 2 Å⁻¹ window. + A ratio falling with k is Debye–Waller damping. +- `same_data_as`: groups holding the identical measurement. + +**The same file twice is the same measurement.** The summary's `same_data` lists groups +whose raw arrays are identical. Each group carries the `file` it was read from, and the +digest its `citation`, which says where and when it was measured. + +### Aligning + +**Linked groups move together, so align refuses them.** Groups whose `reference_id` +points at one reference are one family; a shift to one shifts them all. Align will not +move a group in the standard's family, and refuses the command when nothing else is +left: "The alignment standard and its linked references stay fixed." `larchctl summary` +shows the link as `ref: