Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions backend/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,17 +112,19 @@
async def lifespan(app: FastAPI):
# The anonymous-admin bypass, said out loud once per boot.
#
# AUTH_ALLOW_ANONYMOUS_ADMIN=1 is set in every environment while OAuth is
# unconfigured, so the flag has stopped being noticed — and it is not one
# guard among several, it is the whole of the admin boundary
# AUTH_ALLOW_ANONYMOUS_ADMIN=1 is a local-development convenience that no
# deployed environment sets, so this line on a droplet means the bypass has
# been restored to one — and it is not one guard among several, it is the
# whole of the admin boundary
# (core.users._derive_auth_flags, tests/test_auth.py). The public surfaces
# go to some length to publish displaced receiver positions and to withhold
# private nodes entirely; the admin surfaces behind that boundary serve the
# true ones, and while this is set they serve them to anyone who asks.
#
# A log line, not a refusal: turning it off here would take every
# environment's admin access with it, and the default is deliberately not
# this module's to change. WARNING level so it survives the default
# A log line, not a refusal: local development has no Access assertion and
# no OAuth, so refusing here would leave a laptop with no way into the
# console, and the default is deliberately not this module's to change.
# WARNING level so it survives the default
# LOG_LEVEL and lands in the deploy's own logs rather than only in a
# developer's terminal.
from core.users import AUTH_BYPASS
Expand Down
68 changes: 68 additions & 0 deletions backend/tests/test_anonymous_admin_closed.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
"""No deployed environment may hand out an anonymous administrator.

AUTH_ALLOW_ANONYMOUS_ADMIN makes require_admin return a superuser to any caller,
on every vhost, since the same app answers all of them. ClickUp 86cb1emcx records
what that cost: 1966 irreversible deletes through /api/admin/* in one week, by
nobody in particular.

deploy/check-env-parity.py already refuses to let the three deployed overlays
disagree about it, which catches reintroducing it to one of them. It cannot catch
reintroducing it to all three at once, and that is what this covers.
"""

import re
from pathlib import Path

import pytest

_REPO = Path(__file__).resolve().parents[2]
_FLAG = "AUTH_ALLOW_ANONYMOUS_ADMIN"

#: Everything a real user can reach.
DEPLOYED = ["docker-compose.prod.yml", "docker-compose.staging.yml", "docker-compose.test.yml"]


def _sets_the_flag(overlay: str) -> bool:
text = (_REPO / overlay).read_text()
# An assignment, not the name: the surrounding comment mentions it too.
return bool(re.search(rf"^\s*-\s*{_FLAG}=", text, re.M))


@pytest.mark.parametrize("overlay", DEPLOYED)
def test_no_deployed_overlay_opts_into_the_anonymous_admin(overlay: str) -> None:
assert not _sets_the_flag(overlay), (
f"{overlay} sets {_FLAG}, which serves an anonymous superuser to anyone "
f"who asks, on every vhost. If an environment genuinely needs it, that "
f"is a decision to argue for rather than a line to restore."
)


@pytest.mark.parametrize("overlay", DEPLOYED)
def test_no_deployed_overlay_leaves_the_verifier_unconfigured(overlay: str) -> None:
"""The other half of the same guarantee.

With the bypass gone and no audience set, an environment refuses everybody
including the people who are supposed to get in, and the admin console is
simply unusable there. Failing closed is right, but it should not be reached
by forgetting something.
"""
text = (_REPO / overlay).read_text()
assert re.search(r"^\s*-\s*CF_ACCESS_AUD=\S", text, re.M), (
f"{overlay} sets no CF_ACCESS_AUD, so no Access assertion can be verified "
f"there and every admin request is refused. Create that environment's "
f"Access application and pin its audience here."
)


def test_local_may_keep_the_bypass() -> None:
"""Pinned deliberately, so nobody 'tidies' it away.

A laptop has no Access assertion and no OAuth, the surfaces bind to
localhost, and a developer needs the console. docker-compose.local.yml is
outside check-env-parity.py's OVERLAYS for the related reason that it renders
the template's plain-HTTP branch.
"""
assert _sets_the_flag("docker-compose.local.yml"), (
"docker-compose.local.yml no longer opts into the anonymous admin, so "
"local development cannot reach the admin surfaces at all."
)
8 changes: 4 additions & 4 deletions deploy/check-env-parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,10 @@
# entry, long after the fact. See backend/config/constants.py.
r"^services\.server\.environment\.NODE_FORCE_RETIRE_PREFIXES$",
# AUTH_ALLOW_ANONYMOUS_ADMIN and SYNTHETIC_FLEET_ENABLED are deliberately
# absent from this list: each is set to the same value in every environment,
# so a difference is drift rather than a decision, and CI should fail if one
# environment closes the bypass, or drops the simulation subsystem, without
# the others.
# absent from this list, so a difference between environments is drift
# rather than a decision. SYNTHETIC_FLEET_ENABLED is set in all three;
# AUTH_ALLOW_ANONYMOUS_ADMIN in none, and keeping it off this list is what
# fails CI if it is ever restored to one environment alone.
# Published ports. Production exposes 3012 for real receiver nodes; staging
# has none and closes it, so the two legitimately differ here. Recorded rather
# than silently allowed: if staging ever needs node ingest, it should be
Expand Down
7 changes: 2 additions & 5 deletions docker-compose.prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,8 @@ services:
# bypass onto its own flag, so production can name itself honestly and
# keep all four.
- RETINA_ENV=production
# The bypass itself, now explicit: no OAuth is configured yet, so every
# caller is treated as an anonymous admin. This is the open door, and the
# only thing here that should worry a reader. Closing it (OAuth keys in
# backend/.env, then drop this line) is tracked in ClickUp 86cb1emcx.
- AUTH_ALLOW_ANONYMOUS_ADMIN=1
# This environment's Access application, on admin.retina.fm.
- CF_ACCESS_AUD=44f723ed886782d1e85a5a2cd39a4bd96fded1791b2c2fbfebbced5e217aa5f0
# The simulation subsystem, likewise asked for by name: for now the ingest
# write path, in time the fleet container as well, which is why the flag is
# named for the subsystem rather than for that one route. Every environment
Expand Down
6 changes: 2 additions & 4 deletions docker-compose.staging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,8 @@ services:
# in ClickUp 86cb1emcx; revert to the real environment name there, not
# here.
- RETINA_ENV=test
# Anonymous admin, asked for explicitly since ClickUp 86cb49d29 took it
# off RETINA_ENV. Staging has no OAuth configured, so without this line
# the dashboard and every admin route start answering 401.
- AUTH_ALLOW_ANONYMOUS_ADMIN=1
# This environment's Access application, on staging-admin.retina.fm.
- CF_ACCESS_AUD=e81258e72360b51069e64c82ca9762b52556898d8096b20873833ff50f7eb66e
# The E2E suite retires the nodes it registers, which needs force because
# a registered node is live until something evicts it. Confining force to
# the suite's own prefixes means a bug in the teardown cannot reach a real
Expand Down
9 changes: 1 addition & 8 deletions docker-compose.test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,7 @@ services:
# replay put 20 at a tenth of the turning breaks for 20% worse Doppler RMS;
# main keeps the library's 0.5 until this measures the same live.
- TRACKER_PROCESS_NOISE_DOPPLER=20
# Anonymous admin, asked for explicitly since ClickUp 86cb49d29 took it
# off RETINA_ENV. No OAuth is configured here either.
- AUTH_ALLOW_ANONYMOUS_ADMIN=1
# Audience of this environment's Access application, on test-admin.retina.fm.
# Pinning it is what stops a token minted for one of the fleet's node
# applications being accepted here; the team domain they share is in the
# base compose file. Staging and production carry their own once their
# applications exist.
# This environment's Access application, on test-admin.retina.fm.
- CF_ACCESS_AUD=e5ff9de8d1ca5fbc62b38d102d92a1fc7d910d5f89ef388caf63c83e828493b3
# The simulation subsystem, likewise asked for by name rather than
# inherited from what this environment is called. This droplet is a
Expand Down
5 changes: 3 additions & 2 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,9 @@ services:
# One team for the whole org, so it belongs here rather than in an overlay:
# a per-environment copy could drift to another team and the audience check
# alone would not catch it. The per-application audience does differ, and
# is CF_ACCESS_AUD in each overlay. Not a secret — it appears in the login
# URL of every Access redirect.
# is CF_ACCESS_AUD in each overlay: pinning it is what stops a token minted
# for one of the fleet's seventeen node applications opening the console.
# Not a secret — it appears in the login URL of every Access redirect.
- CF_ACCESS_TEAM_DOMAIN=offworldlab.cloudflareaccess.com
# bash, not sh: start.sh's supervisor uses `wait -n`, a bash builtin that
# dash rejects ("Illegal option -n"), which crash-loops the server. The
Expand Down
13 changes: 10 additions & 3 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,16 @@ per-node trust residuals, and the feed's `adsb_single_node` display section).
## Auth model

Cookie-based JWT issued via OAuth (Google/GitHub), shared across surfaces on the
same origin. `AUTH_ALLOW_ANONYMOUS_ADMIN=1` with no OAuth configured grants the
anonymous-admin bypass, independent of `RETINA_ENV`; every environment currently
sets it while OAuth is unconfigured. Node ownership maps
same origin. Administrators arrive instead through Cloudflare Access: the origin
verifies the `Cf-Access-Jwt-Assertion` itself against the team's published keys,
with `aud` pinned per environment to `CF_ACCESS_AUD`, and the verified email is
the identity (`backend/core/access_identity.py`). Enforcement is backend-side
because every vhost proxies `/api/` to the same app, so gating one hostname at
the edge would protect that hostname's HTML and nothing else; it is also why
`api.retina.fm`, the fleet's ingest hostname, carries no Access application.
`AUTH_ALLOW_ANONYMOUS_ADMIN=1` still grants the anonymous-admin bypass,
independent of `RETINA_ENV`, but only `docker-compose.local.yml` sets it. Node
ownership maps
`node_id → user_id`; the `/ws/aircraft/owner` feed and dashboard use it to scope
data to a user's own nodes.

Expand Down
31 changes: 22 additions & 9 deletions docs/runbook.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,6 @@ arriving on production means it is working; confirm it positively by checking
that the real nodes appear in the test droplet's `/api/radar/analytics`, which
names them by `node_ref` rather than by node id.

Real receiver and transmitter geometry now lands on a droplet running
`AUTH_ALLOW_ANONYMOUS_ADMIN=1`.

---

## Server basics
Expand All @@ -124,9 +121,25 @@ advertised. Get them from the DigitalOcean console or your own `~/.ssh/config`.
| **Restart (no rebuild)** | `docker compose restart` |
| **Rebuild and restart** | `docker compose up -d --build` (wait ~5 s before testing) |
| **Health endpoint** | `curl -sk https://localhost/api/health` |
| **Metrics endpoint** | `curl -sk https://localhost/api/admin/metrics` |
| **Metrics endpoint** | `adm /api/admin/metrics` (defined below) |
| **Dashboard** | `curl -sk https://localhost/api/test/dashboard` |

`/api/admin/*` requires an administrator, and a request to localhost arrives
below Cloudflare with no assertion for the origin to verify, so a bare `curl`
answers 401. Define `adm` once per shell from a browser session on the console:

```bash
# Sign in at the admin console, then copy the CF_Authorization cookie
# (devtools, Application, Cookies). It holds the same signed assertion that
# Cloudflare injects on a proxied request, so the origin verifies it identically.
export CF_ASSERTION='<the cookie value>'
adm() { curl -sk -H "Cf-Access-Jwt-Assertion: ${CF_ASSERTION}" "https://localhost$1"; }
```

It expires with the Access session and carries the email of whoever signed in, so
anything you reach with it is attributed to that person in `/api/admin/events`.
Every `adm` below assumes it.

All state is **in-memory**. A container restart loses all connected nodes, active tracks, and in-flight frame data. State is snapshotted to disk every 60 s and restored on next startup (trust scores, reputations, accuracy samples, node identities).

### Database migrations
Expand Down Expand Up @@ -242,7 +255,7 @@ uptime monitor). Details are never exposed on the endpoint — read them from lo

```bash
docker compose logs --tail=200 | grep "Health check degraded"
curl -sk https://localhost/api/admin/metrics | python3 -m json.tool
adm /api/admin/metrics | python3 -m json.tool
```

---
Expand Down Expand Up @@ -361,7 +374,7 @@ What does catch it is the delay residual: compare the node's published `adsb[].e

**Check:**
```bash
curl -sk https://localhost/api/admin/metrics | python3 -c \
adm /api/admin/metrics | python3 -c \
"import sys,json; m=json.load(sys.stdin); print('queue_pct:', m['solver_queue_pct'], 'drops:', m['solver_queue_drops'], 'avg_latency:', m['solver_avg_latency_s'])"
```

Expand Down Expand Up @@ -506,8 +519,8 @@ working, so this alert means a reading well outside even that.

**Check per-node miss rates:**
```bash
curl -sk https://localhost/api/admin/leaderboard | python3 -c \
"import sys,json; rows=json.load(sys.stdin); [print(r['node_id'], r.get('miss_rate','?')) for r in rows]"
adm /api/admin/leaderboard | python3 -c \
"import sys,json; rows=json.load(sys.stdin)['leaderboard']; [print(r['node_ref'], r.get('miss_rate','?')) for r in rows]"
```

**Common causes:**
Expand All @@ -534,7 +547,7 @@ its own history.
```bash
# On server — check if backup exists on R2:
# (if R2 is configured)
curl -sk https://localhost/api/admin/storage
adm /api/admin/storage
```

Server will start with empty state if snapshot is corrupt. Trust scores and reputation data need to rebuild from scratch — this takes hours under normal node load. Not a functional outage.
Expand Down
18 changes: 5 additions & 13 deletions frontend/e2e/api.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,20 +137,12 @@ test.describe("API admin endpoints", () => {
await ctx.dispose();
});

test("GET /api/admin/leaderboard returns per-node list", async () => {
// 401 rather than a body: the api vhost has no Access application in front of
// it, so an anonymous caller is refused by require_admin itself. See the same
// assertion in dashboard.spec.ts for why that hostname must stay ungated.
test("GET /api/admin/leaderboard refuses an anonymous caller", async () => {
const res = await ctx.get(`${API}/api/admin/leaderboard`);
expect(res.status()).toBe(200);

const body = await res.json();
// Response shape: {leaderboard: [...], total: N}
expect(body).toHaveProperty("leaderboard");
expect(Array.isArray(body.leaderboard)).toBe(true);
// Each entry has required fields
if (body.leaderboard.length > 0) {
const first = body.leaderboard[0];
expect(first).toHaveProperty("node_ref");
expect(first).toHaveProperty("name");
}
expect(res.status()).toBe(401);
});

// Deliberately on the frontend host, not API: /api/config is served by
Expand Down
Loading
Loading