From 45ef3e6925f68f0b85dc585a41055d0b373dfa11 Mon Sep 17 00:00:00 2001 From: Joseph McCarron Date: Wed, 13 May 2026 08:38:12 -0700 Subject: [PATCH 1/5] refactor(api)!: rename /v1/detect -> /v1/find, /v1/sanitize -> /v1/replace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User-facing terminology landed for new consumers: - Paths: `/v1/detect` -> `/v1/find` ("Find sensitive data"); `/v1/sanitize` -> `/v1/replace` ("Replace sensitive data"). - Schemas: `DetectRequest`/`DetectResponse` -> `FindRequest`/`FindResponse`; `SanitizeRequest`/`SanitizeResponse` -> `ReplaceRequest`/`ReplaceResponse`; `SanitizedSpan` -> `ReplacedSpan`; `SanitizeMode` -> `ReplaceMode`. - Response body: `sanitized_text` -> `replaced_text` (pairs naturally with the per-span `replacement` field). - Tags: `Detect`/`Sanitize` -> `Find`/`Replace`. - Internal: `_run_detect` -> `_run_find`, `_DetectInput` -> `_FindInput`, route handlers `detect`/`sanitize` -> `find`/`replace`. - Bump `info.version` 0.3.0 -> 0.4.0 (breaking shape change pre-1.0). - Rename `docs/guides/sanitize-modes.md` -> `replace-modes.md`; update README + every guide ref. Regenerate spec. Old route names are gone — no aliases. Pre-1.0 contract; clients on the old names need a one-line change. Backend method `detector.detect()` (in opf_eval) keeps its name — that's the in-process model API, not the public HTTP contract, and renaming it spans an external package. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 4 +- api/pyproject.toml | 2 +- api/src/opf_api/main.py | 14 +- api/src/opf_api/routes.py | 60 +-- api/src/opf_api/schemas.py | 38 +- api/tests/test_routes.py | 102 ++--- docs/api/openapi.json | 392 +++++++++--------- docs/api/openapi.yaml | 274 ++++++------ docs/guides/auth-and-env.md | 2 +- docs/guides/overview.md | 4 +- .../{sanitize-modes.md => replace-modes.md} | 6 +- uv.lock | 2 +- 12 files changed, 450 insertions(+), 450 deletions(-) rename docs/guides/{sanitize-modes.md => replace-modes.md} (82%) diff --git a/README.md b/README.md index 0b415d7..687821d 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,7 @@ Three in-app reference UIs render the same spec — `/scalar` (Scalar), `/docs` - [overview](docs/guides/overview.md) — what the API does and how to run it. - [detectors](docs/guides/detectors.md) — the registered backends. - [labels](docs/guides/labels.md) — the 15-label canonical taxonomy. -- [sanitize modes](docs/guides/sanitize-modes.md) — the four `/v1/sanitize` modes. +- [replace modes](docs/guides/replace-modes.md) — the four `/v1/replace` modes. - [auth and env](docs/guides/auth-and-env.md) — env-var matrix and error codes. ### Server env @@ -301,4 +301,4 @@ Three in-app reference UIs render the same spec — `/scalar` (Scalar), `/docs` | `OPF_DEVICE` | `cpu`, `cuda`, `mps`. Default `cpu`. | | `OPF_DECODE_MODE` | `viterbi` or `argmax`. OPF-only. Default `viterbi`. | | `SKYFLOW_VAULT_URL` / `_ID` / `_BEARER_TOKEN` | Required for the `skyflow` **detector**. | -| `SKYFLOW_TOKEN_VAULT_URL` / `_ID` / `_BEARER_TOKEN` | Required for `/v1/sanitize` `label_token` mode. See [docs/token-vault-setup.md](docs/token-vault-setup.md). | +| `SKYFLOW_TOKEN_VAULT_URL` / `_ID` / `_BEARER_TOKEN` | Required for `/v1/replace` `label_token` mode. See [docs/token-vault-setup.md](docs/token-vault-setup.md). | diff --git a/api/pyproject.toml b/api/pyproject.toml index 32a0ab6..ec36dd0 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "opf-api" -version = "0.3.0" +version = "0.4.0" description = "Unified privacy-detection API — one contract, multiple backends" requires-python = ">=3.10" dependencies = [ diff --git a/api/src/opf_api/main.py b/api/src/opf_api/main.py index 8bd4d60..afd3d3d 100644 --- a/api/src/opf_api/main.py +++ b/api/src/opf_api/main.py @@ -60,8 +60,8 @@ async def lifespan(app: FastAPI): ## Endpoints -- `POST /v1/detect` — return spans only, no text rewriting. -- `POST /v1/sanitize` — detect and rewrite spans under one of four modes. +- `POST /v1/find` — find sensitive data; return spans only, no text rewriting. +- `POST /v1/replace` — find spans and replace each one under one of four modes. - `GET /v1/detectors` — list registered detectors and their category coverage. - `GET /v1/health` — liveness probe; does not exercise detector backends. @@ -82,13 +82,13 @@ async def lifespan(app: FastAPI): OPENAPI_TAGS = [ { - "name": "Detect", - "description": "Detection-only endpoint. Returns spans without rewriting the input.", + "name": "Find", + "description": "Find-only endpoint. Returns spans without rewriting the input.", }, { - "name": "Sanitize", + "name": "Replace", "description": ( - "Detection plus rewriting under one of four modes: " + "Find plus rewriting under one of four modes: " "`redact`, `label`, `label_number`, `label_token`." ), }, @@ -100,7 +100,7 @@ async def lifespan(app: FastAPI): app = FastAPI( title="Privacy-detection API", - version="0.3.0", + version="0.4.0", summary="Unified PII detection across OPF, GLiNER, Presidio, and Skyflow.", description=API_DESCRIPTION, lifespan=lifespan, diff --git a/api/src/opf_api/routes.py b/api/src/opf_api/routes.py index b51bb72..6f9d382 100644 --- a/api/src/opf_api/routes.py +++ b/api/src/opf_api/routes.py @@ -22,12 +22,12 @@ DetectorInfo, DetectorsResponse, DetectorOptions, - DetectRequest, - DetectResponse, + FindRequest, + FindResponse, HealthResponse, - SanitizedSpan, - SanitizeRequest, - SanitizeResponse, + ReplacedSpan, + ReplaceRequest, + ReplaceResponse, SpanOut, ) from .vault_tokens import TokenVaultClient @@ -129,14 +129,14 @@ def _resolve_detector(request: Request, name: str | None) -> tuple[str, Detector return chosen, entry -class _DetectInput(Protocol): +class _FindInput(Protocol): text: str detector: str | None categories: list[CanonicalLabel] | None options: DetectorOptions | None -async def _run_detect(request: Request, body: _DetectInput) -> tuple[str, list[Span]]: +async def _run_find(request: Request, body: _FindInput) -> tuple[str, list[Span]]: name, entry = _resolve_detector(request, body.detector) detector = await entry.get() @@ -158,13 +158,13 @@ async def _run_detect(request: Request, body: _DetectInput) -> tuple[str, list[S @router.post( - "/detect", - response_model=DetectResponse, - tags=["Detect"], - summary="Detect PII spans in text", + "/find", + response_model=FindResponse, + tags=["Find"], + summary="Find sensitive data in text", description=( "Run the chosen detector over `text` and return canonical-labelled spans.\n\n" - "No rewriting is performed — call `/v1/sanitize` for that.\n\n" + "No rewriting is performed — call `/v1/replace` for that.\n\n" "Filter detector output to a subset of canonical labels with `categories`.\n" "OPF-only: pass `options.opf.decode_mode` to override the default Viterbi decoding." ), @@ -175,9 +175,9 @@ async def _run_detect(request: Request, body: _DetectInput) -> tuple[str, list[S 502: _ERROR_502_EXAMPLE, }, ) -async def detect(request: Request, body: DetectRequest) -> DetectResponse: - """Detect-only: returns spans, no text rewriting.""" - name, spans = await _run_detect(request, body) +async def find(request: Request, body: FindRequest) -> FindResponse: + """Find-only: returns spans, no text rewriting.""" + name, spans = await _run_find(request, body) ordered = sorted(spans, key=lambda s: (s["start"], s["end"])) out_spans = [ SpanOut( @@ -190,7 +190,7 @@ async def detect(request: Request, body: DetectRequest) -> DetectResponse: for s in ordered ] by_label = Counter(s.label for s in out_spans) - return DetectResponse( + return FindResponse( detector=name, text=body.text, detected_spans=out_spans, @@ -212,12 +212,12 @@ def _build_label_token_renderer_raising_http( @router.post( - "/sanitize", - response_model=SanitizeResponse, - tags=["Sanitize"], - summary="Detect and rewrite PII spans", + "/replace", + response_model=ReplaceResponse, + tags=["Replace"], + summary="Replace sensitive data in text", description=( - "Detect spans, then rewrite each one under the chosen `mode`. Four modes, " + "Find spans, then rewrite each one under the chosen `mode`. Four modes, " "in increasing strength of identity preservation:\n\n" "| `mode` | Looks like | What it preserves |\n" "|---|---|---|\n" @@ -228,7 +228,7 @@ def _build_label_token_renderer_raising_http( "| `label_token` | `[EMAIL_MGaE1Bo]` | Identity **across requests and detectors** via a " "Skyflow vault. Deterministic — same plaintext maps to the same 7-char token forever. |\n\n" "**Overlapping spans:** the earlier-starting span wins; later overlaps are skipped in " - "`sanitized_text` (they still appear in `detected_spans`).\n\n" + "`replaced_text` (they still appear in `detected_spans`).\n\n" "**`label_token` requirements:** `SKYFLOW_TOKEN_VAULT_URL`, `SKYFLOW_TOKEN_VAULT_ID`, " "and a bearer (`SKYFLOW_TOKEN_BEARER_TOKEN`, falling back to `SKYFLOW_BEARER_TOKEN`). " "The vault must be configured per the token-vault setup guide — one table with one " @@ -236,15 +236,15 @@ def _build_label_token_renderer_raising_http( "`^[A-Za-z0-9]{7}$`. Spans whose canonical label has no vault column fall back to " "`[LABEL]` for that span only." ), - response_description="Sanitized text, the spans that were rewritten, and per-label counts.", + response_description="Replaced text, the spans that were rewritten, and per-label counts.", responses={ 400: _ERROR_400_EXAMPLE, 422: _ERROR_422_EXAMPLE, 502: _ERROR_502_EXAMPLE, }, ) -async def sanitize(request: Request, body: SanitizeRequest) -> SanitizeResponse: - """Detect + rewrite the input text under the chosen `mode`. +async def replace(request: Request, body: ReplaceRequest) -> ReplaceResponse: + """Find + rewrite the input text under the chosen `mode`. Modes: - `redact` -> fixed-length asterisks (`********`) @@ -252,7 +252,7 @@ async def sanitize(request: Request, body: SanitizeRequest) -> SanitizeResponse: - `label_number` -> `[EMAIL_1]`, per-request counter - `label_token` -> `[EMAIL_jRc7QGn]`, deterministic Skyflow vault token """ - name, spans = await _run_detect(request, body) + name, spans = await _run_find(request, body) mode = body.mode if mode == "redact": @@ -289,7 +289,7 @@ async def sanitize(request: Request, body: SanitizeRequest) -> SanitizeResponse: rendered_pairs: list[tuple[Span, str]] = [(s, render(s)) for s in ordered] out_spans = [ - SanitizedSpan( + ReplacedSpan( label=s["label"], raw_label=s["raw_label"], start=s["start"], @@ -299,15 +299,15 @@ async def sanitize(request: Request, body: SanitizeRequest) -> SanitizeResponse: ) for s, replacement in rendered_pairs ] - sanitized = splice_pieces(body.text, rendered_pairs) + replaced = splice_pieces(body.text, rendered_pairs) by_label = Counter(s.label for s in out_spans) - return SanitizeResponse( + return ReplaceResponse( detector=name, mode=mode, text=body.text, detected_spans=out_spans, - sanitized_text=sanitized, + replaced_text=replaced, summary={"span_count": len(out_spans), "by_label": dict(by_label)}, warning=None, ) diff --git a/api/src/opf_api/schemas.py b/api/src/opf_api/schemas.py index fc51c0d..6caca61 100644 --- a/api/src/opf_api/schemas.py +++ b/api/src/opf_api/schemas.py @@ -10,12 +10,12 @@ DecodeMode = Literal["viterbi", "argmax"] -# Four sanitization modes, in increasing strength of identity preservation: +# Four replacement modes, in increasing strength of identity preservation: # redact -> "********" (fixed-length asterisks; no information leaks) # label -> "[EMAIL]" (default; category label only) # label_number -> "[EMAIL_1]" (per-request counters; duplicates reuse numbers) # label_token -> "[EMAIL_jRc7QGn]" (deterministic Skyflow vault token) -SanitizeMode = Literal["redact", "label", "label_number", "label_token"] +ReplaceMode = Literal["redact", "label", "label_number", "label_token"] class CanonicalLabel(str, Enum): @@ -77,8 +77,8 @@ class DetectorOptions(BaseModel): ) -class DetectRequest(BaseModel): - """Base request for `/v1/detect` — no text-rewriting fields.""" +class FindRequest(BaseModel): + """Base request for `/v1/find` — no text-rewriting fields.""" model_config = ConfigDict( json_schema_extra={ @@ -126,8 +126,8 @@ class DetectRequest(BaseModel): ) -class SanitizeRequest(DetectRequest): - """Request for `/v1/sanitize` — `DetectRequest` plus a `mode` field that picks +class ReplaceRequest(FindRequest): + """Request for `/v1/replace` — `FindRequest` plus a `mode` field that picks how detected spans are rewritten.""" model_config = ConfigDict( @@ -142,7 +142,7 @@ class SanitizeRequest(DetectRequest): } ) - mode: SanitizeMode = Field( + mode: ReplaceMode = Field( default="label", description=( "How to rewrite each detected span. " @@ -156,8 +156,8 @@ class SanitizeRequest(DetectRequest): ) -class SanitizedSpan(BaseModel): - """One detected span plus the string it was rewritten to in `sanitized_text`.""" +class ReplacedSpan(BaseModel): + """One detected span plus the string it was rewritten to in `replaced_text`.""" label: str = Field( ..., @@ -176,7 +176,7 @@ class SanitizedSpan(BaseModel): ) replacement: str = Field( ..., - description="The string this span was rewritten to in `sanitized_text`.", + description="The string this span was rewritten to in `replaced_text`.", examples=["[EMAIL_MGaE1Bo]"], ) @@ -194,8 +194,8 @@ class SummaryOut(BaseModel): ) -class SanitizeResponse(BaseModel): - """Response from `/v1/sanitize`.""" +class ReplaceResponse(BaseModel): + """Response from `/v1/replace`.""" model_config = ConfigDict( json_schema_extra={ @@ -204,7 +204,7 @@ class SanitizeResponse(BaseModel): "detector": "presidio", "mode": "label_token", "text": "Email alice@x.com or call +1-415-555-0100.", - "sanitized_text": "Email [EMAIL_MGaE1Bo] or call [PHONE_vRXiWKZ].", + "replaced_text": "Email [EMAIL_MGaE1Bo] or call [PHONE_vRXiWKZ].", "detected_spans": [ { "label": "EMAIL", @@ -235,16 +235,16 @@ class SanitizeResponse(BaseModel): description="Detector that produced the spans.", examples=["presidio"], ) - mode: SanitizeMode = Field( + mode: ReplaceMode = Field( ..., description="Echo of the request `mode`.", examples=["label_token"] ) text: str = Field( ..., description="Echo of the request `text`.", examples=["Email alice@x.com."] ) - detected_spans: list[SanitizedSpan] = Field( + detected_spans: list[ReplacedSpan] = Field( ..., description="Spans detected by the chosen detector, with their replacements." ) - sanitized_text: str = Field( + replaced_text: str = Field( ..., description=( "`text` with each detected span replaced by its `replacement`. " @@ -261,7 +261,7 @@ class SanitizeResponse(BaseModel): class SpanOut(BaseModel): - """Plain span for `/v1/detect` — no replacement text.""" + """Plain span for `/v1/find` — no replacement text.""" label: str = Field(..., description="Canonical label.", examples=["EMAIL"]) raw_label: str = Field( @@ -276,8 +276,8 @@ class SpanOut(BaseModel): ) -class DetectResponse(BaseModel): - """Response from `/v1/detect`.""" +class FindResponse(BaseModel): + """Response from `/v1/find`.""" model_config = ConfigDict( json_schema_extra={ diff --git a/api/tests/test_routes.py b/api/tests/test_routes.py index 00f22f9..718b38e 100644 --- a/api/tests/test_routes.py +++ b/api/tests/test_routes.py @@ -1,7 +1,7 @@ """End-to-end route tests using a stub detector — avoids loading real models. The tests inject a deterministic FakeDetector into the registry and exercise -the /v1 surface for /detect, /sanitize, /detectors, /health. +the /v1 surface for /find, /replace, /detectors, /health. """ from __future__ import annotations @@ -119,27 +119,27 @@ def _client(**kwargs): return AsyncClient(transport=transport, base_url="http://test") -# --- /v1/detect ----------------------------------------------------------- +# --- /v1/find ----------------------------------------------------------- @pytest.mark.asyncio -async def test_detect_default(): +async def test_find_default(): async with _client() as c: - r = await c.post("/v1/detect", json={"text": "Joe at joe@example.com lives in Elgin, TX."}) + r = await c.post("/v1/find", json={"text": "Joe at joe@example.com lives in Elgin, TX."}) assert r.status_code == 200, r.text body = r.json() assert body["detector"] == "fake" - assert "sanitized_text" not in body + assert "replaced_text" not in body labels = {s["label"] for s in body["detected_spans"]} assert labels == {"EMAIL", "ADDRESS"} assert body["summary"] == {"span_count": 2, "by_label": {"EMAIL": 1, "ADDRESS": 1}} @pytest.mark.asyncio -async def test_detect_filter_categories(): +async def test_find_filter_categories(): async with _client() as c: r = await c.post( - "/v1/detect", + "/v1/find", json={ "text": "Joe at joe@example.com lives in Elgin, TX.", "categories": ["EMAIL"], @@ -150,12 +150,12 @@ async def test_detect_filter_categories(): @pytest.mark.asyncio -async def test_detect_empty_categories_returns_zero_spans(): +async def test_find_empty_categories_returns_zero_spans(): # `[]` is a deliberate "match nothing" filter — distinct from `None`, # which means "no filter". Regression guard for that contract. async with _client() as c: r = await c.post( - "/v1/detect", + "/v1/find", json={ "text": "Joe at joe@example.com lives in Elgin, TX.", "categories": [], @@ -168,10 +168,10 @@ async def test_detect_empty_categories_returns_zero_spans(): @pytest.mark.asyncio -async def test_detect_null_categories_returns_all_spans(): +async def test_find_null_categories_returns_all_spans(): async with _client() as c: r = await c.post( - "/v1/detect", + "/v1/find", json={"text": "Joe at joe@example.com lives in Elgin, TX.", "categories": None}, ) assert r.status_code == 200, r.text @@ -180,10 +180,10 @@ async def test_detect_null_categories_returns_all_spans(): @pytest.mark.asyncio -async def test_detect_invalid_category(): +async def test_find_invalid_category(): async with _client() as c: r = await c.post( - "/v1/detect", + "/v1/find", json={"text": "x", "categories": ["NOT_REAL"]}, ) assert r.status_code == 422 @@ -193,10 +193,10 @@ async def test_detect_invalid_category(): @pytest.mark.asyncio -async def test_detect_unknown_opf_option_rejected(): +async def test_find_unknown_opf_option_rejected(): async with _client() as c: r = await c.post( - "/v1/detect", + "/v1/find", json={"text": "x", "options": {"opf": {"decod_mode": "argmax"}}}, ) assert r.status_code == 422 @@ -208,10 +208,10 @@ async def test_detect_unknown_opf_option_rejected(): @pytest.mark.asyncio -async def test_detect_valid_opf_options_accepted(): +async def test_find_valid_opf_options_accepted(): async with _client() as c: r = await c.post( - "/v1/detect", + "/v1/find", json={ "text": "Joe at joe@example.com lives in Elgin, TX.", "options": {"opf": {"decode_mode": "argmax"}}, @@ -223,46 +223,46 @@ async def test_detect_valid_opf_options_accepted(): @pytest.mark.asyncio -async def test_detect_unknown_detector(): +async def test_find_unknown_detector(): async with _client() as c: - r = await c.post("/v1/detect", json={"text": "x", "detector": "ghost"}) + r = await c.post("/v1/find", json={"text": "x", "detector": "ghost"}) assert r.status_code == 400 -# --- /v1/sanitize --------------------------------------------------------- +# --- /v1/replace --------------------------------------------------------- @pytest.mark.asyncio -async def test_sanitize_default_mode_is_label(): +async def test_replace_default_mode_is_label(): async with _client() as c: r = await c.post( - "/v1/sanitize", + "/v1/replace", json={"text": "Joe at joe@example.com lives in Elgin, TX."}, ) assert r.status_code == 200, r.text body = r.json() assert body["mode"] == "label" - assert body["sanitized_text"] == "Joe at [EMAIL] lives in [ADDRESS]." + assert body["replaced_text"] == "Joe at [EMAIL] lives in [ADDRESS]." replacements = {s["label"]: s["replacement"] for s in body["detected_spans"]} assert replacements == {"EMAIL": "[EMAIL]", "ADDRESS": "[ADDRESS]"} @pytest.mark.asyncio -async def test_sanitize_redact_mode_fixed_length_asterisks(): +async def test_replace_redact_mode_fixed_length_asterisks(): async with _client() as c: r = await c.post( - "/v1/sanitize", + "/v1/replace", json={"text": "Joe at joe@example.com lives in Elgin, TX.", "mode": "redact"}, ) assert r.status_code == 200 body = r.json() - assert body["sanitized_text"] == "Joe at ******** lives in ********." + assert body["replaced_text"] == "Joe at ******** lives in ********." for s in body["detected_spans"]: assert s["replacement"] == "********" @pytest.mark.asyncio -async def test_sanitize_label_number_duplicate_reuses_number(): +async def test_replace_label_number_duplicate_reuses_number(): detector = MultiEntityDetector( [ ("EMAIL", "EMAIL_ADDRESS", "alice@x.com"), @@ -274,11 +274,11 @@ async def test_sanitize_label_number_duplicate_reuses_number(): text = "Email Alice (alice@x.com) and Bob (bob@x.com). Alice will reply from alice@x.com." async with _client(detector_instance=detector) as c: r = await c.post( - "/v1/sanitize", + "/v1/replace", json={"text": text, "mode": "label_number"}, ) assert r.status_code == 200, r.text - out = r.json()["sanitized_text"] + out = r.json()["replaced_text"] assert ( out == "Email [PERSON_1] ([EMAIL_1]) and [PERSON_2] ([EMAIL_2]). " @@ -287,7 +287,7 @@ async def test_sanitize_label_number_duplicate_reuses_number(): @pytest.mark.asyncio -async def test_sanitize_label_number_independent_counters(): +async def test_replace_label_number_independent_counters(): detector = MultiEntityDetector( [ ("EMAIL", "EMAIL_ADDRESS", "a@x.com"), @@ -298,18 +298,18 @@ async def test_sanitize_label_number_independent_counters(): text = "Alice has a@x.com and b@x.com." async with _client(detector_instance=detector) as c: r = await c.post( - "/v1/sanitize", + "/v1/replace", json={"text": text, "mode": "label_number"}, ) - assert r.json()["sanitized_text"] == "[PERSON_1] has [EMAIL_1] and [EMAIL_2]." + assert r.json()["replaced_text"] == "[PERSON_1] has [EMAIL_1] and [EMAIL_2]." @pytest.mark.asyncio -async def test_sanitize_label_token(): +async def test_replace_label_token(): vault = FakeTokenVaultClient() async with _client(token_vault_client=vault) as c: r = await c.post( - "/v1/sanitize", + "/v1/replace", json={ "text": "Joe at joe@example.com lives in Elgin, TX.", "mode": "label_token", @@ -317,8 +317,8 @@ async def test_sanitize_label_token(): ) assert r.status_code == 200, r.text body = r.json() - assert "[EMAIL_TE00001]" in body["sanitized_text"] - assert "[ADDRESS_TA00001]" in body["sanitized_text"] + assert "[EMAIL_TE00001]" in body["replaced_text"] + assert "[ADDRESS_TA00001]" in body["replaced_text"] assert len(vault.calls) == 1 assert set(vault.calls[0]) == { ("EMAIL", "joe@example.com"), @@ -327,25 +327,25 @@ async def test_sanitize_label_token(): @pytest.mark.asyncio -async def test_sanitize_label_token_dedupe_batch(): +async def test_replace_label_token_dedupe_batch(): detector = MultiEntityDetector([("EMAIL", "EMAIL_ADDRESS", "a@x.com")]) vault = FakeTokenVaultClient() async with _client(detector_instance=detector, token_vault_client=vault) as c: r = await c.post( - "/v1/sanitize", + "/v1/replace", json={"text": "a@x.com then a@x.com again", "mode": "label_token"}, ) assert r.status_code == 200 assert vault.calls == [[("EMAIL", "a@x.com")]] body = r.json() - assert body["sanitized_text"].count("[EMAIL_TE00001]") == 2 + assert body["replaced_text"].count("[EMAIL_TE00001]") == 2 @pytest.mark.asyncio -async def test_sanitize_label_token_unconfigured_400(): +async def test_replace_label_token_unconfigured_400(): async with _client(token_vault_client=None) as c: r = await c.post( - "/v1/sanitize", + "/v1/replace", json={"text": "joe@example.com", "mode": "label_token"}, ) assert r.status_code == 400 @@ -353,11 +353,11 @@ async def test_sanitize_label_token_unconfigured_400(): @pytest.mark.asyncio -async def test_sanitize_label_token_vault_failure_502(): +async def test_replace_label_token_vault_failure_502(): vault = FakeTokenVaultClient(raise_on_call=True) async with _client(token_vault_client=vault) as c: r = await c.post( - "/v1/sanitize", + "/v1/replace", json={"text": "joe@example.com", "mode": "label_token"}, ) assert r.status_code == 502 @@ -365,10 +365,10 @@ async def test_sanitize_label_token_vault_failure_502(): @pytest.mark.asyncio -async def test_sanitize_filter_categories(): +async def test_replace_filter_categories(): async with _client() as c: r = await c.post( - "/v1/sanitize", + "/v1/replace", json={ "text": "Joe at joe@example.com lives in Elgin, TX.", "mode": "label_number", @@ -376,29 +376,29 @@ async def test_sanitize_filter_categories(): }, ) body = r.json() - assert body["sanitized_text"] == "Joe at [EMAIL_1] lives in Elgin, TX." + assert body["replaced_text"] == "Joe at [EMAIL_1] lives in Elgin, TX." assert [s["label"] for s in body["detected_spans"]] == ["EMAIL"] @pytest.mark.asyncio -async def test_sanitize_unknown_detector(): +async def test_replace_unknown_detector(): async with _client() as c: r = await c.post( - "/v1/sanitize", + "/v1/replace", json={"text": "x", "detector": "ghost", "mode": "label"}, ) assert r.status_code == 400 @pytest.mark.asyncio -async def test_sanitize_no_spans_passthrough(): +async def test_replace_no_spans_passthrough(): async with _client() as c: r = await c.post( - "/v1/sanitize", + "/v1/replace", json={"text": "nothing to detect here", "mode": "label_number"}, ) body = r.json() - assert body["sanitized_text"] == "nothing to detect here" + assert body["replaced_text"] == "nothing to detect here" assert body["detected_spans"] == [] assert body["summary"] == {"span_count": 0, "by_label": {}} diff --git a/docs/api/openapi.json b/docs/api/openapi.json index 2f2c38a..5dcf44f 100644 --- a/docs/api/openapi.json +++ b/docs/api/openapi.json @@ -23,8 +23,139 @@ "title": "CanonicalLabel", "type": "string" }, - "DetectRequest": { - "description": "Base request for `/v1/detect` \u2014 no text-rewriting fields.", + "DetectorInfo": { + "description": "One row in the detector registry.", + "properties": { + "categories": { + "description": "Canonical categories this detector can produce.", + "examples": [ + [ + "PERSON", + "EMAIL", + "PHONE" + ] + ], + "items": { + "$ref": "#/components/schemas/CanonicalLabel" + }, + "title": "Categories", + "type": "array" + }, + "loaded": { + "description": "`true` once the detector has been initialised. Flips on first use, or at startup if listed in `EAGER_LOAD`.", + "examples": [ + true + ], + "title": "Loaded", + "type": "boolean" + }, + "name": { + "description": "Registry key.", + "examples": [ + "opf" + ], + "title": "Name", + "type": "string" + }, + "proxy": { + "description": "`true` if the detector calls an external service (e.g. Skyflow).", + "examples": [ + false + ], + "title": "Proxy", + "type": "boolean" + } + }, + "required": [ + "name", + "categories", + "loaded", + "proxy" + ], + "title": "DetectorInfo", + "type": "object" + }, + "DetectorOptions": { + "description": "Per-detector options namespace. Each key carries options for that one\ndetector. Only the entry matching the top-level `detector` is consulted \u2014\nother keys are accepted but ignored, so a client can carry one options\nblob across detector swaps without restructuring it.", + "properties": { + "opf": { + "anyOf": [ + { + "$ref": "#/components/schemas/OpfOptions" + }, + { + "type": "null" + } + ], + "description": "Options for the `opf` detector. Ignored by other detectors." + } + }, + "title": "DetectorOptions", + "type": "object" + }, + "DetectorsResponse": { + "description": "Response from `/v1/detectors`.", + "examples": [ + { + "default": "opf", + "detectors": [ + { + "categories": [ + "PERSON", + "EMAIL" + ], + "loaded": false, + "name": "gliner", + "proxy": false + }, + { + "categories": [ + "PERSON", + "EMAIL", + "PHONE" + ], + "loaded": true, + "name": "opf", + "proxy": false + }, + { + "categories": [ + "PERSON", + "EMAIL" + ], + "loaded": false, + "name": "skyflow", + "proxy": true + } + ] + } + ], + "properties": { + "default": { + "description": "Detector used when a request omits the `detector` field.", + "examples": [ + "opf" + ], + "title": "Default", + "type": "string" + }, + "detectors": { + "items": { + "$ref": "#/components/schemas/DetectorInfo" + }, + "title": "Detectors", + "type": "array" + } + }, + "required": [ + "default", + "detectors" + ], + "title": "DetectorsResponse", + "type": "object" + }, + "FindRequest": { + "description": "Base request for `/v1/find` \u2014 no text-rewriting fields.", "examples": [ { "categories": [ @@ -103,11 +234,11 @@ "required": [ "text" ], - "title": "DetectRequest", + "title": "FindRequest", "type": "object" }, - "DetectResponse": { - "description": "Response from `/v1/detect`.", + "FindResponse": { + "description": "Response from `/v1/find`.", "examples": [ { "detected_spans": [ @@ -170,138 +301,7 @@ "detected_spans", "summary" ], - "title": "DetectResponse", - "type": "object" - }, - "DetectorInfo": { - "description": "One row in the detector registry.", - "properties": { - "categories": { - "description": "Canonical categories this detector can produce.", - "examples": [ - [ - "PERSON", - "EMAIL", - "PHONE" - ] - ], - "items": { - "$ref": "#/components/schemas/CanonicalLabel" - }, - "title": "Categories", - "type": "array" - }, - "loaded": { - "description": "`true` once the detector has been initialised. Flips on first use, or at startup if listed in `EAGER_LOAD`.", - "examples": [ - true - ], - "title": "Loaded", - "type": "boolean" - }, - "name": { - "description": "Registry key.", - "examples": [ - "opf" - ], - "title": "Name", - "type": "string" - }, - "proxy": { - "description": "`true` if the detector calls an external service (e.g. Skyflow).", - "examples": [ - false - ], - "title": "Proxy", - "type": "boolean" - } - }, - "required": [ - "name", - "categories", - "loaded", - "proxy" - ], - "title": "DetectorInfo", - "type": "object" - }, - "DetectorOptions": { - "description": "Per-detector options namespace. Each key carries options for that one\ndetector. Only the entry matching the top-level `detector` is consulted \u2014\nother keys are accepted but ignored, so a client can carry one options\nblob across detector swaps without restructuring it.", - "properties": { - "opf": { - "anyOf": [ - { - "$ref": "#/components/schemas/OpfOptions" - }, - { - "type": "null" - } - ], - "description": "Options for the `opf` detector. Ignored by other detectors." - } - }, - "title": "DetectorOptions", - "type": "object" - }, - "DetectorsResponse": { - "description": "Response from `/v1/detectors`.", - "examples": [ - { - "default": "opf", - "detectors": [ - { - "categories": [ - "PERSON", - "EMAIL" - ], - "loaded": false, - "name": "gliner", - "proxy": false - }, - { - "categories": [ - "PERSON", - "EMAIL", - "PHONE" - ], - "loaded": true, - "name": "opf", - "proxy": false - }, - { - "categories": [ - "PERSON", - "EMAIL" - ], - "loaded": false, - "name": "skyflow", - "proxy": true - } - ] - } - ], - "properties": { - "default": { - "description": "Detector used when a request omits the `detector` field.", - "examples": [ - "opf" - ], - "title": "Default", - "type": "string" - }, - "detectors": { - "items": { - "$ref": "#/components/schemas/DetectorInfo" - }, - "title": "Detectors", - "type": "array" - } - }, - "required": [ - "default", - "detectors" - ], - "title": "DetectorsResponse", + "title": "FindResponse", "type": "object" }, "HealthResponse": { @@ -373,8 +373,8 @@ "title": "OpfOptions", "type": "object" }, - "SanitizeRequest": { - "description": "Request for `/v1/sanitize` \u2014 `DetectRequest` plus a `mode` field that picks\nhow detected spans are rewritten.", + "ReplaceRequest": { + "description": "Request for `/v1/replace` \u2014 `FindRequest` plus a `mode` field that picks\nhow detected spans are rewritten.", "examples": [ { "detector": "presidio", @@ -467,11 +467,11 @@ "required": [ "text" ], - "title": "SanitizeRequest", + "title": "ReplaceRequest", "type": "object" }, - "SanitizeResponse": { - "description": "Response from `/v1/sanitize`.", + "ReplaceResponse": { + "description": "Response from `/v1/replace`.", "examples": [ { "detected_spans": [ @@ -494,7 +494,7 @@ ], "detector": "presidio", "mode": "label_token", - "sanitized_text": "Email [EMAIL_MGaE1Bo] or call [PHONE_vRXiWKZ].", + "replaced_text": "Email [EMAIL_MGaE1Bo] or call [PHONE_vRXiWKZ].", "summary": { "by_label": { "EMAIL": 1, @@ -509,7 +509,7 @@ "detected_spans": { "description": "Spans detected by the chosen detector, with their replacements.", "items": { - "$ref": "#/components/schemas/SanitizedSpan" + "$ref": "#/components/schemas/ReplacedSpan" }, "title": "Detected Spans", "type": "array" @@ -536,12 +536,12 @@ "title": "Mode", "type": "string" }, - "sanitized_text": { + "replaced_text": { "description": "`text` with each detected span replaced by its `replacement`. Overlapping spans: the earlier-starting span wins; later overlaps are skipped here (they still appear in `detected_spans`).", "examples": [ "Email [EMAIL_MGaE1Bo]." ], - "title": "Sanitized Text", + "title": "Replaced Text", "type": "string" }, "summary": { @@ -573,14 +573,14 @@ "mode", "text", "detected_spans", - "sanitized_text", + "replaced_text", "summary" ], - "title": "SanitizeResponse", + "title": "ReplaceResponse", "type": "object" }, - "SanitizedSpan": { - "description": "One detected span plus the string it was rewritten to in `sanitized_text`.", + "ReplacedSpan": { + "description": "One detected span plus the string it was rewritten to in `replaced_text`.", "properties": { "end": { "description": "Exclusive character offset in `text`.", @@ -607,7 +607,7 @@ "type": "string" }, "replacement": { - "description": "The string this span was rewritten to in `sanitized_text`.", + "description": "The string this span was rewritten to in `replaced_text`.", "examples": [ "[EMAIL_MGaE1Bo]" ], @@ -639,11 +639,11 @@ "text", "replacement" ], - "title": "SanitizedSpan", + "title": "ReplacedSpan", "type": "object" }, "SpanOut": { - "description": "Plain span for `/v1/detect` \u2014 no replacement text.", + "description": "Plain span for `/v1/find` \u2014 no replacement text.", "properties": { "end": { "description": "Exclusive character offset in `text`.", @@ -736,22 +736,44 @@ "name": "local-privacy", "url": "https://github.com/jstjoe/local-privacy" }, - "description": "Unified PII detection across **OPF**, **GLiNER**, **Presidio**, and **Skyflow**.\nPick a backend with the `detector` field; canonical labels apply uniformly across all of them.\n\n## Endpoints\n\n- `POST /v1/detect` \u2014 return spans only, no text rewriting.\n- `POST /v1/sanitize` \u2014 detect and rewrite spans under one of four modes.\n- `GET /v1/detectors` \u2014 list registered detectors and their category coverage.\n- `GET /v1/health` \u2014 liveness probe; does not exercise detector backends.\n\n## Versioning\n\nThe API is pre-`1.0`. `info.version` is the only version surface \u2014 every\nbreaking change to request or response shape lands as a minor bump\n(`0.x.0 -> 0.(x+1).0`). Non-breaking additions land as patch bumps. The\n`/v1/` URL prefix is reserved for the eventual `1.0` cutover.\n\n## Canonical labels\n\nEvery detector's raw output maps into a 15-label taxonomy:\n`PERSON`, `EMAIL`, `PHONE`, `ADDRESS`, `URL`, `DATE`, `ACCOUNT`, `SECRET`, `USERNAME`,\n`DEMOGRAPHIC`, `ORGANIZATION`, `OCCUPATION`, `MONEY`, `VEHICLE`, `PHYSICAL`.\nDetectors vary in coverage \u2014 `GET /v1/detectors` reports each detector's category list.\n", + "description": "Unified PII detection across **OPF**, **GLiNER**, **Presidio**, and **Skyflow**.\nPick a backend with the `detector` field; canonical labels apply uniformly across all of them.\n\n## Endpoints\n\n- `POST /v1/find` \u2014 find sensitive data; return spans only, no text rewriting.\n- `POST /v1/replace` \u2014 find spans and replace each one under one of four modes.\n- `GET /v1/detectors` \u2014 list registered detectors and their category coverage.\n- `GET /v1/health` \u2014 liveness probe; does not exercise detector backends.\n\n## Versioning\n\nThe API is pre-`1.0`. `info.version` is the only version surface \u2014 every\nbreaking change to request or response shape lands as a minor bump\n(`0.x.0 -> 0.(x+1).0`). Non-breaking additions land as patch bumps. The\n`/v1/` URL prefix is reserved for the eventual `1.0` cutover.\n\n## Canonical labels\n\nEvery detector's raw output maps into a 15-label taxonomy:\n`PERSON`, `EMAIL`, `PHONE`, `ADDRESS`, `URL`, `DATE`, `ACCOUNT`, `SECRET`, `USERNAME`,\n`DEMOGRAPHIC`, `ORGANIZATION`, `OCCUPATION`, `MONEY`, `VEHICLE`, `PHYSICAL`.\nDetectors vary in coverage \u2014 `GET /v1/detectors` reports each detector's category list.\n", "summary": "Unified PII detection across OPF, GLiNER, Presidio, and Skyflow.", "title": "Privacy-detection API", - "version": "0.3.0" + "version": "0.4.0" }, "openapi": "3.1.0", "paths": { - "/v1/detect": { + "/v1/detectors": { + "get": { + "description": "Return every detector registered in this deployment plus the canonical categories each can produce. `loaded` flips to `true` after first use (or at startup if the detector is listed in `EAGER_LOAD`). `proxy=true` means the detector calls an external service.", + "operationId": "list_detectors_v1_detectors_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DetectorsResponse" + } + } + }, + "description": "Registry snapshot." + } + }, + "summary": "List registered detectors", + "tags": [ + "Meta" + ] + } + }, + "/v1/find": { "post": { - "description": "Run the chosen detector over `text` and return canonical-labelled spans.\n\nNo rewriting is performed \u2014 call `/v1/sanitize` for that.\n\nFilter detector output to a subset of canonical labels with `categories`.\nOPF-only: pass `options.opf.decode_mode` to override the default Viterbi decoding.", - "operationId": "detect_v1_detect_post", + "description": "Run the chosen detector over `text` and return canonical-labelled spans.\n\nNo rewriting is performed \u2014 call `/v1/replace` for that.\n\nFilter detector output to a subset of canonical labels with `categories`.\nOPF-only: pass `options.opf.decode_mode` to override the default Viterbi decoding.", + "operationId": "find_v1_find_post", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DetectRequest" + "$ref": "#/components/schemas/FindRequest" } } }, @@ -762,7 +784,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DetectResponse" + "$ref": "#/components/schemas/FindResponse" } } }, @@ -855,31 +877,9 @@ "description": "Detector backend failure or vault call failure." } }, - "summary": "Detect PII spans in text", - "tags": [ - "Detect" - ] - } - }, - "/v1/detectors": { - "get": { - "description": "Return every detector registered in this deployment plus the canonical categories each can produce. `loaded` flips to `true` after first use (or at startup if the detector is listed in `EAGER_LOAD`). `proxy=true` means the detector calls an external service.", - "operationId": "list_detectors_v1_detectors_get", - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DetectorsResponse" - } - } - }, - "description": "Registry snapshot." - } - }, - "summary": "List registered detectors", + "summary": "Find sensitive data in text", "tags": [ - "Meta" + "Find" ] } }, @@ -905,15 +905,15 @@ ] } }, - "/v1/sanitize": { + "/v1/replace": { "post": { - "description": "Detect spans, then rewrite each one under the chosen `mode`. Four modes, in increasing strength of identity preservation:\n\n| `mode` | Looks like | What it preserves |\n|---|---|---|\n| `redact` | `********` | Nothing \u2014 fixed 8-char asterisk run regardless of span length. |\n| `label` | `[EMAIL]` | Category only. Default. |\n| `label_number` | `[EMAIL_1]` | Identity **within one request** via per-label counter; duplicate `(label, text)` reuses its number. |\n| `label_token` | `[EMAIL_MGaE1Bo]` | Identity **across requests and detectors** via a Skyflow vault. Deterministic \u2014 same plaintext maps to the same 7-char token forever. |\n\n**Overlapping spans:** the earlier-starting span wins; later overlaps are skipped in `sanitized_text` (they still appear in `detected_spans`).\n\n**`label_token` requirements:** `SKYFLOW_TOKEN_VAULT_URL`, `SKYFLOW_TOKEN_VAULT_ID`, and a bearer (`SKYFLOW_TOKEN_BEARER_TOKEN`, falling back to `SKYFLOW_BEARER_TOKEN`). The vault must be configured per the token-vault setup guide \u2014 one table with one `tok_