diff --git a/README.md b/README.md index 0b415d7..02f3f59 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,7 @@ Three in-app reference UIs render the same spec — `/scalar` (Scalar), `/docs` - [overview](docs/guides/overview.md) — what the API does and how to run it. - [detectors](docs/guides/detectors.md) — the registered backends. - [labels](docs/guides/labels.md) — the 15-label canonical taxonomy. -- [sanitize modes](docs/guides/sanitize-modes.md) — the four `/v1/sanitize` modes. +- [replace modes](docs/guides/replace-modes.md) — the four `/api/replace` modes. - [auth and env](docs/guides/auth-and-env.md) — env-var matrix and error codes. ### Server env @@ -301,4 +301,4 @@ Three in-app reference UIs render the same spec — `/scalar` (Scalar), `/docs` | `OPF_DEVICE` | `cpu`, `cuda`, `mps`. Default `cpu`. | | `OPF_DECODE_MODE` | `viterbi` or `argmax`. OPF-only. Default `viterbi`. | | `SKYFLOW_VAULT_URL` / `_ID` / `_BEARER_TOKEN` | Required for the `skyflow` **detector**. | -| `SKYFLOW_TOKEN_VAULT_URL` / `_ID` / `_BEARER_TOKEN` | Required for `/v1/sanitize` `label_token` mode. See [docs/token-vault-setup.md](docs/token-vault-setup.md). | +| `SKYFLOW_TOKEN_VAULT_URL` / `_ID` / `_BEARER_TOKEN` | Required for `/api/replace` `label_token` mode. See [docs/token-vault-setup.md](docs/token-vault-setup.md). | diff --git a/api/Dockerfile b/api/Dockerfile index 60307ea..7325e3e 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -50,7 +50,7 @@ RUN if [ "$INCLUDE_PRESIDIO_MULTILANG" = "1" ]; then \ python -m spacy download es_core_news_lg; \ fi -# Bake OPF checkpoint into the image so first /v1/redact doesn't pay the +# Bake OPF checkpoint into the image so the first request doesn't pay the # 2.8 GB download cost. Skip with --build-arg INCLUDE_OPF=0. RUN if [ "$INCLUDE_OPF" = "1" ]; then \ python -c "from opf._common.checkpoint_download import ensure_default_checkpoint; ensure_default_checkpoint()"; \ diff --git a/api/pyproject.toml b/api/pyproject.toml index 32a0ab6..a3d117c 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "opf-api" -version = "0.3.0" +version = "0.6.0" description = "Unified privacy-detection API — one contract, multiple backends" requires-python = ">=3.10" dependencies = [ diff --git a/api/src/opf_api/main.py b/api/src/opf_api/main.py index 8bd4d60..ec200d5 100644 --- a/api/src/opf_api/main.py +++ b/api/src/opf_api/main.py @@ -33,10 +33,10 @@ async def lifespan(app: FastAPI): app.state.token_vault_client = TokenVaultClient.from_env() if app.state.token_vault_client is None: logger.info( - "token vault not configured; /v1/tokenize 'vault_token' mode will 400" + "token vault not configured; /api/replace 'label_token' mode will 400" ) else: - logger.info("token vault configured for /v1/tokenize 'vault_token' mode") + logger.info("token vault configured for /api/replace 'label_token' mode") eager = os.environ.get("EAGER_LOAD", default) eager_names = [n.strip() for n in eager.split(",") if n.strip()] @@ -60,35 +60,39 @@ async def lifespan(app: FastAPI): ## Endpoints -- `POST /v1/detect` — return spans only, no text rewriting. -- `POST /v1/sanitize` — detect and rewrite spans under one of four modes. -- `GET /v1/detectors` — list registered detectors and their category coverage. -- `GET /v1/health` — liveness probe; does not exercise detector backends. +- `POST /api/find` — find sensitive data; return spans only, no text rewriting. +- `POST /api/replace` — find spans and replace each one under one of four modes. +- `GET /api/detectors` — list registered detectors and their category coverage. +- `GET /api/health` — liveness probe; does not exercise detector backends. ## Versioning -The API is pre-`1.0`. `info.version` is the only version surface — every -breaking change to request or response shape lands as a minor bump -(`0.x.0 -> 0.(x+1).0`). Non-breaking additions land as patch bumps. The -`/v1/` URL prefix is reserved for the eventual `1.0` cutover. +The API is pre-`1.0`. `info.version` is the only version surface today — +every breaking change to request or response shape lands as a minor bump +(`0.x.0 -> 0.(x+1).0`); non-breaking additions land as patch bumps. + +URL paths are unversioned (`/api/...`). When the API stabilizes, the +intent is to move to **header-based date-string versioning** in the style +of Stripe — clients will pin to a release date via `API-Version: +2026-05-13`. Until that header lands, treat `info.version` as the contract. ## Canonical labels Every detector's raw output maps into a 15-label taxonomy: `PERSON`, `EMAIL`, `PHONE`, `ADDRESS`, `URL`, `DATE`, `ACCOUNT`, `SECRET`, `USERNAME`, `DEMOGRAPHIC`, `ORGANIZATION`, `OCCUPATION`, `MONEY`, `VEHICLE`, `PHYSICAL`. -Detectors vary in coverage — `GET /v1/detectors` reports each detector's category list. +Detectors vary in coverage — `GET /api/detectors` reports each detector's category list. """ OPENAPI_TAGS = [ { - "name": "Detect", - "description": "Detection-only endpoint. Returns spans without rewriting the input.", + "name": "Find", + "description": "Find-only endpoint. Returns spans without rewriting the input.", }, { - "name": "Sanitize", + "name": "Replace", "description": ( - "Detection plus rewriting under one of four modes: " + "Find plus rewriting under one of four modes: " "`redact`, `label`, `label_number`, `label_token`." ), }, @@ -100,7 +104,7 @@ async def lifespan(app: FastAPI): app = FastAPI( title="Privacy-detection API", - version="0.3.0", + version="0.6.0", summary="Unified PII detection across OPF, GLiNER, Presidio, and Skyflow.", description=API_DESCRIPTION, lifespan=lifespan, @@ -113,7 +117,7 @@ async def lifespan(app: FastAPI): {"url": "http://localhost:8000", "description": "Local dev"}, ], ) -app.include_router(router, prefix="/v1") +app.include_router(router, prefix="/api") @app.get("/scalar", include_in_schema=False) diff --git a/api/src/opf_api/registry.py b/api/src/opf_api/registry.py index 57917a4..d65c41a 100644 --- a/api/src/opf_api/registry.py +++ b/api/src/opf_api/registry.py @@ -81,7 +81,7 @@ def _device() -> str: return os.environ.get("OPF_DEVICE", "cpu") -# All GLiNER variants run at the same threshold so /v1/detect output is +# All GLiNER variants run at the same threshold so /api/find output is # comparable across them and operators have one knob to reason about. # Gretel's own recommendation is 0.7; nvidia's model card suggests 0.3 # but that produces noticeably more false positives — 0.7 was chosen as diff --git a/api/src/opf_api/routes.py b/api/src/opf_api/routes.py index b51bb72..f04b8e2 100644 --- a/api/src/opf_api/routes.py +++ b/api/src/opf_api/routes.py @@ -22,12 +22,12 @@ DetectorInfo, DetectorsResponse, DetectorOptions, - DetectRequest, - DetectResponse, + FindRequest, + FindResponse, HealthResponse, - SanitizedSpan, - SanitizeRequest, - SanitizeResponse, + ReplacedSpan, + ReplaceRequest, + ReplaceResponse, SpanOut, ) from .vault_tokens import TokenVaultClient @@ -129,14 +129,14 @@ def _resolve_detector(request: Request, name: str | None) -> tuple[str, Detector return chosen, entry -class _DetectInput(Protocol): +class _FindInput(Protocol): text: str detector: str | None categories: list[CanonicalLabel] | None options: DetectorOptions | None -async def _run_detect(request: Request, body: _DetectInput) -> tuple[str, list[Span]]: +async def _run_find(request: Request, body: _FindInput) -> tuple[str, list[Span]]: name, entry = _resolve_detector(request, body.detector) detector = await entry.get() @@ -158,13 +158,13 @@ async def _run_detect(request: Request, body: _DetectInput) -> tuple[str, list[S @router.post( - "/detect", - response_model=DetectResponse, - tags=["Detect"], - summary="Detect PII spans in text", + "/find", + response_model=FindResponse, + tags=["Find"], + summary="Find sensitive data in text", description=( "Run the chosen detector over `text` and return canonical-labelled spans.\n\n" - "No rewriting is performed — call `/v1/sanitize` for that.\n\n" + "No rewriting is performed — call `/api/replace` for that.\n\n" "Filter detector output to a subset of canonical labels with `categories`.\n" "OPF-only: pass `options.opf.decode_mode` to override the default Viterbi decoding." ), @@ -175,9 +175,9 @@ async def _run_detect(request: Request, body: _DetectInput) -> tuple[str, list[S 502: _ERROR_502_EXAMPLE, }, ) -async def detect(request: Request, body: DetectRequest) -> DetectResponse: - """Detect-only: returns spans, no text rewriting.""" - name, spans = await _run_detect(request, body) +async def find(request: Request, body: FindRequest) -> FindResponse: + """Find-only: returns spans, no text rewriting.""" + name, spans = await _run_find(request, body) ordered = sorted(spans, key=lambda s: (s["start"], s["end"])) out_spans = [ SpanOut( @@ -190,7 +190,7 @@ async def detect(request: Request, body: DetectRequest) -> DetectResponse: for s in ordered ] by_label = Counter(s.label for s in out_spans) - return DetectResponse( + return FindResponse( detector=name, text=body.text, detected_spans=out_spans, @@ -212,23 +212,26 @@ def _build_label_token_renderer_raising_http( @router.post( - "/sanitize", - response_model=SanitizeResponse, - tags=["Sanitize"], - summary="Detect and rewrite PII spans", + "/replace", + response_model=ReplaceResponse, + tags=["Replace"], + summary="Replace sensitive data in text", description=( - "Detect spans, then rewrite each one under the chosen `mode`. Four modes, " + "Find spans, then rewrite each one under the chosen `mode`. Four modes, " "in increasing strength of identity preservation:\n\n" "| `mode` | Looks like | What it preserves |\n" "|---|---|---|\n" "| `redact` | `********` | Nothing — fixed 8-char asterisk run regardless of span length. |\n" "| `label` | `[EMAIL]` | Category only. Default. |\n" "| `label_number` | `[EMAIL_1]` | Identity **within one request** via per-label counter; " - "duplicate `(label, text)` reuses its number. |\n" + "duplicate `(label, text)` reuses its number. Dropped-overlap spans " + "(`replaced=false`) still consume a counter slot, so the kept sequence " + "may skip numbers — see the overlap notes below. |\n" "| `label_token` | `[EMAIL_MGaE1Bo]` | Identity **across requests and detectors** via a " "Skyflow vault. Deterministic — same plaintext maps to the same 7-char token forever. |\n\n" "**Overlapping spans:** the earlier-starting span wins; later overlaps are skipped in " - "`sanitized_text` (they still appear in `detected_spans`).\n\n" + "`replaced_text` but still appear in `detected_spans` with `replaced=false`. " + "Filter to `replaced=true` to reconstruct exactly what landed.\n\n" "**`label_token` requirements:** `SKYFLOW_TOKEN_VAULT_URL`, `SKYFLOW_TOKEN_VAULT_ID`, " "and a bearer (`SKYFLOW_TOKEN_BEARER_TOKEN`, falling back to `SKYFLOW_BEARER_TOKEN`). " "The vault must be configured per the token-vault setup guide — one table with one " @@ -236,15 +239,15 @@ def _build_label_token_renderer_raising_http( "`^[A-Za-z0-9]{7}$`. Spans whose canonical label has no vault column fall back to " "`[LABEL]` for that span only." ), - response_description="Sanitized text, the spans that were rewritten, and per-label counts.", + response_description="Replaced text, the spans that were rewritten, and per-label counts.", responses={ 400: _ERROR_400_EXAMPLE, 422: _ERROR_422_EXAMPLE, 502: _ERROR_502_EXAMPLE, }, ) -async def sanitize(request: Request, body: SanitizeRequest) -> SanitizeResponse: - """Detect + rewrite the input text under the chosen `mode`. +async def replace(request: Request, body: ReplaceRequest) -> ReplaceResponse: + """Find + rewrite the input text under the chosen `mode`. Modes: - `redact` -> fixed-length asterisks (`********`) @@ -252,7 +255,7 @@ async def sanitize(request: Request, body: SanitizeRequest) -> SanitizeResponse: - `label_number` -> `[EMAIL_1]`, per-request counter - `label_token` -> `[EMAIL_jRc7QGn]`, deterministic Skyflow vault token """ - name, spans = await _run_detect(request, body) + name, spans = await _run_find(request, body) mode = body.mode if mode == "redact": @@ -288,26 +291,40 @@ async def sanitize(request: Request, body: SanitizeRequest) -> SanitizeResponse: ordered = sorted(spans, key=lambda s: (s["start"], s["end"])) rendered_pairs: list[tuple[Span, str]] = [(s, render(s)) for s in ordered] + # Mirror splice_pieces' overlap rule (earlier-starting span wins; later + # overlaps are skipped) so each ReplacedSpan can carry a faithful + # `replaced` flag. Clients filtering to `replaced=true` get exactly the + # set of spans that landed in `replaced_text`. + cursor = 0 + replaced_flags: list[bool] = [] + for s, _ in rendered_pairs: + if s["start"] >= cursor: + replaced_flags.append(True) + cursor = s["end"] + else: + replaced_flags.append(False) + out_spans = [ - SanitizedSpan( + ReplacedSpan( label=s["label"], raw_label=s["raw_label"], start=s["start"], end=s["end"], text=s["text"], replacement=replacement, + replaced=flag, ) - for s, replacement in rendered_pairs + for (s, replacement), flag in zip(rendered_pairs, replaced_flags) ] - sanitized = splice_pieces(body.text, rendered_pairs) + replaced_text = splice_pieces(body.text, rendered_pairs) by_label = Counter(s.label for s in out_spans) - return SanitizeResponse( + return ReplaceResponse( detector=name, mode=mode, text=body.text, detected_spans=out_spans, - sanitized_text=sanitized, + replaced_text=replaced_text, summary={"span_count": len(out_spans), "by_label": dict(by_label)}, warning=None, ) @@ -349,7 +366,7 @@ async def list_detectors(request: Request) -> DetectorsResponse: summary="Liveness probe", description=( "Always returns `200` when the process is up. Does **not** probe detector backends — " - "use `/v1/detectors` to inspect which detectors have been loaded." + "use `/api/detectors` to inspect which detectors have been loaded." ), response_description="Liveness status plus which detectors are currently loaded.", ) diff --git a/api/src/opf_api/schemas.py b/api/src/opf_api/schemas.py index fc51c0d..e1b7747 100644 --- a/api/src/opf_api/schemas.py +++ b/api/src/opf_api/schemas.py @@ -10,12 +10,12 @@ DecodeMode = Literal["viterbi", "argmax"] -# Four sanitization modes, in increasing strength of identity preservation: +# Four replacement modes, in increasing strength of identity preservation: # redact -> "********" (fixed-length asterisks; no information leaks) # label -> "[EMAIL]" (default; category label only) # label_number -> "[EMAIL_1]" (per-request counters; duplicates reuse numbers) # label_token -> "[EMAIL_jRc7QGn]" (deterministic Skyflow vault token) -SanitizeMode = Literal["redact", "label", "label_number", "label_token"] +ReplaceMode = Literal["redact", "label", "label_number", "label_token"] class CanonicalLabel(str, Enum): @@ -77,8 +77,8 @@ class DetectorOptions(BaseModel): ) -class DetectRequest(BaseModel): - """Base request for `/v1/detect` — no text-rewriting fields.""" +class FindRequest(BaseModel): + """Base request for `/api/find` — no text-rewriting fields.""" model_config = ConfigDict( json_schema_extra={ @@ -100,7 +100,7 @@ class DetectRequest(BaseModel): detector: str | None = Field( default=None, description=( - "Detector name from `GET /v1/detectors`. Omit to use the server's " + "Detector name from `GET /api/detectors`. Omit to use the server's " "`DEFAULT_DETECTOR` (set via env, typically `opf`)." ), examples=["presidio", "opf", "gliner"], @@ -126,8 +126,8 @@ class DetectRequest(BaseModel): ) -class SanitizeRequest(DetectRequest): - """Request for `/v1/sanitize` — `DetectRequest` plus a `mode` field that picks +class ReplaceRequest(FindRequest): + """Request for `/api/replace` — `FindRequest` plus a `mode` field that picks how detected spans are rewritten.""" model_config = ConfigDict( @@ -142,7 +142,7 @@ class SanitizeRequest(DetectRequest): } ) - mode: SanitizeMode = Field( + mode: ReplaceMode = Field( default="label", description=( "How to rewrite each detected span. " @@ -156,8 +156,8 @@ class SanitizeRequest(DetectRequest): ) -class SanitizedSpan(BaseModel): - """One detected span plus the string it was rewritten to in `sanitized_text`.""" +class ReplacedSpan(BaseModel): + """One detected span plus the string it was rewritten to in `replaced_text`.""" label: str = Field( ..., @@ -176,9 +176,23 @@ class SanitizedSpan(BaseModel): ) replacement: str = Field( ..., - description="The string this span was rewritten to in `sanitized_text`.", + description=( + "The string the renderer produced for this span. Only spliced into " + "`replaced_text` when `replaced=true`; for skipped overlaps this " + "value is still populated (the renderer ran) but did not land." + ), examples=["[EMAIL_MGaE1Bo]"], ) + replaced: bool = Field( + ..., + description=( + "`true` if this span was actually spliced into `replaced_text`; " + "`false` if it was suppressed as a later-starting overlap of an " + "earlier-starting span. To reconstruct exactly what changed, filter " + "to `replaced=true`." + ), + examples=[True], + ) class SummaryOut(BaseModel): @@ -194,8 +208,8 @@ class SummaryOut(BaseModel): ) -class SanitizeResponse(BaseModel): - """Response from `/v1/sanitize`.""" +class ReplaceResponse(BaseModel): + """Response from `/api/replace`.""" model_config = ConfigDict( json_schema_extra={ @@ -204,7 +218,7 @@ class SanitizeResponse(BaseModel): "detector": "presidio", "mode": "label_token", "text": "Email alice@x.com or call +1-415-555-0100.", - "sanitized_text": "Email [EMAIL_MGaE1Bo] or call [PHONE_vRXiWKZ].", + "replaced_text": "Email [EMAIL_MGaE1Bo] or call [PHONE_vRXiWKZ].", "detected_spans": [ { "label": "EMAIL", @@ -213,6 +227,7 @@ class SanitizeResponse(BaseModel): "end": 17, "text": "alice@x.com", "replacement": "[EMAIL_MGaE1Bo]", + "replaced": True, }, { "label": "PHONE", @@ -221,6 +236,7 @@ class SanitizeResponse(BaseModel): "end": 42, "text": "+1-415-555-0100", "replacement": "[PHONE_vRXiWKZ]", + "replaced": True, }, ], "summary": {"span_count": 2, "by_label": {"EMAIL": 1, "PHONE": 1}}, @@ -235,16 +251,16 @@ class SanitizeResponse(BaseModel): description="Detector that produced the spans.", examples=["presidio"], ) - mode: SanitizeMode = Field( + mode: ReplaceMode = Field( ..., description="Echo of the request `mode`.", examples=["label_token"] ) text: str = Field( ..., description="Echo of the request `text`.", examples=["Email alice@x.com."] ) - detected_spans: list[SanitizedSpan] = Field( + detected_spans: list[ReplacedSpan] = Field( ..., description="Spans detected by the chosen detector, with their replacements." ) - sanitized_text: str = Field( + replaced_text: str = Field( ..., description=( "`text` with each detected span replaced by its `replacement`. " @@ -261,7 +277,7 @@ class SanitizeResponse(BaseModel): class SpanOut(BaseModel): - """Plain span for `/v1/detect` — no replacement text.""" + """Plain span for `/api/find` — no replacement text.""" label: str = Field(..., description="Canonical label.", examples=["EMAIL"]) raw_label: str = Field( @@ -276,8 +292,8 @@ class SpanOut(BaseModel): ) -class DetectResponse(BaseModel): - """Response from `/v1/detect`.""" +class FindResponse(BaseModel): + """Response from `/api/find`.""" model_config = ConfigDict( json_schema_extra={ @@ -338,7 +354,7 @@ class DetectorInfo(BaseModel): class DetectorsResponse(BaseModel): - """Response from `/v1/detectors`.""" + """Response from `/api/detectors`.""" model_config = ConfigDict( json_schema_extra={ @@ -379,7 +395,7 @@ class DetectorsResponse(BaseModel): class HealthResponse(BaseModel): - """Response from `/v1/health`.""" + """Response from `/api/health`.""" model_config = ConfigDict( json_schema_extra={ diff --git a/api/src/opf_api/vault_tokens.py b/api/src/opf_api/vault_tokens.py index 1012ee1..49cc08b 100644 --- a/api/src/opf_api/vault_tokens.py +++ b/api/src/opf_api/vault_tokens.py @@ -3,7 +3,7 @@ Distinct from `opf_eval.detectors.skyflow`, which uses the Detect API. This module inserts already-detected entity values into a vault configured with deterministic format-preserving tokens (7-char alphanumeric), and returns -the tokens back to the /v1/tokenize endpoint. +the tokens back to the `/api/replace` route (`label_token` mode). Vault schema (one row per insert; only one column populated per row): @@ -50,9 +50,9 @@ def column_for_label(canonical_label: str) -> str | None: class TokenVaultClient: """Inserts entity values into the token vault and returns deterministic tokens. - Single batch call per /v1/tokenize request. One record per unique - (label, value) pair; the corresponding `tok_