diff --git a/desktop/VALIDATION.md b/desktop/VALIDATION.md index eb18da10e4..006d7f5023 100644 --- a/desktop/VALIDATION.md +++ b/desktop/VALIDATION.md @@ -28,6 +28,10 @@ The real Server harness covers SQLite with anonymous loopback, static Bearer, HT Selecting a compatibility profile does not attest the remote binary identity. Existing `sqlite-1.1.1-v1` / `sqlite-63f918b7-v1` / `sqlite-ab43e3a7-v1` / `sqlite-f1089f4e-v1` selections require explicitly choosing the new profile and rechecking the connection. CLI diagnostics separately verify the registered executable path, digest and exact version. +`sqlite-legacy-58f7f4f6-v1` identifies Server sources at `58f7f4f6cbadd698bac35b3fca019d9f48f55b81`, contract digest `9f4ba7b35ecd3b3639af4e648aff84b713a7f30b51d5a4c30c8809aeb8dc8024`, and qualification-wheel digest `7203cf1a98cb9dfd8631e142f7a0f5e74c39845da4fb7d862117a49fa1ef38af`. Its original ten Desktop operations and Legacy Memory protocol were exercised by the native `ConnectionManager` against the isolated built-wheel SQLite Server in all four harness modes: anonymous loopback, Bearer, HTTPS with explicit CA/base path, and injected-provider/enforced access. These checks cover save/search/exact historical citation reads, identity changes, revocation, ambiguous writes without replay, and Scope pagination. The original `sqlite-6e237568-v1` record retains its identities and operations. Current qualification requires explicitly selecting a profile whose recorded contract matches the bundled Desktop contract and rechecking the connection. + +This qualification used macOS 26.2 ARM64, Rust 1.95.0 and Python 3.12.7. An external system Python launcher was used for native process creation because this environment can terminate native subprocesses before their first request. The wheel, native client, fixtures and business assertions were preserved. The real CLI's fixed version, service-status and integration commands were also exercised with an isolated home and PATH; their actual outputs passed the native projections. These checks establish the stated SQLite compatibility and CLI response handling. Windows registered-executable checks, credentials, process containment and installed-package acceptance require their actual Windows CI checks. No remote legacy deployment was tested in this qualification. + ## Remaining release qualification - Clean standard-user Windows 11, including absent/present WebView2 and bootstrap recovery. diff --git a/desktop/src-tauri/examples/server_probe.rs b/desktop/src-tauri/examples/server_probe.rs index 30341bdd77..98a07ee0e0 100644 --- a/desktop/src-tauri/examples/server_probe.rs +++ b/desktop/src-tauri/examples/server_probe.rs @@ -123,9 +123,23 @@ async fn main() { .unwrap(), ); let raw: serde_json::Value = serde_json::from_slice(&std::fs::read(&args[1]).unwrap()).unwrap(); - let compatibility = manager.state().unwrap().compatibility_profiles[0] + let contract: serde_json::Value = + serde_json::from_str(include_str!("../src/transport/operations.json")).unwrap(); + let contract_sha256 = contract["contractSha256"].as_str().unwrap(); + let state = manager.state().unwrap(); + let mut qualified = state + .compatibility_profiles + .iter() + .filter(|profile| profile.contract_sha256 == contract_sha256); + let compatibility = qualified + .next() + .expect("qualified fixture contract") .id .clone(); + assert!( + qualified.next().is_none(), + "ambiguous fixture qualification" + ); let credential = raw["token"] .as_str() .map(|value| serde_json::json!({"secret":value,"storage":"session_only"})); diff --git a/desktop/src-tauri/src/connections/compatibility.json b/desktop/src-tauri/src/connections/compatibility.json index e97d0975da..e05d324af4 100644 --- a/desktop/src-tauri/src/connections/compatibility.json +++ b/desktop/src-tauri/src/connections/compatibility.json @@ -17,5 +17,24 @@ "get_memory_entry" ], "evidence": "desktop/VALIDATION.md" + }, + { + "id": "sqlite-legacy-58f7f4f6-v1", + "serverCommit": "58f7f4f6cbadd698bac35b3fca019d9f48f55b81", + "contractSha256": "9f4ba7b35ecd3b3639af4e648aff84b713a7f30b51d5a4c30c8809aeb8dc8024", + "artifactSha256": "7203cf1a98cb9dfd8631e142f7a0f5e74c39845da4fb7d862117a49fa1ef38af", + "operations": [ + "get_liveness", + "get_readiness", + "get_access_principal", + "get_capabilities", + "list_scopes", + "get_scope", + "get_default_scope", + "remember_memory", + "search_memory", + "get_memory_entry" + ], + "evidence": "desktop/VALIDATION.md" } ] diff --git a/desktop/src-tauri/src/diagnostics.rs b/desktop/src-tauri/src/diagnostics.rs index d16327e309..bb023cb1f8 100644 --- a/desktop/src-tauri/src/diagnostics.rs +++ b/desktop/src-tauri/src/diagnostics.rs @@ -191,6 +191,17 @@ pub fn project( Ok(report) } +#[cfg(any(windows, test))] +fn supported_cli_version(version: &str) -> bool { + ((version == "1.0.1" || version.starts_with("1.0.1.dev")) + || (version == "1.1.1" || version.starts_with("1.1.1.dev")) + || (version == "1.2.1" || version.starts_with("1.2.1.dev"))) + && version.len() <= 128 + && version + .bytes() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, b'.' | b'+' | b'-')) +} + #[cfg(windows)] #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -261,13 +272,7 @@ fn execute( serde_json::from_slice(&bytes).map_err(|_| SafeError::InvalidInput)?; // This is an explicit local-installation pin, not publisher-signature verification. if config.source != "explicit_local_installation" - || !((config.version == "1.0.1" || config.version.starts_with("1.0.1.dev")) - || (config.version == "1.1.1" || config.version.starts_with("1.1.1.dev"))) - || config.version.len() > 128 - || !config - .version - .bytes() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, b'.' | b'+' | b'-')) + || !supported_cli_version(&config.version) || config.sha256.len() != 64 || !config.sha256.bytes().all(|v| v.is_ascii_hexdigit()) || !config.executable.is_absolute() @@ -341,3 +346,44 @@ impl DiagnosticHost { } } } + +#[cfg(test)] +mod tests { + use super::supported_cli_version; + + #[test] + fn qualified_cli_versions_include_the_actual_atomic_ci_build() { + for version in [ + "1.0.1", + "1.0.1.dev61+g63f918b7e.d20260919", + "1.1.1", + "1.1.1.dev1+g6e237568", + "1.2.1", + "1.2.1.dev86+g3d3a6290d", + ] { + assert!(supported_cli_version(version), "unqualified {version}"); + } + } + + #[test] + fn unqualified_or_malformed_cli_versions_remain_rejected() { + for version in [ + "", + "1.2.0", + "1.2.2.dev86+g3d3a6290d", + "1.2.10", + "1.2.10.dev86+g3d3a6290d", + "1.3.1", + "1.2.1rc1", + "1.2.1.dev86 with spaces", + "1.2.1.dev86\n", + "1.2.1.dev86+未知", + ] { + assert!(!supported_cli_version(version), "qualified {version}"); + } + assert!(!supported_cli_version(&format!( + "1.2.1.dev{}", + "a".repeat(128) + ))); + } +} diff --git a/desktop/src-tauri/src/transport/operations.json b/desktop/src-tauri/src/transport/operations.json index 93eb65341a..cabc73625d 100644 --- a/desktop/src-tauri/src/transport/operations.json +++ b/desktop/src-tauri/src/transport/operations.json @@ -1,5 +1,5 @@ { - "contractSha256": "9af88b2b779c372a21ae9f398d0ca75b333f2e5da2afa3626df879911a442edd", + "contractSha256": "9f4ba7b35ecd3b3639af4e648aff84b713a7f30b51d5a4c30c8809aeb8dc8024", "operations": { "get_liveness": { "method": "GET", diff --git a/desktop/src-tauri/tests/memory.rs b/desktop/src-tauri/tests/memory.rs index 36d1be2fac..61c56ccc10 100644 --- a/desktop/src-tauri/tests/memory.rs +++ b/desktop/src-tauri/tests/memory.rs @@ -150,9 +150,23 @@ async fn setup() -> Fixture { let manager = Arc::new(ConnectionManager::new( ProfileRepository::open(dir.path().join("profiles.json"), Arc::new(WindowsVault)).unwrap(), )); - let compatibility = manager.state().unwrap().compatibility_profiles[0] + let contract: serde_json::Value = + serde_json::from_str(include_str!("../src/transport/operations.json")).unwrap(); + let contract_sha256 = contract["contractSha256"].as_str().unwrap(); + let state = manager.state().unwrap(); + let mut qualified = state + .compatibility_profiles + .iter() + .filter(|profile| profile.contract_sha256 == contract_sha256); + let compatibility = qualified + .next() + .expect("qualified fixture contract") .id .clone(); + assert!( + qualified.next().is_none(), + "ambiguous fixture qualification" + ); let mut ids = vec![]; for name in ["A", "B"] { let state = manager.save_profile(serde_json::from_value(serde_json::json!({"id":null,"revision":null,"name":name,"endpoint":endpoint,"authentication":"unauthenticated_loopback","caPem":null,"compatibility":compatibility,"keepCredential":false,"credential":null})).unwrap()).unwrap(); diff --git a/desktop/src-tauri/tests/session.rs b/desktop/src-tauri/tests/session.rs index be08df88bc..1363ac883d 100644 --- a/desktop/src-tauri/tests/session.rs +++ b/desktop/src-tauri/tests/session.rs @@ -108,9 +108,23 @@ async fn fixture() -> Fixture { } } fn add(manager: &ConnectionManager, name: &str, endpoint: &str) -> String { - let compatibility = manager.state().unwrap().compatibility_profiles[0] + let contract: serde_json::Value = + serde_json::from_str(include_str!("../src/transport/operations.json")).unwrap(); + let contract_sha256 = contract["contractSha256"].as_str().unwrap(); + let state = manager.state().unwrap(); + let mut qualified = state + .compatibility_profiles + .iter() + .filter(|profile| profile.contract_sha256 == contract_sha256); + let compatibility = qualified + .next() + .expect("qualified fixture contract") .id .clone(); + assert!( + qualified.next().is_none(), + "ambiguous fixture qualification" + ); let state = manager.save_profile(serde_json::from_value(serde_json::json!({ "id":null,"revision":null,"name":name,"endpoint":endpoint, "authentication":"unauthenticated_loopback","caPem":null,"compatibility":compatibility, diff --git a/desktop/tests/installed_fixture.py b/desktop/tests/installed_fixture.py index 15880f74f7..7242c24427 100644 --- a/desktop/tests/installed_fixture.py +++ b/desktop/tests/installed_fixture.py @@ -33,6 +33,16 @@ from real_server import HarnessFailure, control_pipe +def fixture_compatibility_profile() -> str: + desktop = Path(__file__).resolve().parents[1] + contract = json.loads((desktop / "src-tauri/src/transport/operations.json").read_text(encoding="utf-8")) + profiles = json.loads((desktop / "src-tauri/src/connections/compatibility.json").read_text(encoding="utf-8")) + matches = [profile["id"] for profile in profiles if profile["contractSha256"] == contract["contractSha256"]] + if len(matches) != 1: + raise HarnessFailure("installed_server_compatibility_profile") + return matches[0] + + def wait_ready(client: httpx.Client, process: subprocess.Popen[bytes]) -> None: for _ in range(150): if process.poll() is not None: diff --git a/desktop/tests/installed_workflow.py b/desktop/tests/installed_workflow.py index 579811a312..34cab14191 100644 --- a/desktop/tests/installed_workflow.py +++ b/desktop/tests/installed_workflow.py @@ -23,7 +23,7 @@ import httpx from installed_boundaries import exercise_note_budget, exercise_search_limit -from installed_fixture import isolated_server +from installed_fixture import fixture_compatibility_profile, isolated_server from real_server import HarnessFailure ELEMENT = "element-6066-11e4-a52e-4f735466cecf" @@ -110,7 +110,8 @@ def connect(self, name: str, endpoint: str) -> None: self.button("连接") self.type("连接名称", name) self.type("Server 地址", endpoint) - self.click("//label[normalize-space(text())='已验证兼容配置']/select/option[@value='sqlite-6e237568-v1']") + compatibility = fixture_compatibility_profile() + self.click(f"//label[normalize-space(text())='已验证兼容配置']/select/option[@value='{compatibility}']") self.button("保存配置") self.activate(name) diff --git a/desktop/ui/src/generated/api.d.ts b/desktop/ui/src/generated/api.d.ts index d235ec9f2e..4bb9f35529 100644 --- a/desktop/ui/src/generated/api.d.ts +++ b/desktop/ui/src/generated/api.d.ts @@ -596,6 +596,26 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/handoff/hint": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Prepare optional compact continuity hints from a selected Handoff + * @description Explicitly request untrusted historical orientation for a fresh session. Select an exact Revision, a complete transferred PreparedHandoff, or latest only after resolving the intended workstream Scope. Hints project existing Handoff fields and authorized evidence references without transcript summaries or model generation. They never replace full Handoff reads, evidence checks, current instructions, or live validation. The complete rendered UTF-8 hint is limited to max_bytes (at most 4000); omit it rather than truncate when budget or evidence is insufficient. Blocked hints include the full recorded state only when the complete hint fits. Authorization matches Continue: exact/latest require artifact.read and handoff.evidence.inspect on the selected Handoff, covering only its citation manifest without general Scope read; prepared requires scope.read. General evidence APIs retain their independent permission checks. Prepared selection provides no exact Revision reference; the receiver must retain the complete transferred PreparedHandoff. + */ + post: operations["prepare_handoff_hint"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/topic-memory/flush": { parameters: { query?: never; @@ -616,6 +636,26 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/scopes/{scope_id}/artifacts/{family}/search": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Search one Artifact Family in a Scope + * @description Search registered Artifact Families using their supported retrieval controls. Results contain complete exact Artifact revisions. Score metadata is returned only when requested; min_score compares normalized retrieval scores. + */ + post: operations["search_artifacts"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/topic-memory/search": { parameters: { query?: never; @@ -2753,6 +2793,19 @@ export interface components { prepared?: components["schemas"]["PreparedHandoff"]; revision?: components["schemas"]["ArtifactReference"]; }; + PrepareHandoffHintRequest: { + scope_id: string; + selection: components["schemas"]["HandoffSelection"]; + /** @description Required only for prepared selection; retain this complete value for continuation. */ + prepared?: components["schemas"]["PreparedHandoff"]; + /** @description Required only for exact selection. */ + revision?: components["schemas"]["ArtifactReference"]; + /** + * @description Complete rendered UTF-8 text budget including notice, boundaries, and exact references. + * @default 2000 + */ + max_bytes: number; + }; FinalizeHandoffRequest: { scope_id: string; draft: components["schemas"]["HandoffDraft"]; @@ -3712,6 +3765,58 @@ export interface components { score: number; matched_by: components["schemas"]["TopicMemoryMatchedBy"][]; }; + ArtifactSearchFusion: { + method: string; + /** @default {} */ + params: { + [key: string]: unknown; + }; + }; + SearchArtifactsRequest: { + query: string; + /** @default 10 */ + limit: number; + mode?: string; + filters?: { + [key: string]: unknown; + }; + admission?: { + [key: string]: unknown; + }; + fusion?: components["schemas"]["ArtifactSearchFusion"]; + min_score?: number; + /** @default false */ + include_scores: boolean; + rerank?: { + [key: string]: unknown; + }; + }; + ArtifactChannelScore: { + raw: number; + metric: string; + higher_is_better: boolean; + }; + ArtifactSearchScores: { + retrieval: number; + channels: { + [key: string]: components["schemas"]["ArtifactChannelScore"]; + }; + }; + ArtifactSearchItem: { + family: string; + artifact_id: string; + revision: number; + content: { + [key: string]: unknown; + }; + lineage: { + [key: string]: unknown; + }; + scores?: components["schemas"]["ArtifactSearchScores"]; + }; + SearchArtifactsResponse: { + results: components["schemas"]["ArtifactSearchItem"][]; + }; SearchTopicMemoryRequest: { scope_id: string; query: string; @@ -5522,6 +5627,37 @@ export interface operations { 503: components["responses"]["Unavailable"]; }; }; + prepare_handoff_hint: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PrepareHandoffHintRequest"]; + }; + }; + responses: { + /** @description Bounded orientation text, or empty when no complete supported hint can be delivered. */ + 200: { + headers: { + "X-PowerContext-Request-ID": components["headers"]["RequestId"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PreparedContext"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 422: components["responses"]["InvalidRequest"]; + 500: components["responses"]["InternalError"]; + 503: components["responses"]["Unavailable"]; + }; + }; flush_topic_memory: { parameters: { query?: never; @@ -5552,6 +5688,39 @@ export interface operations { 503: components["responses"]["Unavailable"]; }; }; + search_artifacts: { + parameters: { + query?: never; + header?: never; + path: { + scope_id: string; + family: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SearchArtifactsRequest"]; + }; + }; + responses: { + /** @description Complete matching Artifact revisions in retrieval order. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SearchArtifactsResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 422: components["responses"]["InvalidRequest"]; + 500: components["responses"]["InternalError"]; + 503: components["responses"]["Unavailable"]; + }; + }; search_topic_memory: { parameters: { query?: never; diff --git a/desktop/ui/src/generated/operations.ts b/desktop/ui/src/generated/operations.ts index dedc8162bc..13f7b4a8dc 100644 --- a/desktop/ui/src/generated/operations.ts +++ b/desktop/ui/src/generated/operations.ts @@ -15,7 +15,7 @@ */ // Generated from openapi/powercontext.yaml. Do not edit. -export const contractSha256 = "9af88b2b779c372a21ae9f398d0ca75b333f2e5da2afa3626df879911a442edd"; +export const contractSha256 = "9f4ba7b35ecd3b3639af4e648aff84b713a7f30b51d5a4c30c8809aeb8dc8024"; export const operations = { "get_liveness": { "method": "GET", diff --git a/docs/en/docs/workflows/artifacts.md b/docs/en/docs/workflows/artifacts.md index 1096afeed3..a9c02118b7 100644 --- a/docs/en/docs/workflows/artifacts.md +++ b/docs/en/docs/workflows/artifacts.md @@ -48,6 +48,13 @@ Topic Memory also uses these generic interfaces: creation and replacement submit `detail` text without semantic generation. See [Use Topic Memory](topic-memory.md) for its dedicated search and scoped reads. +## Search relevant current heads + +Use `POST /v1/scopes/{scope_id}/artifacts/{family}/search` for relevance search within one Scope. The built-in route +supports Experience, managed Skill, and Topic Memory. Results contain complete content and exact revisions; optional +scores belong to the search response. See [Search Artifacts](search-artifacts.md) for Family support, admission, +thresholds, and failure behavior, and [Fusion algorithms and parameters](search-fusion.md) for Topic RRF controls. + ## Change content through its workflow - [Memory](memory-and-context.md): explicitly write, revise, or retire entries. diff --git a/docs/en/docs/workflows/index.md b/docs/en/docs/workflows/index.md index 2114effd9c..80700cd06c 100644 --- a/docs/en/docs/workflows/index.md +++ b/docs/en/docs/workflows/index.md @@ -26,6 +26,7 @@ those belong to [deployment and operations](../operate/index.md), outside this d | Isolate projects and control access | Scope | [Scopes and access](scopes-and-access.md) | | Capture raw evidence or references to external material | Source | [Sources and capture](sources.md), [Ingest text files](ingest-text-files-with-opendal.md) | | Read revisions and choose a write workflow | Artifact | [Artifacts](artifacts.md) | +| Find relevant current artifacts and configure Topic fusion | Artifact search | [Search Artifacts](search-artifacts.md), [Fusion algorithms and parameters](search-fusion.md) | | Save and recall durable facts, decisions, and constraints | Memory | [Memory and context](memory-and-context.md) | | Build a reusable lesson and review it | Experience | [Create and review Experience](create-and-review-experience.md), [Experience lifecycle](experience-and-skill-lifecycle.md) | | Manage an exportable instruction set | Skill | [Create and export Skill](create-and-export-skill.md), [Configure Skill targets](configure-agent-skill-targets.md) | diff --git a/docs/en/docs/workflows/meta.json b/docs/en/docs/workflows/meta.json index ff33beae70..b56f2dfcda 100644 --- a/docs/en/docs/workflows/meta.json +++ b/docs/en/docs/workflows/meta.json @@ -10,6 +10,8 @@ "ingest-text-files-with-opendal", "---Artifacts and knowledge---", "artifacts", + "search-artifacts", + "search-fusion", "memory-and-context", "---Experience---", "create-and-review-experience", diff --git a/docs/en/docs/workflows/search-artifacts.md b/docs/en/docs/workflows/search-artifacts.md new file mode 100644 index 0000000000..5bd7299759 --- /dev/null +++ b/docs/en/docs/workflows/search-artifacts.md @@ -0,0 +1,191 @@ +--- +title: Search Artifacts +description: Search supported Artifact families within a Scope and interpret retrieval scores. +--- + +# Search Artifacts + +Use the unified search route to find relevant current Artifact heads in one Scope. Each result contains the exact +`family`, `artifact_id`, and `revision`, complete family-specific `content`, and `lineage`. Search does not enumerate +every Artifact or search across Scopes. Use [Manage Artifacts](artifacts.md) for lists and exact historical reads; +use [Prepare context text](prepare-context-text.md) when an Agent needs bounded context instead of complete content. + +## Send a search request + +The caller needs `scope.read` access. Put the Scope and Family in the path, and the search controls in the body. +Identity, access, and audit context come from server authentication and cannot be supplied in the request body: + +```http +POST /v1/scopes/project-a/artifacts/experience/search +Content-Type: application/json + +{ + "query": "release rollback", + "limit": 5, + "include_scores": true +} +``` + +Use the Server URL and credentials from your trusted environment. Include `Authorization: Bearer ` when +Access is enabled. The response is `{"results": [...]}`; an empty `results` array is a successful search with no +admitted result. Results retain their actual content fields and exact revision lineage. + +## Family support + +The built-in deployment currently exposes these capabilities through this route: + +| Family | Unified search | Modes and default | Searchable content | +| --- | --- | --- | --- | +| `experience` | Supported | `text`; omitted mode selects text | Approved, active Experience heads | +| `skill` | Supported | `text`; omitted mode selects text | Approved, active managed Skill heads; external Skill catalogs are excluded | +| `topic-memory` | Supported | `text`, `vector`, `hybrid`; default is hybrid with configured Embedding, otherwise text | Current Topic Memory heads, including title/summary and detail channels | +| `memory` | Unsupported | Existing Memory search remains available | See [Memory and context](memory-and-context.md) | +| `profile` | Unsupported | No relevance search through this route | See [Use Profiles](use-profiles.md) | +| `handoff` | Unsupported | No relevance search through this route | See [Memory and Handoff](memory-and-handoff.md) | +| `prompt` | Unsupported | No relevance search through this route | See [Manage Prompts](manage-prompts.md) | + +Sources are evidence and Tags are metadata; neither is an Artifact Family for this route. The deployment must have +the selected Family's retrieval index enabled. Topic vector and hybrid modes also require a compatible configured +Embedding model and index Profile. See [Configure vector search](configure-vector-search.md) for deployment settings. + +## Common controls and limits + +| Field | Type and default | Supported behavior | +| --- | --- | --- | +| `query` | Required non-empty string | Surrounding whitespace is trimmed. Experience and Topic allow at most 8192 characters; Skill allows 2000. | +| `limit` | Integer, default `10` | Experience and Skill: `1`–`200`; Topic: `1`–`20`. This is a maximum, not a promise to fill the result. | +| `mode` | Optional string | Select one of the Family's supported modes, or omit it to use its default. | +| `filters` | Object, default `{}` | Only an empty object is supported by these three Families. | +| `admission` | Optional object | Controls whether a retrieved candidate may contribute; fields are described below. | +| `min_score` | Optional finite number in `[0, 1]` | Keeps only results whose normalized retrieval score is at least this value. Omitted means no score threshold. | +| `include_scores` | Boolean, default `false` | Adds retrieval and available channel scores to each result. | +| `fusion` | Optional object | Only Topic exposes `rrf`; see [Fusion algorithms and parameters](search-fusion.md). | + +Omit optional fields to use their defaults. Explicit `null`, unknown fields, numeric strings, and booleans in numeric +fields are rejected. `limit` must be an integer; `include_scores` must be a JSON boolean. Non-empty `filters`, `rerank`, +and the `weighted_score` fusion method are unsupported. Experience and Skill reject `fusion` entirely. + +Thresholds are applied after admission and score calculation, before the final limit. The search does not add lower +scoring candidates to fill a result after `min_score` removes hits. Enabling scores changes only response metadata; +it does not change candidate admission, identities, ordering, or thresholds. + +## Experience and Skill text search + +Both Families search one `text` channel. Their admission fields are: + +| Field | Default | Range | +| --- | --- | --- | +| `admission.lexical_coverage` | `0.25` | Finite number in `[0, 1]` | +| `admission.lexical_min_matched_terms` | `2` | Integer at least `1` | + +Let `n` be the number of distinct analyzed query terms. A query with at most two terms requires one matching term. +Longer queries require at least `max(lexical_min_matched_terms, ceil(lexical_coverage × n))` matching terms. This admission +check precedes scoring and truncation. Unapproved or inactive heads cannot enter the result. + +Their normalized score is `relevance / (1 + relevance)`, where relevance is the negated SQLite BM25 raw score or the +OceanBase MATCH raw score. Both backends produce scores in `[0, 1]`, but their scales differ. Choose `min_score` from +observed results for your backend and workload; there is no shared recommended threshold across backends. + +## Topic Memory controls + +Topic has four named channels: + +| Public mode | Enabled channels | +| --- | --- | +| `text` | `topic_fts`, `detail_fts` | +| `vector` | `topic_vector`, `detail_vector` | +| `hybrid` | All four channels | + +Text searches title/summary and detail separately. Vector searches use the deployment's Embedding Profile. Text and +hybrid queries allow at most 64 distinct analyzed terms; vector-only queries use the 8192-character limit. + +Topic supports the two lexical admission fields above and `admission.min_semantic_similarity`, a finite number in +`[-1, 1]` with default `0.3`. Each lexical channel applies the matching-term rule independently. Each vector channel +requires `clip(1 - distance² / 2, -1, 1) >= min_semantic_similarity`, using the normalized vectors' L2 distance. A failure +in one channel removes that channel's contribution, while another admitted channel may still retain the topic. + +Topic combines admitted rankings with RRF. Omitted `fusion` uses `rrf`, rank constant `60`, and weight `1` per enabled +channel. The normalized RRF score is described in [Fusion algorithms and parameters](search-fusion.md). + +```http +POST /v1/scopes/project-a/artifacts/topic-memory/search +Content-Type: application/json + +{ + "query": "release rollback", + "mode": "hybrid", + "limit": 5, + "admission": {"lexical_coverage": 0.5}, + "fusion": { + "method": "rrf", + "params": { + "rank_constant": 60, + "weights": {"topic_fts": 2, "detail_vector": 0.5} + } + }, + "include_scores": true +} +``` + +Parameter combinations are validated even when the Scope contains no topics: + +- `text` rejects an explicitly supplied semantic admission field and any vector weight key, including weight `0`. +- `vector` rejects explicitly supplied lexical admission fields and any FTS weight key, including weight `0`. +- A weight key must name an enabled channel. Unknown channels and an enabled set with total weight `0` are rejected. +- Explicit `vector` or `hybrid`, an explicit semantic threshold, or any vector weight key requires vector capability. + An explicitly supplied value equal to its default still carries this requirement. + +With omitted mode, Topic uses hybrid when Embedding is configured and text otherwise. If the default hybrid request +has no explicit vector requirement, a temporarily unavailable or timed-out Embedding call may fall back to text, +provided the FTS channels have positive total weight. Explicit vector/hybrid requests and explicit vector requirements +never fall back. If both FTS weights are `0`, an Embedding failure remains a service failure. Invalid vectors, +incompatible Profiles, and storage failures do not trigger text fallback. + +## Read the scores + +With `include_scores: true`, each result includes a `scores` object. This example shows only that field: + +```json +{ + "scores": { + "retrieval": 0.5, + "channels": { + "topic_fts": { + "raw": -0.000004, + "metric": "sqlite_bm25", + "higher_is_better": false + } + } + } +} +``` + +`retrieval` is the normalized score in `[0, 1]` used for ordering and `min_score`. `channels` contains the actual raw +scores of channels that admitted this result: + +| Metric | Meaning | Better direction | +| --- | --- | --- | +| `sqlite_bm25` | Signed SQLite FTS BM25 score | Lower; matching scores are negative | +| `oceanbase_match` | OceanBase MATCH relevance | Higher | +| `l2_distance` | Topic vector L2 distance | Lower | + +Experience and Skill use the `text` channel name. Topic uses the four names above. Missing channels are omitted, +with no fabricated zero. An admitted zero-weight Topic channel can still appear in metadata when another channel +selects the same topic. Detail raw scores belong to the representative detail chunk selected by that channel. + +Raw values are backend and query dependent; they are not a universal similarity scale across queries or backends. +The Topic channel raw values are FTS scores or vector distances, not RRF contributions. Scores belong to this search +response and do not alter persisted Artifact content or lineage. With `include_scores: false`, `scores` is omitted. + +## Failures and existing interfaces + +An unknown Family returns `404`. A known Family without unified search, an unavailable configured search capability, +or an unsupported parameter returns `422`. Invalid requests report the parameter path; changing the limit or using +an empty Scope does not bypass validation. Model timeout/unavailability returns a service error when fallback is +not permitted (`503`). Implementation and storage failures remain failures, rather than successful empty results. + +The existing Memory and Topic HTTP search routes, Skill Library search, and existing SDK/internal Experience recall +keep their request and response contracts. In particular, `POST /v1/topic-memory/search` still chooses the deployment +default and returns summary hits with its legacy score scale of `0`–`100`; it does not expose these advanced controls. +The unified Topic route returns complete Artifact content and uses retrieval scores in `[0, 1]`. See +[Use Topic Memory](topic-memory.md) for the dedicated summary search and exact detail-read workflow. diff --git a/docs/en/docs/workflows/search-fusion.md b/docs/en/docs/workflows/search-fusion.md new file mode 100644 index 0000000000..a63328cddb --- /dev/null +++ b/docs/en/docs/workflows/search-fusion.md @@ -0,0 +1,105 @@ +--- +title: Fusion algorithms and parameters +description: Configure Topic Memory RRF ranks, weights, and normalized retrieval scores. +--- + +# Fusion algorithms and parameters + +The implemented fusion method is `rrf` (Reciprocal Rank Fusion). The unified Topic Memory search exposes it through +`fusion`; Experience and Skill expose one text channel and do not accept fusion parameters. See +[Search Artifacts](search-artifacts.md) for Family support, admission, modes, and request limits. + +## RRF calculation + +RRF combines the ranks of candidates admitted by each enabled channel. A better rank contributes more. The algorithm +uses ranks rather than mixing BM25 scores and vector distances, whose units and directions differ. + +For candidate `x`, let `rank_c(x)` be its positive, one-based rank in channel `c`, `w_c` the channel weight, and `K` +the rank constant: + +```text +raw_rrf(x) = Σ [w_c / (K + rank_c(x))] over channels containing x +upper = Σ [w_c / (K + 1)] over every enabled channel +retrieval(x) = raw_rrf(x) / upper +``` + +`retrieval` is finite and normalized to `[0, 1]`. It reaches `1` when a candidate ranks first in every positive-weight +enabled channel. An enabled channel with no admitted candidates still contributes to `upper`. A missing candidate +contributes nothing to that channel's numerator; it is not assigned a zero raw score. + +Ranks retain gaps after duplicate identities are removed within a channel. For example, ranks `1` and `3` remain `1` +and `3`; they are not renumbered. Candidates found only in zero-weight channels are excluded. After fusion, Topic +applies `min_score` and the requested limit. Equal fused scores use Artifact ID UTF-8 order, then descending Revision. + +## Parameters + +| Field | Type | Default | Valid values | +| --- | --- | --- | --- | +| `fusion.method` | String | `rrf` when `fusion` is omitted | Only `rrf` | +| `fusion.params` | JSON object | `{}` | Only `rank_constant` and `weights` | +| `fusion.params.rank_constant` | Integer | `60` | Positive integer | +| `fusion.params.weights` | Object from channel name to number | `{}` | Finite, non-negative numbers; enabled channels must have positive total weight | + +`rank_constant` is the only location for `K`; a top-level `K` or `rank_constant` is invalid. Numeric strings, booleans, +explicit `null`, non-finite values, unknown parameter keys, and negative weights are rejected. + +Weight overrides are partial: every enabled channel omitted from `weights` keeps weight `1`. Weight `0` removes a +channel's contribution to both the numerator and denominator. Scaling all weights by the same positive factor leaves +the normalized score unchanged. A larger `K` reduces the relative advantage of nearby top ranks; a smaller `K` makes +their rank differences matter more. + +Topic permits only the channel names enabled by the chosen mode: + +| Mode | Legal weight keys | +| --- | --- | +| `text` | `topic_fts`, `detail_fts` | +| `vector` | `topic_vector`, `detail_vector` | +| `hybrid` | `topic_fts`, `topic_vector`, `detail_fts`, `detail_vector` | + +An inactive or unknown key is rejected even if its weight is `0`. With omitted mode, supplying any vector weight key +requires vector capability and prevents the default text fallback. Admission and deployment checks still apply; +changing a weight cannot enable an unavailable retrieval channel. + +## Example: text channels with different weights + +```http +POST /v1/scopes/project-a/artifacts/topic-memory/search +Content-Type: application/json + +{ + "query": "release rollback", + "mode": "text", + "fusion": { + "method": "rrf", + "params": { + "rank_constant": 60, + "weights": {"topic_fts": 2} + } + }, + "include_scores": true +} +``` + +`topic_fts` has weight `2`; omitted `detail_fts` retains weight `1`. The normalization denominator is `3 / 61`, even +when detail search returns nothing. A candidate first in `topic_fts` and absent from detail has score `2 / 3`. + +For two equal-weight channels with `K = 60`: + +| Candidate ranks | Raw RRF | Normalized score | +| --- | --- | --- | +| First in both channels | `2 / 61` | `1` | +| First in one, absent from the other | `1 / 61` | `0.5` | +| First in one, other channel empty | `1 / 61` | `0.5` | +| Third in one, absent from the other | `1 / 63` | `61 / 126` | + +An empty enabled channel therefore remains visible in the score's interpretation. Scores from different enabled +channel sets or weight plans should not be treated as the same relevance calibration. + +## Scores and current boundaries + +`scores.retrieval` exposes normalized RRF. `scores.channels` exposes original FTS scores or L2 distances with a metric +and direction; it does not expose the internal raw RRF sum. See [Read the scores](search-artifacts.md#read-the-scores). + +Only RRF is available. `weighted_score`, reranking, and non-empty Family filters are unsupported. Fusion does not relax +channel admission or fill results below `min_score`. The original Topic-specific search keeps its default RRF behavior +and legacy `0`–`100` score scale; advanced fusion controls are available through the unified route. diff --git a/docs/en/docs/workflows/topic-memory.md b/docs/en/docs/workflows/topic-memory.md index fd1e045850..0050afaa01 100644 --- a/docs/en/docs/workflows/topic-memory.md +++ b/docs/en/docs/workflows/topic-memory.md @@ -53,8 +53,9 @@ Use the configuration wizard to select full memory, or configure the deployment - meaningful Source evidence in the Scope. Embedding is optional for Topic Memory itself. Configure a compatible Embedding model and dimension when the deployment -should offer vector or hybrid retrieval. The Server exposes the selected retrieval mode in each search response; callers -do not choose arbitrary retrieval controls. +should offer vector or hybrid retrieval. The dedicated Topic search exposes the selected retrieval mode in each +response and uses the deployment default. The [unified Artifact search](search-artifacts.md) also accepts explicit +modes and supported search controls. The schedule is an admission interval, not a completion deadline. Topic generation also has provider and Worker timeouts. An unset schedule disables new automatic admission but does not discard already accepted work. For SQLite deployments, Topic Workers @@ -108,8 +109,13 @@ Content-Type: application/json ``` The response reports the deployment's actual `mode` (`fts` or `hybrid`) and `hits`. Each hit contains an exact `artifact` -reference, `title`, `summary`, an optional `snippet`, `score`, and `matched_by`. Search sees only current Topic Memory -heads in the current Scope; it does not search across Scopes or accept a caller-selected retrieval mode. +reference, `title`, `summary`, an optional `snippet`, `score`, and `matched_by`. This dedicated search sees only current +Topic Memory heads in the current Scope; it does not search across Scopes or accept a caller-selected retrieval mode. + +For complete Artifact results, explicit text/vector/hybrid modes, admission, score thresholds, or channel scores, use +[Search Artifacts](search-artifacts.md). Topic fusion controls are documented in +[Fusion algorithms and parameters](search-fusion.md). The dedicated route keeps its existing summary response and +`0`–`100` score scale. ## Read exact detail diff --git a/docs/zh/docs/workflows/artifacts.md b/docs/zh/docs/workflows/artifacts.md index f8c7115714..fb3c15e029 100644 --- a/docs/zh/docs/workflows/artifacts.md +++ b/docs/zh/docs/workflows/artifacts.md @@ -45,6 +45,12 @@ Python Client 将该字段表示为字符串，请直接使用，不再访问枚 Topic Memory 也使用上述通用接口：创建和整体替换提交完整的 `title`、`summary` 和 `detail`，内容不经过语义生成。 它的专用搜索和范围化读取见[使用 Topic Memory](topic-memory.md)。 +## 检索相关的当前 head + +使用 `POST /v1/scopes/{scope_id}/artifacts/{family}/search` 在一个 Scope 内按相关性检索。 +内置入口支持 Experience、托管 Skill 和 Topic Memory，返回完整内容及精确 Revision；可选评分属于本次搜索响应。 +Family 支持范围、准入、阈值和故障行为见[检索 Artifact](search-artifacts.md)，Topic 的 RRF 控制参数见[融合算法与参数](search-fusion.md)。 + ## 按对应工作流修改 - [Memory](memory-and-context.md)：显式写入、修订或退役条目。 diff --git a/docs/zh/docs/workflows/index.md b/docs/zh/docs/workflows/index.md index 9c4be192f3..6a63c47366 100644 --- a/docs/zh/docs/workflows/index.md +++ b/docs/zh/docs/workflows/index.md @@ -24,6 +24,7 @@ Scope 管理 Source、Artifact 和 Candidate。Source 保存证据；制品可 | 隔离项目并控制访问范围 | Scope | [Scope 与访问控制](scopes-and-access.md) | | 保存原始证据或外部材料引用 | Source | [Sources 与采集](sources.md)、[从文本文件采集](ingest-text-files-with-opendal.md) | | 读取制品版本并选择写入方式 | Artifact | [Artifact](artifacts.md) | +| 查找相关的当前制品并配置 Topic 融合 | Artifact 检索 | [检索 Artifact](search-artifacts.md)、[融合算法与参数](search-fusion.md) | | 保存和召回长期事实、决定与约束 | Memory | [Memory 与上下文](memory-and-context.md) | | 形成可复用的经验并经过审核 | Experience | [创建和审核 Experience](create-and-review-experience.md)、[Experience 生命周期](experience-and-skill-lifecycle.md) | | 管理可导出的指令集合 | Skill | [创建和导出 Skill](create-and-export-skill.md)、[配置 Skill 目标](configure-agent-skill-targets.md) | diff --git a/docs/zh/docs/workflows/meta.json b/docs/zh/docs/workflows/meta.json index a955d8ae09..169d8b56b2 100644 --- a/docs/zh/docs/workflows/meta.json +++ b/docs/zh/docs/workflows/meta.json @@ -10,6 +10,8 @@ "ingest-text-files-with-opendal", "---Artifact 与知识---", "artifacts", + "search-artifacts", + "search-fusion", "memory-and-context", "---Experience---", "create-and-review-experience", diff --git a/docs/zh/docs/workflows/search-artifacts.md b/docs/zh/docs/workflows/search-artifacts.md new file mode 100644 index 0000000000..8eacc0d78a --- /dev/null +++ b/docs/zh/docs/workflows/search-artifacts.md @@ -0,0 +1,184 @@ +--- +title: 检索 Artifact +description: 在一个 Scope 内检索受支持的 Artifact 家族，并理解检索评分。 +--- + +# 检索 Artifact + +统一检索入口用于查找一个 Scope 内相关的 Artifact 当前 head。每条结果包含精确的 `family`、`artifact_id`、 +`revision`、该家族的完整 `content` 和 `lineage`。搜索不会完整枚举制品，也不跨 Scope 检索。 +列举制品和读取历史版本见[管理 Artifact](artifacts.md)；Agent 需要有界上下文时，使用[准备上下文文本](prepare-context-text.md)， +避免直接注入完整正文。 + +## 发起检索 + +调用方需要 `scope.read` 权限。Scope 和 Family 放在路径中，检索参数放在请求体中。 +调用者身份、访问控制和审计上下文来自服务端认证，不能通过请求体提供： + +```http +POST /v1/scopes/project-a/artifacts/experience/search +Content-Type: application/json + +{ + "query": "release rollback", + "limit": 5, + "include_scores": true +} +``` + +使用受信任环境中的 Server 地址和凭据；启用 Access 时，添加 `Authorization: Bearer ` Header。 +响应为 `{"results": [...]}`。`results` 为空表示搜索成功，但没有通过准入的结果。返回内容保留实际的家族字段和精确版本 lineage。 + +## 各 Family 的支持范围 + +内置部署通过此入口提供以下能力： + +| Family | 统一检索 | 模式与默认策略 | 检索内容 | +| --- | --- | --- | --- | +| `experience` | 支持 | `text`；省略模式时使用文本检索 | 已批准、处于 active 状态的 Experience head | +| `skill` | 支持 | `text`；省略模式时使用文本检索 | 已批准、处于 active 状态的托管 Skill head；不检索外部 Skill 目录 | +| `topic-memory` | 支持 | `text`、`vector`、`hybrid`；配置 Embedding 时默认混合检索，否则默认文本检索 | Topic Memory 当前 head，分别检索标题/摘要和详情通道 | +| `memory` | 不支持 | 保留原有 Memory 检索入口 | 见 [Memory 与上下文](memory-and-context.md) | +| `profile` | 不支持 | 此入口不提供相关性检索 | 见[使用 Profile](use-profiles.md) | +| `handoff` | 不支持 | 此入口不提供相关性检索 | 见 [Memory 与 Handoff](memory-and-handoff.md) | +| `prompt` | 不支持 | 此入口不提供相关性检索 | 见[管理 Prompt](manage-prompts.md) | + +Source 是证据，Tag 是元数据，两者都不是此入口检索的 Artifact Family。部署必须启用所选 Family 的检索索引。 +Topic 向量或混合检索还需要兼容的 Embedding 模型和索引 Profile。部署设置见[配置向量检索](configure-vector-search.md)。 + +## 通用参数与限制 + +| 字段 | 类型与默认值 | 支持范围 | +| --- | --- | --- | +| `query` | 必填非空字符串 | 去除首尾空白。Experience 和 Topic 最多 8192 个字符，Skill 最多 2000 个字符。 | +| `limit` | 整数，默认 `10` | Experience、Skill 为 `1`–`200`；Topic 为 `1`–`20`。这是返回上限，不保证填满。 | +| `mode` | 可省略的字符串 | 选择该 Family 支持的模式；省略时使用默认策略。 | +| `filters` | 对象，默认 `{}` | 这三个 Family 只支持空对象。 | +| `admission` | 可省略的对象 | 决定检索到的候选能否参与评分；字段见下文。 | +| `min_score` | 可省略的有限数值，范围 `[0, 1]` | 只保留归一化检索分数不低于该值的结果；省略时不设评分阈值。 | +| `include_scores` | 布尔值，默认 `false` | 在结果中附加检索分数和实际存在的通道原分。 | +| `fusion` | 可省略的对象 | 只有 Topic 开放 `rrf`，见[融合算法与参数](search-fusion.md)。 | + +使用默认值时省略对应字段。显式 `null`、未知字段、数字字符串、数值字段中的布尔值均会被拒绝。 +`limit` 必须是整数，`include_scores` 必须是 JSON 布尔值。非空 `filters`、`rerank` 和 `weighted_score` 融合方法 +均未开放。Experience 和 Skill 不接受 `fusion`。 + +准入和评分完成后，先应用 `min_score`，再截取最终 `limit`。阈值过滤后不会用更低分的候选补齐数量。 +开启评分只增加响应元数据，不改变候选准入、结果身份、顺序或阈值行为。 + +## Experience 和 Skill 文本检索 + +这两个 Family 都使用一个 `text` 通道，支持以下准入字段： + +| 字段 | 默认值 | 范围 | +| --- | --- | --- | +| `admission.lexical_coverage` | `0.25` | 有限数值，范围 `[0, 1]` | +| `admission.lexical_min_matched_terms` | `2` | 不小于 `1` 的整数 | + +设 `n` 为分析后去重的查询词数量。查询最多含两个词时，至少匹配一个词；更长的查询至少需要匹配 +`max(lexical_min_matched_terms, ceil(lexical_coverage × n))` 个词。准入检查先于评分和截断。 +未批准或非 active 的 head 不会进入结果。 + +归一化评分为 `relevance / (1 + relevance)`。SQLite 的 relevance 为 BM25 原分取负，OceanBase 的 relevance +为 MATCH 原分。两个 backend 都返回 `[0, 1]` 范围的分数，但标尺不同。请根据实际 backend 和查询样本选择 +`min_score`，没有可跨 backend 直接套用的统一推荐阈值。 + +## Topic Memory 参数 + +Topic 使用四个命名通道： + +| 公开模式 | 启用通道 | +| --- | --- | +| `text` | `topic_fts`、`detail_fts` | +| `vector` | `topic_vector`、`detail_vector` | +| `hybrid` | 全部四个通道 | + +文本模式分别检索标题/摘要和详情；向量模式使用部署配置的 Embedding Profile。文本和混合查询最多支持 +64 个分析后去重的查询词；纯向量查询使用 8192 字符限制。 + +Topic 支持上述两个词法准入字段，以及 `admission.min_semantic_similarity`：有限数值，范围 `[-1, 1]`，默认 `0.3`。 +每个词法通道独立应用匹配词数量规则。每个向量通道要求归一化向量的 L2 距离满足 +`clip(1 - distance² / 2, -1, 1) >= min_semantic_similarity`。某个通道未通过准入时，只移除该通道的贡献； +其他通道通过准入后，仍可能保留同一个主题。 + +Topic 使用 RRF 合并通过准入的排名。省略 `fusion` 时，默认使用 `rrf`、排名常数 `60`，每个启用通道的权重为 `1`。 +归一化公式见[融合算法与参数](search-fusion.md)。 + +```http +POST /v1/scopes/project-a/artifacts/topic-memory/search +Content-Type: application/json + +{ + "query": "release rollback", + "mode": "hybrid", + "limit": 5, + "admission": {"lexical_coverage": 0.5}, + "fusion": { + "method": "rrf", + "params": { + "rank_constant": 60, + "weights": {"topic_fts": 2, "detail_vector": 0.5} + } + }, + "include_scores": true +} +``` + +即使 Scope 内没有主题，也会校验参数组合： + +- `text` 不接受显式语义准入字段和任何向量权重键，包括权重为 `0` 的键。 +- `vector` 不接受显式词法准入字段和任何 FTS 权重键，包括权重为 `0` 的键。 +- 权重键必须属于当前启用通道。未知通道或启用通道总权重为 `0` 均会被拒绝。 +- 显式 `vector`、`hybrid`、语义阈值或任意向量权重键都要求向量能力。显式提供与默认值相同的值也带有这一要求。 + +省略模式时，已配置 Embedding 的 Topic 默认使用混合检索，否则使用文本检索。默认混合请求没有显式向量要求、 +且 FTS 通道总权重为正时，Embedding 暂时不可用或超时可以回退到文本。显式向量/混合请求，以及带有显式向量要求 +的请求都不会回退。两个 FTS 权重都为 `0` 时，Embedding 失败仍是服务错误。非法向量、不兼容的 Profile 和存储错误 +不会触发文本回退。 + +## 理解评分 + +`include_scores: true` 会在每条结果中增加 `scores`。下面只展示该字段： + +```json +{ + "scores": { + "retrieval": 0.5, + "channels": { + "topic_fts": { + "raw": -0.000004, + "metric": "sqlite_bm25", + "higher_is_better": false + } + } + } +} +``` + +`retrieval` 是 `[0, 1]` 范围的归一化检索分数，用于排序和 `min_score` 筛选。 +`channels` 保存该结果通过准入的通道原分： + +| 度量 | 含义 | 更好的方向 | +| --- | --- | --- | +| `sqlite_bm25` | SQLite FTS 的带符号 BM25 原分 | 越小越好；匹配分数为负数 | +| `oceanbase_match` | OceanBase MATCH 相关性原分 | 越大越好 | +| `l2_distance` | Topic 向量 L2 距离 | 越小越好 | + +Experience 和 Skill 的通道名为 `text`；Topic 使用上表列出的四个通道名。没有命中的通道不会出现，也不会补造 `0`。 +如果其他通道选中了同一个主题，通过准入但权重为 `0` 的 Topic 通道仍可出现在评分元数据中。 +详情原分对应各通道实际选出的代表分块。 + +原分依赖 backend 和查询，不是跨查询、跨 backend 通用的相似度。Topic 通道原分是 FTS 分数或向量距离， +不是 RRF 贡献值。评分属于本次搜索响应，不修改已保存的 Artifact 正文或 lineage。 +`include_scores: false` 时，响应省略 `scores`。 + +## 故障与原有接口 + +未知 Family 返回 `404`；已知但不支持统一搜索的 Family、未配置的检索能力或不支持的参数返回 `422`。 +非法请求会指出参数路径；缩小 limit 或使用空 Scope 都不会绕过校验。不能回退时，模型超时或不可用返回 `503` 服务错误。 +实现和存储故障会保留失败状态，不转换成成功的空结果。 + +原有 Memory、Topic HTTP 搜索路由、Skill Library 搜索，以及已有 SDK/内部 Experience 召回保留原来的请求与响应契约。 +其中 `POST /v1/topic-memory/search` 仍按部署默认策略检索，返回摘要命中和 `0`–`100` 的旧评分， +不开放上述高级参数。统一 Topic 入口返回完整 Artifact 正文，归一化检索分数为 `[0, 1]`。 +专用摘要搜索和精确详情读取流程见[使用 Topic Memory](topic-memory.md)。 diff --git a/docs/zh/docs/workflows/search-fusion.md b/docs/zh/docs/workflows/search-fusion.md new file mode 100644 index 0000000000..3c6fc46b31 --- /dev/null +++ b/docs/zh/docs/workflows/search-fusion.md @@ -0,0 +1,100 @@ +--- +title: 融合算法与参数 +description: 配置 Topic Memory 的 RRF 排名常数、通道权重和归一化检索评分。 +--- + +# 融合算法与参数 + +当前实现的融合方法为 `rrf`（Reciprocal Rank Fusion，倒数排名融合）。统一 Topic Memory 检索通过 `fusion` +开放这些参数。Experience 和 Skill 只有一个文本通道，不接受融合参数。 +Family 支持范围、准入、模式和请求限制见[检索 Artifact](search-artifacts.md)。 + +## RRF 计算方式 + +RRF 合并各启用通道中通过准入的候选排名，排名越靠前，贡献越大。算法使用排名，不直接混合单位和方向不同的 +BM25 原分与向量距离。 + +对于候选 `x`，设 `rank_c(x)` 为它在通道 `c` 中从 `1` 开始的正整数排名，`w_c` 为通道权重，`K` 为排名常数： + +```text +raw_rrf(x) = Σ [w_c / (K + rank_c(x))] 对包含 x 的通道求和 +upper = Σ [w_c / (K + 1)] 对所有启用通道求和 +retrieval(x) = raw_rrf(x) / upper +``` + +`retrieval` 是有限数值，归一化范围为 `[0, 1]`。候选在所有正权重的启用通道中都排名第一时，分数为 `1`。 +即使某个启用通道没有通过准入的候选，也仍计入 `upper`。候选没有在某个通道出现时，该通道不提供分子贡献， +也不会给它补造原分 `0`。 + +同一通道内去除重复身份后保留排名间隔。例如排名 `1`、`3` 仍使用 `1`、`3`，不会重新编号。 +只在零权重通道出现的候选不进入结果。融合完成后，Topic 再应用 `min_score` 和请求的 limit。 +融合分相同的结果按 Artifact ID 的 UTF-8 升序排序，再按 Revision 降序排序。 + +## 参数 + +| 字段 | 类型 | 默认值 | 合法范围 | +| --- | --- | --- | --- | +| `fusion.method` | 字符串 | 省略 `fusion` 时使用 `rrf` | 只支持 `rrf` | +| `fusion.params` | JSON 对象 | `{}` | 只接受 `rank_constant` 和 `weights` | +| `fusion.params.rank_constant` | 整数 | `60` | 正整数 | +| `fusion.params.weights` | 通道名到数值的对象 | `{}` | 有限非负数；启用通道总权重必须为正 | + +`K` 只通过 `rank_constant` 设置，顶层的 `K` 或 `rank_constant` 都是非法字段。 +数字字符串、布尔值、显式 `null`、非有限数值、未知参数键和负权重都会被拒绝。 + +权重可以只覆盖部分通道，未在 `weights` 中出现的启用通道仍使用权重 `1`。 +权重 `0` 会同时移除该通道在分子和分母中的贡献。所有权重同时乘以同一个正数，不改变归一化分数。 +`K` 越大，前几名之间的相对差距越小；`K` 越小，排名差异的影响越大。 + +Topic 只接受所选模式启用的通道名： + +| 模式 | 合法权重键 | +| --- | --- | +| `text` | `topic_fts`、`detail_fts` | +| `vector` | `topic_vector`、`detail_vector` | +| `hybrid` | `topic_fts`、`topic_vector`、`detail_fts`、`detail_vector` | + +未启用或未知的通道键即使权重为 `0` 也会被拒绝。省略模式时，任何显式向量权重键都会要求向量能力，并禁止默认文本回退。 +准入和部署能力检查仍然有效，修改权重不能启用部署不具备的检索通道。 + +## 示例：为两个文本通道设置不同权重 + +```http +POST /v1/scopes/project-a/artifacts/topic-memory/search +Content-Type: application/json + +{ + "query": "release rollback", + "mode": "text", + "fusion": { + "method": "rrf", + "params": { + "rank_constant": 60, + "weights": {"topic_fts": 2} + } + }, + "include_scores": true +} +``` + +`topic_fts` 的权重为 `2`；未指定的 `detail_fts` 仍使用权重 `1`。归一化分母为 `3 / 61`，详情检索没有结果时也一样。 +候选在 `topic_fts` 排名第一、未在详情通道出现时，分数为 `2 / 3`。 + +两个等权重通道、`K = 60` 时： + +| 候选排名 | RRF 原始和 | 归一化分数 | +| --- | --- | --- | +| 两个通道都排名第一 | `2 / 61` | `1` | +| 一个通道排名第一，另一个通道未命中 | `1 / 61` | `0.5` | +| 一个通道排名第一，另一个通道为空 | `1 / 61` | `0.5` | +| 一个通道排名第三，另一个通道未命中 | `1 / 63` | `61 / 126` | + +启用但为空的通道仍影响分数的含义。不同启用通道集合或权重方案下的分数，不应直接视为同一套相关性标尺。 + +## 评分字段与当前边界 + +`scores.retrieval` 返回归一化 RRF 分数。`scores.channels` 返回带有度量和方向的 FTS 原分或 L2 距离， +不返回内部 RRF 原始和。详见[理解评分](search-artifacts.md#理解评分)。 + +当前只支持 RRF，未开放 `weighted_score`、重排和非空 Family 过滤条件。融合不会放宽通道准入，也不会用低于 +`min_score` 的结果补齐数量。原 Topic 专用搜索保留默认 RRF 行为和 `0`–`100` 的旧评分；高级融合参数通过统一入口使用。 diff --git a/docs/zh/docs/workflows/topic-memory.md b/docs/zh/docs/workflows/topic-memory.md index 96909af274..ac826b7c5f 100644 --- a/docs/zh/docs/workflows/topic-memory.md +++ b/docs/zh/docs/workflows/topic-memory.md @@ -47,8 +47,8 @@ Source → 后台处理游标 → flush 请求 → 新建或更新不可变 Topi - 将 `POWERCONTEXT_SERVER_RUNTIME_TOPIC_MEMORY_SCHEDULE_SECONDS` 设为正数，以接纳新的自动处理任务； - Scope 中存在可归纳的 Source 证据。 -Topic Memory 本身不强制要求 Embedding。需要向量或混合检索时，再配置兼容的 Embedding 模型和维度。Server 会在每次 -搜索响应中返回实际使用的检索模式；调用方不能任意指定检索控制参数。 +Topic Memory 本身不强制要求 Embedding。需要向量或混合检索时，再配置兼容的 Embedding 模型和维度。专用搜索会在每次 +响应中返回实际使用的检索模式，并使用部署默认策略。[统一 Artifact 检索](search-artifacts.md)还支持显式模式和已开放的检索参数。 调度周期是接纳间隔，不是完成时限。Topic 生成还受到 Provider 请求超时和 Worker 总超时控制。调度未设置时，不再接纳新的 自动任务，但已经接纳的任务不会因此丢失。使用 SQLite 部署时，配置了 Generation 模型的 Topic Worker 要求持久化的 @@ -97,9 +97,12 @@ Content-Type: application/json ``` 响应包含部署实际使用的 `mode`（`fts` 或 `hybrid`）和 `hits`。每个命中包括 `artifact` 精确引用、`title`、 -`summary`、可为空的 `snippet`、`score` 和 `matched_by`。搜索只查看当前 Scope 的 Topic Memory head，不跨 Scope +`summary`、可为空的 `snippet`、`score` 和 `matched_by`。此专用搜索只查看当前 Scope 的 Topic Memory head，不跨 Scope 检索，也不接受调用方自选的检索模式。 +需要完整 Artifact 结果、显式文本/向量/混合模式、准入、评分阈值或通道原分时，使用[检索 Artifact](search-artifacts.md)。 +Topic 的融合参数见[融合算法与参数](search-fusion.md)。专用入口保留原来的摘要响应和 `0`–`100` 评分。 + ## 读取精确详情 不要根据标题自行拼接最新版本。把搜索结果中的 `artifact`（`family`、`artifact_id`、`revision`）原样传给 diff --git a/integrations/dify/plugin/powercontext_dify/contract.json b/integrations/dify/plugin/powercontext_dify/contract.json index 6752feea01..b3168b4d99 100644 --- a/integrations/dify/plugin/powercontext_dify/contract.json +++ b/integrations/dify/plugin/powercontext_dify/contract.json @@ -1,5 +1,5 @@ { - "openapi_sha256": "790df0d51dd4ca600b34b77a4b6cce8b064305cbc65666e2813f96b23f3fd9fa", + "openapi_sha256": "9f4ba7b35ecd3b3639af4e648aff84b713a7f30b51d5a4c30c8809aeb8dc8024", "api_version": "1.2.0", "operations": { "get_scope": { diff --git a/integrations/dsh/plugins/powercontext/lib/index.js b/integrations/dsh/plugins/powercontext/lib/index.js index 599ef8648c..efaa3073f8 100644 --- a/integrations/dsh/plugins/powercontext/lib/index.js +++ b/integrations/dsh/plugins/powercontext/lib/index.js @@ -561,6 +561,17 @@ const OPERATIONS$1 = { successStatuses: [200], emptyStatuses: [] }, + search_artifacts: { + method: "POST", + path: "/v1/scopes/{scope_id}/artifacts/{family}/search", + location: "body", + scopeMode: "none", + pathParameters: ["scope_id", "family"], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, search_topic_memory: { method: "POST", path: "/v1/topic-memory/search", diff --git a/integrations/dsh/plugins/powercontext/src/operations.generated.ts b/integrations/dsh/plugins/powercontext/src/operations.generated.ts index 7a71802687..a69ed4865b 100644 --- a/integrations/dsh/plugins/powercontext/src/operations.generated.ts +++ b/integrations/dsh/plugins/powercontext/src/operations.generated.ts @@ -53,6 +53,7 @@ export const OPERATIONS = { continue_handoff: { method: 'POST', path: '/v1/handoff/continue', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, prepare_handoff_hint: { method: 'POST', path: '/v1/handoff/hint', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_topic_memory: { method: 'POST', path: '/v1/topic-memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + search_artifacts: { method: 'POST', path: '/v1/scopes/{scope_id}/artifacts/{family}/search', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'family'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_topic_memory: { method: 'POST', path: '/v1/topic-memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_topic_memory: { method: 'POST', path: '/v1/topic-memory/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_memory: { method: 'POST', path: '/v1/memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/integrations/opencode/plugins/powercontext/src/operations.generated.ts b/integrations/opencode/plugins/powercontext/src/operations.generated.ts index 7a71802687..a69ed4865b 100644 --- a/integrations/opencode/plugins/powercontext/src/operations.generated.ts +++ b/integrations/opencode/plugins/powercontext/src/operations.generated.ts @@ -53,6 +53,7 @@ export const OPERATIONS = { continue_handoff: { method: 'POST', path: '/v1/handoff/continue', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, prepare_handoff_hint: { method: 'POST', path: '/v1/handoff/hint', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_topic_memory: { method: 'POST', path: '/v1/topic-memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + search_artifacts: { method: 'POST', path: '/v1/scopes/{scope_id}/artifacts/{family}/search', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'family'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_topic_memory: { method: 'POST', path: '/v1/topic-memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_topic_memory: { method: 'POST', path: '/v1/topic-memory/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_memory: { method: 'POST', path: '/v1/memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/integrations/pi/plugins/powercontext/src/operations.generated.ts b/integrations/pi/plugins/powercontext/src/operations.generated.ts index 7a71802687..a69ed4865b 100644 --- a/integrations/pi/plugins/powercontext/src/operations.generated.ts +++ b/integrations/pi/plugins/powercontext/src/operations.generated.ts @@ -53,6 +53,7 @@ export const OPERATIONS = { continue_handoff: { method: 'POST', path: '/v1/handoff/continue', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, prepare_handoff_hint: { method: 'POST', path: '/v1/handoff/hint', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_topic_memory: { method: 'POST', path: '/v1/topic-memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + search_artifacts: { method: 'POST', path: '/v1/scopes/{scope_id}/artifacts/{family}/search', location: "body", scopeMode: 'none', pathParameters: ['scope_id', 'family'], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_topic_memory: { method: 'POST', path: '/v1/topic-memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_topic_memory: { method: 'POST', path: '/v1/topic-memory/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, flush_memory: { method: 'POST', path: '/v1/memory/flush', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/openapi/powercontext.yaml b/openapi/powercontext.yaml index d7b3f00cf1..e4d8deefef 100644 --- a/openapi/powercontext.yaml +++ b/openapi/powercontext.yaml @@ -1404,6 +1404,49 @@ paths: $ref: "#/components/responses/Unavailable" "500": $ref: "#/components/responses/InternalError" + /v1/scopes/{scope_id}/artifacts/{family}/search: + post: + tags: [artifacts] + summary: Search one Artifact Family in a Scope + description: >- + Search registered Artifact Families using their supported retrieval controls. Results contain complete exact + Artifact revisions. Score metadata is returned only when requested; min_score compares normalized retrieval scores. + operationId: search_artifacts + x-powercontext-access: {action: scope.read, resource: {type: scope, scope-id-from: scope_id}} + parameters: + - name: scope_id + in: path + required: true + schema: {type: string, minLength: 1, maxLength: 256, pattern: '.*\S.*'} + - name: family + in: path + required: true + schema: {type: string, minLength: 1} + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/SearchArtifactsRequest" + responses: + "200": + description: Complete matching Artifact revisions in retrieval order. + content: + application/json: + schema: + $ref: "#/components/schemas/SearchArtifactsResponse" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "422": + $ref: "#/components/responses/InvalidRequest" + "503": + $ref: "#/components/responses/Unavailable" + "500": + $ref: "#/components/responses/InternalError" /v1/topic-memory/search: post: tags: [topic-memory] @@ -9384,6 +9427,71 @@ components: uniqueItems: true items: $ref: "#/components/schemas/TopicMemoryMatchedBy" + ArtifactSearchFusion: + type: object + additionalProperties: false + x-powercontext-artifact-search-validation: fusion + required: [method] + properties: + method: {type: string, minLength: 1, pattern: '.*\S.*'} + params: {type: object, additionalProperties: true, default: {}} + SearchArtifactsRequest: + type: object + additionalProperties: false + x-powercontext-artifact-search-validation: request + required: [query] + properties: + query: {type: string, minLength: 1, maxLength: 8192, pattern: '.*\S.*'} + limit: {type: integer, minimum: 1, maximum: 200, default: 10} + mode: {type: string, minLength: 1, pattern: '.*\S.*'} + filters: {type: object, additionalProperties: true} + admission: {type: object, additionalProperties: true} + fusion: + $ref: "#/components/schemas/ArtifactSearchFusion" + min_score: {type: number, minimum: 0, maximum: 1} + include_scores: {type: boolean, default: false} + rerank: {type: object, additionalProperties: true} + ArtifactChannelScore: + type: object + additionalProperties: false + x-powercontext-artifact-search-validation: score + required: [raw, metric, higher_is_better] + properties: + raw: {type: number} + metric: {type: string, minLength: 1, pattern: '.*\S.*'} + higher_is_better: {type: boolean} + ArtifactSearchScores: + type: object + additionalProperties: false + x-powercontext-artifact-search-validation: score + required: [retrieval, channels] + properties: + retrieval: {type: number, minimum: 0, maximum: 1} + channels: + type: object + additionalProperties: + $ref: "#/components/schemas/ArtifactChannelScore" + ArtifactSearchItem: + type: object + additionalProperties: false + required: [family, artifact_id, revision, content, lineage] + properties: + family: {type: string, minLength: 1} + artifact_id: {type: string, minLength: 1, maxLength: 128} + revision: {type: integer, minimum: 1} + content: {type: object, additionalProperties: true} + lineage: {type: object, additionalProperties: true} + scores: + $ref: "#/components/schemas/ArtifactSearchScores" + SearchArtifactsResponse: + type: object + additionalProperties: false + required: [results] + properties: + results: + type: array + items: + $ref: "#/components/schemas/ArtifactSearchItem" SearchTopicMemoryRequest: type: object additionalProperties: false diff --git a/scripts/generate_api.py b/scripts/generate_api.py index c835526bb1..89c58f1166 100644 --- a/scripts/generate_api.py +++ b/scripts/generate_api.py @@ -145,9 +145,68 @@ def _generate_models( evidence_models = _candidate_evidence_models(transport_contract.components.schemas) source = _with_nested_model_defaults(f"{result.rstrip()}\n") source = _with_code_validation(source, transport_contract.components.schemas) + source = _with_artifact_search_validation(source, transport_contract.components.schemas) return _with_candidate_evidence_limits(source, evidence_models) +def _with_artifact_search_validation(source: str, schemas: dict[str, Schema | Reference]) -> str: + """Keep Artifact search's omission, trim, and finite-number semantics in generated models.""" + + for name, schema in schemas.items(): + if not isinstance(schema, Schema): + continue + kind = (schema.model_extra or {}).get("x-powercontext-artifact-search-validation") + if kind is None: + continue + if kind not in {"request", "fusion", "score"}: + raise ContractGenerationError("x-powercontext-artifact-search-validation", kind) + if " field_validator,\n" not in source: + source = source.replace("from pydantic import (\n", "from pydantic import (\n field_validator,\n", 1) + header = f"class {name}(BaseModel):" + start = source.find(header) + if start < 0: + raise ContractGenerationError("generated Artifact search model", name) # noqa: TRY003 + next_class = source.find("\nclass ", start + len(header)) + insert_at = next_class if next_class >= 0 else len(source.rstrip()) + validator = """ + @field_validator("*", mode="after") + @classmethod + def _reject_invalid_search_value(cls, value: Any) -> Any: + from math import isfinite + from pydantic import ValidationError + from pydantic_core import InitErrorDetails + + if value is None: + raise ValueError("omit the field instead of sending null") # noqa: TRY003 + errors: list[InitErrorDetails] = [] + pending: list[tuple[Any, tuple[str | int, ...]]] = [(value, ())] + visited: set[int] = set() + while pending: + item, path = pending.pop() + if isinstance(item, float) and not isfinite(item): + errors.append({"type": "finite_number", "loc": path, "input": item}) + elif isinstance(item, (dict, list, tuple)) and id(item) not in visited: + visited.add(id(item)) + entries = item.items() if isinstance(item, dict) else enumerate(item) + pending.extend((nested, (*path, key)) for key, nested in entries) + if errors: + raise ValidationError.from_exception_data(cls.__name__, errors) + return value +""" + if kind == "request": + validator = ( + """ + @field_validator("query", mode="before") + @classmethod + def _trim_search_query(cls, value: Any) -> Any: + return value.strip() if isinstance(value, str) else value +""" + + validator + ) + source = f"{source[:insert_at].rstrip()}\n{validator.rstrip()}\n\n{source[insert_at:].lstrip()}" + return source + + def _with_code_validation(source: str, schemas: dict[str, Schema | Reference]) -> str: """Preserve semantic constraints that OpenAPI 3.0 cannot express natively.""" validators = { diff --git a/src/powercontext/artifacts/__init__.py b/src/powercontext/artifacts/__init__.py index 59fef9d456..be5eb9b1d9 100644 --- a/src/powercontext/artifacts/__init__.py +++ b/src/powercontext/artifacts/__init__.py @@ -23,6 +23,17 @@ MemoryCitation, ) from powercontext.artifacts.protocols import ArtifactCatalog, ArtifactStore +from powercontext.artifacts.search import ( + ArtifactSearchContractError, + ArtifactSearcher, + ArtifactSearchExecutionContext, + ArtifactSearchFamilyNotFound, + ArtifactSearchMatch, + ArtifactSearchOutcome, + ArtifactSearchQuery, + ArtifactSearchUnsupported, + ChannelScore, +) __all__ = [ "Artifact", @@ -31,6 +42,15 @@ "ArtifactDraft", "ArtifactLineage", "ArtifactRef", + "ArtifactSearchContractError", + "ArtifactSearchExecutionContext", + "ArtifactSearchFamilyNotFound", + "ArtifactSearchMatch", + "ArtifactSearchOutcome", + "ArtifactSearchQuery", + "ArtifactSearchUnsupported", + "ArtifactSearcher", "ArtifactStore", + "ChannelScore", "MemoryCitation", ] diff --git a/src/powercontext/artifacts/fusion.py b/src/powercontext/artifacts/fusion.py new file mode 100644 index 0000000000..af5d73a326 --- /dev/null +++ b/src/powercontext/artifacts/fusion.py @@ -0,0 +1,228 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Family-independent reciprocal rank fusion with explicit enabled channels.""" + +from __future__ import annotations + +import json +import math +import sys +from collections.abc import Callable, Hashable, Iterable +from dataclasses import dataclass +from decimal import MAX_EMAX, MIN_EMIN, Decimal, localcontext +from fractions import Fraction +from numbers import Real +from typing import Annotated, Any, Generic, TypeVar + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, StrictInt + +KeyT = TypeVar("KeyT", bound=Hashable) + + +class FusionSelection(BaseModel): + """A method name and its JSON parameters, without algorithm-specific fields.""" + + model_config = ConfigDict(extra="forbid", strict=True, str_strip_whitespace=True, allow_inf_nan=False) + + method: str = Field(min_length=1) + params: dict[str, JsonValue] = Field(default_factory=dict) + + +class RrfParameters(BaseModel): + """RRF parameters; a Family applies partial weights to its enabled channels.""" + + model_config = ConfigDict(extra="forbid", strict=True) + + rank_constant: StrictInt = Field(default=60, ge=1) + weights: dict[str, Annotated[float, Field(ge=0, allow_inf_nan=False)]] = Field(default_factory=dict) + + +@dataclass(frozen=True) +class FusionCandidate(Generic[KeyT]): + """A hashable candidate identity at its original positive rank.""" + + key: KeyT + rank: int + + +@dataclass(frozen=True) +class FusionChannel(Generic[KeyT]): + """One enabled channel, including channels with no admitted candidates.""" + + name: str + weight: float + candidates: tuple[FusionCandidate[KeyT], ...] + + +@dataclass(frozen=True) +class FusionHit(Generic[KeyT]): + """A normalized score and raw RRF sum, ordered with the Family's tie break. + + Ordinary raw sums remain floats. A Decimal preserves positive raw values + when float accumulation would overflow, underflow, or produce a subnormal + result. This internal value is never a public channel score. + """ + + key: KeyT + score: float + raw_score: float | Decimal + + +def fuse_rrf( + channels: Iterable[FusionChannel[KeyT]], + params: RrfParameters, + *, + tie_break: Callable[[KeyT], Any], +) -> tuple[FusionHit[KeyT], ...]: + """Fuse admitted candidates, preserving enabled empties and original ranks.""" + + enabled = tuple(channels) + _validate_channels(enabled) + if type(params.rank_constant) is not int or params.rank_constant < 1: + raise ValueError("rank_constant must be a positive integer") # noqa: TRY003 + terms: dict[KeyT, list[tuple[float, int]]] = {} + for channel in enabled: + if channel.weight > 0: + for candidate in channel.candidates: + terms.setdefault(candidate.key, []).append((channel.weight, candidate.rank)) + if not terms: + return () + + upper = _raw_sum([(channel.weight, 1) for channel in enabled if channel.weight > 0], params.rank_constant) + exact_upper: Fraction | None = None + hits = [] + for key, contributions in terms.items(): + raw = _raw_sum(contributions, params.rank_constant) + if isinstance(raw, float) and isinstance(upper, float): + score = raw / upper + else: + # Round only the combined ratio, including at subnormal midpoints. + if exact_upper is None: + exact_upper = sum((Fraction.from_float(float(channel.weight)) for channel in enabled), Fraction(0)) / ( + params.rank_constant + 1 + ) + exact_raw = sum( + (Fraction.from_float(float(weight)) / (params.rank_constant + rank) for weight, rank in contributions), + Fraction(0), + ) + score = float(exact_raw / exact_upper) + hits.append(FusionHit(key, score, raw)) + return tuple( + sorted( + hits, + key=lambda hit: ( + hit.raw_score.copy_negate() if isinstance(hit.raw_score, Decimal) else -hit.raw_score, + tie_break(hit.key), + ), + ) + ) + + +def _validate_channels(channels: tuple[FusionChannel[KeyT], ...]) -> None: + names: set[str] = set() + positive_weight = False + for channel in channels: + if not isinstance(channel.name, str) or not channel.name.strip() or channel.name in names: + raise ValueError("fusion channel names must be non-empty and unique") # noqa: TRY003 + names.add(channel.name) + try: + valid_weight = ( + not isinstance(channel.weight, bool) + and isinstance(channel.weight, Real) + and math.isfinite(channel.weight) + and channel.weight >= 0 + ) + except OverflowError: + valid_weight = False + if not valid_weight: + raise ValueError("fusion channel weights must be finite nonnegative numbers") # noqa: TRY003 + positive_weight |= channel.weight > 0 + keys: set[KeyT] = set() + previous_rank = 0 + for candidate in channel.candidates: + if type(candidate.rank) is not int or candidate.rank <= previous_rank: + raise ValueError("fusion ranks must be positive, unique, and ordered") # noqa: TRY003 + previous_rank = candidate.rank + try: + duplicate = candidate.key in keys + keys.add(candidate.key) + except TypeError as exc: + raise ValueError("fusion candidate keys must be hashable") from exc # noqa: TRY003 + if duplicate: + raise ValueError("fusion candidate keys must be unique within a channel") # noqa: TRY003 + if not positive_weight: + raise ValueError("enabled fusion channels must have a positive total weight") # noqa: TRY003 + + +def _raw_sum(terms: list[tuple[float, int]], rank_constant: int) -> float | Decimal: + try: + # The legacy Topic score is based on sequential float additions. + raw = 0.0 + for weight, rank in terms: + raw += weight / (rank_constant + rank) + # Subnormal sums can lose rank differences before normalization. + if math.isfinite(raw) and raw >= sys.float_info.min: + return raw + except OverflowError: + pass + largest_denominator = max(rank_constant + rank for _, rank in terms) + with localcontext() as context: + # Enough digits to retain rank differences even when K is an enormous + # integer, rather than merging distinct candidates into false ties. + context.prec = max(34, largest_denominator.bit_length() * 30103 // 100000 + 34) + context.Emax = MAX_EMAX + context.Emin = MIN_EMIN + return sum( + (Decimal.from_float(float(weight)) / Decimal(rank_constant + rank) for weight, rank in terms), + Decimal(0), + ) + + +_FUSION_METHODS = {"rrf": (RrfParameters, fuse_rrf)} + + +def parse_fusion_parameters(selection: FusionSelection) -> RrfParameters: + """Strictly parse parameters using the selected method's own model.""" + + entry = _FUSION_METHODS.get(selection.method) + if entry is None: + raise ValueError(f"unsupported fusion method: {selection.method}") # noqa: TRY003 + model, _ = entry + return model.model_validate_json(json.dumps(selection.params, ensure_ascii=False, allow_nan=False), strict=True) + + +def fuse( + channels: Iterable[FusionChannel[KeyT]], + selection: FusionSelection, + *, + tie_break: Callable[[KeyT], Any], +) -> tuple[FusionHit[KeyT], ...]: + """Select the supported method and fuse already admitted Family candidates.""" + + params = parse_fusion_parameters(selection) + _, implementation = _FUSION_METHODS[selection.method] + return implementation(channels, params, tie_break=tie_break) + + +__all__ = [ + "FusionCandidate", + "FusionChannel", + "FusionHit", + "FusionSelection", + "RrfParameters", + "fuse", + "fuse_rrf", + "parse_fusion_parameters", +] diff --git a/src/powercontext/artifacts/search.py b/src/powercontext/artifacts/search.py new file mode 100644 index 0000000000..64ec275f77 --- /dev/null +++ b/src/powercontext/artifacts/search.py @@ -0,0 +1,185 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Caller-neutral Artifact search requests and Family-owned result contracts.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from numbers import Real +from typing import Any, Protocol, TypeVar + +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, field_validator + +from powercontext.artifacts.models import Artifact, ArtifactRef +from powercontext.errors import PowerContextError + + +class ArtifactSearchQuery(BaseModel): + """Common constraints inherited by each Family's public search request.""" + + model_config = ConfigDict(extra="forbid", strict=True) + + query: str = Field(min_length=1, max_length=8192) + limit: StrictInt = Field(default=10, ge=1, le=200) + include_scores: StrictBool = False + + @field_validator("query", mode="before") + @classmethod + def trim_query(cls, value: Any) -> Any: + return value.strip() if isinstance(value, str) else value + + @field_validator("*", mode="after") + @classmethod + def reject_explicit_null(cls, value: Any) -> Any: + if value is None: + raise ValueError("omit the field instead of sending null") # noqa: TRY003 + return value + + +@dataclass(frozen=True) +class ArtifactSearchExecutionContext: + """Trusted invocation metadata kept outside the caller's search payload. + + The host owns the principal, access service, and audit objects. A missing + context retains a Family's local SDK policy; HTTP supplies an explicit one. + """ + + principal: Any = None + access: Any = None + audit: Any = None + trusted_local: bool = False + + +@dataclass(frozen=True) +class ChannelScore: + """A collected channel value before normalization or direction changes.""" + + raw: float + metric: str + higher_is_better: bool + + def __post_init__(self) -> None: + _require_finite_real(self.raw, "channel raw score") + if not isinstance(self.metric, str) or not self.metric.strip(): + raise ValueError("channel metric must be a non-empty string") # noqa: TRY003 + if type(self.higher_is_better) is not bool: + raise ValueError("channel score direction must be a boolean") # noqa: TRY003 + + +@dataclass(frozen=True) +class ArtifactSearchMatch: + """One exact revision, its retrieval score, and optional collected metadata.""" + + artifact_ref: ArtifactRef + retrieval_score: float + channel_scores: Mapping[str, ChannelScore] | None = None + + def __post_init__(self) -> None: + if not isinstance(self.artifact_ref, ArtifactRef): + raise TypeError("search match requires an exact ArtifactRef") # noqa: TRY003 + ArtifactRef.model_validate(self.artifact_ref.model_dump(), strict=True) + _require_finite_real(self.retrieval_score, "retrieval score") + if not 0 <= self.retrieval_score <= 1: + raise ValueError("retrieval score must be between zero and one") # noqa: TRY003 + if self.channel_scores is not None: + if not isinstance(self.channel_scores, Mapping): + raise ValueError("channel scores must be a mapping") # noqa: TRY003 + for name, value in self.channel_scores.items(): + if not isinstance(name, str) or not name.strip() or not isinstance(value, ChannelScore): + raise ValueError("channel scores require named ChannelScore values") # noqa: TRY003 + ChannelScore(value.raw, value.metric, value.higher_is_better) + + +class ArtifactSearchOutcome(Protocol): + """A Family outcome retaining its final order and actual Artifact values.""" + + @property + def matches(self) -> Sequence[ArtifactSearchMatch]: ... + + @property + def artifacts(self) -> Sequence[Artifact[Any]] | None: ... + + +RequestT = TypeVar("RequestT", bound=ArtifactSearchQuery) +OutcomeT_co = TypeVar("OutcomeT_co", bound=ArtifactSearchOutcome, covariant=True) + + +class ArtifactSearcher(Protocol[RequestT, OutcomeT_co]): + """A Family's request model and callable bound by one registration.""" + + @property + def family(self) -> str: ... + + @property + def request_type(self) -> type[RequestT]: ... + + async def search( + self, + scope_id: str, + request: RequestT, + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> OutcomeT_co: ... + + +class ArtifactSearchFamilyNotFound(PowerContextError, LookupError): + """The Artifact repository does not own the requested Family.""" + + def __init__(self, family: str) -> None: + self.family = family + super().__init__(f"Artifact Family was not found: {family}") + + +class ArtifactSearchUnsupported(PowerContextError, ValueError): + """A known Family cannot search, or cannot use a requested search field.""" + + def __init__(self, family: str, *, field: str | None = None) -> None: + self.family = family + self.field = field + super().__init__(f"Artifact search is unsupported for {family}" + (f": {field}" if field else "")) + + +class ArtifactSearchContractError(PowerContextError, RuntimeError): + """A search implementation returned values that violate the public contract.""" + + def __init__(self, family: str, detail: str) -> None: + self.family = family + self.detail = detail + super().__init__(f"Artifact search result contract failed for {family}: {detail}") + + +def _require_finite_real(value: object, label: str) -> None: + try: + finite = not isinstance(value, bool) and isinstance(value, Real) and math.isfinite(value) + except OverflowError: + finite = False + if not finite: + raise ValueError(f"{label} must be a finite real number") # noqa: TRY003 + + +__all__ = [ + "ArtifactSearchContractError", + "ArtifactSearchExecutionContext", + "ArtifactSearchFamilyNotFound", + "ArtifactSearchMatch", + "ArtifactSearchOutcome", + "ArtifactSearchQuery", + "ArtifactSearchUnsupported", + "ArtifactSearcher", + "ChannelScore", +] diff --git a/src/powercontext/builtin/artifacts/experience/__init__.py b/src/powercontext/builtin/artifacts/experience/__init__.py index 280eaf8c5b..bcc48abe20 100644 --- a/src/powercontext/builtin/artifacts/experience/__init__.py +++ b/src/powercontext/builtin/artifacts/experience/__init__.py @@ -55,6 +55,7 @@ from powercontext.builtin.artifacts.experience.search import ( ExperienceSearchHit, ExperienceSearchOutcome, + ExperienceSearchRequest, experience_search_text, experience_searchable_text, render_experience, @@ -87,6 +88,7 @@ "ExperienceIncubationOutput", "ExperienceSearchHit", "ExperienceSearchOutcome", + "ExperienceSearchRequest", "FailureRecord", "FailureSignature", "FailureVerification", diff --git a/src/powercontext/builtin/artifacts/experience/search.py b/src/powercontext/builtin/artifacts/experience/search.py index 719c4ee2a5..53a4673005 100644 --- a/src/powercontext/builtin/artifacts/experience/search.py +++ b/src/powercontext/builtin/artifacts/experience/search.py @@ -17,12 +17,23 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Literal -from pydantic import BaseModel +from pydantic import BaseModel, Field from powercontext.artifacts import ArtifactRef -from powercontext.builtin.artifacts.experience.models import ExperienceContent -from powercontext.builtin.artifacts.search import AdmissionCounts, analyze_text +from powercontext.artifacts.search import ArtifactSearchMatch, ArtifactSearchQuery, ChannelScore +from powercontext.builtin.artifacts.experience.models import Experience, ExperienceContent +from powercontext.builtin.artifacts.search import AdmissionCounts, EmptyFilters, LexicalSearchAdmission, analyze_text + + +class ExperienceSearchRequest(ArtifactSearchQuery): + """Public text search controls for approved, active Experience heads.""" + + mode: Literal["text"] | None = None + filters: EmptyFilters = Field(default_factory=EmptyFilters) + admission: LexicalSearchAdmission = Field(default_factory=LexicalSearchAdmission) + min_score: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False) @dataclass(frozen=True) @@ -40,6 +51,8 @@ class ExperienceSearchOutcome: hits: tuple[ExperienceSearchHit, ...] = () admission: AdmissionCounts | None = None + matches: tuple[ArtifactSearchMatch, ...] = () + artifacts: tuple[Experience, ...] | None = None class ExperienceSearchHit(BaseModel): @@ -47,6 +60,8 @@ class ExperienceSearchHit(BaseModel): artifact_ref: ArtifactRef content: ExperienceContent + retrieval_score: float | None = None + channel_scores: dict[str, ChannelScore] | None = None def render_experience(content: ExperienceContent, /) -> str: @@ -88,6 +103,7 @@ def experience_search_text(content: ExperienceContent, /) -> str: __all__ = [ "ExperienceSearchHit", "ExperienceSearchOutcome", + "ExperienceSearchRequest", "experience_search_text", "experience_searchable_text", "render_experience", diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index a2a35d3313..fb158749c2 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -22,6 +22,10 @@ from dataclasses import dataclass from itertools import pairwise +from pydantic import BaseModel, ConfigDict, Field, StrictInt, field_validator + +from powercontext.artifacts.search import ChannelScore + _FTS_MIN_QUERY_COVERAGE = 0.25 _FTS_MIN_MATCHED_TERMS = 2 _FTS_SHORT_QUERY_MAX_TERMS = 2 @@ -117,6 +121,67 @@ class AdmissionFloor: DEFAULT_ADMISSION_FLOOR = AdmissionFloor() +class EmptyFilters(BaseModel): + """An explicitly empty filter object for families without filter fields.""" + + model_config = ConfigDict(extra="forbid", strict=True) + + +class LexicalSearchAdmission(BaseModel): + """Strict lexical admission controls for text-only Artifact searches.""" + + model_config = ConfigDict(extra="forbid", strict=True) + + lexical_coverage: float = Field(default=_FTS_MIN_QUERY_COVERAGE, ge=0, le=1, allow_inf_nan=False) + lexical_min_matched_terms: StrictInt = Field(default=_FTS_MIN_MATCHED_TERMS, ge=1) + + @field_validator("*", mode="before") + @classmethod + def reject_explicit_null(cls, value: object) -> object: + if value is None: + raise ValueError("omit the field instead of sending null") # noqa: TRY003 + return value + + def as_floor(self) -> AdmissionFloor: + """Adapt the request controls to the existing lexical admission contract.""" + + return AdmissionFloor( + lexical_coverage=self.lexical_coverage, + lexical_min_matched_terms=self.lexical_min_matched_terms, + ) + + +class InvalidSearchScore(RuntimeError): + """The backend returned an unavailable or invalid lexical relevance value.""" + + +class UnsupportedScoreMetric(RuntimeError): + """The backend did not identify a supported lexical score metric.""" + + +def lexical_search_score(raw: object, metric: object, /) -> tuple[float, ChannelScore]: + """Normalize an explicit backend metric while preserving its true raw score.""" + + if not isinstance(metric, str) or metric not in ("sqlite_bm25", "oceanbase_match"): + raise UnsupportedScoreMetric(f"unsupported lexical score metric: {metric!r}") # noqa: TRY003 + if isinstance(raw, bool) or not isinstance(raw, int | float): + raise InvalidSearchScore("lexical raw score must be a finite number") # noqa: TRY003 + try: + raw_score = float(raw) + except OverflowError as error: + raise InvalidSearchScore("lexical raw score must be finite") from error # noqa: TRY003 + relevance = -raw_score if metric == "sqlite_bm25" else raw_score + if not math.isfinite(relevance) or relevance < 0: + raise InvalidSearchScore("lexical relevance must be finite and nonnegative") # noqa: TRY003 + # This form avoids overflow in 1 + relevance for the largest finite values. + normalized = relevance / (1 + relevance) if relevance <= 1 else 1 / (1 + 1 / relevance) + return normalized, ChannelScore( + raw=raw_score, + metric=metric, + higher_is_better=metric == "oceanbase_match", + ) + + @dataclass(frozen=True) class AdmissionCounts: """Per-family, per-scope admission accounting for one search. @@ -294,10 +359,15 @@ def _is_cjk(character: str) -> bool: "DEFAULT_ADMISSION_FLOOR", "AdmissionCounts", "AdmissionFloor", + "EmptyFilters", + "InvalidSearchScore", + "LexicalSearchAdmission", + "UnsupportedScoreMetric", "admits_fts_text", "analyze_fts_query", "analyze_text", "analyze_text_with_spans", "fts_match_query", "fts_query_requirements", + "lexical_search_score", ] diff --git a/src/powercontext/builtin/artifacts/skill/__init__.py b/src/powercontext/builtin/artifacts/skill/__init__.py index 5816e862df..7f43be53ec 100644 --- a/src/powercontext/builtin/artifacts/skill/__init__.py +++ b/src/powercontext/builtin/artifacts/skill/__init__.py @@ -89,7 +89,13 @@ SKILL_GENERATION_INSTRUCTIONS_VERSION, ) from powercontext.builtin.artifacts.skill.provenance import SkillOrigin, SkillOriginKind -from powercontext.builtin.artifacts.skill.search import SkillSearchHit, skill_search_text, skill_searchable_text +from powercontext.builtin.artifacts.skill.search import ( + SkillSearchHit, + SkillSearchOutcome, + SkillSearchRequest, + skill_search_text, + skill_searchable_text, +) __all__ = [ "MAX_EXTERNAL_SKILL_DESCRIPTION_LENGTH", @@ -150,6 +156,8 @@ "SkillRuntimeRequirements", "SkillRuntimeVariant", "SkillSearchHit", + "SkillSearchOutcome", + "SkillSearchRequest", "assess_skill_compatibility", "build_instruction_skill_package", "capture_skill_archive", diff --git a/src/powercontext/builtin/artifacts/skill/search.py b/src/powercontext/builtin/artifacts/skill/search.py index ba18d2dff1..dd3d05b1b3 100644 --- a/src/powercontext/builtin/artifacts/skill/search.py +++ b/src/powercontext/builtin/artifacts/skill/search.py @@ -16,21 +16,46 @@ from __future__ import annotations -from pydantic import BaseModel +from dataclasses import dataclass +from typing import Literal + +from pydantic import BaseModel, Field from powercontext.artifacts import ArtifactRef -from powercontext.builtin.artifacts.search import analyze_text -from powercontext.builtin.artifacts.skill.models import SkillContent +from powercontext.artifacts.search import ArtifactSearchMatch, ArtifactSearchQuery, ChannelScore +from powercontext.builtin.artifacts.search import EmptyFilters, LexicalSearchAdmission, analyze_text +from powercontext.builtin.artifacts.skill.models import Skill, SkillContent from powercontext.builtin.artifacts.skill.package import SkillPackageSnapshot, package_file _MAX_INDEXED_FILE_BYTES = 128 * 1024 +class SkillSearchRequest(ArtifactSearchQuery): + """Public text search controls for approved, active managed Skill heads.""" + + query: str = Field(min_length=1, max_length=2000) + mode: Literal["text"] | None = None + filters: EmptyFilters = Field(default_factory=EmptyFilters) + admission: LexicalSearchAdmission = Field(default_factory=LexicalSearchAdmission) + min_score: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False) + + class SkillSearchHit(BaseModel): """One relevant approved, active managed Skill head.""" artifact_ref: ArtifactRef content: SkillContent + retrieval_score: float | None = None + channel_scores: dict[str, ChannelScore] | None = None + + +@dataclass(frozen=True) +class SkillSearchOutcome: + """Managed Skill hits and their optional exact Artifact materialization.""" + + hits: tuple[SkillSearchHit, ...] = () + matches: tuple[ArtifactSearchMatch, ...] = () + artifacts: tuple[Skill, ...] | None = None def skill_search_text(content: SkillContent, package: SkillPackageSnapshot | None = None, /) -> str: @@ -71,4 +96,4 @@ def _is_indexed_text(path: str) -> bool: return path == "SKILL.md" or (path.startswith("references/") and path.endswith((".md", ".txt"))) -__all__ = ["SkillSearchHit", "skill_search_text", "skill_searchable_text"] +__all__ = ["SkillSearchHit", "SkillSearchOutcome", "SkillSearchRequest", "skill_search_text", "skill_searchable_text"] diff --git a/src/powercontext/builtin/artifacts/topic_memory/__init__.py b/src/powercontext/builtin/artifacts/topic_memory/__init__.py index 19ddeb7c18..7461538d1c 100644 --- a/src/powercontext/builtin/artifacts/topic_memory/__init__.py +++ b/src/powercontext/builtin/artifacts/topic_memory/__init__.py @@ -40,6 +40,7 @@ MAX_TOPIC_MEMORY_SUMMARY_LENGTH, MAX_TOPIC_MEMORY_TITLE_LENGTH, PublishedTopicMemory, + TopicArtifactSearchRequest, TopicMemory, TopicMemoryBrowseCursor, TopicMemoryCapabilities, @@ -56,6 +57,7 @@ TopicMemorySearchRequest, TopicMemorySearchResult, TopicMemoryUsedSearchMode, + TopicSearchAdmission, ) TOPIC_MEMORY_SOURCE_WINDOW_BINDING = "topic-memory-source-window" @@ -76,6 +78,7 @@ "TOPIC_MEMORY_CHUNK_TARGET_CHARACTERS", "TOPIC_MEMORY_SOURCE_WINDOW_BINDING", "PublishedTopicMemory", + "TopicArtifactSearchRequest", "TopicMemory", "TopicMemoryBrowseCursor", "TopicMemoryCapabilities", @@ -96,6 +99,7 @@ "TopicMemorySearchResult", "TopicMemoryStorageInvariantError", "TopicMemoryUsedSearchMode", + "TopicSearchAdmission", "chunk_topic_memory_detail", "fuse_topic_memory_rankings", "prepare_topic_memory_projection", diff --git a/src/powercontext/builtin/artifacts/topic_memory/fusion.py b/src/powercontext/builtin/artifacts/topic_memory/fusion.py index 7beb583279..dd19e58be7 100644 --- a/src/powercontext/builtin/artifacts/topic_memory/fusion.py +++ b/src/powercontext/builtin/artifacts/topic_memory/fusion.py @@ -20,7 +20,16 @@ from collections import Counter from collections.abc import Sequence -from powercontext.builtin.artifacts.search import AdmissionFloor, admits_fts_text, analyze_text, analyze_text_with_spans +from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf +from powercontext.artifacts.search import ChannelScore +from powercontext.builtin.artifacts.search import ( + AdmissionFloor, + InvalidSearchScore, + admits_fts_text, + analyze_text, + analyze_text_with_spans, + lexical_search_score, +) from powercontext.builtin.artifacts.topic_memory.models import ( TopicMemoryChannelHit, TopicMemoryFusionOutcome, @@ -29,21 +38,12 @@ TopicMemorySearchHit, TopicMemoryUsedSearchMode, ) +from powercontext.builtin.artifacts.topic_memory.search import TOPIC_MODE_CHANNELS, validate_topic_weights -_RRF_CONSTANT = 60 _MIN_SEMANTIC_SIMILARITY = 0.3 _SNIPPET_MAX_CHARACTERS = 480 -_CHANNEL_ORDER: tuple[TopicMemoryMatchedBy, ...] = ( - "topic_fts", - "topic_vector", - "detail_fts", - "detail_vector", -) -_MODE_CHANNELS: dict[TopicMemoryUsedSearchMode, tuple[TopicMemoryMatchedBy, ...]] = { - "fts": ("topic_fts", "detail_fts"), - "vector": ("topic_vector", "detail_vector"), - "hybrid": _CHANNEL_ORDER, -} +_CHANNEL_ORDER = TOPIC_MODE_CHANNELS["hybrid"] +_MODE_CHANNELS = TOPIC_MODE_CHANNELS def fuse_topic_memory_rankings( @@ -71,18 +71,14 @@ def _fuse_topic_memory_rankings( *, mode: TopicMemoryUsedSearchMode = "hybrid", admission: AdmissionFloor | None = None, + fusion: RrfParameters | None = None, + min_score: float | None = None, + include_scores: bool = False, ) -> TopicMemoryFusionOutcome: - """Return fused hits plus internal admission accounting. - - The returned outcome also reports how many channel hits the resolved mode actually - retrieved and how many survived admission, measured around the two admit helpers so the - numbers cannot drift from the admission rule itself. - """ + """Admit per channel, fuse exact identities, and retain rich Topic evidence.""" - candidates: dict[tuple[str, int], TopicMemoryChannelHit] = {} - snippets: dict[tuple[str, int], str] = {} - scores: dict[tuple[str, int], float] = {} - matched: dict[tuple[str, int], set[TopicMemoryMatchedBy]] = {} + params = fusion if fusion is not None else RrfParameters() + validate_topic_weights(mode, params) rankings: tuple[tuple[TopicMemoryMatchedBy, tuple[TopicMemoryChannelHit, ...]], ...] = ( ("topic_fts", _admit_fts(query, channels.topic_fts, admission)), ("topic_vector", _admit_vector(channels.topic_vector, admission)), @@ -105,37 +101,41 @@ def _fuse_topic_memory_rankings( for (channel, raw_hits), (_same_channel, admitted_hits) in zip(channel_inputs, rankings, strict=True) if channel in enabled_channels ) - max_score = len(enabled_channels) / (_RRF_CONSTANT + 1) + candidates: dict[tuple[str, int], TopicMemoryChannelHit] = {} + snippets: dict[tuple[str, int], str] = {} + evidence: dict[tuple[str, int], dict[str, TopicMemoryChannelHit]] = {} + fusion_channels: list[FusionChannel[tuple[str, int]]] = [] for channel, ranking in rankings: if channel not in enabled_channels: continue seen: set[tuple[str, int]] = set() + ranked_candidates = [] for rank, candidate in enumerate(ranking, start=1): identity = (candidate.artifact_ref.artifact_id, candidate.artifact_ref.revision) if identity in seen: continue seen.add(identity) + ranked_candidates.append(FusionCandidate(identity, rank)) candidates.setdefault(identity, candidate) if candidate.chunk_text is not None: snippets.setdefault(identity, _snippet(query, candidate.chunk_text, lexical=channel == "detail_fts")) - scores[identity] = scores.get(identity, 0.0) + 1.0 / (_RRF_CONSTANT + rank) - matched.setdefault(identity, set()).add(channel) - - ordered = sorted( - candidates, - key=lambda identity: (-scores[identity], identity[0].encode(), -identity[1]), - )[:limit] + evidence.setdefault(identity, {})[channel] = candidate + fusion_channels.append(FusionChannel(channel, params.weights.get(channel, 1.0), tuple(ranked_candidates))) + fused = fuse_rrf(fusion_channels, params, tie_break=lambda identity: (identity[0].encode(), -identity[1])) + selected = tuple(hit for hit in fused if min_score is None or hit.score >= min_score)[:limit] return TopicMemoryFusionOutcome( hits=tuple( TopicMemorySearchHit( - artifact_ref=candidates[identity].artifact_ref, - title=candidates[identity].title, - summary=candidates[identity].summary, - snippet=snippets.get(identity), - score=min(100.0, scores[identity] / max_score * 100.0), - matched_by=tuple(channel for channel in _CHANNEL_ORDER if channel in matched[identity]), + artifact_ref=candidates[hit.key].artifact_ref, + title=candidates[hit.key].title, + summary=candidates[hit.key].summary, + snippet=snippets.get(hit.key), + score=min(100.0, hit.score * 100.0), + matched_by=tuple(channel for channel in _CHANNEL_ORDER if channel in evidence[hit.key]), + retrieval_score=hit.score, + channel_scores=_channel_scores(evidence[hit.key]) if include_scores else None, ) - for identity in ordered + for hit in selected ), retrieved=retrieved, admitted=admitted, @@ -143,6 +143,18 @@ def _fuse_topic_memory_rankings( ) +def _channel_scores(evidence: dict[str, TopicMemoryChannelHit]) -> dict[str, ChannelScore]: + result = {} + for name, hit in evidence.items(): + if name.endswith("_vector"): + if hit.distance is None: + raise InvalidSearchScore("admitted vector hit has no raw distance") # noqa: TRY003 + result[name] = ChannelScore(hit.distance, "l2_distance", False) + else: + _, result[name] = lexical_search_score(hit.raw_score, hit.metric) + return result + + def _retrieved_count( channels: TopicMemorySearchChannels, channel: TopicMemoryMatchedBy, diff --git a/src/powercontext/builtin/artifacts/topic_memory/models.py b/src/powercontext/builtin/artifacts/topic_memory/models.py index eefb461313..49496525aa 100644 --- a/src/powercontext/builtin/artifacts/topic_memory/models.py +++ b/src/powercontext/builtin/artifacts/topic_memory/models.py @@ -21,10 +21,13 @@ from typing import Annotated, ClassVar, Literal, TypeAlias from pydantic import BaseModel, Field, StrictInt, field_validator, model_validator +from typing_extensions import override from powercontext.artifacts import Artifact, ArtifactDraft, ArtifactRef +from powercontext.artifacts.fusion import FusionSelection +from powercontext.artifacts.search import ArtifactSearchMatch, ArtifactSearchQuery, ChannelScore from powercontext.builtin.artifacts.memory import EmbeddingProfile, MemoryQueryEmbedding -from powercontext.builtin.artifacts.search import AdmissionCounts, AdmissionFloor +from powercontext.builtin.artifacts.search import AdmissionCounts, AdmissionFloor, EmptyFilters, LexicalSearchAdmission from powercontext.builtin.inference import EmbeddingVector MAX_TOPIC_MEMORY_TITLE_LENGTH = 512 @@ -72,6 +75,31 @@ class TopicMemoryDraft(ArtifactDraft[TopicMemoryContent]): TopicMemoryMatchedBy: TypeAlias = Literal["topic_fts", "topic_vector", "detail_fts", "detail_vector"] +class TopicSearchAdmission(LexicalSearchAdmission): + """Topic-local lexical and semantic admission thresholds.""" + + min_semantic_similarity: float = Field(default=0.3, ge=-1, le=1, allow_inf_nan=False) + + @override + def as_floor(self) -> AdmissionFloor: + return AdmissionFloor( + lexical_coverage=self.lexical_coverage, + lexical_min_matched_terms=self.lexical_min_matched_terms, + min_semantic_similarity=self.min_semantic_similarity, + ) + + +class TopicArtifactSearchRequest(ArtifactSearchQuery): + """Strict public Topic search controls, independent of execution metadata.""" + + limit: StrictInt = Field(default=10, ge=1, le=MAX_TOPIC_MEMORY_SEARCH_LIMIT) + mode: Literal["text", "vector", "hybrid"] | None = None + filters: EmptyFilters = Field(default_factory=EmptyFilters) + admission: TopicSearchAdmission = Field(default_factory=TopicSearchAdmission) + fusion: FusionSelection | None = None + min_score: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False) + + class TopicMemoryChunk(BaseModel): """One rebuildable Markdown-aware search chunk without public identity.""" @@ -155,6 +183,8 @@ class TopicMemoryChannelHit(BaseModel): chunk_start: StrictInt | None = Field(default=None, ge=0) chunk_text: str | None = None distance: float | None = Field(default=None, ge=0.0, allow_inf_nan=False) + raw_score: float | None = Field(default=None, allow_inf_nan=False) + metric: str | None = None class TopicMemorySearchChannels(BaseModel): @@ -181,8 +211,10 @@ class TopicMemorySearchHit(BaseModel): title: str summary: str snippet: str | None = None - score: float = Field(gt=0.0, allow_inf_nan=False) + score: float = Field(ge=0.0, allow_inf_nan=False) matched_by: tuple[TopicMemoryMatchedBy, ...] + retrieval_score: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, exclude=True) + channel_scores: dict[str, ChannelScore] | None = Field(default=None, exclude=True) @dataclass(frozen=True) @@ -214,6 +246,8 @@ class TopicMemorySearchResult(BaseModel): admission: AdmissionCounts | None = Field(default=None, exclude=True) query_embedding: MemoryQueryEmbedding | None = Field(default=None, exclude=True) embedding_calls: int = Field(default=0, exclude=True) + matches: tuple[ArtifactSearchMatch, ...] = Field(default=(), exclude=True) + artifacts: tuple[TopicMemory, ...] | None = Field(default=None, exclude=True) class PublishedTopicMemory(BaseModel): diff --git a/src/powercontext/builtin/artifacts/topic_memory/search.py b/src/powercontext/builtin/artifacts/topic_memory/search.py new file mode 100644 index 0000000000..31c6efff50 --- /dev/null +++ b/src/powercontext/builtin/artifacts/topic_memory/search.py @@ -0,0 +1,134 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Topic-owned public mode, capability, and fusion parameter policy.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from pydantic import ValidationError +from pydantic_core import InitErrorDetails + +from powercontext.artifacts.fusion import FusionSelection, RrfParameters, parse_fusion_parameters +from powercontext.builtin.artifacts.topic_memory.models import ( + TopicArtifactSearchRequest, + TopicMemoryCapabilities, + TopicMemoryMatchedBy, + TopicMemoryUsedSearchMode, +) + +TOPIC_MODE_CHANNELS: dict[TopicMemoryUsedSearchMode, tuple[TopicMemoryMatchedBy, ...]] = { + "fts": ("topic_fts", "detail_fts"), + "vector": ("topic_vector", "detail_vector"), + "hybrid": ("topic_fts", "topic_vector", "detail_fts", "detail_vector"), +} + + +def topic_search_error(path: tuple[str | int, ...], message: str) -> ValidationError: + return ValidationError.from_exception_data( + "TopicArtifactSearchRequest", + [{"type": "value_error", "loc": path, "input": None, "ctx": {"error": ValueError(message)}}], + ) + + +def topic_fusion_parameters(selection: FusionSelection | None) -> RrfParameters: + if selection is None: + return RrfParameters() + if selection.method != "rrf": + raise topic_search_error(("fusion", "method"), "supported fusion method is 'rrf'") + try: + return parse_fusion_parameters(selection) + except ValidationError as exc: + errors: list[InitErrorDetails] = [] + for error in exc.errors(include_url=False): + detail: InitErrorDetails = { + "type": error["type"], + "loc": ("fusion", "params", *error["loc"]), + "input": error["input"], + } + if "ctx" in error: + detail["ctx"] = error["ctx"] + errors.append(detail) + raise ValidationError.from_exception_data("TopicArtifactSearchRequest", errors) from None + + +def validate_topic_weights(mode: TopicMemoryUsedSearchMode, params: RrfParameters) -> None: + enabled = TOPIC_MODE_CHANNELS[mode] + for name in params.weights: + if name not in enabled: + raise topic_search_error( + ("fusion", "params", "weights", name), f"enabled channels are {', '.join(enabled)}" + ) + if not any(params.weights.get(name, 1.0) > 0 for name in enabled): + raise topic_search_error(("fusion", "params", "weights"), "enabled channels must have positive total weight") + + +@dataclass(frozen=True) +class TopicSearchPlan: + mode: TopicMemoryUsedSearchMode + fusion: RrfParameters + requires_vector: bool + fallback_allowed: bool + + +def plan_topic_search( + request: TopicArtifactSearchRequest, + capabilities: TopicMemoryCapabilities, + *, + embedding_available: bool, +) -> TopicSearchPlan: + params = topic_fusion_parameters(request.fusion) + for name in params.weights: + if name not in TOPIC_MODE_CHANNELS["hybrid"]: + raise topic_search_error( + ("fusion", "params", "weights", name), + "supported channels are topic_fts, topic_vector, detail_fts, detail_vector", + ) + semantic_explicit = "min_semantic_similarity" in request.admission.model_fields_set + vector_weight_explicit = any(name.endswith("_vector") for name in params.weights) + lexical_explicit = bool(request.admission.model_fields_set & {"lexical_coverage", "lexical_min_matched_terms"}) + if request.mode == "text" and semantic_explicit: + raise topic_search_error( + ("admission", "min_semantic_similarity"), "semantic admission requires vector or hybrid mode" + ) + if request.mode == "vector" and lexical_explicit: + field = next( + name + for name in ("lexical_coverage", "lexical_min_matched_terms") + if name in request.admission.model_fields_set + ) + raise topic_search_error(("admission", field), "lexical admission requires text or hybrid mode") + requires_vector = request.mode in {"vector", "hybrid"} or semantic_explicit or vector_weight_explicit + mode: TopicMemoryUsedSearchMode = ( + "fts" if request.mode == "text" else request.mode or ("hybrid" if embedding_available else "fts") + ) + if request.mode is not None: + validate_topic_weights(mode, params) + if requires_vector and (not capabilities.vector or not embedding_available): + raise topic_search_error( + ("mode",), "vector and hybrid searches require configured vector indexes and an embedding model" + ) + if mode in {"fts", "hybrid"} and not capabilities.fts: + raise topic_search_error(("mode",), "text search is unavailable in this deployment") + if mode == "hybrid" and not capabilities.hybrid: + raise topic_search_error(("mode",), "hybrid search is unavailable in this deployment") + validate_topic_weights(mode, params) + fallback_allowed = ( + request.mode is None + and not requires_vector + and capabilities.fts + and any(params.weights.get(name, 1.0) > 0 for name in TOPIC_MODE_CHANNELS["fts"]) + ) + return TopicSearchPlan(mode, params, requires_vector, fallback_allowed) diff --git a/src/powercontext/builtin/persistence/database.py b/src/powercontext/builtin/persistence/database.py index 04fba4dff4..74f078e554 100644 --- a/src/powercontext/builtin/persistence/database.py +++ b/src/powercontext/builtin/persistence/database.py @@ -25,7 +25,7 @@ from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine -from powercontext.builtin.persistence.errors import DatabaseClosedError +from powercontext.builtin.persistence.errors import DatabaseClosedError, PersistenceError # Repositories that read a whole Scope selection in one statement chunk it to stay # below the lowest bind-parameter ceiling across the supported backends. @@ -98,14 +98,23 @@ def engine(self) -> AsyncEngine: return self._engine @asynccontextmanager - async def transaction(self) -> AsyncIterator[AsyncConnection]: - """Yield a connection in a transaction owned by the calling use case.""" + async def transaction(self, *, consistent_snapshot: bool = False) -> AsyncIterator[AsyncConnection]: + """Yield an owned transaction, optionally pinning its short read snapshot. + + MySQL snapshot isolation applies only to this transaction. Nested + in-memory SQLite reads retain their caller's transaction and writes. + An already borrowed MySQL transaction cannot be upgraded in place. + """ owner = asyncio.current_task() if self._shared_connection is not None and self._transaction_owner is owner: # Nested lookups on a single-connection profile must join their # caller's transaction, not acquire or commit that connection again. - yield self._shared_connection + connection = self._shared_connection + if consistent_snapshot and connection.dialect.name != "sqlite": + raise PersistenceError("cannot establish a snapshot inside an existing transaction") # noqa: TRY003 + async with connection.begin_nested() if consistent_snapshot else nullcontext(): + yield connection return async with self._state_changed: if self._closed or self._closing: @@ -117,12 +126,28 @@ async def transaction(self) -> AsyncIterator[AsyncConnection]: if connection.dialect.name == "mysql": # The MySQL dialect's begin hook is a no-op. Explicitly start # the owned transaction even when the server session uses autocommit. - await connection.exec_driver_sql("START TRANSACTION") + if consistent_snapshot: + try: + await connection.exec_driver_sql("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + await connection.exec_driver_sql("START TRANSACTION WITH CONSISTENT SNAPSHOT") + except BaseException: + # SET may have succeeded before START failed or was + # cancelled. Do not pool its unconsumed next-transaction setting. + await connection.invalidate() + raise + else: + await connection.exec_driver_sql("START TRANSACTION") if self._shared_connection_lock is not None: self._transaction_owner = owner self._shared_connection = connection try: - yield connection + snapshot = ( + connection.begin_nested() + if consistent_snapshot and connection.dialect.name == "sqlite" + else nullcontext() + ) + async with snapshot: + yield connection finally: self._transaction_owner = None self._shared_connection = None diff --git a/src/powercontext/builtin/persistence/experience_index.py b/src/powercontext/builtin/persistence/experience_index.py index b5fcfa748f..76e0c6da84 100644 --- a/src/powercontext/builtin/persistence/experience_index.py +++ b/src/powercontext/builtin/persistence/experience_index.py @@ -23,6 +23,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.artifacts import ArtifactRef +from powercontext.artifacts.search import ChannelScore from powercontext.builtin.artifacts.experience import ( Experience, ExperienceContent, @@ -31,7 +32,13 @@ experience_search_text, experience_searchable_text, ) -from powercontext.builtin.artifacts.search import AdmissionCounts, AdmissionFloor, admits_fts_text +from powercontext.builtin.artifacts.search import ( + AdmissionCounts, + AdmissionFloor, + InvalidSearchScore, + admits_fts_text, + lexical_search_score, +) from powercontext.builtin.artifacts.skill import ( Skill, SkillContent, @@ -95,6 +102,8 @@ async def search( /, *, admission: AdmissionFloor | None = None, + min_score: float | None = None, + require_scores: bool = False, ) -> ExperienceSearchOutcome: ... async def replace_skill( @@ -113,6 +122,10 @@ async def search_skills( query: str, limit: int, /, + *, + admission: AdmissionFloor | None = None, + min_score: float | None = None, + require_scores: bool = False, ) -> tuple[SkillSearchHit, ...]: ... @@ -140,6 +153,8 @@ async def search( /, *, admission: AdmissionFloor | None = None, + min_score: float | None = None, + require_scores: bool = False, ) -> ExperienceSearchOutcome: return ExperienceSearchOutcome() @@ -160,6 +175,10 @@ async def search_skills( _query: str, _limit: int, /, + *, + admission: AdmissionFloor | None = None, + min_score: float | None = None, + require_scores: bool = False, ) -> tuple[SkillSearchHit, ...]: return () @@ -308,25 +327,32 @@ def experience_search_hits( /, *, admission: AdmissionFloor | None = None, + min_score: float | None = None, + require_scores: bool = False, ) -> ExperienceSearchOutcome: """Decode backend-ordered rows and apply the shared lexical admission rule. - ``admission=None`` applies the historical lexical floor bit for bit. ``skill_search_hits`` - keeps its own default behaviour and is intentionally unaffected. + ``admission=None`` applies the historical lexical floor bit for bit. Raw score metadata + is optional for old custom indexes; public searches set ``require_scores=True``. ``retrieved`` counts the rows *examined*, not the rows kept: the loop stops as soon as - ``limit`` hits are admitted, so a backend that returned many rows for a narrow query is + ``limit`` final hits survive, so a backend that returned many rows for a narrow query is only visible through that count. ``scope_id`` is supplied by the caller because the decoder is the only place that knows both the rows and the Scope they were read from. """ hits: list[ExperienceSearchHit] = [] examined = 0 + admitted = 0 for row in rows: examined += 1 content = _content(row["content"]) if not admits_fts_text(query, experience_search_text(content), floor=admission): continue + admitted += 1 + retrieval_score, channel_scores = _search_scores(row, required=require_scores or min_score is not None) + if min_score is not None and (retrieval_score is None or retrieval_score < min_score): + continue hits.append( ExperienceSearchHit( artifact_ref=ArtifactRef( @@ -335,6 +361,8 @@ def experience_search_hits( revision=int(row["revision"]), ), content=content, + retrieval_score=retrieval_score, + channel_scores=channel_scores, ) ) if len(hits) >= limit: @@ -345,7 +373,7 @@ def experience_search_hits( family=Experience.family, scope_id=scope_id, retrieved=examined, - admitted=len(hits), + admitted=admitted, ), ) @@ -355,6 +383,10 @@ def skill_search_hits( query: str, limit: int, /, + *, + admission: AdmissionFloor | None = None, + min_score: float | None = None, + require_scores: bool = False, ) -> tuple[SkillSearchHit, ...]: """Decode backend-ordered Skill rows and apply shared lexical admission.""" @@ -362,7 +394,10 @@ def skill_search_hits( for row in rows: content = _skill_content(row["content"]) searchable = row.get("searchable_text") or skill_search_text(content) - if not admits_fts_text(query, str(searchable)): + if not admits_fts_text(query, str(searchable), floor=admission): + continue + retrieval_score, channel_scores = _search_scores(row, required=require_scores or min_score is not None) + if min_score is not None and (retrieval_score is None or retrieval_score < min_score): continue hits.append( SkillSearchHit( @@ -372,6 +407,8 @@ def skill_search_hits( revision=int(row["revision"]), ), content=content, + retrieval_score=retrieval_score, + channel_scores=channel_scores, ) ) if len(hits) >= limit: @@ -379,6 +416,15 @@ def skill_search_hits( return tuple(hits) +def _search_scores(row: Mapping[Any, Any], /, *, required: bool) -> tuple[float | None, dict[str, ChannelScore] | None]: + if "raw_score" not in row or "score_metric" not in row: + if required or "raw_score" in row or "score_metric" in row: + raise InvalidSearchScore("lexical raw score and metric are required") # noqa: TRY003 + return None, None + retrieval_score, channel = lexical_search_score(row["raw_score"], row["score_metric"]) + return retrieval_score, {"text": channel} + + async def _update_searchable_text( connection: AsyncConnection, *, diff --git a/src/powercontext/builtin/persistence/oceanbase/experience_index.py b/src/powercontext/builtin/persistence/oceanbase/experience_index.py index 16280d5cba..48806edc4e 100644 --- a/src/powercontext/builtin/persistence/oceanbase/experience_index.py +++ b/src/powercontext/builtin/persistence/oceanbase/experience_index.py @@ -16,7 +16,7 @@ from __future__ import annotations -from sqlalchemy import select, text +from sqlalchemy import literal, select, text from sqlalchemy.dialects.mysql import match from sqlalchemy.ext.asyncio import AsyncConnection @@ -87,6 +87,8 @@ async def search( /, *, admission: AdmissionFloor | None = None, + min_score: float | None = None, + require_scores: bool = False, ) -> ExperienceSearchOutcome: analyzed = analyze_fts_query(query) if not analyzed: @@ -98,6 +100,8 @@ async def search( ARTIFACT_HEADS_TABLE.c.artifact_id, ARTIFACT_HEADS_TABLE.c.revision, ARTIFACTS_TABLE.c.content, + score.label("raw_score"), + literal("oceanbase_match").label("score_metric"), ) .join( ARTIFACTS_TABLE, @@ -116,7 +120,9 @@ async def search( .limit(limit * 4) ) ).mappings() - return experience_search_hits(rows, query, limit, scope_id, admission=admission) + return experience_search_hits( + rows, query, limit, scope_id, admission=admission, min_score=min_score, require_scores=require_scores + ) async def replace_skill( self, @@ -135,6 +141,10 @@ async def search_skills( query: str, limit: int, /, + *, + admission: AdmissionFloor | None = None, + min_score: float | None = None, + require_scores: bool = False, ) -> tuple[SkillSearchHit, ...]: analyzed = analyze_fts_query(query) if not analyzed: @@ -147,6 +157,8 @@ async def search_skills( ARTIFACT_HEADS_TABLE.c.revision, ARTIFACT_HEADS_TABLE.c.searchable_text, ARTIFACTS_TABLE.c.content, + score.label("raw_score"), + literal("oceanbase_match").label("score_metric"), ) .join( ARTIFACTS_TABLE, @@ -165,7 +177,9 @@ async def search_skills( .limit(limit * 4) ) ).mappings() - return skill_search_hits(rows, query, limit) + return skill_search_hits( + rows, query, limit, admission=admission, min_score=min_score, require_scores=require_scores + ) __all__ = ["OceanBaseExperienceFTSIndex"] diff --git a/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py b/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py index 0a6ac68a63..82f9f059f2 100644 --- a/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py +++ b/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py @@ -204,6 +204,7 @@ async def search( TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.revision, TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.title, TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.summary, + topic_score.label("raw_score"), ) .where( TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.scope_id == scope_id, @@ -280,6 +281,7 @@ async def search( chunk_candidates.c.chunk_ordinal, chunk_candidates.c.start_offset, chunk_candidates.c.chunk_text, + chunk_candidates.c.score.label("raw_score"), ) .where(chunk_candidates.c.topic_rank == 1, chunk_candidates.c.coverage) .order_by( @@ -551,6 +553,8 @@ def _channel_hit(row: Mapping[Any, Any], channel: TopicMemoryMatchedBy) -> Topic chunk_start=None if row.get("start_offset") is None else int(row["start_offset"]), chunk_text=None if row.get("chunk_text") is None else str(row["chunk_text"]), distance=None if row.get("distance") is None else float(row["distance"]), + raw_score=None if row.get("raw_score") is None else float(row["raw_score"]), + metric="oceanbase_match" if channel.endswith("_fts") else None, ) diff --git a/src/powercontext/builtin/persistence/sqlite/experience_index.py b/src/powercontext/builtin/persistence/sqlite/experience_index.py index 23ef5a19b3..9875069940 100644 --- a/src/powercontext/builtin/persistence/sqlite/experience_index.py +++ b/src/powercontext/builtin/persistence/sqlite/experience_index.py @@ -64,7 +64,8 @@ ) _SEARCH_FTS_SQL = text( """ - SELECT f.artifact_id, f.revision, f.searchable_text, a.content + SELECT f.artifact_id, f.revision, f.searchable_text, a.content, + bm25(pc_artifact_fts) AS raw_score, 'sqlite_bm25' AS score_metric FROM pc_artifact_fts AS f JOIN pc_artifacts AS a ON a.scope_id = f.scope_id @@ -148,6 +149,8 @@ async def search( /, *, admission: AdmissionFloor | None = None, + min_score: float | None = None, + require_scores: bool = False, ) -> ExperienceSearchOutcome: match_query = fts_match_query(query) if match_query is None: @@ -163,7 +166,9 @@ async def search( }, ) ).mappings() - return experience_search_hits(rows, query, limit, scope_id, admission=admission) + return experience_search_hits( + rows, query, limit, scope_id, admission=admission, min_score=min_score, require_scores=require_scores + ) async def replace_skill( self, @@ -196,6 +201,10 @@ async def search_skills( query: str, limit: int, /, + *, + admission: AdmissionFloor | None = None, + min_score: float | None = None, + require_scores: bool = False, ) -> tuple[SkillSearchHit, ...]: match_query = fts_match_query(query) if match_query is None: @@ -211,7 +220,9 @@ async def search_skills( }, ) ).mappings() - return skill_search_hits(rows, query, limit) + return skill_search_hits( + rows, query, limit, admission=admission, min_score=min_score, require_scores=require_scores + ) @staticmethod async def _insert_row(connection: AsyncConnection, row: Mapping[Any, Any]) -> None: diff --git a/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py b/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py index aa9ea0244c..63d3472589 100644 --- a/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py +++ b/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py @@ -154,7 +154,7 @@ FROM pc_topic_memory_topic_fts WHERE pc_topic_memory_topic_fts MATCH :query AND scope_id = :scope_id ) - SELECT artifact_id, revision, title, summary + SELECT artifact_id, revision, title, summary, score AS raw_score FROM scored WHERE coverage >= :required_matches ORDER BY score, artifact_id, revision DESC @@ -189,7 +189,7 @@ ) AS topic_rank FROM scored ) - SELECT artifact_id, revision, title, summary, chunk_ordinal, start_offset, chunk_text + SELECT artifact_id, revision, title, summary, chunk_ordinal, start_offset, chunk_text, score AS raw_score FROM ranked WHERE topic_rank = 1 AND coverage >= :required_matches ORDER BY score, artifact_id, revision DESC, chunk_ordinal @@ -783,6 +783,8 @@ def _channel_hit(row: Mapping[Any, Any], channel: TopicMemoryMatchedBy) -> Topic chunk_start=None if row.get("start_offset") is None else int(row["start_offset"]), chunk_text=None if row.get("chunk_text") is None else str(row["chunk_text"]), distance=None if row.get("distance") is None else float(row["distance"]), + raw_score=None if row.get("raw_score") is None else float(row["raw_score"]), + metric="sqlite_bm25" if channel.endswith("_fts") else None, ) diff --git a/src/powercontext/builtin/persistence/topic_memory.py b/src/powercontext/builtin/persistence/topic_memory.py index d56279480f..9fb0c2cf3f 100644 --- a/src/powercontext/builtin/persistence/topic_memory.py +++ b/src/powercontext/builtin/persistence/topic_memory.py @@ -17,6 +17,7 @@ from __future__ import annotations from datetime import UTC, datetime +from typing import cast from sqlalchemy import and_, delete, func, insert, or_, select, update from sqlalchemy.dialects.mysql import insert as mysql_insert @@ -24,6 +25,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.artifacts import ArtifactAddress, ArtifactRef +from powercontext.artifacts.search import ArtifactSearchContractError, ArtifactSearchMatch from powercontext.builtin.artifacts.memory import EmbeddingProfile from powercontext.builtin.artifacts.memory.canonical import canonical_embedding from powercontext.builtin.artifacts.search import AdmissionCounts, AdmissionFloor, analyze_fts_query, analyze_text @@ -33,6 +35,7 @@ MAX_TOPIC_MEMORY_QUERY_TERMS, MAX_TOPIC_MEMORY_SEARCH_LIMIT, PublishedTopicMemory, + TopicArtifactSearchRequest, TopicMemory, TopicMemoryBrowseCursor, TopicMemoryCapabilityError, @@ -48,8 +51,9 @@ chunk_topic_memory_detail, ) from powercontext.builtin.artifacts.topic_memory.fusion import _fuse_topic_memory_rankings +from powercontext.builtin.artifacts.topic_memory.search import topic_fusion_parameters, validate_topic_weights from powercontext.builtin.persistence.artifacts import ArtifactRepository -from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError +from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError, RepositoryNotFoundError from powercontext.builtin.persistence.supervision import database_utc_now from powercontext.builtin.persistence.tables import ( ARTIFACT_HEADS_TABLE, @@ -445,6 +449,7 @@ async def search( query_vector: tuple[float, ...] | None = None, embedding_profile: EmbeddingProfile | None = None, admission: AdmissionFloor | None = None, + artifact_request: TopicArtifactSearchRequest | None = None, ) -> TopicMemorySearchResult: """Search current complete projections and fuse two or four logical channels. @@ -465,6 +470,11 @@ async def search( ) analyzed = analyze_text(query) used_mode = self._select_mode(mode, query_vector, embedding_profile) + fusion = None + if artifact_request is not None: + fusion = topic_fusion_parameters(artifact_request.fusion) + validate_topic_weights(used_mode, fusion) + admission = artifact_request.admission.as_floor() query_terms = tuple(sorted(set(analyzed.split()))) if used_mode in {"fts", "hybrid"} and len(query_terms) > MAX_TOPIC_MEMORY_QUERY_TERMS: raise InvalidRepositoryArgumentError( @@ -476,7 +486,9 @@ async def search( analyzed = analyze_fts_query(query) query_terms = tuple(sorted(set(analyzed.split()))) if not analyzed and used_mode == "fts": - return TopicMemorySearchResult(mode=used_mode, hits=()) + return TopicMemorySearchResult( + mode=used_mode, hits=(), artifacts=() if artifact_request is not None else None + ) request = TopicMemorySearchRequest( query=query, analyzed_query=" ".join(query_terms), @@ -488,10 +500,35 @@ async def search( ) channels = await self.index.search(connection, scope_id, request) await self._check_retrieval_shape(connection) - outcome = _fuse_topic_memory_rankings(query, channels, limit, mode=used_mode, admission=admission) + outcome = _fuse_topic_memory_rankings( + query, + channels, + limit, + mode=used_mode, + admission=admission, + fusion=fusion, + min_score=None if artifact_request is None else artifact_request.min_score, + include_scores=artifact_request is not None and artifact_request.include_scores, + ) + artifacts = None + if artifact_request is not None: + try: + artifacts = tuple([ + cast(TopicMemory, await self.artifacts.get(connection, scope_id, hit.artifact_ref)) + for hit in outcome.hits + ]) + except RepositoryNotFoundError as error: + raise ArtifactSearchContractError(TopicMemory.family, "selected exact Artifact is missing") from error return TopicMemorySearchResult( mode=used_mode, hits=outcome.hits, + artifacts=artifacts, + matches=tuple( + ArtifactSearchMatch(hit.artifact_ref, cast(float, hit.retrieval_score), hit.channel_scores) + for hit in outcome.hits + ) + if artifact_request is not None + else (), admission=AdmissionCounts( family=TopicMemory.family, scope_id=scope_id, diff --git a/src/powercontext/builtin/runtime/__init__.py b/src/powercontext/builtin/runtime/__init__.py index 4b46eae1fb..7b12321e73 100644 --- a/src/powercontext/builtin/runtime/__init__.py +++ b/src/powercontext/builtin/runtime/__init__.py @@ -49,6 +49,7 @@ SkillGenerationOrigin, ) from powercontext.builtin.runtime.application import ( + ArtifactApplication, BuiltinRuntime, ExperienceApplication, ExternalSkillApplication, @@ -59,6 +60,7 @@ ReviewApplication, ScheduledExperienceProcessor, ScheduledSourceProcessor, + ScopedArtifactApplication, ScopedExperienceApplication, ScopedExternalSkillApplication, ScopedHandoffApplication, @@ -213,6 +215,7 @@ __all__ = [ "ActivateHandoff", "ApproveArtifactCandidateRequest", + "ArtifactApplication", "ArtifactInventoryStatistics", "ArtifactProcessingBinding", "ArtifactProcessingSupervisor", @@ -350,6 +353,7 @@ "ScheduledExperienceProcessor", "ScheduledSourceProcessor", "ScopeStatistics", + "ScopedArtifactApplication", "ScopedDreamApplication", "ScopedExperienceApplication", "ScopedExternalSkillApplication", diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index 3891ff82bc..8fc129623a 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -30,6 +30,11 @@ from powercontext._logging import log_safely from powercontext.artifacts import ArtifactRef +from powercontext.artifacts.search import ( + ArtifactSearchExecutionContext, + ArtifactSearchOutcome, + ArtifactSearchUnsupported, +) from powercontext.builtin.artifacts.experience import ( EXPERIENCE_INCUBATION_WINDOW_LIMIT, Experience, @@ -132,10 +137,6 @@ from powercontext.builtin.evidence.resolver import AuthorizationContext, ScopedEvidenceAuthorizer from powercontext.builtin.inference import ( EmbeddingModel, - InferenceTimeoutError, - InferenceUnavailableError, - InvalidInferenceOutputError, - embed_query, ) from powercontext.builtin.inference.models import InferenceUsage from powercontext.builtin.inference.usage import bind_usage_reporter @@ -167,6 +168,7 @@ ScopeCacheObserver, ScopeEvictor, ) +from powercontext.builtin.runtime.artifact_search import ArtifactSearchService from powercontext.builtin.runtime.decision_model import DecisionModel from powercontext.builtin.runtime.errors import InvalidRuntimeRequestError, TopicMemoryProcessingUnavailableError from powercontext.builtin.runtime.models import ( @@ -249,7 +251,9 @@ build_recall_candidates, recall_effort, ) +from powercontext.builtin.runtime.skill_search import search_skill_library from powercontext.builtin.runtime.statistics import RelationalScopedStatistics, overview_selection +from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher from powercontext.builtin.scope import ScopeApplication, ScopeDescriptor, ScopeSelection from powercontext.builtin.scope.subject_sources import SubjectSourceService from powercontext.builtin.sources import ( @@ -317,6 +321,7 @@ def __call__( embedding_profile: EmbeddingProfile | None = None, admission: AdmissionFloor | None = None, query_embedding: MemoryQueryEmbedding | None = None, + execution_context: ArtifactSearchExecutionContext | None = None, ) -> Awaitable[TopicMemorySearchResult]: ... @@ -1734,6 +1739,38 @@ def for_scope(self, scope_id: str, /) -> ScopedExperienceApplication: return ScopedExperienceApplication(self._runtime, scope_id) +class ScopedArtifactApplication: + """Search a registered Artifact Family within one existing Scope.""" + + def __init__(self, runtime: BuiltinRuntime, scope_id: str) -> None: + self._runtime = runtime + self.scope_id = validate_scope_id(scope_id) + + async def search( + self, + family: str, + payload: Mapping[str, Any], + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> ArtifactSearchOutcome: + service = self._runtime._artifact_search + if service is None: + raise ArtifactSearchUnsupported(family, field="family") + async with self._runtime._scoped_operation(self.scope_id, embedding_purpose=service.embedding_purpose(family)): + return await service.search(self.scope_id, family, payload, execution_context=execution_context) + + +class ArtifactApplication: + """Select the scoped public Artifact search application.""" + + def __init__(self, runtime: BuiltinRuntime) -> None: + self._runtime = runtime + + def for_scope(self, scope_id: str, /) -> ScopedArtifactApplication: + return ScopedArtifactApplication(self._runtime, scope_id) + + class ScopedSkillApplication: """Propose and exactly read managed Skill Artifacts in one scope.""" @@ -1779,6 +1816,13 @@ async def search(self, query: str, limit: int, /) -> tuple[SkillSearchHit, ...]: async with self._runtime._scoped_operation(self.scope_id): return await recall(self.scope_id, query, limit) + async def search_library( + self, query: str, limit: int, /, *, include_deprecated: bool = False + ) -> tuple[tuple[Skill, ArtifactGovernance], ...]: + """Preserve the Skill Library's active search and bounded deprecated append rule.""" + + return await search_skill_library(self, query, limit, include_deprecated=include_deprecated) + async def list( self, *, @@ -2640,9 +2684,10 @@ async def search( query_embedding: MemoryQueryEmbedding | None = None, embedding_timeout_seconds: float | None = None, allow_embedding: bool = True, + execution_context: ArtifactSearchExecutionContext | None = None, ) -> TopicMemorySearchResult: - search = self._runtime._topic_memory_search - if search is None: + searcher = self._runtime._topic_memory_searcher + if searcher is None: raise _RuntimeStateError("topic-memory-search") query = request.query if query != query.strip() or not query or len(query) > MAX_TOPIC_MEMORY_QUERY_LENGTH: @@ -2652,122 +2697,20 @@ async def search( if len(set(analyze_text(query).split())) > MAX_TOPIC_MEMORY_QUERY_TERMS: raise InvalidRuntimeRequestError("topic-memory-query-terms") - used_fallback = False async with self._runtime._scoped_operation( self.scope_id, embedding_purpose=ModelUsagePurpose.TOPIC_MEMORY_RECALL, ): - embedding = self._runtime._topic_memory_embedding_model if allow_embedding else None - browse = self._runtime._topic_memory_browse - if embedding is not None and browse is not None and not await browse(self.scope_id, limit=1, after=None): - embedding = None - if embedding is None: - result = await search( - self.scope_id, - query, - limit=request.limit, - mode="fts", - **_admission_keyword(admission), - ) - result = result.model_copy(update={"embedding_calls": 0}) - else: - result, used_fallback = await self._search_with_embedding( - request, - embedding, - search, - admission, - query_embedding, - embedding_timeout_seconds, - ) - observer = self._runtime._topic_memory_search_observer - if observer is not None: - try: - observer(result.mode, used_fallback) - except Exception as error: - log_safely( - logger, - logging.ERROR, - "Topic Memory search observation failed", - exc_info=error, - extra={ - "event": "topic_memory.search.observation_failed", - "outcome": "failure", - "unit": "topic-memory", - }, - ) - return result - - async def _search_with_embedding( - self, - request: SearchTopicMemoryRequest, - embedding: EmbeddingModel, - search: TopicMemorySearch, - admission: AdmissionFloor | None, - query_embedding: MemoryQueryEmbedding | None, - embedding_timeout_seconds: float | None, - ) -> tuple[TopicMemorySearchResult, bool]: - if query_embedding is not None and query_embedding.embedding_profile == embedding.profile: - result = await search( - self.scope_id, - request.query, - limit=request.limit, - mode="hybrid", - query_vector=query_embedding.query_vector, - embedding_profile=query_embedding.embedding_profile, - **_admission_keyword(admission), - ) - return result.model_copy(update={"query_embedding": query_embedding, "embedding_calls": 0}), False - try: - async with asyncio.timeout(embedding_timeout_seconds): - embedded = await embed_query(embedding, (request.query,)) - if len(embedded.vectors) != 1: - raise InvalidInferenceOutputError("embed", "provider returned the wrong vector count") - except (InferenceUnavailableError, InferenceTimeoutError, TimeoutError) as error: - used_fallback = True - log_safely( - logger, - logging.WARNING, - "Topic Memory search fell back to FTS", - extra={ - "event": "topic_memory.search.embedding_fallback", - "outcome": "fallback", - "mode": "fts", - "error_code": ( - "inference_timeout" - if isinstance(error, (InferenceTimeoutError, TimeoutError)) - else "inference_unavailable" - ), - "unit": "topic-memory", - }, - ) - else: - result = await search( + return await searcher.search_legacy( self.scope_id, request.query, limit=request.limit, - mode="hybrid", - query_vector=embedded.vectors[0], - embedding_profile=embedding.profile, - **_admission_keyword(admission), + admission=admission, + query_embedding=query_embedding, + embedding_timeout_seconds=embedding_timeout_seconds, + allow_embedding=allow_embedding, + execution_context=execution_context, ) - return result.model_copy( - update={ - "query_embedding": MemoryQueryEmbedding( - query_vector=tuple(embedded.vectors[0]), - embedding_profile=embedding.profile, - ), - "embedding_calls": 1, - } - ), False - - result = await search( - self.scope_id, - request.query, - limit=request.limit, - mode="fts", - **_admission_keyword(admission), - ) - return result.model_copy(update={"embedding_calls": 1}), used_fallback async def get(self, request: GetTopicMemoryRequest, /) -> PublishedTopicMemory: if self._runtime._topic_memory_get is None: @@ -3019,6 +2962,8 @@ def __init__( topic_memory_embedding_model: EmbeddingModel | None = None, topic_memory_processing_available: bool = False, topic_memory_search_observer: TopicMemorySearchObserver | None = None, + topic_memory_searcher: TopicMemorySearcher | None = None, + artifact_search: ArtifactSearchService | None = None, external_skill_registry: ExternalSkillRegistryFactory | None = None, external_skill_importer: ExternalSkillImporter | None = None, skill_publication_service: SkillPublicationServiceFactory | None = None, @@ -3075,6 +3020,16 @@ def __init__( self._topic_memory_embedding_model = topic_memory_embedding_model self._topic_memory_processing_available = topic_memory_processing_available self._topic_memory_search_observer = topic_memory_search_observer + self._topic_memory_searcher = topic_memory_searcher + if self._topic_memory_searcher is None and topic_memory_search is not None: + self._topic_memory_searcher = TopicMemorySearcher( + search=topic_memory_search, + get=topic_memory_get, + browse=topic_memory_browse, + embedding_model=topic_memory_embedding_model, + observer=topic_memory_search_observer, + ) + self._artifact_search = artifact_search self._external_skill_registry = external_skill_registry self._external_skill_importer = external_skill_importer self._skill_publication_service = skill_publication_service @@ -3116,6 +3071,7 @@ def __init__( self.code = CodeApplication(self, code_service or CodeService(CodeConfig())) self.context = ContextApplication(self) self.experience = ExperienceApplication(self) + self.artifacts = ArtifactApplication(self) self.dream = DreamApplication(self) self.external_skills = ExternalSkillApplication(self) self.handoff = HandoffApplication(self) diff --git a/src/powercontext/builtin/runtime/artifact_search.py b/src/powercontext/builtin/runtime/artifact_search.py new file mode 100644 index 0000000000..226a420e10 --- /dev/null +++ b/src/powercontext/builtin/runtime/artifact_search.py @@ -0,0 +1,185 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Typed Artifact search registration and public result validation.""" + +from __future__ import annotations + +import json +import math +from collections.abc import Iterable, Mapping, Sequence +from typing import Any, Generic, TypeVar + +from pydantic import ValidationError +from pydantic_core import InitErrorDetails + +from powercontext.artifacts import Artifact +from powercontext.artifacts.search import ( + ArtifactSearchContractError, + ArtifactSearcher, + ArtifactSearchExecutionContext, + ArtifactSearchFamilyNotFound, + ArtifactSearchMatch, + ArtifactSearchOutcome, + ArtifactSearchQuery, + ArtifactSearchUnsupported, +) +from powercontext.builtin.statistics import ModelUsagePurpose + +RequestT = TypeVar("RequestT", bound=ArtifactSearchQuery) +OutcomeT_co = TypeVar("OutcomeT_co", bound=ArtifactSearchOutcome, covariant=True) + + +class _SearcherBinding(Generic[RequestT, OutcomeT_co]): + def __init__( + self, + searcher: ArtifactSearcher[RequestT, OutcomeT_co], + embedding_purpose: ModelUsagePurpose | None, + ) -> None: + self.searcher = searcher + self.embedding_purpose = embedding_purpose + + async def invoke( + self, + scope_id: str, + payload: Mapping[str, Any], + execution_context: ArtifactSearchExecutionContext | None, + ) -> ArtifactSearchOutcome: + encoded = _encode_request_payload(payload, self.searcher.request_type.__name__) + request = self.searcher.request_type.model_validate_json(encoded, strict=True) + outcome = await self.searcher.search(scope_id, request, execution_context=execution_context) + _validate_public_outcome(self.searcher.family, request, outcome) + return outcome + + +class ArtifactSearchService: + """Dispatch repository-known Families without knowing their concrete models.""" + + def __init__(self, *, known_families: Iterable[str]) -> None: + self._known_families = frozenset(known_families) + self._bindings: dict[str, _SearcherBinding[Any, ArtifactSearchOutcome]] = {} + + def register( + self, + searcher: ArtifactSearcher[RequestT, OutcomeT_co], + *, + embedding_purpose: ModelUsagePurpose | None = None, + ) -> None: + """Keep a Family's parser and callable in the same typed binding.""" + + if searcher.family not in self._known_families: + raise ArtifactSearchFamilyNotFound(searcher.family) + if searcher.family in self._bindings: + raise ValueError(f"Artifact searcher is already registered for {searcher.family}") # noqa: TRY003 + if not isinstance(searcher.request_type, type) or not issubclass(searcher.request_type, ArtifactSearchQuery): + raise TypeError("Artifact search request type must inherit ArtifactSearchQuery") # noqa: TRY003 + self._bindings[searcher.family] = _SearcherBinding(searcher, embedding_purpose) + + def embedding_purpose(self, family: str) -> ModelUsagePurpose | None: + """Read invocation metadata while preserving unknown/unsupported failures.""" + + return self._binding(family).embedding_purpose + + async def search( + self, + scope_id: str, + family: str, + payload: Mapping[str, Any], + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> ArtifactSearchOutcome: + """Parse with the registered request model and return its existing outcome.""" + + return await self._binding(family).invoke(scope_id, payload, execution_context) + + def _binding(self, family: str) -> _SearcherBinding[Any, ArtifactSearchOutcome]: + if family not in self._known_families: + raise ArtifactSearchFamilyNotFound(family) + binding = self._bindings.get(family) + if binding is None: + raise ArtifactSearchUnsupported(family, field="family") + return binding + + +def _encode_request_payload(payload: Mapping[str, Any], title: str) -> str: + try: + return json.dumps(payload, ensure_ascii=False, allow_nan=False) + except (ValueError, TypeError, OverflowError): + errors: list[InitErrorDetails] = [] + pending: list[tuple[Any, tuple[str | int, ...]]] = [(payload, ())] + visited: set[int] = set() + while pending: + value, path = pending.pop() + if isinstance(value, float) and not math.isfinite(value): + errors.append({"type": "finite_number", "loc": path, "input": value}) + elif isinstance(value, (Mapping, list, tuple)) and id(value) not in visited: + visited.add(id(value)) + entries = value.items() if isinstance(value, Mapping) else enumerate(value) + pending.extend((item, (*path, key)) for key, item in entries) + if not errors: + errors.append({ + "type": "json_invalid", + "loc": (), + "input": None, + "ctx": {"error": "search payload must contain JSON-compatible values"}, + }) + raise ValidationError.from_exception_data(title, errors) from None + + +def _validate_public_outcome(family: str, request: ArtifactSearchQuery, outcome: ArtifactSearchOutcome) -> None: + try: + matches = outcome.matches + artifacts = outcome.artifacts + except AttributeError as exc: + raise ArtifactSearchContractError(family, "matches and artifacts are required") from exc + if ( + not isinstance(matches, Sequence) + or not isinstance(artifacts, Sequence) + or isinstance(matches, (str, bytes, bytearray)) + or isinstance(artifacts, (str, bytes, bytearray)) + ): + raise ArtifactSearchContractError(family, "public matches and artifacts must be sequences") + if len(matches) != len(artifacts): + raise ArtifactSearchContractError(family, "matches and artifacts must have the same count") + if len(matches) > request.limit: + raise ArtifactSearchContractError(family, "result count exceeds the parsed request limit") + identities: set[tuple[str, str, int]] = set() + for match, artifact in zip(matches, artifacts, strict=True): + identity = _validate_public_result(family, request, match, artifact) + if identity in identities: + raise ArtifactSearchContractError(family, "an exact Artifact revision appears more than once") + identities.add(identity) + + +def _validate_public_result( + family: str, + request: ArtifactSearchQuery, + match: ArtifactSearchMatch, + artifact: Artifact[Any], +) -> tuple[str, str, int]: + if not isinstance(match, ArtifactSearchMatch) or not isinstance(artifact, Artifact): + raise ArtifactSearchContractError(family, "results must contain ArtifactSearchMatch and Artifact values") + try: + ArtifactSearchMatch(match.artifact_ref, match.retrieval_score, match.channel_scores) + artifact_ref = artifact.as_ref() + except (ValueError, TypeError, AttributeError) as exc: + raise ArtifactSearchContractError(family, "invalid exact reference or score metadata") from exc + if match.artifact_ref.family != family or artifact_ref != match.artifact_ref: + raise ArtifactSearchContractError(family, "Artifact identity, Family, and match order must agree") + if request.include_scores and match.channel_scores is None: + raise ArtifactSearchContractError(family, "requested channel scores were not collected") + return (match.artifact_ref.family, match.artifact_ref.artifact_id, match.artifact_ref.revision) + + +__all__ = ["ArtifactSearchService"] diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index d8285b2969..4f6ee5f992 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -126,6 +126,7 @@ ArtifactProcessingSupervisors, SpawnArtifactProcessingWorkerLauncher, ) +from powercontext.builtin.runtime.artifact_search import ArtifactSearchService from powercontext.builtin.runtime.config import BuiltinConfig, ExternalSkillsConfig, InferenceConfig, RuntimeConfig from powercontext.builtin.runtime.decision_model import ( DECISION_INSTRUCTIONS, @@ -137,6 +138,7 @@ FailOpenDecisionModel, LLMDecisionModel, ) +from powercontext.builtin.runtime.experience_search import ExperienceArtifactSearcher from powercontext.builtin.runtime.family_processing import FAMILY_BINDINGS, FamilyWorkerSpec, run_family_worker from powercontext.builtin.runtime.memory_write_gate import build_memory_write_gate from powercontext.builtin.runtime.models import MemorySearchMode, RuntimeCapabilities @@ -156,15 +158,18 @@ ) from powercontext.builtin.runtime.recall_sufficiency import RecallSufficiencyPolicy from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.runtime.skill_search import SkillArtifactSearcher from powercontext.builtin.runtime.topic_memory_processing import ( TopicMemoryWorkerSpec, run_topic_memory_worker, validate_topic_memory_provider_settings, ) +from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher from powercontext.builtin.sources import ( BUILTIN_SOURCE_REGISTRY, TEXT_EVIDENCE_PROJECTION_KEY, ) +from powercontext.builtin.statistics import ModelUsagePurpose from powercontext.errors import InvalidSourceProjectionError, SourceProjectionNotFoundError from powercontext.sources import Source, SourceDefinitionRegistry, SourceProjectionKey @@ -581,6 +586,22 @@ async def run_profile(scope_id, high): if family in registered_families ) topic_memory_processing_available = _topic_memory_processing_available(config, processing_bindings) + artifact_search = ArtifactSearchService(known_families=contexts.repositories.artifacts.families) + artifact_search.register( + ExperienceArtifactSearcher(contexts.database, contexts.repositories.artifacts, contexts.experience_index) + ) + artifact_search.register( + SkillArtifactSearcher(contexts.database, contexts.repositories.artifacts, contexts.experience_index) + ) + topic_memory_searcher = TopicMemorySearcher( + search=contexts.search_topic_memories, + get=contexts.get_topic_memory, + browse=contexts.browse_topic_memories, + embedding_model=configured_embedding, + observer=topic_memory_search_observer, + capabilities=contexts.topic_memory_index.capabilities, + ) + artifact_search.register(topic_memory_searcher, embedding_purpose=ModelUsagePurpose.TOPIC_MEMORY_RECALL) runtime = await resources.enter_async_context( BuiltinRuntime( code_service=await resources.enter_async_context(open_code_service(config.code, config.database)), @@ -617,6 +638,7 @@ async def run_profile(scope_id, high): ), generation_concurrency=config.runtime.generation_concurrency, experience_recall=contexts.search_experience_outcome, + artifact_search=artifact_search, skill_recall=contexts.search_skills, skill_lister=contexts.list_skills, skill_origin_reader=contexts.get_skill_origins, @@ -628,6 +650,7 @@ async def run_profile(scope_id, high): skill_usage_recorder=contexts.record_skill_usage, experience_incubator=contexts.incubate_experience if contexts.experience_incubation else None, topic_memory_search=contexts.search_topic_memories, + topic_memory_searcher=topic_memory_searcher, topic_memory_get=contexts.get_topic_memory, topic_memory_browse=contexts.browse_topic_memories, topic_memory_flush=contexts.request_topic_memory_flush, diff --git a/src/powercontext/builtin/runtime/experience_search.py b/src/powercontext/builtin/runtime/experience_search.py new file mode 100644 index 0000000000..7e15be72a9 --- /dev/null +++ b/src/powercontext/builtin/runtime/experience_search.py @@ -0,0 +1,83 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public Experience search over the existing approved-head lexical index.""" + +from dataclasses import replace +from typing import cast + +from powercontext.artifacts.search import ( + ArtifactSearchContractError, + ArtifactSearchExecutionContext, + ArtifactSearchMatch, + ArtifactSearchUnsupported, +) +from powercontext.builtin.artifacts.experience import Experience, ExperienceSearchOutcome, ExperienceSearchRequest +from powercontext.builtin.artifacts.search import InvalidSearchScore +from powercontext.builtin.persistence.artifacts import ArtifactRepository +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.experience_index import ExperienceIndex, NoExperienceIndex +from powercontext.builtin.sources import validate_scope_id + + +class ExperienceArtifactSearcher: + """Own Experience request controls and exact final-result reads.""" + + family = Experience.family + request_type = ExperienceSearchRequest + + def __init__(self, database: AsyncDatabase, artifacts: ArtifactRepository, index: ExperienceIndex) -> None: + self._database = database + self._artifacts = artifacts + self._index = index + + async def search( + self, + scope_id: str, + request: ExperienceSearchRequest, + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> ExperienceSearchOutcome: + if isinstance(self._index, NoExperienceIndex): + raise ArtifactSearchUnsupported(self.family, field="mode") + scope = validate_scope_id(scope_id) + async with self._database.transaction() as connection: + outcome = await self._index.search( + connection, + scope, + request.query, + request.limit, + admission=request.admission.as_floor(), + min_score=request.min_score, + require_scores=True, + ) + matches: list[ArtifactSearchMatch] = [] + artifacts: list[Experience] = [] + for hit in outcome.hits: + if hit.retrieval_score is None or hit.channel_scores is None or "text" not in hit.channel_scores: + raise InvalidSearchScore("public Experience search requires true text scoring metadata") # noqa: TRY003 + matches.append(ArtifactSearchMatch(hit.artifact_ref, hit.retrieval_score, hit.channel_scores)) + try: + artifact = await self._artifacts.get(connection, scope, hit.artifact_ref) + except RepositoryNotFoundError as error: + raise ArtifactSearchContractError( + self.family, "selected exact Artifact revision is missing" + ) from error + artifacts.append(cast(Experience, artifact)) + return replace(outcome, matches=tuple(matches), artifacts=tuple(artifacts)) + + +__all__ = ["ExperienceArtifactSearcher"] diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index ff080768a8..b4c36e10ae 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -38,6 +38,7 @@ from powercontext._logging import log_safely from powercontext.artifacts import Artifact, ArtifactRef, MemoryCitation +from powercontext.artifacts.search import ArtifactSearchExecutionContext from powercontext.builtin.artifacts.experience import ( EXPERIENCE_INCUBATION_CURSOR_NAME, Experience, @@ -104,6 +105,7 @@ from powercontext.builtin.artifacts.topic_memory import ( TOPIC_MEMORY_SOURCE_WINDOW_BINDING, PublishedTopicMemory, + TopicArtifactSearchRequest, TopicMemory, TopicMemoryBrowseCursor, TopicMemoryCurrentItem, @@ -994,6 +996,8 @@ async def search_topic_memories( embedding_profile: EmbeddingProfile | None = None, admission: AdmissionFloor | None = None, query_embedding: MemoryQueryEmbedding | None = None, + artifact_request: TopicArtifactSearchRequest | None = None, + execution_context: ArtifactSearchExecutionContext | None = None, ) -> TopicMemorySearchResult: """Search current active Topic projections in this deployment.""" @@ -1001,7 +1005,29 @@ async def search_topic_memories( query_vector = query_embedding.query_vector embedding_profile = query_embedding.embedding_profile scope = validate_scope_id(scope_id) - async with self.database.transaction() as connection: + access = None if execution_context is None else execution_context.access + if execution_context is not None and access is None and not execution_context.trusted_local: + from powercontext.server.authz import AccessDeniedError + + raise AccessDeniedError + audit = nullcontext() if access is None else access.defer_decision_audit() + async with audit, self.database.transaction(consistent_snapshot=True) as connection: + if execution_context is not None and access is not None: + # Establish the data snapshot before a remote PDP can suspend. + # Local providers then read their policy through this same connection. + await connection.execute( + select(ARTIFACT_HEADS_TABLE.c.revision) + .where( + ARTIFACT_HEADS_TABLE.c.scope_id == scope, ARTIFACT_HEADS_TABLE.c.family == TopicMemory.family + ) + .limit(1) + ) + await access.require_scope_read( + execution_context.principal, + scope, + connection=connection, + context=execution_context.audit, + ) return await self.repositories.topic_memories.search( connection, scope, @@ -1011,6 +1037,7 @@ async def search_topic_memories( query_vector=query_vector, embedding_profile=embedding_profile, admission=admission, + **({} if artifact_request is None else {"artifact_request": artifact_request}), ) async def search_skills( diff --git a/src/powercontext/builtin/runtime/skill_search.py b/src/powercontext/builtin/runtime/skill_search.py new file mode 100644 index 0000000000..6377bbaa26 --- /dev/null +++ b/src/powercontext/builtin/runtime/skill_search.py @@ -0,0 +1,123 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public managed Skill search over the existing approved-head lexical index.""" + +from typing import Protocol, cast + +from powercontext.artifacts.search import ( + ArtifactSearchContractError, + ArtifactSearchExecutionContext, + ArtifactSearchMatch, + ArtifactSearchUnsupported, +) +from powercontext.builtin.artifacts.search import InvalidSearchScore +from powercontext.builtin.artifacts.skill import Skill, SkillSearchHit, SkillSearchOutcome, SkillSearchRequest +from powercontext.builtin.persistence.artifact_governance import ArtifactGovernance, ArtifactLifecycleState +from powercontext.builtin.persistence.artifacts import ArtifactRepository +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.experience_index import ExperienceIndex, NoExperienceIndex +from powercontext.builtin.runtime.models import GetSkillRequest +from powercontext.builtin.sources import validate_scope_id + + +class _SkillLibrary(Protocol): + async def search(self, query: str, limit: int, /) -> tuple[SkillSearchHit, ...]: ... + async def get(self, request: GetSkillRequest, /) -> Skill: ... + async def governance(self, artifact_id: str, /) -> ArtifactGovernance: ... + async def list( + self, *, include_deprecated: bool = False, limit: int = 100 + ) -> tuple[tuple[Skill, ArtifactGovernance], ...]: ... + + +async def search_skill_library( + scoped: _SkillLibrary, query: str, limit: int, /, *, include_deprecated: bool = False +) -> tuple[tuple[Skill, ArtifactGovernance], ...]: + """Apply the legacy Library rule through existing scoped Skill operations.""" + + query = query.strip() + if not query: + return (await scoped.list(include_deprecated=include_deprecated, limit=limit))[:limit] + values: list[tuple[Skill, ArtifactGovernance]] = [] + for hit in await scoped.search(query, limit): + skill = await scoped.get(GetSkillRequest(artifact=hit.artifact_ref)) + values.append((skill, await scoped.governance(skill.artifact_id))) + if include_deprecated: + seen = {skill.artifact_id for skill, _governance in values} + for skill, governance in await scoped.list(include_deprecated=True, limit=limit): + search_text = "\n".join(( + skill.content.name, + skill.content.description, + skill.content.instructions, + *skill.content.metadata.values(), + )) + if ( + governance.lifecycle_state is ArtifactLifecycleState.DEPRECATED + and skill.artifact_id not in seen + and query.casefold() in search_text.casefold() + ): + values.append((skill, governance)) + return tuple(values[:limit]) + + +class SkillArtifactSearcher: + """Own Skill request controls and exact final-result reads.""" + + family = Skill.family + request_type = SkillSearchRequest + + def __init__(self, database: AsyncDatabase, artifacts: ArtifactRepository, index: ExperienceIndex) -> None: + self._database = database + self._artifacts = artifacts + self._index = index + + async def search( + self, + scope_id: str, + request: SkillSearchRequest, + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> SkillSearchOutcome: + if isinstance(self._index, NoExperienceIndex): + raise ArtifactSearchUnsupported(self.family, field="mode") + scope = validate_scope_id(scope_id) + async with self._database.transaction() as connection: + hits = await self._index.search_skills( + connection, + scope, + request.query, + request.limit, + admission=request.admission.as_floor(), + min_score=request.min_score, + require_scores=True, + ) + matches: list[ArtifactSearchMatch] = [] + artifacts: list[Skill] = [] + for hit in hits: + if hit.retrieval_score is None or hit.channel_scores is None or "text" not in hit.channel_scores: + raise InvalidSearchScore("public Skill search requires true text scoring metadata") # noqa: TRY003 + matches.append(ArtifactSearchMatch(hit.artifact_ref, hit.retrieval_score, hit.channel_scores)) + try: + artifact = await self._artifacts.get(connection, scope, hit.artifact_ref) + except RepositoryNotFoundError as error: + raise ArtifactSearchContractError( + self.family, "selected exact Artifact revision is missing" + ) from error + artifacts.append(cast(Skill, artifact)) + return SkillSearchOutcome(hits=hits, matches=tuple(matches), artifacts=tuple(artifacts)) + + +__all__ = ["SkillArtifactSearcher", "search_skill_library"] diff --git a/src/powercontext/builtin/runtime/topic_memory_search.py b/src/powercontext/builtin/runtime/topic_memory_search.py new file mode 100644 index 0000000000..6c3c99cc04 --- /dev/null +++ b/src/powercontext/builtin/runtime/topic_memory_search.py @@ -0,0 +1,291 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared Topic search orchestration for public Artifact and legacy entry points.""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from typing import Any + +from powercontext._logging import log_safely +from powercontext.artifacts import ArtifactRef +from powercontext.artifacts.search import ArtifactSearchContractError, ArtifactSearchExecutionContext +from powercontext.builtin.artifacts.memory import MemoryQueryEmbedding +from powercontext.builtin.artifacts.search import AdmissionFloor, analyze_text +from powercontext.builtin.artifacts.topic_memory import ( + MAX_TOPIC_MEMORY_QUERY_TERMS, + PublishedTopicMemory, + TopicArtifactSearchRequest, + TopicMemoryCapabilities, + TopicMemoryCurrentItem, + TopicMemorySearchResult, +) +from powercontext.builtin.artifacts.topic_memory.search import TopicSearchPlan, plan_topic_search, topic_search_error +from powercontext.builtin.inference import ( + EmbeddingModel, + EmbeddingResult, + InferenceTimeoutError, + InferenceUnavailableError, + InvalidInferenceOutputError, +) +from powercontext.builtin.inference.protocols import embed_query +from powercontext.builtin.persistence.errors import RepositoryNotFoundError + +logger = logging.getLogger(__name__) + + +class TopicMemorySearcher: + """Own mode selection, embedding reuse, and transient default fallback. + + The calling Runtime facade owns scoped operation and model usage accounting. + Public execution forwards the Family request to the composed backend. Legacy + execution preserves its existing callback keywords and complete-read cost. + """ + + family = "topic-memory" + request_type = TopicArtifactSearchRequest + + def __init__( + self, + *, + search: Callable[..., Awaitable[TopicMemorySearchResult]], + get: Callable[[str, ArtifactRef], Awaitable[PublishedTopicMemory]] | None = None, + browse: Callable[..., Awaitable[tuple[TopicMemoryCurrentItem, ...]]] | None = None, + embedding_model: EmbeddingModel | None = None, + observer: Callable[[str, bool], None] | None = None, + capabilities: TopicMemoryCapabilities | None = None, + ) -> None: + self._search = search + self._get = get + self._browse = browse + self._embedding = embedding_model + self._observer = observer + self._capabilities = capabilities + + async def search( + self, + scope_id: str, + request: TopicArtifactSearchRequest, + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> TopicMemorySearchResult: + capabilities = self._capabilities + if capabilities is None: + raise topic_search_error(("mode",), "Topic search capabilities are not configured") + embedding = self._embedding + plan = plan_topic_search(request, capabilities, embedding_available=embedding is not None) + if ( + plan.mode in {"vector", "hybrid"} + and embedding is not None + and capabilities.embedding_profile != embedding.profile + ): + raise topic_search_error( + ("mode",), "vector and hybrid searches require an embedding model matching the index profile" + ) + if ( + plan.mode in {"fts", "hybrid"} + and len(set(analyze_text(request.query).split())) > MAX_TOPIC_MEMORY_QUERY_TERMS + ): + raise topic_search_error( + ("query",), f"text search accepts at most {MAX_TOPIC_MEMORY_QUERY_TERMS} analyzed terms" + ) + result, fallback = await self._execute( + scope_id, + request.query, + limit=request.limit, + admission=request.admission.as_floor(), + artifact_request=request, + plan=plan, + embedding_model=embedding, + execution_context=execution_context, + ) + if result.artifacts is None: + if self._get is None: + raise ArtifactSearchContractError( + self.family, "public Topic search did not materialize exact Artifacts" + ) + if len(result.matches) != len(result.hits): + raise ArtifactSearchContractError(self.family, "public Topic hits and matches must have the same count") + artifacts = [] + for match in result.matches: + try: + publication = await self._get(scope_id, match.artifact_ref) + except RepositoryNotFoundError as error: + raise ArtifactSearchContractError(self.family, "selected exact Artifact is missing") from error + artifacts.append(publication.topic) + result = result.model_copy(update={"artifacts": tuple(artifacts)}) + self._observe(result, fallback) + return result + + async def search_legacy( + self, + scope_id: str, + query: str, + /, + *, + limit: int, + admission: AdmissionFloor | None = None, + query_embedding: MemoryQueryEmbedding | None = None, + embedding_timeout_seconds: float | None = None, + allow_embedding: bool = True, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> TopicMemorySearchResult: + result, fallback = await self._execute( + scope_id, + query, + limit=limit, + admission=admission, + query_embedding=query_embedding, + embedding_timeout_seconds=embedding_timeout_seconds, + allow_embedding=allow_embedding, + embedding_model=self._embedding, + execution_context=execution_context, + ) + self._observe(result, fallback) + return result + + async def _execute( + self, + scope_id: str, + query: str, + *, + limit: int, + admission: AdmissionFloor | None, + embedding_model: EmbeddingModel | None, + query_embedding: MemoryQueryEmbedding | None = None, + embedding_timeout_seconds: float | None = None, + allow_embedding: bool = True, + artifact_request: TopicArtifactSearchRequest | None = None, + plan: TopicSearchPlan | None = None, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> tuple[TopicMemorySearchResult, bool]: + embedding = embedding_model if allow_embedding and (plan is None or plan.mode != "fts") else None + keywords: dict[str, Any] = {"limit": limit} + if admission is not None: + keywords["admission"] = admission + if artifact_request is not None: + keywords["artifact_request"] = artifact_request + if execution_context is not None: + keywords["execution_context"] = execution_context + if ( + embedding is not None + and self._browse is not None + and (execution_context is None or (execution_context.access is None and execution_context.trusted_local)) + and not await self._browse(scope_id, limit=1, after=None) + ): + if artifact_request is not None and plan is not None and not plan.fallback_allowed: + return TopicMemorySearchResult(mode=plan.mode, artifacts=()), False + embedding = None + if embedding is None: + result = await self._search(scope_id, query, mode="fts", **keywords) + return result.model_copy(update={"embedding_calls": 0}), False + mode = "hybrid" if plan is None else plan.mode + if query_embedding is not None and query_embedding.embedding_profile == embedding.profile: + result = await self._search( + scope_id, + query, + mode=mode, + query_vector=query_embedding.query_vector, + embedding_profile=query_embedding.embedding_profile, + **keywords, + ) + return result.model_copy(update={"query_embedding": query_embedding, "embedding_calls": 0}), False + embedded = await self._embed_for_search( + embedding, + query, + timeout_seconds=embedding_timeout_seconds, + allow_fallback=plan is None or plan.fallback_allowed, + ) + if embedded is None: + result = await self._search(scope_id, query, mode="fts", **keywords) + return result.model_copy(update={"embedding_calls": 1}), True + result = await self._search( + scope_id, + query, + mode=mode, + query_vector=embedded.vectors[0], + embedding_profile=embedding.profile, + **keywords, + ) + return result.model_copy( + update={ + "query_embedding": MemoryQueryEmbedding( + query_vector=tuple(embedded.vectors[0]), embedding_profile=embedding.profile + ), + "embedding_calls": 1, + } + ), False + + async def _embed_for_search( + self, + embedding: EmbeddingModel, + query: str, + *, + timeout_seconds: float | None, + allow_fallback: bool, + ) -> EmbeddingResult | None: + try: + async with asyncio.timeout(timeout_seconds): + embedded = await embed_query(embedding, (query,)) + if len(embedded.vectors) != 1: + raise InvalidInferenceOutputError("embed", "provider returned the wrong vector count") + except (InferenceUnavailableError, InferenceTimeoutError, TimeoutError) as error: + if not allow_fallback: + if isinstance(error, TimeoutError) and not isinstance(error, InferenceTimeoutError): + raise InferenceUnavailableError("embed", "embedding request timed out") from None + raise + self._log_fallback(error) + return None + return embedded + + @staticmethod + def _log_fallback(error: Exception) -> None: + log_safely( + logger, + logging.WARNING, + "Topic Memory search fell back to FTS", + extra={ + "event": "topic_memory.search.embedding_fallback", + "outcome": "fallback", + "mode": "fts", + "error_code": "inference_timeout" + if isinstance(error, (InferenceTimeoutError, TimeoutError)) + else "inference_unavailable", + "unit": "topic-memory", + }, + ) + + def _observe(self, result: TopicMemorySearchResult, fallback: bool) -> None: + if self._observer is not None: + try: + self._observer(result.mode, fallback) + except Exception as error: + log_safely( + logger, + logging.ERROR, + "Topic Memory search observation failed", + exc_info=error, + extra={ + "event": "topic_memory.search.observation_failed", + "outcome": "failure", + "unit": "topic-memory", + }, + ) + + +__all__ = ["TopicMemorySearcher"] diff --git a/src/powercontext/client/client.py b/src/powercontext/client/client.py index 12c04d3c4b..d6c5d3d5ba 100644 --- a/src/powercontext/client/client.py +++ b/src/powercontext/client/client.py @@ -176,6 +176,8 @@ ScopedStats, ScopePage, ScopeQueryField, + SearchArtifactsRequest, + SearchArtifactsResponse, SearchMemoryRequest, SearchMemoryResponse, SearchTopicMemoryRequest, @@ -304,6 +306,7 @@ REVOKE_ACCESS_BINDING, REVOKE_REMOTE_SKILL_TARGET, SCAN_EXTERNAL_SKILLS, + SEARCH_ARTIFACTS, SEARCH_MEMORY, SEARCH_TOPIC_MEMORY, SET_DEFAULT_SCOPE, @@ -915,6 +918,13 @@ async def search_topic_memory(self, request: SearchTopicMemoryRequest) -> Search return await self._request(SEARCH_TOPIC_MEMORY, request) + async def search_artifacts( + self, scope_id: str, family: str, request: SearchArtifactsRequest + ) -> SearchArtifactsResponse: + """Search a registered Artifact Family and retain complete exact revisions.""" + + return await self._request(SEARCH_ARTIFACTS, request, path_parameters={"scope_id": scope_id, "family": family}) + async def get_topic_memory(self, request: GetTopicMemoryRequest) -> TopicMemoryArtifact: """Read one exact immutable Topic Memory revision.""" diff --git a/src/powercontext/http/__init__.py b/src/powercontext/http/__init__.py index 0c7c1b269e..0067729732 100644 --- a/src/powercontext/http/__init__.py +++ b/src/powercontext/http/__init__.py @@ -51,6 +51,7 @@ ArtifactAddress, ArtifactCandidate, ArtifactCandidatePage, + ArtifactChannelScore, ArtifactCollectionItem, ArtifactCreated, ArtifactFamilyAccessCapability, @@ -61,6 +62,9 @@ ArtifactReference, ArtifactRevision, ArtifactRevisionPage, + ArtifactSearchFusion, + ArtifactSearchItem, + ArtifactSearchScores, ArtifactTagPage, ArtifactTagSet, ArtifactTagTarget, @@ -347,6 +351,8 @@ ScopePage, ScopeQueryField, ScopeSelection, + SearchArtifactsRequest, + SearchArtifactsResponse, SearchMemoryHit, SearchMemoryRequest, SearchMemoryResponse, @@ -450,6 +456,7 @@ "ArtifactAddress", "ArtifactCandidate", "ArtifactCandidatePage", + "ArtifactChannelScore", "ArtifactCollectionItem", "ArtifactCreated", "ArtifactFamilyAccessCapability", @@ -460,6 +467,9 @@ "ArtifactReference", "ArtifactRevision", "ArtifactRevisionPage", + "ArtifactSearchFusion", + "ArtifactSearchItem", + "ArtifactSearchScores", "ArtifactTagPage", "ArtifactTagSet", "ArtifactTagTarget", @@ -746,6 +756,8 @@ "ScopeQueryField", "ScopeSelection", "ScopedStats", + "SearchArtifactsRequest", + "SearchArtifactsResponse", "SearchMemoryHit", "SearchMemoryRequest", "SearchMemoryResponse", diff --git a/src/powercontext/http/_generated/models.py b/src/powercontext/http/_generated/models.py index 5b40ec854f..485ad65115 100644 --- a/src/powercontext/http/_generated/models.py +++ b/src/powercontext/http/_generated/models.py @@ -17,6 +17,7 @@ StrictFloat, StrictInt, StrictStr, + field_validator, model_validator, ) @@ -1768,6 +1769,170 @@ class ReviseMemoryEntryRequest(BaseModel): reason: Annotated[StrictStr | None, Field(max_length=512)] = None +class ArtifactSearchFusion(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + method: Annotated[StrictStr, Field(min_length=1, pattern=".*\\S.*")] + params: dict[str, Any] = {} + + @field_validator("*", mode="after") + @classmethod + def _reject_invalid_search_value(cls, value: Any) -> Any: + from math import isfinite + + from pydantic import ValidationError + from pydantic_core import InitErrorDetails + + if value is None: + raise ValueError("omit the field instead of sending null") # noqa: TRY003 + errors: list[InitErrorDetails] = [] + pending: list[tuple[Any, tuple[str | int, ...]]] = [(value, ())] + visited: set[int] = set() + while pending: + item, path = pending.pop() + if isinstance(item, float) and not isfinite(item): + errors.append({"type": "finite_number", "loc": path, "input": item}) + elif isinstance(item, (dict, list, tuple)) and id(item) not in visited: + visited.add(id(item)) + entries = item.items() if isinstance(item, dict) else enumerate(item) + pending.extend((nested, (*path, key)) for key, nested in entries) + if errors: + raise ValidationError.from_exception_data(cls.__name__, errors) + return value + + +class SearchArtifactsRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + query: Annotated[StrictStr, Field(max_length=8192, min_length=1, pattern=".*\\S.*")] + limit: Annotated[StrictInt, Field(ge=1, le=200)] = 10 + mode: Annotated[StrictStr | None, Field(min_length=1, pattern=".*\\S.*")] = None + filters: dict[str, Any] | None = None + admission: dict[str, Any] | None = None + fusion: ArtifactSearchFusion | None = None + min_score: Annotated[StrictFloat | None, Field(ge=0.0, le=1.0)] = None + include_scores: StrictBool = False + rerank: dict[str, Any] | None = None + + @field_validator("query", mode="before") + @classmethod + def _trim_search_query(cls, value: Any) -> Any: + return value.strip() if isinstance(value, str) else value + + @field_validator("*", mode="after") + @classmethod + def _reject_invalid_search_value(cls, value: Any) -> Any: + from math import isfinite + + from pydantic import ValidationError + from pydantic_core import InitErrorDetails + + if value is None: + raise ValueError("omit the field instead of sending null") # noqa: TRY003 + errors: list[InitErrorDetails] = [] + pending: list[tuple[Any, tuple[str | int, ...]]] = [(value, ())] + visited: set[int] = set() + while pending: + item, path = pending.pop() + if isinstance(item, float) and not isfinite(item): + errors.append({"type": "finite_number", "loc": path, "input": item}) + elif isinstance(item, (dict, list, tuple)) and id(item) not in visited: + visited.add(id(item)) + entries = item.items() if isinstance(item, dict) else enumerate(item) + pending.extend((nested, (*path, key)) for key, nested in entries) + if errors: + raise ValidationError.from_exception_data(cls.__name__, errors) + return value + + +class ArtifactChannelScore(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + raw: StrictFloat + metric: Annotated[StrictStr, Field(min_length=1, pattern=".*\\S.*")] + higher_is_better: StrictBool + + @field_validator("*", mode="after") + @classmethod + def _reject_invalid_search_value(cls, value: Any) -> Any: + from math import isfinite + + from pydantic import ValidationError + from pydantic_core import InitErrorDetails + + if value is None: + raise ValueError("omit the field instead of sending null") # noqa: TRY003 + errors: list[InitErrorDetails] = [] + pending: list[tuple[Any, tuple[str | int, ...]]] = [(value, ())] + visited: set[int] = set() + while pending: + item, path = pending.pop() + if isinstance(item, float) and not isfinite(item): + errors.append({"type": "finite_number", "loc": path, "input": item}) + elif isinstance(item, (dict, list, tuple)) and id(item) not in visited: + visited.add(id(item)) + entries = item.items() if isinstance(item, dict) else enumerate(item) + pending.extend((nested, (*path, key)) for key, nested in entries) + if errors: + raise ValidationError.from_exception_data(cls.__name__, errors) + return value + + +class ArtifactSearchScores(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + retrieval: Annotated[StrictFloat, Field(ge=0.0, le=1.0)] + channels: dict[str, ArtifactChannelScore] + + @field_validator("*", mode="after") + @classmethod + def _reject_invalid_search_value(cls, value: Any) -> Any: + from math import isfinite + + from pydantic import ValidationError + from pydantic_core import InitErrorDetails + + if value is None: + raise ValueError("omit the field instead of sending null") # noqa: TRY003 + errors: list[InitErrorDetails] = [] + pending: list[tuple[Any, tuple[str | int, ...]]] = [(value, ())] + visited: set[int] = set() + while pending: + item, path = pending.pop() + if isinstance(item, float) and not isfinite(item): + errors.append({"type": "finite_number", "loc": path, "input": item}) + elif isinstance(item, (dict, list, tuple)) and id(item) not in visited: + visited.add(id(item)) + entries = item.items() if isinstance(item, dict) else enumerate(item) + pending.extend((nested, (*path, key)) for key, nested in entries) + if errors: + raise ValidationError.from_exception_data(cls.__name__, errors) + return value + + +class ArtifactSearchItem(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + family: Annotated[StrictStr, Field(min_length=1)] + artifact_id: Annotated[StrictStr, Field(max_length=128, min_length=1)] + revision: Annotated[StrictInt, Field(ge=1)] + content: dict[str, Any] + lineage: dict[str, Any] + scores: ArtifactSearchScores | None = None + + +class SearchArtifactsResponse(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + results: list[ArtifactSearchItem] + + class SearchTopicMemoryRequest(BaseModel): model_config = ConfigDict( extra="forbid", diff --git a/src/powercontext/http/_generated/operations.py b/src/powercontext/http/_generated/operations.py index 31c7e617e0..96162041ee 100644 --- a/src/powercontext/http/_generated/operations.py +++ b/src/powercontext/http/_generated/operations.py @@ -155,6 +155,8 @@ ScopeDescriptor, ScopedStats, ScopePage, + SearchArtifactsRequest, + SearchArtifactsResponse, SearchMemoryRequest, SearchMemoryResponse, SearchTopicMemoryRequest, @@ -1148,6 +1150,30 @@ class AccessRequirement(BaseModel): ), ) +SEARCH_ARTIFACTS = Operation[SearchArtifactsRequest, SearchArtifactsResponse]( + method="POST", + path="/v1/scopes/{scope_id}/artifacts/{family}/search", + operation_id="search_artifacts", + request_type=SearchArtifactsRequest, + request_location="body", + path_parameters=("scope_id", "family"), + response_type=SearchArtifactsResponse, + success_status=200, + summary="Search one Artifact Family in a Scope", + tags=("artifacts",), + scope_mode="none", + responses={ + 200: {"description": "Complete matching Artifact revisions in retrieval order."}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + 500: {"$ref": "#/components/responses/InternalError"}, + }, + access=AccessRequirement(action="scope.read", resource="scope", scope_id_field="scope_id", resolver="request"), +) + SEARCH_TOPIC_MEMORY = Operation[SearchTopicMemoryRequest, SearchTopicMemoryResponse]( method="POST", path="/v1/topic-memory/search", diff --git a/src/powercontext/http/_generated/schema.py b/src/powercontext/http/_generated/schema.py index 5ddf027b03..342808089c 100644 --- a/src/powercontext/http/_generated/schema.py +++ b/src/powercontext/http/_generated/schema.py @@ -1436,6 +1436,72 @@ "x-powercontext-scope-mode": "current", } }, + "/v1/scopes/{scope_id}/artifacts/{family}/search": { + "post": { + "tags": ["artifacts"], + "summary": "Search one Artifact Family in a Scope", + "description": "Search " + "registered " + "Artifact " + "Families " + "using " + "their " + "supported " + "retrieval " + "controls. " + "Results " + "contain " + "complete " + "exact " + "Artifact " + "revisions. " + "Score " + "metadata " + "is " + "returned " + "only when " + "requested; " + "min_score " + "compares " + "normalized " + "retrieval " + "scores.", + "operationId": "search_artifacts", + "x-powercontext-access": { + "action": "scope.read", + "resource": {"type": "scope", "scope-id-from": "scope_id"}, + }, + "parameters": [ + { + "name": "scope_id", + "in": "path", + "required": True, + "schema": {"type": "string", "minLength": 1, "maxLength": 256, "pattern": ".*\\S.*"}, + }, + {"name": "family", "in": "path", "required": True, "schema": {"type": "string", "minLength": 1}}, + ], + "requestBody": { + "required": True, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/SearchArtifactsRequest"}} + }, + }, + "responses": { + "200": { + "description": "Complete matching Artifact revisions in retrieval order.", + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/SearchArtifactsResponse"}} + }, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + "500": {"$ref": "#/components/responses/InternalError"}, + }, + } + }, "/v1/topic-memory/search": { "post": { "tags": ["topic-memory"], @@ -8621,6 +8687,78 @@ "type": "object", "required": ["artifact", "title", "summary", "snippet", "score", "matched_by"], }, + "ArtifactSearchFusion": { + "properties": { + "method": {"type": "string", "minLength": 1, "pattern": ".*\\S.*"}, + "params": {"additionalProperties": True, "type": "object", "default": {}}, + }, + "additionalProperties": False, + "type": "object", + "required": ["method"], + "x-powercontext-artifact-search-validation": "fusion", + }, + "SearchArtifactsRequest": { + "properties": { + "query": {"type": "string", "maxLength": 8192, "minLength": 1, "pattern": ".*\\S.*"}, + "limit": {"type": "integer", "maximum": 200.0, "minimum": 1.0, "default": 10}, + "mode": {"type": "string", "minLength": 1, "pattern": ".*\\S.*"}, + "filters": {"additionalProperties": True, "type": "object"}, + "admission": {"additionalProperties": True, "type": "object"}, + "fusion": {"$ref": "#/components/schemas/ArtifactSearchFusion"}, + "min_score": {"type": "number", "maximum": 1.0, "minimum": 0.0}, + "include_scores": {"type": "boolean", "default": False}, + "rerank": {"additionalProperties": True, "type": "object"}, + }, + "additionalProperties": False, + "type": "object", + "required": ["query"], + "x-powercontext-artifact-search-validation": "request", + }, + "ArtifactChannelScore": { + "properties": { + "raw": {"type": "number"}, + "metric": {"type": "string", "minLength": 1, "pattern": ".*\\S.*"}, + "higher_is_better": {"type": "boolean"}, + }, + "additionalProperties": False, + "type": "object", + "required": ["raw", "metric", "higher_is_better"], + "x-powercontext-artifact-search-validation": "score", + }, + "ArtifactSearchScores": { + "properties": { + "retrieval": {"type": "number", "maximum": 1.0, "minimum": 0.0}, + "channels": { + "additionalProperties": {"$ref": "#/components/schemas/ArtifactChannelScore"}, + "type": "object", + }, + }, + "additionalProperties": False, + "type": "object", + "required": ["retrieval", "channels"], + "x-powercontext-artifact-search-validation": "score", + }, + "ArtifactSearchItem": { + "properties": { + "family": {"type": "string", "minLength": 1}, + "artifact_id": {"type": "string", "maxLength": 128, "minLength": 1}, + "revision": {"type": "integer", "minimum": 1.0}, + "content": {"additionalProperties": True, "type": "object"}, + "lineage": {"additionalProperties": True, "type": "object"}, + "scores": {"$ref": "#/components/schemas/ArtifactSearchScores"}, + }, + "additionalProperties": False, + "type": "object", + "required": ["family", "artifact_id", "revision", "content", "lineage"], + }, + "SearchArtifactsResponse": { + "properties": { + "results": {"items": {"$ref": "#/components/schemas/ArtifactSearchItem"}, "type": "array"} + }, + "additionalProperties": False, + "type": "object", + "required": ["results"], + }, "SearchTopicMemoryRequest": { "properties": { "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, diff --git a/src/powercontext/server/app.py b/src/powercontext/server/app.py index 814f5dcc41..23288c35c7 100644 --- a/src/powercontext/server/app.py +++ b/src/powercontext/server/app.py @@ -49,6 +49,11 @@ from powercontext._logging import log_safely from powercontext.artifacts import ArtifactAddress, ArtifactRef +from powercontext.artifacts.search import ( + ArtifactSearchExecutionContext, + ArtifactSearchFamilyNotFound, + ArtifactSearchUnsupported, +) from powercontext.builtin.artifacts.experience import Experience from powercontext.builtin.artifacts.handoff import ( HandoffCitation, @@ -303,6 +308,7 @@ SubmitSourceObservation as RuntimeSubmitSourceObservation, ) from powercontext.builtin.runtime.application import BuiltinRuntime, PromptApplication +from powercontext.builtin.runtime.skill_search import search_skill_library from powercontext.builtin.scope import ( ScopeApplication, ScopeBindingNotFoundError, @@ -541,6 +547,7 @@ ScopePage, ScopeQueryField, ScopeSelection, + SearchArtifactsRequest, SearchMemoryRequest, SearchMemoryResponse, SearchTopicMemoryRequest, @@ -746,6 +753,7 @@ REVOKE_ACCESS_BINDING, REVOKE_REMOTE_SKILL_TARGET, SCAN_EXTERNAL_SKILLS, + SEARCH_ARTIFACTS, SEARCH_MEMORY, SEARCH_TOPIC_MEMORY, SET_DEFAULT_SCOPE, @@ -1189,7 +1197,13 @@ def for_scope(self, scope_id: str, /) -> _ScopedMemoryApplication: ... class _ScopedTopicMemoryApplication(Protocol): - async def search(self, request: RuntimeSearchTopicMemoryRequest, /) -> TopicMemorySearchResult: ... + async def search( + self, + request: RuntimeSearchTopicMemoryRequest, + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> TopicMemorySearchResult: ... async def get(self, request: RuntimeGetTopicMemoryRequest, /) -> PublishedTopicMemory: ... @@ -1418,6 +1432,7 @@ async def unexpected_error(request: Request, error: Exception) -> JSONResponse: _add_route(app, CAPTURE_CONTENT_SOURCE, capture_content_source) _add_route(app, FLUSH_TOPIC_MEMORY, flush_topic_memory) _add_route(app, SEARCH_TOPIC_MEMORY, search_topic_memory) + _add_route(app, SEARCH_ARTIFACTS, search_artifacts) _add_route(app, GET_TOPIC_MEMORY, get_topic_memory) _add_route(app, REGISTER_SOURCE_DEFINITION, register_source_definition) _add_route(app, GET_CONNECTOR_CHECKPOINT, get_connector_checkpoint) @@ -2447,13 +2462,48 @@ async def flush_topic_memory( async def search_topic_memory( request: SearchTopicMemoryRequest, application: Annotated[ServerApplication, Depends(_require_application)], + http_request: Request, ) -> SearchTopicMemoryResponse: + access = access_control_for_mode(http_request.app.state.access_control, mode=http_request.app.state.access_mode) + execution_context = ArtifactSearchExecutionContext( + principal=_require_principal() if access is not None else None, + access=access, + audit=_access_audit_context(SEARCH_TOPIC_MEMORY.operation_id), + trusted_local=access is None and http_request.app.state.access_mode == "disabled", + ) result = await application.topic_memory.for_scope(request.scope_id).search( - mapping.topic_memory_search_request(request) + mapping.topic_memory_search_request(request), + **({} if access is None else {"execution_context": execution_context}), ) return mapping.topic_memory_search_response(result) +async def search_artifacts( + scope_id: _ScopePathId, + family: Annotated[str, Path(min_length=1)], + request: SearchArtifactsRequest, + application: Annotated[ServerApplication, Depends(_require_application)], + http_request: Request, +) -> Response: + artifacts = getattr(application, "artifacts", None) + if artifacts is None: + raise ArtifactSearchUnsupported(family, field="family") + access = access_control_for_mode(http_request.app.state.access_control, mode=http_request.app.state.access_mode) + execution_context = ArtifactSearchExecutionContext( + principal=_require_principal() if access is not None else None, + access=access, + audit=_access_audit_context(SEARCH_ARTIFACTS.operation_id), + trusted_local=access is None and http_request.app.state.access_mode == "disabled", + ) + outcome = await artifacts.for_scope(scope_id).search( + family, + request.model_dump(mode="json", exclude_unset=True), + execution_context=execution_context, + ) + result = mapping.artifact_search_response(outcome, include_scores=request.include_scores) + return JSONResponse(content=result.model_dump(mode="json", exclude_unset=True)) + + async def get_topic_memory( request: GetTopicMemoryRequest, application: Annotated[ServerApplication, Depends(_require_application)], @@ -3337,29 +3387,12 @@ async def list_managed_skills( application: Annotated[ServerApplication, Depends(_require_application)], ) -> ListManagedSkillsResponse: scoped = application.skill.for_scope(request.scope_id) - values: list[tuple[Skill, ArtifactGovernance]] = [] - query = "" if request.query is None else request.query.strip() - if query: - for hit in await scoped.search(query, request.limit): - skill = await scoped.get(RuntimeGetSkillRequest(artifact=hit.artifact_ref)) - values.append((skill, await scoped.governance(skill.artifact_id))) - else: - values.extend(await scoped.list(include_deprecated=request.include_deprecated, limit=request.limit)) - if query and request.include_deprecated: - seen = {skill.artifact_id for skill, _governance in values} - for skill, governance in await scoped.list(include_deprecated=True, limit=request.limit): - search_text = "\n".join(( - skill.content.name, - skill.content.description, - skill.content.instructions, - *skill.content.metadata.values(), - )) - if ( - governance.lifecycle_state is ArtifactLifecycleState.DEPRECATED - and skill.artifact_id not in seen - and query.casefold() in search_text.casefold() - ): - values.append((skill, governance)) + values = await search_skill_library( + scoped, + "" if request.query is None else request.query, + request.limit, + include_deprecated=request.include_deprecated, + ) return ListManagedSkillsResponse( skills=[mapping.managed_skill_library_entry(skill, governance) for skill, governance in values[: request.limit]] ) @@ -4356,6 +4389,7 @@ def _add_route( # Collection permission allows identity discovery, but content remains unavailable # until every committed identity has its immutable owner relation. _COLLECTION_CONTENT_OPERATIONS = frozenset({ + "search_artifacts", "search_memory", "list_memory_entries", "get_memory_capacity", @@ -5196,6 +5230,15 @@ def _set_error_headers(response: Response, error: Exception) -> None: def _map_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | None]: + if isinstance(error, ArtifactSearchFamilyNotFound): + return status.HTTP_404_NOT_FOUND, "artifact_family_not_found", "The Artifact Family was not found.", None + if isinstance(error, ArtifactSearchUnsupported): + return ( + status.HTTP_422_UNPROCESSABLE_CONTENT, + "artifact_search_not_supported", + "The Artifact search capability is unavailable.", + None if error.field is None else {"field": error.field}, + ) if isinstance(error, CodeError): return error.status, error.code, "The code query could not be completed.", None access_error = _map_access_error(error) diff --git a/src/powercontext/server/authz/casbin.py b/src/powercontext/server/authz/casbin.py index 27360ffec2..9bc4b8f9f8 100644 --- a/src/powercontext/server/authz/casbin.py +++ b/src/powercontext/server/authz/casbin.py @@ -72,6 +72,11 @@ def __init__( self._deployment_id = deployment_id self._clock = clock or (lambda: datetime.now(UTC)) + def with_repository(self, repository: AccessRepository) -> CasbinAuthorizationProvider: + """Preserve the fixed policy while binding reads to the caller's transaction.""" + + return CasbinAuthorizationProvider(repository, deployment_id=self._deployment_id, clock=self._clock) + async def check(self, request: AccessRequest, /) -> AccessDecision: return (await self.check_batch((request,)))[0] diff --git a/src/powercontext/server/authz/service.py b/src/powercontext/server/authz/service.py index a4b0155fb6..11a56d4f88 100644 --- a/src/powercontext/server/authz/service.py +++ b/src/powercontext/server/authz/service.py @@ -621,6 +621,42 @@ def with_connection(self, connection): static_scope_principal=self._static_scope_principal, ) + async def require_scope_read(self, principal, scope_id: str, *, connection, context: AccessAuditContext): + """Use a local policy snapshot when available, preserving external point decisions.""" + + access = self + provider = self.provider + canonical = type(provider) is BuiltinAuthorizationProvider + if not canonical and type(provider).__module__ == "powercontext.server.authz.casbin": + from powercontext.server.authz.casbin import CasbinAuthorizationProvider + + canonical = type(provider) is CasbinAuthorizationProvider + if canonical: + from powercontext.server.authz.repository import RelationalAccessRepository + + repository = provider._repository + # Binding either a Provider or Repository subclass could discard its custom rules. + if type(repository) is RelationalAccessRepository: + try: + bound_repository = repository.with_connection(connection) + except AccessUnavailableError as error: + if error.code != "transactional_relationships_unavailable": + raise + # A separate policy database still supports its original point decision. + else: + access = AccessControlService( + provider.with_repository(bound_repository), + relationships=self.relationships, + audit=self.audit, + deployment_id=self.deployment_id, + provider_capabilities=self.provider_capabilities, + clock=self._clock, + cursor_secret=self._cursor_secret, + static_scope_principal=self._static_scope_principal, + ) + access._deferred_decisions = self._deferred_decisions + return await access.require(principal, AccessAction.SCOPE_READ, ResourceRef.scope(scope_id), context=context) + async def bootstrap_subject_scope(self, connection, principal, scope_id, *, context): """Grant only a newly created ordinary Scope, in the Source transaction.""" from powercontext.server.authz.repository import RelationalAccessRepository diff --git a/src/powercontext/server/mapping.py b/src/powercontext/server/mapping.py index bd13026be5..2062017137 100644 --- a/src/powercontext/server/mapping.py +++ b/src/powercontext/server/mapping.py @@ -21,6 +21,7 @@ from pydantic import ValidationError from powercontext.artifacts import ArtifactRef +from powercontext.artifacts.search import ArtifactSearchOutcome from powercontext.builtin.artifacts.experience import ( Experience, ExperienceContent, @@ -254,6 +255,7 @@ ReviseMemoryEntryRequest, ScanExternalSkillsResponse, ScopedStats, + SearchArtifactsResponse, SearchMemoryHit, SearchMemoryRequest, SearchMemoryResponse, @@ -353,6 +355,34 @@ from powercontext.sources import SourceRef +def artifact_search_response(outcome: ArtifactSearchOutcome, /, *, include_scores: bool) -> SearchArtifactsResponse: + """Serialize actual Artifact content while retaining the Family's final match order.""" + + if outcome.artifacts is None: + raise ValueError("public Artifact search requires complete Artifacts") # noqa: TRY003 + results: list[dict[str, Any]] = [] + for match, artifact in zip(outcome.matches, outcome.artifacts, strict=True): + item: dict[str, Any] = { + "family": artifact.family, + "artifact_id": artifact.artifact_id, + "revision": artifact.revision, + "content": artifact.content.model_dump(mode="json", by_alias=True, serialize_as_any=True), + "lineage": artifact.lineage.model_dump(mode="json", by_alias=True, serialize_as_any=True), + } + if include_scores: + if match.channel_scores is None: + raise ValueError("requested Artifact channel scores were not collected") # noqa: TRY003 + item["scores"] = { + "retrieval": match.retrieval_score, + "channels": { + name: {"raw": value.raw, "metric": value.metric, "higher_is_better": value.higher_is_better} + for name, value in match.channel_scores.items() + }, + } + results.append(item) + return SearchArtifactsResponse.model_validate({"results": results}) + + def capture_request(value: CaptureContentSourceRequest) -> CaptureSource: return CaptureSource( source_id=value.source_id, diff --git a/tests/builtin/artifacts/test_lexical_search_requests.py b/tests/builtin/artifacts/test_lexical_search_requests.py new file mode 100644 index 0000000000..6b48c24a8d --- /dev/null +++ b/tests/builtin/artifacts/test_lexical_search_requests.py @@ -0,0 +1,93 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import math + +import pytest +from pydantic import ValidationError + +from powercontext.builtin.artifacts.experience import search as experience_search +from powercontext.builtin.artifacts.skill import search as skill_search + + +@pytest.mark.parametrize( + "module,name", [(experience_search, "ExperienceSearchRequest"), (skill_search, "SkillSearchRequest")] +) +def test_lexical_search_request_defaults_and_json_numbers(module, name: str) -> None: + request_type = getattr(module, name) + request = request_type.model_validate({"query": " contract client ", "filters": {}, "admission": {}}) + + assert request.query == "contract client" + assert request.limit == 10 + assert request.include_scores is False + assert request.mode is None + assert request.min_score is None + assert request.admission.lexical_coverage == 0.25 + assert request.admission.lexical_min_matched_terms == 2 + assert request.model_fields_set == {"query", "filters", "admission"} + configured = request_type.model_validate_json( + '{"query":"client","mode":"text","admission":{"lexical_coverage":0,"lexical_min_matched_terms":1},"min_score":1}' + ) + assert configured.admission.lexical_coverage == 0 + assert configured.min_score == 1 + + +@pytest.mark.parametrize( + "module,name", [(experience_search, "ExperienceSearchRequest"), (skill_search, "SkillSearchRequest")] +) +@pytest.mark.parametrize( + "fields", + [ + {"query": " "}, + {"limit": True}, + {"limit": "10"}, + {"limit": 201}, + {"include_scores": 1}, + {"filters": {"scope": "other"}}, + {"filters": []}, + {"admission": {"min_semantic_similarity": 0.2}}, + {"admission": {"lexical_coverage": True}}, + {"admission": {"lexical_coverage": "0.5"}}, + {"admission": {"lexical_coverage": math.inf}}, + {"admission": {"lexical_min_matched_terms": True}}, + {"admission": {"lexical_min_matched_terms": 0}}, + {"mode": "vector"}, + {"fusion": {}}, + {"rerank": {}}, + {"min_score": True}, + {"min_score": "0.5"}, + {"min_score": math.nan}, + {"min_score": -0.1}, + {"min_score": 1.1}, + {"unknown": 1}, + *({field: None} for field in ("query", "limit", "include_scores", "mode", "filters", "admission", "min_score")), + {"admission": {"lexical_coverage": None}}, + ], +) +def test_lexical_search_request_rejects_invalid_fields(module, name: str, fields: dict[str, object]) -> None: + with pytest.raises(ValidationError): + getattr(module, name).model_validate({"query": "client", **fields}) + + +@pytest.mark.parametrize( + "module,name,maximum", + [(experience_search, "ExperienceSearchRequest", 8192), (skill_search, "SkillSearchRequest", 2000)], +) +def test_lexical_search_request_owns_query_length(module, name: str, maximum: int) -> None: + request_type = getattr(module, name) + assert request_type(query="x" * maximum).query == "x" * maximum + with pytest.raises(ValidationError): + request_type(query="x" * (maximum + 1)) diff --git a/tests/builtin/artifacts/topic_memory/test_search.py b/tests/builtin/artifacts/topic_memory/test_search.py new file mode 100644 index 0000000000..ba5e2e6420 --- /dev/null +++ b/tests/builtin/artifacts/topic_memory/test_search.py @@ -0,0 +1,166 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public Topic request and configurable fusion behavior.""" + +import json + +import pytest +from pydantic import ValidationError + +from powercontext.artifacts import ArtifactRef +from powercontext.builtin.artifacts.topic_memory import TopicMemoryChannelHit, TopicMemorySearchChannels + + +def test_topic_request_has_family_defaults_and_preserves_explicit_fields(): + from powercontext.builtin.artifacts.topic_memory import TopicArtifactSearchRequest + + request = TopicArtifactSearchRequest.model_validate_json( + '{"query":" needle ","admission":{"lexical_coverage":0}}' + ) + assert request.query == "needle" + assert request.mode is None + assert request.limit == 10 + assert request.admission.min_semantic_similarity == 0.3 + assert request.admission.model_fields_set == {"lexical_coverage"} + assert request.model_fields_set == {"query", "admission"} + + +@pytest.mark.parametrize( + "fields", + [ + {"mode": None}, + {"min_score": None}, + {"fusion": None}, + {"filters": {"tag": "x"}}, + {"mode": "fts"}, + {"mode": "auto"}, + {"limit": 21}, + {"rerank": {}}, + {"admission": {"min_semantic_similarity": True}}, + {"admission": {"lexical_min_matched_terms": 1.0}}, + {"min_score": "0.5"}, + ], +) +def test_topic_request_rejects_unsupported_or_non_strict_fields(fields): + from powercontext.builtin.artifacts.topic_memory import TopicArtifactSearchRequest + + with pytest.raises(ValidationError): + TopicArtifactSearchRequest.model_validate_json(json.dumps({"query": "needle", **fields}), strict=True) + + +def _hit(channel, artifact_id="topic", *, distance=None, raw_score=-2.0, metric="sqlite_bm25"): + return TopicMemoryChannelHit( + artifact_ref=ArtifactRef(family="topic-memory", artifact_id=artifact_id, revision=1), + title="needle", + summary="needle evidence", + channel=channel, + chunk_text="needle detail" if channel.startswith("detail") else None, + chunk_ordinal=0 if channel.startswith("detail") else None, + chunk_start=0 if channel.startswith("detail") else None, + distance=distance, + raw_score=raw_score, + metric=metric, + ) + + +def test_topic_fusion_weights_scores_and_metadata_keep_channel_evidence(): + from powercontext.artifacts.fusion import RrfParameters + from powercontext.builtin.artifacts.topic_memory.fusion import _fuse_topic_memory_rankings + + channels = TopicMemorySearchChannels( + topic_fts=(_hit("topic_fts"),), + detail_fts=(_hit("detail_fts", raw_score=-3),), + topic_vector=(_hit("topic_vector", distance=0.1),), + ) + result = _fuse_topic_memory_rankings( + "needle", channels, 10, fusion=RrfParameters(weights={"topic_fts": 0}), include_scores=True + ) + assert result.hits[0].retrieval_score == pytest.approx(2 / 3) + assert result.hits[0].score == pytest.approx(200 / 3) + assert result.hits[0].matched_by == ("topic_fts", "topic_vector", "detail_fts") + assert result.hits[0].snippet == "needle detail" + metadata = result.hits[0].channel_scores + assert metadata is not None + assert metadata["topic_fts"].raw == -2 + assert metadata["topic_fts"].higher_is_better is False + assert metadata["topic_vector"].raw == 0.1 + assert metadata["topic_vector"].metric == "l2_distance" + assert "detail_vector" not in metadata + + +def test_topic_fusion_threshold_does_not_fill_with_lower_candidates(): + from powercontext.artifacts.fusion import RrfParameters + from powercontext.builtin.artifacts.topic_memory.fusion import _fuse_topic_memory_rankings + + channels = TopicMemorySearchChannels(topic_fts=(_hit("topic_fts", "first"), _hit("topic_fts", "second"))) + result = _fuse_topic_memory_rankings("needle", channels, 10, mode="fts", fusion=RrfParameters(), min_score=0.5) + assert [hit.artifact_ref.artifact_id for hit in result.hits] == ["first"] + + +def test_topic_fusion_keeps_duplicate_rank_gaps_and_score_toggle_order(): + from powercontext.builtin.artifacts.topic_memory.fusion import _fuse_topic_memory_rankings + + channels = TopicMemorySearchChannels( + topic_fts=(_hit("topic_fts", "first"), _hit("topic_fts", "first"), _hit("topic_fts", "second")) + ) + plain = _fuse_topic_memory_rankings("needle", channels, 10, mode="fts") + scored = _fuse_topic_memory_rankings("needle", channels, 10, mode="fts", include_scores=True) + assert [hit.artifact_ref for hit in plain.hits] == [hit.artifact_ref for hit in scored.hits] + assert plain.hits[1].retrieval_score == pytest.approx(61 / 126) + assert plain.hits[0].channel_scores is None + + +@pytest.mark.parametrize("weights", [{"unknown": 0}, {"topic_vector": 0}, {"topic_fts": 0, "detail_fts": 0}]) +def test_topic_fusion_rejects_inactive_unknown_and_all_zero_weights(weights): + from powercontext.artifacts.fusion import RrfParameters + from powercontext.builtin.artifacts.topic_memory.fusion import _fuse_topic_memory_rankings + + with pytest.raises(ValidationError): + _fuse_topic_memory_rankings( + "needle", TopicMemorySearchChannels(), 10, mode="fts", fusion=RrfParameters(weights=weights) + ) + + +def test_topic_fusion_custom_rank_constant_and_zero_weight_phantoms(): + from powercontext.artifacts.fusion import RrfParameters + from powercontext.builtin.artifacts.topic_memory.fusion import _fuse_topic_memory_rankings + + channels = TopicMemorySearchChannels( + topic_fts=(_hit("topic_fts", "phantom"),), + detail_fts=(_hit("detail_fts", "first"), _hit("detail_fts", "first"), _hit("detail_fts", "second")), + ) + result = _fuse_topic_memory_rankings( + "needle", channels, 10, mode="fts", fusion=RrfParameters(rank_constant=1, weights={"topic_fts": 0}) + ) + assert [hit.artifact_ref.artifact_id for hit in result.hits] == ["first", "second"] + assert [hit.retrieval_score for hit in result.hits] == [1.0, 0.5] + + +def test_topic_admission_rejection_removes_only_that_channel_contribution(): + from powercontext.builtin.artifacts.topic_memory.fusion import _fuse_topic_memory_rankings + + lexical = _hit("topic_fts").model_copy(update={"title": "alpha", "summary": "unrelated"}) + semantic = _hit("topic_vector", distance=0.1) + outcome = _fuse_topic_memory_rankings( + "alpha beta gamma", + TopicMemorySearchChannels(topic_fts=(lexical,), topic_vector=(semantic,)), + 10, + include_scores=True, + ) + assert outcome.hits[0].matched_by == ("topic_vector",) + assert outcome.hits[0].retrieval_score == 0.25 + assert outcome.hits[0].channel_scores is not None + assert set(outcome.hits[0].channel_scores) == {"topic_vector"} + assert outcome.rejected == 1 diff --git a/tests/builtin/persistence/test_database_snapshots.py b/tests/builtin/persistence/test_database_snapshots.py new file mode 100644 index 0000000000..3caf2cc339 --- /dev/null +++ b/tests/builtin/persistence/test_database_snapshots.py @@ -0,0 +1,174 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace +from typing import cast + +import pytest +from sqlalchemy import Column, Integer, MetaData, Table, insert, select, update +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine + +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.errors import PersistenceError +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile + +_TABLE = Table("snapshot_values", MetaData(), Column("value", Integer, nullable=False)) + + +def test_snapshot_transaction_keeps_sqlite_reads_at_the_same_revision(tmp_path) -> None: + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'snapshot.db'}") + async with SQLiteProfile.open(config, tables=(_TABLE,)) as profile: + async with profile.database.transaction() as writer: + await writer.execute(insert(_TABLE).values(value=1)) + async with profile.database.transaction(consistent_snapshot=True) as reader: + assert await reader.scalar(select(_TABLE.c.value)) == 1 + async with profile.database.transaction() as writer: + await writer.execute(update(_TABLE).values(value=2)) + assert await reader.scalar(select(_TABLE.c.value)) == 1 + async with profile.database.transaction() as reader: + assert await reader.scalar(select(_TABLE.c.value)) == 2 + + asyncio.run(scenario()) + + +def test_nested_snapshot_does_not_commit_the_shared_sqlite_write() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=(_TABLE,)) as profile: + with pytest.raises(RuntimeError, match="abort outer write"): + async with profile.database.transaction() as writer: + await writer.execute(insert(_TABLE).values(value=1)) + async with profile.database.transaction(consistent_snapshot=True) as reader: + assert await reader.scalar(select(_TABLE.c.value)) == 1 + assert await writer.scalar(select(_TABLE.c.value)) == 1 + raise RuntimeError("abort outer write") # noqa: TRY003 + async with profile.database.transaction() as reader: + assert (await reader.execute(select(_TABLE.c.value))).all() == [] + + asyncio.run(scenario()) + + +class _MySQLConnection: + """Expose the startup boundary without requiring an external backend. + + Actual SQL/isolation behavior is covered by the OceanBase E2E parameter. + These faults represent a driver failing after applying a one-shot setting. + """ + + dialect = SimpleNamespace(name="mysql") + + def __init__(self, *, fail_at: str | None = None, cancellation: bool = False) -> None: + self.fail_at = fail_at + self.cancellation = cancellation + self.statements: list[str] = [] + self.invalidated = False + self.reached = asyncio.Event() + + async def exec_driver_sql(self, statement: str) -> None: + self.statements.append(statement) + if self.fail_at is not None and statement.startswith(self.fail_at): + if self.cancellation: + self.reached.set() + await asyncio.Event().wait() + raise RuntimeError("snapshot setup failed") # noqa: TRY003 + + async def invalidate(self) -> None: + self.invalidated = True + + +class _MySQLEngine: + def __init__(self, connection: _MySQLConnection) -> None: + self.connection = connection + self.disposed = False + + @asynccontextmanager + async def begin(self): + yield cast(AsyncConnection, self.connection) + + async def dispose(self) -> None: + self.disposed = True + + +@pytest.mark.parametrize("fail_at", ["SET", "START"]) +@pytest.mark.parametrize("cancellation", [False, True]) +def test_failed_mysql_snapshot_setup_discards_the_connection(fail_at: str, cancellation: bool) -> None: + async def scenario() -> None: + connection = _MySQLConnection(fail_at=fail_at, cancellation=cancellation) + engine = _MySQLEngine(connection) + database = AsyncDatabase.own(cast(AsyncEngine, engine)) + + async def enter() -> None: + async with database.transaction(consistent_snapshot=True): + pytest.fail("failed setup must not enter the transaction body") + + if cancellation: + task = asyncio.create_task(enter()) + try: + await asyncio.wait_for(connection.reached.wait(), timeout=1) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + with pytest.raises(RuntimeError, match="snapshot setup failed"): + await enter() + assert connection.invalidated, "an unconsumed next-transaction setting must not return to the pool" + # Failure/cancellation must release lifecycle accounting as well. + await asyncio.wait_for(database.close(), timeout=1) + assert engine.disposed + + asyncio.run(scenario()) + + +def test_mysql_snapshot_is_opt_in_and_does_not_change_the_session_default() -> None: + async def scenario() -> None: + connection = _MySQLConnection() + database = AsyncDatabase.attach(cast(AsyncEngine, _MySQLEngine(connection))) + async with database.transaction(): + pass + assert connection.statements == ["START TRANSACTION"] + connection.statements.clear() + async with database.transaction(consistent_snapshot=True): + pass + assert connection.statements == [ + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ", + "START TRANSACTION WITH CONSISTENT SNAPSHOT", + ] + connection.statements.clear() + async with database.transaction(): + pass + assert connection.statements == ["START TRANSACTION"] + assert not connection.invalidated + await database.close() + + asyncio.run(scenario()) + + +def test_snapshot_request_cannot_upgrade_a_borrowed_mysql_transaction() -> None: + async def scenario() -> None: + connection = _MySQLConnection() + database = AsyncDatabase.own(cast(AsyncEngine, _MySQLEngine(connection)), shared_connection=True) + async with database.transaction(): + with pytest.raises(PersistenceError, match="existing transaction"): + async with database.transaction(consistent_snapshot=True): + pytest.fail("a borrowed READ COMMITTED transaction cannot become a stable snapshot") + assert connection.statements == ["START TRANSACTION"] + assert not connection.invalidated, "the outer transaction still belongs to its caller" + await database.close() + + asyncio.run(scenario()) diff --git a/tests/builtin/persistence/test_experience_index.py b/tests/builtin/persistence/test_experience_index.py index e730078092..91b6f2f582 100644 --- a/tests/builtin/persistence/test_experience_index.py +++ b/tests/builtin/persistence/test_experience_index.py @@ -15,11 +15,13 @@ from __future__ import annotations import asyncio +import math import sqlite3 from types import SimpleNamespace from typing import cast from unittest.mock import AsyncMock +import pytest from sqlalchemy import select, text from sqlalchemy.dialects import mysql from sqlalchemy.ext.asyncio import AsyncConnection @@ -28,7 +30,12 @@ from powercontext.builtin.artifacts.experience import Experience, ExperienceContent from powercontext.builtin.artifacts.skill import Skill, SkillContent from powercontext.builtin.persistence.artifact_governance import ArtifactLifecycleState -from powercontext.builtin.persistence.experience_index import ensure_artifact_head_searchable_text +from powercontext.builtin.persistence.experience_index import ( + ensure_artifact_head_searchable_text, + experience_search_hits, + skill_search_hits, +) +from powercontext.builtin.persistence.oceanbase.experience_index import OceanBaseExperienceFTSIndex from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, BUILTIN_TABLES from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts @@ -146,6 +153,21 @@ async def scenario() -> None: assert first_outcome.admission is not None assert first_outcome.admission.admitted == 1 assert first_outcome.admission.retrieved >= 1 + scored_hit = first_outcome.hits[0] + assert scored_hit.retrieval_score is not None + assert scored_hit.channel_scores is not None + raw = scored_hit.channel_scores["text"] + assert raw.metric == "sqlite_bm25" + assert raw.raw < 0 + assert raw.higher_is_better is False + assert scored_hit.retrieval_score == -raw.raw / (1 - raw.raw) + async with contexts.database.transaction() as connection: + excluded = await contexts.experience_index.search( + connection, "project", "hamsterlegacy", 8, min_score=1, require_scores=True + ) + assert excluded.hits == () + assert excluded.admission is not None + assert excluded.admission.admitted == 1 assert await contexts.search_experience("other-project", "hamsterlegacy", 8) == () assert await contexts.search_experience("project", "situation outcome", 8) == () @@ -177,6 +199,9 @@ async def scenario() -> None: assert skill_approval.result_artifact is not None skill_hits = await contexts.search_skills("project", "regenerate client", 8) assert tuple(hit.artifact_ref for hit in skill_hits) == (skill_approval.result_artifact,) + assert skill_hits[0].retrieval_score is not None + assert skill_hits[0].channel_scores is not None + assert skill_hits[0].channel_scores["text"].metric == "sqlite_bm25" governance = await contexts.update_skill_lifecycle( "project", skill_approval.result_artifact.artifact_id, @@ -233,3 +258,142 @@ async def scenario() -> None: assert tuple(hit.artifact_ref for hit in rebuilt) == (replaced.result_artifact,) asyncio.run(scenario()) + + +def _experience_row(artifact_id: str, raw_score: float, metric: str, *, keyword: str = "client") -> dict[str, object]: + return { + "artifact_id": artifact_id, + "revision": 1, + "content": _experience(keyword, "Validate the generated contract.").model_dump_json().encode(), + "raw_score": raw_score, + "score_metric": metric, + } + + +@pytest.mark.parametrize("metric,raw", [("sqlite_bm25", -2.0), ("oceanbase_match", 2.0)]) +def test_experience_scores_normalize_backend_direction_and_preserve_raw(metric: str, raw: float) -> None: + outcome = experience_search_hits([_experience_row("one", raw, metric)], "client", 8, "project", require_scores=True) + + assert outcome.hits[0].retrieval_score == pytest.approx(2 / 3) + assert outcome.hits[0].channel_scores is not None + channel = outcome.hits[0].channel_scores["text"] + assert channel.raw == raw + assert channel.metric == metric + assert channel.higher_is_better is (metric == "oceanbase_match") + + +@pytest.mark.parametrize("raw", [0.0, 1e308]) +def test_experience_scores_accept_zero_and_large_finite_relevance(raw: float) -> None: + hit = experience_search_hits( + [_experience_row("one", raw, "oceanbase_match")], "client", 8, "project", require_scores=True + ).hits[0] + + assert hit.retrieval_score == (0 if raw == 0 else 1) + assert hit.retrieval_score is not None + assert math.isfinite(hit.retrieval_score) + + +@pytest.mark.parametrize( + "metric,raw,error_name", + [ + ("sqlite_bm25", 1.0, "InvalidSearchScore"), + ("oceanbase_match", -1.0, "InvalidSearchScore"), + ("oceanbase_match", math.inf, "InvalidSearchScore"), + ("oceanbase_match", math.nan, "InvalidSearchScore"), + ("unknown", 1.0, "UnsupportedScoreMetric"), + ], +) +def test_experience_scores_reject_invalid_relevance(metric: str, raw: float, error_name: str) -> None: + from powercontext.builtin.artifacts import search + + with pytest.raises(getattr(search, error_name)): + experience_search_hits([_experience_row("one", raw, metric)], "client", 8, "project", require_scores=True) + + +def test_experience_threshold_keeps_equal_scores_and_counts_lexical_admission() -> None: + rows = [ + _experience_row("rejected", math.nan, "oceanbase_match", keyword="unrelated"), + _experience_row("equal", 1.0, "oceanbase_match"), + _experience_row("below", 0.25, "oceanbase_match"), + ] + # The invalid score belongs to a lexically rejected row and is never normalized. + rows[0]["content"] = ( + ExperienceContent(situation="unrelated", action="ignore", outcome="absent", lesson="none") + .model_dump_json() + .encode() + ) + outcome = experience_search_hits(rows, "client", 3, "project", min_score=0.5, require_scores=True) + + assert tuple(hit.artifact_ref.artifact_id for hit in outcome.hits) == ("equal",) + assert outcome.admission is not None + assert (outcome.admission.retrieved, outcome.admission.admitted) == (3, 2) + + +def test_experience_legacy_stops_at_limit_and_accepts_unscored_custom_rows() -> None: + rows = [_experience_row("one", 2.0, "oceanbase_match"), _experience_row("two", 1.0, "oceanbase_match")] + for row in rows: + del row["raw_score"] + del row["score_metric"] + outcome = experience_search_hits(rows, "client", 1, "project") + + assert tuple(hit.artifact_ref.artifact_id for hit in outcome.hits) == ("one",) + assert outcome.hits[0].retrieval_score is None + assert outcome.admission is not None + assert (outcome.admission.retrieved, outcome.admission.admitted) == (1, 1) + with pytest.raises(RuntimeError): + experience_search_hits(rows, "client", 1, "project", require_scores=True) + + +def test_skill_threshold_uses_same_score_and_admission_contract() -> None: + from powercontext.builtin.artifacts.search import AdmissionFloor + + rows = [ + { + "artifact_id": "equal", + "revision": 1, + "content": _skill().model_dump_json().encode(), + "raw_score": -1.0, + "score_metric": "sqlite_bm25", + }, + { + "artifact_id": "below", + "revision": 1, + "content": _skill().model_dump_json().encode(), + "raw_score": -0.25, + "score_metric": "sqlite_bm25", + }, + ] + hits = skill_search_hits( + rows, + "client unrelated absent", + 3, + admission=AdmissionFloor(lexical_min_matched_terms=1), + min_score=0.5, + require_scores=True, + ) + + assert tuple(hit.artifact_ref.artifact_id for hit in hits) == ("equal",) + assert hits[0].retrieval_score == 0.5 + assert hits[0].channel_scores is not None + assert hits[0].channel_scores["text"].raw == -1 + + +def test_oceanbase_search_selects_match_raw_and_preserves_candidate_window() -> None: + rows = [_experience_row("one", 2.0, "oceanbase_match")] + connection = SimpleNamespace(execute=AsyncMock(return_value=SimpleNamespace(mappings=lambda: rows))) + outcome = asyncio.run( + OceanBaseExperienceFTSIndex().search( + cast(AsyncConnection, connection), "project", "client", 8, require_scores=True + ) + ) + statement = connection.execute.await_args.args[0] + compiled = statement.compile(dialect=mysql.dialect()) + sql = str(compiled) + + assert "AS raw_score" in sql + assert "MATCH (pc_artifact_heads.searchable_text) AGAINST" in sql + assert "ORDER BY MATCH" in sql + assert " DESC, pc_artifact_heads.artifact_id, pc_artifact_heads.revision" in sql + assert 32 in compiled.params.values() + assert outcome.hits[0].channel_scores is not None + assert outcome.hits[0].channel_scores["text"].metric == "oceanbase_match" diff --git a/tests/builtin/persistence/test_topic_memory.py b/tests/builtin/persistence/test_topic_memory.py index 96d058012a..d7c4c5433a 100644 --- a/tests/builtin/persistence/test_topic_memory.py +++ b/tests/builtin/persistence/test_topic_memory.py @@ -139,6 +139,191 @@ def _fts_index() -> CompositeTopicMemoryIndex: return CompositeTopicMemoryIndex(SQLiteTopicMemoryFTSIndex()) +def test_public_topic_search_keeps_signed_fts_metadata_and_exact_artifacts() -> None: + from powercontext.builtin.artifacts.topic_memory import TopicArtifactSearchRequest + + async def scenario() -> None: + index = _fts_index() + repository = TopicMemoryRepository(index=index) + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES + index.tables) as profile: + async with profile.database.transaction() as connection: + await repository.initialize(connection) + content = _content("Needle", "needle") + publication = await repository.publish_create( + connection, "scope-a", "topic-1", _draft(content), prepare_topic_memory_projection(content) + ) + plain = await repository.search( + connection, + "scope-a", + "needle", + limit=10, + artifact_request=TopicArtifactSearchRequest(query="needle"), + ) + scored = await repository.search( + connection, + "scope-a", + "needle", + limit=10, + artifact_request=TopicArtifactSearchRequest(query="needle", include_scores=True), + ) + legacy = await repository.search(connection, "scope-a", "needle", limit=10) + empty = await repository.search( + connection, "empty", "needle", limit=10, artifact_request=TopicArtifactSearchRequest(query="needle") + ) + assert tuple(hit.artifact_ref for hit in plain.hits) == tuple(hit.artifact_ref for hit in scored.hits) + assert plain.artifacts is not None + assert plain.artifacts[0] == publication.topic + assert plain.artifacts[0].as_ref() == scored.matches[0].artifact_ref + assert legacy.artifacts is None + assert empty.artifacts == () + assert plain.matches[0].channel_scores is None + assert scored.matches[0].channel_scores is not None + assert scored.matches[0].channel_scores["topic_fts"].raw < 0 + assert scored.matches[0].channel_scores["topic_fts"].metric == "sqlite_bm25" + assert scored.matches[0].channel_scores["detail_fts"].raw < 0 + + asyncio.run(scenario()) + + +def test_public_topic_missing_selected_exact_revision_is_a_contract_failure(monkeypatch) -> None: + from powercontext.artifacts.search import ArtifactSearchContractError + from powercontext.builtin.artifacts.topic_memory import TopicArtifactSearchRequest + from powercontext.builtin.persistence.errors import RepositoryNotFoundError + + async def scenario() -> None: + index = _fts_index() + repository = TopicMemoryRepository(index=index) + async with ( + SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES + index.tables) as profile, + profile.database.transaction() as connection, + ): + await repository.initialize(connection) + content = _content("Needle", "needle") + await repository.publish_create( + connection, "scope-a", "topic-1", _draft(content), prepare_topic_memory_projection(content) + ) + + async def missing(connection, scope, ref): + raise RepositoryNotFoundError("artifact", ref) + + monkeypatch.setattr(repository.artifacts, "get", missing) + legacy = await repository.search(connection, "scope-a", "needle", limit=10) + assert len(legacy.hits) == 1 + with pytest.raises(ArtifactSearchContractError): + await repository.search( + connection, + "scope-a", + "needle", + limit=10, + artifact_request=TopicArtifactSearchRequest(query="needle"), + ) + + asyncio.run(scenario()) + + +def test_public_topic_detail_score_belongs_to_the_selected_representative_chunk() -> None: + from powercontext.builtin.artifacts.topic_memory import TopicArtifactSearchRequest + + async def scenario() -> None: + index = _fts_index() + repository = TopicMemoryRepository(index=index) + content = TopicMemoryContent( + title="Evidence", summary="Detailed search", detail="alpha " * 400 + "\n\n" + "alpha beta gamma " * 40 + ) + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES + index.tables) as profile: + async with profile.database.transaction() as connection: + await repository.initialize(connection) + await repository.publish_create( + connection, "scope-a", "topic", _draft(content), prepare_topic_memory_projection(content) + ) + query = "alpha beta gamma" + channels = await index.search( + connection, + "scope-a", + TopicMemorySearchRequest(query=query, analyzed_query=query, candidate_limit=32, mode="fts"), + ) + row_scores = ( + await connection.execute( + text( + "SELECT chunk_ordinal, bm25(pc_topic_memory_chunk_fts) AS raw_score FROM pc_topic_memory_chunk_fts WHERE pc_topic_memory_chunk_fts MATCH :query AND scope_id = :scope" + ), + {"query": fts_match_query(query), "scope": "scope-a"}, + ) + ).mappings() + actual_scores = {int(row["chunk_ordinal"]): float(row["raw_score"]) for row in row_scores} + scored = await repository.search( + connection, + "scope-a", + query, + limit=10, + artifact_request=TopicArtifactSearchRequest(query=query, include_scores=True), + ) + representative = channels.detail_fts[0] + assert representative.chunk_text is not None + assert representative.chunk_ordinal is not None + assert scored.matches[0].channel_scores is not None + assert "beta" in representative.chunk_text and "gamma" in representative.chunk_text + assert representative.raw_score == actual_scores[representative.chunk_ordinal] + assert scored.matches[0].channel_scores["detail_fts"].raw == representative.raw_score + + asyncio.run(scenario()) + + +def test_public_topic_vector_metadata_keeps_actual_l2_distance_and_direction() -> None: + from powercontext.builtin.artifacts.topic_memory import TopicArtifactSearchRequest + + async def scenario() -> None: + embedding_profile = EmbeddingProfile(profile_id="topic-v1", model="test", dimension=2) + index = CompositeTopicMemoryIndex(SQLiteTopicMemoryFTSIndex(), SQLiteTopicMemoryVectorIndex(embedding_profile)) + repository = TopicMemoryRepository(index=index) + content = _content("Vector", "detail") + base = prepare_topic_memory_projection(content) + projection = base.model_copy( + update={ + "topic_embedding": (0.8, 0.6), + "chunk_embeddings": tuple((1.0, 0.0) for _ in base.chunks), + "embedding_profile": embedding_profile, + } + ) + async with SQLiteProfile.open( + SQLiteConfig(), tables=BUILTIN_TABLES + index.tables, load_vector_extension=True + ) as profile: + async with profile.database.transaction() as connection: + await repository.initialize(connection) + await repository.publish_create(connection, "scope-a", "topic", _draft(content), projection) + request = TopicArtifactSearchRequest(query="semantic", mode="vector", include_scores=True) + channels = await index.search( + connection, + "scope-a", + TopicMemorySearchRequest( + query="semantic", + candidate_limit=32, + mode="vector", + query_vector=(1.0, 0.0), + embedding_profile=embedding_profile, + ), + ) + result = await repository.search( + connection, + "scope-a", + "semantic", + limit=10, + mode="vector", + query_vector=(1.0, 0.0), + embedding_profile=embedding_profile, + artifact_request=request, + ) + metadata = result.matches[0].channel_scores + assert metadata is not None + assert metadata["topic_vector"].raw == channels.topic_vector[0].distance + assert metadata["detail_vector"].raw == 0.0 + assert metadata["topic_vector"].metric == "l2_distance" + assert metadata["topic_vector"].higher_is_better is False + assert "topic_fts" not in metadata + + asyncio.run(scenario()) + + def test_publication_retains_exact_revisions_and_searches_only_the_current_scope_head() -> None: async def scenario() -> None: index = _fts_index() diff --git a/tests/builtin/persistence/test_topic_memory_oceanbase.py b/tests/builtin/persistence/test_topic_memory_oceanbase.py index 24c7904561..8bbc4a48d1 100644 --- a/tests/builtin/persistence/test_topic_memory_oceanbase.py +++ b/tests/builtin/persistence/test_topic_memory_oceanbase.py @@ -119,8 +119,10 @@ async def scenario() -> None: "ON pc_topic_memory_active_chunks (searchable_text) WITH PARSER SPACE", ] assert "MATCH (pc_topic_memory_active_topics.searchable_text) AGAINST" in query_statements[0] + assert " AS raw_score" in query_statements[0] assert "pc_topic_memory_active_topics.scope_id" in query_statements[0] assert "MATCH (pc_topic_memory_active_chunks.searchable_text) AGAINST" in query_statements[1] + assert " AS raw_score" in query_statements[1] assert "pc_topic_memory_active_chunks.scope_id" in query_statements[1] assert "row_number() OVER" in query_statements[1] assert "anon_1.topic_rank" in query_statements[1] diff --git a/tests/builtin/review/test_service.py b/tests/builtin/review/test_service.py index d4f9210dcc..d595cbc9c6 100644 --- a/tests/builtin/review/test_service.py +++ b/tests/builtin/review/test_service.py @@ -103,6 +103,8 @@ async def search( /, *, admission: object = None, + min_score: float | None = None, + require_scores: bool = False, ) -> ExperienceSearchOutcome: return ExperienceSearchOutcome() @@ -123,6 +125,10 @@ async def search_skills( _query: str, _limit: int, /, + *, + admission: object = None, + min_score: float | None = None, + require_scores: bool = False, ) -> tuple[SkillSearchHit, ...]: return () diff --git a/tests/builtin/runtime/test_artifact_search.py b/tests/builtin/runtime/test_artifact_search.py new file mode 100644 index 0000000000..125fce4516 --- /dev/null +++ b/tests/builtin/runtime/test_artifact_search.py @@ -0,0 +1,370 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from typing import Any, ClassVar, cast + +import pytest +from pydantic import BaseModel, ValidationError + +from powercontext.artifacts import Artifact, ArtifactRef +from powercontext.artifacts.search import ( + ArtifactSearchContractError, + ArtifactSearchExecutionContext, + ArtifactSearchMatch, + ArtifactSearchQuery, + ArtifactSearchUnsupported, + ChannelScore, +) +from powercontext.builtin.artifacts.experience import ExperienceContent, ExperienceSearchHit, ExperienceSearchOutcome +from powercontext.builtin.artifacts.search import InvalidSearchScore +from powercontext.builtin.artifacts.skill import Skill, SkillContent, SkillSearchHit +from powercontext.builtin.persistence.artifact_governance import ArtifactGovernance, ArtifactLifecycleState +from powercontext.builtin.persistence.experience_index import NoExperienceIndex +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.sqlite.experience_index import SQLiteExperienceFTSIndex +from powercontext.builtin.runtime import ( + ApproveArtifactCandidateRequest, + BuiltinConfig, + BuiltinRuntime, + CaptureSource, + GetExperienceRequest, + GetSkillRequest, + ProposeExperienceRequest, + ProposeSkillRequest, + RuntimeCapabilities, + open_builtin_contexts, + open_builtin_runtime, +) +from powercontext.builtin.runtime.artifact_search import ArtifactSearchService +from powercontext.builtin.scope import ScopeDraft, ScopeNotFoundError + + +async def _scope(runtime: BuiltinRuntime) -> str: + assert runtime.scopes is not None + return ( + await runtime.scopes.create(ScopeDraft(title="Search", summary="Artifact tests", idempotency_key="search")) + ).scope_id + + +def _content(family: str, marker: str) -> ExperienceContent | SkillContent: + if family == "experience": + return ExperienceContent( + situation=marker, action="Inspect generated client", outcome="Correct contract", lesson="Validate" + ) + return SkillContent( + name="generated-client", + description=marker, + instructions="Inspect generated contract", + validation=("Validate contract",), + ) + + +async def _approve(runtime: BuiltinRuntime, scope: str, family: str, marker: str, target: ArtifactRef | None = None): + source = await runtime.sources.for_scope(scope).capture( + CaptureSource(source_id=marker, content=marker, metadata={}) + ) + proposal = _content(family, marker) + lineage = () if target is None else (target,) + if isinstance(proposal, ExperienceContent): + candidate = await runtime.experience.for_scope(scope).propose( + ProposeExperienceRequest(proposal=proposal, sources=(source.source_ref,), artifacts=lineage, target=target) + ) + else: + candidate = await runtime.skill.for_scope(scope).propose( + ProposeSkillRequest(proposal=proposal, sources=(source.source_ref,), artifacts=lineage, target=target) + ) + approved = await runtime.review.for_scope(scope).approve( + ApproveArtifactCandidateRequest(candidate_id=candidate.candidate_id, expected_version=candidate.version) + ) + assert approved.result_artifact is not None + return approved.result_artifact, source.source_ref + + +@pytest.mark.parametrize("family", ["experience", "skill"]) +def test_public_lexical_search_returns_current_exact_artifacts_and_true_scores(family: str) -> None: + async def scenario() -> None: + async with open_builtin_runtime(BuiltinConfig(database=SQLiteConfig())) as runtime: + scope = await _scope(runtime) + first, _ = await _approve(runtime, scope, family, "oldmarker") + current, source = await _approve(runtime, scope, family, "currentmarker", first) + application = runtime.artifacts.for_scope(scope) + outcome = await application.search(family, {"query": "currentmarker", "limit": 200, "include_scores": True}) + assert outcome.artifacts is not None + assert tuple(match.artifact_ref for match in outcome.matches) == (current,) + assert tuple(artifact.as_ref() for artifact in outcome.artifacts) == (current,) + artifact = outcome.artifacts[0] + exact = ( + await runtime.experience.for_scope(scope).get(GetExperienceRequest(artifact=current)) + if family == "experience" + else await runtime.skill.for_scope(scope).get(GetSkillRequest(artifact=current)) + ) + assert artifact == exact + assert artifact.lineage.sources == (source,) + assert artifact.lineage.artifacts == (first,) + channel = outcome.matches[0].channel_scores + assert channel is not None + assert channel["text"].metric == "sqlite_bm25" + assert channel["text"].raw < 0 + assert 0 < outcome.matches[0].retrieval_score < 1 + plain = await application.search(family, {"query": "currentmarker", "limit": 200}) + assert tuple(match.artifact_ref for match in plain.matches) == (current,) + assert plain.artifacts == outcome.artifacts + assert (await application.search(family, {"query": "oldmarker"})).matches == () + excluded = await application.search(family, {"query": "currentmarker", "min_score": 1}) + assert excluded.matches == () + assert excluded.artifacts == () + with pytest.raises(ValidationError): + await application.search(family, {"query": "currentmarker", "filters": {"unknown": True}}) + with pytest.raises(ValidationError): + await application.search(family, {"query": "currentmarker", "limit": 201}) + with pytest.raises(ScopeNotFoundError): + await runtime.artifacts.for_scope("missing-scope").search(family, {"query": "currentmarker"}) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("family", ["experience", "skill"]) +def test_public_lexical_search_reports_disabled_index_while_legacy_stays_empty(family: str) -> None: + async def scenario() -> None: + from powercontext.builtin.runtime.experience_search import ExperienceArtifactSearcher + from powercontext.builtin.runtime.skill_search import SkillArtifactSearcher + + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + index = NoExperienceIndex() + service = ArtifactSearchService(known_families={family}) + if family == "experience": + service.register(ExperienceArtifactSearcher(contexts.database, contexts.repositories.artifacts, index)) + else: + service.register(SkillArtifactSearcher(contexts.database, contexts.repositories.artifacts, index)) + with pytest.raises(ArtifactSearchUnsupported): + await service.search("project", family, {"query": "client"}) + async with contexts.database.transaction() as connection: + assert (await index.search(connection, "project", "", 500)).hits == () + assert await index.search_skills(connection, "project", "", 500) == () + + asyncio.run(scenario()) + + +class _UnusedProvider: + async def get(self, scope_id: str, /) -> Any: + raise AssertionError(scope_id) + + +class _Scopes: + async def get(self, _scope_id: str, /) -> Any: + return object() + + +def test_runtime_facade_preserves_custom_family_outcome_and_optional_injection() -> None: + class CustomContent(BaseModel): + title: str + + class CustomArtifact(Artifact[CustomContent]): + family: ClassVar[str] = "custom" + + artifact = CustomArtifact(artifact_id="custom", revision=1, content=CustomContent(title="value")) + supplied_context = ArtifactSearchExecutionContext(principal=object()) + seen_contexts: list[ArtifactSearchExecutionContext | None] = [] + + @dataclass + class Outcome: + matches: tuple[ArtifactSearchMatch, ...] = (ArtifactSearchMatch(artifact.as_ref(), 0.5),) + artifacts: tuple[CustomArtifact, ...] = (artifact,) + extra: str = "preserved" + + class Searcher: + family = "custom" + request_type = ArtifactSearchQuery + + async def search( + self, + scope_id: str, + request: ArtifactSearchQuery, + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> Outcome: + assert scope_id == "project" and request.query == "value" + seen_contexts.append(execution_context) + return Outcome() + + async def scenario() -> None: + service = ArtifactSearchService(known_families={"custom"}) + service.register(Searcher()) + async with BuiltinRuntime( + provider=_UnusedProvider(), + capabilities=RuntimeCapabilities(memory_extraction=False, memory_search_modes=()), + scope_application=cast(Any, _Scopes()), + artifact_search=service, + ) as runtime: + result = await runtime.artifacts.for_scope("project").search("custom", {"query": "value"}) + assert isinstance(result, Outcome) + assert result.extra == "preserved" + await runtime.artifacts.for_scope("project").search( + "custom", {"query": "value"}, execution_context=supplied_context + ) + assert seen_contexts[0] is None + assert seen_contexts[1] is supplied_context + async with BuiltinRuntime( + provider=_UnusedProvider(), + capabilities=RuntimeCapabilities(memory_extraction=False, memory_search_modes=()), + scope_application=cast(Any, _Scopes()), + ) as runtime: + assert await runtime.skill.for_scope("project").search("", 500) == () + with pytest.raises(ArtifactSearchUnsupported): + await runtime.artifacts.for_scope("project").search("custom", {"query": "value"}) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("include_scores", [False, True]) +def test_public_experience_requires_raw_metadata_even_when_scores_are_hidden(include_scores: bool) -> None: + from powercontext.builtin.runtime.experience_search import ExperienceArtifactSearcher + + class Index(SQLiteExperienceFTSIndex): + async def search(self, *_args: Any, **_kwargs: Any) -> ExperienceSearchOutcome: + return ExperienceSearchOutcome( + hits=( + ExperienceSearchHit( + artifact_ref=ArtifactRef(family="experience", artifact_id="missing-raw", revision=1), + content=cast(ExperienceContent, _content("experience", "needle")), + retrieval_score=0.5, + ), + ) + ) + + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + service = ArtifactSearchService(known_families={"experience"}) + service.register(ExperienceArtifactSearcher(contexts.database, contexts.repositories.artifacts, Index())) + with pytest.raises(InvalidSearchScore): + await service.search("project", "experience", {"query": "needle", "include_scores": include_scores}) + + asyncio.run(scenario()) + + +def test_skill_library_preserves_combined_window_whole_query_and_active_order() -> None: + skills = tuple( + Skill( + artifact_id=key, + revision=1, + content=SkillContent(name=key, description=text, instructions="check", validation=("Validate",)), + ) + for key, text in ( + ("a-active", "needle phrase"), + ("b-deprecated", "needle phrase"), + ("c-deprecated", "needle and phrase"), + ("y-active", "needle phrase"), + ("z-deprecated", "needle phrase"), + ) + ) + governance = { + skill.artifact_id: ArtifactGovernance( + artifact=skill.as_ref(), + governance_generation=0, + lifecycle_state=ArtifactLifecycleState.ACTIVE + if skill.artifact_id.endswith("active") + else ArtifactLifecycleState.DEPRECATED, + ) + for skill in skills + } + + async def recall(_scope: str, query: str, limit: int): + assert query == "needle phrase" + return tuple( + SkillSearchHit(artifact_ref=skill.as_ref(), content=skill.content) for skill in (skills[3], skills[0]) + ) + + async def lister(_scope: str, include_deprecated: bool, limit: int): + return tuple( + (skill, governance[skill.artifact_id]) + for skill in skills + if include_deprecated or governance[skill.artifact_id].lifecycle_state is ArtifactLifecycleState.ACTIVE + )[:limit] + + async def reader(_scope: str, key: str): + return governance[key] + + class Review: + async def get_skill(self, ref: ArtifactRef): + return next(skill for skill in skills if skill.as_ref() == ref) + + async def scenario() -> None: + async with BuiltinRuntime( + provider=_UnusedProvider(), + capabilities=RuntimeCapabilities(memory_extraction=False, memory_search_modes=()), + scope_application=cast(Any, _Scopes()), + skill_recall=recall, + skill_lister=lister, + skill_governance_reader=reader, + review_service=cast(Any, lambda _scope: Review()), + ) as runtime: + scoped = runtime.skill.for_scope("project") + values = await scoped.search_library(" needle phrase ", 3, include_deprecated=True) + assert tuple(skill.artifact_id for skill, _ in values) == ("y-active", "a-active", "b-deprecated") + assert tuple( + skill.artifact_id for skill, _ in await scoped.search_library("", 2, include_deprecated=True) + ) == ("a-active", "b-deprecated") + assert tuple( + skill.artifact_id + for skill, _ in await scoped.search_library("needle phrase", 3, include_deprecated=False) + ) == ("y-active", "a-active") + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("family", ["experience", "skill"]) +def test_public_search_missing_selected_revision_is_an_internal_contract_failure(family: str) -> None: + from powercontext.builtin.runtime.experience_search import ExperienceArtifactSearcher + from powercontext.builtin.runtime.skill_search import SkillArtifactSearcher + + ref = ArtifactRef(family=family, artifact_id="missing-selected-revision", revision=1) + scores = {"text": ChannelScore(-1.0, "sqlite_bm25", False)} + + class Index(SQLiteExperienceFTSIndex): + async def search(self, *_args: Any, **_kwargs: Any) -> ExperienceSearchOutcome: + return ExperienceSearchOutcome( + hits=( + ExperienceSearchHit( + artifact_ref=ref, + content=cast(ExperienceContent, _content(family, "needle")), + retrieval_score=0.5, + channel_scores=scores, + ), + ) + ) + + async def search_skills(self, *_args: Any, **_kwargs: Any) -> tuple[SkillSearchHit, ...]: + return ( + SkillSearchHit( + artifact_ref=ref, + content=cast(SkillContent, _content(family, "needle")), + retrieval_score=0.5, + channel_scores=scores, + ), + ) + + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + service = ArtifactSearchService(known_families={family}) + searcher = ExperienceArtifactSearcher if family == "experience" else SkillArtifactSearcher + service.register(searcher(contexts.database, contexts.repositories.artifacts, Index())) + with pytest.raises(ArtifactSearchContractError): + await service.search("project", family, {"query": "needle"}) + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_artifact_search_registry.py b/tests/builtin/runtime/test_artifact_search_registry.py new file mode 100644 index 0000000000..0737d4d2c1 --- /dev/null +++ b/tests/builtin/runtime/test_artifact_search_registry.py @@ -0,0 +1,413 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public search dispatch accepts arbitrary Artifact Families and checks outcomes.""" + +import asyncio +import json +import math +from dataclasses import FrozenInstanceError, dataclass +from typing import Any, ClassVar, cast + +import pytest +from pydantic import BaseModel, Field, StrictInt, ValidationError + +from powercontext.artifacts import Artifact, ArtifactRef, ArtifactSearchExecutionContext + + +class CustomContent(BaseModel): + title: str + + +class CustomArtifact(Artifact[CustomContent]): + family: ClassVar[str] = "custom" + + +@pytest.mark.parametrize("local_default", [False, True]) +def test_registry_passes_the_same_trusted_execution_context_to_a_custom_family(local_default): + from powercontext.artifacts.search import ArtifactSearchMatch, ArtifactSearchQuery + from powercontext.builtin.runtime.artifact_search import ArtifactSearchService + + context = ( + None if local_default else ArtifactSearchExecutionContext(principal=object(), access=object(), audit=object()) + ) + if context is not None: + assert context.trusted_local is False + with pytest.raises(FrozenInstanceError): + cast(Any, context).principal = object() + artifact = CustomArtifact(artifact_id="item", revision=1, content=CustomContent(title="value")) + + @dataclass + class Outcome: + matches: tuple[ArtifactSearchMatch, ...] = (ArtifactSearchMatch(artifact.as_ref(), 0.5),) + artifacts: tuple[CustomArtifact, ...] = (artifact,) + + class Searcher: + family = "custom" + request_type = ArtifactSearchQuery + + async def search(self, scope, request, /, *, execution_context=None) -> Outcome: + assert scope == "scope" + assert request.model_fields_set == {"query"} + assert execution_context is context + return Outcome() + + service = ArtifactSearchService(known_families={"custom"}) + service.register(Searcher()) + result = asyncio.run(service.search("scope", "custom", {"query": "value"}, execution_context=context)) + assert isinstance(result, Outcome) + + +def test_query_trims_and_validates_inherited_fields_with_json_semantics(): + from powercontext.artifacts.search import ArtifactSearchQuery + + class CustomQuery(ArtifactSearchQuery): + tags: tuple[str, ...] = () + optional: str | None = None + limit: StrictInt = Field(default=1, ge=1, le=2) + + request = CustomQuery.model_validate_json(json.dumps({"query": " 你好 ", "tags": ["a"]}), strict=True) + assert request.query == "你好" + assert request.tags == ("a",) + assert request.limit == 1 + assert request.optional is None + assert request.model_fields_set == {"query", "tags"} + with pytest.raises(ValidationError): + CustomQuery.model_validate_json('{"query":"valid","optional":null}', strict=True) + + +@pytest.mark.parametrize( + "payload", + [ + {"query": " "}, + {"query": 1}, + {"query": "x" * 8193}, + {"query": "x", "limit": True}, + {"query": "x", "limit": "1"}, + {"query": "x", "limit": 1.0}, + {"query": "x", "limit": 0}, + {"query": "x", "limit": 201}, + {"query": "x", "include_scores": 1}, + {"query": "x", "include_scores": "true"}, + {"query": "x", "include_scores": None}, + ], +) +def test_query_rejects_invalid_public_parameters(payload): + from powercontext.artifacts.search import ArtifactSearchQuery + + with pytest.raises(ValidationError): + ArtifactSearchQuery.model_validate_json(json.dumps(payload), strict=True) + + +def test_registry_dispatches_custom_request_and_preserves_outcome(): + from powercontext.artifacts.search import ArtifactSearchMatch, ArtifactSearchQuery, ChannelScore + from powercontext.builtin.runtime.artifact_search import ArtifactSearchService + from powercontext.builtin.statistics import ModelUsagePurpose + + class CustomQuery(ArtifactSearchQuery): + tags: tuple[str, ...] = () + limit: StrictInt = Field(default=1, ge=1, le=2) + + artifact = CustomArtifact(artifact_id="item", revision=3, content=CustomContent(title="你好")) + + @dataclass + class CustomOutcome: + matches: tuple[ArtifactSearchMatch, ...] + artifacts: tuple[CustomArtifact, ...] + additional_field: str = "preserved" + + class CustomSearcher: + family = "custom" + request_type = CustomQuery + + async def search( + self, + scope_id: str, + request: CustomQuery, + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> CustomOutcome: + assert scope_id == "scope" + assert request.tags == ("a",) + assert request.query == "你好" + assert request.model_fields_set == {"query", "tags", "include_scores"} + return CustomOutcome( + matches=(ArtifactSearchMatch(artifact.as_ref(), 0.8, {"text": ChannelScore(-2.5, "distance", False)}),), + artifacts=(artifact,), + ) + + service = ArtifactSearchService(known_families={"custom", "unsupported"}) + service.register(CustomSearcher(), embedding_purpose=ModelUsagePurpose.MEMORY_RECALL) + result = asyncio.run(service.search("scope", "custom", {"query": " 你好 ", "tags": ["a"], "include_scores": True})) + assert isinstance(result, CustomOutcome) + assert result.additional_field == "preserved" + assert result.artifacts[0] is artifact + assert result.artifacts[0].content.title == "你好" + assert service.embedding_purpose("custom") == ModelUsagePurpose.MEMORY_RECALL + + +def test_registry_distinguishes_unknown_family_and_unsupported_search(): + from powercontext.artifacts.search import ArtifactSearchFamilyNotFound, ArtifactSearchUnsupported + from powercontext.builtin.runtime.artifact_search import ArtifactSearchService + + service = ArtifactSearchService(known_families={"custom"}) + with pytest.raises(ArtifactSearchFamilyNotFound) as missing: + asyncio.run(service.search("scope", "unknown", {"query": "x"})) + assert missing.value.family == "unknown" + with pytest.raises(ArtifactSearchUnsupported) as unsupported: + service.embedding_purpose("custom") + assert unsupported.value.family == "custom" + assert unsupported.value.field == "family" + + +def test_registry_refuses_unknown_and_duplicate_registration(): + from powercontext.artifacts.search import ArtifactSearchFamilyNotFound, ArtifactSearchQuery + from powercontext.builtin.runtime.artifact_search import ArtifactSearchService + + class Searcher: + family = "custom" + request_type = ArtifactSearchQuery + + async def search( + self, + scope_id: str, + request: ArtifactSearchQuery, + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> Any: + raise AssertionError("not called") # noqa: TRY003 + + with pytest.raises(ArtifactSearchFamilyNotFound): + ArtifactSearchService(known_families=()).register(Searcher()) + service = ArtifactSearchService(known_families={"custom"}) + service.register(Searcher()) + with pytest.raises(ValueError, match="already registered"): + service.register(Searcher()) + + +@pytest.mark.parametrize( + "defect", + [ + "missing-artifacts", + "count", + "order", + "wrong-family", + "duplicates", + "limit", + "missing-scores", + "nonfinite-score", + "nonfinite-raw", + "invalid-reference", + ], +) +def test_registry_reports_malformed_public_outcomes_as_implementation_failure(defect): # noqa: C901 + from powercontext.artifacts.search import ( + ArtifactSearchContractError, + ArtifactSearchMatch, + ArtifactSearchQuery, + ChannelScore, + ) + from powercontext.builtin.runtime.artifact_search import ArtifactSearchService + + class Query(ArtifactSearchQuery): + limit: StrictInt = Field(default=1, ge=1, le=200) + + first = CustomArtifact(artifact_id="first", revision=1, content=CustomContent(title="first")) + second = CustomArtifact(artifact_id="second", revision=2, content=CustomContent(title="second")) + score = ChannelScore(1.0, "relevance", True) + match = ArtifactSearchMatch(first.as_ref(), 1.0, {"text": score}) + matches = (match,) + artifacts = (first,) + if defect == "missing-artifacts": + artifacts = None + elif defect == "count": + artifacts = () + elif defect == "order": + artifacts = (second,) + elif defect == "wrong-family": + object.__setattr__(match, "artifact_ref", ArtifactRef(family="other", artifact_id="first", revision=1)) + elif defect == "duplicates": + matches = (match, match) + artifacts = (first, first) + elif defect == "limit": + matches = (match, ArtifactSearchMatch(second.as_ref(), 0.5, {"text": score})) + artifacts = (first, second) + elif defect == "missing-scores": + object.__setattr__(match, "channel_scores", None) + elif defect == "nonfinite-score": + object.__setattr__(match, "retrieval_score", math.inf) + elif defect == "nonfinite-raw": + object.__setattr__(score, "raw", math.nan) + else: + object.__setattr__(match, "artifact_ref", {"family": "custom", "artifact_id": "first", "revision": 1}) + + @dataclass + class Outcome: + matches: Any + artifacts: Any + + class Searcher: + family = "custom" + request_type = Query + + async def search( + self, scope_id: str, request: Query, /, *, execution_context: ArtifactSearchExecutionContext | None = None + ) -> Outcome: + return Outcome(matches, artifacts) + + service = ArtifactSearchService(known_families={"custom"}) + service.register(Searcher()) + with pytest.raises(ArtifactSearchContractError): + asyncio.run(service.search("scope", "custom", {"query": "x", "include_scores": True})) + + +def test_registry_preserves_runtime_failures(): + from powercontext.artifacts.search import ArtifactSearchQuery + from powercontext.builtin.runtime.artifact_search import ArtifactSearchService + + class Searcher: + family = "custom" + request_type = ArtifactSearchQuery + + async def search( + self, + scope_id: str, + request: ArtifactSearchQuery, + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> Any: + raise RuntimeError("retrieval failed") # noqa: TRY003 + + service = ArtifactSearchService(known_families={"custom"}) + service.register(Searcher()) + with pytest.raises(RuntimeError, match="retrieval failed"): + asyncio.run(service.search("scope", "custom", {"query": "x"})) + + +@pytest.mark.parametrize("channel_scores", [None, {}]) +def test_registry_accepts_unrequested_or_collected_empty_metadata(channel_scores): + from powercontext.artifacts.search import ArtifactSearchMatch, ArtifactSearchQuery + from powercontext.builtin.runtime.artifact_search import ArtifactSearchService + + artifact = CustomArtifact(artifact_id="item", revision=1, content=CustomContent(title="item")) + + @dataclass + class Outcome: + matches: tuple[ArtifactSearchMatch, ...] + artifacts: tuple[CustomArtifact, ...] + + outcome = Outcome((ArtifactSearchMatch(artifact.as_ref(), 0.5, channel_scores),), (artifact,)) + + class Searcher: + family = "custom" + request_type = ArtifactSearchQuery + + async def search( + self, + scope_id: str, + request: ArtifactSearchQuery, + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> Outcome: + return outcome + + service = ArtifactSearchService(known_families={"custom"}) + service.register(Searcher()) + assert asyncio.run(service.search("scope", "custom", {"query": "x"})) is outcome + if channel_scores is not None: + assert asyncio.run(service.search("scope", "custom", {"query": "x", "include_scores": True})) is outcome + + +@pytest.mark.parametrize( + "kwargs", + [ + {"raw": math.nan, "metric": "relevance", "higher_is_better": True}, + {"raw": True, "metric": "relevance", "higher_is_better": True}, + {"raw": "1", "metric": "relevance", "higher_is_better": True}, + {"raw": 0, "metric": " ", "higher_is_better": True}, + {"raw": 0, "metric": "relevance", "higher_is_better": 1}, + ], +) +def test_channel_scores_require_meaningful_finite_values(kwargs): + from powercontext.artifacts.search import ChannelScore + + with pytest.raises(ValueError): + ChannelScore(**kwargs) + + +@pytest.mark.parametrize("score", [math.inf, math.nan, True, "1", -0.1, 1.1]) +def test_match_scores_require_finite_normalized_values(score): + from powercontext.artifacts.search import ArtifactSearchMatch + + with pytest.raises(ValueError): + ArtifactSearchMatch(ArtifactRef(family="custom", artifact_id="item", revision=1), score) + + +@pytest.mark.parametrize("value", ["", b"", bytearray()]) +def test_registry_rejects_string_like_result_sequences(value): + from powercontext.artifacts.search import ArtifactSearchContractError, ArtifactSearchQuery + from powercontext.builtin.runtime.artifact_search import ArtifactSearchService + + @dataclass + class BadOutcome: + matches: Any + artifacts: Any + + class Searcher: + family = "custom" + request_type = ArtifactSearchQuery + + async def search( + self, + scope_id: str, + request: ArtifactSearchQuery, + /, + *, + execution_context: ArtifactSearchExecutionContext | None = None, + ) -> BadOutcome: + return BadOutcome(value, value) + + service = ArtifactSearchService(known_families={"custom"}) + service.register(Searcher()) + with pytest.raises(ArtifactSearchContractError): + asyncio.run(service.search("scope", "custom", {"query": "x"})) + + +@pytest.mark.parametrize("value", [math.nan, math.inf, -math.inf]) +def test_registry_reports_nonfinite_json_input_with_a_located_validation_error(value): + from powercontext.artifacts.search import ArtifactSearchQuery + from powercontext.builtin.runtime.artifact_search import ArtifactSearchService + + class Query(ArtifactSearchQuery): + filters: dict[str, float] = Field(default_factory=dict) + + class Searcher: + family = "custom" + request_type = Query + + async def search( + self, scope_id: str, request: Query, /, *, execution_context: ArtifactSearchExecutionContext | None = None + ) -> Any: + raise AssertionError("invalid input must not be dispatched") # noqa: TRY003 + + service = ArtifactSearchService(known_families={"custom"}) + service.register(Searcher()) + with pytest.raises(ValidationError) as error: + asyncio.run(service.search("scope", "custom", {"query": "private query", "filters": {"value": value}})) + assert error.value.errors()[0]["loc"] == ("filters", "value") + assert "private query" not in str(error.value) diff --git a/tests/builtin/runtime/test_model_usage_recorder.py b/tests/builtin/runtime/test_model_usage_recorder.py index 3a06ff1f8f..809a20fa42 100644 --- a/tests/builtin/runtime/test_model_usage_recorder.py +++ b/tests/builtin/runtime/test_model_usage_recorder.py @@ -206,9 +206,17 @@ async def scenario() -> None: await recorder.flush(target) assert await _rows(database) == () await _assert_connection_restored(database) + # Recovery checks the same recorder's usability, independently + # of the short deadline exercised while the writer lock was held. + recorder._write_timeout_seconds = 5.0 _offer(recorder) - await recorder.flush() - assert (await _rows(database))[0].requests == 1 + target = recorder.checkpoint() + async with asyncio.timeout(6.0): + while recorder._settled < target: + await recorder.flush(target) + rows = await _rows(database) + assert len(rows) == 1 + assert rows[0].requests == 1 finally: await recorder.close() diff --git a/tests/builtin/runtime/test_topic_memory_search.py b/tests/builtin/runtime/test_topic_memory_search.py new file mode 100644 index 0000000000..95603d0565 --- /dev/null +++ b/tests/builtin/runtime/test_topic_memory_search.py @@ -0,0 +1,329 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Topic search modes and embedding orchestration at the shared service boundary.""" + +import asyncio +import sys + +import pytest +from pydantic import ValidationError + +from powercontext.builtin.artifacts.memory import EmbeddingProfile, MemoryQueryEmbedding +from powercontext.builtin.artifacts.topic_memory import ( + TopicArtifactSearchRequest, + TopicMemoryCapabilities, + TopicMemorySearchResult, +) +from powercontext.builtin.inference import EmbeddingResult, InferenceUnavailableError, InvalidInferenceOutputError + + +class Embedding: + profile = EmbeddingProfile(profile_id="topic-v1", model="test", dimension=2) + + def __init__(self, failure: Exception | None = None): + self.failure = failure + self.calls = 0 + + async def embed(self, texts, /): + self.calls += 1 + if self.failure is not None: + raise self.failure + return EmbeddingResult(vectors=((1.0, 0.0),)) + + +def _service(*, embedding=None, capabilities=None, browse=None, calls=None, observer=None): + from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher + + async def search(scope_id, query, /, **kwargs): + if calls is not None: + calls.append(kwargs) + return TopicMemorySearchResult(mode=kwargs["mode"], artifacts=() if "artifact_request" in kwargs else None) + + return TopicMemorySearcher( + search=search, get=None, browse=browse, embedding_model=embedding, capabilities=capabilities, observer=observer + ) + + +@pytest.mark.parametrize("mode", ["text", "vector", "hybrid"]) +def test_topic_service_dispatches_explicit_modes(mode): + embedding = Embedding() + calls = [] + capabilities = TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=embedding.profile) + service = _service(embedding=embedding, capabilities=capabilities, calls=calls) + result = asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle", mode=mode))) + assert result.mode == ("fts" if mode == "text" else mode) + assert result.artifacts == () + assert embedding.calls == (0 if mode == "text" else 1) + assert calls[0]["artifact_request"].mode == mode + + +@pytest.mark.parametrize( + "fields", + [ + {"mode": "vector"}, + {"mode": "hybrid"}, + {"admission": {"min_semantic_similarity": 0.3}}, + {"fusion": {"method": "rrf", "params": {"weights": {"topic_vector": 0}}}}, + ], +) +def test_topic_service_validates_vector_requirements_before_empty_scope(fields): + calls = [] + + async def browse(scope_id, *, limit, after): + raise AssertionError("invalid request must fail before browsing") # noqa: TRY003 + + service = _service(capabilities=TopicMemoryCapabilities(fts=True), browse=browse, calls=calls) + with pytest.raises(ValidationError): + asyncio.run(service.search("scope", TopicArtifactSearchRequest.model_validate({"query": "needle", **fields}))) + assert calls == [] + + +def test_topic_service_default_transient_fallback_and_explicit_mode_failure(): + calls = [] + observations = [] + failure = InferenceUnavailableError("embed", "temporarily unavailable") + embedding = Embedding(failure) + capabilities = TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=embedding.profile) + service = _service( + embedding=embedding, + capabilities=capabilities, + calls=calls, + observer=lambda mode, fallback: observations.append((mode, fallback)), + ) + result = asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle"))) + assert result.mode == "fts" + assert result.embedding_calls == 1 + assert observations == [("fts", True)] + with pytest.raises(InferenceUnavailableError): + asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle", mode="hybrid"))) + assert len(calls) == 1 + + +def test_topic_service_does_not_fallback_to_all_zero_text_weights(): + failure = InferenceUnavailableError("embed", "temporarily unavailable") + embedding = Embedding(failure) + service = _service( + embedding=embedding, + capabilities=TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=embedding.profile), + ) + request = TopicArtifactSearchRequest.model_validate({ + "query": "needle", + "fusion": {"method": "rrf", "params": {"weights": {"topic_fts": 0, "detail_fts": 0}}}, + }) + with pytest.raises(InferenceUnavailableError): + asyncio.run(service.search("scope", request)) + + +def test_topic_service_legacy_profile_reuse_has_no_new_callback_keywords(): + embedding = Embedding() + calls = [] + + async def legacy(scope, query, /, *, limit, mode, query_vector, embedding_profile): + calls.append((scope, query, query_vector, embedding_profile)) + return TopicMemorySearchResult(mode=mode) + + from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher + + service = TopicMemorySearcher(search=legacy, get=None, embedding_model=embedding) + result = asyncio.run( + service.search_legacy( + "scope", + "needle", + limit=4, + query_embedding=MemoryQueryEmbedding(query_vector=(1.0, 0.0), embedding_profile=embedding.profile), + ) + ) + assert result.embedding_calls == 0 + assert result.artifacts is None + assert embedding.calls == 0 + assert calls[0][2] == (1.0, 0.0) + + +def test_topic_service_invalid_output_propagates_without_fallback(): + embedding = Embedding(InvalidInferenceOutputError("embed", "invalid shape")) + service = _service( + embedding=embedding, + capabilities=TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=embedding.profile), + ) + with pytest.raises(InvalidInferenceOutputError): + asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle"))) + + +def test_topic_service_requires_declared_public_search_capabilities(): + service = _service() + with pytest.raises(ValidationError): + asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle", mode="text"))) + + +@pytest.mark.parametrize("mode", [None, "text", "vector", "hybrid"]) +def test_topic_empty_scope_rejects_a_supplied_context_without_authority(mode): + from powercontext.artifacts.search import ArtifactSearchExecutionContext + from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts + from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher + from powercontext.server.authz import AccessDeniedError + + async def scenario(): + embedding = Embedding() + async with open_builtin_contexts(BuiltinConfig(), embedding_model=embedding) as contexts: + service = TopicMemorySearcher( + search=contexts.search_topic_memories, + browse=contexts.browse_topic_memories, + embedding_model=embedding, + capabilities=contexts.topic_memory_index.capabilities, + ) + request = TopicArtifactSearchRequest.model_validate({ + "query": "needle", + **({} if mode is None else {"mode": mode}), + }) + with pytest.raises(AccessDeniedError): + await service.search("scope", request, execution_context=ArtifactSearchExecutionContext()) + # Missing SDK context and an explicitly trusted local invocation retain local policy. + assert (await service.search("scope", request)).hits == () + assert ( + await service.search( + "scope", request, execution_context=ArtifactSearchExecutionContext(trusted_local=True) + ) + ).hits == () + with pytest.raises(AccessDeniedError): + await service.search_legacy( + "scope", "needle", limit=4, execution_context=ArtifactSearchExecutionContext() + ) + + asyncio.run(scenario()) + + +def test_topic_service_rejects_mismatched_embedding_profile_before_empty_scope(): + embedding = Embedding() + profile = EmbeddingProfile(profile_id="other", model="other", dimension=2) + + async def browse(scope, *, limit, after): + return () + + service = _service( + embedding=embedding, + capabilities=TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=profile), + browse=browse, + ) + with pytest.raises(ValidationError): + asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle"))) + assert embedding.calls == 0 + + +@pytest.mark.parametrize( + "fields,path", + [ + ({"mode": "text", "admission": {"min_semantic_similarity": 0.3}}, ("admission", "min_semantic_similarity")), + ({"mode": "vector", "admission": {"lexical_coverage": 0.25}}, ("admission", "lexical_coverage")), + ( + {"mode": "text", "fusion": {"method": "rrf", "params": {"weights": {"topic_vector": 0}}}}, + ("fusion", "params", "weights", "topic_vector"), + ), + ({"fusion": {"method": "unknown"}}, ("fusion", "method")), + ], +) +def test_topic_service_reports_mode_parameter_conflicts_before_browsing(fields, path): + embedding = Embedding() + capabilities = TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=embedding.profile) + service = _service(embedding=embedding, capabilities=capabilities) + with pytest.raises(ValidationError) as error: + asyncio.run(service.search("scope", TopicArtifactSearchRequest.model_validate({"query": "needle", **fields}))) + assert error.value.errors()[0]["loc"] == path + + +def test_topic_service_default_empty_scope_skips_embedding_but_validates_explicit_parameters(): + embedding = Embedding() + calls = [] + + async def browse(scope, *, limit, after): + return () + + capabilities = TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=embedding.profile) + service = _service(embedding=embedding, capabilities=capabilities, browse=browse, calls=calls) + result = asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle"))) + assert result.mode == "fts" + assert embedding.calls == 0 + with pytest.raises(ValidationError): + asyncio.run( + service.search( + "scope", + TopicArtifactSearchRequest.model_validate({ + "query": "needle", + "fusion": {"method": "rrf", "params": {"weights": {"unknown": 0}}}, + }), + ) + ) + + +def test_topic_service_enforces_public_fts_query_term_limit_but_allows_vector_queries(): + embedding = Embedding() + capabilities = TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=embedding.profile) + service = _service(embedding=embedding, capabilities=capabilities) + query = " ".join(f"term{index}" for index in range(65)) + with pytest.raises(ValidationError): + asyncio.run(service.search("scope", TopicArtifactSearchRequest(query=query, mode="text"))) + result = asyncio.run(service.search("scope", TopicArtifactSearchRequest(query=query, mode="vector"))) + assert result.mode == "vector" + + +def test_topic_service_public_bare_embedding_timeout_is_a_service_failure(): + embedding = Embedding(TimeoutError("provider timeout")) + service = _service( + embedding=embedding, + capabilities=TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=embedding.profile), + ) + with pytest.raises(InferenceUnavailableError, match="embedding request timed out"): + asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle", mode="vector"))) + + +def test_topic_service_text_fallback_runs_without_provider_exception_context(): + from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher + + async def search(scope, query, /, **kwargs): + assert sys.exception() is None + return TopicMemorySearchResult(mode="fts") + + service = TopicMemorySearcher( + search=search, embedding_model=Embedding(InferenceUnavailableError("embed", "provider unavailable")) + ) + result = asyncio.run(service.search_legacy("scope", "needle", limit=2)) + assert result.mode == "fts" + + +def test_topic_service_missing_selected_exact_revision_is_a_contract_failure(): + from powercontext.artifacts import ArtifactRef + from powercontext.artifacts.search import ArtifactSearchContractError, ArtifactSearchMatch + from powercontext.builtin.artifacts.topic_memory import TopicMemorySearchHit + from powercontext.builtin.persistence.errors import RepositoryNotFoundError + from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher + + ref = ArtifactRef(family="topic-memory", artifact_id="selected", revision=1) + + async def search(scope, query, /, **kwargs): + return TopicMemorySearchResult( + mode="fts", + hits=( + TopicMemorySearchHit( + artifact_ref=ref, title="Needle", summary="Needle", score=50, matched_by=("topic_fts",) + ), + ), + matches=(ArtifactSearchMatch(ref, 0.5),), + ) + + async def get(scope, artifact_ref): + raise RepositoryNotFoundError("artifact", artifact_ref) + + service = TopicMemorySearcher(search=search, get=get, capabilities=TopicMemoryCapabilities(fts=True)) + with pytest.raises(ArtifactSearchContractError): + asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle", mode="text"))) diff --git a/tests/builtin/test_publication.py b/tests/builtin/test_publication.py index c2a9ba96af..f1798e565a 100644 --- a/tests/builtin/test_publication.py +++ b/tests/builtin/test_publication.py @@ -49,13 +49,17 @@ async def initialize(self, _connection, /) -> None: async def replace(self, _connection, _scope_id, _experience, /) -> None: raise _IndexUnavailableError - async def search(self, _connection, _scope_id, _query, _limit, /, *, admission=None): + async def search( + self, _connection, _scope_id, _query, _limit, /, *, admission=None, min_score=None, require_scores=False + ): return ExperienceSearchOutcome() async def replace_skill(self, _connection, _scope_id, _skill, _package, /) -> None: pass - async def search_skills(self, _connection, _scope_id, _query, _limit, /): + async def search_skills( + self, _connection, _scope_id, _query, _limit, /, *, admission=None, min_score=None, require_scores=False + ): return () diff --git a/tests/e2e/test_recall_sufficiency_gate.py b/tests/e2e/test_recall_sufficiency_gate.py index 104a80d9f8..6ceaf4596a 100644 --- a/tests/e2e/test_recall_sufficiency_gate.py +++ b/tests/e2e/test_recall_sufficiency_gate.py @@ -63,6 +63,7 @@ REASON_SUFFICIENT, RecallSufficiencyGate, ) +from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher from powercontext.builtin.scope import ScopeDraft # A three-term query is required: the round-zero floor only differs from the round-one floor @@ -370,7 +371,14 @@ async def scenario() -> None: ) as runtime: scope_id = await _create_scope(runtime, "topic-timeout") await _seed_topic_memories(runtime, scope_id, 1) - runtime._topic_memory_embedding_model = embedding + assert runtime._topic_memory_search is not None + runtime._topic_memory_searcher = TopicMemorySearcher( + search=runtime._topic_memory_search, + get=runtime._topic_memory_get, + browse=runtime._topic_memory_browse, + embedding_model=embedding, + observer=runtime._topic_memory_search_observer, + ) request = _memory_request(assembly=_TOPIC_MEMORY_ONLY) for attempt in (1, 2): build, effort = await _prepare_build(runtime, scope_id, request) diff --git a/tests/e2e/test_topic_search_authorization_snapshot.py b/tests/e2e/test_topic_search_authorization_snapshot.py new file mode 100644 index 0000000000..63bd9d35fa --- /dev/null +++ b/tests/e2e/test_topic_search_authorization_snapshot.py @@ -0,0 +1,535 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Real backend HTTP searches authorize and read one snapshot after inference.""" + +from __future__ import annotations + +import asyncio +import importlib.util +import json +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import replace +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import uuid4 + +import httpx +import pytest +from pydantic import SecretStr +from sqlalchemy.engine import make_url + +from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.inference import EmbeddingResult, InferenceUnavailableError +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile +from powercontext.builtin.persistence.seekdb import SeekDBConfig +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository +from powercontext.builtin.runtime.config import DatabaseConfig, RuntimeConfig +from powercontext.server.authentication import AuthenticationRejectedError, AuthenticationResult, ProviderReadiness +from powercontext.server.authz import AccessAction, AccessDecision, PrincipalRef +from powercontext.server.authz.authzen import AuthZenAuthorizationProvider +from powercontext.server.authz.composition import open_builtin_access_control, open_casbin_access_control +from powercontext.server.authz.repository import RelationalAccessRepository +from powercontext.server.authz.service import ( + AccessControlService, + AccessProviderCapabilities, + BuiltinAuthorizationProvider, +) +from powercontext.server.factory import create_server_app +from powercontext.server.settings import AccessControlConfig, McpConfig, MetricsConfig, ServerSettings + +ADMIN = PrincipalRef(type="service", id="admin") +VIEWER = PrincipalRef(type="user", id="viewer") +VIEWER_HEADERS = {"Authorization": "Bearer viewer"} + + +class _CustomBuiltin(BuiltinAuthorizationProvider): + def __init__(self, repository, policy): + super().__init__(repository, deployment_id="topic-snapshot") + self._policy = policy + + async def check_batch(self, requests, /): + decisions = await super().check_batch(requests) + return tuple( + AccessDecision(False, "custom-denial", decision.policy_revision) + if request.subject == VIEWER and request.action is AccessAction.SCOPE_READ and not self._policy["allowed"] + else decision + for request, decision in zip(requests, decisions, strict=True) + ) + + +class _IndependentAudit: + def __init__(self, repository): + self._repository = repository + + async def append_audit(self, event, /): + return await self._repository.append_audit(event) + + async def list_audit(self, **kwargs): + return await self._repository.list_audit(**kwargs) + + +class _CustomPolicyRepository(RelationalAccessRepository): + def __init__(self, database, policy): + super().__init__(database) + self._policy = policy + + async def decision_snapshot(self, subjects, **kwargs): + state = await super().decision_snapshot(subjects, **kwargs) + return replace(state, bindings=()) if VIEWER in subjects and not self._policy["allowed"] else state + + +class _Authentication: + async def authenticate(self, request): + principal = {"Bearer admin": ADMIN, "Bearer viewer": VIEWER}.get(request.headers.get("authorization")) + if principal is None: + raise AuthenticationRejectedError + return AuthenticationResult(subject=principal) + + async def readiness(self): + return ProviderReadiness(ready=True) + + +class _Embedding: + profile = EmbeddingProfile(profile_id="topic-snapshot", model="test", dimension=3, normalization="unit") + + def __init__(self): + self.started = asyncio.Event() + self.resume = asyncio.Event() + self.finished = asyncio.Event() + self.pause_query = False + self.unavailable = False + + async def embed(self, texts, /): + return EmbeddingResult(vectors=((1.0, 0.0, 0.0),) * len(texts)) + + async def embed_query(self, texts, /): + if self.pause_query: + self.started.set() + await self.resume.wait() + if self.unavailable: + raise InferenceUnavailableError("embed", "temporarily unavailable") + result = await self.embed(texts) + self.finished.set() + return result + + +@asynccontextmanager +async def _isolated_database(backend: str, tmp_path: Path) -> AsyncIterator[DatabaseConfig]: + if backend == "sqlite": + yield SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'topics.db'}") + return + if backend == "seekdb": + if importlib.util.find_spec("pylibseekdb") is None: + pytest.skip("install powercontext[seekdb] for the real embedded backend") + with TemporaryDirectory(prefix="pc-topic-", dir="/tmp") as directory: + yield SeekDBConfig(path=Path(directory) / "db") + return + configured_url = os.environ.get("POWERCONTEXT_TEST_OCEANBASE_URL") + if not configured_url: + pytest.skip("set POWERCONTEXT_TEST_OCEANBASE_URL with test database creation and deletion privileges") + name = f"pc_topic_{uuid4().hex}" + async with OceanBaseProfile.open(OceanBaseConfig(url=SecretStr(configured_url)), tables=()) as profile: + async with profile.database.transaction() as connection: + await connection.exec_driver_sql(f"CREATE DATABASE `{name}`") + try: + url = make_url(configured_url).set(database=name).render_as_string(hide_password=False) + yield OceanBaseConfig(url=SecretStr(url)) + finally: + async with profile.database.transaction() as connection: + await connection.exec_driver_sql(f"DROP DATABASE `{name}`") + + +@asynccontextmanager +async def _server( + tmp_path, provider, embedding, *, database=None, external_policy=None, access_factory=None, access_database=None +): + database = database or SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'topics.db'}") + opener = open_casbin_access_control if provider == "casbin" else open_builtin_access_control + + async def evaluate(request): + payload = json.loads(request.content) + + def allowed(evaluation): + return external_policy is None or evaluation["subject"]["id"] != VIEWER.id or external_policy["allowed"] + + if request.url.path.endswith("/evaluations"): + return httpx.Response(200, json={"evaluations": [{"decision": allowed(p)} for p in payload["evaluations"]]}) + decision = allowed(payload) + if ( + external_policy is not None + and external_policy.get("pause_point") + and payload["subject"]["id"] == VIEWER.id + and payload["action"]["name"] == "scope.read" + and embedding.finished.is_set() + ): + external_policy["decision_started"].set() + await external_policy["resume_decision"].wait() + return httpx.Response(200, json={"decision": decision}) + + async with ( + opener( + access_database or database, bootstrap_administrators=(ADMIN,), deployment_id="topic-snapshot" + ) as access, + httpx.AsyncClient(transport=httpx.MockTransport(evaluate)) as pdp, + ): + if external_policy is not None: + access = AccessControlService( + AuthZenAuthorizationProvider("http://127.0.0.1:9876", http_client=pdp), + relationships=None, + audit=access.audit, + deployment_id="topic-snapshot", + provider_capabilities=AccessProviderCapabilities( + safe_resource_filtering=False, multi_requirement_check=True, relationship_management=False + ), + ) + if access_factory is not None: + access = await access_factory(access) + app = create_server_app( + settings=ServerSettings( + database=database, + runtime=RuntimeConfig(artifact_processing_families=()), + access=AccessControlConfig(mode="enforced", deployment_id="topic-snapshot"), + metrics=MetricsConfig(enabled=False), + mcp=McpConfig(enabled=False), + ), + scheduler_path=tmp_path / "scheduler.db", + authentication_provider=_Authentication(), + access_control=access, + embedding_model=embedding, + ) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://testserver", + headers={"Authorization": "Bearer admin"}, + ) as client, + ): + yield client + + +def _content(label): + return {"title": f"Alpha {label}", "summary": f"Alpha {label} summary", "detail": f"Alpha {label} complete detail."} + + +async def _create(client, scope_id, label): + result = await client.post( + f"/v1/scopes/{scope_id}/artifacts", json={"family": "topic-memory", "content": _content(label)} + ) + assert result.status_code == 201, result.text + return result + + +async def _seed(client, *, grant=True): + created_scope = await client.post( + "/v1/scopes", json={"title": "Snapshot", "summary": "Snapshot", "idempotency_key": "snapshot"} + ) + assert created_scope.status_code == 201, created_scope.text + scope_id = created_scope.json()["scope_id"] + original = await _create(client, scope_id, "original") + if not grant: + return scope_id, original, None + binding = await client.post( + "/v1/access/bindings/create", + json={ + "subject": {"type": VIEWER.type, "id": VIEWER.id}, + "resource": {"type": "scope", "scope_id": scope_id}, + "role": "scope.viewer", + "idempotency_key": "viewer-scope", + }, + ) + assert binding.status_code == 201, binding.text + return scope_id, original, binding.json() + + +async def _revoke_and_write(client, scope_id, binding, original=None): + revoked = await client.post( + "/v1/access/bindings/revoke", + json={ + "binding_id": binding["binding_id"], + "expected_version": binding["version"], + "idempotency_key": "revoke-viewer", + }, + ) + assert revoked.status_code == 200, revoked.text + assert revoked.json()["state"] == "revoked" + created = await _create(client, scope_id, "new secret") + if original is not None: + revised = await client.put( + original.headers["Location"], + headers={"If-Match": original.headers["ETag"]}, + json={"content": _content("revised secret")}, + ) + assert revised.status_code == 200, revised.text + assert revised.json()["revision"] > original.json()["revision"] + return created + + +async def _search(client, scope_id, surface, mode): + if surface == "legacy": + return await client.post( + "/v1/topic-memory/search", headers=VIEWER_HEADERS, json={"scope_id": scope_id, "query": "alpha"} + ) + return await client.post( + f"/v1/scopes/{scope_id}/artifacts/topic-memory/search", + headers=VIEWER_HEADERS, + json={"query": "alpha", **({} if mode in {None, "fallback"} else {"mode": mode})}, + ) + + +async def _assert_current_access_revoked(client, scope_id, surface, mode, created): + fresh = await _search(client, scope_id, surface, mode) + assert fresh.status_code == 403, fresh.text + exact = await client.get(created.headers["Location"], headers=VIEWER_HEADERS) + assert exact.status_code == 403, exact.text + + +def _injected_access(configuration, control): + async def configure(original): + provider = original.provider + if configuration == "custom-provider": + provider = _CustomBuiltin(original.relationships, control) + elif configuration == "custom-repository": + provider = BuiltinAuthorizationProvider( + _CustomPolicyRepository(original.relationships._database, control), deployment_id="topic-snapshot" + ) + return AccessControlService( + provider, + relationships=None if configuration == "no-relationships" else original.relationships, + audit=_IndependentAudit(original.audit) if configuration == "independent-audit" else original.audit, + deployment_id="topic-snapshot", + ) + + return configure + + +def _assert_original(result, surface, original, original_read): + assert result.status_code == 200, result.text + if surface == "artifact": + results = result.json()["results"] + assert len(results) == 1, result.text + assert results[0]["artifact_id"] == original.json()["artifact_id"] + assert results[0]["revision"] == original.json()["revision"] + assert results[0]["content"] == original_read.json()["content"] + assert results[0]["lineage"]["sources"] == original_read.json()["sources"] + assert results[0]["lineage"]["artifacts"] == original_read.json()["artifacts"] + else: + hits = result.json()["hits"] + assert len(hits) == 1, result.text + assert hits[0]["artifact"]["artifact_id"] == original.json()["artifact_id"] + assert hits[0]["artifact"]["revision"] == original.json()["revision"] + assert hits[0]["title"] == _content("original")["title"] + assert hits[0]["summary"] == _content("original")["summary"] + + +@pytest.mark.parametrize("provider", ["builtin", "casbin"]) +@pytest.mark.parametrize("backend", ["sqlite", "oceanbase", "seekdb"]) +@pytest.mark.parametrize( + "surface,mode", + [("artifact", "vector"), ("artifact", "hybrid"), ("artifact", None), ("artifact", "fallback"), ("legacy", None)], +) +def test_topic_search_rechecks_scope_access_after_embedding(tmp_path, provider, backend, surface, mode): + async def scenario(): + embedding = _Embedding() + async with ( + _isolated_database(backend, tmp_path) as database, + _server(tmp_path, provider, embedding, database=database) as client, + ): + scope_id, _, binding = await _seed(client) + embedding.pause_query = True + embedding.unavailable = mode == "fallback" + pending = asyncio.create_task(_search(client, scope_id, surface, mode)) + try: + await asyncio.wait_for(embedding.started.wait(), timeout=20) + # Both commits must finish while inference is still paused. + created = await asyncio.wait_for(_revoke_and_write(client, scope_id, binding), timeout=20) + finally: + embedding.resume.set() + result = await asyncio.wait_for(pending, timeout=20) + assert result.status_code == 403, result.text + await _assert_current_access_revoked(client, scope_id, surface, mode, created) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("surface", ["artifact", "legacy"]) +def test_topic_search_preserves_authzen_point_reads_and_rechecks_after_embedding(tmp_path, surface): + async def scenario(): + policy = {"allowed": True} + embedding = _Embedding() + async with _server(tmp_path, "authzen", embedding, external_policy=policy) as client: + scope_id, _, _ = await _seed(client, grant=False) + allowed = await _search(client, scope_id, surface, "hybrid") + assert allowed.status_code == 200, allowed.text + embedding.pause_query = True + pending = asyncio.create_task(_search(client, scope_id, surface, "hybrid")) + try: + await asyncio.wait_for(embedding.started.wait(), timeout=20) + policy["allowed"] = False + created = await asyncio.wait_for(_create(client, scope_id, "new secret"), timeout=20) + finally: + embedding.resume.set() + result = await asyncio.wait_for(pending, timeout=20) + assert result.status_code == 403, result.text + await _assert_current_access_revoked(client, scope_id, surface, "hybrid", created) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("surface", ["artifact", "legacy"]) +@pytest.mark.parametrize("configuration", ["custom-provider", "custom-repository"]) +def test_topic_search_preserves_injected_custom_point_denials(tmp_path, surface, configuration): + async def scenario(): + control = {"allowed": True} + embedding = _Embedding() + async with _server(tmp_path, "builtin", embedding) as admin: + scope_id, _, _ = await _seed(admin) + async with _server( + tmp_path, "builtin", embedding, access_factory=_injected_access(configuration, control) + ) as client: + embedding.pause_query = True + pending = asyncio.create_task(_search(client, scope_id, surface, "hybrid")) + try: + await asyncio.wait_for(embedding.started.wait(), timeout=20) + control["allowed"] = False + created = await asyncio.wait_for(_create(admin, scope_id, "new secret"), timeout=20) + finally: + embedding.resume.set() + result = await asyncio.wait_for(pending, timeout=20) + assert result.status_code == 403, result.text + await _assert_current_access_revoked(client, scope_id, surface, "hybrid", created) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("surface", ["artifact", "legacy"]) +@pytest.mark.parametrize("configuration", ["no-relationships", "independent-audit", "different-database"]) +def test_topic_search_preserves_injected_builtin_point_reads(tmp_path, surface, configuration): + async def scenario(): + control = {} + embedding = _Embedding() + access_database = ( + SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'separate-access.db'}") + if configuration == "different-database" + else None + ) + async with _server(tmp_path, "builtin", embedding, access_database=access_database) as admin: + scope_id, _, binding = await _seed(admin) + async with _server( + tmp_path, + "builtin", + embedding, + access_factory=_injected_access(configuration, control), + access_database=access_database, + ) as client: + allowed = await _search(client, scope_id, surface, "hybrid") + assert allowed.status_code == 200, allowed.text + embedding.pause_query = True + pending = asyncio.create_task(_search(client, scope_id, surface, "hybrid")) + try: + await asyncio.wait_for(embedding.started.wait(), timeout=20) + created = await asyncio.wait_for(_revoke_and_write(admin, scope_id, binding), timeout=20) + finally: + embedding.resume.set() + result = await asyncio.wait_for(pending, timeout=20) + assert result.status_code == 403, result.text + await _assert_current_access_revoked(client, scope_id, surface, "hybrid", created) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("surface", ["artifact", "legacy"]) +def test_topic_search_pins_data_before_awaiting_authzen_point_read(tmp_path, surface): + async def scenario(): + policy = { + "allowed": True, + "pause_point": True, + "decision_started": asyncio.Event(), + "resume_decision": asyncio.Event(), + } + embedding = _Embedding() + async with _server(tmp_path, "authzen", embedding, external_policy=policy) as client: + scope_id, original, _ = await _seed(client, grant=False) + original_read = await client.get(original.headers["Location"], headers=VIEWER_HEADERS) + assert original_read.status_code == 200, original_read.text + embedding.pause_query = True + pending = asyncio.create_task(_search(client, scope_id, surface, "hybrid")) + try: + await asyncio.wait_for(embedding.started.wait(), timeout=20) + embedding.resume.set() + await asyncio.wait_for(policy["decision_started"].wait(), timeout=20) + policy["allowed"] = False + created = await asyncio.wait_for(_create(client, scope_id, "new secret"), timeout=20) + revised = await asyncio.wait_for( + client.put( + original.headers["Location"], + headers={"If-Match": original.headers["ETag"]}, + json={"content": _content("revised secret")}, + ), + timeout=20, + ) + assert revised.status_code == 200, revised.text + finally: + embedding.resume.set() + policy["resume_decision"].set() + result = await asyncio.wait_for(pending, timeout=20) + _assert_original(result, surface, original, original_read) + await _assert_current_access_revoked(client, scope_id, surface, "hybrid", created) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("provider", ["builtin", "casbin"]) +@pytest.mark.parametrize("backend", ["sqlite", "oceanbase", "seekdb"]) +@pytest.mark.parametrize( + "surface,mode", [("artifact", "text"), ("artifact", "vector"), ("artifact", "hybrid"), ("legacy", None)] +) +def test_topic_search_pins_authority_candidates_and_complete_content( + tmp_path, provider, backend, surface, mode, monkeypatch +): + async def scenario(): + async with ( + _isolated_database(backend, tmp_path) as database, + _server(tmp_path, provider, _Embedding(), database=database) as client, + ): + scope_id, original, binding = await _seed(client) + original_read = await client.get(original.headers["Location"], headers=VIEWER_HEADERS) + assert original_read.status_code == 200, original_read.text + authorized = asyncio.Event() + resume = asyncio.Event() + search = TopicMemoryRepository.search + + async def pause_candidates(*args, **kwargs): + authorized.set() + await resume.wait() + return await search(*args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(TopicMemoryRepository, "search", pause_candidates) + pending = asyncio.create_task(_search(client, scope_id, surface, mode)) + try: + await asyncio.wait_for(authorized.wait(), timeout=20) + # Writers commit after read authority is pinned, before candidate retrieval resumes. + created = await asyncio.wait_for(_revoke_and_write(client, scope_id, binding, original), timeout=20) + finally: + resume.set() + result = await asyncio.wait_for(pending, timeout=20) + _assert_original(result, surface, original, original_read) + await _assert_current_access_revoked(client, scope_id, surface, mode, created) + + asyncio.run(scenario()) diff --git a/tests/e2e/topic_memory_product/harness.py b/tests/e2e/topic_memory_product/harness.py index fb84c8da47..12aa927f39 100644 --- a/tests/e2e/topic_memory_product/harness.py +++ b/tests/e2e/topic_memory_product/harness.py @@ -1092,7 +1092,7 @@ def run_e2( # noqa: C901 fallback_server = None unavailable_embedding_server = None fallback_capture = _EmbeddingFallbackCapture() - search_logger = logging.getLogger("powercontext.builtin.runtime.application") + search_logger = logging.getLogger("powercontext.builtin.runtime") search_logger.addHandler(fallback_capture) temporary_openai_key = False result: dict[str, object] | None = None diff --git a/tests/test_access_adapters.py b/tests/test_access_adapters.py index 5368e7b7b8..1973aea636 100644 --- a/tests/test_access_adapters.py +++ b/tests/test_access_adapters.py @@ -16,6 +16,8 @@ import asyncio import json +from collections.abc import Sequence +from dataclasses import replace from datetime import UTC, datetime import httpx @@ -29,9 +31,11 @@ AccessBinding, AccessBindingState, AccessControlService, + AccessDeniedError, AccessProviderCapabilities, AccessRequest, AccessRole, + AccessSubjectRef, AccessUnavailableError, AuthZenAuthorizationProvider, BuiltinAuthorizationProvider, @@ -45,6 +49,7 @@ ) from powercontext.server.authz.composition import open_casbin_access_control from powercontext.server.authz.repository import ACCESS_TABLES, RelationalAccessRepository +from powercontext.server.authz.service import DecisionState ADMIN = PrincipalRef(type="service", id="admin") ALICE = PrincipalRef(type="user", id="alice", description="Alice") @@ -114,6 +119,66 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_casbin_transaction_binding_preserves_custom_repository_policy() -> None: + class ReadOnlyScopeRepository(RelationalAccessRepository): + async def decision_snapshot( + self, + subjects: Sequence[AccessSubjectRef], + *, + now: datetime, + artifact_resources: Sequence[ResourceRef] = (), + owned_by: PrincipalRef | None = None, + ) -> DecisionState: + state = await super().decision_snapshot( + subjects, + now=now, + artifact_resources=artifact_resources, + owned_by=owned_by, + ) + return replace( + state, + bindings=tuple( + binding for binding in state.bindings if binding.role is not AccessRole.SCOPE_CONTRIBUTOR + ), + ) + + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=ACCESS_TABLES) as profile: + repository = RelationalAccessRepository(profile.database) + await _seed_admin(repository) + scope = ResourceRef.scope("scope-a") + setup = AccessControlService( + BuiltinAuthorizationProvider(repository), relationships=repository, audit=repository + ) + for role in (AccessRole.SCOPE_VIEWER, AccessRole.SCOPE_CONTRIBUTOR): + await setup.create_binding( + ADMIN, + CreateBinding(subject=BOB, resource=scope, role=role, idempotency_key=role.value), + context=AUDIT, + ) + + service = AccessControlService( + CasbinAuthorizationProvider(ReadOnlyScopeRepository(profile.database)), + relationships=repository, + audit=repository, + ) + assert (await service.require(BOB, AccessAction.SCOPE_READ, scope, context=AUDIT)).allowed + with pytest.raises(AccessDeniedError): + await service.require(BOB, AccessAction.SCOPE_CONTRIBUTE, scope, context=AUDIT) + assert any( + binding.role is AccessRole.SCOPE_CONTRIBUTOR + for binding in await repository.active_bindings((BOB,), now=datetime.now(UTC)) + ) + + async with profile.database.transaction() as connection: + bound = service.with_connection(connection) + assert (await bound.require(BOB, AccessAction.SCOPE_READ, scope, context=AUDIT)).allowed + with pytest.raises(AccessDeniedError): + await bound.require(BOB, AccessAction.SCOPE_CONTRIBUTE, scope, context=AUDIT) + + asyncio.run(scenario()) + + def test_authzen_uses_logical_identity_and_trusted_powercontext_context() -> None: seen: list[dict[str, object]] = [] subject = PrincipalRef(type="user", id="workforce:alice", description="Alice") diff --git a/tests/test_access_http.py b/tests/test_access_http.py index 6a2c76093f..10107e701c 100644 --- a/tests/test_access_http.py +++ b/tests/test_access_http.py @@ -18,7 +18,7 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from types import SimpleNamespace -from typing import Self +from typing import Any, Self, cast import httpx import pytest @@ -26,7 +26,7 @@ from pydantic import SecretStr from starlette.middleware import Middleware -from powercontext.artifacts import ArtifactAddress, ArtifactRef +from powercontext.artifacts import ArtifactAddress, ArtifactRef, ArtifactSearchExecutionContext from powercontext.builtin.artifacts.memory import MemoryEntryVersion from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.publication import ArtifactPublication, ArtifactPublicationRequest @@ -47,6 +47,7 @@ AccessRole, BuiltinAuthorizationProvider, CreateBinding, + GroupRef, MemoryEntrySelector, PrincipalRef, ResourceRef, @@ -1025,6 +1026,97 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("access_mode", ["enforced", "disabled"]) +def test_artifact_search_uses_path_scope_permission_and_waits_for_content_owners(access_mode) -> None: + group = GroupRef(type="group", id="search-group") + + class Provider(_ActingAuthenticationProvider): + async def authenticate(self, request) -> AuthenticationResult: + return AuthenticationResult(subject=BOB, actor=ADMIN, subject_groups=(group,)) + + class Application: + def __init__(self) -> None: + self.records = self.artifacts = self + self.scope_id = "" + self.reads: list[str] = [] + self.contexts: list[ArtifactSearchExecutionContext | None] = [] + + def for_scope(self, scope_id: str): + self.scope_id = scope_id + return self + + async def logical_artifacts(self): + return (SimpleNamespace(family="skill", artifact_id="pending", entry_id=None),) + + async def search(self, family, payload, *, execution_context: ArtifactSearchExecutionContext | None = None): + self.reads.append(self.scope_id) + self.contexts.append(execution_context) + return SimpleNamespace(matches=(), artifacts=()) + + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=ACCESS_TABLES) as profile: + repository = RelationalAccessRepository(profile.database) + await _seed_admin(repository) + service = AccessControlService( + BuiltinAuthorizationProvider(repository), relationships=repository, audit=repository + ) + await service.create_binding( + ADMIN, + CreateBinding( + subject=BOB, + resource=ResourceRef.scope("scope-a"), + role=AccessRole.SCOPE_VIEWER, + idempotency_key="search-viewer", + ), + context=AUDIT, + ) + application = Application() + provider = Provider() + app = create_app( + application=cast(Any, application), + access_control=service, + access_mode=access_mode, + authentication_provider=provider, + middleware=(Middleware(AuthenticationMiddleware, provider=provider),), + ) + path = "/v1/scopes/scope-a/artifacts/custom-family/search" + async with _client(app) as client: + denied = await client.post( + path.replace("scope-a", "scope-b"), headers=_auth("search-token"), json={"query": "x"} + ) + injected = await client.post( + path, headers=_auth("search-token"), json={"query": "x", "scope_id": "scope-b"} + ) + pending = await client.post(path, headers=_auth("search-token"), json={"query": "x"}) + assert denied.status_code == 403 + assert injected.status_code == 422 + assert pending.status_code == 503 + assert pending.json()["error"]["code"] == "artifact_owner_pending" + assert application.reads == [] + await service.establish_artifact_owner( + ResourceRef.artifact("scope-a", family="skill", artifact_id="pending"), + BOB, + idempotency_key="search-owner", + context=AUDIT, + ) + ready = await client.post(path, headers=_auth("search-token"), json={"query": "x"}) + assert ready.status_code == 200 + assert ready.json() == {"results": []} + assert application.reads == ["scope-a"] + context = application.contexts[0] + assert context is not None + assert context.principal == BOB + assert context.access is service + assert context.trusted_local is False + assert context.audit.actor == ADMIN + assert context.audit.subject_groups == (group,) + assert context.audit.operation == "search_artifacts" + assert context.audit.transport == "http" + assert context.audit.request_id == ready.headers["x-powercontext-request-id"] + + asyncio.run(scenario()) + + def _app(service: AccessControlService, *, principal: PrincipalRef, token: str, application=None): provider = StaticBearerAuthenticationProvider(token, principal) return create_app( diff --git a/tests/test_api_contract.py b/tests/test_api_contract.py index 7df86ab2f3..3b6a4694c4 100644 --- a/tests/test_api_contract.py +++ b/tests/test_api_contract.py @@ -751,6 +751,7 @@ def test_base_access_contract_includes_revision_history_and_tags() -> None: paths = contract["paths"] expected_operations = { + ("/v1/scopes/{scope_id}/artifacts/{family}/search", "post"): "search_artifacts", ("/v1/scopes/{scope_id}/artifacts/{family}/{artifact_id}/tags", "get"): "get_artifact_tags", ("/v1/scopes/{scope_id}/artifacts/{family}/{artifact_id}/tags", "put"): "replace_artifact_tags", ( @@ -782,7 +783,7 @@ def test_base_access_contract_includes_revision_history_and_tags() -> None: } assert actual_operations == expected_operations assert not any( - operation_id in {"search_sources", "search_artifacts", "delete_artifact", "list_scopes"} + operation_id in {"search_sources", "delete_artifact", "list_scopes"} for operation_id in actual_operations.values() ) assert not any("search-results" in path for path in paths) diff --git a/tests/test_artifact_fusion.py b/tests/test_artifact_fusion.py new file mode 100644 index 0000000000..453a2110bd --- /dev/null +++ b/tests/test_artifact_fusion.py @@ -0,0 +1,278 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Behavior contracts for family-independent Artifact fusion.""" + +import math +import sys +from decimal import Decimal + +import pytest +from pydantic import ValidationError + + +def test_rrf_preserves_missing_channels_and_rank_gaps(): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + channels = ( + FusionChannel("text", 1.0, (FusionCandidate("both", 1), FusionCandidate("text-only", 3))), + FusionChannel("vector", 1.0, (FusionCandidate("both", 1),)), + ) + hits = fuse_rrf(channels, RrfParameters(), tie_break=lambda key: key) + + assert [hit.key for hit in hits] == ["both", "text-only"] + assert hits[0].score == 1.0 + assert hits[0].raw_score == 2 / 61 + assert hits[1].score == pytest.approx(61 / 126) + assert hits[1].raw_score == 1 / 63 + + +def test_rrf_empty_enabled_channel_contributes_to_normalization(): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + hits = fuse_rrf( + (FusionChannel("text", 1, (FusionCandidate("a", 1),)), FusionChannel("vector", 1, ())), + RrfParameters(), + tie_break=lambda key: key, + ) + assert hits[0].score == 0.5 + + +def test_rrf_uses_family_tie_break_and_excludes_zero_weight_only_candidates(): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + hits = fuse_rrf( + ( + FusionChannel("left", 1, (FusionCandidate("b", 1),)), + FusionChannel("right", 1, (FusionCandidate("a", 1),)), + FusionChannel("ignored", 0, (FusionCandidate("phantom", 1),)), + ), + RrfParameters(), + tie_break=lambda key: key, + ) + assert [hit.key for hit in hits] == ["a", "b"] + assert [hit.score for hit in hits] == [0.5, 0.5] + + +@pytest.mark.parametrize("value", [True, "1", 1.5, 0, -1]) +def test_rrf_rank_constant_is_a_positive_strict_integer(value): + from powercontext.artifacts.fusion import RrfParameters + + with pytest.raises(ValidationError): + RrfParameters(rank_constant=value) + + +@pytest.mark.parametrize("value", [True, "1", -0.1, math.inf, -math.inf, math.nan]) +def test_rrf_weights_require_finite_nonnegative_real_numbers(value): + from powercontext.artifacts.fusion import RrfParameters + + with pytest.raises(ValidationError): + RrfParameters(weights={"text": value}) + + +def test_fusion_selection_owns_method_specific_parameters(): + from powercontext.artifacts.fusion import FusionSelection, RrfParameters, parse_fusion_parameters + + selection = FusionSelection(method="rrf", params={"weights": {"text": 0.0}, "rank_constant": 7}) + params = parse_fusion_parameters(selection) + assert isinstance(params, RrfParameters) + assert params.weights == {"text": 0.0} + assert params.rank_constant == 7 + with pytest.raises(ValidationError): + FusionSelection.model_validate({"method": "rrf", "rank_constant": 7}) + with pytest.raises(ValidationError): + parse_fusion_parameters(FusionSelection(method="rrf", params={"unknown": 1})) + with pytest.raises(ValueError, match="unsupported fusion method"): + parse_fusion_parameters(FusionSelection(method="weighted_score")) + + +def test_fusion_method_dispatch_preserves_family_supplied_channel_weights(): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, FusionSelection, fuse + + hits = fuse( + (FusionChannel("text", 3, (FusionCandidate("a", 1),)), FusionChannel("vector", 1, ())), + FusionSelection(method="rrf"), + tie_break=lambda key: key, + ) + assert hits[0].score == 0.75 + + +@pytest.mark.parametrize("payload", [{"method": " "}, {"method": 1}, {"method": "rrf", "params": []}]) +def test_fusion_selection_requires_nonempty_method_and_object_parameters(payload): + from powercontext.artifacts.fusion import FusionSelection + + with pytest.raises(ValidationError): + FusionSelection.model_validate(payload) + + +@pytest.mark.parametrize( + "defect", + ["duplicate-channel", "duplicate-key", "unordered-ranks", "duplicate-rank", "bad-rank", "bad-weight", "all-zero"], +) +def test_rrf_rejects_invalid_internal_input(defect): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + if defect == "duplicate-channel": + channels = (FusionChannel("text", 1, ()), FusionChannel("text", 1, ())) + elif defect == "duplicate-key": + channels = (FusionChannel("text", 1, (FusionCandidate("a", 1), FusionCandidate("a", 2))),) + elif defect == "unordered-ranks": + channels = (FusionChannel("text", 1, (FusionCandidate("a", 3), FusionCandidate("b", 1))),) + elif defect == "duplicate-rank": + channels = (FusionChannel("text", 1, (FusionCandidate("a", 1), FusionCandidate("b", 1))),) + elif defect == "bad-rank": + channels = (FusionChannel("text", 1, (FusionCandidate("a", True),)),) + elif defect == "bad-weight": + channels = (FusionChannel("text", math.inf, ()),) + else: + channels = (FusionChannel("text", 0, ()),) + with pytest.raises(ValueError): + fuse_rrf(channels, RrfParameters(), tie_break=lambda key: key) + + +def test_rrf_large_weights_normalize_without_overflow(): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + hits = fuse_rrf( + (FusionChannel("text", 1e308, (FusionCandidate("a", 1),)), FusionChannel("vector", 1e308, ())), + RrfParameters(), + tie_break=lambda key: key, + ) + assert hits[0].score == 0.5 + assert math.isfinite(hits[0].raw_score) + + +@pytest.mark.parametrize( + "weight", [math.ulp(0.0), 1e-320, sys.float_info.min, sys.float_info.min * 30.5, sys.float_info.max] +) +def test_rrf_uniform_weight_scaling_preserves_normalized_scores_and_threshold(weight): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + def search(channel_weight): + return fuse_rrf( + ( + FusionChannel("left", channel_weight, (FusionCandidate("a", 1), FusionCandidate("b", 2))), + FusionChannel("right", channel_weight, (FusionCandidate("b", 1), FusionCandidate("a", 2))), + ), + RrfParameters(), + tie_break=lambda key: key, + ) + + ordinary = search(1.0) + scaled = search(weight) + assert [hit.key for hit in scaled] == [hit.key for hit in ordinary] + assert [hit.score for hit in scaled] == pytest.approx([hit.score for hit in ordinary], rel=1e-15, abs=0) + assert [hit.key for hit in scaled if hit.score >= 0.999] == [] + + +@pytest.mark.parametrize("weight", [math.ulp(0.0), 1e-320, sys.float_info.min, sys.float_info.max]) +def test_rrf_uniform_weight_scaling_preserves_distinct_candidate_order(weight): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + hits = fuse_rrf( + ( + FusionChannel("left", weight, (FusionCandidate("z", 1), FusionCandidate("a", 2))), + FusionChannel("right", weight, (FusionCandidate("a", 3), FusionCandidate("z", 4))), + ), + RrfParameters(), + tie_break=lambda key: key, + ) + assert [hit.key for hit in hits] == ["z", "a"] + assert hits[0].score > hits[1].score + + +def test_rrf_mixed_normal_and_subnormal_weights_preserve_normal_raw_score(): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + hits = fuse_rrf( + ( + FusionChannel("normal", 1.0, (FusionCandidate("both", 1),)), + FusionChannel("subnormal", math.ulp(0.0), (FusionCandidate("both", 1), FusionCandidate("tiny", 2))), + ), + RrfParameters(), + tie_break=lambda key: key, + ) + assert [hit.key for hit in hits] == ["both", "tiny"] + assert hits[0].score == 1.0 + assert isinstance(hits[0].raw_score, float) + assert hits[0].raw_score == 1 / 61 + assert hits[1].raw_score > 0 + + +@pytest.mark.parametrize("empty_weight, expected_score", [(120.0, math.ulp(0.0)), (124.0, math.ulp(0.0)), (244.0, 0.0)]) +def test_rrf_subnormal_contributions_combine_before_normalization(empty_weight, expected_score): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + hits = fuse_rrf( + ( + FusionChannel("empty", empty_weight, ()), + FusionChannel("left", 61 * math.ulp(0.0), (FusionCandidate("a", 1),)), + FusionChannel("right", 61 * math.ulp(0.0), (FusionCandidate("a", 1),)), + ), + RrfParameters(), + tie_break=lambda key: key, + ) + assert hits[0].raw_score > 0 + assert hits[0].score == expected_score + + +def test_rrf_large_integer_rank_constant_normalizes_without_overflow(): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + hits = fuse_rrf( + (FusionChannel("text", 1, (FusionCandidate("a", 1),)), FusionChannel("vector", 1, ())), + RrfParameters(rank_constant=10**400), + tie_break=lambda key: key, + ) + assert hits[0].score == 0.5 + assert hits[0].raw_score > 0 + + +def test_rrf_raw_overflow_uses_a_finite_decimal(): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + hits = fuse_rrf( + tuple(FusionChannel(str(index), 1e308, (FusionCandidate("a", 1),)) for index in range(4)), + RrfParameters(rank_constant=1), + tie_break=lambda key: key, + ) + assert hits[0].score == 1.0 + assert isinstance(hits[0].raw_score, Decimal) + assert hits[0].raw_score.is_finite() + assert hits[0].raw_score > Decimal("1e308") + + +def test_rrf_decimal_raw_order_preserves_ranks_beyond_float_precision(): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + hits = fuse_rrf( + (FusionChannel("text", 1, (FusionCandidate("z", 1), FusionCandidate("a", 3))),), + RrfParameters(rank_constant=10**400), + tie_break=lambda key: key, + ) + assert [hit.key for hit in hits] == ["z", "a"] + assert hits[0].raw_score > hits[1].raw_score + + +def test_rrf_normal_float_path_preserves_legacy_topic_accumulation(): + from powercontext.artifacts.fusion import FusionCandidate, FusionChannel, RrfParameters, fuse_rrf + + ranks = (1, 1, 1, 9) + hits = fuse_rrf( + tuple(FusionChannel(str(index), 1, (FusionCandidate("a", rank),)) for index, rank in enumerate(ranks)), + RrfParameters(), + tie_break=lambda key: key, + ) + assert hits[0].raw_score == 0.06367308149204086 + assert hits[0].score * 100 == 97.10144927536231 diff --git a/tests/test_artifact_search_server.py b/tests/test_artifact_search_server.py new file mode 100644 index 0000000000..68622f7215 --- /dev/null +++ b/tests/test_artifact_search_server.py @@ -0,0 +1,459 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import math +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any, ClassVar, cast + +import pytest +from fastapi.testclient import TestClient +from pydantic import BaseModel, Field, ValidationError + +from powercontext import http +from powercontext.artifacts import Artifact, ArtifactLineage +from powercontext.artifacts.search import ( + ArtifactSearchExecutionContext, + ArtifactSearchMatch, + ArtifactSearchQuery, + ChannelScore, +) +from powercontext.builtin.inference import InferenceTimeoutError, InferenceUnavailableError +from powercontext.builtin.runtime import BuiltinRuntime, RuntimeCapabilities +from powercontext.builtin.runtime.artifact_search import ArtifactSearchService +from powercontext.server.app import create_app +from powercontext.sources import SourceRef + +PATH = "/v1/scopes/scope-a/artifacts/custom-family/search" + + +class Content(BaseModel): + title: str + detail: str + + +class CustomArtifact(Artifact[Content]): + family: ClassVar[str] = "custom-family" + + +class Query(ArtifactSearchQuery): + filters: dict[str, Any] = Field(default_factory=dict) + min_score: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False) + + +@dataclass +class Outcome: + matches: tuple[ArtifactSearchMatch, ...] + artifacts: tuple[CustomArtifact, ...] + + +class Searcher: + family = "custom-family" + request_type = Query + + def __init__(self, error: Exception | None = None) -> None: + self.calls: list[Query] = [] + self.contexts: list[ArtifactSearchExecutionContext | None] = [] + self.error = error + + async def search( + self, scope_id: str, request: Query, /, *, execution_context: ArtifactSearchExecutionContext | None = None + ) -> Outcome: + assert scope_id == "scope-a" + self.calls.append(request) + self.contexts.append(execution_context) + if self.error is not None: + raise self.error + artifacts = tuple( + CustomArtifact( + artifact_id=key, + revision=3, + content=Content(title="中文", detail="complete detail"), + lineage=ArtifactLineage(sources=(SourceRef(source_type="content", source_id="proof"),)), + ) + for key in ("z-first", "a-second") + ) + pairs = [ + (artifact, score) + for artifact, score in zip(artifacts, (0.8, 0.2), strict=True) + if request.min_score is None or score >= request.min_score + ] + return Outcome( + matches=tuple( + ArtifactSearchMatch(artifact.as_ref(), score, {"text": ChannelScore(-2.0, "custom_raw", False)}) + for artifact, score in pairs + ), + artifacts=tuple(artifact for artifact, _ in pairs), + ) + + +class Scopes: + async def get(self, scope_id: str, /) -> Any: + return SimpleNamespace(scope_id=scope_id) + + +class Provider: + async def get(self, scope_id: str, /) -> Any: + raise AssertionError(scope_id) + + +def _client(searcher: Searcher | None = None) -> TestClient: + service = ArtifactSearchService(known_families={"custom-family", "known-disabled"}) + service.register(searcher or Searcher()) + runtime = BuiltinRuntime( + provider=Provider(), + scope_application=cast(Any, Scopes()), + capabilities=RuntimeCapabilities(memory_extraction=False, memory_search_modes=()), + artifact_search=service, + ) + return TestClient(create_app(application=cast(Any, runtime)), raise_server_exceptions=False) + + +def test_artifact_search_http_returns_arbitrary_family_full_content_and_optional_raw_scores() -> None: + searcher = Searcher() + with _client(searcher) as client: + plain = client.post(PATH, json={"query": " focused query ", "filters": {"tags": ["a"]}}) + scored = client.post(PATH, json={"query": "focused query", "include_scores": True}) + threshold = client.post(PATH, json={"query": "focused query", "min_score": 0.8}) + assert plain.status_code == scored.status_code == threshold.status_code == 200 + assert [item["artifact_id"] for item in plain.json()["results"]] == ["z-first", "a-second"] + assert [item["artifact_id"] for item in scored.json()["results"]] == ["z-first", "a-second"] + item = plain.json()["results"][0] + assert item["family"] == "custom-family" + assert item["content"] == {"title": "中文", "detail": "complete detail"} + assert item["lineage"]["sources"] == [{"source_type": "content", "source_id": "proof"}] + assert "scores" not in item + assert scored.json()["results"][0]["scores"] == { + "retrieval": 0.8, + "channels": {"text": {"raw": -2.0, "metric": "custom_raw", "higher_is_better": False}}, + } + assert [item["artifact_id"] for item in threshold.json()["results"]] == ["z-first"] + assert searcher.calls[0].model_fields_set == {"query", "filters"} + + +def test_artifact_search_http_preserves_custom_content_json_aliases() -> None: + class AliasedContent(Content): + schema_: str = Field(alias="schema") + + class AliasedSearcher(Searcher): + async def search( + self, scope_id: str, request: Query, /, *, execution_context: ArtifactSearchExecutionContext | None = None + ) -> Outcome: + outcome = await super().search(scope_id, request, execution_context=execution_context) + content = AliasedContent.model_validate({ + "title": "中文", + "detail": "complete detail", + "schema": "custom-v1", + }) + return Outcome( + matches=outcome.matches, + artifacts=tuple(artifact.model_copy(update={"content": content}) for artifact in outcome.artifacts), + ) + + with _client(AliasedSearcher()) as client: + response = client.post(PATH, json={"query": "needle"}) + + assert response.status_code == 200 + assert response.json()["results"][0]["content"] == { + "title": "中文", + "detail": "complete detail", + "schema": "custom-v1", + } + + +@pytest.mark.parametrize( + "body", + [ + {"query": " "}, + {"query": "x" * 8193}, + {"limit": True}, + {"limit": "10"}, + {"limit": 201}, + {"include_scores": 1}, + {"min_score": True}, + {"min_score": "0.5"}, + {"min_score": -0.1}, + {"min_score": 1.1}, + {"filters": []}, + {"fusion": {}}, + {"fusion": {"method": "rrf", "unknown": 1}}, + *( + {field: None} + for field in ( + "query", + "limit", + "include_scores", + "mode", + "filters", + "admission", + "fusion", + "min_score", + "rerank", + ) + ), + *( + {field: "injected"} + for field in ( + "scope_id", + "family", + "principal", + "access", + "audit", + "trusted_local", + "execution_context", + "page", + "offset", + "cursor", + "unknown", + ) + ), + ], +) +def test_artifact_search_http_rejects_fixed_body_contract_before_read(body: dict[str, object]) -> None: + searcher = Searcher() + response = _client(searcher).post(PATH, json={"query": "private-query", **body}) + assert response.status_code == 422 + assert response.json()["error"]["code"] == "invalid_request" + assert searcher.calls == [] + assert "private-query" not in str(response.json()) + + +def test_artifact_search_http_distinguishes_unknown_disabled_and_internal_failure() -> None: + with _client() as client: + unknown = client.post(PATH.replace("custom-family", "unknown"), json={"query": "needle"}) + disabled = client.post(PATH.replace("custom-family", "known-disabled"), json={"query": "needle"}) + assert unknown.status_code == 404 + assert unknown.json()["error"]["code"] == "artifact_family_not_found" + assert disabled.status_code == 422 + assert disabled.json()["error"]["code"] == "artifact_search_not_supported" + failed = _client(Searcher(RuntimeError("private-failure"))).post(PATH, json={"query": "needle"}) + assert failed.status_code == 500 + assert failed.json()["error"]["code"] == "internal_error" + unavailable = _client(Searcher(InferenceUnavailableError("embed"))).post(PATH, json={"query": "needle"}) + assert unavailable.status_code == 503 + assert unavailable.json()["error"]["code"] == "inference_unavailable" + missing = TestClient(create_app(application=SimpleNamespace()), raise_server_exceptions=False).post( + PATH, json={"query": "needle"} + ) + assert missing.status_code == 422 + assert missing.json()["error"]["code"] == "artifact_search_not_supported" + + +def test_artifact_search_transport_model_trims_before_bounds_and_rejects_nonfinite() -> None: + request_type = http.SearchArtifactsRequest + assert request_type(query=" " + "x" * 8192 + " ").query == "x" * 8192 + for value in (math.inf, math.nan, -math.inf): + with pytest.raises(ValidationError): + request_type(query="needle", min_score=value) + with pytest.raises(ValidationError) as nested: + request_type(query="needle", filters={"weights": [value]}) + assert nested.value.errors()[0]["loc"] == ("filters", "weights", 0) + + +@pytest.mark.parametrize("error", [InferenceTimeoutError("embed", 0.1), InferenceUnavailableError("embed")]) +def test_artifact_search_http_retains_typed_inference_unavailable_status(error: Exception) -> None: + response = _client(Searcher(error)).post(PATH, json={"query": "needle"}) + assert response.status_code == 503 + assert response.json()["error"]["code"] in {"inference_timeout", "inference_unavailable"} + + +def test_artifact_search_http_disabled_access_supplies_explicit_trusted_execution_context() -> None: + searcher = Searcher() + with _client(searcher) as client: + response = client.post(PATH, json={"query": "needle"}) + assert response.status_code == 200 + context = searcher.contexts[0] + assert context is not None + assert context.principal is None + assert context.access is None + assert context.trusted_local is True + assert context.audit.transport == "http" + assert context.audit.operation == "search_artifacts" + assert context.audit.request_id == response.headers["x-powercontext-request-id"] + assert context.audit.subject_groups == () + assert "execution_context" not in response.json()["results"][0] + + +def test_artifact_search_http_reports_family_validation_paths_and_nonfinite_nested_values() -> None: + searcher = Searcher() + with _client(searcher) as client: + mode = client.post(PATH, json={"query": "private-query", "mode": "semantic"}) + nested = client.post( + PATH, + content='{"query":"private-query","filters":{"weight":NaN}}', + headers={"content-type": "application/json"}, + ) + assert mode.status_code == nested.status_code == 422 + assert mode.json()["error"]["code"] == nested.json()["error"]["code"] == "invalid_request" + assert any("mode" in detail["loc"] for detail in mode.json()["error"]["details"]["errors"]) + assert any("weight" in detail["loc"] for detail in nested.json()["error"]["details"]["errors"]) + assert searcher.calls == [] + assert "private-query" not in str(mode.json()) + str(nested.json()) + + +@pytest.mark.parametrize("family", ["experience", "skill"]) +def test_sqlite_artifact_search_http_returns_exact_current_revision_and_lineage(family: str, tmp_path) -> None: + from powercontext.builtin.artifacts.experience import ExperienceContent + from powercontext.builtin.artifacts.skill import SkillContent + from powercontext.builtin.persistence.sqlite import SQLiteConfig + from powercontext.builtin.runtime import ( + ApproveArtifactCandidateRequest, + CaptureSource, + GetExperienceRequest, + GetSkillRequest, + ProposeExperienceRequest, + ProposeSkillRequest, + ) + from powercontext.builtin.scope import ScopeDraft + from powercontext.server.factory import create_server_app + from powercontext.server.settings import ServerSettings + + app = create_server_app( + settings=ServerSettings(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'http.db'}")) + ) + + async def seed(): + runtime = app.state.application + scope = ( + await runtime.scopes.create(ScopeDraft(title="Search", summary="HTTP", idempotency_key="http")) + ).scope_id + source = await runtime.sources.for_scope(scope).capture( + CaptureSource(source_id="proof", content="needle", metadata={}) + ) + references = [] + for marker in ("oldmarker", "newmarker"): + target = references[-1] if references else None + if family == "experience": + content = ExperienceContent(situation=marker, action="Inspect", outcome="Complete", lesson="Verify") + candidate = await runtime.experience.for_scope(scope).propose( + ProposeExperienceRequest( + proposal=content, sources=(source.source_ref,), artifacts=tuple(references), target=target + ) + ) + else: + content = SkillContent( + name=marker, description="Inspect", instructions="Complete", validation=("Verify",) + ) + candidate = await runtime.skill.for_scope(scope).propose( + ProposeSkillRequest( + proposal=content, sources=(source.source_ref,), artifacts=tuple(references), target=target + ) + ) + approved = await runtime.review.for_scope(scope).approve( + ApproveArtifactCandidateRequest(candidate_id=candidate.candidate_id, expected_version=candidate.version) + ) + assert approved.result_artifact is not None + references.append(approved.result_artifact) + exact = ( + await runtime.experience.for_scope(scope).get(GetExperienceRequest(artifact=references[-1])) + if family == "experience" + else await runtime.skill.for_scope(scope).get(GetSkillRequest(artifact=references[-1])) + ) + return scope, references, source.source_ref, exact.content + + with TestClient(app) as client: + assert client.portal is not None + scope, refs, source, content = client.portal.call(seed) + path = f"/v1/scopes/{scope}/artifacts/{family}/search" + plain = client.post(path, json={"query": "newmarker", "limit": 200}) + scored = client.post(path, json={"query": "newmarker", "limit": 200, "include_scores": True}) + rejected = client.post(path, json={"query": "newmarker", "min_score": 1}) + old = client.post(path, json={"query": "oldmarker"}) + for field, value in ( + ("mode", "semantic"), + ("filters", {"tags": ["unsupported"]}), + ("fusion", {"method": "rrf"}), + ("rerank", {}), + ): + invalid = client.post(path, json={"query": "newmarker", field: value}) + assert invalid.status_code == 422 + assert invalid.json()["error"]["code"] == "invalid_request" + assert any(field in detail["loc"] for detail in invalid.json()["error"]["details"]["errors"]) + assert plain.status_code == scored.status_code == rejected.status_code == old.status_code == 200 + item = plain.json()["results"][0] + assert (item["family"], item["artifact_id"], item["revision"]) == (family, refs[-1].artifact_id, refs[-1].revision) + assert item["content"] == content.model_dump(mode="json") + assert item["lineage"]["sources"] == [source.model_dump(mode="json")] + assert item["lineage"]["artifacts"] == [refs[0].model_dump(mode="json")] + assert "scores" not in item + assert scored.json()["results"][0]["artifact_id"] == item["artifact_id"] + assert scored.json()["results"][0]["scores"]["channels"]["text"]["raw"] < 0 + assert rejected.json() == old.json() == {"results": []} + + +def test_skill_library_http_preserves_custom_scoped_protocol_and_combined_window() -> None: + from powercontext.builtin.artifacts.skill import Skill, SkillContent, SkillSearchHit + from powercontext.builtin.persistence.artifact_governance import ArtifactGovernance, ArtifactLifecycleState + + skills = tuple( + Skill( + artifact_id=key, + revision=1, + content=SkillContent( + name=key, + description=text, + instructions="Inspect", + validation=("Verify",), + ), + ) + for key, text in ( + ("a-active", "needle phrase"), + ("b-deprecated", "needle phrase"), + ("z-deprecated", "needle phrase"), + ) + ) + governance = { + skill.artifact_id: ArtifactGovernance( + artifact=skill.as_ref(), + governance_generation=0, + lifecycle_state=ArtifactLifecycleState.ACTIVE + if skill.artifact_id.endswith("active") + else ArtifactLifecycleState.DEPRECATED, + ) + for skill in skills + } + + class Scoped: + def for_scope(self, scope_id): + assert scope_id == "scope-a" + return self + + async def search(self, query, limit): + assert query == "needle phrase" + return (SkillSearchHit(artifact_ref=skills[0].as_ref(), content=skills[0].content),) + + async def get(self, request): + return next(skill for skill in skills if skill.as_ref() == request.artifact) + + async def governance(self, artifact_id): + return governance[artifact_id] + + async def list(self, *, include_deprecated=False, limit=100): + return tuple( + (skill, governance[skill.artifact_id]) + for skill in skills + if include_deprecated or governance[skill.artifact_id].lifecycle_state is ArtifactLifecycleState.ACTIVE + )[:limit] + + with TestClient(create_app(application=SimpleNamespace(skill=Scoped()))) as client: + matched = client.post( + "/v1/skill/library", + json={"scope_id": "scope-a", "query": " needle phrase ", "include_deprecated": True, "limit": 2}, + ) + listed = client.post("/v1/skill/library", json={"scope_id": "scope-a", "include_deprecated": True, "limit": 2}) + assert matched.status_code == listed.status_code == 200 + assert [item["artifact"]["artifact_id"] for item in matched.json()["skills"]] == ["a-active", "b-deprecated"] + assert matched.json() == listed.json() diff --git a/tests/test_client.py b/tests/test_client.py index 09df64fa0c..d692c10959 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -58,11 +58,63 @@ ScopeId, ScopeQueryField, ScopeSelection, + SearchArtifactsRequest, SearchTopicMemoryRequest, UpdateScopeRequest, ) +def test_artifact_search_client_preserves_path_identity_unset_fields_and_complete_results() -> None: + async def scenario() -> None: + sent: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response( + 200, + json={ + "results": [ + { + "family": "custom-family", + "artifact_id": "result", + "revision": 2, + "content": {"title": "中文", "detail": {"evidence": [1, 2]}}, + "lineage": {"sources": [], "artifacts": []}, + "scores": { + "retrieval": 0.5, + "channels": {"text": {"raw": -1.0, "metric": "sqlite_bm25", "higher_is_better": False}}, + }, + } + ] + }, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as transport: + client = PowerContextClient("https://memory.example", http_client=transport) + response = await client.search_artifacts( + "scope-a", "custom-family", SearchArtifactsRequest(query=" needle ") + ) + await client.search_artifacts( + "scope-a", + "custom-family", + SearchArtifactsRequest(query="needle", limit=200, include_scores=True, filters={}), + ) + assert [request.url.path for request in sent] == [ + "/v1/scopes/scope-a/artifacts/custom-family/search", + "/v1/scopes/scope-a/artifacts/custom-family/search", + ] + assert json.loads(sent[0].content) == {"query": "needle"} + assert json.loads(sent[1].content) == {"query": "needle", "limit": 200, "include_scores": True, "filters": {}} + item = response.results[0] + assert item.family == "custom-family" + assert item.revision == 2 + assert item.content == {"title": "中文", "detail": {"evidence": [1, 2]}} + assert item.scores is not None + assert item.scores.channels["text"].raw == -1.0 + + asyncio.run(scenario()) + + def test_prepare_client_preserves_omitted_and_explicit_assembly() -> None: async def scenario() -> None: bodies = []