From a063ef68cdefebb7d88870dc9cb3a7307743fa9e Mon Sep 17 00:00:00 2001 From: alkhatibdev Date: Sat, 14 Nov 2026 21:36:22 +0400 Subject: [PATCH] fix: meaning search compares every vector, so a search is the same every time Signed-off-by: alkhatibdev --- evals/retrieval.md | 6 +++ packages/dpolens/src/dpolens/cli/index.py | 10 ++-- .../src/dpolens/engine/embedding/index.py | 36 ++----------- packages/dpolens/src/dpolens/engine/jobs.py | 2 +- .../src/dpolens/engine/search/vector.py | 13 ++--- .../dpolens/src/dpolens/engine/session.py | 17 ------ .../versions/0021_exact_meaning_search.py | 52 +++++++++++++++++++ tests/conftest.py | 11 ---- tests/integration/test_api_routes.py | 10 ++-- tests/integration/test_cli.py | 19 +++++++ tests/integration/test_embedding_index.py | 50 ++++++++++++++++++ tests/integration/test_mcp_end_to_end.py | 4 +- tests/integration/test_migrations.py | 33 ++++++++++++ tests/integration/test_query_log.py | 4 +- tests/integration/test_selected_laws.py | 26 +++++++--- 15 files changed, 203 insertions(+), 90 deletions(-) create mode 100644 packages/dpolens/src/dpolens/migrations/versions/0021_exact_meaning_search.py diff --git a/evals/retrieval.md b/evals/retrieval.md index c5cfe98..d3848df 100644 --- a/evals/retrieval.md +++ b/evals/retrieval.md @@ -20,6 +20,12 @@ Meaning search is a multilingual embedding model running on the CPU inside the c through ONNX Runtime. No API key, no network call after the first start, nothing to configure. An embedding model is not an LLM: it turns text into numbers and writes nothing. +A question is compared with the vector of every clause, with no approximate index in between. +The laws and an organisation's policies come to thousands of vectors, which takes +milliseconds. With an approximate index, the database decides from its own statistics whether +to use it, and other clauses come back when it does, so a search, and a score measured on it, +would not be the same twice. + **Languages.** A question is searched in the language of the letters it is written in: Arabic letters, Arabic, and anything else, English. Keyword search keeps one index per language, because an index fixes how it splits and folds words when it is built. Meaning diff --git a/packages/dpolens/src/dpolens/cli/index.py b/packages/dpolens/src/dpolens/cli/index.py index 1e7a743..8411a4d 100644 --- a/packages/dpolens/src/dpolens/cli/index.py +++ b/packages/dpolens/src/dpolens/cli/index.py @@ -1,4 +1,4 @@ -"""Building and inspecting the vector index.""" +"""Embedding the clauses, and seeing how much of the corpus is embedded.""" from __future__ import annotations @@ -13,10 +13,12 @@ from dpolens.engine.embedding.encode import usable_cpus from dpolens.engine.embedding.index import build_index from dpolens.engine.search.questions import embed_questions, mark_current -from dpolens.engine.session import maintenance_scope, session_scope +from dpolens.engine.session import session_scope from dpolens.settings import load_settings -app = typer.Typer(name="index", help="Build and inspect the vector index.", no_args_is_help=True) +app = typer.Typer( + name="index", help="Embed the clauses, and see how much is embedded.", no_args_is_help=True +) @app.command() @@ -44,7 +46,7 @@ def build( Embedder( get_model(model), cache_dir=cache_dir, intra_op_num_threads=usable_cpus() ) as embedder, - maintenance_scope(load_settings()) as session, + session_scope(load_settings()) as session, ): result = build_index(session, embedder, skip_embedded=skip_embedded) # A law loaded or updated since its questions were may have changed words under them. diff --git a/packages/dpolens/src/dpolens/engine/embedding/index.py b/packages/dpolens/src/dpolens/engine/embedding/index.py index de28569..1766418 100644 --- a/packages/dpolens/src/dpolens/engine/embedding/index.py +++ b/packages/dpolens/src/dpolens/engine/embedding/index.py @@ -1,8 +1,11 @@ -"""Building the vector index over the corpus. +"""Embedding the corpus: a vector for every clause that has text. Every clause with text gets one vector per model. Clauses without text, such as an article that only holds paragraphs, are skipped: an empty string embeds to a vector that matches everything weakly and drags on every query. + +The vectors are stored and nothing is built over them: a search compares the +question with every one, so there is no approximate index to keep in step. """ from __future__ import annotations @@ -23,12 +26,6 @@ # memory for attention grows: 64 clauses of 64 tokens, 16 of 128, or one of 512. BATCH_BUDGET = 64 * 64 * 64 -# Chosen so the index is not what loses recall, and measured against a -# brute-force scan in the evals. -HNSW_M = 16 -HNSW_EF_CONSTRUCTION = 64 -EF_SEARCH = 100 - # The clauses a model embeds: every one with text, in a published version. _PUBLISHED_TEXT = """ FROM node_texts t @@ -86,18 +83,13 @@ def build_index( embedder: Embedder, activate: bool = True, skip_embedded: bool = False, - create_index: bool = True, ) -> IndexResult: - """Embed every clause that has text, then build the vector index. + """Embed every clause that has text. With `skip_embedded`, a clause this model has already embedded under the current recipe is left alone. That makes a second run cheap, which is what an instance restarting wants, and it is why the recipe is part of the condition: a change to the recipe has to embed everything again. - - Without `create_index`, the index is left as it is: Postgres adds each new - vector to it, and creating one needs the role that owns the table, which the - worker embedding a newly published policy does not hold. """ model_id = register_model(session, embedder.model, activate=activate) @@ -187,9 +179,6 @@ def build_index( ) embedded += len(batch) - if create_index: - _create_index(session, model_id, embedder.model.dimensions) - return IndexResult( model=embedder.model.name, recipe=RECIPE_VERSION, @@ -263,18 +252,3 @@ def _ancestor_headings( {"node_id": node_id, "lang": lang, "applies_from": applies_from}, ).all() return [row.heading for row in rows if row.heading] - - -def _create_index(session: Session, model_id: uuid.UUID, dimensions: int) -> None: - """One index per model, built after the rows are in, which is far faster.""" - name = f"node_embeddings_hnsw_{str(model_id).replace('-', '')}" - session.execute( - text( - f""" - CREATE INDEX IF NOT EXISTS {name} ON node_embeddings - USING hnsw ((embedding::vector({dimensions})) vector_cosine_ops) - WITH (m = {HNSW_M}, ef_construction = {HNSW_EF_CONSTRUCTION}) - WHERE embedding_model_id = '{model_id}' - """ - ) - ) diff --git a/packages/dpolens/src/dpolens/engine/jobs.py b/packages/dpolens/src/dpolens/engine/jobs.py index f3b87e2..e04f5f1 100644 --- a/packages/dpolens/src/dpolens/engine/jobs.py +++ b/packages/dpolens/src/dpolens/engine/jobs.py @@ -96,7 +96,7 @@ def _embed_what_is_left(session: Session, settings: Settings) -> int: return 0 ensure_cached(model) with Embedder(model, intra_op_num_threads=settings.intra_op_num_threads) as embedder: - return build_index(session, embedder, skip_embedded=True, create_index=False).embedded + return build_index(session, embedder, skip_embedded=True).embedded def give_back_memory() -> None: diff --git a/packages/dpolens/src/dpolens/engine/search/vector.py b/packages/dpolens/src/dpolens/engine/search/vector.py index 902e6c8..d3ba3a4 100644 --- a/packages/dpolens/src/dpolens/engine/search/vector.py +++ b/packages/dpolens/src/dpolens/engine/search/vector.py @@ -15,7 +15,6 @@ from sqlalchemy.orm import Session from dpolens.engine.embedding.encode import Embedder -from dpolens.engine.embedding.index import EF_SEARCH from dpolens.engine.search.keyword import Candidate, scoped @@ -48,16 +47,12 @@ def vector_search( ) vector = embedder.encode_query(query) if vector is None else vector - session.execute(text(f"SET LOCAL hnsw.ef_search = {EF_SEARCH}")) - # The index finds the nearest vectors first and the filter runs after it, so a - # search kept to some laws could end with fewer candidates than it asked for. - # An iterative scan goes on until enough pass. Set either way, because SET - # LOCAL lasts for the transaction and the next search may not be kept. - session.execute( - text(f"SET LOCAL hnsw.iterative_scan = {'strict_order' if packs is not None else 'off'}") - ) scope_join, scope_condition = scoped(packs) + # Every vector is compared: nothing approximate stands between the question and the + # clauses. Laws and policies come to thousands of vectors, which takes milliseconds. + # With an approximate index, Postgres decides from its statistics whether to use it and + # other clauses come back when it does, so a search would not be the same twice. rows = session.execute( text( f""" diff --git a/packages/dpolens/src/dpolens/engine/session.py b/packages/dpolens/src/dpolens/engine/session.py index 275ab35..6c5cbd9 100644 --- a/packages/dpolens/src/dpolens/engine/session.py +++ b/packages/dpolens/src/dpolens/engine/session.py @@ -158,23 +158,6 @@ def session_from(engine: Engine) -> Iterator[Session]: session.commit() -@contextmanager -def maintenance_scope(settings: Settings) -> Iterator[Session]: - """A session for work that changes the schema, such as creating an index. - - PostgreSQL requires ownership of a table to create an index on it, and the - application role deliberately owns nothing: that is what keeps it from - rewriting the governance log. So index building connects as the role that - owns the tables, the same one migrations use, and falls back to the ordinary - URL on a development database where both are the same role. - """ - settings_for_owner = settings.model_copy( - update={"database_url": settings.migration_database_url or settings.database_url} - ) - with session_scope(settings_for_owner) as session: - yield session - - @contextmanager def session_scope(settings: Settings) -> Iterator[Session]: """A session and an engine of its own, for a command that then exits. diff --git a/packages/dpolens/src/dpolens/migrations/versions/0021_exact_meaning_search.py b/packages/dpolens/src/dpolens/migrations/versions/0021_exact_meaning_search.py new file mode 100644 index 0000000..0ccaa46 --- /dev/null +++ b/packages/dpolens/src/dpolens/migrations/versions/0021_exact_meaning_search.py @@ -0,0 +1,52 @@ +"""Meaning search compares every vector, so the approximate indexes go + +Revision ID: 0021 +Revises: 0020 +Create Date: 2026-11-14 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +revision: str = "0021" +down_revision: str | None = "0020" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # `dpolens index build` made one for each embedding model, named after the model. Whether + # a search went through it or compared every vector was the planner's choice, made from + # statistics that change as rows arrive, and the two do not return the same clauses. + op.execute( + """ + DO $$ + DECLARE + approximate record; + BEGIN + FOR approximate IN + SELECT built.relname AS name + FROM pg_index + JOIN pg_class AS built ON built.oid = pg_index.indexrelid + JOIN pg_class AS held ON held.oid = pg_index.indrelid + JOIN pg_namespace ON pg_namespace.oid = held.relnamespace + JOIN pg_am ON pg_am.oid = built.relam + WHERE held.relname = 'node_embeddings' + AND pg_namespace.nspname = current_schema() + AND pg_am.amname = 'hnsw' + LOOP + EXECUTE format('DROP INDEX %I', approximate.name); + END LOOP; + END + $$ + """ + ) + + +def downgrade() -> None: + """Nothing to put back: the version before this one builds its index again when + `dpolens index build` next runs, which is at every start, and searches without it until + then.""" diff --git a/tests/conftest.py b/tests/conftest.py index 6a1b489..fef8b78 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -174,17 +174,6 @@ def _migrated_database(database_url: str, app_role: str, name: str) -> str: ) -@pytest.fixture(scope="session") -def committed_owner_url(committed_database_url: str, database_url: str) -> str: - """The committing tests' database, reached as the role that owns the tables. - - Creating an index needs ownership, which the application role deliberately - does not have, so loading a corpus for a test connects the way an operator's - maintenance command does. - """ - return _as_owner(committed_database_url, database_url) - - @pytest.fixture(scope="session") def committed_database_url(database_url: str, app_role: str) -> str: """A database of its own for tests that commit, reached as the application role. diff --git a/tests/integration/test_api_routes.py b/tests/integration/test_api_routes.py index ebb9315..87d4b1a 100644 --- a/tests/integration/test_api_routes.py +++ b/tests/integration/test_api_routes.py @@ -37,13 +37,9 @@ @pytest.fixture(scope="module") -def corpus(committed_owner_url: str) -> Iterator[Engine]: - """The fixture pack, loaded and indexed once for this module. - - As the owning role, because building a vector index creates one and - PostgreSQL requires ownership for that. It is maintenance, not serving. - """ - engine = create_engine(committed_owner_url) +def corpus(committed_database_url: str) -> Iterator[Engine]: + """The fixture pack, loaded and embedded once for this module.""" + engine = create_engine(committed_database_url) with session_from(engine) as opened: bootstrap(opened, actor=ACTOR) load_pack(opened, TESTLAW) diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index 9b3c628..6d22247 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -69,6 +69,25 @@ def test_loading_twice_is_a_no_op(loaded: None) -> None: assert "already loaded" in result.output +def test_embedding_needs_only_the_application_role( + app_database_url: str, + clean_tables: None, + testlaw_pack: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """It writes rows and creates nothing, so the role that answers requests can run it, + with no connection as the role that owns the tables.""" + monkeypatch.setenv("DPOLENS_DATABASE_URL", app_database_url) + monkeypatch.delenv("DPOLENS_MIGRATION_DATABASE_URL", raising=False) + loaded = runner.invoke(app, ["pack", "load", str(testlaw_pack)]) + assert loaded.exit_code == 0, loaded.output + + built = runner.invoke(app, ["index", "build"]) + + assert built.exit_code == 0, built.output + assert "clauses with e5-small" in said(built) + + def test_showing_an_article(loaded: None) -> None: result = runner.invoke(app, ["clause", "show", "gdpr:art-17"]) diff --git a/tests/integration/test_embedding_index.py b/tests/integration/test_embedding_index.py index b07b70d..e1ebf1e 100644 --- a/tests/integration/test_embedding_index.py +++ b/tests/integration/test_embedding_index.py @@ -266,3 +266,53 @@ def test_a_new_recipe_embeds_every_clause_again_in_place( """ ) ).one() == (before, before, RECIPE_VERSION, RECIPE_VERSION) + + +def test_meaning_search_compares_every_vector(indexed: Session, embedder: Embedder) -> None: + """Each score is the clause's own closeness to the question, and no clause is left + out, which an approximate index does not promise.""" + question = "how do I delete someone's account" + asked = embedder.encode_query(question) + rows = indexed.execute( + text( + """ + SELECT n.canonical_key AS key, e.embedding::text AS embedding + FROM node_embeddings e + JOIN node_texts t ON t.id = e.node_text_id + JOIN document_nodes n ON n.id = t.node_id + """ + ) + ).all() + closeness: dict[str, float] = {} + for row in rows: + # A clause counts once, at whichever of its texts is closer. + closest = float(np.dot(json.loads(row.embedding), asked)) + closeness[row.key] = max(closeness.get(row.key, -1.0), closest) + + found = vector_search(indexed, embedder, question, limit=len(rows)) + + assert {candidate.key for candidate in found} == set(closeness) + for candidate in found: + assert candidate.score == pytest.approx(closeness[candidate.key], abs=1e-5) + assert [candidate.score for candidate in found] == sorted( + (candidate.score for candidate in found), reverse=True + ) + + +def test_embedding_builds_no_approximate_index(indexed: Session) -> None: + """Postgres would use one or not as its statistics change, and other clauses come back + when it does, so a search would not be the same twice.""" + methods = indexed.scalars( + text( + """ + SELECT pg_am.amname + FROM pg_index + JOIN pg_class AS built ON built.oid = pg_index.indexrelid + JOIN pg_class AS held ON held.oid = pg_index.indrelid + JOIN pg_am ON pg_am.oid = built.relam + WHERE held.relname IN ('node_embeddings', 'search_question_embeddings') + """ + ) + ).all() + + assert set(methods) == {"btree"} diff --git a/tests/integration/test_mcp_end_to_end.py b/tests/integration/test_mcp_end_to_end.py index 1bf221f..10a274d 100644 --- a/tests/integration/test_mcp_end_to_end.py +++ b/tests/integration/test_mcp_end_to_end.py @@ -54,8 +54,8 @@ def anyio_backend() -> str: @pytest.fixture(scope="module") -def corpus(committed_owner_url: str) -> Iterator[Engine]: - engine = create_engine(committed_owner_url) +def corpus(committed_database_url: str) -> Iterator[Engine]: + engine = create_engine(committed_database_url) with session_from(engine) as opened: bootstrap(opened, actor=ACTOR) load_pack(opened, TESTLAW) diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index 224e637..0f1c14b 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -122,6 +122,39 @@ def test_a_table_added_later_is_reachable_by_the_application_role( connection.execute(text("DROP TABLE later_table")) +def test_0021_drops_the_approximate_index_an_earlier_version_built( + owner_url: str, app_role: str +) -> None: + """Whichever model it was built for, and nothing else the table has.""" + config = alembic(owner_url, app_role) + engine = create_engine(owner_url) + indexes = text("SELECT indexname FROM pg_indexes WHERE tablename = 'node_embeddings'") + try: + command.upgrade(config, "head") + command.downgrade(config, "0020") + with engine.begin() as connection: + kept = set(connection.scalars(indexes)) + connection.execute( + text( + """ + CREATE INDEX node_embeddings_hnsw_0a1b ON node_embeddings + USING hnsw ((embedding::vector(384)) vector_cosine_ops) + WITH (m = 16, ef_construction = 64) + WHERE embedding_model_id = '00000000-0000-0000-0000-00000000000a' + """ + ) + ) + assert set(connection.scalars(indexes)) == kept | {"node_embeddings_hnsw_0a1b"} + + command.upgrade(config, "head") + + with engine.connect() as connection: + assert kept + assert set(connection.scalars(indexes)) == kept + finally: + engine.dispose() + + def test_0019_names_what_was_stored_before(owner_url: str, app_role: str) -> None: """A law keeps its name and title, in English as they were written, until its pack loads again, and a policy keeps its title, in the language of its text.""" diff --git a/tests/integration/test_query_log.py b/tests/integration/test_query_log.py index cb3e93e..119fa7d 100644 --- a/tests/integration/test_query_log.py +++ b/tests/integration/test_query_log.py @@ -44,8 +44,8 @@ @pytest.fixture(scope="module") -def corpus(committed_owner_url: str) -> Iterator[Engine]: - engine = create_engine(committed_owner_url) +def corpus(committed_database_url: str) -> Iterator[Engine]: + engine = create_engine(committed_database_url) with session_from(engine) as opened: bootstrap(opened, actor=ACTOR) load_pack(opened, TESTLAW) diff --git a/tests/integration/test_selected_laws.py b/tests/integration/test_selected_laws.py index 2faeea4..ad2214f 100644 --- a/tests/integration/test_selected_laws.py +++ b/tests/integration/test_selected_laws.py @@ -176,15 +176,29 @@ def test_both_retrievers_keep_to_the_laws_they_are_given( assert {candidate.key.split(":")[0] for candidate in keyword} == {"bilingual"} assert {candidate.key.split(":")[0] for candidate in meaning} == {"bilingual"} - def test_a_kept_search_does_not_keep_the_next_one( + def test_a_search_kept_to_one_law_leaves_none_of_its_clauses_out( self, two_laws: Session, embedder: Embedder ) -> None: - """SET LOCAL lasts for the transaction, so each search sets the scan its own way.""" - vector_search(two_laws, embedder, BOTH, packs=["bilingual"]) - assert two_laws.execute(text("SHOW hnsw.iterative_scan")).scalar() == "strict_order" + """Every vector is compared, so however many of the other law's clauses sit closer + to the question, the law asked for still returns all it holds.""" + held = two_laws.scalars( + text( + """ + SELECT n.canonical_key + FROM node_texts t + JOIN document_nodes n ON n.id = t.node_id + JOIN document_versions v ON v.id = n.document_version_id + JOIN documents d ON d.id = v.document_id + JOIN packs p ON p.id = d.pack_id + WHERE p.slug = 'bilingual' AND length(trim(t.body_text)) > 0 + """ + ) + ).all() + + meaning = vector_search(two_laws, embedder, BOTH, packs=["bilingual"], limit=len(held)) - vector_search(two_laws, embedder, BOTH) - assert two_laws.execute(text("SHOW hnsw.iterative_scan")).scalar() == "off" + assert held + assert {candidate.key for candidate in meaning} == set(held) def test_an_eval_run_searches_only_the_law_it_tests( self, two_laws: Session, embedder: Embedder