Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions evals/retrieval.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ Meaning search is a multilingual embedding model running on the CPU inside the c
through ONNX Runtime. No API key, no network call after the first start, nothing to
configure. An embedding model is not an LLM: it turns text into numbers and writes nothing.

A question is compared with the vector of every clause, with no approximate index in between.
The laws and an organisation's policies come to thousands of vectors, which takes
milliseconds. With an approximate index, the database decides from its own statistics whether
to use it, and other clauses come back when it does, so a search, and a score measured on it,
would not be the same twice.

**Languages.** A question is searched in the language of the letters it is written in:
Arabic letters, Arabic, and anything else, English. Keyword search keeps one index per
language, because an index fixes how it splits and folds words when it is built. Meaning
Expand Down
10 changes: 6 additions & 4 deletions packages/dpolens/src/dpolens/cli/index.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Building and inspecting the vector index."""
"""Embedding the clauses, and seeing how much of the corpus is embedded."""

from __future__ import annotations

Expand All @@ -13,10 +13,12 @@
from dpolens.engine.embedding.encode import usable_cpus
from dpolens.engine.embedding.index import build_index
from dpolens.engine.search.questions import embed_questions, mark_current
from dpolens.engine.session import maintenance_scope, session_scope
from dpolens.engine.session import session_scope
from dpolens.settings import load_settings

app = typer.Typer(name="index", help="Build and inspect the vector index.", no_args_is_help=True)
app = typer.Typer(
name="index", help="Embed the clauses, and see how much is embedded.", no_args_is_help=True
)


@app.command()
Expand Down Expand Up @@ -44,7 +46,7 @@ def build(
Embedder(
get_model(model), cache_dir=cache_dir, intra_op_num_threads=usable_cpus()
) as embedder,
maintenance_scope(load_settings()) as session,
session_scope(load_settings()) as session,
):
result = build_index(session, embedder, skip_embedded=skip_embedded)
# A law loaded or updated since its questions were may have changed words under them.
Expand Down
36 changes: 5 additions & 31 deletions packages/dpolens/src/dpolens/engine/embedding/index.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
"""Building the vector index over the corpus.
"""Embedding the corpus: a vector for every clause that has text.

Every clause with text gets one vector per model. Clauses without text, such as
an article that only holds paragraphs, are skipped: an empty string embeds to a
vector that matches everything weakly and drags on every query.

The vectors are stored and nothing is built over them: a search compares the
question with every one, so there is no approximate index to keep in step.
"""

from __future__ import annotations
Expand All @@ -23,12 +26,6 @@
# memory for attention grows: 64 clauses of 64 tokens, 16 of 128, or one of 512.
BATCH_BUDGET = 64 * 64 * 64

# Chosen so the index is not what loses recall, and measured against a
# brute-force scan in the evals.
HNSW_M = 16
HNSW_EF_CONSTRUCTION = 64
EF_SEARCH = 100

# The clauses a model embeds: every one with text, in a published version.
_PUBLISHED_TEXT = """
FROM node_texts t
Expand Down Expand Up @@ -86,18 +83,13 @@ def build_index(
embedder: Embedder,
activate: bool = True,
skip_embedded: bool = False,
create_index: bool = True,
) -> IndexResult:
"""Embed every clause that has text, then build the vector index.
"""Embed every clause that has text.

With `skip_embedded`, a clause this model has already embedded under the
current recipe is left alone. That makes a second run cheap, which is what an
instance restarting wants, and it is why the recipe is part of the condition:
a change to the recipe has to embed everything again.

Without `create_index`, the index is left as it is: Postgres adds each new
vector to it, and creating one needs the role that owns the table, which the
worker embedding a newly published policy does not hold.
"""
model_id = register_model(session, embedder.model, activate=activate)

Expand Down Expand Up @@ -187,9 +179,6 @@ def build_index(
)
embedded += len(batch)

if create_index:
_create_index(session, model_id, embedder.model.dimensions)

return IndexResult(
model=embedder.model.name,
recipe=RECIPE_VERSION,
Expand Down Expand Up @@ -263,18 +252,3 @@ def _ancestor_headings(
{"node_id": node_id, "lang": lang, "applies_from": applies_from},
).all()
return [row.heading for row in rows if row.heading]


def _create_index(session: Session, model_id: uuid.UUID, dimensions: int) -> None:
"""One index per model, built after the rows are in, which is far faster."""
name = f"node_embeddings_hnsw_{str(model_id).replace('-', '')}"
session.execute(
text(
f"""
CREATE INDEX IF NOT EXISTS {name} ON node_embeddings
USING hnsw ((embedding::vector({dimensions})) vector_cosine_ops)
WITH (m = {HNSW_M}, ef_construction = {HNSW_EF_CONSTRUCTION})
WHERE embedding_model_id = '{model_id}'
"""
)
)
2 changes: 1 addition & 1 deletion packages/dpolens/src/dpolens/engine/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ def _embed_what_is_left(session: Session, settings: Settings) -> int:
return 0
ensure_cached(model)
with Embedder(model, intra_op_num_threads=settings.intra_op_num_threads) as embedder:
return build_index(session, embedder, skip_embedded=True, create_index=False).embedded
return build_index(session, embedder, skip_embedded=True).embedded


def give_back_memory() -> None:
Expand Down
13 changes: 4 additions & 9 deletions packages/dpolens/src/dpolens/engine/search/vector.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
from sqlalchemy.orm import Session

from dpolens.engine.embedding.encode import Embedder
from dpolens.engine.embedding.index import EF_SEARCH
from dpolens.engine.search.keyword import Candidate, scoped


Expand Down Expand Up @@ -48,16 +47,12 @@ def vector_search(
)

vector = embedder.encode_query(query) if vector is None else vector
session.execute(text(f"SET LOCAL hnsw.ef_search = {EF_SEARCH}"))
# The index finds the nearest vectors first and the filter runs after it, so a
# search kept to some laws could end with fewer candidates than it asked for.
# An iterative scan goes on until enough pass. Set either way, because SET
# LOCAL lasts for the transaction and the next search may not be kept.
session.execute(
text(f"SET LOCAL hnsw.iterative_scan = {'strict_order' if packs is not None else 'off'}")
)
scope_join, scope_condition = scoped(packs)

# Every vector is compared: nothing approximate stands between the question and the
# clauses. Laws and policies come to thousands of vectors, which takes milliseconds.
# With an approximate index, Postgres decides from its statistics whether to use it and
# other clauses come back when it does, so a search would not be the same twice.
rows = session.execute(
text(
f"""
Expand Down
17 changes: 0 additions & 17 deletions packages/dpolens/src/dpolens/engine/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,23 +158,6 @@ def session_from(engine: Engine) -> Iterator[Session]:
session.commit()


@contextmanager
def maintenance_scope(settings: Settings) -> Iterator[Session]:
"""A session for work that changes the schema, such as creating an index.

PostgreSQL requires ownership of a table to create an index on it, and the
application role deliberately owns nothing: that is what keeps it from
rewriting the governance log. So index building connects as the role that
owns the tables, the same one migrations use, and falls back to the ordinary
URL on a development database where both are the same role.
"""
settings_for_owner = settings.model_copy(
update={"database_url": settings.migration_database_url or settings.database_url}
)
with session_scope(settings_for_owner) as session:
yield session


@contextmanager
def session_scope(settings: Settings) -> Iterator[Session]:
"""A session and an engine of its own, for a command that then exits.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
"""Meaning search compares every vector, so the approximate indexes go

Revision ID: 0021
Revises: 0020
Create Date: 2026-11-14
"""

from __future__ import annotations

from collections.abc import Sequence

from alembic import op

revision: str = "0021"
down_revision: str | None = "0020"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None


def upgrade() -> None:
# `dpolens index build` made one for each embedding model, named after the model. Whether
# a search went through it or compared every vector was the planner's choice, made from
# statistics that change as rows arrive, and the two do not return the same clauses.
op.execute(
"""
DO $$
DECLARE
approximate record;
BEGIN
FOR approximate IN
SELECT built.relname AS name
FROM pg_index
JOIN pg_class AS built ON built.oid = pg_index.indexrelid
JOIN pg_class AS held ON held.oid = pg_index.indrelid
JOIN pg_namespace ON pg_namespace.oid = held.relnamespace
JOIN pg_am ON pg_am.oid = built.relam
WHERE held.relname = 'node_embeddings'
AND pg_namespace.nspname = current_schema()
AND pg_am.amname = 'hnsw'
LOOP
EXECUTE format('DROP INDEX %I', approximate.name);
END LOOP;
END
$$
"""
)


def downgrade() -> None:
"""Nothing to put back: the version before this one builds its index again when
`dpolens index build` next runs, which is at every start, and searches without it until
then."""
11 changes: 0 additions & 11 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,17 +174,6 @@ def _migrated_database(database_url: str, app_role: str, name: str) -> str:
)


@pytest.fixture(scope="session")
def committed_owner_url(committed_database_url: str, database_url: str) -> str:
"""The committing tests' database, reached as the role that owns the tables.

Creating an index needs ownership, which the application role deliberately
does not have, so loading a corpus for a test connects the way an operator's
maintenance command does.
"""
return _as_owner(committed_database_url, database_url)


@pytest.fixture(scope="session")
def committed_database_url(database_url: str, app_role: str) -> str:
"""A database of its own for tests that commit, reached as the application role.
Expand Down
10 changes: 3 additions & 7 deletions tests/integration/test_api_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,9 @@


@pytest.fixture(scope="module")
def corpus(committed_owner_url: str) -> Iterator[Engine]:
"""The fixture pack, loaded and indexed once for this module.

As the owning role, because building a vector index creates one and
PostgreSQL requires ownership for that. It is maintenance, not serving.
"""
engine = create_engine(committed_owner_url)
def corpus(committed_database_url: str) -> Iterator[Engine]:
"""The fixture pack, loaded and embedded once for this module."""
engine = create_engine(committed_database_url)
with session_from(engine) as opened:
bootstrap(opened, actor=ACTOR)
load_pack(opened, TESTLAW)
Expand Down
19 changes: 19 additions & 0 deletions tests/integration/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,25 @@ def test_loading_twice_is_a_no_op(loaded: None) -> None:
assert "already loaded" in result.output


def test_embedding_needs_only_the_application_role(
app_database_url: str,
clean_tables: None,
testlaw_pack: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""It writes rows and creates nothing, so the role that answers requests can run it,
with no connection as the role that owns the tables."""
monkeypatch.setenv("DPOLENS_DATABASE_URL", app_database_url)
monkeypatch.delenv("DPOLENS_MIGRATION_DATABASE_URL", raising=False)
loaded = runner.invoke(app, ["pack", "load", str(testlaw_pack)])
assert loaded.exit_code == 0, loaded.output

built = runner.invoke(app, ["index", "build"])

assert built.exit_code == 0, built.output
assert "clauses with e5-small" in said(built)


def test_showing_an_article(loaded: None) -> None:
result = runner.invoke(app, ["clause", "show", "gdpr:art-17"])

Expand Down
50 changes: 50 additions & 0 deletions tests/integration/test_embedding_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,3 +266,53 @@ def test_a_new_recipe_embeds_every_clause_again_in_place(
"""
)
).one() == (before, before, RECIPE_VERSION, RECIPE_VERSION)


def test_meaning_search_compares_every_vector(indexed: Session, embedder: Embedder) -> None:
"""Each score is the clause's own closeness to the question, and no clause is left
out, which an approximate index does not promise."""
question = "how do I delete someone's account"
asked = embedder.encode_query(question)
rows = indexed.execute(
text(
"""
SELECT n.canonical_key AS key, e.embedding::text AS embedding
FROM node_embeddings e
JOIN node_texts t ON t.id = e.node_text_id
JOIN document_nodes n ON n.id = t.node_id
"""
)
).all()
closeness: dict[str, float] = {}
for row in rows:
# A clause counts once, at whichever of its texts is closer.
closest = float(np.dot(json.loads(row.embedding), asked))
closeness[row.key] = max(closeness.get(row.key, -1.0), closest)

found = vector_search(indexed, embedder, question, limit=len(rows))

assert {candidate.key for candidate in found} == set(closeness)
for candidate in found:
assert candidate.score == pytest.approx(closeness[candidate.key], abs=1e-5)
assert [candidate.score for candidate in found] == sorted(
(candidate.score for candidate in found), reverse=True
)


def test_embedding_builds_no_approximate_index(indexed: Session) -> None:
"""Postgres would use one or not as its statistics change, and other clauses come back
when it does, so a search would not be the same twice."""
methods = indexed.scalars(
text(
"""
SELECT pg_am.amname
FROM pg_index
JOIN pg_class AS built ON built.oid = pg_index.indexrelid
JOIN pg_class AS held ON held.oid = pg_index.indrelid
JOIN pg_am ON pg_am.oid = built.relam
WHERE held.relname IN ('node_embeddings', 'search_question_embeddings')
"""
)
).all()

assert set(methods) == {"btree"}
4 changes: 2 additions & 2 deletions tests/integration/test_mcp_end_to_end.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ def anyio_backend() -> str:


@pytest.fixture(scope="module")
def corpus(committed_owner_url: str) -> Iterator[Engine]:
engine = create_engine(committed_owner_url)
def corpus(committed_database_url: str) -> Iterator[Engine]:
engine = create_engine(committed_database_url)
with session_from(engine) as opened:
bootstrap(opened, actor=ACTOR)
load_pack(opened, TESTLAW)
Expand Down
33 changes: 33 additions & 0 deletions tests/integration/test_migrations.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,39 @@ def test_a_table_added_later_is_reachable_by_the_application_role(
connection.execute(text("DROP TABLE later_table"))


def test_0021_drops_the_approximate_index_an_earlier_version_built(
owner_url: str, app_role: str
) -> None:
"""Whichever model it was built for, and nothing else the table has."""
config = alembic(owner_url, app_role)
engine = create_engine(owner_url)
indexes = text("SELECT indexname FROM pg_indexes WHERE tablename = 'node_embeddings'")
try:
command.upgrade(config, "head")
command.downgrade(config, "0020")
with engine.begin() as connection:
kept = set(connection.scalars(indexes))
connection.execute(
text(
"""
CREATE INDEX node_embeddings_hnsw_0a1b ON node_embeddings
USING hnsw ((embedding::vector(384)) vector_cosine_ops)
WITH (m = 16, ef_construction = 64)
WHERE embedding_model_id = '00000000-0000-0000-0000-00000000000a'
"""
)
)
assert set(connection.scalars(indexes)) == kept | {"node_embeddings_hnsw_0a1b"}

command.upgrade(config, "head")

with engine.connect() as connection:
assert kept
assert set(connection.scalars(indexes)) == kept
finally:
engine.dispose()


def test_0019_names_what_was_stored_before(owner_url: str, app_role: str) -> None:
"""A law keeps its name and title, in English as they were written, until its pack loads
again, and a policy keeps its title, in the language of its text."""
Expand Down
Loading
Loading