Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions agent/corpus_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -460,10 +460,10 @@ def query_chunks_lexical(
actual_backend = "projection_substring"
if not backend_warning:
backend_warning = "FTS unavailable or yielded no tokenized query; using projection substring fallback."
candidates = _query_chunk_search_fallback(conn, query_text=raw_query)
candidates = _query_chunk_search_fallback(conn, query_text=raw_query, limit=fetch_limit)
else:
actual_backend = "projection_substring"
candidates = _query_chunk_search_fallback(conn, query_text=raw_query)
candidates = _query_chunk_search_fallback(conn, query_text=raw_query, limit=fetch_limit)
Comment on lines +463 to +466

Copilot AI Mar 23, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The fallback path is now bounded via limit=fetch_limit, but there’s no regression test asserting that the projection-substring backend doesn’t return an unbounded candidate set when FTS5 is unavailable. Adding a focused test (e.g., patching _ensure_chunk_search_fts to false and inserting many chunk_search rows, then asserting the candidate fetch is capped) would prevent accidental reintroduction of the unbounded query.

Copilot uses AI. Check for mistakes.

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@copilot apply changes based on this feedback

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in commit 0c8fc41. Added FallbackLimitRegressionTests to tests/test_retrieval_contract.py with three focused tests that exercise _query_chunk_search_fallback directly against an in-memory SQLite DB: (1) result count is capped at the given limit when 200 matching rows exist, (2) all rows are returned when total matches fall below the limit, and (3) every returned row includes a backend_score float.


if not candidates:
return []
Expand Down Expand Up @@ -716,6 +716,7 @@ def _query_chunk_search_fallback(
conn: sqlite3.Connection,
*,
query_text: str,
limit: int,
) -> list[dict[str, object]]:
rows = conn.execute(
"""
Expand Down Expand Up @@ -746,6 +747,8 @@ def _query_chunk_search_fallback(
OR instr(lower(journal_entry_date), lower(?)) > 0
OR instr(lower(layer), lower(?)) > 0
OR instr(lower(register), lower(?)) > 0
Comment thread
duck-lint marked this conversation as resolved.
ORDER BY chunk_key
LIMIT ?
""",
(
query_text,
Expand All @@ -759,6 +762,7 @@ def _query_chunk_search_fallback(
query_text,
query_text,
query_text,
max(1, int(limit)),
),
).fetchall()
out: list[dict[str, object]] = []
Expand Down
86 changes: 86 additions & 0 deletions tests/test_retrieval_contract.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
from __future__ import annotations

from datetime import datetime, timezone
import sqlite3
import unittest
from unittest.mock import patch

from agent.corpus_db import _query_chunk_search_fallback
from agent.embeddings import sync_embeddings
from agent.retrieval import RetrievedChunk, _apply_bounded_rerank
from tests.support import AppFixture, dummy_embedder_factory
Expand Down Expand Up @@ -262,5 +264,89 @@ def test_bounded_rerank_falls_back_to_mtime_when_journal_entry_date_is_missing(s
self.assertEqual(reranked[0].chunk_key, "newer-mtime")


class FallbackLimitRegressionTests(unittest.TestCase):
"""Regression tests asserting that _query_chunk_search_fallback respects its LIMIT."""

def _make_conn(self) -> sqlite3.Connection:
conn = sqlite3.connect(":memory:")
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys = OFF") # no documents/sources tables in this minimal fixture
conn.execute(
"""
CREATE TABLE chunk_search (
id INTEGER PRIMARY KEY,
doc_id INTEGER NOT NULL,
chunk_key TEXT NOT NULL UNIQUE,
chunk_kind TEXT NOT NULL,
rel_path TEXT NOT NULL,
body_text TEXT NOT NULL,
chunk_title TEXT NOT NULL,
heading_path TEXT NOT NULL,
canonical_name TEXT NOT NULL,
aliases_text TEXT NOT NULL,
tags_text TEXT NOT NULL,
note_type TEXT NOT NULL,
journal_entry_date TEXT,
layer TEXT NOT NULL,
register TEXT NOT NULL,
updated_at REAL NOT NULL
)
"""
)
return conn

def _insert_rows(self, conn: sqlite3.Connection, count: int, *, query_word: str) -> None:
for i in range(count):
conn.execute(
"""
INSERT INTO chunk_search
(doc_id, chunk_key, chunk_kind, rel_path, body_text,
chunk_title, heading_path, canonical_name, aliases_text,
tags_text, note_type, journal_entry_date, layer, register, updated_at)
VALUES (?, ?, 'content', ?, ?, ?, '', '', '', '', 'knowledge', NULL, '', '', 0.0)
""",
(
i,
f"chunk-{i}",
f"doc-{i}.md",
f"{query_word} body text for row {i}",
f"{query_word} title {i}",
),
)
conn.commit()

def test_fallback_candidate_count_is_capped_at_limit(self) -> None:
conn = self._make_conn()
total_rows = 200
limit = 10
self._insert_rows(conn, total_rows, query_word="common")

results = _query_chunk_search_fallback(conn, query_text="common", limit=limit)

self.assertLessEqual(
len(results),
limit,
f"Expected at most {limit} rows but got {len(results)}",
)

def test_fallback_returns_all_matching_when_below_limit(self) -> None:
conn = self._make_conn()
self._insert_rows(conn, 5, query_word="raretoken")

results = _query_chunk_search_fallback(conn, query_text="raretoken", limit=50)

self.assertEqual(len(results), 5)

def test_fallback_rows_include_backend_score(self) -> None:
conn = self._make_conn()
self._insert_rows(conn, 3, query_word="scorecheck")

results = _query_chunk_search_fallback(conn, query_text="scorecheck", limit=10)

for row in results:
self.assertIn("backend_score", row)
self.assertIsInstance(row["backend_score"], float)


if __name__ == "__main__":
unittest.main()
Loading