drag0sd0g · drag0sd0g · Jul 25, 2026 · Jul 25, 2026
diff --git a/.env.example b/.env.example
@@ -31,6 +31,9 @@ CLAUDE_MODEL=claude-opus-4-6
 
 # --- Embedding Model ---
 EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+# Chunks embedded per batch in the embedding worker. Larger batches are
+# faster on machines with plenty of RAM (e.g. 128 on a 32 GB+ machine).
+EMBEDDING_BATCH_SIZE=64
 
 # --- Query API ---
 API_KEYS=dev-key-1,dev-key-2

diff --git a/db/migrations/002_hybrid_search_and_facts.sql b/db/migrations/002_hybrid_search_and_facts.sql
@@ -0,0 +1,39 @@
+-- FinDocDRAG Migration 002 — Hybrid search + structured financial facts
+-- See: docs/technical-design-document.md Section 5.3
+--
+-- 1. Full-text search support on document_chunks (lexical leg of hybrid
+--    retrieval — fused with vector search via Reciprocal Rank Fusion).
+-- 2. financial_facts table for curated annual XBRL facts, injected into
+--    query context for numeric financial questions.
+--
+-- All statements are idempotent: this file may be re-applied safely.
+
+-- ============================================================
+-- Full-text search column + GIN index
+-- ============================================================
+ALTER TABLE document_chunks
+    ADD COLUMN IF NOT EXISTS chunk_tsv tsvector
+    GENERATED ALWAYS AS (to_tsvector('english', chunk_text)) STORED;
+
+CREATE INDEX IF NOT EXISTS idx_chunks_tsv
+    ON document_chunks USING GIN (chunk_tsv);
+
+-- ============================================================
+-- Structured financial facts (XBRL companyfacts)
+-- ============================================================
+CREATE TABLE IF NOT EXISTS financial_facts (
+    ticker       VARCHAR(10)  NOT NULL,
+    cik          BIGINT       NOT NULL,
+    concept      VARCHAR(120) NOT NULL,   -- us-gaap tag, e.g. NetIncomeLoss
+    label        VARCHAR(120) NOT NULL,   -- human-readable label
+    unit         VARCHAR(20)  NOT NULL,   -- USD | USD/shares
+    fiscal_year  INTEGER      NOT NULL,   -- year of the fiscal period end
+    period_end   DATE         NOT NULL,
+    value        NUMERIC      NOT NULL,
+    filed        DATE         NOT NULL,   -- filing date of the reporting 10-K
+    updated_at   TIMESTAMPTZ  NOT NULL DEFAULT NOW(),
+    PRIMARY KEY (ticker, concept, unit, period_end)
+);
+
+CREATE INDEX IF NOT EXISTS idx_facts_ticker_year
+    ON financial_facts(ticker, fiscal_year);
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -213,7 +213,7 @@ services:
     deploy:
       resources:
         limits:
-          cpus: "8.0"
+          cpus: "12.0"
           memory: 16G
     depends_on:
       db-migrate:
@@ -228,6 +228,7 @@ services:
       POSTGRES_PASSWORD: changeme
       KAFKA_BOOTSTRAP_SERVERS: kafka:9092
       EMBEDDING_MODEL: "sentence-transformers/all-MiniLM-L6-v2"
+      EMBEDDING_BATCH_SIZE: "${EMBEDDING_BATCH_SIZE:-64}"
       LOG_LEVEL: "${LOG_LEVEL:-INFO}"
     ports:
       - "8002:8002"

diff --git a/docs/evaluation-results.md b/docs/evaluation-results.md
@@ -73,3 +73,61 @@ Scores range from 0 to 1; higher is better. Target thresholds: ≥ 0.70 for all
 | What risks does Amazon identify related to its internat… | AMZN | 0.679 | 0.625 | 1.000 |
 | What was Amazon's operating income for fiscal year 2024? | AMZN | 0.000 | 0.333 | 0.750 |
 | **Mean** |  | **0.185** | **0.696** | **0.697** |
+
+## Evaluation Run — 2026-07-19 12:44:53 UTC
+
+**Samples evaluated:** 6 / 30  
+**Full report:** `services/eval/results/eval_report_2026-07-19_12-44-53_UTC.json`
+
+**Retrieval metrics (LLM-free):** section_hit_rate = 1.000 · section_mrr = 0.833 · ticker_accuracy = 1.000
+
+| Question | Ticker | Answer Relevancy | Faithfulness | Context Precision |
+| --- | --- | --- | --- | --- |
+| What was Apple's total net revenue for fiscal year 2024? | AAPL | 0.966 | 1.000 | — |
+| What are Apple's primary supply chain risk factors disc… | AAPL | 0.908 | 1.000 | — |
+| What was Apple's Services segment revenue in fiscal yea… | AAPL | 0.000 | — | — |
+| How much did Apple spend on research and development in… | AAPL | 1.000 | 1.000 | — |
+| Which geographic markets contribute most to Apple's net… | AAPL | 0.989 | 1.000 | — |
+| What competitive risks does Apple identify in its annua… | AAPL | 0.954 | — | — |
+| **Mean** |  | **0.803** | **1.000** | **nan** |
+
+## Evaluation Run — 2026-07-19 13:25:20 UTC
+
+**Samples evaluated:** 30 / 30  
+**Full report:** `services/eval/results/eval_report_2026-07-19_13-25-20_UTC.json`
+
+**Retrieval metrics (LLM-free):** section_hit_rate = 0.800 · section_mrr = 0.678 · ticker_accuracy = 1.000
+
+| Question | Ticker | Answer Relevancy | Faithfulness | Context Precision |
+| --- | --- | --- | --- | --- |
+| What was Apple's total net revenue for fiscal year 2024? | AAPL | 1.000 | 1.000 | 0.667 |
+| What are Apple's primary supply chain risk factors disc… | AAPL | 0.913 | 1.000 | 0.000 |
+| What was Apple's Services segment revenue in fiscal yea… | AAPL | 0.000 | 0.750 | 0.000 |
+| How much did Apple spend on research and development in… | AAPL | 1.000 | 1.000 | 0.833 |
+| Which geographic markets contribute most to Apple's net… | AAPL | 0.999 | 1.000 | 0.500 |
+| What competitive risks does Apple identify in its annua… | AAPL | 0.936 | 1.000 | 1.000 |
+| What was Microsoft's total revenue for fiscal year 2024? | MSFT | 1.000 | 1.000 | 0.667 |
+| What was the revenue from Microsoft's Intelligent Cloud… | MSFT | 1.000 | 1.000 | 0.250 |
+| What are the key competition-related risk factors Micro… | MSFT | 0.924 | 0.947 | 0.000 |
+| How much did Microsoft spend on research and developmen… | MSFT | 1.000 | 1.000 | 0.917 |
+| How does Microsoft describe its approach to returning c… | MSFT | 0.946 | 1.000 | 0.500 |
+| What are Microsoft's three main business segments as de… | MSFT | 0.774 | 1.000 | 1.000 |
+| What was Alphabet's total revenue for fiscal year 2024? | GOOGL | 0.999 | 1.000 | 0.700 |
+| What was Google Cloud's revenue for fiscal year 2024? | GOOGL | 1.000 | 1.000 | 0.333 |
+| What regulatory and legal risks does Alphabet disclose … | GOOGL | 0.969 | 1.000 | 0.500 |
+| What is Alphabet's primary source of revenue according … | GOOGL | 0.902 | — | 0.000 |
+| How much did Alphabet spend on research and development… | GOOGL | 1.000 | 1.000 | 1.000 |
+| How does Alphabet describe its artificial intelligence … | GOOGL | 0.823 | 0.700 | 0.000 |
+| What was Amazon's total net sales for fiscal year 2024? | AMZN | 0.958 | 1.000 | 1.000 |
+| What was Amazon Web Services net sales for fiscal year … | AMZN | 0.000 | 1.000 | 0.000 |
+| What competition-related risks does Amazon disclose in … | AMZN | 0.967 | 0.893 | 0.000 |
+| How does Amazon describe its fulfilment and logistics n… | AMZN | 0.876 | 0.583 | 0.000 |
+| What risks does Amazon identify related to its internat… | AMZN | 0.978 | 1.000 | 0.500 |
+| What was Amazon's operating income for fiscal year 2024? | AMZN | 1.000 | 1.000 | 1.000 |
+| What was JPMorgan Chase's total net revenue for fiscal … | JPM | 1.000 | 1.000 | 1.000 |
+| What was JPMorgan Chase's net interest income for fisca… | JPM | 0.000 | 1.000 | 0.000 |
+| What credit risk factors does JPMorgan Chase identify i… | JPM | 0.925 | 1.000 | 0.000 |
+| How does JPMorgan Chase describe its capital management… | JPM | 0.000 | 1.000 | 0.000 |
+| What was JPMorgan Chase's provision for credit losses i… | JPM | 1.000 | 1.000 | 0.000 |
+| What are JPMorgan Chase's four main business segments a… | JPM | 0.859 | 0.625 | 0.000 |
+| **Mean** |  | **0.825** | **0.948** | **0.412** |
diff --git a/helm/findoc-rag/templates/configmaps.yaml b/helm/findoc-rag/templates/configmaps.yaml
@@ -65,6 +65,30 @@ data:
 
     CREATE INDEX IF NOT EXISTS idx_chunks_ticker ON document_chunks(ticker);
     CREATE INDEX IF NOT EXISTS idx_chunks_accession ON document_chunks(accession_number);
+  002_hybrid_search_and_facts.sql: |
+    ALTER TABLE document_chunks
+        ADD COLUMN IF NOT EXISTS chunk_tsv tsvector
+        GENERATED ALWAYS AS (to_tsvector('english', chunk_text)) STORED;
+
+    CREATE INDEX IF NOT EXISTS idx_chunks_tsv
+        ON document_chunks USING GIN (chunk_tsv);
+
+    CREATE TABLE IF NOT EXISTS financial_facts (
+        ticker       VARCHAR(10)  NOT NULL,
+        cik          BIGINT       NOT NULL,
+        concept      VARCHAR(120) NOT NULL,
+        label        VARCHAR(120) NOT NULL,
+        unit         VARCHAR(20)  NOT NULL,
+        fiscal_year  INTEGER      NOT NULL,
+        period_end   DATE         NOT NULL,
+        value        NUMERIC      NOT NULL,
+        filed        DATE         NOT NULL,
+        updated_at   TIMESTAMPTZ  NOT NULL DEFAULT NOW(),
+        PRIMARY KEY (ticker, concept, unit, period_end)
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_facts_ticker_year
+        ON financial_facts(ticker, fiscal_year);
 {{- if .Values.prometheus.enabled }}
 ---
 apiVersion: v1

diff --git a/helm/findoc-rag/tests/configmaps_test.yaml b/helm/findoc-rag/tests/configmaps_test.yaml
@@ -149,6 +149,26 @@ tests:
           path: data["001_initial_schema.sql"]
           pattern: "m = 16"
 
+  - it: db-migrations configmap contains the hybrid search migration
+    documentIndex: 1
+    asserts:
+      - isNotEmpty:
+          path: data["002_hybrid_search_and_facts.sql"]
+
+  - it: db-migrations 002 adds the tsvector column for full-text search
+    documentIndex: 1
+    asserts:
+      - matchRegex:
+          path: data["002_hybrid_search_and_facts.sql"]
+          pattern: "chunk_tsv tsvector"
+
+  - it: db-migrations 002 creates the financial_facts table
+    documentIndex: 1
+    asserts:
+      - matchRegex:
+          path: data["002_hybrid_search_and_facts.sql"]
+          pattern: "CREATE TABLE IF NOT EXISTS financial_facts"
+
   # ── prometheus-config configmap (documentIndex 2) ─────────────────
 
   - it: prometheus-config configmap has correct kind

diff --git a/services/embedding-worker/src/chunker.py b/services/embedding-worker/src/chunker.py
@@ -1,9 +1,17 @@
 """Section-aware chunking for 10-K filings.
 
-Implements the three-stage chunking strategy from TDD Section 5.2.2:
+Implements the chunking strategy from TDD Section 5.2.2:
   1. Section split — by 10-K item headers (Item 1, 1A, 7, etc.)
   2. Paragraph split — by double newlines within each section
-  3. Token-based windowing — 512-token windows with 64-token overlap
+  3. Paragraph packing — consecutive paragraphs are greedily packed into
+     chunks of up to 512 tokens, so short paragraphs (headings, single
+     sentences) never become their own low-signal chunks
+  4. Token-based windowing — only paragraphs that alone exceed 512 tokens
+     are split into 512-token windows with 64-token overlap
+
+Each chunk also exposes ``embedding_text`` — the chunk text prefixed with
+a contextual header (ticker, filing date, section) that is embedded but
+not stored, which measurably improves retrieval on corpus-wide queries.
 
 References:
   - TDD: FR-7 (section-aware splitting with 512/64 token window)
@@ -55,6 +63,20 @@ class Chunk:
     text: str
     token_count: int
 
+    @property
+    def embedding_text(self) -> str:
+        """Chunk text prefixed with a contextual header, used for embedding only.
+
+        The header anchors the vector to the filing's identity so that
+        queries like "Apple supply chain risks" match AAPL chunks even when
+        the chunk body never repeats the company name.  The stored/displayed
+        text (``self.text``) is unchanged.
+        """
+        return (
+            f"[{self.ticker} | 10-K | filed {self.filing_date} | {self.section_name}]\n"
+            f"{self.text}"
+        )
+
 
 # ── Tokeniser (cached) ──────────────────────────────────────────
 
@@ -159,6 +181,44 @@ def split_by_token_window(
     return windows
 
 
+# ── Stage 2b: Paragraph packing ─────────────────────────────────
+
+def pack_paragraphs(paragraphs: list[str], max_tokens: int = DEFAULT_CHUNK_SIZE) -> list[str]:
+    """Greedily pack consecutive paragraphs into groups of ≤ max_tokens.
+
+    Short paragraphs (headings, one-liners, table fragments) are merged
+    with their neighbours instead of becoming their own low-signal chunks.
+    A paragraph that alone exceeds max_tokens is emitted as its own group
+    (the caller window-splits it).
+    """
+    groups: list[str] = []
+    current: list[str] = []
+    current_tokens = 0
+
+    for paragraph in paragraphs:
+        tokens = count_tokens(paragraph)
+
+        if tokens > max_tokens:
+            if current:
+                groups.append("\n\n".join(current))
+                current, current_tokens = [], 0
+            groups.append(paragraph)  # oversize — window-split downstream
+            continue
+
+        # +1 accounts for the "\n\n" joiner between paragraphs.
+        if current and current_tokens + tokens + 1 > max_tokens:
+            groups.append("\n\n".join(current))
+            current, current_tokens = [], 0
+
+        current.append(paragraph)
+        current_tokens += tokens + 1
+
+    if current:
+        groups.append("\n\n".join(current))
+
+    return groups
+
+
 # ── Public API: full chunking pipeline ───────────────────────────
 
 def chunk_filing(
@@ -169,7 +229,7 @@ def chunk_filing(
     max_tokens: int = DEFAULT_CHUNK_SIZE,
     overlap: int = DEFAULT_OVERLAP,
 ) -> list[Chunk]:
-    """Run the full three-stage chunking pipeline on a filing.
+    """Run the full chunking pipeline on a filing.
 
     Returns a list of Chunk objects ready for embedding and storage.
     Implements FR-7, FR-8.
@@ -181,9 +241,10 @@ def chunk_filing(
 
     for section_name, section_text in sections:
         paragraphs = split_into_paragraphs(section_text)
+        groups = pack_paragraphs(paragraphs, max_tokens)
 
-        for paragraph in paragraphs:
-            windows = split_by_token_window(paragraph, max_tokens, overlap)
+        for group in groups:
+            windows = split_by_token_window(group, max_tokens, overlap)
 
             for window_text in windows:
                 token_count = count_tokens(window_text)

diff --git a/services/embedding-worker/src/main.py b/services/embedding-worker/src/main.py
@@ -55,6 +55,7 @@
     f"/{os.getenv('POSTGRES_DB', 'findocdrag')}"
 )
 EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
+EMBEDDING_BATCH_SIZE = int(os.getenv("EMBEDDING_BATCH_SIZE", "64"))
 LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
 
 TOPIC_RAW = "filings.raw"
@@ -186,10 +187,11 @@ def _process_filing(
     for c in chunks:
         CHUNK_TOKENS.observe(c.token_count)
 
-    # Embed (with batch duration timing)
-    texts = [c.text for c in chunks]
+    # Embed (with batch duration timing).  embedding_text prefixes each
+    # chunk with its filing context header — embedded, never stored.
+    texts = [c.embedding_text for c in chunks]
     t0 = time.perf_counter()
-    embeddings = embedder.embed(texts)
+    embeddings = embedder.embed(texts, batch_size=EMBEDDING_BATCH_SIZE)
     embed_elapsed = time.perf_counter() - t0
     BATCH_DURATION.observe(embed_elapsed)
 

diff --git a/services/embedding-worker/tests/test_chunker.py b/services/embedding-worker/tests/test_chunker.py
@@ -11,6 +11,7 @@
     chunk_filing,
     count_tokens,
     make_chunk_id,
+    pack_paragraphs,
     split_by_token_window,
     split_into_paragraphs,
     split_into_sections,
@@ -166,3 +167,64 @@ def test_chunk_ids_are_unique(self) -> None:
         )
         ids = [c.chunk_id for c in chunks]
         assert len(ids) == len(set(ids))  # all unique
+
+
+# ── Paragraph packing ────────────────────────────────────────────
+
+class TestPackParagraphs:
+    def test_empty_list(self) -> None:
+        assert pack_paragraphs([]) == []
+
+    def test_short_paragraphs_are_merged(self) -> None:
+        paragraphs = ["Item 7. MD&A", "Revenue grew 5% year over year.", "Margins expanded."]
+        groups = pack_paragraphs(paragraphs, max_tokens=512)
+        assert len(groups) == 1
+        assert groups[0] == "\n\n".join(paragraphs)
+
+    def test_budget_is_respected(self) -> None:
+        paragraph = "word " * 50  # ~50 tokens
+        groups = pack_paragraphs([paragraph.strip()] * 10, max_tokens=120)
+        assert len(groups) > 1
+        for group in groups:
+            assert count_tokens(group) <= 120
+
+    def test_oversize_paragraph_emitted_alone(self) -> None:
+        small = "A short line."
+        huge = "token " * 600  # exceeds the budget on its own
+        groups = pack_paragraphs([small, huge.strip(), small], max_tokens=512)
+        assert len(groups) == 3
+        assert groups[0] == small
+        assert groups[1] == huge.strip()
+        assert groups[2] == small
+
+    def test_order_is_preserved(self) -> None:
+        paragraphs = [f"Paragraph number {i}." for i in range(20)]
+        groups = pack_paragraphs(paragraphs, max_tokens=30)
+        assert "\n\n".join(groups) == "\n\n".join(paragraphs)
+
+
+# ── Contextual embedding text ────────────────────────────────────
+
+class TestEmbeddingText:
+    def _chunk(self) -> Chunk:
+        return Chunk(
+            chunk_id="x" * 64,
+            accession_number="ACC001",
+            ticker="AAPL",
+            filing_date="2024-11-01",
+            section_name="Item 1A",
+            chunk_index=0,
+            text="The Company faces supply chain risks.",
+            token_count=7,
+        )
+
+    def test_header_contains_filing_identity(self) -> None:
+        chunk = self._chunk()
+        header, body = chunk.embedding_text.split("\n", 1)
+        assert header == "[AAPL | 10-K | filed 2024-11-01 | Item 1A]"
+        assert body == chunk.text
+
+    def test_stored_text_is_unchanged(self) -> None:
+        chunk = self._chunk()
+        _ = chunk.embedding_text
+        assert chunk.text == "The Company faces supply chain risks."