20260323 implementation 1 complete

OPERATOR_QUICKREF.md

@@ -158,6 +158,10 @@ Check these fields first:
   - rebuild corpus and embeddings, then rerun `doctor --require-grounding`
 - `DOCTOR_MEMORY_DANGLING_EVIDENCE`
   - reset or repair durable memory entries that cite removed chunk keys
+- `DUPLICATE_DOCUMENT_IDENTITY`
+  - `python -m agent index --json` found two configured sources that resolve to the same global document identity
+  - if the notes are missing UUIDs, add explicit `uuid` frontmatter to one or both notes, or rename one note so the relative path changes
+  - if the notes already have UUIDs, assign distinct UUIDs unless both files intentionally represent the same document
 
 ## Security Checks
 

README.md

@@ -75,6 +75,13 @@ The corpus ingester ports the vault-aware logic that matters for retrieval quali
 - wikilink extraction
 - date extraction from paths and metadata
 
+Document identity is intentionally strict and global across configured sources:
+
+- frontmatter `uuid` is the authoritative document identity when present
+- notes without `uuid` fall back to a stable `doc_key` derived from the note's relative path
+- fallback `doc_key` values are still globally unique, so two configured sources cannot contain the same relative-path note without UUIDs
+- if that happens, corpus ingest fails fast with a duplicate document identity error; the supported fixes are to add UUIDs or rename one of the notes
+
 The only supported chunking and embedding-preprocess profile is `obsidian_v1`.
 
 ## Storage

agent/app_types.py

@@ -152,6 +152,7 @@ class DocumentRecord:
 class ChunkRecord:
     chunk_key: str
     doc_key: str
+    chunk_kind: str
     chunk_index: int
     section_index: int
     heading_path: str

agent/chunking.py

@@ -11,6 +11,16 @@
 WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(\|([^\]]+))?\]\]")
 HEADING_RE = re.compile(r"^\s{0,3}(#{1,6})\s+(.*)$")
 
+CHUNK_KIND_CONTENT = "content"
+CHUNK_KIND_METADATA = "metadata"
+METADATA_HEADING_PATH = "META: frontmatter"
+METADATA_CHUNK_ANCHOR = "frontmatter"
+METADATA_CHUNK_TITLE = "frontmatter"
+METADATA_CHUNK_INDEX = -1
+METADATA_SECTION_INDEX = -1
+METADATA_PROJECTION_VERSION = "metadata_v2"
+LEXICAL_PROJECTION_VERSION = "lexical_v2"
+
 
 @dataclass(frozen=True)
 class ChunkDraft:
@@ -35,6 +45,18 @@ class Section:
     start_char: int
 
 
+@dataclass(frozen=True)
+class MetadataProjection:
+    note_type: str
+    aliases: list[str]
+    tags: list[str]
+    journal_entry_date: Optional[str]
+    canonical_name: str
+    layer: str
+    register: str
+    text: str
+
+
 def sha256_text(value: str) -> str:
     return hashlib.sha256(value.encode("utf-8", errors="replace")).hexdigest()
 
@@ -76,13 +98,15 @@ def canonicalize_heading_path(heading_path: Any) -> list[str]:
 def stable_chunk_key(
     *,
     source_uri: str,
+    chunk_kind: str,
     heading_path: list[str],
     section_index: int,
     chunk_index: int,
 ) -> str:
     canonical_source = canonicalize_source_uri(source_uri)
     canonical_heading = " > ".join(canonicalize_heading_path(heading_path))
-    return sha256_text(f"{canonical_source}|{canonical_heading}|{section_index}|{chunk_index}")[:32]
+    kind = str(chunk_kind or CHUNK_KIND_CONTENT).strip().lower() or CHUNK_KIND_CONTENT
+    return sha256_text(f"{canonical_source}|{kind}|{canonical_heading}|{section_index}|{chunk_index}")[:32]
 
 
 def split_frontmatter(text: str) -> tuple[str, str]:
@@ -146,6 +170,95 @@ def parse_source_date(meta: dict[str, Any], filename: str) -> Optional[str]:
     return None
 
 
+def parse_string_list_field(meta: dict[str, Any], key: str) -> list[str]:
+    raw = meta.get(key)
+    if raw is None:
+        return []
+    if isinstance(raw, list):
+        items = raw
+    else:
+        items = [raw]
+    seen: set[str] = set()
+    out: list[str] = []
+    for item in items:
+        text = str(item or "").strip()
+        if not text:
+            continue
+        if text in seen:
+            continue
+        seen.add(text)
+        out.append(text)
+    return out
+
+
+def parse_string_field(meta: dict[str, Any], key: str) -> str:
+    raw = meta.get(key)
+    if raw is None:
+        return ""
+    return str(raw).strip()
+
+
+def normalize_doc_type(
+    meta: dict[str, Any],
+    *,
+    folder: str,
+    entry_date: Optional[str],
+) -> str:
+    explicit = str(meta.get("doc_type") or "").strip().lower()
+    if explicit:
+        return explicit
+    note_type = str(meta.get("note_type") or "").strip().lower()
+    if note_type:
+        if note_type in {"journal", "journal_entry", "journal-entry"}:
+            return "journal"
+        return note_type
+    note_status = str(meta.get("note_status") or "").strip().lower()
+    if note_status in {"journal", "journal_entry", "journal-entry"}:
+        return "journal"
+    if entry_date:
+        return "journal"
+    return folder.lower() if folder else "note"
+
+
+def build_metadata_projection(
+    *,
+    meta: dict[str, Any],
+    document_title: str,
+    entry_date: Optional[str],
+) -> MetadataProjection:
+    note_type = parse_string_field(meta, "note_type")
+    aliases = parse_string_list_field(meta, "aliases")
+    tags = parse_string_list_field(meta, "tags")
+    canonical_name = parse_string_field(meta, "canonical_name") or document_title
+    layer = parse_string_field(meta, "layer")
+    register = parse_string_field(meta, "register")
+    lines: list[str] = []
+    if note_type:
+        lines.append(f"note_type: {note_type}")
+    if aliases:
+        lines.append(f"aliases: {', '.join(aliases)}")
+    if tags:
+        lines.append(f"tags: {', '.join(tags)}")
+    if entry_date:
+        lines.append(f"journal_entry_date: {entry_date}")
+    if canonical_name:
+        lines.append(f"canonical_name: {canonical_name}")
+    if layer:
+        lines.append(f"layer: {layer}")
+    if register:
+        lines.append(f"register: {register}")
+    return MetadataProjection(
+        note_type=note_type,
+        aliases=aliases,
+        tags=tags,
+        journal_entry_date=entry_date,
+        canonical_name=canonical_name,
+        layer=layer,
+        register=register,
+        text="\n".join(lines).strip() + "\n",
+    )
+
+
 def normalize_markdown_light(markdown: str) -> str:
     lines = markdown.splitlines()
     out: list[str] = []