Foundation: corpus-source substrate, persistence, retrieval (3 corpora live)

.gitattributes

@@ -0,0 +1 @@
+*.ipynb linguist-documentation

.github/workflows/ci.yml

@@ -0,0 +1,48 @@
+# wads CI — calls the reusable workflow hosted in i2mint/wads.
+#
+# All configuration comes from this repo's pyproject.toml [tool.wads.ci.*].
+# To customize the workflow itself (rare), replace this file with the
+# full inline template `wads/data/github_ci_uv.yml` from i2mint/wads.
+#
+# Pinning: `@master` floats with wads. If you need version stability for
+# a release-sensitive repo, change `@master` to a wads tag (e.g. `@v0.1.81`).
+# CI failure does not block a published release — it blocks the publish
+# step itself — so floating master is generally safe.
+#
+# Permissions: GitHub validates that the caller grants AT LEAST the
+# permissions any job in the called workflow requests — at workflow-parse
+# time, not at run-time, even if the job would be skipped via `if:`.
+# The reusable workflow needs:
+#   contents: write   for the publish job's version-bump push-back
+#                     and for the github-pages job's gh-pages branch push
+#   pages: write      for the github-pages job's REST API Pages config
+# Both default to `write` on org-account GITHUB_TOKEN and need to be
+# granted explicitly on personal-account callers (where the default is
+# read-only). No `id-token: write` needed — the publish-github-pages
+# action uses peaceiris/actions-gh-pages (branch-based) + REST API,
+# not the OIDC `actions/deploy-pages` flow.
+name: Continuous Integration
+on: [push, pull_request]
+jobs:
+  ci:
+    uses: i2mint/wads/.github/workflows/uv-ci.yml@master
+    permissions:
+      contents: write
+      pages: write
+    # Explicit pass-through (not `secrets: inherit`) because `inherit` does
+    # not reliably propagate caller-repo secrets to a reusable workflow owned
+    # by a different account (verified empirically: personal-account caller +
+    # i2mint-org workflow → `${{ secrets.PYPI_PASSWORD }}` resolved to empty).
+    #
+    # This list is the per-repo *transport*: it should contain PYPI_PASSWORD
+    # (for publishing) plus every secret your tests/CI need. It is generated
+    # from [tool.wads.ci.env] in pyproject.toml. To add one, run
+    #   wads-secrets add VAR_NAME            # updates pyproject + this block
+    # or just append a line below. *Which* of these become job env vars (and
+    # which are required) is controlled by [tool.wads.ci.env] — passing a
+    # secret here does not by itself put it in the environment.
+    #
+    # A secret name must also be declared in the reusable workflow's superset
+    # (wads/ci_secrets.py). `wads-secrets add` warns if it is not.
+    secrets:
+      PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

.gitignore

@@ -0,0 +1,120 @@
+.claude/handoffs/
+.claude/scratch/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+
+.DS_Store
+# C extensions
+*.so
+
+# TLS certificates
+## Ignore all PEM files anywhere
+*.pem
+## Also ignore any certs directory
+certs/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+_build
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+docs/*
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# PyCharm
+.idea

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 i2mint
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ir/__init__.py

@@ -0,0 +1,61 @@
+"""``ir`` — an information-retrieval substrate for agentic systems.
+
+One uniform "find the relevant things in this corpus" contract that scales from
+an ad-hoc search over an ephemeral list to a maintained search engine. Retrieval
+is the core; generation/selection/reranking are layered on top.
+
+Quick start::
+
+    import ir
+
+    # Define a corpus source (abstract strategy + parameters, smart defaults):
+    source = ir.CorpusSource.from_md_reports()          # project docs/ reports
+    corpus = ir.build(source)                            # index (incremental)
+    hits = ir.search(corpus, "how do I deploy the app")  # ranked SearchHits
+
+    # Light, dependency-free embedding for fast tests:
+    corpus = ir.build(source, embedder="light")
+
+A corpus source is defined by a ``scope`` (what is in the corpus), a
+``change_signal`` (what counts as stale), an ``indexing_strategy`` (how a raw
+item becomes filter fields + embeddable surfaces), and an ``embedder``. The
+default embedder is a decent *local* model (``all-MiniLM-L6-v2``); ``"light"``
+selects a numpy-only hashing embedder. Data persists under XDG dirs through a
+``dol`` repository layer.
+"""
+
+from __future__ import annotations
+
+from . import embed as _embed  # noqa: F401  (sets USE_TF=0 before transformers)
+from .base import Artifact, IndexPlan, Record, SearchHit, Surface
+from .index import Corpus, build, open_corpus
+from .retrieve import search as _search
+from .sources import CorpusSource
+from .store import CorpusStore
+from .strategy import Chunked, IndexingStrategy, Package, Skill, WholeText
+
+__all__ = [
+    "Artifact",
+    "Surface",
+    "Record",
+    "SearchHit",
+    "IndexPlan",
+    "IndexingStrategy",
+    "WholeText",
+    "Chunked",
+    "Skill",
+    "Package",
+    "CorpusSource",
+    "CorpusStore",
+    "Corpus",
+    "build",
+    "open_corpus",
+    "search",
+]
+
+
+def search(corpus, query, **kwargs):
+    """Search a :class:`~ir.index.Corpus`, or a corpus *name* (reopened lazily)."""
+    if isinstance(corpus, str):
+        corpus = open_corpus(corpus)
+    return _search(corpus, query, **kwargs)

ir/base.py

@@ -0,0 +1,117 @@
+"""Core data model for ``ir``.
+
+Retrieval in ``ir`` flows through four small, explicit types:
+
+- :class:`Artifact` — a logical item in a corpus (a file, a skill, a package).
+  Opaque ``raw`` payload plus ``metadata``.
+- :class:`Surface` — one *embeddable unit* derived from an artifact. A single
+  artifact may yield several heterogeneous surfaces (a short description, an
+  AI-authored synopsis, a list of problem classes, body chunks). The
+  artifact→surfaces decomposition is the job of an
+  :class:`~ir.strategy.IndexingStrategy`.
+- :class:`IndexPlan` — a strategy's output for one artifact: the
+  ``filter_fields`` (hard-filterable metadata, *not* embedded) and the list of
+  surfaces (embedded).
+- :class:`Record` — a stored, embedded surface (one row in the index; maps
+  directly to a ``vd`` ``Document``).
+- :class:`SearchHit` — a scored record returned by retrieval, with a helper to
+  collapse multiple surface-hits of the same artifact.
+
+The split between **filter_fields** (metadata you filter on) and **surfaces**
+(text you embed) is deliberate and central: good retrieval is hard metadata
+filtering *and* semantic ranking, not only embeddings.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+
+FilterFields = Mapping[str, Any]
+"""Non-embedded, hard-filterable metadata for an artifact (name, owner, tags)."""
+
+
+def storage_key(*parts: str) -> str:
+    """Stable, filesystem-safe id from arbitrary string parts (truncated SHA-256)."""
+    h = hashlib.sha256("␟".join(parts).encode("utf-8")).hexdigest()
+    return h[:24]
+
+
+@dataclass
+class Artifact:
+    """A logical corpus item before decomposition into surfaces."""
+
+    id: str
+    raw: Any
+    metadata: dict = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class Surface:
+    """One embeddable unit derived from an artifact.
+
+    ``kind`` names the surface type (e.g. ``"description"``, ``"synopsis"``,
+    ``"problem_class"``, ``"chunk"``) so a query can match the *right part* of
+    an artifact. ``granularity`` is a coarse hint (``"document"`` / ``"chunk"``
+    / ``"field"``). ``metadata`` is surface-local (e.g. chunk offsets).
+    """
+
+    artifact_id: str
+    kind: str
+    text: str
+    granularity: str = "document"
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class IndexPlan:
+    """An :class:`~ir.strategy.IndexingStrategy`'s output for one artifact."""
+
+    filter_fields: dict = field(default_factory=dict)
+    surfaces: list[Surface] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class Record:
+    """A stored, embedded surface — one row of the index."""
+
+    id: str
+    artifact_id: str
+    surface_kind: str
+    surface_index: int
+    text: str
+    vector: np.ndarray
+    metadata: dict = field(default_factory=dict)
+
+    @staticmethod
+    def make_id(artifact_id: str, surface_kind: str, surface_index: int) -> str:
+        """Deterministic storage id for a surface of an artifact."""
+        return storage_key(artifact_id, surface_kind, str(surface_index))
+
+
+@dataclass(frozen=True)
+class SearchHit:
+    """A scored record returned by retrieval (higher score = closer)."""
+
+    artifact_id: str
+    surface_kind: str
+    score: float
+    text: str
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+
+def best_per_artifact(hits: Sequence[SearchHit]) -> list[SearchHit]:
+    """Collapse hits to the highest-scoring surface per artifact.
+
+    Returns the surviving hits sorted by score (descending).
+    """
+    seen: dict[str, SearchHit] = {}
+    for h in hits:
+        cur = seen.get(h.artifact_id)
+        if cur is None or h.score > cur.score:
+            seen[h.artifact_id] = h
+    return sorted(seen.values(), key=lambda h: h.score, reverse=True)