feat(finder): make the publisher host free-text searchable

cmd/ans-finder/main.go

@@ -116,6 +116,12 @@ func run(cfgPath string) error {
 			logger.Warn().Err(err).Msg("index close")
 		}
 	}()
+	// Surface schema upgrades: a migration can rewrite the whole FTS
+	// index between "starting" and "listening", so an operator must be
+	// able to confirm from logs alone that it ran.
+	if applied := store.AppliedMigrations(); len(applied) > 0 {
+		logger.Info().Strs("migrations", applied).Msg("applied index migrations")
+	}
 
 	// Feed client + poller.
 	feedClient, err := poller.NewHTTPFeedClient(cfg.Feed.BaseURL, cfg.Feed.AllowHTTP, cfg.Feed.Timeout)

internal/adapter/docsui/openapi/finder.yaml

@@ -106,6 +106,26 @@ paths:
         set when it satisfies the relevance criteria for `text` AND every
         constraint in `filter`.
 
+        **Text matching contract (this registry).** ARDS leaves text
+        relevance criteria registry-defined; this Finder matches as
+        follows. `query.text` splits on whitespace into terms, and every
+        term must match the entry across its searchable fields —
+        `displayName`, `description`, `capabilities`, `tags`, and the
+        publisher host. Matching is case- and diacritic-insensitive but
+        exact-token: no stemming and no prefix expansion, so "translate"
+        does not match "translator". Prefer content keywords — filler
+        words must also match and usually exclude everything. A term
+        containing separators (dots, hyphens) matches as a **phrase**:
+        its sub-tokens adjacent and in order within one field. So
+        `translator.example.com` matches that publisher host, and
+        "translator" alone matches too, but `translator.com` does not —
+        separate the words with spaces (`translator example`) to match
+        them independently. FTS query operators in the input (AND, OR,
+        `*`, quotes, …) are treated as literal text, never as syntax.
+        Results are currently ordered by bm25 relevance; the ranking
+        algorithm is informational, not part of this contract, and may
+        improve without notice.
+
         **Why POST.** The query carries a free-text string plus a
         structured `filter` whose keys are dot-separated field paths
         (ARDS §7.1); this does not fit a query string. The body shape is
@@ -174,7 +194,8 @@ paths:
         returns an aggregation over the matched set rather than ranked
         entries (ARDS §7.3). Explore lets a client introspect the
         registry — for example, "which media types are available?" —
-        narrowed by the same `text` and `filter` as search.
+        narrowed by the same `text` and `filter` as search. `query.text`
+        follows the search operation's **text matching contract**.
 
         For explore, `query.text` and `query.filter` are **both
         optional**; when both are absent, the aggregation covers the
@@ -267,10 +288,13 @@ components:
         text:
           type: string
           description: |
-            Natural-language description of the need (ARDS §7.1). Narrows
-            the result set by semantic relevance. Required for search,
-            optional for explore.
-          example: find me a flight booking agent
+            Free-text keyword terms describing the need (ARDS §7.1).
+            Narrows the result set by relevance under the **text matching
+            contract** documented on the search operation (exact-token,
+            all terms must match) — prefer content keywords over filler
+            words. Applies identically to search and explore. Required
+            for search, optional for explore.
+          example: flight booking
         filter:
           $ref: '#/components/schemas/Filter'
 
@@ -397,8 +421,10 @@ components:
               minimum: 0
               maximum: 100
               description: |
-                Semantic relevance ranking (0–100) computed by the
-                registry (ARDS §7.2). It is **strictly an informational
+                Relevance score (0–100) computed by the registry
+                (ARDS §7.2), normalized **within this result set** — the
+                best match scores 100 — so scores are not comparable
+                across queries. It is **strictly an informational
                 relevance metric** and MUST NOT be interpreted by
                 orchestrators as a cryptographic trust, compliance, or
                 safety rating. Trust evaluation is decoupled and handled

internal/adapter/store/sqlitefinder/apply.go

@@ -233,14 +233,15 @@ func insertActiveRow(ctx context.Context, tx *sqlx.Tx, pe project.ProjectedEntry
 		return fmt.Errorf("sqlitefinder: marshal entry: %w", err)
 	}
 
+	publisher := publisherFromURN(pe.Entry.Identifier)
 	res, err := tx.ExecContext(ctx, `
         INSERT INTO finder_entries (
             ans_name, type, url, identifier, publisher, agent_id, log_id,
             display_name, description, version, updated_at,
             lifecycle, created_at, expires_at, entry_json
         ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
 		pe.AnsName, pe.Entry.Type, pe.Entry.URL, pe.Entry.Identifier,
-		publisherFromURN(pe.Entry.Identifier), pe.AgentID, pe.LogID,
+		publisher, pe.AgentID, pe.LogID,
 		pe.Entry.DisplayName, pe.Entry.Description, pe.Entry.Version, pe.Entry.UpdatedAt,
 		string(project.LifecycleActive), pe.CreatedAt, pe.ExpiresAt, string(entryJSON),
 	)
@@ -261,7 +262,7 @@ func insertActiveRow(ctx context.Context, tx *sqlx.Tx, pe project.ProjectedEntry
 	if err := insertSideValues(ctx, tx, "finder_entry_attestation_types", rowid, attestationTypes(pe.Entry)); err != nil {
 		return err
 	}
-	return insertFTS(ctx, tx, rowid, pe.Entry)
+	return insertFTS(ctx, tx, rowid, pe.Entry, publisher)
 }
 
 // applyTombstone suppresses every ACTIVE row for the tombstone's ansName
@@ -395,13 +396,16 @@ func insertSideValues(ctx context.Context, tx *sqlx.Tx, table string, rowid int6
 }
 
 // insertFTS writes the FTS row for an entry, flattening capabilities and
-// tags into space-joined text columns the tokenizer can index.
-func insertFTS(ctx context.Context, tx *sqlx.Tx, rowid int64, e project.Entry) error {
+// tags into space-joined text columns the tokenizer can index. publisher
+// is the URN <publisher> segment (the agent's host); indexing it lets
+// free-text queries match host words — unicode61 splits on ".", so
+// translator.example.com matches the token "translator".
+func insertFTS(ctx context.Context, tx *sqlx.Tx, rowid int64, e project.Entry, publisher string) error {
 	if _, err := tx.ExecContext(ctx, `
-        INSERT INTO finder_entries_fts (rowid, display_name, description, capabilities_text, tags_text)
-        VALUES (?,?,?,?,?)`,
+        INSERT INTO finder_entries_fts (rowid, display_name, description, capabilities_text, tags_text, publisher)
+        VALUES (?,?,?,?,?,?)`,
 		rowid, e.DisplayName, e.Description,
-		strings.Join(e.Capabilities, " "), strings.Join(e.Tags, " ")); err != nil {
+		strings.Join(e.Capabilities, " "), strings.Join(e.Tags, " "), publisher); err != nil {
 		return fmt.Errorf("sqlitefinder: insert fts: %w", err)
 	}
 	return nil

internal/adapter/store/sqlitefinder/migrate_internal_test.go

@@ -0,0 +1,142 @@
+package sqlitefinder
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+
+	"github.com/agentnameservice/ans/internal/finder/index"
+)
+
+// TestMigration002_RepopulatesFTSFromExistingRows exercises the upgrade
+// path a deployed Finder takes: a database created at schema 001 —
+// with entries already indexed under the four-column FTS shape — is
+// opened by a binary that ships 002. FTS5 has no ALTER TABLE, so 002
+// drops and recreates the FTS table; this test proves the repopulation
+// preserves the existing searchable text (display, capabilities) AND
+// makes the publisher host searchable, without replaying the feed.
+//
+// Setup applies 001 verbatim from the embedded FS (not a hand-copied
+// schema, so the test cannot drift from what shipped), seeds one ACTIVE
+// and one REVOKED row exactly as the 001-era store would have left them
+// (the revoked row has no FTS row — tombstones clear it), then lets
+// Open apply 002.
+func TestMigration002_RepopulatesFTSFromExistingRows(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	path := filepath.Join(t.TempDir(), "finder.db")
+
+	seedSchema001(ctx, t, path)
+
+	// Open applies pending migrations — only 002 here.
+	s, err := Open(ctx, path)
+	if err != nil {
+		t.Fatalf("open at schema 002: %v", err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+
+	// The upgrade must be observable: callers log AppliedMigrations at
+	// startup, so it has to report exactly the migration that ran.
+	if got := s.AppliedMigrations(); len(got) != 1 || got[0] != "002_fts_publisher.sql" {
+		t.Fatalf("AppliedMigrations = %v, want [002_fts_publisher.sql]", got)
+	}
+
+	cases := []struct {
+		name string
+		text string
+		want int
+	}{
+		{name: "publisher host token now matches", text: "translator", want: 1},
+		{name: "pre-existing display text preserved", text: "converter", want: 1},
+		{name: "pre-existing description text preserved", text: "conversion", want: 1},
+		{name: "pre-existing capabilities text preserved", text: "translate_text", want: 1},
+		{name: "pre-existing tags text preserved", text: "linguistics", want: 1},
+		{name: "revoked row not resurrected into FTS", text: "revokedhost", want: 0},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			res, err := s.Search(ctx, index.SearchQuery{Text: tc.text, Limit: 10},
+				time.Date(2099, 1, 1, 0, 0, 0, 0, time.UTC))
+			if err != nil {
+				t.Fatalf("search %q: %v", tc.text, err)
+			}
+			if len(res.Results) != tc.want {
+				t.Fatalf("text %q: got %d results, want %d", tc.text, len(res.Results), tc.want)
+			}
+		})
+	}
+}
+
+// seedSchema001 builds a database exactly as a 001-era store would have
+// left it: 001 applied and recorded, one ACTIVE row with side values and
+// a four-column FTS row, one REVOKED row with blanked display fields and
+// no FTS or side rows.
+func seedSchema001(ctx context.Context, t *testing.T, path string) {
+	t.Helper()
+
+	body, err := migrationsFS.ReadFile("migrations/001_initial.sql")
+	if err != nil {
+		t.Fatalf("read embedded 001: %v", err)
+	}
+
+	db, err := sqlx.ConnectContext(ctx, "sqlite", path+"?_pragma=foreign_keys(1)")
+	if err != nil {
+		t.Fatalf("connect for seed: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	stmts := []string{
+		`CREATE TABLE schema_migrations (
+            version       TEXT PRIMARY KEY,
+            applied_at_ms INTEGER NOT NULL
+        )`,
+		string(body),
+		`INSERT INTO schema_migrations(version, applied_at_ms)
+         VALUES('001_initial.sql', 0)`,
+
+		// ACTIVE row: display fields share no tokens with the host, so a
+		// post-migration host match can only come from the new column.
+		`INSERT INTO finder_entries (
+            id, ans_name, type, url, identifier, publisher, agent_id, log_id,
+            display_name, description, lifecycle, created_at, entry_json
+        ) VALUES (
+            1, 'ans://v1.0.0.translator.example.com',
+            'application/mcp-server-card+json',
+            'https://translator.example.com/.well-known/mcp.json',
+            'urn:air:translator.example.com:agents:converter',
+            'translator.example.com', 'agent-1', 'log-1',
+            'converter', 'language conversion service',
+            'ACTIVE', '2025-01-01T00:00:00Z',
+            '{"identifier":"urn:air:translator.example.com:agents:converter","displayName":"converter"}'
+        )`,
+		`INSERT INTO finder_entry_capabilities (entry_rowid, value)
+         VALUES (1, 'translate_text')`,
+		`INSERT INTO finder_entry_tags (entry_rowid, value)
+         VALUES (1, 'linguistics')`,
+		`INSERT INTO finder_entries_fts (
+            rowid, display_name, description, capabilities_text, tags_text
+        ) VALUES (1, 'converter', 'language conversion service', 'translate_text', 'linguistics')`,
+
+		// REVOKED row, as a tombstone leaves it: blanked display fields,
+		// no FTS row, no side rows. 002 must not give it one.
+		`INSERT INTO finder_entries (
+            id, ans_name, type, url, identifier, publisher, agent_id, log_id,
+            display_name, description, lifecycle, created_at, entry_json
+        ) VALUES (
+            2, 'ans://v1.0.0.revokedhost.example.com',
+            'application/mcp-server-card+json',
+            'https://revokedhost.example.com/.well-known/mcp.json',
+            'urn:air:revokedhost.example.com:agents:gone',
+            'revokedhost.example.com', 'agent-2', 'tomb-2',
+            '', '', 'REVOKED', '2025-02-01T00:00:00Z', '{}'
+        )`,
+	}
+	for _, stmt := range stmts {
+		if _, err := db.ExecContext(ctx, stmt); err != nil {
+			t.Fatalf("seed stmt failed: %v\n%s", err, stmt)
+		}
+	}
+}

internal/adapter/store/sqlitefinder/migrations/002_fts_publisher.sql

@@ -0,0 +1,56 @@
+-- Add the publisher host to the free-text search surface.
+--
+-- The publisher (the agent's verified host, the URN's <publisher>
+-- segment) is the one text every entry reliably carries, and the first
+-- thing a user types when they know the agent's domain — but 001 indexed
+-- it only as a structured-filter column on the base table, so free-text
+-- "translator" found nothing for translator.example.com unless the
+-- publisher happened to repeat the word in its display fields. Adding
+-- the column to the FTS table makes host words match: unicode61 treats
+-- "." as a token separator, so translator.example.com indexes as the
+-- tokens translator / example / com. Shared TLD tokens ("example",
+-- "com") appear in nearly every row and bm25 down-weights them
+-- accordingly, so they add no meaningful noise.
+--
+-- FTS5 virtual tables do not support ALTER TABLE ADD COLUMN, so the
+-- table is dropped and recreated with the new shape, then repopulated
+-- from the base + side tables. Repopulation is self-contained — no feed
+-- replay — because every indexed value survives outside the FTS copy:
+-- display_name/description/publisher on finder_entries, capabilities
+-- and tags in their side tables (group_concat ORDER BY rowid mirrors
+-- insertFTS's space-joining in insertion order). Only ACTIVE rows are
+-- rebuilt: that is the runtime invariant (insertActiveRow adds the FTS
+-- row, tombstone suppression deletes it), and rebuilding a
+-- REVOKED/DEPRECATED row would resurrect it into search results.
+--
+-- Who this migration protects: any deployment created from a released
+-- 001-schema binary (v0.1.x and later ship the finder). Amending 001 in
+-- place would silently skip those databases, whose 001 row is already
+-- recorded in schema_migrations.
+DROP TABLE finder_entries_fts;
+
+CREATE VIRTUAL TABLE finder_entries_fts USING fts5(
+    display_name,
+    description,
+    capabilities_text,
+    tags_text,
+    publisher,
+    tokenize='unicode61 remove_diacritics 2'
+);
+
+INSERT INTO finder_entries_fts (
+    rowid, display_name, description, capabilities_text, tags_text, publisher
+)
+SELECT
+    e.id,
+    e.display_name,
+    e.description,
+    COALESCE((SELECT group_concat(c.value, ' ' ORDER BY c.rowid)
+                FROM finder_entry_capabilities c
+               WHERE c.entry_rowid = e.id), ''),
+    COALESCE((SELECT group_concat(t.value, ' ' ORDER BY t.rowid)
+                FROM finder_entry_tags t
+               WHERE t.entry_rowid = e.id), ''),
+    e.publisher
+FROM finder_entries e
+WHERE e.lifecycle = 'ACTIVE';