diff --git a/internal/storage/sqlite/issues.go b/internal/storage/sqlite/issues.go index a47e6a358e..a44ab03a78 100644 --- a/internal/storage/sqlite/issues.go +++ b/internal/storage/sqlite/issues.go @@ -108,6 +108,7 @@ func insertIssueStrict(ctx context.Context, conn *sql.Conn, issue *types.Issue) crystallizes = 1 } + metadataStr := string(issue.Metadata) _, err := conn.ExecContext(ctx, ` INSERT INTO issues ( id, content_hash, title, description, design, acceptance_criteria, notes, @@ -130,11 +131,17 @@ func insertIssueStrict(ctx context.Context, conn *sql.Conn, issue *types.Issue) issue.AwaitType, issue.AwaitID, int64(issue.Timeout), formatJSONStringArray(issue.Waiters), string(issue.MolType), issue.EventKind, issue.Actor, issue.Target, issue.Payload, - issue.DueAt, issue.DeferUntil, string(issue.Metadata), + issue.DueAt, issue.DeferUntil, metadataStr, ) if err != nil { return fmt.Errorf("failed to insert issue: %w", err) } + + // Update metadata index (GH#1589) + if err := updateMetadataIndex(ctx, conn, issue.ID, metadataStr); err != nil { + return fmt.Errorf("failed to index metadata: %w", err) + } + return nil } @@ -180,7 +187,8 @@ func insertIssues(ctx context.Context, conn *sql.Conn, issues []*types.Issue) er crystallizes = 1 } - _, err = stmt.ExecContext(ctx, + metadataStr := string(issue.Metadata) + res, err := stmt.ExecContext(ctx, issue.ID, issue.ContentHash, issue.Title, issue.Description, issue.Design, issue.AcceptanceCriteria, issue.Notes, issue.Status, issue.Priority, issue.IssueType, issue.Assignee, @@ -191,7 +199,7 @@ func insertIssues(ctx context.Context, conn *sql.Conn, issues []*types.Issue) er issue.AwaitType, issue.AwaitID, int64(issue.Timeout), formatJSONStringArray(issue.Waiters), string(issue.MolType), issue.EventKind, issue.Actor, issue.Target, issue.Payload, - issue.DueAt, issue.DeferUntil, string(issue.Metadata), + issue.DueAt, issue.DeferUntil, metadataStr, ) if err != nil { // INSERT OR IGNORE should handle duplicates, but driver may still return error @@ -200,6 +208,16 @@ func insertIssues(ctx context.Context, conn *sql.Conn, issues []*types.Issue) er return fmt.Errorf("failed to insert issue %s: %w", issue.ID, err) } // Duplicate ID detected and ignored (INSERT OR IGNORE succeeded) + } else { + // Update metadata index (GH#1589) — only if issue was actually inserted. + // INSERT OR IGNORE returns RowsAffected=0 for duplicates; indexing the + // ignored metadata would cause the index to drift from the issues table. + rows, _ := res.RowsAffected() + if rows > 0 { + if err := updateMetadataIndex(ctx, conn, issue.ID, metadataStr); err != nil { + return fmt.Errorf("failed to index metadata for %s: %w", issue.ID, err) + } + } } } return nil @@ -250,6 +268,7 @@ func insertIssuesStrict(ctx context.Context, conn *sql.Conn, issues []*types.Iss crystallizes = 1 } + metadataStr := string(issue.Metadata) _, err = stmt.ExecContext(ctx, issue.ID, issue.ContentHash, issue.Title, issue.Description, issue.Design, issue.AcceptanceCriteria, issue.Notes, issue.Status, @@ -261,11 +280,16 @@ func insertIssuesStrict(ctx context.Context, conn *sql.Conn, issues []*types.Iss issue.AwaitType, issue.AwaitID, int64(issue.Timeout), formatJSONStringArray(issue.Waiters), string(issue.MolType), issue.EventKind, issue.Actor, issue.Target, issue.Payload, - issue.DueAt, issue.DeferUntil, string(issue.Metadata), + issue.DueAt, issue.DeferUntil, metadataStr, ) if err != nil { return fmt.Errorf("failed to insert issue %s: %w", issue.ID, err) } + + // Update metadata index (GH#1589) + if err := updateMetadataIndex(ctx, conn, issue.ID, metadataStr); err != nil { + return fmt.Errorf("failed to index metadata for %s: %w", issue.ID, err) + } } return nil } diff --git a/internal/storage/sqlite/metadata_index.go b/internal/storage/sqlite/metadata_index.go new file mode 100644 index 0000000000..e4888e388f --- /dev/null +++ b/internal/storage/sqlite/metadata_index.go @@ -0,0 +1,121 @@ +package sqlite + +import ( + "context" + "encoding/json" + "fmt" +) + +// updateMetadataIndex refreshes the index rows for a specific issue. +// It must be called within the transaction of Create/Update issue. +// +// For Phase 1, this indexes all top-level scalar values (string, int, float, bool) +// found in the metadata JSON blob. No schema is required. +// +// GH#1589: Phase 1 of the Schema-Indexed Metadata architecture. +func updateMetadataIndex(ctx context.Context, exec dbExecutor, issueID string, metadataJSON string) error { + // Clear existing index entries for this issue. + // We do a full delete/re-insert for simplicity and correctness. + _, err := exec.ExecContext(ctx, `DELETE FROM issue_metadata_index WHERE issue_id = ?`, issueID) + if err != nil { + return fmt.Errorf("failed to clear metadata index for %s: %w", issueID, err) + } + + if metadataJSON == "" || metadataJSON == "{}" { + return nil + } + + // Parse metadata JSON + var meta map[string]any + if err := json.Unmarshal([]byte(metadataJSON), &meta); err != nil { + // Invalid JSON is ignored for indexing; validation happens elsewhere + return nil + } + + // Index top-level scalars + return indexFlatKeys(ctx, exec, issueID, "", meta) +} + +// indexFlatKeys indexes scalar values from a metadata map, supporting one level of nesting +// for namespaced keys (e.g., "jira.story_points"). +func indexFlatKeys(ctx context.Context, exec dbExecutor, issueID, prefix string, meta map[string]any) error { + stmt := `INSERT OR REPLACE INTO issue_metadata_index (issue_id, key, value_text, value_int, value_real) VALUES (?, ?, ?, ?, ?)` + + for key, val := range meta { + fullKey := key + if prefix != "" { + fullKey = prefix + "." + key + } + + switch v := val.(type) { + case string: + if _, err := exec.ExecContext(ctx, stmt, issueID, fullKey, v, nil, nil); err != nil { + return fmt.Errorf("failed to index metadata key %s: %w", fullKey, err) + } + case float64: + // JSON numbers are float64. Check if it's actually an integer. + if v == float64(int64(v)) { + if _, err := exec.ExecContext(ctx, stmt, issueID, fullKey, nil, int64(v), nil); err != nil { + return fmt.Errorf("failed to index metadata key %s: %w", fullKey, err) + } + } else { + if _, err := exec.ExecContext(ctx, stmt, issueID, fullKey, nil, nil, v); err != nil { + return fmt.Errorf("failed to index metadata key %s: %w", fullKey, err) + } + } + case bool: + i := int64(0) + if v { + i = 1 + } + if _, err := exec.ExecContext(ctx, stmt, issueID, fullKey, nil, i, nil); err != nil { + return fmt.Errorf("failed to index metadata key %s: %w", fullKey, err) + } + case map[string]any: + // Support one level of nesting for namespaced keys (e.g., "jira.story_points") + if prefix == "" { + if err := indexFlatKeys(ctx, exec, issueID, key, v); err != nil { + return err + } + } + // Skip deeper nesting + default: + // Skip arrays, nulls, and deeper structures + continue + } + } + return nil +} + +// RebuildMetadataIndex wipes and rebuilds the entire metadata index from the +// canonical metadata column. Safe to call from bd doctor or bd import. +func (s *SQLiteStorage) RebuildMetadataIndex(ctx context.Context) error { + conn, err := s.db.Conn(ctx) + if err != nil { + return fmt.Errorf("failed to acquire connection: %w", err) + } + defer func() { _ = conn.Close() }() + + // Truncate the index + if _, err := conn.ExecContext(ctx, `DELETE FROM issue_metadata_index`); err != nil { + return fmt.Errorf("failed to truncate metadata index: %w", err) + } + + // Scan all issues with non-empty metadata + rows, err := conn.QueryContext(ctx, `SELECT id, metadata FROM issues WHERE metadata IS NOT NULL AND metadata != '' AND metadata != '{}'`) + if err != nil { + return fmt.Errorf("failed to query issues for metadata index rebuild: %w", err) + } + defer func() { _ = rows.Close() }() + + for rows.Next() { + var id, meta string + if err := rows.Scan(&id, &meta); err != nil { + return fmt.Errorf("failed to scan issue for metadata index: %w", err) + } + if err := updateMetadataIndex(ctx, conn, id, meta); err != nil { + return fmt.Errorf("failed to index metadata for %s: %w", id, err) + } + } + return rows.Err() +} diff --git a/internal/storage/sqlite/metadata_index_test.go b/internal/storage/sqlite/metadata_index_test.go new file mode 100644 index 0000000000..1d90ec45b6 --- /dev/null +++ b/internal/storage/sqlite/metadata_index_test.go @@ -0,0 +1,406 @@ +package sqlite + +import ( + "database/sql" + "encoding/json" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +func TestMetadataIndex_CreateIssue(t *testing.T) { + env := newTestEnv(t) + ctx := env.Ctx + + // Create an issue with metadata + issue := &types.Issue{ + Title: "test metadata indexing", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Metadata: json.RawMessage(`{"category":"security","severity":"high","story_points":5}`), + } + + if err := env.Store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("CreateIssue failed: %v", err) + } + + // Verify index rows were created + rows, err := env.Store.db.QueryContext(ctx, `SELECT key, value_text, value_int FROM issue_metadata_index WHERE issue_id = ? ORDER BY key`, issue.ID) + if err != nil { + t.Fatalf("query index failed: %v", err) + } + defer func() { _ = rows.Close() }() + + type indexRow struct { + Key string + ValueText sql.NullString + ValueInt sql.NullInt64 + } + + var got []indexRow + for rows.Next() { + var r indexRow + if err := rows.Scan(&r.Key, &r.ValueText, &r.ValueInt); err != nil { + t.Fatalf("scan failed: %v", err) + } + got = append(got, r) + } + + if len(got) != 3 { + t.Fatalf("expected 3 index rows, got %d: %+v", len(got), got) + } + + // Check specific values + byKey := map[string]indexRow{} + for _, r := range got { + byKey[r.Key] = r + } + + if r, ok := byKey["category"]; !ok || !r.ValueText.Valid || r.ValueText.String != "security" { + t.Errorf("category: expected text 'security', got %+v", byKey["category"]) + } + if r, ok := byKey["severity"]; !ok || !r.ValueText.Valid || r.ValueText.String != "high" { + t.Errorf("severity: expected text 'high', got %+v", byKey["severity"]) + } + if r, ok := byKey["story_points"]; !ok || !r.ValueInt.Valid || r.ValueInt.Int64 != 5 { + t.Errorf("story_points: expected int 5, got %+v", byKey["story_points"]) + } +} + +func TestMetadataIndex_NestedKeys(t *testing.T) { + env := newTestEnv(t) + ctx := env.Ctx + + // Create an issue with namespaced metadata (e.g., tracker sync data) + issue := &types.Issue{ + Title: "test nested metadata", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Metadata: json.RawMessage(`{"jira":{"story_points":8,"sprint":"Sprint 24"},"local_key":"value"}`), + } + + if err := env.Store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("CreateIssue failed: %v", err) + } + + // Verify nested keys are indexed with dot notation + rows, err := env.Store.db.QueryContext(ctx, `SELECT key, value_text, value_int FROM issue_metadata_index WHERE issue_id = ? ORDER BY key`, issue.ID) + if err != nil { + t.Fatalf("query index failed: %v", err) + } + defer func() { _ = rows.Close() }() + + type indexRow struct { + Key string + ValueText sql.NullString + ValueInt sql.NullInt64 + } + + byKey := map[string]indexRow{} + for rows.Next() { + var r indexRow + if err := rows.Scan(&r.Key, &r.ValueText, &r.ValueInt); err != nil { + t.Fatalf("scan failed: %v", err) + } + byKey[r.Key] = r + } + + if r, ok := byKey["jira.story_points"]; !ok || !r.ValueInt.Valid || r.ValueInt.Int64 != 8 { + t.Errorf("jira.story_points: expected int 8, got %+v", byKey["jira.story_points"]) + } + if r, ok := byKey["jira.sprint"]; !ok || !r.ValueText.Valid || r.ValueText.String != "Sprint 24" { + t.Errorf("jira.sprint: expected text 'Sprint 24', got %+v", byKey["jira.sprint"]) + } + if r, ok := byKey["local_key"]; !ok || !r.ValueText.Valid || r.ValueText.String != "value" { + t.Errorf("local_key: expected text 'value', got %+v", byKey["local_key"]) + } +} + +func TestMetadataIndex_UpdateIssue(t *testing.T) { + env := newTestEnv(t) + ctx := env.Ctx + + // Create issue with initial metadata + issue := &types.Issue{ + Title: "test metadata update", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Metadata: json.RawMessage(`{"category":"bug","priority_tag":"low"}`), + } + + if err := env.Store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("CreateIssue failed: %v", err) + } + + // Update metadata + err := env.Store.UpdateIssue(ctx, issue.ID, map[string]interface{}{ + "metadata": `{"category":"feature","new_field":"hello"}`, + }, "test") + if err != nil { + t.Fatalf("UpdateIssue failed: %v", err) + } + + // Verify index was refreshed (old keys removed, new keys added) + rows, err := env.Store.db.QueryContext(ctx, `SELECT key, value_text FROM issue_metadata_index WHERE issue_id = ? ORDER BY key`, issue.ID) + if err != nil { + t.Fatalf("query index failed: %v", err) + } + defer func() { _ = rows.Close() }() + + byKey := map[string]string{} + for rows.Next() { + var key string + var val sql.NullString + if err := rows.Scan(&key, &val); err != nil { + t.Fatalf("scan failed: %v", err) + } + if val.Valid { + byKey[key] = val.String + } + } + + if byKey["category"] != "feature" { + t.Errorf("category: expected 'feature', got %q", byKey["category"]) + } + if byKey["new_field"] != "hello" { + t.Errorf("new_field: expected 'hello', got %q", byKey["new_field"]) + } + if _, exists := byKey["priority_tag"]; exists { + t.Error("priority_tag should have been removed from index after update") + } +} + +func TestMetadataIndex_EmptyMetadata(t *testing.T) { + env := newTestEnv(t) + ctx := env.Ctx + + // Create issue with no metadata + issue := &types.Issue{ + Title: "no metadata", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + } + + if err := env.Store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("CreateIssue failed: %v", err) + } + + // Verify no index rows + var count int + err := env.Store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM issue_metadata_index WHERE issue_id = ?`, issue.ID).Scan(&count) + if err != nil { + t.Fatalf("count query failed: %v", err) + } + if count != 0 { + t.Errorf("expected 0 index rows for empty metadata, got %d", count) + } +} + +func TestMetadataIndex_BoolAndFloat(t *testing.T) { + env := newTestEnv(t) + ctx := env.Ctx + + issue := &types.Issue{ + Title: "test types", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Metadata: json.RawMessage(`{"active":true,"score":3.14,"count":42}`), + } + + if err := env.Store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("CreateIssue failed: %v", err) + } + + rows, err := env.Store.db.QueryContext(ctx, `SELECT key, value_text, value_int, value_real FROM issue_metadata_index WHERE issue_id = ? ORDER BY key`, issue.ID) + if err != nil { + t.Fatalf("query index failed: %v", err) + } + defer func() { _ = rows.Close() }() + + type indexRow struct { + Key string + ValueText sql.NullString + ValueInt sql.NullInt64 + ValueReal sql.NullFloat64 + } + + byKey := map[string]indexRow{} + for rows.Next() { + var r indexRow + if err := rows.Scan(&r.Key, &r.ValueText, &r.ValueInt, &r.ValueReal); err != nil { + t.Fatalf("scan failed: %v", err) + } + byKey[r.Key] = r + } + + // Bool true → int 1 + if r, ok := byKey["active"]; !ok || !r.ValueInt.Valid || r.ValueInt.Int64 != 1 { + t.Errorf("active: expected int 1 (true), got %+v", byKey["active"]) + } + // Float + if r, ok := byKey["score"]; !ok || !r.ValueReal.Valid { + t.Errorf("score: expected real value, got %+v", byKey["score"]) + } else if r.ValueReal.Float64 < 3.13 || r.ValueReal.Float64 > 3.15 { + t.Errorf("score: expected ~3.14, got %f", r.ValueReal.Float64) + } + // Integer + if r, ok := byKey["count"]; !ok || !r.ValueInt.Valid || r.ValueInt.Int64 != 42 { + t.Errorf("count: expected int 42, got %+v", byKey["count"]) + } +} + +func TestMetadataIndex_RebuildMetadataIndex(t *testing.T) { + env := newTestEnv(t) + ctx := env.Ctx + + // Create issues with metadata + issue1 := &types.Issue{ + Title: "issue1", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Metadata: json.RawMessage(`{"team":"platform"}`), + } + issue2 := &types.Issue{ + Title: "issue2", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Metadata: json.RawMessage(`{"team":"frontend","severity":"high"}`), + } + + if err := env.Store.CreateIssue(ctx, issue1, "test"); err != nil { + t.Fatalf("CreateIssue 1 failed: %v", err) + } + if err := env.Store.CreateIssue(ctx, issue2, "test"); err != nil { + t.Fatalf("CreateIssue 2 failed: %v", err) + } + + // Manually wipe the index + if _, err := env.Store.db.ExecContext(ctx, `DELETE FROM issue_metadata_index`); err != nil { + t.Fatalf("failed to wipe index: %v", err) + } + + // Verify it's empty + var count int + if err := env.Store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM issue_metadata_index`).Scan(&count); err != nil { + t.Fatalf("count query failed: %v", err) + } + if count != 0 { + t.Fatalf("expected 0 rows after wipe, got %d", count) + } + + // Rebuild + if err := env.Store.RebuildMetadataIndex(ctx); err != nil { + t.Fatalf("RebuildMetadataIndex failed: %v", err) + } + + // Verify index is repopulated + if err := env.Store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM issue_metadata_index`).Scan(&count); err != nil { + t.Fatalf("count query failed: %v", err) + } + // issue1: team → 1 row; issue2: team + severity → 2 rows → total 3 + if count != 3 { + t.Errorf("expected 3 index rows after rebuild, got %d", count) + } +} + +func TestMetadataIndex_ImportDuplicate(t *testing.T) { + env := newTestEnv(t) + ctx := env.Ctx + + // 1. Create an existing issue with metadata + now := time.Now() + issue := &types.Issue{ + Title: "Original", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + Metadata: json.RawMessage(`{"value":"original"}`), + CreatedAt: now, + UpdatedAt: now, + } + if err := env.Store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("Setup failed: %v", err) + } + + // 2. Simulate an import of the SAME issue with DIFFERENT metadata. + // insertIssues uses INSERT OR IGNORE, so the issues table keeps the old row, + // but a buggy implementation would update the index with the stale import data. + importIssue := *issue + importIssue.Metadata = json.RawMessage(`{"value":"stale_import"}`) + + conn, err := env.Store.db.Conn(ctx) + if err != nil { + t.Fatalf("Failed to get conn: %v", err) + } + defer func() { _ = conn.Close() }() + + if err := insertIssues(ctx, conn, []*types.Issue{&importIssue}); err != nil { + t.Fatalf("insertIssues failed: %v", err) + } + + // 3. Verify the index was NOT updated with the stale import data. + // The issue already existed, so INSERT OR IGNORE did nothing. + // The index must match the DB ("original"), not the ignored import ("stale_import"). + var value string + err = env.Store.db.QueryRowContext(ctx, + `SELECT value_text FROM issue_metadata_index WHERE issue_id = ? AND key = 'value'`, + issue.ID).Scan(&value) + if err != nil { + t.Fatalf("Query failed: %v", err) + } + if value != "original" { + t.Errorf("Index drift detected! Expected 'original', got %q. "+ + "The index was updated even though the issue insert was ignored.", value) + } +} + +func TestMetadataIndex_DeleteCascade(t *testing.T) { + env := newTestEnv(t) + ctx := env.Ctx + + issue := &types.Issue{ + Title: "To be deleted", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + Metadata: json.RawMessage(`{"ghost":"buster"}`), + } + + if err := env.Store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("CreateIssue failed: %v", err) + } + + // Verify index exists + var count int + if err := env.Store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM issue_metadata_index WHERE issue_id = ?`, + issue.ID).Scan(&count); err != nil { + t.Fatalf("count query failed: %v", err) + } + if count == 0 { + t.Fatal("Setup failed: index not created") + } + + // Delete the issue — ON DELETE CASCADE should clean up the index + if err := env.Store.DeleteIssue(ctx, issue.ID); err != nil { + t.Fatalf("DeleteIssue failed: %v", err) + } + + // Verify index rows are gone + if err := env.Store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM issue_metadata_index WHERE issue_id = ?`, + issue.ID).Scan(&count); err != nil { + t.Fatalf("count query failed: %v", err) + } + if count != 0 { + t.Errorf("FK CASCADE failed! Expected 0 index rows after delete, got %d. Ghost data remains.", count) + } +} diff --git a/internal/storage/sqlite/migrations.go b/internal/storage/sqlite/migrations.go index 58065d3a82..cfe5820086 100644 --- a/internal/storage/sqlite/migrations.go +++ b/internal/storage/sqlite/migrations.go @@ -60,6 +60,7 @@ var migrationsList = []Migration{ {"metadata_column", migrations.MigrateMetadataColumn}, {"wisp_type_column", migrations.MigrateWispTypeColumn}, {"spec_id_column", migrations.MigrateSpecIDColumn}, + {"metadata_index_table", migrations.MigrateMetadataIndexTable}, } // migrationInfo contains metadata about a migration for inspection @@ -127,6 +128,7 @@ func getMigrationDescription(name string) string { "metadata_column": "Adds metadata column for arbitrary JSON data (tool annotations, file lists) per GH#1406", "wisp_type_column": "Adds wisp_type column for TTL-based compaction classification (gt-9br)", "spec_id_column": "Adds spec_id column for linking issues to specification documents", + "metadata_index_table": "Adds issue_metadata_index table for fast querying of metadata fields (GH#1589)", } if desc, ok := descriptions[name]; ok { diff --git a/internal/storage/sqlite/migrations/044_metadata_index_table.go b/internal/storage/sqlite/migrations/044_metadata_index_table.go new file mode 100644 index 0000000000..e69e523e0a --- /dev/null +++ b/internal/storage/sqlite/migrations/044_metadata_index_table.go @@ -0,0 +1,60 @@ +package migrations + +import ( + "database/sql" + "fmt" +) + +// MigrateMetadataIndexTable creates the issue_metadata_index table for fast querying +// of metadata fields. This is Phase 1 of the Schema-Indexed Metadata architecture (GH#1589). +// +// The table acts as a cache/index over the canonical metadata JSON blob in the issues table. +// It indexes all top-level scalar values (string, int, float, bool) found in metadata. +// If the index gets out of sync, it can be rebuilt from metadata via bd doctor. +func MigrateMetadataIndexTable(db *sql.DB) error { + // Check if table already exists + var tableExists bool + err := db.QueryRow(` + SELECT COUNT(*) > 0 + FROM sqlite_master + WHERE type='table' AND name='issue_metadata_index' + `).Scan(&tableExists) + if err != nil { + return fmt.Errorf("failed to check issue_metadata_index table: %w", err) + } + + if !tableExists { + _, err = db.Exec(` + CREATE TABLE issue_metadata_index ( + issue_id TEXT NOT NULL, + key TEXT NOT NULL, + value_text TEXT, + value_int INTEGER, + value_real REAL, + PRIMARY KEY (issue_id, key), + FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE + ) + `) + if err != nil { + return fmt.Errorf("failed to create issue_metadata_index table: %w", err) + } + } + + // Create indexes for fast filtering by key+value + _, err = db.Exec(`CREATE INDEX IF NOT EXISTS idx_meta_text ON issue_metadata_index(key, value_text)`) + if err != nil { + return fmt.Errorf("failed to create idx_meta_text index: %w", err) + } + + _, err = db.Exec(`CREATE INDEX IF NOT EXISTS idx_meta_int ON issue_metadata_index(key, value_int)`) + if err != nil { + return fmt.Errorf("failed to create idx_meta_int index: %w", err) + } + + // Note: Full backfill of individual keys requires JSON parsing in Go, + // which is handled by RebuildMetadataIndex. New/updated issues will be + // indexed automatically; existing issues get indexed on next import or + // via 'bd doctor'. + + return nil +} diff --git a/internal/storage/sqlite/queries.go b/internal/storage/sqlite/queries.go index a1c28f3012..f0f2ab912c 100644 --- a/internal/storage/sqlite/queries.go +++ b/internal/storage/sqlite/queries.go @@ -1081,6 +1081,14 @@ func (s *SQLiteStorage) UpdateIssue(ctx context.Context, id string, updates map[ // NOTE: Graph edges now managed via AddDependency() per Decision 004 Phase 4. + // Update metadata index if metadata was changed (GH#1589) + if metaVal, ok := updates["metadata"]; ok { + metaStr, _ := storage.NormalizeMetadataValue(metaVal) + if err := updateMetadataIndex(ctx, conn, id, metaStr); err != nil { + return fmt.Errorf("failed to update metadata index: %w", err) + } + } + // Mark issue as dirty for incremental export if err := markDirty(ctx, conn, id); err != nil { return fmt.Errorf("failed to mark issue dirty: %w", err) diff --git a/internal/storage/sqlite/transaction.go b/internal/storage/sqlite/transaction.go index 3043e3f370..2dccc4170d 100644 --- a/internal/storage/sqlite/transaction.go +++ b/internal/storage/sqlite/transaction.go @@ -530,6 +530,14 @@ func (t *sqliteTxStorage) UpdateIssue(ctx context.Context, id string, updates ma return fmt.Errorf("failed to record event: %w", err) } + // Update metadata index if metadata was changed (GH#1589) + if metaVal, ok := updates["metadata"]; ok { + metaStr, _ := storage.NormalizeMetadataValue(metaVal) + if err := updateMetadataIndex(ctx, t.conn, id, metaStr); err != nil { + return fmt.Errorf("failed to update metadata index: %w", err) + } + } + // Mark issue as dirty if err := markDirty(ctx, t.conn, id); err != nil { return fmt.Errorf("failed to mark issue dirty: %w", err)