Skip to content

[STALE] feat: metadata indexing infrastructure (Phase 1) #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
turian wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[STALE] feat: metadata indexing infrastructure (Phase 1) #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions internal/storage/sqlite/issues.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ func insertIssueStrict(ctx context.Context, conn *sql.Conn, issue *types.Issue)
crystallizes = 1
}

metadataStr := string(issue.Metadata)
_, err := conn.ExecContext(ctx, `
INSERT INTO issues (
id, content_hash, title, description, design, acceptance_criteria, notes,
Expand All @@ -130,11 +131,17 @@ func insertIssueStrict(ctx context.Context, conn *sql.Conn, issue *types.Issue)
issue.AwaitType, issue.AwaitID, int64(issue.Timeout), formatJSONStringArray(issue.Waiters),
string(issue.MolType),
issue.EventKind, issue.Actor, issue.Target, issue.Payload,
issue.DueAt, issue.DeferUntil, string(issue.Metadata),
issue.DueAt, issue.DeferUntil, metadataStr,
)
if err != nil {
return fmt.Errorf("failed to insert issue: %w", err)
}

// Update metadata index (GH#1589)
if err := updateMetadataIndex(ctx, conn, issue.ID, metadataStr); err != nil {
return fmt.Errorf("failed to index metadata: %w", err)
}

return nil
}

Expand Down Expand Up @@ -180,7 +187,8 @@ func insertIssues(ctx context.Context, conn *sql.Conn, issues []*types.Issue) er
crystallizes = 1
}

_, err = stmt.ExecContext(ctx,
metadataStr := string(issue.Metadata)
res, err := stmt.ExecContext(ctx,
issue.ID, issue.ContentHash, issue.Title, issue.Description, issue.Design,
issue.AcceptanceCriteria, issue.Notes, issue.Status,
issue.Priority, issue.IssueType, issue.Assignee,
Expand All @@ -191,7 +199,7 @@ func insertIssues(ctx context.Context, conn *sql.Conn, issues []*types.Issue) er
issue.AwaitType, issue.AwaitID, int64(issue.Timeout), formatJSONStringArray(issue.Waiters),
string(issue.MolType),
issue.EventKind, issue.Actor, issue.Target, issue.Payload,
issue.DueAt, issue.DeferUntil, string(issue.Metadata),
issue.DueAt, issue.DeferUntil, metadataStr,
)
if err != nil {
// INSERT OR IGNORE should handle duplicates, but driver may still return error
Expand All @@ -200,6 +208,16 @@ func insertIssues(ctx context.Context, conn *sql.Conn, issues []*types.Issue) er
return fmt.Errorf("failed to insert issue %s: %w", issue.ID, err)
}
// Duplicate ID detected and ignored (INSERT OR IGNORE succeeded)
} else {
// Update metadata index (GH#1589) — only if issue was actually inserted.
// INSERT OR IGNORE returns RowsAffected=0 for duplicates; indexing the
// ignored metadata would cause the index to drift from the issues table.
rows, _ := res.RowsAffected()
if rows > 0 {
if err := updateMetadataIndex(ctx, conn, issue.ID, metadataStr); err != nil {
return fmt.Errorf("failed to index metadata for %s: %w", issue.ID, err)
}
}
}
}
return nil
Expand Down Expand Up @@ -250,6 +268,7 @@ func insertIssuesStrict(ctx context.Context, conn *sql.Conn, issues []*types.Iss
crystallizes = 1
}

metadataStr := string(issue.Metadata)
_, err = stmt.ExecContext(ctx,
issue.ID, issue.ContentHash, issue.Title, issue.Description, issue.Design,
issue.AcceptanceCriteria, issue.Notes, issue.Status,
Expand All @@ -261,11 +280,16 @@ func insertIssuesStrict(ctx context.Context, conn *sql.Conn, issues []*types.Iss
issue.AwaitType, issue.AwaitID, int64(issue.Timeout), formatJSONStringArray(issue.Waiters),
string(issue.MolType),
issue.EventKind, issue.Actor, issue.Target, issue.Payload,
issue.DueAt, issue.DeferUntil, string(issue.Metadata),
issue.DueAt, issue.DeferUntil, metadataStr,
)
if err != nil {
return fmt.Errorf("failed to insert issue %s: %w", issue.ID, err)
}

// Update metadata index (GH#1589)
if err := updateMetadataIndex(ctx, conn, issue.ID, metadataStr); err != nil {
return fmt.Errorf("failed to index metadata for %s: %w", issue.ID, err)
}
}
return nil
}
121 changes: 121 additions & 0 deletions internal/storage/sqlite/metadata_index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package sqlite

import (
"context"
"encoding/json"
"fmt"
)

// updateMetadataIndex refreshes the index rows for a specific issue.
// It must be called within the transaction of Create/Update issue.
//
// For Phase 1, this indexes all top-level scalar values (string, int, float, bool)
// found in the metadata JSON blob. No schema is required.
//
// GH#1589: Phase 1 of the Schema-Indexed Metadata architecture.
func updateMetadataIndex(ctx context.Context, exec dbExecutor, issueID string, metadataJSON string) error {
// Clear existing index entries for this issue.
// We do a full delete/re-insert for simplicity and correctness.
_, err := exec.ExecContext(ctx, `DELETE FROM issue_metadata_index WHERE issue_id = ?`, issueID)
if err != nil {
return fmt.Errorf("failed to clear metadata index for %s: %w", issueID, err)
}

if metadataJSON == "" || metadataJSON == "{}" {
return nil
}

// Parse metadata JSON
var meta map[string]any
if err := json.Unmarshal([]byte(metadataJSON), &meta); err != nil {
// Invalid JSON is ignored for indexing; validation happens elsewhere
return nil
}

// Index top-level scalars
return indexFlatKeys(ctx, exec, issueID, "", meta)
}

// indexFlatKeys indexes scalar values from a metadata map, supporting one level of nesting
// for namespaced keys (e.g., "jira.story_points").
func indexFlatKeys(ctx context.Context, exec dbExecutor, issueID, prefix string, meta map[string]any) error {
stmt := `INSERT OR REPLACE INTO issue_metadata_index (issue_id, key, value_text, value_int, value_real) VALUES (?, ?, ?, ?, ?)`

for key, val := range meta {
fullKey := key
if prefix != "" {
fullKey = prefix + "." + key
}

switch v := val.(type) {
case string:
if _, err := exec.ExecContext(ctx, stmt, issueID, fullKey, v, nil, nil); err != nil {
return fmt.Errorf("failed to index metadata key %s: %w", fullKey, err)
}
case float64:
// JSON numbers are float64. Check if it's actually an integer.
if v == float64(int64(v)) {
if _, err := exec.ExecContext(ctx, stmt, issueID, fullKey, nil, int64(v), nil); err != nil {
return fmt.Errorf("failed to index metadata key %s: %w", fullKey, err)
}
} else {
if _, err := exec.ExecContext(ctx, stmt, issueID, fullKey, nil, nil, v); err != nil {
return fmt.Errorf("failed to index metadata key %s: %w", fullKey, err)
}
}
case bool:
i := int64(0)
if v {
i = 1
}
if _, err := exec.ExecContext(ctx, stmt, issueID, fullKey, nil, i, nil); err != nil {
return fmt.Errorf("failed to index metadata key %s: %w", fullKey, err)
}
case map[string]any:
// Support one level of nesting for namespaced keys (e.g., "jira.story_points")
if prefix == "" {
if err := indexFlatKeys(ctx, exec, issueID, key, v); err != nil {
return err
}
}
// Skip deeper nesting
default:
// Skip arrays, nulls, and deeper structures
continue
}
}
return nil
}

// RebuildMetadataIndex wipes and rebuilds the entire metadata index from the
// canonical metadata column. Safe to call from bd doctor or bd import.
func (s *SQLiteStorage) RebuildMetadataIndex(ctx context.Context) error {
conn, err := s.db.Conn(ctx)
if err != nil {
return fmt.Errorf("failed to acquire connection: %w", err)
}
defer func() { _ = conn.Close() }()

// Truncate the index
if _, err := conn.ExecContext(ctx, `DELETE FROM issue_metadata_index`); err != nil {
return fmt.Errorf("failed to truncate metadata index: %w", err)
}

// Scan all issues with non-empty metadata
rows, err := conn.QueryContext(ctx, `SELECT id, metadata FROM issues WHERE metadata IS NOT NULL AND metadata != '' AND metadata != '{}'`)
if err != nil {
return fmt.Errorf("failed to query issues for metadata index rebuild: %w", err)
}
defer func() { _ = rows.Close() }()

for rows.Next() {
var id, meta string
if err := rows.Scan(&id, &meta); err != nil {
return fmt.Errorf("failed to scan issue for metadata index: %w", err)
}
if err := updateMetadataIndex(ctx, conn, id, meta); err != nil {
return fmt.Errorf("failed to index metadata for %s: %w", id, err)
}
}
return rows.Err()
}
Loading