Phase 2: embedded store + context injection

cmd/super-ollama/main.go

@@ -8,6 +8,7 @@ import (
 	"log/slog"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strings"
 	"syscall"
 
@@ -18,6 +19,7 @@ import (
 	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/internal/config"
 	"github.com/ollama/ollama/internal/engine"
+	"github.com/ollama/ollama/internal/store"
 	"github.com/ollama/ollama/internal/ui"
 	"github.com/ollama/ollama/logutil"
 	"github.com/ollama/ollama/runner"
@@ -51,7 +53,7 @@ func main() {
 		newStubCmd("email", "Email assistant (coming in a later phase)"),
 		newStubCmd("todo", "TODO manager (coming in a later phase)"),
 		newStubCmd("snap", "Screenshot capture (coming in a later phase)"),
-		newStubCmd("learn", "Learning-loop re-index (coming in a later phase)"),
+		newLearnCmd(),
 		newConfigShowCmd(),
 	)
 
@@ -123,7 +125,21 @@ func newAskCmd() *cobra.Command {
 			}
 			defer eng.Close()
 
-			out, err := eng.Generate(ctx, model, prompt, nil)
+			st, embModel, storeErr := openStore(eng)
+			if storeErr != nil {
+				slog.Warn("store unavailable; running without context injection", "err", storeErr)
+			}
+			if st != nil {
+				defer st.Close()
+			}
+
+			var cs engine.ContextSearcher
+			if st != nil {
+				st.SetEmbedder(eng, embModel)
+				cs = st
+			}
+
+			out, err := eng.GenerateWithContext(ctx, model, prompt, prompt, nil, cs)
 			if err != nil {
 				return err
 			}
@@ -152,6 +168,20 @@ func newChatCmd() *cobra.Command {
 			}
 			defer eng.Close()
 
+			st, embModel, storeErr := openStore(eng)
+			if storeErr != nil {
+				slog.Warn("store unavailable; running without context injection", "err", storeErr)
+			}
+			if st != nil {
+				defer st.Close()
+				st.SetEmbedder(eng, embModel)
+			}
+
+			var cs engine.ContextSearcher
+			if st != nil {
+				cs = st
+			}
+
 			var msgs []api.Message
 			scanner := bufio.NewScanner(os.Stdin)
 			ui.Printf("Model: %s — type /bye to exit\n", model)
@@ -172,7 +202,17 @@ func newChatCmd() *cobra.Command {
 				ch := make(chan string, 64)
 				errCh := make(chan error, 1)
 				go func() {
-					errCh <- eng.StreamChat(ctx, model, msgs, nil, ch)
+					// StreamChat does not support context injection; inject context
+					// into the message list before streaming.
+					msgsToSend := msgs
+					if cs != nil {
+						summaries, serr := cs.SearchSummaries(ctx, line, 10)
+						if serr == nil && len(summaries) > 0 {
+							sysContent := buildContextMessage(summaries)
+							msgsToSend = append([]api.Message{{Role: "system", Content: sysContent}}, msgs...)
+						}
+					}
+					errCh <- eng.StreamChat(ctx, model, msgsToSend, nil, ch)
 				}()
 
 				var reply strings.Builder
@@ -186,6 +226,11 @@ func newChatCmd() *cobra.Command {
 				}
 				ui.Println()
 				msgs = append(msgs, api.Message{Role: "assistant", Content: reply.String()})
+
+				// Best-effort: save the interaction to the store.
+				if cs != nil {
+					_ = cs.SaveInteraction(ctx, line, reply.String())
+				}
 			}
 			if err := scanner.Err(); err != nil {
 				return err
@@ -195,6 +240,18 @@ func newChatCmd() *cobra.Command {
 	}
 }
 
+// buildContextMessage formats retrieved summaries as a system message string.
+func buildContextMessage(summaries []string) string {
+	var b strings.Builder
+	b.WriteString("Relevant context from your personal knowledge base:\n\n")
+	for i, s := range summaries {
+		if s != "" {
+			fmt.Fprintf(&b, "%d. %s\n", i+1, s)
+		}
+	}
+	return b.String()
+}
+
 func newStubCmd(name, short string) *cobra.Command {
 	return &cobra.Command{
 		Use:   name,
@@ -205,6 +262,113 @@ func newStubCmd(name, short string) *cobra.Command {
 	}
 }
 
+// openStore opens the embedded SQLite store using the current config.
+// It returns (nil, "", err) on any failure so callers can degrade gracefully.
+func openStore(_ *engine.Engine) (*store.Store, string, error) {
+	cfg, err := config.Load()
+	if err != nil {
+		return nil, "", fmt.Errorf("load config: %w", err)
+	}
+	dbPath, err := config.ResolvedDBPath(cfg)
+	if err != nil {
+		return nil, "", fmt.Errorf("resolve db path: %w", err)
+	}
+	// Ensure the parent directory exists.
+	dir := filepath.Dir(dbPath)
+	if dir != "" && dir != "." {
+		if mkErr := os.MkdirAll(dir, 0o700); mkErr != nil {
+			slog.Warn("could not create store directory", "dir", dir, "err", mkErr)
+		}
+	}
+	st, err := store.Open(dbPath, nil, "")
+	if err != nil {
+		return nil, "", fmt.Errorf("open store: %w", err)
+	}
+	// Embedder is wired by the caller via st.SetEmbedder after the engine is ready.
+	return st, cfg.EmbeddingModel, nil
+}
+
+// newLearnCmd returns the `learn` command that re-indexes events missing embeddings.
+func newLearnCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "learn",
+		Short: "Re-index events that are missing embeddings",
+		Long: `learn iterates over every event in the local store that has no embedding and
+generates one using the configured embedding model. Run this after restoring a
+backup or first install to bring the semantic search index up to date.`,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
+			defer stop()
+
+			cfg, err := config.Load()
+			if err != nil {
+				return err
+			}
+			dbPath, err := config.ResolvedDBPath(cfg)
+			if err != nil {
+				return err
+			}
+
+			st, err := store.Open(dbPath, nil, "")
+			if err != nil {
+				return fmt.Errorf("open store: %w", err)
+			}
+			defer st.Close()
+
+			missing, err := st.EventsWithoutEmbedding(ctx)
+			if err != nil {
+				return err
+			}
+			if len(missing) == 0 {
+				ui.Println("All events already have embeddings. Nothing to do.")
+				return nil
+			}
+			ui.Printf("Found %d event(s) without embeddings. Starting engine…\n", len(missing))
+
+			eng, err := engine.New(context.Background())
+			if err != nil {
+				return err
+			}
+			defer eng.Close()
+
+			st.SetEmbedder(eng, cfg.EmbeddingModel)
+
+			var indexed, failed int
+			for _, ev := range missing {
+				select {
+				case <-ctx.Done():
+					ui.Printf("Interrupted. Indexed %d/%d events.\n", indexed, len(missing))
+					return ctx.Err()
+				default:
+				}
+
+				text := ev.Summary
+				if text == "" {
+					text = ev.RawText
+				}
+				if text == "" {
+					continue
+				}
+
+				emb, embErr := eng.Embed(ctx, cfg.EmbeddingModel, text)
+				if embErr != nil {
+					slog.Warn("embed failed", "event_id", ev.ID, "err", embErr)
+					failed++
+					continue
+				}
+				if updateErr := st.UpdateEventEmbedding(ctx, ev.ID, emb); updateErr != nil {
+					slog.Warn("update embedding failed", "event_id", ev.ID, "err", updateErr)
+					failed++
+					continue
+				}
+				indexed++
+			}
+			ui.Printf("Done. Indexed %d event(s). Failed: %d.\n", indexed, failed)
+			return nil
+		},
+	}
+}
+
 func newConfigShowCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:   "config",
@@ -218,8 +382,14 @@ func newConfigShowCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			ui.Printf("config file: %s\n", path)
-			ui.Printf("default_model: %s\n", cfg.DefaultModel)
+			dbPath, err := config.ResolvedDBPath(cfg)
+			if err != nil {
+				return err
+			}
+			ui.Printf("config file:     %s\n", path)
+			ui.Printf("default_model:   %s\n", cfg.DefaultModel)
+			ui.Printf("embedding_model: %s\n", cfg.EmbeddingModel)
+			ui.Printf("db_path:         %s\n", dbPath)
 			if strings.TrimSpace(modelFlag) != "" {
 				ui.Printf("active --model override: %s\n", strings.TrimSpace(modelFlag))
 			}

go.mod

@@ -36,6 +36,7 @@ require (
 	golang.org/x/mod v0.30.0
 	golang.org/x/tools v0.38.0
 	gonum.org/v1/gonum v0.15.0
+	modernc.org/sqlite v1.34.5
 )
 
 require (
@@ -53,6 +54,7 @@ require (
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/flatbuffers v24.3.25+incompatible // indirect
@@ -64,15 +66,20 @@ require (
 	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/xtgo/set v1.0.0 // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gorgonia.org/vecf32 v0.9.0 // indirect
 	gorgonia.org/vecf64 v0.9.0 // indirect
+	modernc.org/libc v1.55.3 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.8.0 // indirect
 )
 
 require (

go.sum

@@ -62,6 +62,8 @@ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+
 github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
 github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
 github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/emirpasic/gods/v2 v2.0.0-alpha h1:dwFlh8pBg1VMOXWGipNMRt8v96dKAIvBehtCt6OtunU=
 github.com/emirpasic/gods/v2 v2.0.0-alpha/go.mod h1:W0y4M2dtBB9U5z3YlghmpuUhiaZT2h6yoeE+C1sCp6A=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -134,6 +136,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
+github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -186,6 +190,8 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/nlpodyssey/gopickle v0.3.0 h1:BLUE5gxFLyyNOPzlXxt6GoHEMMxD0qhsE4p0CIQyoLw=
 github.com/nlpodyssey/gopickle v0.3.0/go.mod h1:f070HJ/yR+eLi5WmM1OXJEGaTpuJEUiib19olXgYha0=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
@@ -206,6 +212,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -442,5 +450,29 @@ gorgonia.org/vecf64 v0.9.0 h1:bgZDP5x0OzBF64PjMGC3EvTdOoMEcmfAh1VCUnZFm1A=
 gorgonia.org/vecf64 v0.9.0/go.mod h1:hp7IOWCnRiVQKON73kkC/AUMtEXyf9kGlVrtPQ9ccVA=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
+modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
+modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
+modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
+modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
+modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
+modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
+modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
+modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
+modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
+modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
+modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
+modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g=
+modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
 nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=