Phase 3: screen capture + indexing

cmd/super-ollama/main.go

@@ -16,8 +16,10 @@ import (
 
 	"github.com/ollama/ollama/api"
 	"github.com/ollama/ollama/envconfig"
+	"github.com/ollama/ollama/internal/capture"
 	"github.com/ollama/ollama/internal/config"
 	"github.com/ollama/ollama/internal/engine"
+	"github.com/ollama/ollama/internal/store"
 	"github.com/ollama/ollama/internal/ui"
 	"github.com/ollama/ollama/logutil"
 	"github.com/ollama/ollama/runner"
@@ -50,7 +52,7 @@ func main() {
 		newChatCmd(),
 		newStubCmd("email", "Email assistant (coming in a later phase)"),
 		newStubCmd("todo", "TODO manager (coming in a later phase)"),
-		newStubCmd("snap", "Screenshot capture (coming in a later phase)"),
+		newSnapCmd(),
 		newStubCmd("learn", "Learning-loop re-index (coming in a later phase)"),
 		newConfigShowCmd(),
 	)
@@ -195,6 +197,66 @@ func newChatCmd() *cobra.Command {
 	}
 }
 
+func newSnapCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "snap",
+		Short: "Capture all monitors, OCR, summarise, and index to the data store",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			model, err := resolveModel()
+			if err != nil {
+				return err
+			}
+			cfg, err := config.Load()
+			if err != nil {
+				return err
+			}
+
+			ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
+			defer stop()
+
+			eng, err := engine.New(context.Background())
+			if err != nil {
+				return err
+			}
+			defer eng.Close()
+
+			dbPath, err := store.DefaultPath()
+			if err != nil {
+				return err
+			}
+			st, err := store.Open(dbPath)
+			if err != nil {
+				return err
+			}
+			defer st.Close()
+
+			pf, err := capture.LoadPrivacyFilter()
+			if err != nil {
+				return fmt.Errorf("snap: load privacy filter: %w", err)
+			}
+
+			generateFn := func(ctx context.Context, mdl, prompt string) (string, error) {
+				return eng.Generate(ctx, mdl, prompt, nil)
+			}
+
+			_ = cfg // capture_interval_minutes is used by the daemon; snap is always one-shot
+			cap := capture.New(model, generateFn, st, pf)
+			summaries, err := cap.Snap(ctx)
+			if err != nil {
+				return err
+			}
+			if len(summaries) == 0 {
+				ui.Println("snap: no content captured (privacy filter matched or empty OCR)")
+				return nil
+			}
+			for i, s := range summaries {
+				ui.Printf("Display %d: %s\n", i+1, s)
+			}
+			return nil
+		},
+	}
+}
+
 func newStubCmd(name, short string) *cobra.Command {
 	return &cobra.Command{
 		Use:   name,

go.mod

@@ -24,11 +24,13 @@ require (
 	github.com/d4l3k/go-bfloat16 v0.0.0-20211005043715-690c3bdd05f1
 	github.com/dlclark/regexp2 v1.11.4
 	github.com/emirpasic/gods/v2 v2.0.0-alpha
+	github.com/kbinani/screenshot v0.0.0-20230812210009-b87d31814237
 	github.com/klauspost/compress v1.18.3
 	github.com/mattn/go-runewidth v0.0.16
 	github.com/nlpodyssey/gopickle v0.3.0
 	github.com/pdevine/tensor v0.0.0-20240510204454-f88f4562727c
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
+	github.com/robfig/cron/v3 v3.0.1
 	github.com/tree-sitter/go-tree-sitter v0.25.0
 	github.com/tree-sitter/tree-sitter-cpp v0.23.4
 	github.com/wk8/go-ordered-map/v2 v2.1.8
@@ -54,10 +56,13 @@ require (
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
+	github.com/gen2brain/shm v0.0.0-20230802011745-f2460f5984f7 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/flatbuffers v24.3.25+incompatible // indirect
+	github.com/jezek/xgb v1.1.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/lxn/win v0.0.0-20210218163916-a377121e959e // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-pointer v0.0.1 // indirect

go.sum

@@ -77,6 +77,8 @@ github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/
 github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/gen2brain/shm v0.0.0-20230802011745-f2460f5984f7 h1:VLEKvjGJYAMCXw0/32r9io61tEXnMWDRxMk+peyRVFc=
+github.com/gen2brain/shm v0.0.0-20230802011745-f2460f5984f7/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/cors v1.7.2 h1:oLDHxdg8W/XDoN/8zamqk/Drgt4oVZDvaV0YmvVICQw=
 github.com/gin-contrib/cors v1.7.2/go.mod h1:SUJVARKgQ40dmrzgXEVxj2m7Ig1v1qIboQkPDTQ9t2E=
@@ -140,11 +142,15 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jezek/xgb v1.1.0 h1:wnpxJzP1+rkbGclEkmwpVFQWpuE2PUGNUzP8SbfFobk=
+github.com/jezek/xgb v1.1.0/go.mod h1:nrhwO0FX/enq75I7Y7G8iN1ubpSGZEiA3v9e9GyRFlk=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
+github.com/kbinani/screenshot v0.0.0-20230812210009-b87d31814237 h1:YOp8St+CM/AQ9Vp4XYm4272E77MptJDHkwypQHIRl9Q=
+github.com/kbinani/screenshot v0.0.0-20230812210009-b87d31814237/go.mod h1:e7qQlOY68wOz4b82D7n+DdaptZAi+SHW0+yKiWZzEYE=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
@@ -162,6 +168,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
+github.com/lxn/win v0.0.0-20210218163916-a377121e959e h1:H+t6A/QJMbhCSEH5rAuRxh+CtW96g0Or0Fxa9IKr4uc=
+github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -209,6 +217,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
@@ -349,6 +359,7 @@ golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

internal/capture/capture.go

@@ -0,0 +1,135 @@
+// Package capture implements multi-monitor screenshot capture, OCR, LLM-based
+// summarisation, and indexing to the data store.
+package capture
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"image"
+	"image/png"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/kbinani/screenshot"
+	"github.com/ollama/ollama/internal/store"
+)
+
+// GenerateFunc is the function signature used to call the LLM for summarisation.
+type GenerateFunc func(ctx context.Context, model, prompt string) (string, error)
+
+// Capturer manages screen capture, OCR, summarisation, and indexing.
+type Capturer struct {
+	modelName string
+	generate  GenerateFunc
+	store     *store.Store
+	privacy   *PrivacyFilter
+}
+
+// New creates a Capturer. pf may be nil (no privacy filtering applied).
+func New(modelName string, generate GenerateFunc, st *store.Store, pf *PrivacyFilter) *Capturer {
+	return &Capturer{
+		modelName: modelName,
+		generate:  generate,
+		store:     st,
+		privacy:   pf,
+	}
+}
+
+// CaptureAll captures all active monitors and returns an image per monitor.
+func CaptureAll() ([]image.Image, error) {
+	n := screenshot.NumActiveDisplays()
+	if n == 0 {
+		return nil, fmt.Errorf("capture: no active displays found")
+	}
+	images := make([]image.Image, 0, n)
+	for i := 0; i < n; i++ {
+		bounds := screenshot.GetDisplayBounds(i)
+		img, err := screenshot.CaptureRect(bounds)
+		if err != nil {
+			return nil, fmt.Errorf("capture: display %d: %w", i, err)
+		}
+		images = append(images, img)
+	}
+	return images, nil
+}
+
+// OCR extracts text from img using the tesseract CLI.
+// Returns an empty string (without error) when tesseract is not installed.
+func OCR(img image.Image) (string, error) {
+	tmp, err := os.CreateTemp("", "super-ollama-capture-*.png")
+	if err != nil {
+		return "", fmt.Errorf("ocr: create temp file: %w", err)
+	}
+	tmpName := tmp.Name()
+	defer os.Remove(tmpName)
+
+	if err := png.Encode(tmp, img); err != nil {
+		tmp.Close()
+		return "", fmt.Errorf("ocr: encode png: %w", err)
+	}
+	tmp.Close()
+
+	out, err := exec.Command("tesseract", tmpName, "stdout", "-l", "eng").Output()
+	if err != nil {
+		// If tesseract is not installed, return empty string rather than an error
+		// so captures degrade gracefully on systems without Tesseract.
+		if errors.Is(err, exec.ErrNotFound) {
+			return "", nil
+		}
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return "", fmt.Errorf("ocr: tesseract: %w (stderr: %s)", err, string(exitErr.Stderr))
+		}
+		return "", fmt.Errorf("ocr: tesseract: %w", err)
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+// Summarise calls the LLM to produce a 2-sentence summary of rawText.
+// Returns an empty string without error when rawText is empty.
+func (c *Capturer) Summarise(ctx context.Context, rawText string) (string, error) {
+	if strings.TrimSpace(rawText) == "" {
+		return "", nil
+	}
+	prompt := "Summarise the following screen content in 2 sentences:\n\n" + rawText
+	return c.generate(ctx, c.modelName, prompt)
+}
+
+// IndexCapture saves summary and rawText as a screenshot event in the store.
+func (c *Capturer) IndexCapture(summary, rawText string) error {
+	_, err := c.store.InsertEvent("screenshot", rawText, summary)
+	return err
+}
+
+// Snap performs a one-shot capture pipeline:
+//
+//	screenshots → OCR → privacy check → LLM summarise → index → return summaries
+func (c *Capturer) Snap(ctx context.Context) ([]string, error) {
+	imgs, err := CaptureAll()
+	if err != nil {
+		return nil, err
+	}
+	var summaries []string
+	for i, img := range imgs {
+		rawText, err := OCR(img)
+		if err != nil {
+			return nil, fmt.Errorf("snap: ocr display %d: %w", i, err)
+		}
+		if c.privacy != nil && c.privacy.MatchesText(rawText) {
+			continue
+		}
+		summary, err := c.Summarise(ctx, rawText)
+		if err != nil {
+			return nil, fmt.Errorf("snap: summarise display %d: %w", i, err)
+		}
+		if err := c.IndexCapture(summary, rawText); err != nil {
+			return nil, fmt.Errorf("snap: index display %d: %w", i, err)
+		}
+		if summary != "" {
+			summaries = append(summaries, summary)
+		}
+	}
+	return summaries, nil
+}