diff --git a/.gitignore b/.gitignore index cefaec71..98ff8602 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ vendor/ internal/db/db.go internal/db/models.go internal/db/*.sql.go +internal/db/copyfrom.go # templ generated files — *.templ files are source, *_templ.go are artifacts # produced by `make templ` (or `templ generate`). Keep generated Go out of diff --git a/Makefile b/Makefile index b75484b4..be3ae85c 100644 --- a/Makefile +++ b/Makefile @@ -247,7 +247,7 @@ sqlc-install: fi sqlc: sqlc-install - @rm -f internal/db/*.sql.go internal/db/models.go internal/db/db.go + @rm -f internal/db/*.sql.go internal/db/models.go internal/db/db.go internal/db/copyfrom.go sqlc generate @$(MAKE) sqlc-tag @@ -256,7 +256,7 @@ sqlc: sqlc-install # `emit_build_tag` option (added in 1.27+), so we patch the artifacts in a # follow-up step. CI runs the same target. sqlc-tag: - @for f in internal/db/db.go internal/db/models.go internal/db/*.sql.go; do \ + @for f in internal/db/db.go internal/db/models.go internal/db/copyfrom.go internal/db/*.sql.go; do \ if [ -f "$$f" ] && ! head -1 "$$f" | grep -q '^//go:build'; then \ printf '//go:build !lite\n\n' | cat - "$$f" > "$$f.tmp" && mv "$$f.tmp" "$$f"; \ fi; \ diff --git a/docs/csv-import.md b/docs/csv-import.md new file mode 100644 index 00000000..39f89aeb --- /dev/null +++ b/docs/csv-import.md @@ -0,0 +1,123 @@ +# CSV import + +Breadbox imports transactions from CSV/TSV exports. There are two paths: + +- **v2 — drop anywhere (recommended).** Drag a CSV onto any page. A global overlay + appears; on drop the import flow analyzes the file, detects which account it + belongs to, dedupes against what's already there, shows a preview of exactly + what will change, and applies only on confirm. +- **v1 — wizard page.** The legacy 4-step wizard at + `/admin/connections/import-csv` (upload → map → preview → import). Still works; + kept for now. v2 supersedes it. + +This doc covers v2. v1 internals live in `internal/admin/csv_import.go`. + +## The flow + +``` +drop CSV (any page) + → POST /-/csv/v2/sessions analyze: parse, detect template/mapping/ + currency, apply saved profile, rank accounts + → [account step, if not auto-chosen] + POST /-/csv/v2/sessions/{id}/resolve bind/create account → classify rows + → preview/edit + GET /-/csv/v2/sessions/{id}/rows paginated rows + summary + PATCH /-/csv/v2/sessions/{id}/rows/{rowId} inline edit a row + POST /-/csv/v2/sessions/{id}/bulk include/exclude/set-category/remap + → POST /-/csv/v2/sessions/{id}/apply upsert the included set, save the profile +``` + +All endpoints are admin-only (session-cookie auth), so they carry no OpenAPI +surface. The flow UI is `components.ImportModal` + `$store.csvImport` (drop +overlay + open/close) + the `csvImport` Alpine factory +(`static/js/admin/components/csv_import_v2.js`). See the live variant matrix at +`/design/c/csv-import`. + +## Staging (durable, so preview == apply) + +A drop creates a `csv_import_sessions` row holding the original bytes +(`raw_blob`), detected mapping, resolved account, and (after apply) a result +snapshot. Each parsed row is a `csv_import_rows` row carrying its parsed values, +classification, dedup hashes, and the user's include/exclude intent. Classifying +or remapping replaces the row set wholesale (delete + `:copyfrom`), so up to 50k +rows insert fast. Abandoned sessions are swept hourly past their 24h TTL +(`SweepExpiredImportSessions`, wired in `serve.go`). + +Service layer: `internal/service/csv_import_v2.go` (lifecycle + apply), +`csv_classify.go` (dedup classification), `csv_account_match.go` (account +detection), `csv_profiles.go` (profile CRUD). + +## Account detection + +`MatchCSVAccounts` ranks existing accounts with a deterministic, explainable +weighted score (every point carries a reason string): + +| Signal | Points | +|---|---| +| saved profile's default account | auto-preselect | +| mask / last-4 matches the account | +40 | +| transaction overlap (rows already present) | up to +40 | +| institution name appears in the account name | +15 | +| filename mentions the account | +5 | + +The top account is **pre-selected** only when it scores ≥70 *and* leads the +runner-up by ≥20 (or a profile matched). Otherwise the user picks from the ranked +list or creates a new account inline. Confidence buckets: high ≥70, medium 40–69, +low <40. + +## Deduplication + +Per incoming row, against the **resolved account's** live transactions (any +provider — CSV, Plaid, Teller): + +- **exact_dup** — the row's stable `provider_transaction_id` + (`SHA-256(accountID|date|amount|desc)`) already exists, or its account- + independent `content_hash` (`SHA-256(date|amount|desc)`) matches an existing + row. Default: excluded. +- **probable_dup** — same amount, date within ±1 day, and a similar name (scored + by the shared `internal/textmatch` similarity used by the sync matcher). + Default: excluded. +- **new** — no match. Default: included. +- **error** — unparseable row; surfaced for inline fixing. Excluded. + +The classifier loads candidates with one indexed query +(`transactions_dedup_lookup` on `(account_id, date, amount) WHERE deleted_at IS +NULL`). Apply upserts via `UpsertTransactionV2` (returns `(xmax = 0) AS inserted` +for reliable new-vs-updated counts) and is **idempotent** — re-dropping the same +file into the same account upserts to no-ops. A genuine same-key duplicate the +user force-imports gets an occurrence disambiguator so it inserts as a distinct +row. + +Why re-dropping a later export "just adds the new ones": the overlapping rows hit +exact_dup (and the saved profile auto-resolves the account), so only the genuinely +new rows are classified `new` and imported. + +## Profiles + +A profile is a saved import recipe keyed by a **header fingerprint** (a hash of +the file's normalized header row). On a successful apply the profile is upserted: +created the first time a given header layout is imported, then bumped +(`times_used` / `last_used_at`) and updated to remember the resolved account. On a +future drop whose fingerprint matches, the flow auto-applies the saved +mapping/date-format/sign/currency and pre-selects the remembered account — +collapsing straight to a deduped preview. + +Manage profiles via `GET/PATCH/DELETE /-/csv/v2/profiles[/{id}]` +(`internal/service/csv_profiles.go`). The user's rename survives future imports +(the upsert deliberately doesn't overwrite `name`). + +## Currency + +Detected from the file (currency symbols in the amount column), defaulting to the +target account's currency, confirmable in the flow. Amounts are stored as-is with +their `iso_currency_code`; nothing is converted (honoring the never-sum-across- +currency invariant). + +## Schema + +Additive migrations only (shared dev DB): + +- `transactions.content_hash` + partial index `transactions_dedup_lookup`. +- `csv_import_profiles` (saved recipes; one per user+fingerprint). +- `csv_import_sessions` + `csv_import_rows` (durable staging). +- `csv_import_rows.category_id` (user-chosen Breadbox category applied at import). diff --git a/internal/admin/csv_import_v2.go b/internal/admin/csv_import_v2.go new file mode 100644 index 00000000..6e5a50e4 --- /dev/null +++ b/internal/admin/csv_import_v2.go @@ -0,0 +1,284 @@ +//go:build !headless && !lite + +package admin + +import ( + "errors" + "io" + "net/http" + "strconv" + + "breadbox/internal/app" + "breadbox/internal/pgconv" + "breadbox/internal/service" + + "github.com/alexedwards/scs/v2" + "github.com/go-chi/chi/v5" +) + +// maxCSVV2UploadSize caps the drag-drop upload. Larger than the legacy wizard's +// 10MB because v2 persists the raw bytes and supports up to 50k rows. +const maxCSVV2UploadSize = 30 << 20 // 30MB + +// CSVV2CreateSessionHandler serves POST /-/csv/v2/sessions. +// Accepts a multipart file (field "file") + optional "user_id"; analyzes the +// file, runs account detection, and returns the import analysis. +func CSVV2CreateSessionHandler(a *app.App, sm *scs.SessionManager, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxCSVV2UploadSize) + if err := r.ParseMultipartForm(maxCSVV2UploadSize); err != nil { + writeError(w, http.StatusBadRequest, "FILE_TOO_LARGE", "File too large (max 30MB)") + return + } + file, header, err := r.FormFile("file") + if err != nil { + writeError(w, http.StatusBadRequest, "NO_FILE", "No file uploaded") + return + } + defer file.Close() + raw, err := io.ReadAll(file) + if err != nil { + writeError(w, http.StatusBadRequest, "READ_FAILED", "Failed to read file") + return + } + + userID, err := defaultImportUser(r, svc, r.FormValue("user_id")) + if err != nil { + writeError(w, http.StatusUnprocessableEntity, "VALIDATION_ERROR", err.Error()) + return + } + + filename := r.FormValue("filename") + if filename == "" && header != nil { + filename = header.Filename + } + + analysis, err := svc.CreateImportSession(r.Context(), service.CreateImportSessionParams{ + UserID: userID, + Filename: filename, + Data: raw, + }) + if err != nil { + a.Logger.Debug("csv v2 analyze failed", "error", err) + writeError(w, http.StatusUnprocessableEntity, "PARSE_FAILED", humanizeCSVError(err)) + return + } + writeJSON(w, http.StatusOK, analysis) + } +} + +// CSVV2ResolveHandler serves POST /-/csv/v2/sessions/{id}/resolve. +func CSVV2ResolveHandler(a *app.App, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + AccountID string `json:"account_id"` + CreateNew bool `json:"create_new"` + NewName string `json:"new_name"` + NewType string `json:"new_type"` + NewSubtype string `json:"new_subtype"` + NewCurrency string `json:"new_currency"` + } + if !decodeJSON(w, r, &req) { + return + } + if !req.CreateNew && req.AccountID == "" { + writeError(w, http.StatusUnprocessableEntity, "VALIDATION_ERROR", "account_id or create_new is required") + return + } + sess, err := svc.ResolveImportAccount(r.Context(), chi.URLParam(r, "id"), service.ResolveImportAccountParams{ + AccountID: req.AccountID, + CreateNew: req.CreateNew, + NewName: req.NewName, + NewType: req.NewType, + NewSubtype: req.NewSubtype, + NewCurrency: req.NewCurrency, + }) + if err != nil { + writeImportErr(w, a, "resolve account", err) + return + } + writeJSON(w, http.StatusOK, sess) + } +} + +// CSVV2RowsHandler serves GET /-/csv/v2/sessions/{id}/rows. +func CSVV2RowsHandler(a *app.App, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + pageSize, _ := strconv.Atoi(r.URL.Query().Get("page_size")) + rows, summary, err := svc.ListImportRows(r.Context(), id, r.URL.Query().Get("status"), page, pageSize) + if err != nil { + writeImportErr(w, a, "list rows", err) + return + } + sess, err := svc.GetImportSession(r.Context(), id) + if err != nil { + writeImportErr(w, a, "get session", err) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "session": sess, + "rows": rows, + "summary": summary, + }) + } +} + +// CSVV2EditRowHandler serves PATCH /-/csv/v2/sessions/{id}/rows/{rowId}. +func CSVV2EditRowHandler(a *app.App, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + Date string `json:"date"` + Amount string `json:"amount"` + Desc string `json:"description"` + Merchant string `json:"merchant"` + Include *bool `json:"include"` + } + if !decodeJSON(w, r, &req) { + return + } + row, err := svc.EditImportRow(r.Context(), chi.URLParam(r, "id"), chi.URLParam(r, "rowId"), service.EditImportRowParams{ + Date: req.Date, + Amount: req.Amount, + Desc: req.Desc, + Merchant: req.Merchant, + Include: req.Include, + }) + if err != nil { + writeImportErr(w, a, "edit row", err) + return + } + writeJSON(w, http.StatusOK, row) + } +} + +// CSVV2BulkHandler serves POST /-/csv/v2/sessions/{id}/bulk. +func CSVV2BulkHandler(a *app.App, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + Op string `json:"op"` + Classification string `json:"classification"` + CategoryID string `json:"category_id"` + ColumnMapping map[string]int `json:"column_mapping"` + DateFormat string `json:"date_format"` + PositiveIsDebit bool `json:"positive_is_debit"` + HasDebitCredit bool `json:"has_debit_credit"` + } + if !decodeJSON(w, r, &req) { + return + } + if err := svc.BulkImportOp(r.Context(), chi.URLParam(r, "id"), service.ImportBulkOp{ + Op: req.Op, + Classification: req.Classification, + CategoryID: req.CategoryID, + ColumnMapping: req.ColumnMapping, + DateFormat: req.DateFormat, + PositiveIsDebit: req.PositiveIsDebit, + HasDebitCredit: req.HasDebitCredit, + }); err != nil { + writeImportErr(w, a, "bulk op", err) + return + } + // Return the refreshed summary so the client can update counts. + _, summary, err := svc.ListImportRows(r.Context(), chi.URLParam(r, "id"), "", 1, 1) + if err != nil { + writeImportErr(w, a, "summary", err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"summary": summary}) + } +} + +// CSVV2ApplyHandler serves POST /-/csv/v2/sessions/{id}/apply. +func CSVV2ApplyHandler(a *app.App, sm *scs.SessionManager, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + result, err := svc.ApplyImportSession(r.Context(), chi.URLParam(r, "id"), ActorFromSession(sm, r)) + if err != nil { + writeImportErr(w, a, "apply", err) + return + } + writeJSON(w, http.StatusOK, result) + } +} + +// CSVV2ProfilesListHandler serves GET /-/csv/v2/profiles. +func CSVV2ProfilesListHandler(a *app.App, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + profiles, err := svc.ListCSVProfiles(r.Context()) + if err != nil { + a.Logger.Debug("csv v2 list profiles failed", "error", err) + writeError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "Failed to list profiles") + return + } + writeJSON(w, http.StatusOK, profiles) + } +} + +// CSVV2ProfileRenameHandler serves PATCH /-/csv/v2/profiles/{id}. +func CSVV2ProfileRenameHandler(a *app.App, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + Name string `json:"name"` + } + if !decodeJSON(w, r, &req) { + return + } + if req.Name == "" { + writeError(w, http.StatusUnprocessableEntity, "VALIDATION_ERROR", "name is required") + return + } + prof, err := svc.RenameCSVProfile(r.Context(), chi.URLParam(r, "id"), req.Name) + if err != nil { + writeProfileErr(w, a, "rename", err) + return + } + writeJSON(w, http.StatusOK, prof) + } +} + +// CSVV2ProfileDeleteHandler serves DELETE /-/csv/v2/profiles/{id}. +func CSVV2ProfileDeleteHandler(a *app.App, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if err := svc.DeleteCSVProfile(r.Context(), chi.URLParam(r, "id")); err != nil { + writeProfileErr(w, a, "delete", err) + return + } + writeOK(w) + } +} + +func writeProfileErr(w http.ResponseWriter, a *app.App, op string, err error) { + if errors.Is(err, service.ErrProfileNotFound) { + writeError(w, http.StatusNotFound, "NOT_FOUND", "Import profile not found") + return + } + a.Logger.Debug("csv v2 profile "+op+" failed", "error", err) + writeError(w, http.StatusUnprocessableEntity, "VALIDATION_ERROR", err.Error()) +} + +// defaultImportUser resolves the household user for a new import: the explicit +// value if given, otherwise the sole household member. +func defaultImportUser(r *http.Request, svc *service.Service, explicit string) (string, error) { + if explicit != "" { + return explicit, nil + } + users, err := svc.Queries.ListUsers(r.Context()) + if err != nil { + return "", errors.New("failed to list household members") + } + if len(users) == 0 { + return "", errors.New("create a household member before importing") + } + return pgconv.FormatUUID(users[0].ID), nil +} + +// writeImportErr maps a service error to an HTTP response. +func writeImportErr(w http.ResponseWriter, a *app.App, op string, err error) { + if errors.Is(err, service.ErrImportSessionNotFound) { + writeError(w, http.StatusNotFound, "NOT_FOUND", "Import session not found") + return + } + a.Logger.Debug("csv v2 "+op+" failed", "error", err) + writeError(w, http.StatusUnprocessableEntity, "VALIDATION_ERROR", err.Error()) +} diff --git a/internal/admin/csv_import_v2_integration_test.go b/internal/admin/csv_import_v2_integration_test.go new file mode 100644 index 00000000..36538f2c --- /dev/null +++ b/internal/admin/csv_import_v2_integration_test.go @@ -0,0 +1,103 @@ +//go:build integration && !headless && !lite + +package admin + +import ( + "bytes" + "encoding/json" + "log/slog" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + + "breadbox/internal/app" + "breadbox/internal/service" + "breadbox/internal/testutil" + + "github.com/alexedwards/scs/v2" + "github.com/go-chi/chi/v5" +) + +// TestCSVV2Flow_HTTP drives the v2 endpoints through a real chi router: +// analyze (multipart) → resolve (create new account) → list rows → apply. +func TestCSVV2Flow_HTTP(t *testing.T) { + pool, q := testutil.ServicePool(t) + svc := service.New(q, pool, nil, slog.Default()) + a := &app.App{DB: pool, Queries: q, Logger: slog.Default()} + sm := scs.New() // defaults to an in-memory store + + testutil.MustCreateUser(t, q, "Alice") + + r := chi.NewRouter() + r.Use(sm.LoadAndSave) + r.Post("/-/csv/v2/sessions", CSVV2CreateSessionHandler(a, sm, svc)) + r.Post("/-/csv/v2/sessions/{id}/resolve", CSVV2ResolveHandler(a, svc)) + r.Get("/-/csv/v2/sessions/{id}/rows", CSVV2RowsHandler(a, svc)) + r.Post("/-/csv/v2/sessions/{id}/apply", CSVV2ApplyHandler(a, sm, svc)) + srv := httptest.NewServer(r) + defer srv.Close() + + // 1. Analyze via multipart upload. + var body bytes.Buffer + mw := multipart.NewWriter(&body) + fw, _ := mw.CreateFormFile("file", "chase.csv") + fw.Write([]byte("Date,Amount,Description\n2026-10-01,10.00,COFFEE\n2026-10-02,20.00,LUNCH\n")) + mw.Close() + + resp, err := http.Post(srv.URL+"/-/csv/v2/sessions", mw.FormDataContentType(), &body) + if err != nil { + t.Fatalf("analyze post: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("analyze status %d", resp.StatusCode) + } + var analysis service.ImportAnalysis + json.NewDecoder(resp.Body).Decode(&analysis) + resp.Body.Close() + shortID := analysis.Session.ShortID + if shortID == "" { + t.Fatal("no session short id returned") + } + + // 2. Resolve by creating a new account. + post := func(path string, payload any) *http.Response { + b, _ := json.Marshal(payload) + rsp, err := http.Post(srv.URL+path, "application/json", bytes.NewReader(b)) + if err != nil { + t.Fatalf("post %s: %v", path, err) + } + return rsp + } + rr := post("/-/csv/v2/sessions/"+shortID+"/resolve", map[string]any{"create_new": true, "new_name": "Chase"}) + if rr.StatusCode != http.StatusOK { + t.Fatalf("resolve status %d", rr.StatusCode) + } + rr.Body.Close() + + // 3. List rows. + rowsResp, err := http.Get(srv.URL + "/-/csv/v2/sessions/" + shortID + "/rows") + if err != nil { + t.Fatalf("rows get: %v", err) + } + var rowsBody struct { + Summary service.ImportSummary `json:"summary"` + } + json.NewDecoder(rowsResp.Body).Decode(&rowsBody) + rowsResp.Body.Close() + if rowsBody.Summary.IncludedCount != 2 { + t.Fatalf("included = %d, want 2", rowsBody.Summary.IncludedCount) + } + + // 4. Apply. + ar := post("/-/csv/v2/sessions/"+shortID+"/apply", map[string]any{}) + if ar.StatusCode != http.StatusOK { + t.Fatalf("apply status %d", ar.StatusCode) + } + var result service.CSVImportResult + json.NewDecoder(ar.Body).Decode(&result) + ar.Body.Close() + if result.NewCount != 2 { + t.Fatalf("apply new count = %d, want 2", result.NewCount) + } +} diff --git a/internal/admin/router.go b/internal/admin/router.go index d50169b8..c6abc79a 100644 --- a/internal/admin/router.go +++ b/internal/admin/router.go @@ -578,6 +578,17 @@ func NewAdminRouter(a *app.App, sm *scs.SessionManager, tr *TemplateRenderer, sv r.Post("/csv/preview", CSVPreviewHandler(a, sm)) r.Post("/csv/import", CSVImportHandler(a, sm, svc)) + // CSV import v2 — drag-anywhere flow (analyze → resolve → preview → apply). + r.Post("/csv/v2/sessions", CSVV2CreateSessionHandler(a, sm, svc)) + r.Post("/csv/v2/sessions/{id}/resolve", CSVV2ResolveHandler(a, svc)) + r.Get("/csv/v2/sessions/{id}/rows", CSVV2RowsHandler(a, svc)) + r.Patch("/csv/v2/sessions/{id}/rows/{rowId}", CSVV2EditRowHandler(a, svc)) + r.Post("/csv/v2/sessions/{id}/bulk", CSVV2BulkHandler(a, svc)) + r.Post("/csv/v2/sessions/{id}/apply", CSVV2ApplyHandler(a, sm, svc)) + r.Get("/csv/v2/profiles", CSVV2ProfilesListHandler(a, svc)) + r.Patch("/csv/v2/profiles/{id}", CSVV2ProfileRenameHandler(a, svc)) + r.Delete("/csv/v2/profiles/{id}", CSVV2ProfileDeleteHandler(a, svc)) + // API key + OAuth client revoke/delete — admin only. r.Delete("/api-keys/{id}", RevokeAPIKeyHandler(svc)) r.Delete("/oauth-clients/{id}", RevokeOAuthClientHandler(svc)) diff --git a/internal/admin/templates.go b/internal/admin/templates.go index 494f6da5..adef0cae 100644 --- a/internal/admin/templates.go +++ b/internal/admin/templates.go @@ -93,6 +93,9 @@ var componentRegistry = map[string]componentAdapter{ "ThemeToggle": func(_ any) (templ.Component, error) { return components.ThemeToggle(), nil }, + "ImportModal": func(_ any) (templ.Component, error) { + return components.ImportModal(), nil + }, "TopbarUserMenu": func(data any) (templ.Component, error) { m, ok := data.(map[string]any) if !ok { diff --git a/internal/cli/serve.go b/internal/cli/serve.go index c0ac04ee..3d957bd1 100644 --- a/internal/cli/serve.go +++ b/internal/cli/serve.go @@ -219,6 +219,18 @@ func runServe(_ context.Context, version string, noDashboardFlag bool) error { logger.Warn("pg_dump not found — backup service disabled") } + // Hourly sweep of abandoned/expired CSV import sessions (drops their staged + // rows + raw file blob). Best-effort; never fatal. + if err := scheduler.AddFunc("23 * * * *", func() { + if n, serr := a.Service.SweepExpiredImportSessions(context.Background()); serr != nil { + logger.Warn("csv import session sweep failed", "error", serr) + } else if n > 0 { + logger.Info("csv import session sweep", "deleted", n) + } + }); err != nil { + logger.Error("failed to add csv import sweep cron", "error", err) + } + go scheduler.RunStartupSync(ctx, cfg.SyncIntervalMinutes) router := api.NewRouter(a, version) diff --git a/internal/db/migrations/20260607043318_transactions_content_hash.sql b/internal/db/migrations/20260607043318_transactions_content_hash.sql new file mode 100644 index 00000000..54f8d889 --- /dev/null +++ b/internal/db/migrations/20260607043318_transactions_content_hash.sql @@ -0,0 +1,25 @@ +-- +goose Up + +-- content_hash: an account-INDEPENDENT fingerprint of a transaction's core +-- content (date | amount | normalized description). The existing dedup key +-- `provider_transaction_id` bakes in the account id, so it only catches exact +-- re-imports into the SAME account. The CSV import v2 classifier needs to detect +-- duplicates against ALL transactions already in a resolved account — including +-- Plaid/Teller rows — so it compares on this account-independent hash plus a +-- date+amount+name fuzzy pass. +-- +-- Nullable + additive: CSV v2 writes it going forward; legacy provider rows stay +-- NULL (the classifier falls back to date+amount+name for those). Safe on the +-- shared dev DB alongside running servers. +ALTER TABLE transactions ADD COLUMN IF NOT EXISTS content_hash TEXT; + +-- Fast candidate lookup for classification: "all live transactions in this +-- account near this date with this amount". Partial on deleted_at because the +-- classifier only ever compares against live rows. +CREATE INDEX IF NOT EXISTS transactions_dedup_lookup + ON transactions (account_id, date, amount) + WHERE deleted_at IS NULL; + +-- +goose Down +DROP INDEX IF EXISTS transactions_dedup_lookup; +ALTER TABLE transactions DROP COLUMN IF EXISTS content_hash; diff --git a/internal/db/migrations/20260607043418_csv_import_profiles.sql b/internal/db/migrations/20260607043418_csv_import_profiles.sql new file mode 100644 index 00000000..30a95a97 --- /dev/null +++ b/internal/db/migrations/20260607043418_csv_import_profiles.sql @@ -0,0 +1,46 @@ +-- +goose Up + +-- csv_import_profiles: saved CSV "recipes" so repeat imports from the same +-- source are one click. Keyed by a header fingerprint (a stable hash of the +-- file's normalized header row). On a future import whose fingerprint matches, +-- the flow auto-applies the saved column mapping / date format / sign / currency +-- and pre-selects `default_account_id`, collapsing straight to a deduped preview. +-- +-- A profile is upserted on every successful apply: created the first time a given +-- header layout is imported, then bumped (times_used / last_used_at) and updated +-- to remember the most recently resolved account. One profile per +-- (user, header_fingerprint). +-- +-- Additive-only: one new table. Safe on the shared dev DB. +CREATE TABLE csv_import_profiles ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + short_id TEXT NOT NULL UNIQUE, + user_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE, + name TEXT NOT NULL, + header_fingerprint TEXT NOT NULL, + headers JSONB NOT NULL DEFAULT '[]'::jsonb, + detected_template TEXT NULL, + column_mapping JSONB NOT NULL DEFAULT '{}'::jsonb, + date_format TEXT NOT NULL DEFAULT '', + delimiter TEXT NOT NULL DEFAULT ',', + positive_is_debit BOOLEAN NOT NULL DEFAULT FALSE, + has_debit_credit BOOLEAN NOT NULL DEFAULT FALSE, + iso_currency_code TEXT NOT NULL DEFAULT 'USD', + default_account_id UUID NULL REFERENCES accounts (id) ON DELETE SET NULL, + institution_hint TEXT NULL, + mask_hint TEXT NULL, + times_used INTEGER NOT NULL DEFAULT 0, + last_used_at TIMESTAMPTZ NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (user_id, header_fingerprint) +); + +CREATE TRIGGER set_short_id_csv_import_profiles + BEFORE INSERT ON csv_import_profiles + FOR EACH ROW EXECUTE FUNCTION set_short_id(); + +CREATE INDEX csv_import_profiles_user_idx ON csv_import_profiles (user_id); + +-- +goose Down +DROP TABLE IF EXISTS csv_import_profiles; diff --git a/internal/db/migrations/20260607043518_csv_import_sessions.sql b/internal/db/migrations/20260607043518_csv_import_sessions.sql new file mode 100644 index 00000000..07ddf061 --- /dev/null +++ b/internal/db/migrations/20260607043518_csv_import_sessions.sql @@ -0,0 +1,81 @@ +-- +goose Up + +-- csv_import_sessions + csv_import_rows: durable staging for the CSV import v2 +-- flow. The current wizard holds parsed rows in the HTTP session, which can't +-- satisfy "preview exactly what will happen, then apply that exact set" — the +-- classification depends on live DB state, a 50k-row classified set with edits +-- can't live in a cookie, and apply needs to be atomic + idempotent. +-- +-- A session is created on drop, classified once, persisted here, edited in place, +-- and applied in a single transaction. Rows carry their parsed values + a +-- per-row classification (new / exact_dup / probable_dup / conflict / error / +-- needs_account) and the user's include/exclude intent. +-- +-- Additive-only: two new tables. Safe on the shared dev DB. +CREATE TABLE csv_import_sessions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + short_id TEXT NOT NULL UNIQUE, + user_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE, + -- analyzing | awaiting_account | previewed | applying | applied | failed | abandoned + status TEXT NOT NULL DEFAULT 'analyzing', + filename TEXT NOT NULL DEFAULT '', + file_sha256 TEXT NOT NULL DEFAULT '', + delimiter TEXT NOT NULL DEFAULT ',', + headers JSONB NOT NULL DEFAULT '[]'::jsonb, + raw_blob BYTEA NULL, -- original bytes; nulled after apply + row_count INTEGER NOT NULL DEFAULT 0, + resolved_account_id UUID NULL REFERENCES accounts (id) ON DELETE SET NULL, + resolved_connection_id UUID NULL REFERENCES bank_connections (id) ON DELETE SET NULL, + detected_template TEXT NULL, + column_mapping JSONB NULL, + date_format TEXT NOT NULL DEFAULT '', + positive_is_debit BOOLEAN NOT NULL DEFAULT FALSE, + has_debit_credit BOOLEAN NOT NULL DEFAULT FALSE, + iso_currency_code TEXT NOT NULL DEFAULT 'USD', + profile_id UUID NULL REFERENCES csv_import_profiles (id) ON DELETE SET NULL, + result JSONB NULL, -- snapshot of CSVImportResult after apply + sync_log_id UUID NULL REFERENCES sync_logs (id) ON DELETE SET NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ NULL -- TTL for abandoned-session cleanup +); + +CREATE TRIGGER set_short_id_csv_import_sessions + BEFORE INSERT ON csv_import_sessions + FOR EACH ROW EXECUTE FUNCTION set_short_id(); + +CREATE INDEX csv_import_sessions_user_idx ON csv_import_sessions (user_id); +-- Sweeper lookup: abandoned/expired sessions that were never applied. +CREATE INDEX csv_import_sessions_expires_idx ON csv_import_sessions (expires_at) + WHERE status <> 'applied'; + +CREATE TABLE csv_import_rows ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + session_id UUID NOT NULL REFERENCES csv_import_sessions (id) ON DELETE CASCADE, + row_index INTEGER NOT NULL, + raw JSONB NOT NULL DEFAULT '[]'::jsonb, + parsed_date DATE NULL, + parsed_amount NUMERIC(12,2) NULL, + parsed_desc TEXT NULL, + parsed_merchant TEXT NULL, + parsed_category TEXT NULL, + -- new | exact_dup | probable_dup | conflict | error | needs_account + classification TEXT NOT NULL DEFAULT 'new', + match_txn_id UUID NULL REFERENCES transactions (id) ON DELETE SET NULL, + match_score INTEGER NULL, + match_reason TEXT NULL, + parse_error TEXT NULL, + content_hash TEXT NULL, + provider_txn_id TEXT NULL, + include BOOLEAN NOT NULL DEFAULT TRUE, + user_edited BOOLEAN NOT NULL DEFAULT FALSE, + UNIQUE (session_id, row_index) +); + +-- Summary counts (GROUP BY classification) + filtered pagination per session. +CREATE INDEX csv_import_rows_session_class_idx + ON csv_import_rows (session_id, classification); + +-- +goose Down +DROP TABLE IF EXISTS csv_import_rows; +DROP TABLE IF EXISTS csv_import_sessions; diff --git a/internal/db/migrations/20260607043618_csv_import_rows_category.sql b/internal/db/migrations/20260607043618_csv_import_rows_category.sql new file mode 100644 index 00000000..bd4dcd57 --- /dev/null +++ b/internal/db/migrations/20260607043618_csv_import_rows_category.sql @@ -0,0 +1,11 @@ +-- +goose Up + +-- category_id: the Breadbox category a user assigns to a staged row (via the +-- preview's bulk "set category" or inline edit). NULL = let transaction rules +-- categorize at/after import, matching the legacy CSV behavior. Distinct from +-- parsed_category, which is the raw provider category string from the file. +ALTER TABLE csv_import_rows + ADD COLUMN IF NOT EXISTS category_id UUID NULL REFERENCES categories (id) ON DELETE SET NULL; + +-- +goose Down +ALTER TABLE csv_import_rows DROP COLUMN IF EXISTS category_id; diff --git a/internal/db/queries/csv_import_profiles.sql b/internal/db/queries/csv_import_profiles.sql new file mode 100644 index 00000000..27c286c9 --- /dev/null +++ b/internal/db/queries/csv_import_profiles.sql @@ -0,0 +1,59 @@ +-- name: UpsertCSVImportProfile :one +-- Create a profile for this (user, header layout), or update the existing one. +-- `name` is intentionally NOT in the conflict SET list so a user's rename +-- survives future imports. times_used is bumped and last_used_at refreshed. +INSERT INTO csv_import_profiles ( + user_id, name, header_fingerprint, headers, detected_template, + column_mapping, date_format, delimiter, positive_is_debit, has_debit_credit, + iso_currency_code, default_account_id, institution_hint, mask_hint, + times_used, last_used_at +) VALUES ( + $1, $2, $3, $4, $5, + $6, $7, $8, $9, $10, + $11, $12, $13, $14, + 1, NOW() +) +ON CONFLICT (user_id, header_fingerprint) DO UPDATE SET + headers = EXCLUDED.headers, + detected_template = EXCLUDED.detected_template, + column_mapping = EXCLUDED.column_mapping, + date_format = EXCLUDED.date_format, + delimiter = EXCLUDED.delimiter, + positive_is_debit = EXCLUDED.positive_is_debit, + has_debit_credit = EXCLUDED.has_debit_credit, + iso_currency_code = EXCLUDED.iso_currency_code, + default_account_id = EXCLUDED.default_account_id, + institution_hint = EXCLUDED.institution_hint, + mask_hint = EXCLUDED.mask_hint, + times_used = csv_import_profiles.times_used + 1, + last_used_at = NOW(), + updated_at = NOW() +RETURNING *; + +-- name: GetCSVImportProfileByFingerprint :one +SELECT * FROM csv_import_profiles +WHERE user_id = $1 AND header_fingerprint = $2; + +-- name: GetCSVImportProfile :one +SELECT * FROM csv_import_profiles WHERE id = $1; + +-- name: GetCSVImportProfileByShortID :one +SELECT * FROM csv_import_profiles WHERE short_id = $1; + +-- name: ListCSVImportProfilesByUser :many +SELECT * FROM csv_import_profiles +WHERE user_id = $1 +ORDER BY last_used_at DESC NULLS LAST, created_at DESC; + +-- name: ListCSVImportProfiles :many +SELECT * FROM csv_import_profiles +ORDER BY last_used_at DESC NULLS LAST, created_at DESC; + +-- name: RenameCSVImportProfile :one +UPDATE csv_import_profiles +SET name = $2, updated_at = NOW() +WHERE id = $1 +RETURNING *; + +-- name: DeleteCSVImportProfile :exec +DELETE FROM csv_import_profiles WHERE id = $1; diff --git a/internal/db/queries/csv_import_rows.sql b/internal/db/queries/csv_import_rows.sql new file mode 100644 index 00000000..04f15ee4 --- /dev/null +++ b/internal/db/queries/csv_import_rows.sql @@ -0,0 +1,82 @@ +-- name: CreateCSVImportRows :copyfrom +-- Bulk-insert fully-parsed + classified rows (up to 50k). Classification is +-- computed in Go, so (re)classification is a DeleteCSVImportRows + this copy. +INSERT INTO csv_import_rows ( + session_id, row_index, raw, parsed_date, parsed_amount, parsed_desc, + parsed_merchant, parsed_category, classification, match_txn_id, match_score, + match_reason, parse_error, content_hash, provider_txn_id, include, user_edited, + category_id +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 +); + +-- name: DeleteCSVImportRows :exec +DELETE FROM csv_import_rows WHERE session_id = $1; + +-- name: GetCSVImportRow :one +SELECT * FROM csv_import_rows WHERE id = $1; + +-- name: GetCSVImportRowsPage :many +-- Paginated preview. Empty $2 returns all classifications. +SELECT * FROM csv_import_rows +WHERE session_id = $1 + AND ($2 = '' OR classification = $2) +ORDER BY row_index +LIMIT $3 OFFSET $4; + +-- name: CountCSVImportRowsByClassification :many +SELECT classification, COUNT(*) AS count +FROM csv_import_rows +WHERE session_id = $1 +GROUP BY classification; + +-- name: CountIncludedCSVImportRows :one +SELECT COUNT(*) FROM csv_import_rows +WHERE session_id = $1 + AND include = TRUE + AND classification <> 'error' + AND classification <> 'needs_account'; + +-- name: ListIncludedCSVImportRows :many +-- The exact set that apply will upsert: included, non-error rows in row order. +SELECT * FROM csv_import_rows +WHERE session_id = $1 + AND include = TRUE + AND classification <> 'error' + AND classification <> 'needs_account' +ORDER BY row_index; + +-- name: UpdateCSVImportRow :one +-- Inline edit of a single row (also re-derives its hash/classification in Go). +UPDATE csv_import_rows +SET parsed_date = $2, + parsed_amount = $3, + parsed_desc = $4, + parsed_merchant = $5, + parsed_category = $6, + classification = $7, + match_txn_id = $8, + match_score = $9, + match_reason = $10, + parse_error = $11, + content_hash = $12, + provider_txn_id = $13, + include = $14, + category_id = $15, + user_edited = TRUE +WHERE id = $1 +RETURNING *; + +-- name: SetCSVImportRowInclude :exec +UPDATE csv_import_rows SET include = $2 WHERE id = $1; + +-- name: SetCSVImportRowsIncludeByClassification :exec +UPDATE csv_import_rows +SET include = $3 +WHERE session_id = $1 AND classification = $2; + +-- name: SetCSVImportRowsCategoryAll :exec +-- Bulk "set category" across all included rows in the session. +UPDATE csv_import_rows +SET category_id = $2, user_edited = TRUE +WHERE session_id = $1 AND include = TRUE; diff --git a/internal/db/queries/csv_import_sessions.sql b/internal/db/queries/csv_import_sessions.sql new file mode 100644 index 00000000..6a347ec6 --- /dev/null +++ b/internal/db/queries/csv_import_sessions.sql @@ -0,0 +1,68 @@ +-- name: CreateCSVImportSession :one +INSERT INTO csv_import_sessions ( + user_id, status, filename, file_sha256, delimiter, headers, raw_blob, + row_count, detected_template, column_mapping, date_format, + positive_is_debit, has_debit_credit, iso_currency_code, profile_id, expires_at +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, + $8, $9, $10, $11, + $12, $13, $14, $15, $16 +) +RETURNING *; + +-- name: GetCSVImportSession :one +SELECT * FROM csv_import_sessions WHERE id = $1; + +-- name: GetCSVImportSessionByShortID :one +SELECT * FROM csv_import_sessions WHERE short_id = $1; + +-- name: UpdateCSVImportSessionStatus :exec +UPDATE csv_import_sessions +SET status = $2, updated_at = NOW() +WHERE id = $1; + +-- name: ResolveCSVImportSessionAccount :one +-- Bind the chosen target account/connection + currency and advance status. +UPDATE csv_import_sessions +SET resolved_account_id = $2, + resolved_connection_id = $3, + iso_currency_code = $4, + status = $5, + updated_at = NOW() +WHERE id = $1 +RETURNING *; + +-- name: UpdateCSVImportSessionParse :one +-- Re-map columns / sign / date-format / template (triggers a reclassify). +UPDATE csv_import_sessions +SET column_mapping = $2, + date_format = $3, + positive_is_debit = $4, + has_debit_credit = $5, + detected_template = $6, + updated_at = NOW() +WHERE id = $1 +RETURNING *; + +-- name: SetCSVImportSessionProfile :exec +UPDATE csv_import_sessions +SET profile_id = $2, updated_at = NOW() +WHERE id = $1; + +-- name: FinalizeCSVImportSession :exec +-- After a successful apply: snapshot result, link the sync log, drop the blob. +UPDATE csv_import_sessions +SET status = 'applied', + result = $2, + sync_log_id = $3, + raw_blob = NULL, + updated_at = NOW() +WHERE id = $1; + +-- name: ListExpiredCSVImportSessions :many +SELECT * FROM csv_import_sessions +WHERE status <> 'applied' AND expires_at IS NOT NULL AND expires_at < NOW() +ORDER BY expires_at; + +-- name: DeleteCSVImportSession :exec +DELETE FROM csv_import_sessions WHERE id = $1; diff --git a/internal/db/queries/transactions.sql b/internal/db/queries/transactions.sql index e351ef24..b0560dff 100644 --- a/internal/db/queries/transactions.sql +++ b/internal/db/queries/transactions.sql @@ -48,6 +48,76 @@ ON CONFLICT (provider_transaction_id) DO UPDATE SET END RETURNING *; +-- name: UpsertTransactionV2 :one +-- Like UpsertTransaction but also stores the account-independent content_hash +-- (used by CSV import v2 dedup) and returns whether the row was newly inserted +-- (xmax = 0 on INSERT, non-zero on the ON CONFLICT UPDATE path) so callers get +-- reliable insert-vs-update counts without timestamp heuristics. +INSERT INTO transactions ( + account_id, provider_transaction_id, provider_pending_transaction_id, + amount, iso_currency_code, date, authorized_date, + datetime, authorized_datetime, provider_name, provider_merchant_name, + provider_category_primary, provider_category_detailed, provider_category_confidence, + provider_payment_channel, pending, category_id, provider_raw, merchant_key, + content_hash +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20 +) +ON CONFLICT (provider_transaction_id) DO UPDATE SET + account_id = EXCLUDED.account_id, + merchant_key = EXCLUDED.merchant_key, + provider_pending_transaction_id = EXCLUDED.provider_pending_transaction_id, + amount = EXCLUDED.amount, + iso_currency_code = EXCLUDED.iso_currency_code, + date = EXCLUDED.date, + authorized_date = EXCLUDED.authorized_date, + datetime = EXCLUDED.datetime, + authorized_datetime = EXCLUDED.authorized_datetime, + provider_name = EXCLUDED.provider_name, + provider_merchant_name = EXCLUDED.provider_merchant_name, + provider_category_primary = EXCLUDED.provider_category_primary, + provider_category_detailed = EXCLUDED.provider_category_detailed, + provider_category_confidence = EXCLUDED.provider_category_confidence, + provider_payment_channel = EXCLUDED.provider_payment_channel, + pending = EXCLUDED.pending, + provider_raw = COALESCE(EXCLUDED.provider_raw, transactions.provider_raw), + content_hash = COALESCE(EXCLUDED.content_hash, transactions.content_hash), + category_id = CASE + WHEN transactions.category_override <> 'none' THEN transactions.category_id + WHEN EXCLUDED.category_id IS NOT NULL THEN EXCLUDED.category_id + ELSE transactions.category_id + END, + deleted_at = NULL, + updated_at = CASE + WHEN transactions.amount IS DISTINCT FROM EXCLUDED.amount + OR transactions.provider_name IS DISTINCT FROM EXCLUDED.provider_name + OR transactions.pending IS DISTINCT FROM EXCLUDED.pending + OR transactions.provider_merchant_name IS DISTINCT FROM EXCLUDED.provider_merchant_name + OR transactions.provider_category_primary IS DISTINCT FROM EXCLUDED.provider_category_primary + OR transactions.provider_category_detailed IS DISTINCT FROM EXCLUDED.provider_category_detailed + OR transactions.deleted_at IS NOT NULL + THEN NOW() + ELSE transactions.updated_at + END +RETURNING *, (xmax = 0) AS inserted; + +-- name: ListAccountTransactionsForDedup :many +-- Live transactions in an account within a date window — the candidate set the +-- CSV import classifier compares incoming rows against (provider-agnostic, so it +-- also dedupes against Plaid/Teller rows already in the account). +SELECT id, date, amount, provider_name, provider_merchant_name, + content_hash, provider_transaction_id +FROM transactions +WHERE account_id = $1 AND deleted_at IS NULL AND date BETWEEN $2 AND $3; + +-- name: ListTransactionKeysInRange :many +-- (account_id, date, amount) for all live transactions in a date window, across +-- every account — used to score how well an uploaded CSV overlaps each candidate +-- account during account auto-detection. +SELECT account_id, date, amount +FROM transactions +WHERE deleted_at IS NULL AND account_id IS NOT NULL AND date BETWEEN $1 AND $2; + -- name: SoftDeleteTransactionByExternalID :exec UPDATE transactions SET deleted_at = NOW() WHERE provider_transaction_id = $1 AND deleted_at IS NULL; diff --git a/internal/provider/csv/dedup.go b/internal/provider/csv/dedup.go index 17da67c9..1a73cdf6 100644 --- a/internal/provider/csv/dedup.go +++ b/internal/provider/csv/dedup.go @@ -13,6 +13,10 @@ import ( // GenerateExternalID creates a deterministic external transaction ID by // hashing SHA-256(accountID|YYYY-MM-DD|amount|lower(trim(description))). +// +// The account id is baked in so this id is stable for idempotent upserts into a +// RESOLVED account: re-importing the same file into the same account reproduces +// identical ids and UpsertTransaction no-ops. func GenerateExternalID(accountID string, date time.Time, amount decimal.Decimal, description string) string { input := fmt.Sprintf("%s|%s|%s|%s", accountID, @@ -23,3 +27,36 @@ func GenerateExternalID(accountID string, date time.Time, amount decimal.Decimal hash := sha256.Sum256([]byte(input)) return fmt.Sprintf("%x", hash[:]) } + +// GenerateExternalIDWithOccurrence is GenerateExternalID with a disambiguator +// appended, used when a user force-imports a row that is byte-for-byte identical +// to one already imported (same date+amount+description). Occurrence 0 is +// identical to GenerateExternalID so the common path stays idempotent. +func GenerateExternalIDWithOccurrence(accountID string, date time.Time, amount decimal.Decimal, description string, occurrence int) string { + if occurrence == 0 { + return GenerateExternalID(accountID, date, amount, description) + } + input := fmt.Sprintf("%s|%s|%s|%s|#%d", + accountID, + date.Format("2006-01-02"), + amount.String(), + strings.ToLower(strings.TrimSpace(description)), + occurrence, + ) + hash := sha256.Sum256([]byte(input)) + return fmt.Sprintf("%x", hash[:]) +} + +// GenerateContentHash creates an account-INDEPENDENT fingerprint of a +// transaction's core content: SHA-256(YYYY-MM-DD|amount|lower(trim(desc))). It +// lets the CSV import classifier detect that a row already exists in a target +// account regardless of which provider (CSV, Plaid, Teller) created it. +func GenerateContentHash(date time.Time, amount decimal.Decimal, description string) string { + input := fmt.Sprintf("%s|%s|%s", + date.Format("2006-01-02"), + amount.String(), + strings.ToLower(strings.TrimSpace(description)), + ) + hash := sha256.Sum256([]byte(input)) + return fmt.Sprintf("%x", hash[:]) +} diff --git a/internal/provider/csv/fingerprint.go b/internal/provider/csv/fingerprint.go new file mode 100644 index 00000000..cfa3c3da --- /dev/null +++ b/internal/provider/csv/fingerprint.go @@ -0,0 +1,153 @@ +//go:build !lite + +package csv + +import ( + "crypto/sha256" + "encoding/hex" + "regexp" + "strings" +) + +// NormalizeHeaders lowercases and trims each header. The order is preserved +// because column mapping is index-based — a bank reordering its columns is a +// genuinely different layout that should not reuse the same saved profile. +func NormalizeHeaders(headers []string) []string { + out := make([]string, len(headers)) + for i, h := range headers { + out[i] = strings.ToLower(strings.TrimSpace(h)) + } + return out +} + +// HeaderFingerprint returns a stable hash of the normalized, ordered header row. +// It is the key used to look up a saved CSV import profile: two files from the +// same source export the same header layout and therefore the same fingerprint. +func HeaderFingerprint(headers []string) string { + norm := NormalizeHeaders(headers) + sum := sha256.Sum256([]byte(strings.Join(norm, "\x1f"))) + return hex.EncodeToString(sum[:]) +} + +// maskHeaderRe matches header names that plausibly carry a card/account number. +var maskHeaderRe = regexp.MustCompile(`(?i)\b(card\s*no\.?|card\s*number|account\s*(no\.?|number)|last\s*4)\b`) + +// digitsRe extracts runs of digits. +var digitsRe = regexp.MustCompile(`\d+`) + +// filenameMaskRe finds a 4-digit group preceded by a mask-ish hint in a +// filename, e.g. "x4321", "ending 4321", "acct_4321", "-4321". +// The trailing (?:\D|$) (rather than \b) rejects the case where the 4 digits are +// part of a longer run — "_" counts as a word char so \b would mis-fire on +// "4321_". RE2 has no lookahead, so we consume one non-digit outside the group. +var filenameMaskRe = regexp.MustCompile(`(?i)(?:x|ending|acct|account|card|no|#|-|_)\s*[-_ ]?(\d{4})(?:\D|$)`) + +// instMaskRe matches the very common "" filename shape banks +// use, e.g. "Chase0198_Activity.csv" or "Amex1009.csv" — letters immediately +// followed by exactly 4 digits. Requiring a letter run before the digits avoids +// matching date stamps like "_20260607" (preceded by a separator, not letters). +var instMaskRe = regexp.MustCompile(`(?i)[a-z]{2,}(\d{4})(?:\D|$)`) + +// lastFour returns the trailing 4 digits of s (digits only), or "" if there are +// fewer than 4 digits. +func lastFour(s string) string { + var b strings.Builder + for _, r := range s { + if r >= '0' && r <= '9' { + b.WriteByte(byte(r)) + } + } + d := b.String() + if len(d) < 4 { + return "" + } + return d[len(d)-4:] +} + +// ExtractMask attempts to recover the account's last-4 ("mask") from the file — +// first from a card/account-number column (using a sample of data rows), then +// from the filename. Returns "" when nothing convincing is found. +func ExtractMask(filename string, headers []string, sampleRows [][]string) string { + // 1. A dedicated card/account-number column. + for i, h := range headers { + if !maskHeaderRe.MatchString(h) { + continue + } + for _, row := range sampleRows { + if i >= len(row) { + continue + } + if m := lastFour(row[i]); m != "" { + return m + } + } + } + + // 2. The filename, with a mask-ish hint in front of the 4 digits. + base := filename + if idx := strings.LastIndexAny(base, `/\`); idx >= 0 { + base = base[idx+1:] + } + if m := filenameMaskRe.FindStringSubmatch(base); m != nil { + return m[1] + } + // "" shape (Chase0198, Amex1009). + if m := instMaskRe.FindStringSubmatch(base); m != nil { + return m[1] + } + + return "" +} + +// stopTokens are filename fragments that carry no account-identifying signal. +var stopTokens = map[string]bool{ + "csv": true, "tsv": true, "txt": true, "transactions": true, + "transaction": true, "activity": true, "statement": true, "export": true, + "download": true, "history": true, "data": true, "report": true, +} + +// tokenSplitRe splits a filename into alphabetic/numeric tokens. +var tokenSplitRe = regexp.MustCompile(`[^a-zA-Z0-9]+`) + +// FilenameTokens returns lowercased, meaningful tokens from a filename (no +// extension, no generic words, no pure numbers). Used to match a file against +// an account's name/nickname. +func FilenameTokens(filename string) []string { + base := filename + if idx := strings.LastIndexAny(base, `/\`); idx >= 0 { + base = base[idx+1:] + } + if idx := strings.LastIndex(base, "."); idx >= 0 { + base = base[:idx] + } + var out []string + for _, tok := range tokenSplitRe.Split(strings.ToLower(base), -1) { + if len(tok) < 3 || stopTokens[tok] { + continue + } + if digitsRe.MatchString(tok) && !regexp.MustCompile(`[a-z]`).MatchString(tok) { + continue // pure number + } + out = append(out, tok) + } + return out +} + +// InstitutionTokens returns lowercased significant tokens of an institution / +// account name, suitable for substring overlap scoring. Drops short and generic +// words ("the", "card", "checking", ...). +func InstitutionTokens(name string) []string { + generic := map[string]bool{ + "the": true, "card": true, "credit": true, "checking": true, + "savings": true, "account": true, "bank": true, "of": true, + "and": true, "debit": true, "rewards": true, "cash": true, + } + var out []string + for _, tok := range tokenSplitRe.Split(strings.ToLower(strings.TrimSpace(name)), -1) { + if len(tok) < 3 || generic[tok] { + continue + } + out = append(out, tok) + } + return out +} diff --git a/internal/provider/csv/fingerprint_test.go b/internal/provider/csv/fingerprint_test.go new file mode 100644 index 00000000..a9cac05e --- /dev/null +++ b/internal/provider/csv/fingerprint_test.go @@ -0,0 +1,70 @@ +//go:build !lite + +package csv + +import ( + "reflect" + "testing" +) + +func TestHeaderFingerprintStableAndOrderSensitive(t *testing.T) { + a := HeaderFingerprint([]string{"Date", "Amount", "Description"}) + // Case/whitespace insensitive, same order → same fingerprint. + b := HeaderFingerprint([]string{" date ", "AMOUNT", "description"}) + if a != b { + t.Errorf("fingerprint should ignore case/whitespace: %s != %s", a, b) + } + // Different column order → different fingerprint (mapping is index-based). + c := HeaderFingerprint([]string{"Amount", "Date", "Description"}) + if a == c { + t.Error("fingerprint should be order-sensitive") + } +} + +func TestExtractMaskFromColumn(t *testing.T) { + headers := []string{"Transaction Date", "Card No.", "Description", "Debit"} + rows := [][]string{ + {"2026-01-02", "", "STARBUCKS", "5.00"}, // empty mask cell + {"2026-01-03", "1234567890124321", "AMZN", ""}, // full PAN → last4 + } + if got := ExtractMask("capone.csv", headers, rows); got != "4321" { + t.Errorf("ExtractMask column = %q, want 4321", got) + } +} + +func TestExtractMaskFromFilename(t *testing.T) { + headers := []string{"Date", "Amount", "Description"} + cases := map[string]string{ + "Chase_x4321_Activity.csv": "4321", + "statement-ending 9876.csv": "9876", + "acct_5555_2026.csv": "5555", + "plain_transactions.csv": "", + "/Users/me/Downloads/card-0001": "0001", + // Institution-glued last4 (real Chase/Amex export filenames). + "Chase0198_Activity_20260607.CSV": "0198", + "Amex1009.csv": "1009", + // A bare date stamp must not be mistaken for a mask. + "export_20260607.csv": "", + } + for fn, want := range cases { + if got := ExtractMask(fn, headers, nil); got != want { + t.Errorf("ExtractMask(%q) = %q, want %q", fn, got, want) + } + } +} + +func TestFilenameTokens(t *testing.T) { + got := FilenameTokens("Chase_Sapphire_transactions_2026.csv") + want := []string{"chase", "sapphire"} + if !reflect.DeepEqual(got, want) { + t.Errorf("FilenameTokens = %v, want %v", got, want) + } +} + +func TestInstitutionTokens(t *testing.T) { + got := InstitutionTokens("Chase Sapphire Credit Card") + want := []string{"chase", "sapphire"} + if !reflect.DeepEqual(got, want) { + t.Errorf("InstitutionTokens = %v, want %v", got, want) + } +} diff --git a/internal/provider/csv/parser.go b/internal/provider/csv/parser.go index 3bae9c6d..e5846b37 100644 --- a/internal/provider/csv/parser.go +++ b/internal/provider/csv/parser.go @@ -33,6 +33,12 @@ func ParseFile(raw []byte) (*ParsedFile, error) { r.Comma = delim r.LazyQuotes = true r.TrimLeadingSpace = true + // Allow a variable number of fields per row. Real bank exports are ragged: + // Chase checking CSVs append a trailing comma to every data row (8 fields vs + // a 7-column header), and other banks pad/omit trailing empties. Column + // mapping is index-based and bounds-checked downstream, so extra/short + // trailing fields are harmless — failing the whole import over them is not. + r.FieldsPerRecord = -1 var allRows [][]string for { diff --git a/internal/provider/csv/parser_test.go b/internal/provider/csv/parser_test.go index 21737bc3..37f3b867 100644 --- a/internal/provider/csv/parser_test.go +++ b/internal/provider/csv/parser_test.go @@ -26,6 +26,29 @@ func TestParseFileCSV(t *testing.T) { } } +// TestParseFileRaggedRows covers real bank exports whose data rows carry a +// trailing comma (more fields than the header) — e.g. Chase checking CSVs with +// a "Check or Slip #" column plus a trailing comma. These must parse, not fail. +func TestParseFileRaggedRows(t *testing.T) { + input := "Details,Posting Date,Description,Amount,Type,Balance,Check or Slip #\n" + + "CREDIT,06/05/2026,\"META PAYROLL PPD ID: 9111\",8108.27,ACH_CREDIT,18754.72,,\n" + + "DEBIT,06/02/2026,\"AMEX ACH PMT WEB ID: 9493\",-2213.33,ACH_DEBIT,10646.45,,\n" + + pf, err := ParseFile([]byte(input)) + if err != nil { + t.Fatalf("ParseFile ragged: %v", err) + } + if len(pf.Rows) != 2 { + t.Fatalf("rows: got %d, want 2", len(pf.Rows)) + } + if DetectTemplate(pf.Headers) == nil { + t.Error("expected Chase Checking template to match") + } + if pf.Rows[0][3] != "8108.27" { + t.Errorf("amount cell: got %q, want 8108.27", pf.Rows[0][3]) + } +} + func TestParseFileTSV(t *testing.T) { input := "Date\tAmount\tDescription\n2024-01-15\t42.50\tCoffee\n2024-01-16\t10.00\tGrocery\n" diff --git a/internal/service/csv_account_match.go b/internal/service/csv_account_match.go new file mode 100644 index 00000000..88b991ce --- /dev/null +++ b/internal/service/csv_account_match.go @@ -0,0 +1,307 @@ +//go:build !lite + +package service + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "breadbox/internal/db" + "breadbox/internal/pgconv" + csvpkg "breadbox/internal/provider/csv" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +// Account-detection scoring weights. Deterministic and explainable — every point +// awarded carries a human-readable reason. +const ( + scoreMaskMatch = 40 // file's last-4 matches the account mask + scoreOverlapMax = 40 // proportional to how many rows already exist there + scoreInstitution = 15 // template bank name appears in the account name + scoreFilenameHint = 5 // filename mentions the account + overlapSampleSize = 200 + preselectMinScore = 70 // top must reach this… + preselectMinGap = 20 // …and lead the runner-up by this + confidenceHighScore = 70 + confidenceMediumScore = 40 +) + +// CSVAccountMatch is one ranked account suggestion for an uploaded file. +type CSVAccountMatch struct { + AccountID string `json:"account_id"` + ShortID string `json:"short_id"` + AccountName string `json:"account_name"` + Institution string `json:"institution"` + Mask string `json:"mask"` + Currency string `json:"currency"` + Score int `json:"score"` + Confidence string `json:"confidence"` // high | medium | low + Reasons []string `json:"reasons"` + ProfileMatch bool `json:"profile_match"` +} + +// CSVAccountSuggestion is the full account-detection result: ranked matches plus +// whether one is confident enough to pre-select. +type CSVAccountSuggestion struct { + Matches []CSVAccountMatch `json:"matches"` + Preselect string `json:"preselect"` // account id to auto-select, "" = ask the user + ProfileID string `json:"profile_id"` +} + +// CSVDetectionSignals are the file-derived inputs to account detection. +type CSVDetectionSignals struct { + Filename string + Headers []string + DetectedTemplate string // bank template name, "" if none matched + Mask string // last-4 extracted from the file/filename, "" if none +} + +// MatchCSVAccounts ranks existing accounts for an uploaded file using the file +// signals plus how strongly the parsed rows overlap each account's existing +// transactions. A saved profile (matched by header fingerprint) short-circuits +// to its remembered account. +func (s *Service) MatchCSVAccounts( + ctx context.Context, + userID pgtype.UUID, + signals CSVDetectionSignals, + parsed []CSVParsedRow, +) (*CSVAccountSuggestion, error) { + accounts, err := s.Queries.ListAccounts(ctx) + if err != nil { + return nil, fmt.Errorf("list accounts: %w", err) + } + + // Profile lookup by header fingerprint (the strongest possible signal). + fingerprint := csvpkg.HeaderFingerprint(signals.Headers) + var profileAccountID, profileID string + if prof, err := s.Queries.GetCSVImportProfileByFingerprint(ctx, db.GetCSVImportProfileByFingerprintParams{ + UserID: userID, + HeaderFingerprint: fingerprint, + }); err == nil { + profileID = formatUUID(prof.ID) + if prof.DefaultAccountID.Valid { + profileAccountID = formatUUID(prof.DefaultAccountID) + } + } else if !errors.Is(err, pgx.ErrNoRows) { + s.Logger.Warn("csv account match: profile lookup failed", "error", err) + } + + overlap := s.overlapRatios(ctx, accounts, parsed) + fileTokens := csvpkg.FilenameTokens(signals.Filename) + + matches := make([]CSVAccountMatch, 0, len(accounts)) + for _, a := range accounts { + accID := formatUUID(a.ID) + mask := pgconv.TextOr(a.Mask, "") + institution := pgconv.TextOr(a.InstitutionName, "") + + m := CSVAccountMatch{ + AccountID: accID, + ShortID: a.ShortID, + AccountName: a.Name, + Institution: institution, + Mask: mask, + Currency: pgconv.TextOr(a.IsoCurrencyCode, "USD"), + } + + // Mask / last-4. + if signals.Mask != "" && mask != "" && signals.Mask == mask { + m.Score += scoreMaskMatch + m.Reasons = append(m.Reasons, "card ending "+mask+" matches") + } + + // Transaction overlap. + if ratio := overlap[accID]; ratio > 0 { + pts := int(ratio*float64(scoreOverlapMax) + 0.5) + if pts > 0 { + m.Score += pts + m.Reasons = append(m.Reasons, fmt.Sprintf("%d%% of rows already here", int(ratio*100+0.5))) + } + } + + // Institution name (template bank name appears in the account name). + if signals.DetectedTemplate != "" && institutionOverlap(signals.DetectedTemplate, a.Name, institution) { + m.Score += scoreInstitution + m.Reasons = append(m.Reasons, "institution looks like "+signals.DetectedTemplate) + } + + // Filename mentions the account. + if tokensOverlap(fileTokens, csvpkg.InstitutionTokens(a.Name)) { + m.Score += scoreFilenameHint + m.Reasons = append(m.Reasons, "filename mentions this account") + } + + // Profile-remembered account wins outright. + if profileAccountID != "" && accID == profileAccountID { + m.ProfileMatch = true + m.Score = 100 + m.Reasons = append([]string{"saved import profile"}, m.Reasons...) + } + + m.Confidence = confidenceFor(m.Score) + matches = append(matches, m) + } + + sort.SliceStable(matches, func(i, j int) bool { + if matches[i].Score != matches[j].Score { + return matches[i].Score > matches[j].Score + } + return matches[i].AccountName < matches[j].AccountName + }) + + return &CSVAccountSuggestion{ + Matches: matches, + Preselect: choosePreselect(matches, profileAccountID), + ProfileID: profileID, + }, nil +} + +// overlapRatios returns, per account id, the fraction of sampled parseable rows +// whose (date, amount) already exists in that account. +func (s *Service) overlapRatios(ctx context.Context, accounts []db.ListAccountsRow, parsed []CSVParsedRow) map[string]float64 { + sample := sampleParsedRows(parsed, overlapSampleSize) + if len(sample) == 0 { + return nil + } + + var minDate, maxDate time.Time + for i, r := range sample { + if i == 0 || r.Date.Before(minDate) { + minDate = r.Date + } + if i == 0 || r.Date.After(maxDate) { + maxDate = r.Date + } + } + + keys, err := s.Queries.ListTransactionKeysInRange(ctx, db.ListTransactionKeysInRangeParams{ + Date: pgconv.Date(minDate), + Date_2: pgconv.Date(maxDate), + }) + if err != nil { + s.Logger.Warn("csv account match: overlap query failed", "error", err) + return nil + } + + // Per-account set of date|amount keys present in the DB. + present := map[string]map[string]struct{}{} + for _, k := range keys { + if !k.AccountID.Valid { + continue + } + acc := formatUUID(k.AccountID) + set := present[acc] + if set == nil { + set = map[string]struct{}{} + present[acc] = set + } + set[dateAmountKey(k.Date.Time, numericToDecimal(k.Amount).String())] = struct{}{} + } + + ratios := make(map[string]float64, len(accounts)) + for _, a := range accounts { + acc := formatUUID(a.ID) + set := present[acc] + if len(set) == 0 { + continue + } + matched := 0 + for _, r := range sample { + if _, ok := set[dateAmountKey(r.Date, r.Amount.String())]; ok { + matched++ + } + } + if matched > 0 { + ratios[acc] = float64(matched) / float64(len(sample)) + } + } + return ratios +} + +func dateAmountKey(d time.Time, amount string) string { + return d.Format("2006-01-02") + "|" + amount +} + +// sampleParsedRows returns up to n parseable rows, strided to spread coverage +// across the file rather than clustering at the start. +func sampleParsedRows(parsed []CSVParsedRow, n int) []CSVParsedRow { + valid := make([]CSVParsedRow, 0, len(parsed)) + for _, r := range parsed { + if r.ParseError == "" { + valid = append(valid, r) + } + } + if len(valid) <= n { + return valid + } + out := make([]CSVParsedRow, 0, n) + stride := float64(len(valid)) / float64(n) + for i := 0; i < n; i++ { + out = append(out, valid[int(float64(i)*stride)]) + } + return out +} + +func institutionOverlap(template, name, institution string) bool { + tmplTokens := csvpkg.InstitutionTokens(template) + target := csvpkg.InstitutionTokens(name) + target = append(target, csvpkg.InstitutionTokens(institution)...) + return tokensOverlap(tmplTokens, target) +} + +func tokensOverlap(a, b []string) bool { + if len(a) == 0 || len(b) == 0 { + return false + } + set := make(map[string]struct{}, len(b)) + for _, t := range b { + set[t] = struct{}{} + } + for _, t := range a { + if _, ok := set[t]; ok { + return true + } + } + return false +} + +func confidenceFor(score int) string { + switch { + case score >= confidenceHighScore: + return "high" + case score >= confidenceMediumScore: + return "medium" + default: + return "low" + } +} + +// choosePreselect returns the account id to auto-select, or "" to ask the user. +// A profile-remembered account always pre-selects; otherwise the top match must +// be both strong and a clear leader. +func choosePreselect(matches []CSVAccountMatch, profileAccountID string) string { + if profileAccountID != "" { + for _, m := range matches { + if m.AccountID == profileAccountID { + return profileAccountID + } + } + } + if len(matches) == 0 { + return "" + } + top := matches[0] + second := 0 + if len(matches) > 1 { + second = matches[1].Score + } + if top.Score >= preselectMinScore && top.Score-second >= preselectMinGap { + return top.AccountID + } + return "" +} diff --git a/internal/service/csv_account_match_integration_test.go b/internal/service/csv_account_match_integration_test.go new file mode 100644 index 00000000..73699e8e --- /dev/null +++ b/internal/service/csv_account_match_integration_test.go @@ -0,0 +1,123 @@ +//go:build integration && !lite + +package service_test + +import ( + "context" + "testing" + + "breadbox/internal/db" + "breadbox/internal/pgconv" + csvpkg "breadbox/internal/provider/csv" + "breadbox/internal/service" + "breadbox/internal/testutil" +) + +// overlapRows builds CSV rows matching the given (date, amount) pairs. +func overlapRows(pairs [][2]string) [][]string { + rows := make([][]string, len(pairs)) + for i, p := range pairs { + rows[i] = []string{p[0], p[1], "TXN " + p[0]} + } + return rows +} + +func TestAccountMatch_OverlapRanksCorrectAccount(t *testing.T) { + svc, q, _ := newService(t) + user := testutil.MustCreateUser(t, q, "Alice") + conn := testutil.MustCreateConnection(t, q, user.ID, "item_1") + accA := testutil.MustCreateAccount(t, q, conn.ID, "extA", "Checking A") + accB := testutil.MustCreateAccount(t, q, conn.ID, "extB", "Savings B") + + // Seed account B with three transactions; A gets an unrelated one. + testutil.MustCreateTransaction(t, q, accB.ID, "b1", "TXN 2026-04-01", 1000, "2026-04-01") + testutil.MustCreateTransaction(t, q, accB.ID, "b2", "TXN 2026-04-02", 2000, "2026-04-02") + testutil.MustCreateTransaction(t, q, accB.ID, "b3", "TXN 2026-04-03", 3000, "2026-04-03") + testutil.MustCreateTransaction(t, q, accA.ID, "a1", "OTHER", 9999, "2026-04-01") + + rows := overlapRows([][2]string{{"2026-04-01", "10.00"}, {"2026-04-02", "20.00"}, {"2026-04-03", "30.00"}}) + parsed := service.ParseCSVRows(classifyCfg, rows) + + sug, err := svc.MatchCSVAccounts(context.Background(), user.ID, service.CSVDetectionSignals{}, parsed) + if err != nil { + t.Fatalf("MatchCSVAccounts: %v", err) + } + if len(sug.Matches) == 0 || sug.Matches[0].AccountID != pgconv.FormatUUID(accB.ID) { + t.Fatalf("expected account B on top, got %+v", sug.Matches) + } + if sug.Matches[0].Score < 40 { + t.Errorf("full overlap should score the overlap max, got %d", sug.Matches[0].Score) + } +} + +func TestAccountMatch_PreselectOnMaskPlusOverlap(t *testing.T) { + svc, q, pool := newService(t) + user := testutil.MustCreateUser(t, q, "Alice") + conn := testutil.MustCreateConnection(t, q, user.ID, "item_1") + acc := testutil.MustCreateAccount(t, q, conn.ID, "extA", "Chase Checking") + // Give the account a mask. + if _, err := pool.Exec(context.Background(), "UPDATE accounts SET mask = '4321' WHERE id = $1", acc.ID); err != nil { + t.Fatalf("set mask: %v", err) + } + testutil.MustCreateTransaction(t, q, acc.ID, "t1", "TXN 2026-05-01", 1000, "2026-05-01") + testutil.MustCreateTransaction(t, q, acc.ID, "t2", "TXN 2026-05-02", 2000, "2026-05-02") + + rows := overlapRows([][2]string{{"2026-05-01", "10.00"}, {"2026-05-02", "20.00"}}) + parsed := service.ParseCSVRows(classifyCfg, rows) + + sug, err := svc.MatchCSVAccounts(context.Background(), user.ID, service.CSVDetectionSignals{Mask: "4321"}, parsed) + if err != nil { + t.Fatalf("MatchCSVAccounts: %v", err) + } + if sug.Preselect != pgconv.FormatUUID(acc.ID) { + t.Fatalf("expected preselect of the matched account, got %q (top score %d)", sug.Preselect, topScore(sug)) + } +} + +func TestAccountMatch_ProfilePreselect(t *testing.T) { + svc, q, _ := newService(t) + user := testutil.MustCreateUser(t, q, "Alice") + conn := testutil.MustCreateConnection(t, q, user.ID, "item_1") + acc := testutil.MustCreateAccount(t, q, conn.ID, "extA", "Chase Checking") + + headers := []string{"Date", "Amount", "Description"} + fp := csvpkg.HeaderFingerprint(headers) + _, err := q.UpsertCSVImportProfile(context.Background(), db.UpsertCSVImportProfileParams{ + UserID: user.ID, + Name: "Chase CSV", + HeaderFingerprint: fp, + Headers: []byte(`["Date","Amount","Description"]`), + ColumnMapping: []byte(`{"date":0,"amount":1,"description":2}`), + DateFormat: "2006-01-02", + Delimiter: ",", + IsoCurrencyCode: "USD", + DefaultAccountID: acc.ID, + }) + if err != nil { + t.Fatalf("seed profile: %v", err) + } + + rows := overlapRows([][2]string{{"2026-06-01", "10.00"}}) + parsed := service.ParseCSVRows(classifyCfg, rows) + + sug, err := svc.MatchCSVAccounts(context.Background(), user.ID, service.CSVDetectionSignals{Headers: headers}, parsed) + if err != nil { + t.Fatalf("MatchCSVAccounts: %v", err) + } + if sug.ProfileID == "" { + t.Error("expected ProfileID to be set on a fingerprint hit") + } + if sug.Preselect != pgconv.FormatUUID(acc.ID) { + t.Fatalf("profile should preselect its default account, got %q", sug.Preselect) + } + if len(sug.Matches) == 0 || !sug.Matches[0].ProfileMatch { + t.Error("top match should be flagged ProfileMatch") + } +} + +func topScore(s *service.CSVAccountSuggestion) int { + if len(s.Matches) == 0 { + return 0 + } + return s.Matches[0].Score +} diff --git a/internal/service/csv_classify.go b/internal/service/csv_classify.go new file mode 100644 index 00000000..c7b823b4 --- /dev/null +++ b/internal/service/csv_classify.go @@ -0,0 +1,350 @@ +//go:build !lite + +package service + +import ( + "context" + "fmt" + "math/big" + "strings" + "time" + + "breadbox/internal/db" + "breadbox/internal/pgconv" + csvpkg "breadbox/internal/provider/csv" + "breadbox/internal/textmatch" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/shopspring/decimal" +) + +// CSVRowClassification is the per-row verdict produced when an analyzed CSV is +// compared against the transactions already in a resolved target account. +type CSVRowClassification string + +const ( + CSVRowNew CSVRowClassification = "new" // not present — import it (default include) + CSVRowExactDup CSVRowClassification = "exact_dup" // identical row already exists (default exclude) + CSVRowProbableDup CSVRowClassification = "probable_dup" // same date+amount, similar name (default exclude) + CSVRowConflict CSVRowClassification = "conflict" // would materially change an existing row + CSVRowError CSVRowClassification = "error" // unparseable (default exclude) + CSVRowNeedsAccount CSVRowClassification = "needs_account" // no account resolved yet +) + +// DefaultDedupToleranceDays is the +/- day window for the fuzzy duplicate pass. +const DefaultDedupToleranceDays = 1 + +// CSVParsedRow is a single CSV data row after applying the column mapping. +type CSVParsedRow struct { + RowIndex int + Raw []string + Date time.Time + Amount decimal.Decimal + Desc string + Merchant string + Category string + ParseError string // non-empty → classification is forced to error +} + +// CSVClassifiedRow is a parsed row plus its verdict against a target account. +type CSVClassifiedRow struct { + CSVParsedRow + Classification CSVRowClassification + MatchTxnID string // existing transaction this row matched (exact/probable) + MatchScore int // name-similarity score (0-3) for probable matches + MatchReason string + ContentHash string + ProviderTxnID string // stable upsert id, set once the account is known + Include bool // default user intent +} + +// CSVParseConfig captures everything needed to turn raw rows into parsed rows. +type CSVParseConfig struct { + ColumnMapping map[string]int + DateFormat string + PositiveIsDebit bool + HasDebitCredit bool +} + +// ParseCSVRows applies the column mapping to raw rows, producing CSVParsedRow +// values. Rows that fail to parse carry a ParseError instead of being dropped, +// so the preview can show and let the user fix them. This mirrors the per-row +// logic in ImportCSV but never touches the DB. +func ParseCSVRows(cfg CSVParseConfig, rows [][]string) []CSVParsedRow { + dateCol := cfg.ColumnMapping["date"] + amountCol := cfg.ColumnMapping["amount"] + descCol := cfg.ColumnMapping["description"] + catCol, hasCat := cfg.ColumnMapping["category"] + merchantCol, hasMerchant := cfg.ColumnMapping["merchant_name"] + debitCol := cfg.ColumnMapping["debit"] + creditCol := cfg.ColumnMapping["credit"] + + out := make([]CSVParsedRow, 0, len(rows)) + for i, row := range rows { + pr := CSVParsedRow{RowIndex: i, Raw: row} + + maxCol := max(dateCol, descCol) + if cfg.HasDebitCredit { + maxCol = max(maxCol, max(debitCol, creditCol)) + } else { + maxCol = max(maxCol, amountCol) + } + if maxCol >= len(row) { + pr.ParseError = "not enough columns" + out = append(out, pr) + continue + } + + dateVal, err := csvpkg.ParseDate(row[dateCol], cfg.DateFormat) + if err != nil { + pr.ParseError = err.Error() + out = append(out, pr) + continue + } + pr.Date = dateVal + + var amount decimal.Decimal + if cfg.HasDebitCredit { + amount, err = csvpkg.ParseDualColumns(row[debitCol], row[creditCol]) + } else { + amount, err = csvpkg.ParseAmount(row[amountCol]) + } + if err != nil { + pr.ParseError = err.Error() + out = append(out, pr) + continue + } + pr.Amount = csvpkg.NormalizeSign(amount, cfg.PositiveIsDebit) + + desc := strings.TrimSpace(row[descCol]) + if desc == "" { + pr.ParseError = "empty description" + out = append(out, pr) + continue + } + if len(desc) > 500 { + desc = desc[:500] + } + pr.Desc = desc + + if hasCat && catCol < len(row) { + pr.Category = strings.TrimSpace(row[catCol]) + } + if hasMerchant && merchantCol < len(row) { + m := strings.TrimSpace(row[merchantCol]) + if len(m) > 200 { + m = m[:200] + } + pr.Merchant = m + } + + out = append(out, pr) + } + return out +} + +// dedupCandidate is an existing transaction reduced to the fields the classifier +// compares against. +type dedupCandidate struct { + id string + date time.Time + amount decimal.Decimal + name string + merchant string + contentHash string + providerTxnID string +} + +// ClassifyCSVRows compares parsed rows against the live transactions in the +// resolved account and assigns each a classification. accountIDStr is the +// canonical account UUID string (used to derive the stable provider_transaction_id). +func (s *Service) ClassifyCSVRows( + ctx context.Context, + accountID pgtype.UUID, + accountIDStr string, + rows []CSVParsedRow, + toleranceDays int, +) ([]CSVClassifiedRow, error) { + if toleranceDays < 0 { + toleranceDays = 0 + } + + // Date window covering all parseable rows, padded by the tolerance. + var minDate, maxDate time.Time + haveDate := false + for _, r := range rows { + if r.ParseError != "" { + continue + } + if !haveDate || r.Date.Before(minDate) { + minDate = r.Date + } + if !haveDate || r.Date.After(maxDate) { + maxDate = r.Date + } + haveDate = true + } + + // Existing-row indexes. + byProviderTxnID := map[string]dedupCandidate{} + byContentHash := map[string]dedupCandidate{} + byAmount := map[string][]dedupCandidate{} + + if haveDate { + tol := time.Duration(toleranceDays) * 24 * time.Hour + existing, err := s.Queries.ListAccountTransactionsForDedup(ctx, db.ListAccountTransactionsForDedupParams{ + AccountID: accountID, + Date: pgconv.Date(minDate.Add(-tol)), + Date_2: pgconv.Date(maxDate.Add(tol)), + }) + if err != nil { + return nil, fmt.Errorf("load dedup candidates: %w", err) + } + for _, e := range existing { + cand := dedupCandidate{ + id: formatUUID(e.ID), + date: e.Date.Time, + amount: numericToDecimal(e.Amount), + name: e.ProviderName, + merchant: pgconv.TextOr(e.ProviderMerchantName, ""), + contentHash: pgconv.TextOr(e.ContentHash, ""), + providerTxnID: e.ProviderTransactionID, + } + byProviderTxnID[cand.providerTxnID] = cand + if cand.contentHash != "" { + byContentHash[cand.contentHash] = cand + } + byAmount[cand.amount.String()] = append(byAmount[cand.amount.String()], cand) + } + } + + out := make([]CSVClassifiedRow, 0, len(rows)) + for _, pr := range rows { + cr := CSVClassifiedRow{CSVParsedRow: pr} + + if pr.ParseError != "" { + cr.Classification = CSVRowError + cr.Include = false + out = append(out, cr) + continue + } + + cr.ContentHash = csvpkg.GenerateContentHash(pr.Date, pr.Amount, pr.Desc) + cr.ProviderTxnID = csvpkg.GenerateExternalID(accountIDStr, pr.Date, pr.Amount, pr.Desc) + + // Exact duplicate: identical row already imported (same provider id) or + // identical content already present from any source. + if cand, ok := byProviderTxnID[cr.ProviderTxnID]; ok { + cr.Classification = CSVRowExactDup + cr.MatchTxnID = cand.id + cr.MatchReason = "already imported" + cr.Include = false + out = append(out, cr) + continue + } + if cand, ok := byContentHash[cr.ContentHash]; ok { + cr.Classification = CSVRowExactDup + cr.MatchTxnID = cand.id + cr.MatchReason = "identical transaction already exists" + cr.Include = false + out = append(out, cr) + continue + } + + // Probable duplicate: same amount, date within tolerance, name similar. + best, bestScore, bestReason := bestFuzzyMatch(pr, byAmount[pr.Amount.String()], toleranceDays) + if best != nil { + cr.Classification = CSVRowProbableDup + cr.MatchTxnID = best.id + cr.MatchScore = bestScore + cr.MatchReason = bestReason + cr.Include = false + out = append(out, cr) + continue + } + + cr.Classification = CSVRowNew + cr.Include = true + out = append(out, cr) + } + return out, nil +} + +// bestFuzzyMatch finds the strongest same-amount, within-tolerance candidate for +// a parsed row. Returns nil when none qualifies. A candidate with an exact +// date+amount match qualifies even with no name overlap (score 0). +func bestFuzzyMatch(pr CSVParsedRow, sameAmount []dedupCandidate, toleranceDays int) (*dedupCandidate, int, string) { + descLower := strings.ToLower(pr.Desc) + merchLower := strings.ToLower(pr.Merchant) + + var best *dedupCandidate + bestScore := -1 + bestExact := false + var bestReason string + + for i := range sameAmount { + cand := &sameAmount[i] + dayDiff := daysBetween(pr.Date, cand.date) + if dayDiff > toleranceDays { + continue + } + exactDate := dayDiff == 0 + score, fields := textmatch.ScoreLowered(pr.Desc, descLower, pr.Merchant, merchLower, cand.name, cand.merchant) + + // Only a same-date+amount row qualifies on its own with no name overlap. + if score == 0 && !exactDate { + continue + } + + // Prefer higher name score; tie-break toward an exact-date match. + better := score > bestScore || (score == bestScore && exactDate && !bestExact) + if best == nil || better { + best = cand + bestScore = score + bestExact = exactDate + bestReason = fuzzyReason(score, fields, exactDate) + } + } + if best == nil { + return nil, 0, "" + } + return best, bestScore, bestReason +} + +func fuzzyReason(score int, fields []string, exactDate bool) string { + datePart := "same amount" + if exactDate { + datePart = "same date & amount" + } else { + datePart = "same amount, nearby date" + } + if score >= 2 && len(fields) > 0 { + return fmt.Sprintf("%s, %s matches", datePart, strings.ReplaceAll(fields[0], "_", " ")) + } + if score == 1 { + return datePart + ", similar name" + } + return datePart +} + +// daysBetween returns the absolute whole-day difference between two dates. +func daysBetween(a, b time.Time) int { + ay, am, ad := a.Date() + by, bm, bd := b.Date() + da := time.Date(ay, am, ad, 0, 0, 0, 0, time.UTC) + db := time.Date(by, bm, bd, 0, 0, 0, 0, time.UTC) + diff := int(da.Sub(db).Hours() / 24) + if diff < 0 { + diff = -diff + } + return diff +} + +// numericToDecimal converts a pgtype.Numeric to a shopspring Decimal. Invalid / +// NaN / infinity values become zero (they never occur for transaction amounts). +func numericToDecimal(n pgtype.Numeric) decimal.Decimal { + if !n.Valid || n.NaN || n.Int == nil || n.InfinityModifier != pgtype.Finite { + return decimal.Zero + } + return decimal.NewFromBigInt(new(big.Int).Set(n.Int), n.Exp) +} diff --git a/internal/service/csv_classify_integration_test.go b/internal/service/csv_classify_integration_test.go new file mode 100644 index 00000000..3d659317 --- /dev/null +++ b/internal/service/csv_classify_integration_test.go @@ -0,0 +1,108 @@ +//go:build integration && !lite + +package service_test + +import ( + "context" + "testing" + + "breadbox/internal/pgconv" + csvpkg "breadbox/internal/provider/csv" + "breadbox/internal/service" + "breadbox/internal/testutil" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/shopspring/decimal" +) + +// classifyCfg is the column mapping used across these tests: date|amount|desc. +var classifyCfg = service.CSVParseConfig{ + ColumnMapping: map[string]int{"date": 0, "amount": 1, "description": 2}, + DateFormat: "2006-01-02", + PositiveIsDebit: true, // positive already = debit (Breadbox convention) +} + +func classify(t *testing.T, svc *service.Service, acct pgtype.UUID, rows [][]string) []service.CSVClassifiedRow { + t.Helper() + parsed := service.ParseCSVRows(classifyCfg, rows) + out, err := svc.ClassifyCSVRows(context.Background(), acct, pgconv.FormatUUID(acct), parsed, service.DefaultDedupToleranceDays) + if err != nil { + t.Fatalf("ClassifyCSVRows: %v", err) + } + return out +} + +func TestClassify_AllNewIntoEmptyAccount(t *testing.T) { + svc, q, _ := newService(t) + acct := seedTxnFixture(t, q) + + rows := [][]string{ + {"2026-01-10", "12.50", "STARBUCKS"}, + {"2026-01-11", "40.00", "SHELL GAS"}, + } + got := classify(t, svc, acct, rows) + for _, r := range got { + if r.Classification != service.CSVRowNew || !r.Include { + t.Errorf("row %d: got %s include=%v, want new/included", r.RowIndex, r.Classification, r.Include) + } + } +} + +func TestClassify_ExactDupByProviderID(t *testing.T) { + svc, q, _ := newService(t) + acct := seedTxnFixture(t, q) + acctStr := pgconv.FormatUUID(acct) + + // Pre-seed a transaction whose provider id is exactly what re-importing the + // same row would generate. + date := testutil.MustParseDate("2026-01-15") + extID := csvpkg.GenerateExternalID(acctStr, date, decimal.RequireFromString("5.00"), "STARBUCKS") + testutil.MustCreateTransaction(t, q, acct, extID, "STARBUCKS", 500, "2026-01-15") + + got := classify(t, svc, acct, [][]string{{"2026-01-15", "5.00", "STARBUCKS"}}) + if got[0].Classification != service.CSVRowExactDup || got[0].Include { + t.Fatalf("got %s include=%v, want exact_dup/excluded", got[0].Classification, got[0].Include) + } +} + +func TestClassify_ProbableDupCrossProvider(t *testing.T) { + svc, q, _ := newService(t) + // A Teller-sourced txn (no content_hash) with same date+amount and similar + // name. Re-importing a CSV that overlaps must flag it as a probable dup. + user := testutil.MustCreateUser(t, q, "Bob") + conn := testutil.MustCreateTellerConnection(t, q, user.ID, "teller_1") + acct := testutil.MustCreateAccount(t, q, conn.ID, "tlr_acct", "Teller Checking") + testutil.MustCreateTransaction(t, q, acct.ID, "teller_txn_1", "STARBUCKS STORE 123", 500, "2026-02-01") + + // Same amount/date, name is a superset — should fuzzy-match. + got := classify(t, svc, acct.ID, [][]string{{"2026-02-01", "5.00", "STARBUCKS"}}) + if got[0].Classification != service.CSVRowProbableDup || got[0].Include { + t.Fatalf("got %s include=%v reason=%q, want probable_dup/excluded", + got[0].Classification, got[0].Include, got[0].MatchReason) + } + if got[0].MatchTxnID == "" { + t.Error("probable dup should record the matched txn id") + } +} + +func TestClassify_NearbyDateNoNameStaysNew(t *testing.T) { + svc, q, _ := newService(t) + acct := seedTxnFixture(t, q) + // Existing txn: same amount, 5 days away (outside tolerance), different name. + testutil.MustCreateTransaction(t, q, acct, "x1", "WALMART", 500, "2026-03-01") + + got := classify(t, svc, acct, [][]string{{"2026-03-06", "5.00", "TARGET"}}) + if got[0].Classification != service.CSVRowNew { + t.Fatalf("got %s, want new (amount matches but date far + name differs)", got[0].Classification) + } +} + +func TestClassify_ParseErrorRow(t *testing.T) { + svc, q, _ := newService(t) + acct := seedTxnFixture(t, q) + got := classify(t, svc, acct, [][]string{{"not-a-date", "5.00", "X"}}) + if got[0].Classification != service.CSVRowError || got[0].Include { + t.Fatalf("got %s include=%v, want error/excluded", got[0].Classification, got[0].Include) + } +} + diff --git a/internal/service/csv_import_v2.go b/internal/service/csv_import_v2.go new file mode 100644 index 00000000..7ba8904a --- /dev/null +++ b/internal/service/csv_import_v2.go @@ -0,0 +1,1051 @@ +//go:build !lite + +package service + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "breadbox/internal/db" + "breadbox/internal/pgconv" + csvpkg "breadbox/internal/provider/csv" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/shopspring/decimal" +) + +// importSessionTTL is how long an un-applied session lives before the sweeper +// may reap it. +const importSessionTTL = 24 * time.Hour + +// Session status values. +const ( + importStatusAnalyzing = "analyzing" + importStatusAwaitingAccount = "awaiting_account" + importStatusPreviewed = "previewed" + importStatusApplied = "applied" +) + +// ErrImportSessionNotFound is returned when a session id/short id doesn't resolve. +var ErrImportSessionNotFound = errors.New("import session not found") + +// ImportSession is the service-layer view of a csv_import_sessions row. +type ImportSession struct { + ID string `json:"id"` + ShortID string `json:"short_id"` + UserID string `json:"user_id"` + Status string `json:"status"` + Filename string `json:"filename"` + Delimiter string `json:"delimiter"` + Headers []string `json:"headers"` + RowCount int `json:"row_count"` + ResolvedAccountID string `json:"resolved_account_id"` + ResolvedConnectionID string `json:"resolved_connection_id"` + DetectedTemplate string `json:"detected_template"` + ColumnMapping map[string]int `json:"column_mapping"` + DateFormat string `json:"date_format"` + PositiveIsDebit bool `json:"positive_is_debit"` + HasDebitCredit bool `json:"has_debit_credit"` + IsoCurrencyCode string `json:"iso_currency_code"` + ProfileID string `json:"profile_id"` + Result *CSVImportResult `json:"result,omitempty"` +} + +// ImportRowView is the service-layer view of one staged row. +type ImportRowView struct { + ID string `json:"id"` + RowIndex int `json:"row_index"` + Raw []string `json:"raw"` + Date string `json:"date"` + Amount string `json:"amount"` + Desc string `json:"description"` + Merchant string `json:"merchant"` + Category string `json:"category"` + Classification string `json:"classification"` + MatchTxnID string `json:"match_txn_id"` + MatchScore int `json:"match_score"` + MatchReason string `json:"match_reason"` + ParseError string `json:"parse_error"` + Include bool `json:"include"` + UserEdited bool `json:"user_edited"` +} + +// ImportSummary is the per-classification breakdown for a session. +type ImportSummary struct { + Total int `json:"total"` + Counts map[string]int `json:"counts"` + IncludedCount int `json:"included_count"` +} + +// ImportAnalysis is returned from CreateImportSession: the session plus account +// suggestions plus a first summary. +type ImportAnalysis struct { + Session *ImportSession `json:"session"` + Accounts *CSVAccountSuggestion `json:"accounts"` + Summary ImportSummary `json:"summary"` +} + +// CreateImportSessionParams are the inputs for analyzing a dropped file. +type CreateImportSessionParams struct { + UserID string + Filename string + Data []byte +} + +// CreateImportSession parses + analyzes a dropped CSV, persists a staging +// session, applies any saved profile, runs account detection, and (when an +// account can be confidently chosen) classifies rows so the preview is ready. +func (s *Service) CreateImportSession(ctx context.Context, p CreateImportSessionParams) (*ImportAnalysis, error) { + userID, err := s.resolveUserID(ctx, p.UserID) + if err != nil { + return nil, fmt.Errorf("invalid user: %w", err) + } + + pf, err := csvpkg.ParseFile(p.Data) + if err != nil { + return nil, fmt.Errorf("parse csv: %w", err) + } + + cfg, detectedTemplate := detectParseConfig(pf) + currency := detectCurrency(pf.Rows, cfg.ColumnMapping) + + // Apply a saved profile (keyed by header fingerprint), if any. + fingerprint := csvpkg.HeaderFingerprint(pf.Headers) + var profileID pgtype.UUID + if prof, perr := s.Queries.GetCSVImportProfileByFingerprint(ctx, db.GetCSVImportProfileByFingerprintParams{ + UserID: userID, + HeaderFingerprint: fingerprint, + }); perr == nil { + profileID = prof.ID + if m := decodeMapping(prof.ColumnMapping); len(m) > 0 { + cfg.ColumnMapping = m + } + cfg.DateFormat = prof.DateFormat + cfg.PositiveIsDebit = prof.PositiveIsDebit + cfg.HasDebitCredit = prof.HasDebitCredit + if prof.IsoCurrencyCode != "" { + currency = prof.IsoCurrencyCode + } + if prof.DetectedTemplate.Valid { + detectedTemplate = prof.DetectedTemplate.String + } + } else if !errors.Is(perr, pgx.ErrNoRows) { + s.Logger.Warn("csv import: profile lookup failed", "error", perr) + } + + headersJSON, _ := json.Marshal(pf.Headers) + mappingJSON, _ := json.Marshal(cfg.ColumnMapping) + sum := sha256.Sum256(p.Data) + + sess, err := s.Queries.CreateCSVImportSession(ctx, db.CreateCSVImportSessionParams{ + UserID: userID, + Status: importStatusAnalyzing, + Filename: p.Filename, + FileSha256: hex.EncodeToString(sum[:]), + Delimiter: string(pf.Delimiter), + Headers: headersJSON, + RawBlob: p.Data, + RowCount: int32(len(pf.Rows)), + DetectedTemplate: pgconv.TextIfNotEmpty(detectedTemplate), + ColumnMapping: mappingJSON, + DateFormat: cfg.DateFormat, + PositiveIsDebit: cfg.PositiveIsDebit, + HasDebitCredit: cfg.HasDebitCredit, + IsoCurrencyCode: currency, + ProfileID: profileID, + ExpiresAt: pgconv.Timestamptz(time.Now().Add(importSessionTTL)), + }) + if err != nil { + return nil, fmt.Errorf("create session: %w", err) + } + + parsed := ParseCSVRows(cfg, pf.Rows) + + // Account detection. + suggestion, err := s.MatchCSVAccounts(ctx, userID, CSVDetectionSignals{ + Filename: p.Filename, + Headers: pf.Headers, + DetectedTemplate: detectedTemplate, + Mask: csvpkg.ExtractMask(p.Filename, pf.Headers, sampleRaw(pf.Rows, 50)), + }, parsed) + if err != nil { + return nil, err + } + + // If we can confidently pick an account, resolve + classify now so the + // preview is immediately ready; otherwise stage the rows pending an account. + if suggestion.Preselect != "" { + if _, err := s.resolveAndClassify(ctx, sess, suggestion.Preselect, currency, cfg, pf.Rows); err != nil { + return nil, err + } + } else { + if err := s.stageNeedsAccount(ctx, sess.ID, cfg, pf.Rows); err != nil { + return nil, err + } + _ = s.Queries.UpdateCSVImportSessionStatus(ctx, db.UpdateCSVImportSessionStatusParams{ + ID: sess.ID, + Status: importStatusAwaitingAccount, + }) + } + + out, err := s.GetImportSession(ctx, sess.ShortID) + if err != nil { + return nil, err + } + summary, err := s.importSummary(ctx, sess.ID) + if err != nil { + return nil, err + } + return &ImportAnalysis{Session: out, Accounts: suggestion, Summary: summary}, nil +} + +// ResolveImportAccountParams selects (or creates) the target account. +type ResolveImportAccountParams struct { + AccountID string // existing account (UUID or short id) + + // Create-new (used when AccountID is empty). + CreateNew bool + NewName string + NewType string + NewSubtype string + NewCurrency string +} + +// ResolveImportAccount binds the session to an account (existing or newly +// created) and reclassifies every row against it. Status → previewed. +func (s *Service) ResolveImportAccount(ctx context.Context, sessionIDOrShort string, p ResolveImportAccountParams) (*ImportSession, error) { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return nil, err + } + rows, err := s.rawRows(sess) + if err != nil { + return nil, err + } + cfg := sessionParseConfig(sess) + + accountID := p.AccountID + currency := sess.IsoCurrencyCode + if p.CreateNew { + name := strings.TrimSpace(p.NewName) + if name == "" { + name = "CSV Import" + } + cur := p.NewCurrency + if cur == "" { + cur = currency + } + _, acctID, err := s.createCSVConnectionAccount(ctx, sess.UserID, name, cur, orDefault(p.NewType, "depository"), orDefault(p.NewSubtype, "checking")) + if err != nil { + return nil, err + } + accountID = formatUUID(acctID) + currency = cur + } else { + // Adopt the chosen account's currency when the file didn't detect one. + if accID, err := s.resolveAccountID(ctx, accountID); err == nil { + if acc, err := s.Queries.GetAccount(ctx, accID); err == nil { + if c := pgconv.TextOr(acc.IsoCurrencyCode, ""); c != "" { + currency = c + } + } + } + } + + if _, err := s.resolveAndClassify(ctx, sess, accountID, currency, cfg, rows); err != nil { + return nil, err + } + return s.GetImportSession(ctx, sess.ShortID) +} + +// resolveAndClassify binds the account on the session, classifies all rows +// against it, and persists them. Returns the number of rows persisted. +func (s *Service) resolveAndClassify(ctx context.Context, sess db.CsvImportSession, accountIDOrStr, currency string, cfg CSVParseConfig, rows [][]string) (int, error) { + accountID, err := s.resolveAccountID(ctx, accountIDOrStr) + if err != nil { + return 0, fmt.Errorf("resolve account: %w", err) + } + acctStr := formatUUID(accountID) + + acc, err := s.Queries.GetAccount(ctx, accountID) + if err != nil { + return 0, fmt.Errorf("get account: %w", err) + } + + parsed := ParseCSVRows(cfg, rows) + classified, err := s.ClassifyCSVRows(ctx, accountID, acctStr, parsed, DefaultDedupToleranceDays) + if err != nil { + return 0, err + } + + tx, err := s.Pool.Begin(ctx) + if err != nil { + return 0, err + } + defer tx.Rollback(ctx) + q := s.Queries.WithTx(tx) + + if _, err := q.ResolveCSVImportSessionAccount(ctx, db.ResolveCSVImportSessionAccountParams{ + ID: sess.ID, + ResolvedAccountID: accountID, + ResolvedConnectionID: acc.ConnectionID, + IsoCurrencyCode: currency, + Status: importStatusPreviewed, + }); err != nil { + return 0, err + } + if err := q.DeleteCSVImportRows(ctx, sess.ID); err != nil { + return 0, err + } + if _, err := q.CreateCSVImportRows(ctx, classifiedToParams(sess.ID, classified)); err != nil { + return 0, err + } + if err := tx.Commit(ctx); err != nil { + return 0, err + } + return len(classified), nil +} + +// stageNeedsAccount persists parsed rows in the needs_account state (no account +// resolved yet) so the row count + parse errors are visible before resolution. +func (s *Service) stageNeedsAccount(ctx context.Context, sessionID pgtype.UUID, cfg CSVParseConfig, rows [][]string) error { + parsed := ParseCSVRows(cfg, rows) + staged := make([]CSVClassifiedRow, 0, len(parsed)) + for _, pr := range parsed { + cr := CSVClassifiedRow{CSVParsedRow: pr, Include: pr.ParseError == ""} + if pr.ParseError != "" { + cr.Classification = CSVRowError + } else { + cr.Classification = CSVRowNeedsAccount + cr.ContentHash = csvpkg.GenerateContentHash(pr.Date, pr.Amount, pr.Desc) + } + staged = append(staged, cr) + } + tx, err := s.Pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + q := s.Queries.WithTx(tx) + if err := q.DeleteCSVImportRows(ctx, sessionID); err != nil { + return err + } + if _, err := q.CreateCSVImportRows(ctx, classifiedToParams(sessionID, staged)); err != nil { + return err + } + return tx.Commit(ctx) +} + +// ListImportRows returns a page of staged rows plus the full per-classification +// summary. classFilter "" returns all classifications. +func (s *Service) ListImportRows(ctx context.Context, sessionIDOrShort, classFilter string, page, pageSize int) ([]ImportRowView, ImportSummary, error) { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return nil, ImportSummary{}, err + } + if page < 1 { + page = 1 + } + if pageSize <= 0 || pageSize > 500 { + pageSize = 100 + } + rows, err := s.Queries.GetCSVImportRowsPage(ctx, db.GetCSVImportRowsPageParams{ + SessionID: sess.ID, + Column2: classFilter, + Limit: int32(pageSize), + Offset: int32((page - 1) * pageSize), + }) + if err != nil { + return nil, ImportSummary{}, err + } + views := make([]ImportRowView, len(rows)) + for i, r := range rows { + views[i] = rowToView(r) + } + summary, err := s.importSummary(ctx, sess.ID) + if err != nil { + return nil, ImportSummary{}, err + } + return views, summary, nil +} + +// EditImportRowParams carries the editable fields of a single row. +type EditImportRowParams struct { + Date string // "2006-01-02" + Amount string // decimal string + Desc string + Merchant string + Include *bool +} + +// EditImportRow updates a single row's parsed fields and reclassifies just that +// row against the resolved account. +func (s *Service) EditImportRow(ctx context.Context, sessionIDOrShort, rowID string, p EditImportRowParams) (*ImportRowView, error) { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return nil, err + } + if !sess.ResolvedAccountID.Valid { + return nil, errors.New("resolve an account before editing rows") + } + rowUUID, err := pgconv.ParseUUID(rowID) + if err != nil { + return nil, fmt.Errorf("invalid row id: %w", err) + } + existing, err := s.Queries.GetCSVImportRow(ctx, rowUUID) + if err != nil { + return nil, fmt.Errorf("get row: %w", err) + } + + pr := CSVParsedRow{RowIndex: int(existing.RowIndex)} + _ = json.Unmarshal(existing.Raw, &pr.Raw) + + date, derr := time.Parse("2006-01-02", strings.TrimSpace(p.Date)) + amount, aerr := decimal.NewFromString(strings.TrimSpace(p.Amount)) + pr.Desc = strings.TrimSpace(p.Desc) + pr.Merchant = strings.TrimSpace(p.Merchant) + switch { + case derr != nil: + pr.ParseError = "unparseable date" + case aerr != nil: + pr.ParseError = "unparseable amount" + case pr.Desc == "": + pr.ParseError = "empty description" + default: + pr.Date, pr.Amount = date, amount + } + + acctStr := formatUUID(sess.ResolvedAccountID) + classified, err := s.ClassifyCSVRows(ctx, sess.ResolvedAccountID, acctStr, []CSVParsedRow{pr}, DefaultDedupToleranceDays) + if err != nil { + return nil, err + } + cr := classified[0] + + include := cr.Include + if p.Include != nil { + include = *p.Include + } + + updated, err := s.Queries.UpdateCSVImportRow(ctx, db.UpdateCSVImportRowParams{ + ID: rowUUID, + ParsedDate: dateOrNull(cr.Date, cr.ParseError == ""), + ParsedAmount: numericOrNull(cr.Amount, cr.ParseError == ""), + ParsedDesc: pgconv.TextIfNotEmpty(cr.Desc), + ParsedMerchant: pgconv.TextIfNotEmpty(cr.Merchant), + ParsedCategory: pgconv.TextIfNotEmpty(cr.Category), + Classification: string(cr.Classification), + MatchTxnID: optionalUUID(cr.MatchTxnID), + MatchScore: pgconv.Int4(int32(cr.MatchScore)), + MatchReason: pgconv.TextIfNotEmpty(cr.MatchReason), + ParseError: pgconv.TextIfNotEmpty(cr.ParseError), + ContentHash: pgconv.TextIfNotEmpty(cr.ContentHash), + ProviderTxnID: pgconv.TextIfNotEmpty(cr.ProviderTxnID), + Include: include, + CategoryID: existing.CategoryID, + }) + if err != nil { + return nil, err + } + v := rowToView(updated) + return &v, nil +} + +// ImportBulkOp describes a bulk preview action. +type ImportBulkOp struct { + Op string // "include" | "exclude" | "set_category" | "remap" + Classification string // for include/exclude scoping ("" = all dup classes) + CategoryID string // for set_category + + // for remap + ColumnMapping map[string]int + DateFormat string + PositiveIsDebit bool + HasDebitCredit bool +} + +// BulkImportOp applies a bulk action to a session's staged rows. +func (s *Service) BulkImportOp(ctx context.Context, sessionIDOrShort string, op ImportBulkOp) error { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return err + } + switch op.Op { + case "include", "exclude": + include := op.Op == "include" + classes := []string{op.Classification} + if op.Classification == "" { + classes = []string{string(CSVRowExactDup), string(CSVRowProbableDup)} + } + for _, c := range classes { + if err := s.Queries.SetCSVImportRowsIncludeByClassification(ctx, db.SetCSVImportRowsIncludeByClassificationParams{ + SessionID: sess.ID, + Classification: c, + Include: include, + }); err != nil { + return err + } + } + return nil + case "set_category": + var catID pgtype.UUID + if op.CategoryID != "" { + id, err := s.resolveCategoryID(ctx, op.CategoryID) + if err != nil { + return fmt.Errorf("resolve category: %w", err) + } + catID = id + } + return s.Queries.SetCSVImportRowsCategoryAll(ctx, db.SetCSVImportRowsCategoryAllParams{ + SessionID: sess.ID, + CategoryID: catID, + }) + case "remap": + if !sess.ResolvedAccountID.Valid { + return errors.New("resolve an account before remapping") + } + mappingJSON, _ := json.Marshal(op.ColumnMapping) + if _, err := s.Queries.UpdateCSVImportSessionParse(ctx, db.UpdateCSVImportSessionParseParams{ + ID: sess.ID, + ColumnMapping: mappingJSON, + DateFormat: op.DateFormat, + PositiveIsDebit: op.PositiveIsDebit, + HasDebitCredit: op.HasDebitCredit, + DetectedTemplate: sess.DetectedTemplate, + }); err != nil { + return err + } + fresh, err := s.Queries.GetCSVImportSession(ctx, sess.ID) + if err != nil { + return err + } + rows, err := s.rawRows(fresh) + if err != nil { + return err + } + _, err = s.resolveAndClassify(ctx, fresh, formatUUID(fresh.ResolvedAccountID), fresh.IsoCurrencyCode, sessionParseConfig(fresh), rows) + return err + default: + return fmt.Errorf("unknown bulk op %q", op.Op) + } +} + +// ApplyImportSession upserts the exact included set into the resolved account in +// one transaction, idempotently, and saves/updates the source profile. +func (s *Service) ApplyImportSession(ctx context.Context, sessionIDOrShort string, actor Actor) (*CSVImportResult, error) { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return nil, err + } + if sess.Status == importStatusApplied { + return nil, errors.New("session already applied") + } + if !sess.ResolvedAccountID.Valid || !sess.ResolvedConnectionID.Valid { + return nil, errors.New("resolve an account before applying") + } + acctStr := formatUUID(sess.ResolvedAccountID) + + included, err := s.Queries.ListIncludedCSVImportRows(ctx, sess.ID) + if err != nil { + return nil, err + } + + var uncategorizedID pgtype.UUID + _ = s.Pool.QueryRow(ctx, "SELECT id FROM categories WHERE slug = 'uncategorized'").Scan(&uncategorizedID) + + tx, err := s.Pool.Begin(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback(ctx) + q := s.Queries.WithTx(tx) + + now := time.Now().UTC() + syncLog, err := q.CreateSyncLog(ctx, db.CreateSyncLogParams{ + ConnectionID: sess.ResolvedConnectionID, + Trigger: db.SyncTriggerManual, + Status: db.SyncStatusInProgress, + StartedAt: pgconv.Timestamptz(now), + }) + if err != nil { + return nil, fmt.Errorf("create sync log: %w", err) + } + + result := &CSVImportResult{ + ConnectionID: formatUUID(sess.ResolvedConnectionID), + AccountID: acctStr, + TotalRows: int(sess.RowCount), + } + + seen := map[string]int{} // provider_txn_id base → next occurrence + for _, r := range included { + amount := numericToDecimal(r.ParsedAmount) + desc := pgconv.TextOr(r.ParsedDesc, "") + date := r.ParsedDate.Time + + // Disambiguate genuine same-key duplicates so "import anyway" inserts a + // distinct row instead of no-op upserting onto the original. + base := pgconv.TextOr(r.ProviderTxnID, csvpkg.GenerateExternalID(acctStr, date, amount, desc)) + occ := seen[base] + if r.Classification == string(CSVRowExactDup) && occ == 0 { + occ = 1 // explicit re-import of an identical row → force a new id + } + seen[base] = occ + 1 + providerTxnID := csvpkg.GenerateExternalIDWithOccurrence(acctStr, date, amount, desc, occ) + + categoryID := uncategorizedID + if r.CategoryID.Valid { + categoryID = r.CategoryID + } + + var amountNumeric pgtype.Numeric + _ = amountNumeric.Scan(amount.String()) + + up, err := q.UpsertTransactionV2(ctx, db.UpsertTransactionV2Params{ + AccountID: sess.ResolvedAccountID, + ProviderTransactionID: providerTxnID, + Amount: amountNumeric, + IsoCurrencyCode: pgconv.Text(sess.IsoCurrencyCode), + Date: pgconv.Date(date), + ProviderName: desc, + ProviderMerchantName: r.ParsedMerchant, + ProviderCategoryPrimary: r.ParsedCategory, + ProviderPaymentChannel: pgconv.Text("other"), + Pending: false, + CategoryID: categoryID, + ProviderRaw: r.Raw, + ContentHash: pgconv.TextIfNotEmpty(pgconv.TextOr(r.ContentHash, csvpkg.GenerateContentHash(date, amount, desc))), + }) + if err != nil { + result.SkippedCount++ + result.SkipReasons = append(result.SkipReasons, fmt.Sprintf("row %d: %s", r.RowIndex+2, err.Error())) + continue + } + if up.Inserted { + result.NewCount++ + } else { + result.UpdatedCount++ + } + } + + syncStatus := db.SyncStatusSuccess + var errMsg pgtype.Text + if result.NewCount+result.UpdatedCount == 0 && result.SkippedCount > 0 { + syncStatus = db.SyncStatusError + errMsg = pgconv.Text(fmt.Sprintf("all %d rows skipped", result.SkippedCount)) + } + if err := q.UpdateSyncLog(ctx, db.UpdateSyncLogParams{ + ID: syncLog.ID, + Status: syncStatus, + CompletedAt: pgconv.Timestamptz(time.Now().UTC()), + AddedCount: int32(result.NewCount), + ModifiedCount: int32(result.UpdatedCount), + RemovedCount: 0, + ErrorMessage: errMsg, + }); err != nil { + s.Logger.Error("csv import: update sync log", "error", err) + } + + // Save / refresh the source profile so future imports of this layout are + // one click. Best-effort — never fails the apply. + if err := s.upsertProfileFromSession(ctx, q, sess); err != nil { + s.Logger.Warn("csv import: profile upsert failed", "error", err) + } + + resultJSON, _ := json.Marshal(result) + if err := q.FinalizeCSVImportSession(ctx, db.FinalizeCSVImportSessionParams{ + ID: sess.ID, + Result: resultJSON, + SyncLogID: syncLog.ID, + }); err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, err + } + return result, nil +} + +// SweepExpiredImportSessions deletes un-applied sessions past their TTL. CASCADE +// removes their staged rows; the raw_blob goes with them. Returns the count. +func (s *Service) SweepExpiredImportSessions(ctx context.Context) (int, error) { + expired, err := s.Queries.ListExpiredCSVImportSessions(ctx) + if err != nil { + return 0, err + } + n := 0 + for _, sess := range expired { + if err := s.Queries.DeleteCSVImportSession(ctx, sess.ID); err != nil { + s.Logger.Warn("csv import: sweep delete failed", "session", sess.ShortID, "error", err) + continue + } + n++ + } + return n, nil +} + +// GetImportSession returns the service view of a session by id or short id. +func (s *Service) GetImportSession(ctx context.Context, sessionIDOrShort string) (*ImportSession, error) { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return nil, err + } + v := sessionToView(sess) + return &v, nil +} + +// --- helpers --- + +func (s *Service) getSession(ctx context.Context, idOrShort string) (db.CsvImportSession, error) { + if len(idOrShort) == 8 { + sess, err := s.Queries.GetCSVImportSessionByShortID(ctx, idOrShort) + if err != nil { + return db.CsvImportSession{}, ErrImportSessionNotFound + } + return sess, nil + } + uid, err := pgconv.ParseUUID(idOrShort) + if err != nil { + return db.CsvImportSession{}, ErrImportSessionNotFound + } + sess, err := s.Queries.GetCSVImportSession(ctx, uid) + if err != nil { + return db.CsvImportSession{}, ErrImportSessionNotFound + } + return sess, nil +} + +func (s *Service) importSummary(ctx context.Context, sessionID pgtype.UUID) (ImportSummary, error) { + counts, err := s.Queries.CountCSVImportRowsByClassification(ctx, sessionID) + if err != nil { + return ImportSummary{}, err + } + out := ImportSummary{Counts: map[string]int{}} + for _, c := range counts { + out.Counts[c.Classification] = int(c.Count) + out.Total += int(c.Count) + } + included, err := s.Queries.CountIncludedCSVImportRows(ctx, sessionID) + if err != nil { + return ImportSummary{}, err + } + out.IncludedCount = int(included) + return out, nil +} + +// rawRows re-parses the stored file blob into rows. +func (s *Service) rawRows(sess db.CsvImportSession) ([][]string, error) { + if len(sess.RawBlob) == 0 { + return nil, errors.New("session file is no longer available") + } + pf, err := csvpkg.ParseFile(sess.RawBlob) + if err != nil { + return nil, fmt.Errorf("re-parse csv: %w", err) + } + return pf.Rows, nil +} + +// createCSVConnectionAccount mints a fresh CSV connection + account (generalized +// from the legacy ImportCSV new-import branch). +func (s *Service) createCSVConnectionAccount(ctx context.Context, userID pgtype.UUID, name, currency, accType, subtype string) (pgtype.UUID, pgtype.UUID, error) { + b := make([]byte, 16) + _, _ = rand.Read(b) + externalID := fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) + + conn, err := s.Queries.CreateBankConnection(ctx, db.CreateBankConnectionParams{ + UserID: userID, + Provider: db.ProviderTypeCsv, + InstitutionName: pgconv.Text(name), + ExternalID: pgconv.Text(externalID), + Status: db.ConnectionStatusActive, + }) + if err != nil { + return pgtype.UUID{}, pgtype.UUID{}, fmt.Errorf("create connection: %w", err) + } + acct, err := s.Queries.UpsertAccount(ctx, db.UpsertAccountParams{ + ConnectionID: conn.ID, + ExternalAccountID: externalID, + Name: name, + Type: accType, + Subtype: pgconv.Text(subtype), + IsoCurrencyCode: pgconv.Text(currency), + }) + if err != nil { + return pgtype.UUID{}, pgtype.UUID{}, fmt.Errorf("create account: %w", err) + } + return conn.ID, acct.ID, nil +} + +func (s *Service) upsertProfileFromSession(ctx context.Context, q *db.Queries, sess db.CsvImportSession) error { + headers := decodeHeaders(sess.Headers) + name := strings.TrimSpace(sess.Filename) + if name == "" { + name = "CSV import" + } + _, err := q.UpsertCSVImportProfile(ctx, db.UpsertCSVImportProfileParams{ + UserID: sess.UserID, + Name: name, + HeaderFingerprint: csvpkg.HeaderFingerprint(headers), + Headers: sess.Headers, + DetectedTemplate: sess.DetectedTemplate, + ColumnMapping: sess.ColumnMapping, + DateFormat: sess.DateFormat, + Delimiter: sess.Delimiter, + PositiveIsDebit: sess.PositiveIsDebit, + HasDebitCredit: sess.HasDebitCredit, + IsoCurrencyCode: sess.IsoCurrencyCode, + DefaultAccountID: sess.ResolvedAccountID, + }) + return err +} + +func classifiedToParams(sessionID pgtype.UUID, rows []CSVClassifiedRow) []db.CreateCSVImportRowsParams { + out := make([]db.CreateCSVImportRowsParams, len(rows)) + for i, cr := range rows { + rawJSON, _ := json.Marshal(cr.Raw) + ok := cr.ParseError == "" + out[i] = db.CreateCSVImportRowsParams{ + SessionID: sessionID, + RowIndex: int32(cr.RowIndex), + Raw: rawJSON, + ParsedDate: dateOrNull(cr.Date, ok), + ParsedAmount: numericOrNull(cr.Amount, ok), + ParsedDesc: pgconv.TextIfNotEmpty(cr.Desc), + ParsedMerchant: pgconv.TextIfNotEmpty(cr.Merchant), + ParsedCategory: pgconv.TextIfNotEmpty(cr.Category), + Classification: string(cr.Classification), + MatchTxnID: optionalUUID(cr.MatchTxnID), + MatchScore: pgconv.Int4(int32(cr.MatchScore)), + MatchReason: pgconv.TextIfNotEmpty(cr.MatchReason), + ParseError: pgconv.TextIfNotEmpty(cr.ParseError), + ContentHash: pgconv.TextIfNotEmpty(cr.ContentHash), + ProviderTxnID: pgconv.TextIfNotEmpty(cr.ProviderTxnID), + Include: cr.Include, + UserEdited: false, + } + } + return out +} + +func rowToView(r db.CsvImportRow) ImportRowView { + var raw []string + _ = json.Unmarshal(r.Raw, &raw) + v := ImportRowView{ + ID: formatUUID(r.ID), + RowIndex: int(r.RowIndex), + Raw: raw, + Desc: pgconv.TextOr(r.ParsedDesc, ""), + Merchant: pgconv.TextOr(r.ParsedMerchant, ""), + Category: pgconv.TextOr(r.ParsedCategory, ""), + Classification: r.Classification, + MatchReason: pgconv.TextOr(r.MatchReason, ""), + ParseError: pgconv.TextOr(r.ParseError, ""), + Include: r.Include, + UserEdited: r.UserEdited, + } + if r.MatchTxnID.Valid { + v.MatchTxnID = formatUUID(r.MatchTxnID) + } + if r.MatchScore.Valid { + v.MatchScore = int(r.MatchScore.Int32) + } + if r.ParsedDate.Valid { + v.Date = r.ParsedDate.Time.Format("2006-01-02") + } + if r.ParsedAmount.Valid { + v.Amount = numericToDecimal(r.ParsedAmount).String() + } + return v +} + +func sessionToView(s db.CsvImportSession) ImportSession { + v := ImportSession{ + ID: formatUUID(s.ID), + ShortID: s.ShortID, + UserID: formatUUID(s.UserID), + Status: s.Status, + Filename: s.Filename, + Delimiter: s.Delimiter, + Headers: decodeHeaders(s.Headers), + RowCount: int(s.RowCount), + DetectedTemplate: pgconv.TextOr(s.DetectedTemplate, ""), + ColumnMapping: decodeMapping(s.ColumnMapping), + DateFormat: s.DateFormat, + PositiveIsDebit: s.PositiveIsDebit, + HasDebitCredit: s.HasDebitCredit, + IsoCurrencyCode: s.IsoCurrencyCode, + } + if s.ResolvedAccountID.Valid { + v.ResolvedAccountID = formatUUID(s.ResolvedAccountID) + } + if s.ResolvedConnectionID.Valid { + v.ResolvedConnectionID = formatUUID(s.ResolvedConnectionID) + } + if s.ProfileID.Valid { + v.ProfileID = formatUUID(s.ProfileID) + } + if len(s.Result) > 0 { + var res CSVImportResult + if json.Unmarshal(s.Result, &res) == nil { + v.Result = &res + } + } + return v +} + +func sessionParseConfig(s db.CsvImportSession) CSVParseConfig { + return CSVParseConfig{ + ColumnMapping: decodeMapping(s.ColumnMapping), + DateFormat: s.DateFormat, + PositiveIsDebit: s.PositiveIsDebit, + HasDebitCredit: s.HasDebitCredit, + } +} + +// detectParseConfig builds a parse config from a parsed file using the bank +// template (if matched) or generic column detection. Returns the template name. +func detectParseConfig(pf *csvpkg.ParsedFile) (CSVParseConfig, string) { + cfg := CSVParseConfig{ColumnMapping: map[string]int{}} + if t := csvpkg.DetectTemplate(pf.Headers); t != nil && t.HeaderPatterns != nil { + idx := func(name string) (int, bool) { + for i, h := range pf.Headers { + if strings.EqualFold(strings.TrimSpace(h), name) { + return i, true + } + } + return 0, false + } + if i, ok := idx(t.DateColumn); ok { + cfg.ColumnMapping["date"] = i + } + if i, ok := idx(t.AmountColumn); ok { + cfg.ColumnMapping["amount"] = i + } + if i, ok := idx(t.DescriptionColumn); ok { + cfg.ColumnMapping["description"] = i + } + if t.CategoryColumn != "" { + if i, ok := idx(t.CategoryColumn); ok { + cfg.ColumnMapping["category"] = i + } + } + if t.MerchantColumn != "" { + if i, ok := idx(t.MerchantColumn); ok { + cfg.ColumnMapping["merchant_name"] = i + } + } + if t.HasDebitCredit { + if i, ok := idx(t.DebitColumn); ok { + cfg.ColumnMapping["debit"] = i + } + if i, ok := idx(t.CreditColumn); ok { + cfg.ColumnMapping["credit"] = i + } + } + cfg.DateFormat = t.DateFormat + cfg.PositiveIsDebit = t.PositiveIsDebit + cfg.HasDebitCredit = t.HasDebitCredit + return cfg, t.Name + } + + cfg.ColumnMapping = csvpkg.DetectColumns(pf.Headers) + if dateCol, ok := cfg.ColumnMapping["date"]; ok { + samples := make([]string, 0, 20) + for _, r := range pf.Rows { + if dateCol < len(r) { + samples = append(samples, r[dateCol]) + } + if len(samples) >= 20 { + break + } + } + if df, err := csvpkg.DetectDateFormat(samples); err == nil { + cfg.DateFormat = df + } + } + return cfg, "" +} + +// detectCurrency guesses the ISO currency from symbols in the amount column, +// defaulting to USD. +func detectCurrency(rows [][]string, mapping map[string]int) string { + col, ok := mapping["amount"] + if !ok { + return "USD" + } + for i, r := range rows { + if i >= 50 || col >= len(r) { + continue + } + switch { + case strings.Contains(r[col], "€"): + return "EUR" + case strings.Contains(r[col], "£"): + return "GBP" + case strings.Contains(r[col], "¥"): + return "JPY" + } + } + return "USD" +} + +func decodeHeaders(b []byte) []string { + var h []string + _ = json.Unmarshal(b, &h) + return h +} + +func decodeMapping(b []byte) map[string]int { + m := map[string]int{} + _ = json.Unmarshal(b, &m) + return m +} + +func sampleRaw(rows [][]string, n int) [][]string { + if len(rows) <= n { + return rows + } + return rows[:n] +} + +func dateOrNull(t time.Time, ok bool) pgtype.Date { + if !ok { + return pgtype.Date{} + } + return pgconv.Date(t) +} + +func numericOrNull(d decimal.Decimal, ok bool) pgtype.Numeric { + if !ok { + return pgtype.Numeric{} + } + var n pgtype.Numeric + _ = n.Scan(d.String()) + return n +} + +func optionalUUID(idStr string) pgtype.UUID { + if idStr == "" { + return pgtype.UUID{} + } + u, err := pgconv.ParseUUID(idStr) + if err != nil { + return pgtype.UUID{} + } + return u +} + +func orDefault(v, def string) string { + if strings.TrimSpace(v) == "" { + return def + } + return v +} diff --git a/internal/service/csv_import_v2_integration_test.go b/internal/service/csv_import_v2_integration_test.go new file mode 100644 index 00000000..332256fe --- /dev/null +++ b/internal/service/csv_import_v2_integration_test.go @@ -0,0 +1,178 @@ +//go:build integration && !lite + +package service_test + +import ( + "context" + "testing" + + "breadbox/internal/pgconv" + "breadbox/internal/service" + "breadbox/internal/testutil" +) + +func countTxns(t *testing.T, svc *service.Service, accountID string) int { + t.Helper() + uid, err := pgconv.ParseUUID(accountID) + if err != nil { + t.Fatalf("parse account id: %v", err) + } + var n int + if err := svc.Pool.QueryRow(context.Background(), + "SELECT count(*) FROM transactions WHERE account_id = $1 AND deleted_at IS NULL", uid).Scan(&n); err != nil { + t.Fatalf("count txns: %v", err) + } + return n +} + +func TestImportV2_FullLifecycleAndProfileRedrop(t *testing.T) { + svc, q, _ := newService(t) + ctx := context.Background() + user := testutil.MustCreateUser(t, q, "Alice") + + file1 := []byte("Date,Amount,Description\n2026-07-01,10.00,COFFEE\n2026-07-02,20.00,LUNCH\n") + + // Analyze — no accounts yet, so it awaits account resolution. + an, err := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "chase.csv", Data: file1, + }) + if err != nil { + t.Fatalf("CreateImportSession: %v", err) + } + if an.Session.Status != "awaiting_account" { + t.Fatalf("status = %q, want awaiting_account", an.Session.Status) + } + + // Resolve by creating a new account. + sess, err := svc.ResolveImportAccount(ctx, an.Session.ShortID, service.ResolveImportAccountParams{ + CreateNew: true, NewName: "Chase Checking", + }) + if err != nil { + t.Fatalf("ResolveImportAccount: %v", err) + } + if sess.Status != "previewed" || sess.ResolvedAccountID == "" { + t.Fatalf("after resolve: status=%q account=%q", sess.Status, sess.ResolvedAccountID) + } + acctA := sess.ResolvedAccountID + + _, summary, err := svc.ListImportRows(ctx, sess.ShortID, "", 1, 100) + if err != nil { + t.Fatalf("ListImportRows: %v", err) + } + if summary.Counts["new"] != 2 || summary.IncludedCount != 2 { + t.Fatalf("summary = %+v, want 2 new/included", summary) + } + + // Apply. + res, err := svc.ApplyImportSession(ctx, sess.ShortID, service.SystemActor()) + if err != nil { + t.Fatalf("ApplyImportSession: %v", err) + } + if res.NewCount != 2 || res.UpdatedCount != 0 { + t.Fatalf("apply result = %+v, want 2 new", res) + } + if n := countTxns(t, svc, acctA); n != 2 { + t.Fatalf("account has %d txns, want 2", n) + } + + // Re-drop the same file plus one new row. The saved profile should now + // auto-resolve to the same account, dedupe the two, and stage one new row. + file2 := []byte("Date,Amount,Description\n2026-07-01,10.00,COFFEE\n2026-07-02,20.00,LUNCH\n2026-07-03,30.00,DINNER\n") + an2, err := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "chase.csv", Data: file2, + }) + if err != nil { + t.Fatalf("CreateImportSession redrop: %v", err) + } + if an2.Session.Status != "previewed" { + t.Fatalf("redrop status = %q, want previewed (profile auto-resolve)", an2.Session.Status) + } + if an2.Session.ResolvedAccountID != acctA { + t.Fatalf("redrop resolved to %q, want original account %q", an2.Session.ResolvedAccountID, acctA) + } + if an2.Summary.Counts["exact_dup"] != 2 || an2.Summary.Counts["new"] != 1 { + t.Fatalf("redrop summary = %+v, want 2 exact_dup + 1 new", an2.Summary) + } + + res2, err := svc.ApplyImportSession(ctx, an2.Session.ShortID, service.SystemActor()) + if err != nil { + t.Fatalf("apply redrop: %v", err) + } + if res2.NewCount != 1 { + t.Fatalf("redrop apply = %+v, want 1 new", res2) + } + if n := countTxns(t, svc, acctA); n != 3 { + t.Fatalf("after redrop account has %d txns, want 3", n) + } +} + +func TestImportV2_ApplyIsIdempotent(t *testing.T) { + svc, q, _ := newService(t) + ctx := context.Background() + user := testutil.MustCreateUser(t, q, "Alice") + file := []byte("Date,Amount,Description\n2026-08-01,5.00,A\n2026-08-02,6.00,B\n") + + an, err := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "f.csv", Data: file, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + sess, err := svc.ResolveImportAccount(ctx, an.Session.ShortID, service.ResolveImportAccountParams{CreateNew: true, NewName: "Acct"}) + if err != nil { + t.Fatalf("resolve: %v", err) + } + if _, err := svc.ApplyImportSession(ctx, sess.ShortID, service.SystemActor()); err != nil { + t.Fatalf("apply1: %v", err) + } + // A fresh session over the same file into the same account must add nothing. + an2, err := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "f.csv", Data: file, + }) + if err != nil { + t.Fatalf("create2: %v", err) + } + res2, err := svc.ApplyImportSession(ctx, an2.Session.ShortID, service.SystemActor()) + if err != nil { + t.Fatalf("apply2: %v", err) + } + if res2.NewCount != 0 { + t.Fatalf("second apply added %d rows, want 0 (idempotent)", res2.NewCount) + } + if n := countTxns(t, svc, sess.ResolvedAccountID); n != 2 { + t.Fatalf("account has %d txns after double import, want 2", n) + } +} + +func TestImportV2_AccountChangeBeforeConfirm(t *testing.T) { + svc, q, _ := newService(t) + ctx := context.Background() + user := testutil.MustCreateUser(t, q, "Alice") + conn := testutil.MustCreateConnection(t, q, user.ID, "item_1") + accA := testutil.MustCreateAccount(t, q, conn.ID, "extA", "Account A") + accB := testutil.MustCreateAccount(t, q, conn.ID, "extB", "Account B") + + file := []byte("Date,Amount,Description\n2026-09-01,12.00,X\n") + an, err := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "f.csv", Data: file, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + // Resolve to A, then change to B before applying. + if _, err := svc.ResolveImportAccount(ctx, an.Session.ShortID, service.ResolveImportAccountParams{AccountID: pgconv.FormatUUID(accA.ID)}); err != nil { + t.Fatalf("resolve A: %v", err) + } + if _, err := svc.ResolveImportAccount(ctx, an.Session.ShortID, service.ResolveImportAccountParams{AccountID: pgconv.FormatUUID(accB.ID)}); err != nil { + t.Fatalf("resolve B: %v", err) + } + if _, err := svc.ApplyImportSession(ctx, an.Session.ShortID, service.SystemActor()); err != nil { + t.Fatalf("apply: %v", err) + } + if n := countTxns(t, svc, pgconv.FormatUUID(accB.ID)); n != 1 { + t.Fatalf("account B has %d txns, want 1", n) + } + if n := countTxns(t, svc, pgconv.FormatUUID(accA.ID)); n != 0 { + t.Fatalf("account A has %d txns, want 0 (import went to B)", n) + } +} diff --git a/internal/service/csv_profiles.go b/internal/service/csv_profiles.go new file mode 100644 index 00000000..ce8c2b01 --- /dev/null +++ b/internal/service/csv_profiles.go @@ -0,0 +1,107 @@ +//go:build !lite + +package service + +import ( + "context" + "errors" + "fmt" + "time" + + "breadbox/internal/db" + "breadbox/internal/pgconv" + + "github.com/jackc/pgx/v5/pgtype" +) + +// ErrProfileNotFound is returned when a profile id/short id doesn't resolve. +var ErrProfileNotFound = errors.New("import profile not found") + +// CSVProfile is the service-layer view of a saved CSV import recipe. +type CSVProfile struct { + ID string `json:"id"` + ShortID string `json:"short_id"` + Name string `json:"name"` + DetectedTemplate string `json:"detected_template"` + DefaultAccountID string `json:"default_account_id"` + DefaultAccountName string `json:"default_account_name"` + IsoCurrencyCode string `json:"iso_currency_code"` + TimesUsed int `json:"times_used"` + LastUsedAt *time.Time `json:"last_used_at"` + Headers []string `json:"headers"` +} + +// ListCSVProfiles returns all saved import profiles, newest-used first, with the +// default account's display name resolved. +func (s *Service) ListCSVProfiles(ctx context.Context) ([]CSVProfile, error) { + rows, err := s.Queries.ListCSVImportProfiles(ctx) + if err != nil { + return nil, fmt.Errorf("list profiles: %w", err) + } + out := make([]CSVProfile, 0, len(rows)) + for _, r := range rows { + out = append(out, s.profileToView(ctx, r)) + } + return out, nil +} + +// RenameCSVProfile updates a profile's display name. +func (s *Service) RenameCSVProfile(ctx context.Context, idOrShort, name string) (*CSVProfile, error) { + id, err := s.resolveProfileID(ctx, idOrShort) + if err != nil { + return nil, err + } + row, err := s.Queries.RenameCSVImportProfile(ctx, db.RenameCSVImportProfileParams{ID: id, Name: name}) + if err != nil { + return nil, fmt.Errorf("rename profile: %w", err) + } + v := s.profileToView(ctx, row) + return &v, nil +} + +// DeleteCSVProfile removes a saved profile. +func (s *Service) DeleteCSVProfile(ctx context.Context, idOrShort string) error { + id, err := s.resolveProfileID(ctx, idOrShort) + if err != nil { + return err + } + return s.Queries.DeleteCSVImportProfile(ctx, id) +} + +func (s *Service) resolveProfileID(ctx context.Context, idOrShort string) (pgtype.UUID, error) { + if len(idOrShort) == 8 { + prof, err := s.Queries.GetCSVImportProfileByShortID(ctx, idOrShort) + if err != nil { + return pgtype.UUID{}, ErrProfileNotFound + } + return prof.ID, nil + } + uid, err := pgconv.ParseUUID(idOrShort) + if err != nil { + return pgtype.UUID{}, ErrProfileNotFound + } + return uid, nil +} + +func (s *Service) profileToView(ctx context.Context, r db.CsvImportProfile) CSVProfile { + v := CSVProfile{ + ID: formatUUID(r.ID), + ShortID: r.ShortID, + Name: r.Name, + DetectedTemplate: pgconv.TextOr(r.DetectedTemplate, ""), + IsoCurrencyCode: r.IsoCurrencyCode, + TimesUsed: int(r.TimesUsed), + Headers: decodeHeaders(r.Headers), + } + if r.LastUsedAt.Valid { + t := r.LastUsedAt.Time + v.LastUsedAt = &t + } + if r.DefaultAccountID.Valid { + v.DefaultAccountID = formatUUID(r.DefaultAccountID) + if acc, err := s.Queries.GetAccount(ctx, r.DefaultAccountID); err == nil { + v.DefaultAccountName = acc.Name + } + } + return v +} diff --git a/internal/service/csv_profiles_integration_test.go b/internal/service/csv_profiles_integration_test.go new file mode 100644 index 00000000..b95c996e --- /dev/null +++ b/internal/service/csv_profiles_integration_test.go @@ -0,0 +1,80 @@ +//go:build integration && !lite + +package service_test + +import ( + "context" + "testing" + + "breadbox/internal/pgconv" + "breadbox/internal/service" + "breadbox/internal/testutil" +) + +func TestCSVProfiles_ListRenameDelete(t *testing.T) { + svc, q, _ := newService(t) + ctx := context.Background() + user := testutil.MustCreateUser(t, q, "Alice") + file := []byte("Date,Amount,Description\n2026-11-01,9.00,A\n") + + // Apply once → a profile is created from the file's header layout. + an, err := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "amex.csv", Data: file, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + sess, err := svc.ResolveImportAccount(ctx, an.Session.ShortID, service.ResolveImportAccountParams{CreateNew: true, NewName: "Amex"}) + if err != nil { + t.Fatalf("resolve: %v", err) + } + if _, err := svc.ApplyImportSession(ctx, sess.ShortID, service.SystemActor()); err != nil { + t.Fatalf("apply: %v", err) + } + + profiles, err := svc.ListCSVProfiles(ctx) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(profiles) != 1 { + t.Fatalf("got %d profiles, want 1", len(profiles)) + } + p := profiles[0] + if p.TimesUsed != 1 { + t.Errorf("times_used = %d, want 1", p.TimesUsed) + } + if p.DefaultAccountName != "Amex" { + t.Errorf("default account name = %q, want Amex", p.DefaultAccountName) + } + + // Rename. + renamed, err := svc.RenameCSVProfile(ctx, p.ShortID, "My Amex") + if err != nil { + t.Fatalf("rename: %v", err) + } + if renamed.Name != "My Amex" { + t.Errorf("name = %q, want My Amex", renamed.Name) + } + + // A second import of the same layout must NOT overwrite the rename, and must + // bump times_used. + an2, _ := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "amex.csv", Data: file, + }) + if _, err := svc.ApplyImportSession(ctx, an2.Session.ShortID, service.SystemActor()); err != nil { + t.Fatalf("apply2: %v", err) + } + after, _ := svc.ListCSVProfiles(ctx) + if len(after) != 1 || after[0].Name != "My Amex" || after[0].TimesUsed != 2 { + t.Fatalf("after redrop: %+v (want 1 profile 'My Amex' times_used=2)", after) + } + + // Delete. + if err := svc.DeleteCSVProfile(ctx, p.ShortID); err != nil { + t.Fatalf("delete: %v", err) + } + final, _ := svc.ListCSVProfiles(ctx) + if len(final) != 0 { + t.Fatalf("got %d profiles after delete, want 0", len(final)) + } +} diff --git a/internal/sync/matcher.go b/internal/sync/matcher.go index 7feca75a..440f32b8 100644 --- a/internal/sync/matcher.go +++ b/internal/sync/matcher.go @@ -10,6 +10,7 @@ import ( "breadbox/internal/db" "breadbox/internal/pgconv" + "breadbox/internal/textmatch" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" @@ -240,40 +241,11 @@ func pickBestCandidate(depName, depMerchant string, candidates []matchCandidate) } // nameSimilarityScore computes how similar two transactions' names are. -// Returns score (0-3) and which fields matched. +// Returns score (0-3) and which fields matched. The scoring lives in the +// tag-free internal/textmatch package so the CSV import classifier can reuse +// the exact same logic without importing the sync package. func nameSimilarityScore(depName, depMerchant, priName, priMerchant string) (int, []string) { - // Exact merchant_name match (highest signal). - // EqualFold covers case-insensitive equality without allocating. - if depMerchant != "" && priMerchant != "" && - strings.EqualFold(depMerchant, priMerchant) { - return 3, []string{"merchant_name"} - } - - // Merchant name contains or is contained. - if depMerchant != "" && priMerchant != "" { - depMerchantLower := strings.ToLower(depMerchant) - priMerchantLower := strings.ToLower(priMerchant) - if strings.Contains(depMerchantLower, priMerchantLower) || - strings.Contains(priMerchantLower, depMerchantLower) { - return 2, []string{"merchant_name"} - } - } - - // Exact name match. - if strings.EqualFold(depName, priName) { - return 2, []string{"name"} - } - - // Name contains or is contained. - depNameLower := strings.ToLower(depName) - priNameLower := strings.ToLower(priName) - if strings.Contains(depNameLower, priNameLower) || - strings.Contains(priNameLower, depNameLower) { - return 1, []string{"name"} - } - - // No name similarity — still valid (date + amount matched). - return 0, nil + return textmatch.Score(depName, depMerchant, priName, priMerchant) } // nameSimilarityScoreLowered is the same as nameSimilarityScore but accepts @@ -285,37 +257,11 @@ func nameSimilarityScoreLowered( depMerchant, depMerchantLower, priName, priMerchant string, ) (int, []string) { - // Exact merchant_name match (highest signal). - if depMerchant != "" && priMerchant != "" && - strings.EqualFold(depMerchant, priMerchant) { - return 3, []string{"merchant_name"} - } - - // Merchant name contains or is contained. We reuse depMerchantLower - // across candidates and only lower priMerchant on demand. - if depMerchant != "" && priMerchant != "" { - priMerchantLower := strings.ToLower(priMerchant) - if strings.Contains(depMerchantLower, priMerchantLower) || - strings.Contains(priMerchantLower, depMerchantLower) { - return 2, []string{"merchant_name"} - } - } - - // Exact name match. - if strings.EqualFold(depName, priName) { - return 2, []string{"name"} - } - - // Name contains or is contained. Reuse the pre-lowered depName and - // lower priName on demand. - priNameLower := strings.ToLower(priName) - if strings.Contains(depNameLower, priNameLower) || - strings.Contains(priNameLower, depNameLower) { - return 1, []string{"name"} - } - - // No name similarity — still valid (date + amount matched). - return 0, nil + return textmatch.ScoreLowered( + depName, depNameLower, + depMerchant, depMerchantLower, + priName, priMerchant, + ) } // buildMatchedOn returns which name fields matched between two transactions. diff --git a/internal/templates/components/import_modal.templ b/internal/templates/components/import_modal.templ new file mode 100644 index 00000000..ddefd863 --- /dev/null +++ b/internal/templates/components/import_modal.templ @@ -0,0 +1,269 @@ +package components + +// ImportModal is the drop-anywhere CSV import flow: a large daisy modal that +// walks analyze → resolve account → preview/edit → apply → result. It is +// mounted once globally from base.html and driven by the $store.csvImport store +// (drop overlay + open/close) plus the csvImport Alpine factory +// (static/js/admin/components/csv_import_v2.js). +templ ImportModal() { + + +} + +templ importModalAnalyzing() { +
+ +
Analyzing your file…
+
+} + +templ importModalError() { +
+
+ @LucideIcon("alert-circle", "w-5 h-5") + +
+
+ +
+
+} + +// Account selection: ranked suggestions + create-new. +templ importModalAccount() { +
+
+
Which account is this?
+
We ranked your accounts by how well the file matches. Pick one, or create a new account.
+
+
+ +
No existing accounts yet — create one below.
+
+ + +
+} + +// Preview: summary chips + classification filter + paginated table. +templ importModalPreview() { +
+ +
+
+ + + + + +
+
+ + +
+
+ +
+ + + + + + + + + + + + + + +
DateAmountDescriptionStatus
+
No rows in this view.
+
+ +
+
+ + + +
+
+
+} + +templ importModalResult() { +
+
+ @LucideIcon("check", "w-7 h-7 text-success") +
+
+
Import complete
+
+ added + · updated + · skipped +
+
+ +
+} diff --git a/internal/templates/components/pages/design_sections.templ b/internal/templates/components/pages/design_sections.templ index 55fc955c..cd8c2a15 100644 --- a/internal/templates/components/pages/design_sections.templ +++ b/internal/templates/components/pages/design_sections.templ @@ -3508,3 +3508,101 @@ templ designSettingsCreateButton() { templ designSettingsSaveButton(label string) { } + +// SectionCSVImport showcases the drop-anywhere CSV import flow's building +// blocks (static markup — the live flow is components.ImportModal driven by the +// csvImport Alpine factory + $store.csvImport). Variant matrix: drop overlay, +// ranked account picker rows, classification badges, a preview table slice, and +// the success screen. +templ SectionCSVImport() { +
+ @designExample("Drop overlay", "Shown full-window when a file is dragged over any page.") { +
+
+
+ @components.LucideIcon("upload-cloud", "w-10 h-10 text-primary") +
Drop your CSV to import
+
We'll detect the account and preview changes before saving.
+
+
+
+ } + @designExample("Account suggestions", "Ranked by mask, transaction overlap, institution, and saved profiles.") { +
+ + + +
+ } + @designExample("Classification badges", "Per-row verdict in the preview table.") { +
+ New + Maybe dup + Duplicate + Error +
+ } + @designExample("Preview table", "Server-paginated; checkbox toggles include, pencil edits inline.") { +
+ + + + + + + + + + + + + + + + +
DateAmountDescriptionStatus
2026-07-0330.00DINNERNew
2026-07-0110.00COFFEE + Duplicate +
already imported
+
+
+ } + @designExample("Success", "Result screen after apply.") { +
+
+ @components.LucideIcon("check", "w-7 h-7 text-success") +
+
+
Import complete
+
1 added · 2 skipped
+
+
+ } +
+} diff --git a/internal/templates/components/pages/design_types.go b/internal/templates/components/pages/design_types.go index 5d221b87..03fe517b 100644 --- a/internal/templates/components/pages/design_types.go +++ b/internal/templates/components/pages/design_types.go @@ -414,6 +414,13 @@ func DesignSections() []DesignSection { }, // ── Patterns ──────────────────────────────────────────────── + { + Slug: "csv-import", + Title: "CSV import flow", + Description: "Drop-anywhere CSV import (components.ImportModal + $store.csvImport + the csvImport Alpine factory): the full-window drop overlay, ranked account suggestions (mask / overlap / institution / saved profile), per-row classification badges (new / maybe-dup / duplicate / error), the server-paginated preview table, and the success screen.", + Group: "patterns", + Render: func() templ.Component { return SectionCSVImport() }, + }, { Slug: "multi-select-toolbar", Title: "Multi-select toolbar", diff --git a/internal/templates/layout/base.html b/internal/templates/layout/base.html index 8d3d9ed6..4e28a649 100644 --- a/internal/templates/layout/base.html +++ b/internal/templates/layout/base.html @@ -212,6 +212,89 @@ }); + +