diff --git a/cmd/server/account_export.go b/cmd/server/account_export.go new file mode 100644 index 000000000..9169f0ada --- /dev/null +++ b/cmd/server/account_export.go @@ -0,0 +1,200 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "time" + + "github.com/meshcore-analyzer/users" +) + +// accountExport is GET /api/account/export: everything users.db holds about +// the logged-in account (docs/specs/2026-10-08-account-export-and-users-backup-design.md). +// Credentials are left out on purpose: the password hash, session, link and +// unsubscribe tokens; of a pending email change only the new address is +// exported. Other accounts appear as ids only (activatedBy, audit). +type accountExport struct { + FormatVersion int `json:"formatVersion"` + ExportedAt string `json:"exportedAt"` + Instance string `json:"instance"` + Profile exportProfile `json:"profile"` + Sessions []exportSession `json:"sessions"` + Settings *exportSettings `json:"settings"` + Proposals []exportProposal `json:"proposals"` + Notifications exportNotifications `json:"notifications"` + Audit []exportAuditEntry `json:"audit"` + Mail []exportMail `json:"mail"` +} + +type exportProfile struct { + ID int64 `json:"id"` + Email string `json:"email"` + PendingEmail *string `json:"pendingEmail"` // unconfirmed email change: the address only + DisplayName string `json:"displayName"` + Role users.Role `json:"role"` + Status users.Status `json:"status"` + CreatedAt string `json:"createdAt"` + LastLoginAt *string `json:"lastLoginAt"` + EmailBouncing bool `json:"emailBouncing"` + ActivatedAt *string `json:"activatedAt"` + ActivatedBy *int64 `json:"activatedBy"` +} + +type exportSession struct { + CreatedAt string `json:"createdAt"` + LastSeenAt string `json:"lastSeenAt"` + UserAgent string `json:"userAgent,omitempty"` +} + +type exportSettings struct { + Revision int64 `json:"revision"` + UpdatedAt string `json:"updatedAt"` + Doc settingsDoc `json:"doc"` +} + +type exportProposal struct { + Kind string `json:"kind"` + Subject string `json:"subject"` + Status users.ProposalStatus `json:"status"` + Note string `json:"note"` + CreatedAt string `json:"createdAt"` + DecidedAt *string `json:"decidedAt"` +} + +type exportNotifications struct { + Prefs *exportNotifyPrefs `json:"prefs"` + Watches []exportWatch `json:"watches"` +} + +type exportNotifyPrefs struct { + Enabled bool `json:"enabled"` + Events []string `json:"events"` +} + +type exportWatch struct { + Pubkey string `json:"pubkey"` + CreatedAt string `json:"createdAt"` +} + +type exportAuditEntry struct { + At string `json:"at"` + Action string `json:"action"` + ActorUserID *int64 `json:"actorUserId"` + TargetUserID *int64 `json:"targetUserId"` + Detail map[string]string `json:"detail"` +} + +type exportMail struct { + To string `json:"to"` + Purpose string `json:"purpose"` + SentAt string `json:"sentAt"` + Status string `json:"status"` + StatusAt string `json:"statusAt"` + Reason string `json:"reason,omitempty"` + Events []mailEventJSON `json:"events"` +} + +// buildAccountExport reads every section for u. It only reads users.db. +func (a *authService) buildAccountExport(u *users.User, now time.Time) (*accountExport, error) { + x := &accountExport{ + FormatVersion: 1, + ExportedAt: rfc3339(now), + Instance: a.set.baseURL.String(), + Profile: exportProfile{ID: u.ID, Email: u.Email, DisplayName: u.DisplayName, Role: u.Role, Status: u.Status, + CreatedAt: rfc3339(u.CreatedAt), LastLoginAt: rfc3339Ptr(u.LastLoginAt), EmailBouncing: u.EmailBouncing, + ActivatedAt: rfc3339Ptr(u.ActivatedAt), ActivatedBy: u.ActivatedBy}, + Sessions: []exportSession{}, + Proposals: []exportProposal{}, + Notifications: exportNotifications{Watches: []exportWatch{}}, + Audit: []exportAuditEntry{}, + Mail: []exportMail{}, + } + pending, err := a.st.PendingEmailChange(u.ID) + if err != nil { + return nil, fmt.Errorf("pending email: %w", err) + } + if pending != "" { + x.Profile.PendingEmail = &pending + } + sessions, err := a.st.ListSessions(u.ID) + if err != nil { + return nil, fmt.Errorf("sessions: %w", err) + } + for _, s := range sessions { + x.Sessions = append(x.Sessions, exportSession{CreatedAt: rfc3339(s.CreatedAt), LastSeenAt: rfc3339(s.LastSeenAt), UserAgent: s.UserAgent}) + } + rec, err := a.st.SettingsRecordFor(u.ID) + if err != nil { + return nil, fmt.Errorf("settings: %w", err) + } + if rec != nil { + var doc settingsDoc + if err := json.Unmarshal([]byte(rec.Doc), &doc); err != nil { + return nil, fmt.Errorf("settings decode (%d bytes): %w", len(rec.Doc), err) + } + x.Settings = &exportSettings{Revision: rec.Version.Revision, UpdatedAt: rfc3339(rec.UpdatedAt), Doc: doc} + } + props, err := a.st.AllProposalsByUser(u.ID) + if err != nil { + return nil, fmt.Errorf("proposals: %w", err) + } + for _, p := range props { + x.Proposals = append(x.Proposals, exportProposal{Kind: p.Kind, Subject: p.Subject, Status: p.Status, Note: p.Note, + CreatedAt: rfc3339(p.CreatedAt), DecidedAt: rfc3339Ptr(p.DecidedAt)}) + } + prefs, err := a.st.StoredNotifyPrefs(u.ID) + if err != nil { + return nil, fmt.Errorf("notification prefs: %w", err) + } + if prefs != nil { + x.Notifications.Prefs = &exportNotifyPrefs{Enabled: prefs.Enabled, Events: prefs.Events} + } + watches, err := a.st.WatchesFor(u.ID) + if err != nil { + return nil, fmt.Errorf("watches: %w", err) + } + for _, w := range watches { + x.Notifications.Watches = append(x.Notifications.Watches, exportWatch{Pubkey: w.Pubkey, CreatedAt: rfc3339(w.CreatedAt)}) + } + entries, err := a.st.AuditAllFor(u.ID) + if err != nil { + return nil, fmt.Errorf("audit: %w", err) + } + for _, e := range entries { + x.Audit = append(x.Audit, exportAuditEntry{At: rfc3339(e.At), Action: e.Action, + ActorUserID: e.ActorUserID, TargetUserID: e.TargetUserID, Detail: e.Detail}) + } + mails, err := a.st.MailAllForUser(u.ID) + if err != nil { + return nil, fmt.Errorf("mail: %w", err) + } + for _, m := range mails { + em := exportMail{To: m.ToEmail, Purpose: m.Purpose, SentAt: rfc3339(m.SentAt), Status: m.LastEvent, + StatusAt: rfc3339(m.LastEventAt), Reason: m.LastReason, Events: []mailEventJSON{}} + for _, e := range m.Events { + em.Events = append(em.Events, mailEventJSON{Event: e.Event, At: rfc3339(e.At), Reason: e.Reason}) + } + x.Mail = append(x.Mail, em) + } + return x, nil +} + +// handleAccountExport answers the export as a download. GET, so a plain +// link works and no CSRF token is needed. The user.export audit row is +// written after the export is built, so the file shows the state before it. +func (s *Server) handleAccountExport(w http.ResponseWriter, _ *http.Request, u *users.User, _ *users.Session) { + a := s.auth + now := time.Now().UTC() + x, err := a.buildAccountExport(u, now) + if err != nil { + log.Printf("[users] export for user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + a.audit(idPtr(u.ID), "user.export", idPtr(u.ID), nil) + w.Header().Set("Content-Disposition", `attachment; filename="corescope-account-`+now.Format("2006-01-02")+`.json"`) + w.Header().Set("Cache-Control", "no-store") + writeJSON(w, x) +} diff --git a/cmd/server/account_export_test.go b/cmd/server/account_export_test.go new file mode 100644 index 000000000..440fec438 --- /dev/null +++ b/cmd/server/account_export_test.go @@ -0,0 +1,241 @@ +package main + +import ( + "regexp" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/users" +) + +const exportPubkey = "aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899" + +var exportFilenameRE = regexp.MustCompile(`^attachment; filename="corescope-account-\d{4}-\d{2}-\d{2}\.json"$`) + +func TestAccountExportRequiresSession(t *testing.T) { + f := newAuthFixture(t) + expectStatus(t, f.do("GET", "/api/account/export", nil), 401) +} + +func TestAccountExportSections(t *testing.T) { + f, boss, uma := adminFixture(t) + st, uid, bossID := f.st, uma.me.ID, boss.me.ID + if _, err := st.PutSettings(uid, users.SettingsVersion{}, `{"v":1,"keys":{"meshcore-theme":"dark"}}`); err != nil { + t.Fatal(err) + } + if _, err := st.Propose(users.KindHashtagChannel, "#export-test", uid, users.ProposalLimits{}); err != nil { + t.Fatal(err) + } + prefs, err := st.SetNotifyPrefs(uid, true, []string{users.NotifyNodeOffline}) + if err != nil { + t.Fatal(err) + } + if err := st.AddWatch(uid, exportPubkey, 10); err != nil { + t.Fatal(err) + } + if err := st.Audit(&bossID, "user.role", &uid, map[string]string{"role": "user"}); err != nil { + t.Fatal(err) + } + if _, err := st.LogMail(&uid, "uma@example.org", "reset", ""); err != nil { + t.Fatal(err) + } + if _, _, err := st.RecordMailEvent("", "delivered", time.Now(), ""); err != nil { + t.Fatal(err) + } + reset, err := st.IssueToken(uid, users.PurposeReset, time.Hour, "") + if err != nil { + t.Fatal(err) + } + change, err := st.IssueToken(uid, users.PurposeEmailChange, time.Hour, "uma-new@example.org") + if err != nil { + t.Fatal(err) + } + full, err := st.GetByID(uid) + if err != nil { + t.Fatal(err) + } + + w := f.do("GET", "/api/account/export", nil, as(uma)) + expectStatus(t, w, 200) + if ct := w.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("Content-Type = %q", ct) + } + if cc := w.Header().Get("Cache-Control"); cc != "no-store" { + t.Errorf("Cache-Control = %q", cc) + } + if cd := w.Header().Get("Content-Disposition"); !exportFilenameRE.MatchString(cd) { + t.Errorf("Content-Disposition = %q", cd) + } + body := w.Body.String() + x := decode[accountExport](t, w) + + if x.FormatVersion != 1 || x.Instance != testBase || x.ExportedAt == "" { + t.Fatalf("header fields = %d %q %q", x.FormatVersion, x.Instance, x.ExportedAt) + } + p := x.Profile + if p.ID != uid || p.Email != "uma@example.org" || p.DisplayName != "Uma" || p.Role != users.RoleUser || + p.Status != users.StatusActive || p.CreatedAt == "" || p.EmailBouncing { + t.Fatalf("profile = %+v", p) + } + if p.PendingEmail == nil || *p.PendingEmail != "uma-new@example.org" { + t.Fatalf("pendingEmail = %v; want the unconfirmed address", p.PendingEmail) + } + if full.ActivatedAt == nil || p.ActivatedAt == nil || *p.ActivatedAt != rfc3339(*full.ActivatedAt) || p.ActivatedBy != nil { + t.Fatalf("activation (link): profile = %+v, stored at %v", p, full.ActivatedAt) + } + if len(x.Sessions) != 1 || x.Sessions[0].CreatedAt == "" || x.Sessions[0].LastSeenAt == "" { + t.Fatalf("sessions = %+v", x.Sessions) + } + if x.Settings == nil || x.Settings.Revision != 1 || x.Settings.UpdatedAt == "" || x.Settings.Doc.Keys["meshcore-theme"] != "dark" { + t.Fatalf("settings = %+v", x.Settings) + } + if len(x.Proposals) != 1 || x.Proposals[0].Subject != "#export-test" || x.Proposals[0].Kind != users.KindHashtagChannel || + x.Proposals[0].Status != users.ProposalPending || x.Proposals[0].DecidedAt != nil { + t.Fatalf("proposals = %+v", x.Proposals) + } + n := x.Notifications + if n.Prefs == nil || !n.Prefs.Enabled || len(n.Prefs.Events) != 1 || n.Prefs.Events[0] != users.NotifyNodeOffline || + len(n.Watches) != 1 || n.Watches[0].Pubkey != exportPubkey || n.Watches[0].CreatedAt == "" { + t.Fatalf("notifications = %+v", n) + } + var role *exportAuditEntry + for i := range x.Audit { + switch x.Audit[i].Action { + case "user.role": + role = &x.Audit[i] + case "user.export": + t.Fatal("the export contains its own audit row") + } + } + if role == nil || role.ActorUserID == nil || *role.ActorUserID != bossID || role.TargetUserID == nil || + *role.TargetUserID != uid || role.Detail["role"] != "user" || role.At == "" { + t.Fatalf("user.role audit entry = %+v", role) + } + var resetMail *exportMail + for i := range x.Mail { + if x.Mail[i].Purpose == "reset" { + resetMail = &x.Mail[i] + } + } + if resetMail == nil || resetMail.To != "uma@example.org" || resetMail.Status != "delivered" || resetMail.SentAt == "" || + len(resetMail.Events) != 1 || resetMail.Events[0].Event != "delivered" { + t.Fatalf("reset mail = %+v", resetMail) + } + + secrets := map[string]string{ + "password hash": full.PasswordHash, + "session cookie": uma.cookie.Value, + "session token hash": users.HashToken(uma.cookie.Value), + "csrf token": uma.csrf, + "unsubscribe token": prefs.UnsubToken, + "reset token": reset, + "reset token hash": users.HashToken(reset), + "email change token": change, + "email change hash": users.HashToken(change), + } + for name, v := range secrets { + if v == "" { + t.Fatalf("%s is empty: the check below would prove nothing", name) + } + if strings.Contains(body, v) { + t.Errorf("export contains the %s", name) + } + } + for _, other := range []string{"boss@example.org", `"Boss"`} { + if strings.Contains(body, other) { + t.Errorf("export contains another account's %s", other) + } + } +} + +func TestAccountExportWritesAuditRowAfterwards(t *testing.T) { + f := newAuthFixture(t) + uma := f.registerAndActivate(t, "uma@example.org", "Uma", pw) + expectStatus(t, f.do("GET", "/api/account/export", nil, as(uma)), 200) + entries, err := f.st.AuditList(users.AuditFilter{Actions: []string{"user.export"}}) + if err != nil || len(entries) != 1 { + t.Fatalf("user.export rows = %d, %v", len(entries), err) + } + e := entries[0] + if e.ActorUserID == nil || *e.ActorUserID != uma.me.ID || e.TargetUserID == nil || *e.TargetUserID != uma.me.ID { + t.Fatalf("user.export row = %+v", e) + } + second := decode[accountExport](t, f.do("GET", "/api/account/export", nil, as(uma))) + seen := 0 + for _, a := range second.Audit { + if a.Action == "user.export" { + seen++ + } + } + if seen != 1 { + t.Fatalf("second export lists %d user.export rows; want the 1 from the first export", seen) + } +} + +func TestAccountExportEmptySectionsAndNoWrites(t *testing.T) { + f := newAuthFixture(t) + uma := f.registerAndActivate(t, "uma@example.org", "Uma", pw) + w := f.do("GET", "/api/account/export", nil, as(uma)) + expectStatus(t, w, 200) + body := w.Body.String() + for _, want := range []string{`"settings":null`, `"proposals":[]`, `"prefs":null`, `"watches":[]`, `"pendingEmail":null`} { + if !strings.Contains(body, want) { + t.Errorf("export lacks %s: %s", want, body) + } + } + if p, err := f.st.StoredNotifyPrefs(uma.me.ID); err != nil || p != nil { + t.Fatalf("the export created notification prefs: %+v, %v", p, err) + } +} + +func TestAccountExportNoEventsIsEmptyList(t *testing.T) { + f := newAuthFixture(t) + uma := f.registerAndActivate(t, "uma@example.org", "Uma", pw) + if _, err := f.st.SetNotifyPrefs(uma.me.ID, false, nil); err != nil { + t.Fatal(err) + } + w := f.do("GET", "/api/account/export", nil, as(uma)) + expectStatus(t, w, 200) + if body := w.Body.String(); !strings.Contains(body, `"prefs":{"enabled":false,"events":[]}`) { + t.Fatalf("prefs without events: %s", body) + } +} + +func TestAccountExportStoreFailure(t *testing.T) { + f := newAuthFixture(t) + uma := f.registerAndActivate(t, "uma@example.org", "Uma", pw) + f.breakTable(t, "proposals") + w := f.do("GET", "/api/account/export", nil, as(uma)) + expectStatus(t, w, 500) + if cd := w.Header().Get("Content-Disposition"); cd != "" { + t.Fatalf("Content-Disposition on a failed export: %q", cd) + } + if entries, _ := f.st.AuditList(users.AuditFilter{Actions: []string{"user.export"}}); len(entries) != 0 { + t.Fatal("user.export recorded for a failed export") + } +} + +func TestAccountExportManualActivation(t *testing.T) { + f, boss, _ := adminFixture(t) + f.do("POST", "/api/auth/register", registerRequest{Email: "late@example.org", DisplayName: "Late", Password: pw}) + late, err := f.st.GetByEmail("late@example.org") + if err != nil { + t.Fatal(err) + } + expectStatus(t, f.do("POST", userPath(late.ID, "/activate"), nil, as(boss)), 200) + lateC := f.login(t, "late@example.org", pw) + x := decode[accountExport](t, f.do("GET", "/api/account/export", nil, as(lateC))) + if x.Profile.ActivatedBy == nil || *x.Profile.ActivatedBy != boss.me.ID || x.Profile.ActivatedAt == nil { + t.Fatalf("activation (manual): profile = %+v", x.Profile) + } + var activate *exportMail + for i := range x.Mail { + if x.Mail[i].Purpose == "activate" { + activate = &x.Mail[i] + } + } + if activate == nil || activate.To != "late@example.org" { + t.Fatalf("activation mail = %+v", activate) + } +} diff --git a/cmd/server/auth_routes.go b/cmd/server/auth_routes.go index af25f490b..354c21023 100644 --- a/cmd/server/auth_routes.go +++ b/cmd/server/auth_routes.go @@ -27,6 +27,7 @@ func (s *Server) registerAuthRoutes(r *mux.Router) { r.HandleFunc("/api/account/settings", s.withUser(s.handleSettingsGet)).Methods("GET") r.HandleFunc("/api/account/settings", s.withUser(s.handleSettingsPut)).Methods("PUT") r.HandleFunc("/api/account/settings", s.withUser(s.handleSettingsDelete)).Methods("DELETE") + r.HandleFunc("/api/account/export", s.withUser(s.handleAccountExport)).Methods("GET") r.HandleFunc("/api/admin/users", s.withAdmin(s.handleAdminUsers)).Methods("GET") r.HandleFunc("/api/admin/users/{id}", s.withAdmin(s.handleAdminUserDetail)).Methods("GET") r.HandleFunc("/api/admin/users/{id}", s.withAdmin(s.handleAdminDelete)).Methods("DELETE") @@ -38,6 +39,7 @@ func (s *Server) registerAuthRoutes(r *mux.Router) { r.HandleFunc("/api/admin/users/{id}/mail/{mailId}/refresh", s.withAdmin(s.handleAdminMailRefresh)).Methods("POST") r.HandleFunc("/api/admin/audit", s.withAdmin(s.handleAdminAudit)).Methods("GET") r.HandleFunc("/api/admin/stats", s.withAdmin(s.handleAdminStats)).Methods("GET") + r.HandleFunc("/api/admin/users-backup", s.withAdmin(s.handleAdminUsersBackup)).Methods("GET") if s.auth.set.proposals.enabled { s.registerProposalRoutes(r) } diff --git a/cmd/server/auth_service.go b/cmd/server/auth_service.go index 44de7a881..0a4fc84ac 100644 --- a/cmd/server/auth_service.go +++ b/cmd/server/auth_service.go @@ -133,6 +133,7 @@ func (a *authService) janitor(every time.Duration) { defer t.Stop() for { a.prune() + a.maybeBackup(time.Now()) select { case <-a.stop: return diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index b68255a83..3f2f29dcd 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -76,6 +76,7 @@ func routeDescriptions() map[string]routeMeta { "GET /api/account/settings": {Summary: "Get own synced settings", Description: "Returns {revision, generation, doc, allowlist}. doc is {v: 1, keys: {: }} or null at revision 0. generation is a random id of the stored document (empty at revision 0); revisions restart at 1 after a DELETE, the generation does not repeat. allowlist lists the keys the server accepts: {key, kind: set|scalar, id?}, where id names the field that identifies an item of a set (absent: the item itself).", Tag: "users", Session: true}, "PUT /api/account/settings": {Summary: "Replace own synced settings", Description: "Body {baseRevision, baseGeneration, doc}. 200 {revision, generation}. 409 {revision, generation, doc} when baseRevision is not the stored revision or, with a stored document, baseGeneration is not its generation. baseRevision 0 without a stored document starts a new generation. 400 when doc is not {v: 1, keys} or holds a key that is never synced or not allowlisted. 413 above 256 KiB. 429 above 60 PUT requests per hour per user.", Tag: "users", Session: true}, "DELETE /api/account/settings": {Summary: "Delete own synced settings", Description: "Removes the stored document; the next PUT (baseRevision 0) starts a new one in a new generation.", Tag: "users", Session: true}, + "GET /api/account/export": {Summary: "Download own account data", Description: "One JSON file (Content-Disposition attachment corescope-account-.json): {formatVersion: 1, exportedAt, instance, profile, sessions, settings, proposals, notifications: {prefs, watches}, audit, mail}. Every row, no caps. Leaves out the password hash and every session, link and unsubscribe token. Audit rows name other accounts by id only. Includes activatedAt, activatedBy (id) and each mail row's to-address. Records a user.export audit row after the file is built.", Tag: "users", Session: true}, "GET /api/admin/users": {Summary: "List users (admin)", Tag: "users", Session: true, QueryParams: []paramMeta{{Name: "status", Description: "pending | active | disabled", Type: "string"}, {Name: "role", Description: "user | admin", Type: "string"}, {Name: "q", Description: "Substring of email or display name", Type: "string"}, {Name: "bouncing", Description: "1 = only addresses whose mail bounces", Type: "string"}}}, "GET /api/admin/users/{id}": {Summary: "User detail with sessions, mail log and audit (admin)", Tag: "users", Session: true}, "DELETE /api/admin/users/{id}": {Summary: "Delete a user (admin)", Tag: "users", Session: true}, @@ -87,6 +88,7 @@ func routeDescriptions() map[string]routeMeta { "POST /api/admin/users/{id}/mail/{mailId}/refresh": {Summary: "Pull delivery events for one mail from the provider (admin)", Tag: "users", Session: true}, "GET /api/admin/audit": {Summary: "Global audit log (admin)", Description: "Newest first, keyset-paginated: {entries, next}. Pass next as before for the following page; next is null on the last page. Entry: {id, at, action, actor, target, detail}; actor and target are {id, displayName, email}, {id, deleted: true} for a removed account, or null for the system. Login rows (user.login, user.login.failed) are kept 90 days. 400 on an invalid parameter.", Tag: "users", Session: true, QueryParams: []paramMeta{{Name: "action", Description: "One action, or a group ending in .* (user.login.* matches user.login and user.login.failed)", Type: "string"}, {Name: "user", Description: "User id, as actor or target", Type: "integer"}, {Name: "from", Description: "RFC 3339 time, inclusive", Type: "string"}, {Name: "to", Description: "RFC 3339 time, inclusive", Type: "string"}, {Name: "before", Description: "Only entries with a smaller id (next of the previous page)", Type: "integer"}, {Name: "limit", Description: "Page size, default 100, capped at 500", Type: "integer"}}}, "GET /api/admin/stats": {Summary: "User figures for the admin overview (admin)", Description: "Accounts by status, active admins, pending accounts older than 24 hours (stuckPending), bouncing addresses, registrations in 7 and 30 days and per UTC day for 30 days (newPerDay, oldest first), users active in 7 and 30 days, logins and failed logins in 24 hours, mail sent in 7 days by latest delivery status (mail7d), and accounts with 5 or more failed logins in 24 hours (guessing: [{userId, displayName, failed}]).", Tag: "users", Session: true}, + "GET /api/admin/users-backup": {Summary: "Download a users.db snapshot (admin)", Description: "A fresh, consistent copy of users.db (VACUUM INTO) as application/octet-stream, attachment corescope-users-.db (UTC). It holds every password hash and address: store it encrypted. Records a user.backup audit row. The analyzer database has its own route, GET /api/backup.", Tag: "users", Session: true}, "POST /api/proposals": {Summary: "Propose a hashtag channel", Description: "Body {kind: \"hashtag_channel\", subject}. subject is trimmed and gets a leading # when missing; at most 31 UTF-8 bytes including the #, no control or invisible characters (ZWJ allowed), not Public. 201 with the proposal {id, kind, subject, status, note, createdAt, decidedAt}. 400 invalid kind or name, 409 already pending, already approved or rejected, or already decrypted on this instance (a name in config.json hashChannels or channelKeys, case-sensitive), 429 when the caller created userManagement.channelProposals.perUserPerDay proposals in 24 hours or maxPending proposals wait for review. Registered only when userManagement.channelProposals.enabled.", Tag: "users", Session: true}, "GET /api/account/proposals": {Summary: "List own proposals", Description: "The caller's proposals, newest first: [{id, kind, subject, status, note, createdAt, decidedAt}]. note is the reviewer's reason.", Tag: "users", Session: true}, "GET /api/admin/proposals": {Summary: "List proposals (admin)", Description: "Newest first, at most 500: the proposal fields plus proposer and reviewer ({id, displayName, email}, or null once the account is deleted). 400 on an invalid filter.", Tag: "users", Session: true, QueryParams: []paramMeta{{Name: "status", Description: "pending | approved | rejected | revoked", Type: "string"}, {Name: "kind", Description: "hashtag_channel", Type: "string"}}}, diff --git a/cmd/server/user_mgmt_config.go b/cmd/server/user_mgmt_config.go index 6cd9980d6..6683685d0 100644 --- a/cmd/server/user_mgmt_config.go +++ b/cmd/server/user_mgmt_config.go @@ -23,6 +23,7 @@ type UserManagementConfig struct { Mail UserMailConfig `json:"mail"` ChannelProposals *ChannelProposalsConfig `json:"channelProposals,omitempty"` Notifications *NotificationsConfig `json:"notifications,omitempty"` + Backup *UsersBackupConfig `json:"backup,omitempty"` } // UserMailConfig is userManagement.mail. @@ -121,6 +122,41 @@ func resolveNotifications(c *NotificationsConfig) notifySettings { } } +// UsersBackupConfig is userManagement.backup: scheduled users.db +// snapshots (docs/specs/2026-10-08-account-export-and-users-backup-design.md). +// On by default: an absent block, or one without "enabled", means on. +type UsersBackupConfig struct { + Enabled *bool `json:"enabled,omitempty"` + Dir string `json:"dir,omitempty"` + Keep int `json:"keep,omitempty"` +} + +// backupSettings is the resolved form; the zero value means off. +type backupSettings struct { + enabled bool + dir string // relative paths resolve against the working directory, like dbPath + keep int // snapshots kept after each run +} + +const defaultBackupKeep = 7 + +// resolveBackup fills the defaults: dir "backups" next to users.db, keep 7 +// for an absent, zero or negative value. Only "enabled": false turns it off. +func resolveBackup(c *UsersBackupConfig, dbPath string) backupSettings { + if c != nil && c.Enabled != nil && !*c.Enabled { + return backupSettings{} + } + b := backupSettings{enabled: true, dir: filepath.Join(filepath.Dir(dbPath), "backups"), keep: defaultBackupKeep} + if c == nil { + return b + } + if d := strings.TrimSpace(c.Dir); d != "" { + b.dir = d + } + b.keep = positiveOr(c.Keep, defaultBackupKeep) + return b +} + // UserManagementEnabled reports whether optional accounts are on. Nil config // or absent section means off (the default). func (c *Config) UserManagementEnabled() bool { @@ -142,6 +178,7 @@ type userMgmtSettings struct { webhookSecret string proposals proposalSettings notify notifySettings + backup backupSettings } const defaultSessionDays = 30 @@ -219,6 +256,7 @@ func resolveUserManagement(u *UserManagementConfig, measurementDBPath string, ge } set.proposals = resolveProposals(u.ChannelProposals) set.notify = resolveNotifications(u.Notifications) + set.backup = resolveBackup(u.Backup, set.dbPath) return set, nil } diff --git a/cmd/server/user_mgmt_config_test.go b/cmd/server/user_mgmt_config_test.go index fa9a201a3..09ccb6dcf 100644 --- a/cmd/server/user_mgmt_config_test.go +++ b/cmd/server/user_mgmt_config_test.go @@ -140,3 +140,38 @@ func TestResolveNotifications(t *testing.T) { t.Fatal("enabled false resolved as on") } } + +func TestResolveUsersBackup(t *testing.T) { + u := validUM() + meas := filepath.Join("data", "meshcore.db") + defaults := backupSettings{enabled: true, dir: filepath.Join("data", "backups"), keep: 7} + set, err := resolveUserManagement(u, meas, noEnv) + if err != nil { + t.Fatal(err) + } + if set.backup != defaults { + t.Fatalf("absent block = %+v; want %+v", set.backup, defaults) + } + u.DBPath = filepath.Join("accounts", "users.db") + if set, _ = resolveUserManagement(u, meas, noEnv); set.backup.dir != filepath.Join("accounts", "backups") { + t.Fatalf("dir follows dbPath: %q", set.backup.dir) + } + u.DBPath = "" + off, on := false, true + u.Backup = &UsersBackupConfig{Enabled: &off, Dir: "x", Keep: 3} + if set, _ = resolveUserManagement(u, meas, noEnv); set.backup != (backupSettings{}) { + t.Fatalf("enabled false = %+v; want off", set.backup) + } + u.Backup = &UsersBackupConfig{Dir: " bk ", Keep: 3} + if set, _ = resolveUserManagement(u, meas, noEnv); set.backup != (backupSettings{enabled: true, dir: "bk", keep: 3}) { + t.Fatalf("enabled omitted = %+v; want on with dir bk, keep 3", set.backup) + } + u.Backup = &UsersBackupConfig{Enabled: &on, Dir: " ", Keep: -1} + if set, _ = resolveUserManagement(u, meas, noEnv); set.backup != defaults { + t.Fatalf("blank dir and negative keep = %+v; want defaults", set.backup) + } + u.Backup = &UsersBackupConfig{Keep: 0} + if set, _ = resolveUserManagement(u, meas, noEnv); set.backup.keep != 7 { + t.Fatalf("keep 0 = %d; want 7", set.backup.keep) + } +} diff --git a/cmd/server/user_mgmt_off_test.go b/cmd/server/user_mgmt_off_test.go index 83eb2bb41..2b78f9bf3 100644 --- a/cmd/server/user_mgmt_off_test.go +++ b/cmd/server/user_mgmt_off_test.go @@ -15,7 +15,7 @@ func TestUserManagementOffIsUnchanged(t *testing.T) { if srv.auth != nil { t.Fatal("auth built without config") } - for _, p := range []string{"/api/auth/me", "/api/admin/users", "/api/admin/audit", "/api/admin/stats", "/api/account/sessions", "/api/account/settings"} { + for _, p := range []string{"/api/auth/me", "/api/admin/users", "/api/admin/audit", "/api/admin/stats", "/api/account/sessions", "/api/account/settings", "/api/account/export", "/api/admin/users-backup"} { w := httptest.NewRecorder() router.ServeHTTP(w, httptest.NewRequest("GET", p, nil)) if w.Code != 404 { diff --git a/cmd/server/users_backup.go b/cmd/server/users_backup.go new file mode 100644 index 000000000..bebe788da --- /dev/null +++ b/cmd/server/users_backup.go @@ -0,0 +1,229 @@ +package main + +import ( + "errors" + "io" + "io/fs" + "log" + "net/http" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/meshcore-analyzer/users" +) + +// users.db snapshots (docs/specs/2026-10-08-account-export-and-users-backup-design.md). +const ( + usersBackupInterval = 24 * time.Hour + usersBackupLayout = "20060102-150405" // UTC, in the file name +) + +// usersBackupName matches the snapshot files this code writes and rotates. +// Apart from orphaned temporary files (usersBackupTemp), nothing else in +// the directory is ever touched. +var usersBackupName = regexp.MustCompile(`^users-\d{8}-\d{6}\.db$`) + +// usersBackupTemp matches the temporary name a snapshot is written under. +// One is only left behind when the process died mid-write. +var usersBackupTemp = regexp.MustCompile(`^users-\d{8}-\d{6}\.db\.tmp$`) + +// usersBackupTempMaxAge is how old an orphaned temporary snapshot must be +// before the sweep removes it; no snapshot write takes this long. +const usersBackupTempMaxAge = 24 * time.Hour + +func usersBackupFile(t time.Time) string { return "users-" + t.UTC().Format(usersBackupLayout) + ".db" } + +// listUsersBackups returns the snapshot file names in dir, oldest first +// (the UTC timestamp sorts by name). A missing dir has none. +func listUsersBackups(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var out []string + for _, e := range entries { + if e.Type().IsRegular() && usersBackupName.MatchString(e.Name()) { + out = append(out, e.Name()) + } + } + sort.Strings(out) + return out, nil +} + +// usersBackupDue reports whether a snapshot is due at now: the newest +// snapshot whose name time is not in the future is at least +// usersBackupInterval old, or there is none. Future names (a clock stepped +// back) are skipped so they cannot stop the backups. +func usersBackupDue(names []string, now time.Time) bool { + for i := len(names) - 1; i >= 0; i-- { + t, err := time.Parse(usersBackupLayout, strings.TrimSuffix(strings.TrimPrefix(names[i], "users-"), ".db")) + if err != nil || t.After(now) { + continue + } + return now.Sub(t) >= usersBackupInterval + } + return true +} + +// writeUsersBackup snapshots st into dir (created 0700) under a temporary +// name and renames it when complete. It returns the path and size. +func writeUsersBackup(st *users.Store, dir string, now time.Time) (string, int64, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", 0, err + } + path := filepath.Join(dir, usersBackupFile(now)) + tmp := path + ".tmp" + if err := st.Snapshot(tmp); err != nil { + return "", 0, err + } + if err := os.Rename(tmp, path); err != nil { + os.Remove(tmp) + return "", 0, err + } + fi, err := os.Stat(path) + if err != nil { + return "", 0, err + } + return path, fi.Size(), nil +} + +// sweepUsersBackupTemps removes orphaned temporary snapshots (regular files +// named like usersBackupTemp, modified more than usersBackupTempMaxAge +// before now) and returns how many it removed. A missing dir has none. +func sweepUsersBackupTemps(dir string, now time.Time) (int, error) { + entries, err := os.ReadDir(dir) + if errors.Is(err, fs.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, err + } + removed := 0 + for _, e := range entries { + if !e.Type().IsRegular() || !usersBackupTemp.MatchString(e.Name()) { + continue + } + fi, err := e.Info() + if err != nil { + return removed, err + } + if now.Sub(fi.ModTime()) <= usersBackupTempMaxAge { + continue + } + if err := os.Remove(filepath.Join(dir, e.Name())); err != nil { + return removed, err + } + removed++ + } + return removed, nil +} + +// rotateUsersBackups deletes the oldest snapshots beyond keep and returns +// how many remain. It never deletes justWritten, which a future-dated name +// (a clock that ran ahead) would otherwise push out as the oldest. +func rotateUsersBackups(dir string, keep int, justWritten string) (int, error) { + names, err := listUsersBackups(dir) + if err != nil { + return 0, err + } + kept := len(names) + for _, n := range names { + if kept <= keep { + break + } + if n == justWritten { + continue + } + if err := os.Remove(filepath.Join(dir, n)); err != nil { + return kept, err + } + kept-- + } + return kept, nil +} + +// maybeBackup sweeps orphaned temporary files, takes a snapshot when one +// is due and then rotates. The +// janitor calls it every hour, the first time at startup. A failure is +// logged and leaves the existing snapshots; the next run tries again. +func (a *authService) maybeBackup(now time.Time) { + b := a.set.backup + if !b.enabled { + return + } + if n, err := sweepUsersBackupTemps(b.dir, now); err != nil { + log.Printf("[users] backup temp sweep failed: %v", err) + } else if n > 0 { + log.Printf("[users] backup removed %d orphaned temporary file(s)", n) + } + names, err := listUsersBackups(b.dir) + if err != nil { + log.Printf("[users] backup failed: %v", err) + return + } + if !usersBackupDue(names, now) { + return + } + path, size, err := writeUsersBackup(a.st, b.dir, now) + if err != nil { + log.Printf("[users] backup failed: %v", err) + return + } + kept, err := rotateUsersBackups(b.dir, b.keep, filepath.Base(path)) + if err != nil { + log.Printf("[users] backup rotation failed: %v", err) + } + log.Printf("[users] backup written: %s (%d bytes, kept %d)", absForLog(path), size, kept) +} + +// handleAdminUsersBackup streams a fresh users.db snapshot from a temp +// directory, removed afterwards. The file holds password hashes and +// addresses: admin only, audited as user.backup. +func (s *Server) handleAdminUsersBackup(w http.ResponseWriter, _ *http.Request, admin *users.User, _ *users.Session) { + a := s.auth + tmpDir, err := os.MkdirTemp("", "corescope-users-") + if err != nil { + log.Printf("[users] backup download: temp dir: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + log.Printf("[users] backup download cleanup: %v", err) + } + }() + name := "corescope-users-" + time.Now().UTC().Format(usersBackupLayout) + ".db" + path := filepath.Join(tmpDir, name) + if err := a.st.Snapshot(path); err != nil { + log.Printf("[users] backup download: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + f, err := os.Open(path) + if err != nil { + log.Printf("[users] backup download: open snapshot: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + defer f.Close() // runs before the RemoveAll above (LIFO), which Windows needs + if fi, err := f.Stat(); err == nil { + w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10)) + } + a.audit(idPtr(admin.ID), "user.backup", nil, nil) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`) + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, f); err != nil { + log.Printf("[users] backup download stream: %v", err) + } +} diff --git a/cmd/server/users_backup_test.go b/cmd/server/users_backup_test.go new file mode 100644 index 000000000..c57117d25 --- /dev/null +++ b/cmd/server/users_backup_test.go @@ -0,0 +1,299 @@ +package main + +import ( + "bytes" + "database/sql" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strconv" + "testing" + "time" + + "github.com/meshcore-analyzer/users" +) + +var backupNow = time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + +// backupService is a test auth service with backups on in a fresh, not yet +// created directory. +func backupService(t *testing.T, keep int) (*authService, string) { + t.Helper() + a, _ := newTestAuthService(t) + dir := filepath.Join(t.TempDir(), "backups") + a.set.backup = backupSettings{enabled: true, dir: dir, keep: keep} + return a, dir +} + +func snapshotNames(t *testing.T, dir string) []string { + t.Helper() + names, err := listUsersBackups(dir) + if err != nil { + t.Fatal(err) + } + return names +} + +func writeBackupFile(t *testing.T, dir, name string) { + t.Helper() + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte("old"), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestUsersBackupWrittenWhenNoneExists(t *testing.T) { + a, dir := backupService(t, 7) + if _, err := a.st.CreatePending("a@example.org", "Aaa", "$argon2id$placeholder"); err != nil { + t.Fatal(err) + } + a.maybeBackup(backupNow) + names := snapshotNames(t, dir) + if len(names) != 1 || names[0] != usersBackupFile(backupNow) || names[0] != "users-20261008-120000.db" { + t.Fatalf("snapshots = %v", names) + } + db, err := sql.Open("sqlite", filepath.Join(dir, names[0])) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM users WHERE email = 'a@example.org'`).Scan(&n); err != nil || n != 1 { + t.Fatalf("user rows in the snapshot = %d, %v", n, err) + } + if left, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(left) != 0 { + t.Fatalf("temporary files left: %v", left) + } +} + +func TestUsersBackupFileMode(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Unix file modes") + } + a, dir := backupService(t, 7) + a.maybeBackup(backupNow) + fi, err := os.Stat(filepath.Join(dir, usersBackupFile(backupNow))) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o600 { + t.Fatalf("snapshot mode = %v; want 0600", fi.Mode().Perm()) + } + di, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if di.Mode().Perm() != 0o700 { + t.Fatalf("directory mode = %v; want 0700", di.Mode().Perm()) + } +} + +func TestUsersBackupSkippedWhenFresh(t *testing.T) { + a, dir := backupService(t, 7) + fresh := usersBackupFile(backupNow.Add(-time.Hour)) + writeBackupFile(t, dir, fresh) + a.maybeBackup(backupNow) + a.maybeBackup(backupNow.Add(22 * time.Hour)) + if names := snapshotNames(t, dir); !reflect.DeepEqual(names, []string{fresh}) { + t.Fatalf("snapshots = %v; want only %s", names, fresh) + } +} + +func TestUsersBackupWrittenWhenStale(t *testing.T) { + a, dir := backupService(t, 7) + stale := usersBackupFile(backupNow.Add(-25 * time.Hour)) + writeBackupFile(t, dir, stale) + a.maybeBackup(backupNow) + if names := snapshotNames(t, dir); !reflect.DeepEqual(names, []string{stale, usersBackupFile(backupNow)}) { + t.Fatalf("snapshots = %v", names) + } +} + +func TestUsersBackupFutureSnapshotDoesNotBlock(t *testing.T) { + a, dir := backupService(t, 7) + future := usersBackupFile(backupNow.Add(48 * time.Hour)) + writeBackupFile(t, dir, future) + a.maybeBackup(backupNow) + want := []string{usersBackupFile(backupNow), future} + if names := snapshotNames(t, dir); !reflect.DeepEqual(names, want) { + t.Fatalf("snapshots = %v; want %v", names, want) + } + // The snapshot just written is the newest one not in the future: fresh. + a.maybeBackup(backupNow.Add(time.Hour)) + if names := snapshotNames(t, dir); !reflect.DeepEqual(names, want) { + t.Fatalf("after an hour = %v; want %v", names, want) + } +} + +func TestUsersBackupRotationNeverDeletesTheNewSnapshot(t *testing.T) { + a, dir := backupService(t, 1) + future := usersBackupFile(backupNow.Add(48 * time.Hour)) + writeBackupFile(t, dir, future) + a.maybeBackup(backupNow) + want := []string{usersBackupFile(backupNow)} + if names := snapshotNames(t, dir); !reflect.DeepEqual(names, want) { + t.Fatalf("snapshots = %v; want %v", names, want) + } +} + +func TestUsersBackupRotationKeepsNewestAndForeignFiles(t *testing.T) { + a, dir := backupService(t, 3) + var old []string // oldest first + for i := 5; i >= 1; i-- { + n := usersBackupFile(backupNow.Add(-time.Duration(i) * 25 * time.Hour)) + writeBackupFile(t, dir, n) + old = append(old, n) + } + foreign := []string{"notes.txt", "users-before-upgrade.db", "users.db", usersBackupFile(backupNow.Add(-300*time.Hour)) + ".tmp"} + for _, n := range foreign { + writeBackupFile(t, dir, n) + } + a.maybeBackup(backupNow) + want := []string{old[3], old[4], usersBackupFile(backupNow)} + if names := snapshotNames(t, dir); !reflect.DeepEqual(names, want) { + t.Fatalf("after rotation = %v; want %v", names, want) + } + for _, n := range foreign { + if _, err := os.Stat(filepath.Join(dir, n)); err != nil { + t.Errorf("foreign file %s was touched: %v", n, err) + } + } +} + +func TestUsersBackupSweepsOldTempFiles(t *testing.T) { + a, dir := backupService(t, 7) + setAge := func(name string, age time.Duration) { + t.Helper() + at := backupNow.Add(-age) + if err := os.Chtimes(filepath.Join(dir, name), at, at); err != nil { + t.Fatal(err) + } + } + orphan := usersBackupFile(backupNow.Add(-30*time.Hour)) + ".tmp" + recent := usersBackupFile(backupNow.Add(-time.Hour)) + ".tmp" + foreign := []string{"other.db.tmp", "users-20261001-000000.db.tmp.bak", "users-before-upgrade.db.tmp"} + for _, n := range append([]string{orphan, recent}, foreign...) { + writeBackupFile(t, dir, n) + setAge(n, 25*time.Hour) + } + setAge(recent, time.Hour) + dirLike := usersBackupFile(backupNow.Add(-40*time.Hour)) + ".tmp" + if err := os.Mkdir(filepath.Join(dir, dirLike), 0o700); err != nil { + t.Fatal(err) + } + setAge(dirLike, 25*time.Hour) + // A fresh snapshot: no new one is due, the sweep still runs. + writeBackupFile(t, dir, usersBackupFile(backupNow.Add(-time.Hour))) + + a.maybeBackup(backupNow) + if _, err := os.Stat(filepath.Join(dir, orphan)); !os.IsNotExist(err) { + t.Errorf("orphaned temp file older than 24h still there: %v", err) + } + for _, n := range append([]string{recent, dirLike}, foreign...) { + if _, err := os.Stat(filepath.Join(dir, n)); err != nil { + t.Errorf("%s was touched: %v", n, err) + } + } +} + +func TestUsersBackupFailureKeepsOldSnapshots(t *testing.T) { + a, dir := backupService(t, 1) + stale := usersBackupFile(backupNow.Add(-48 * time.Hour)) + writeBackupFile(t, dir, stale) + a.st.Close() // every store call fails from here; the fixture's cleanup closes again, which is a no-op + a.maybeBackup(backupNow) + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name() != stale { + var got []string + for _, e := range entries { + got = append(got, e.Name()) + } + t.Fatalf("directory after a failed snapshot = %v; want only %s", got, stale) + } +} + +func TestUsersBackupDisabled(t *testing.T) { + a, dir := backupService(t, 7) + a.set.backup.enabled = false + a.maybeBackup(backupNow) + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Fatalf("backup directory created while disabled: %v", err) + } +} + +func TestInitUserManagementBacksUpAtStartup(t *testing.T) { + dir := t.TempDir() + srv := &Server{cfg: &Config{UserManagement: &UserManagementConfig{ + Enabled: true, PublicBaseURL: testBase, + Mail: UserMailConfig{BrevoAPIKey: "k", FromEmail: "noreply@example.org"}, + }}} + if err := srv.initUserManagement(filepath.Join(dir, "meshcore.db")); err != nil { + t.Fatal(err) + } + // The janitor's first pass (prune, then backup) runs before it sees stop. + srv.closeUserManagement() + if names := snapshotNames(t, filepath.Join(dir, "backups")); len(names) != 1 { + t.Fatalf("snapshots after startup = %v; want 1", names) + } +} + +var usersBackupFilenameRE = regexp.MustCompile(`^attachment; filename="corescope-users-\d{8}-\d{6}\.db"$`) + +func TestAdminUsersBackupDownload(t *testing.T) { + f, boss, uma := adminFixture(t) + tmp := t.TempDir() + for _, k := range []string{"TMPDIR", "TMP", "TEMP"} { + t.Setenv(k, tmp) + } + expectStatus(t, f.do("GET", "/api/admin/users-backup", nil), 401) + expectStatus(t, f.do("GET", "/api/admin/users-backup", nil, as(uma)), 403) + + w := f.do("GET", "/api/admin/users-backup", nil, as(boss)) + expectStatus(t, w, 200) + if ct := w.Header().Get("Content-Type"); ct != "application/octet-stream" { + t.Errorf("Content-Type = %q", ct) + } + if cc := w.Header().Get("Cache-Control"); cc != "no-store" { + t.Errorf("Cache-Control = %q", cc) + } + if cd := w.Header().Get("Content-Disposition"); !usersBackupFilenameRE.MatchString(cd) { + t.Errorf("Content-Disposition = %q", cd) + } + body := w.Body.Bytes() + if !bytes.HasPrefix(body, []byte("SQLite format 3\x00")) { + t.Fatalf("body is not a SQLite file (%d bytes)", len(body)) + } + if cl := w.Header().Get("Content-Length"); cl != strconv.Itoa(len(body)) { + t.Errorf("Content-Length = %q; body is %d bytes", cl, len(body)) + } + path := filepath.Join(t.TempDir(), "download.db") + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM users`).Scan(&n); err != nil || n != 2 { + t.Fatalf("users in the download = %d, %v; want 2", n, err) + } + if left, _ := os.ReadDir(tmp); len(left) != 0 { + t.Fatalf("temporary files left in the temp dir: %d", len(left)) + } + entries, err := f.st.AuditList(users.AuditFilter{Actions: []string{"user.backup"}}) + if err != nil || len(entries) != 1 { + t.Fatalf("user.backup rows = %d, %v; want 1", len(entries), err) + } + if e := entries[0]; e.ActorUserID == nil || *e.ActorUserID != boss.me.ID || e.TargetUserID != nil { + t.Fatalf("user.backup row = %+v", e) + } +} diff --git a/config.example.json b/config.example.json index fead9d9e6..012a35ec6 100644 --- a/config.example.json +++ b/config.example.json @@ -455,11 +455,17 @@ "perUserPerDay": 20, "maxMailsPerDay": 100, "maxWatchesPerUser": 50 + }, + "backup": { + "enabled": true, + "dir": "", + "keep": 7 } }, "_comment_userManagement": "Optional accounts (off by default; see docs/user-guide/accounts.md). When enabled: visitors can register with email + password and activate through a mailed link; addresses in adminEmails become admins; admins manage users at #/admin/users and can use the operator endpoints without the apiKey. The dashboard stays public. dbPath defaults to users.db next to the analyzer database: a separate file, the server never writes measurement data. publicBaseUrl is required and is used for every link in every mail. Mail goes through Brevo's transactional API (a free account is enough); brevoApiKey and webhookSecret may instead come from CORESCOPE_BREVO_API_KEY and CORESCOPE_BREVO_WEBHOOK_SECRET. Set webhookSecret (16+ chars) and point a Brevo transactional webhook at /api/mail/brevo/webhook with bearer auth to see delivery status per user. trustedProxies (CIDRs) lets login rate limits use X-Forwarded-For behind a reverse proxy. Startup fails if enabled with an incomplete mail setup.", "_comment_channelProposals": "Optional, needs userManagement.enabled (docs/user-guide/accounts.md, Channel proposals). Logged-in users propose hashtag channels; admins approve, reject or revoke them on the Proposals tab. An approved channel is decrypted by the ingestor from its next refresh (once a minute, no restart) and listed for every visitor on the Channels page. maxApproved bounds the decryption work (every approved key is tried on every group message); perUserPerDay and maxPending limit proposals. The ingestor reads users.db read-only from userManagement.dbPath, or next to the analyzer database; set dbPath explicitly when the server and the ingestor are not given the same analyzer database path.", "_comment_notifications": "Optional, needs userManagement.enabled (docs/user-guide/accounts.md, Node notifications). Logged-in users watch nodes and get one mail per check when a watched node goes offline, comes back or reports a low battery; admins can also watch for new foreign nodes and observers going offline. Thresholds are healthThresholds and batteryThresholds. A check runs every intervalMinutes; at most perUserPerDay mails per user and maxMailsPerDay in total go out per 24 hours (over a limit the change is recorded and dropped, never queued). maxMailsPerDay counts notification mail only: keep it below the mail provider's daily quota so activation, password-reset and address-change mail still go out (the default 100 leaves 200 of Brevo's free 300 a day); maxWatchesPerUser bounds each watch list. Every mail has a one-click unsubscribe.", + "_comment_usersBackup": "Part of userManagement (docs/user-guide/accounts.md, Backups). With accounts on, the server snapshots users.db at startup when the newest snapshot is older than 24 hours, then about every 24 hours, to users-.db (UTC, mode 0600) in dir (empty: backups/ next to users.db), and deletes the oldest beyond keep (default 7), never the one just written. On unless enabled is false. Snapshots hold password hashes, and deleted accounts stay in them for up to keep days: protect the directory like users.db. A dir that already exists is not tightened to 0700: set that yourself. Admins download a fresh snapshot at /api/admin/users-backup. A restore brings back accounts deleted after the snapshot (also self-deletions), old passwords, revoked sessions and used links: follow the Restore steps in the doc (note the deletions from the current audit log first, then disable those accounts and clear sessions and tokens before starting, and delete them again in the admin area).", "customizer": { "disabledTabs": [], "_comment_disabledTabs": "Issue #1508. List of customizer-modal tab ids to hide from end users. Useful when operators want to expose only daily-use viewer controls and keep one-shot admin chrome out of the way. Recognized ids: branding, theme, nodes, home, display, geofilter, export. Example: [\"branding\", \"geofilter\", \"export\"] hides the admin tabs and leaves theme/colors/home/display visible. Default [] keeps the legacy behavior (all tabs visible). Operators edit this in config.json directly — there is no in-app UI for this list (by design)." diff --git a/docs/api-spec.md b/docs/api-spec.md index ffdd932e1..9875009e8 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -123,6 +123,7 @@ These routes are registered only when `userManagement.enabled` is true. When it | `GET /api/account/settings` | session | -> `{revision, generation, doc, allowlist}`. `doc` is `{v: 1, keys: {: }}`, or `null` at revision 0. `generation` is a random id of the stored document (`""` at revision 0): revisions restart at 1 after a `DELETE`, the generation does not repeat. `allowlist` is `[{key, kind, id?}]`: `kind` is `set` (JSON array merged per item; `id` names the field that identifies an item, absent means the item itself) or `scalar` | | `PUT /api/account/settings` | session | `{baseRevision, baseGeneration, doc}` -> `{revision, generation}`. `409` `{revision, generation, doc}` when `baseRevision` is not the stored revision or, with a stored document, `baseGeneration` is not its generation; `baseRevision` 0 without a stored document starts a new generation; `400` for another shape, a key that is never synced (`corescope_channel_*`, `live-channel-colors`, `meshcore-api-key`) or a key not in the allowlist; `413` when `doc` is over 256 KiB (body cap 264 KiB); `429` above 60 PUT requests per hour per user | | `DELETE /api/account/settings` | session | -> `{ok}`. The next `PUT` with `baseRevision` 0 starts a new document in a new generation | +| `GET /api/account/export` | session | -> attachment `corescope-account-.json`: `{formatVersion: 1, exportedAt, instance, profile, sessions, settings, proposals, notifications, audit, mail}`. `profile` is `{id, email, displayName, role, status, createdAt, lastLoginAt, emailBouncing, activatedAt, activatedBy}` (`activatedBy` is the admin id, `null` for a link activation); `sessions` `[{createdAt, lastSeenAt, userAgent?}]`; `settings` `{revision, updatedAt, doc}` or `null`; `proposals` `[{kind, subject, status, note, createdAt, decidedAt}]`; `notifications` `{prefs: {enabled, events} or null, watches: [{pubkey, createdAt}]}`; `audit` `[{at, action, actorUserId, targetUserId, detail}]` (other accounts by id only); `mail` `[{to, purpose, sentAt, status, statusAt, reason?, events}]`. No row caps. No password hash and no session, link or unsubscribe token. Writes a `user.export` audit row afterwards | | `DELETE /api/account` | session | `{currentPassword}` -> `{ok, message}`. `409` you are the last admin | | `GET /api/admin/users?status=&role=&q=&bouncing=` | admin | -> `[adminUser]`, at most 1000, newest first. `status` is `pending`, `active` or `disabled`; `role` is `user` or `admin`; `bouncing=1` keeps only addresses whose mail bounces (any other value is `400`) | | `GET /api/admin/users/{id}` | admin | -> `{user, sessions, mail, audit}` | @@ -135,6 +136,7 @@ These routes are registered only when `userManagement.enabled` is true. When it | `POST /api/admin/users/{id}/mail/{mailId}/refresh` | admin | -> mail record with `events`, pulled from the provider. `502` provider unavailable | | `GET /api/admin/audit?action=&user=&from=&to=&before=&limit=` | admin | -> `{entries, next}`, newest first. `action` is one action or a group ending in `.*` (`user.login.*`); `user` matches actor or target; `from`/`to` are RFC 3339 and inclusive; pass `next` as `before` for the next page (`null` on the last page); `limit` defaults to 100, capped at 500. Entry: `{id, at, action, actor, target, detail}`, with actor and target `{id, displayName, email}`, `{id, deleted: true}` or `null` (system). Login rows are kept 90 days. `400` on an invalid parameter | | `GET /api/admin/stats` | admin | -> `{total, active, pending, disabled, admins, stuckPending, bouncing, new7d, new30d, newPerDay, active7d, active30d, logins24h, failedLogins24h, mail7d, guessing}`. `newPerDay` is `[{day, count}]` for 30 UTC days, oldest first; registrations count `user.register` audit rows. `stuckPending` counts pending accounts older than 24 hours. `mail7d` is `{delivered, bounced, blocked, spam, pending, other}` by latest delivery event. `guessing` is `[{userId, displayName, failed}]`: accounts with 5 or more failed logins in 24 hours | +| `GET /api/admin/users-backup` | admin | -> `application/octet-stream`, attachment `corescope-users-.db` (UTC): a fresh consistent copy of `users.db`. It holds every password hash and address, so store it encrypted. Writes a `user.backup` audit row. Restore: docs/user-guide/accounts.md, Backups | | `POST /api/mail/brevo/webhook` | `Authorization: Bearer ` | Brevo transactional event payload -> `{ok}`. `401` on a wrong secret | `adminUser` = `{id, email, displayName, role, status, createdAt, activatedAt, activatedManually, activatedBy, lastLoginAt, emailBouncing, configAdmin, lastMail}`. `status` is `pending`, `active` or `disabled`. diff --git a/docs/specs/2026-10-08-account-export-and-users-backup-design.md b/docs/specs/2026-10-08-account-export-and-users-backup-design.md new file mode 100644 index 000000000..354c16cb1 --- /dev/null +++ b/docs/specs/2026-10-08-account-export-and-users-backup-design.md @@ -0,0 +1,154 @@ +# Optional User Management: Account Data Export and users.db Backup, Design Spec + +**Status:** approved in conversation 2026-10-08. +**Builds on:** A to E of optional user management (`docs/specs/2026-10-06-user-management-design.md` +and the later parts). Roadmap #2128. Small follow-up, no new schema version. + +## Problem + +1. An instance with accounts holds personal data (address, name, settings, audit and mail + history). A user has no way to get a copy of it. On an EU-hosted instance the user has + a right to that copy (GDPR art. 15 and 20). Self-delete already exists + (`DELETE /api/account`). +2. `users.db` is the only file the server writes. Nothing backs it up: the existing + `GET /api/backup` snapshots the analyzer database only. Losing `users.db` loses every + account, setting, proposal and watch. + +## Goals + +1. A logged-in user downloads all data the instance holds about them as one JSON file. +2. The server keeps rotating local snapshots of `users.db`, and an admin can download a + consistent snapshot to keep off the server. +3. With user management off, nothing changes. + +## Non-goals + +- Importing an export into another instance. +- Backing up the analyzer database (exists: `GET /api/backup`). +- Off-site upload (S3, rsync). The admin download and the documented restore cover it. + +## 1. Data export + +- `GET /api/account/export` (withUser; GET, so no CSRF token is needed and a plain link + works). Response `application/json`, `Content-Disposition: attachment; + filename="corescope-account-.json"`, `Cache-Control: no-store`. +- Body: one typed Go struct (no `map[string]interface{}`): + - `exportedAt`, `instance` (the configured `publicBaseUrl`), `formatVersion: 1` + - `profile`: id, email, displayName, role, status, createdAt, lastLoginAt, + emailBouncing + - `sessions`: per session createdAt, lastSeenAt and the stored user agent if any; + no token or token hash + - `settings`: the synced settings document as stored (B), revision and updatedAt, or + null + - `proposals`: the user's own proposals (kind, subject, status, note, createdAt, + decidedAt) + - `notifications`: prefs (enabled, events) and watches (pubkey, createdAt); no + unsubscribe token + - `audit`: every audit row where the user is actor or target (time, action, detail); + another account in a row appears as its id only, never its address or name + - `mail`: every mail log row for the user (purpose, sentAt, final status and events) +- Excluded on purpose: password hash, session and activation/reset token hashes, the + unsubscribe token. They are credentials, not data about the person, and an export file + is easily mailed around. +- Audit: `user.export` (actor and target the user). The export reflects the state before + that row is written. +- No limits on row counts: per-user data is small (audit and mail are pruned by the + janitor). +- Frontend: a "Download my data" button on the account page (a plain link to the route), + with one line explaining what it contains. + +## 2. users.db backup + +### Scheduled snapshots + +- Config: + + ```json + "userManagement": { + "backup": { "enabled": true, "dir": "", "keep": 7 } + } + ``` + + Absent block: enabled with the defaults (user management on implies a backup). + `enabled: false` turns it off. `dir` empty means `backups/` next to `users.db`. + `keep` at or below 0 falls back to 7. +- The server takes a snapshot with `VACUUM INTO` through `internal/users` + (`Store.Snapshot(path)`), the same technique as `GET /api/backup`. The server's write + path stays `internal/users` only (AGENTS.md, read/write separation). +- Schedule: once at startup when the newest snapshot is older than 24 hours (or none + exists), then every 24 hours from the janitor loop. File name + `users-.db`, written to a temporary name and renamed when complete, mode + 0600, directory created with 0700. +- Rotation: after a successful snapshot, delete the oldest `users-*.db` files in that + directory beyond `keep`. Files not matching that pattern are never touched. +- A failed snapshot logs `[users] backup failed: ` and leaves existing snapshots in + place; the next run tries again. +- Log on success: `[users] backup written: ( bytes, kept )`. + +### Admin download + +- `GET /api/admin/users-backup` (withAdmin): takes a fresh snapshot into a temp + directory and streams it as `application/octet-stream`, + `filename="corescope-users-.db"`, `Cache-Control: no-store`; the temp + file is removed afterwards. +- Audit: `user.backup` (actor the admin). +- The file contains password hashes and addresses, so the route is admin-only and the + docs say to store the download encrypted. + +### Restore (documented, not automated) + +Stop the server, copy the snapshot over `users.db` (remove `users.db-wal` and +`users.db-shm`), start the server. The schema version check refuses a snapshot from a +newer binary. + +## Security and privacy + +- The export is for the logged-in user only; there is no admin "export any user" route. +- Snapshot files and the download contain credentials (hashes); 0600 files, admin-only + download, audit rows for both actions. + +## Performance + +`users.db` is small (kilobytes to a few megabytes); `VACUUM INTO` takes milliseconds and +runs once a day plus on admin request. The export is a handful of per-user queries. No +analyzer hot path is touched. + +## Testing + +- `internal/users`: `Snapshot` produces a valid database with the same rows; refuses an + existing target path. +- `cmd/server` export: every section present with the right rows; credentials absent + (assert no hash or token value appears anywhere in the body); other accounts appear as + id only; 401 without session; audit row; feature off: route absent. +- Scheduled backup: snapshot written at startup when none exists, skipped when a fresh + one exists, rotation keeps `keep` newest and ignores foreign files, failure keeps old + files, file mode 0600 (skipped on Windows), disabled by config. +- Admin download: 403 for a user, 401 without session, body is a valid SQLite file, + temp file removed, audit row. +- Frontend unit (vm): the button renders for a logged-in user and links to the route. +- Playwright (e2etest build): the account page shows the button; fetching the export as + the logged-in user returns JSON with the user's address. + +## Amendments + +Added after the final review of the implementation (2026-10-08): + +- Rotation never deletes the snapshot it just wrote. Without this, future-dated names + (a clock that ran ahead) numbering `keep` or more pushed the new snapshot out right + after it was written. The logged `kept ` is the number actually left. +- Each backup run deletes orphaned temporary snapshots in the backup directory: regular + files named exactly `users-.db.tmp` last modified more than 24 hours + ago. Other files stay untouched. +- Export `profile.pendingEmail`: the new address of an unused, unexpired email change, or + null. The token and its hash stay out. `notifications.prefs.events` is `[]` when no + event is chosen (pinned by a test; the store already returned an empty list). +- `notification_state` (which notifications were already sent) is not exported: it is + the server's operational state, not data the user gave. The user guide says so. +- Docs: snapshots are taken "about every 24 hours" (hourly check). The Backups section + states what a restore brings back (accounts deleted after the snapshot including + self-deletions, old passwords, revoked sessions, used links) and the procedure: + list `user.delete` / `user.delete.self` audit rows newer than the snapshot from the + current `users.db` before overwriting, then with the server stopped disable those + accounts and `DELETE FROM sessions; DELETE FROM tokens;`, and delete the accounts again + in the admin area after start. Deleted accounts remain in snapshots for up to `keep` + days and in downloaded copies. An existing `backup.dir` is not tightened to 0700. diff --git a/docs/user-guide/accounts.md b/docs/user-guide/accounts.md index f0db8e1d1..2be2eff77 100644 --- a/docs/user-guide/accounts.md +++ b/docs/user-guide/accounts.md @@ -191,8 +191,82 @@ watched node changes state. Admins can also watch the instance. ### Backups -`users.db` holds password hashes and addresses. Back it up together with the analyzer -database, and protect it the same way. Deleting it removes all accounts and nothing else. +`users.db` holds password hashes and addresses. The server keeps its own snapshots of +it: at startup when the newest snapshot is older than 24 hours (or there is none), then +about every 24 hours (the check runs hourly). A snapshot is a complete copy named `users-.db` (UTC), +readable by the server's user only, in `backups/` next to `users.db`. After each new +snapshot the oldest ones beyond `keep` are deleted, never the one just written. +Temporary files of an interrupted snapshot (`users-.db.tmp`) are +deleted once they are older than 24 hours; other files in that directory are never +touched. A failed snapshot is logged (`[users] backup failed: ...`) and the next run +tries again. + +The server creates the default directory, or a `backup.dir` that does not exist yet, +with mode 0700. An existing `backup.dir` is not tightened: make it readable by the +server's user only yourself (`chmod 700 `). + +A deleted account, including one the user deleted themselves, stays in the snapshots +for up to `keep` days (one snapshot a day), and in every downloaded copy until you +delete that copy. + +```json +"userManagement": { + "backup": { "enabled": true, "dir": "", "keep": 7 } +} +``` + +| Key | Meaning | +|---|---| +| `backup.enabled` | Default `true`, also when the block is absent. `false` turns the snapshots off. | +| `backup.dir` | Where snapshots go. Empty: `backups/` next to `users.db`. A relative path is relative to the server's working directory. | +| `backup.keep` | How many snapshots are kept. Default 7, also for 0 or less. | + +Snapshots on the same disk are lost with that disk. To keep a copy elsewhere, log in as +an admin and open `/api/admin/users-backup` in the same browser: it downloads a fresh +snapshot (`corescope-users-.db`) and records it in the audit log +(`user.backup`). Store the download encrypted: it holds every password hash and address. +The analyzer database has its own backup route, `GET /api/backup`. + +**What a restore brings back.** `users.db` returns to the moment of the snapshot: + +- accounts deleted after it, including accounts users deleted themselves; +- old passwords of users who changed them since; +- sessions that were logged out or revoked since (valid again until they expire, for + anyone who still holds the cookie); +- activation, reset and email change links used since (valid again until they expire). + +The audit log is replaced too, so note the deletions before you overwrite it. + +**Restore** (not automated). The commands use the `sqlite3` command-line tool; replace +`2026-10-08 12:00:00` with the snapshot's time from its file name (UTC). + +1. Stop the server. +2. List the accounts deleted since the snapshot, from the current `users.db`: + + ```sh + sqlite3 users.db "SELECT target_user_id, action, datetime(at, 'unixepoch') FROM audit_log + WHERE action IN ('user.delete', 'user.delete.self') + AND at >= CAST(strftime('%s', '2026-10-08 12:00:00') AS INTEGER);" + ``` + +3. Copy the snapshot over `users.db`, and delete `users.db-wal` and `users.db-shm` if + they exist. +4. Still with the server stopped, disable the accounts from step 2 (replace `12, 34` + with their ids) and clear all sessions and links, so everyone logs in again: + + ```sh + sqlite3 users.db "UPDATE users SET status = 'disabled' WHERE id IN (12, 34); + DELETE FROM sessions; DELETE FROM tokens;" + ``` + +5. Start the server. +6. Delete the accounts from step 2 again in the admin area. Deleting there, not in + `sqlite3`, also replaces the addresses in the mail history with a hash and records + the deletion in the audit log. + +A snapshot made by a newer CoreScope version is refused at startup ("database schema +version N is newer than this binary supports"): run that version or newer. Deleting +`users.db` removes all accounts and nothing else. ## For users @@ -205,6 +279,15 @@ database, and protect it the same way. Deleting it removes all accounts and noth - **My account:** change your display name, password or address, see your logged-in devices, or delete your account. A new address is confirmed from a link sent to it, and your old address gets a notice. Changing your password logs out your other devices. +- **Download my data:** *My account, My data, Download my data* saves one JSON file with + your profile (including when and by whom the account was activated), logged-in devices, + synced settings, proposals, notification settings and watched nodes, and the history of + your account and of the mail sent to you (including the address each mail went to). + An address change you have not confirmed yet is included as `pendingEmail`. Password + hashes and login, link and unsubscribe tokens are not in it, and neither is the + server's internal bookkeeping of which notifications it already sent. Other accounts + in your history appear as a number only. Each download is recorded in the audit log + (`user.export`). - **Settings sync:** while you are logged in, your settings follow you: your nodes, favorites, theme and customizer settings, saved packet filters, and the filter, sort and view choices of each page. Log in on another browser or phone and they are diff --git a/internal/users/audit.go b/internal/users/audit.go index 6f3fda575..ff3f47e6b 100644 --- a/internal/users/audit.go +++ b/internal/users/audit.go @@ -49,11 +49,21 @@ func (s *Store) Audit(actor *int64, action string, target *int64, detail map[str return err } -// AuditFor returns entries where userID is the target or the actor, newest first. +// AuditFor returns entries where userID is the target or the actor, newest +// first, at most limit (default 100). func (s *Store) AuditFor(userID int64, limit int) ([]AuditEntry, error) { if limit <= 0 { limit = 100 } + return s.auditFor(userID, limit) +} + +// AuditAllFor is AuditFor without a cap (the account export). +func (s *Store) AuditAllFor(userID int64) ([]AuditEntry, error) { + return s.auditFor(userID, noLimit) +} + +func (s *Store) auditFor(userID int64, limit int) ([]AuditEntry, error) { rows, err := s.db.Query(`SELECT id, at, actor_user_id, action, target_user_id, detail FROM audit_log WHERE target_user_id = ? OR actor_user_id = ? ORDER BY at DESC, id DESC LIMIT ?`, userID, userID, limit) if err != nil { diff --git a/internal/users/export_test.go b/internal/users/export_test.go new file mode 100644 index 000000000..8da4e2b24 --- /dev/null +++ b/internal/users/export_test.go @@ -0,0 +1,161 @@ +package users + +import ( + "errors" + "fmt" + "testing" + "time" +) + +func TestAuditAllForHasNoCap(t *testing.T) { + st, clk := newTestStore(t) + u := mustCreate(t, st, "a@example.org", "Aaa") + uid := u.ID + for i := 0; i < 120; i++ { + if err := st.Audit(&uid, "user.login", &uid, nil); err != nil { + t.Fatal(err) + } + clk.Advance(time.Second) + } + capped, err := st.AuditFor(uid, 0) + if err != nil || len(capped) != 100 { + t.Fatalf("AuditFor default = %d, %v; want 100", len(capped), err) + } + all, err := st.AuditAllFor(uid) + if err != nil || len(all) != 120 { + t.Fatalf("AuditAllFor = %d, %v; want 120", len(all), err) + } + if !all[0].At.After(all[119].At) { + t.Fatal("AuditAllFor is not newest first") + } +} + +func TestMailAllForUserHasNoCap(t *testing.T) { + st, clk := newTestStore(t) + u := mustCreate(t, st, "a@example.org", "Aaa") + uid := u.ID + for i := 0; i < 60; i++ { + if _, err := st.LogMail(&uid, "a@example.org", "reset", ""); err != nil { + t.Fatal(err) + } + } + if _, err := st.LogMail(&uid, "a@example.org", "activate", ""); err != nil { + t.Fatal(err) + } + if _, _, err := st.RecordMailEvent("", "delivered", clk.Now(), ""); err != nil { + t.Fatal(err) + } + capped, err := st.MailForUser(uid, 0) + if err != nil || len(capped) != 50 { + t.Fatalf("MailForUser default = %d, %v; want 50", len(capped), err) + } + all, err := st.MailAllForUser(uid) + if err != nil || len(all) != 61 { + t.Fatalf("MailAllForUser = %d, %v; want 61", len(all), err) + } + if all[0].Purpose != "activate" || len(all[0].Events) != 1 || all[0].Events[0].Event != "delivered" { + t.Fatalf("newest mail = %+v", all[0]) + } +} + +func TestAllProposalsByUserHasNoCap(t *testing.T) { + st, clk := newTestStore(t) + u := mustCreate(t, st, "a@example.org", "Aaa") + tx, err := st.db.Begin() + if err != nil { + t.Fatal(err) + } + for i := 0; i <= ProposalListMax; i++ { + if _, err := tx.Exec(`INSERT INTO proposals (kind, subject, status, proposer_id, created_at) VALUES (?, ?, 'rejected', ?, ?)`, + KindHashtagChannel, fmt.Sprintf("#p%d", i), u.ID, unix(clk.Now())); err != nil { + tx.Rollback() + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + capped, err := st.ProposalsByUser(u.ID) + if err != nil || len(capped) != ProposalListMax { + t.Fatalf("ProposalsByUser = %d, %v; want %d", len(capped), err, ProposalListMax) + } + all, err := st.AllProposalsByUser(u.ID) + if err != nil || len(all) != ProposalListMax+1 { + t.Fatalf("AllProposalsByUser = %d, %v; want %d", len(all), err, ProposalListMax+1) + } +} + +func TestSettingsRecordFor(t *testing.T) { + st, clk := newTestStore(t) + u := mustCreate(t, st, "a@example.org", "Aaa") + if r, err := st.SettingsRecordFor(u.ID); err != nil || r != nil { + t.Fatalf("without a document = %+v, %v; want nil, nil", r, err) + } + doc := `{"v":1,"keys":{"meshcore-theme":"dark"}}` + if _, err := st.PutSettings(u.ID, SettingsVersion{}, doc); err != nil { + t.Fatal(err) + } + r, err := st.SettingsRecordFor(u.ID) + if err != nil || r == nil { + t.Fatalf("SettingsRecordFor = %+v, %v", r, err) + } + if r.Doc != doc || r.Version.Revision != 1 || r.Version.Generation == "" || !r.UpdatedAt.Equal(clk.Now()) { + t.Fatalf("record = %+v", r) + } +} + +func TestStoredNotifyPrefsNeverCreates(t *testing.T) { + st, _ := newTestStore(t) + u := mustCreate(t, st, "a@example.org", "Aaa") + if p, err := st.StoredNotifyPrefs(u.ID); err != nil || p != nil { + t.Fatalf("before first use = %+v, %v; want nil, nil", p, err) + } + if _, err := st.getPrefs(u.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("StoredNotifyPrefs created a row: %v", err) + } + want, err := st.NotifyPrefsFor(u.ID) + if err != nil { + t.Fatal(err) + } + p, err := st.StoredNotifyPrefs(u.ID) + if err != nil || p == nil || p.UnsubToken != want.UnsubToken || p.Enabled != want.Enabled { + t.Fatalf("after NotifyPrefsFor = %+v, %v; want %+v", p, err, want) + } +} + +func TestPendingEmailChange(t *testing.T) { + st, clk := newTestStore(t) + u := mustCreate(t, st, "a@example.org", "Aaa") + if got, err := st.PendingEmailChange(u.ID); err != nil || got != "" { + t.Fatalf("none issued = %q, %v; want empty", got, err) + } + if _, err := st.IssueToken(u.ID, PurposeReset, time.Hour, ""); err != nil { + t.Fatal(err) + } + if _, err := st.IssueToken(u.ID, PurposeEmailChange, time.Hour, "first@example.org"); err != nil { + t.Fatal(err) + } + if _, err := st.IssueToken(u.ID, PurposeEmailChange, time.Hour, "second@example.org"); err != nil { + t.Fatal(err) + } + if got, err := st.PendingEmailChange(u.ID); err != nil || got != "second@example.org" { + t.Fatalf("pending = %q, %v; want the newest address", got, err) + } + clk.Advance(2 * time.Hour) + if got, err := st.PendingEmailChange(u.ID); err != nil || got != "" { + t.Fatalf("after expiry = %q, %v; want empty", got, err) + } + if _, err := st.IssueToken(u.ID, PurposeEmailChange, time.Hour, "third@example.org"); err != nil { + t.Fatal(err) + } + other := mustCreate(t, st, "b@example.org", "Bbb") + if got, err := st.PendingEmailChange(other.ID); err != nil || got != "" { + t.Fatalf("other user = %q, %v; want empty", got, err) + } + if err := st.InvalidateTokens(u.ID, PurposeEmailChange); err != nil { + t.Fatal(err) + } + if got, err := st.PendingEmailChange(u.ID); err != nil || got != "" { + t.Fatalf("after invalidation = %q, %v; want empty", got, err) + } +} diff --git a/internal/users/mail.go b/internal/users/mail.go index 1ec4897ec..3bb2ca8b4 100644 --- a/internal/users/mail.go +++ b/internal/users/mail.go @@ -134,11 +134,21 @@ func (s *Store) MailByID(id int64) (*MailRecord, error) { return m, nil } -// MailForUser returns a user's mails, newest first, each with its events. +// MailForUser returns a user's mails, newest first, each with its events, +// at most limit (default 50). func (s *Store) MailForUser(userID int64, limit int) ([]MailRecord, error) { if limit <= 0 { limit = 50 } + return s.mailForUser(userID, limit) +} + +// MailAllForUser is MailForUser without a cap (the account export). +func (s *Store) MailAllForUser(userID int64) ([]MailRecord, error) { + return s.mailForUser(userID, noLimit) +} + +func (s *Store) mailForUser(userID int64, limit int) ([]MailRecord, error) { rows, err := s.db.Query(`SELECT `+mailCols+` FROM mail_log WHERE user_id = ? ORDER BY sent_at DESC, id DESC LIMIT ?`, userID, limit) if err != nil { return nil, err diff --git a/internal/users/notify.go b/internal/users/notify.go index caa7d41e4..4bad64573 100644 --- a/internal/users/notify.go +++ b/internal/users/notify.go @@ -162,6 +162,16 @@ func (s *Store) NotifyPrefsFor(userID int64) (NotifyPrefs, error) { return *p, nil } +// StoredNotifyPrefs returns the user's stored preferences, or nil when +// none were ever created. Unlike NotifyPrefsFor it never writes. +func (s *Store) StoredNotifyPrefs(userID int64) (*NotifyPrefs, error) { + p, err := s.getPrefs(userID) + if errors.Is(err, ErrNotFound) { + return nil, nil + } + return p, err +} + // SetNotifyPrefs stores enabled and events (unknown events dropped, // canonical order) and deletes the state rows of events no longer chosen, // so choosing an event again starts with a silent first evaluation. The diff --git a/internal/users/onetime.go b/internal/users/onetime.go index c1af9bee8..89d3556ba 100644 --- a/internal/users/onetime.go +++ b/internal/users/onetime.go @@ -149,6 +149,20 @@ func (s *Store) InvalidateTokens(userID int64, p Purpose) error { return err } +// PendingEmailChange returns the new address of the user's unused, +// unexpired email change, or "" when there is none. Only the address is +// returned, never the token or its hash. +func (s *Store) PendingEmailChange(userID int64) (string, error) { + var ne sql.NullString + err := s.db.QueryRow(`SELECT new_email FROM tokens + WHERE user_id = ? AND purpose = ? AND used_at IS NULL AND expires_at > ? + ORDER BY expires_at DESC LIMIT 1`, userID, string(PurposeEmailChange), unix(s.now())).Scan(&ne) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return ne.String, err +} + // PruneTokens deletes tokens that expired more than keep ago. func (s *Store) PruneTokens(keep time.Duration) (int64, error) { res, err := s.db.Exec(`DELETE FROM tokens WHERE expires_at < ?`, unix(s.now())-int64(keep/time.Second)) diff --git a/internal/users/proposals.go b/internal/users/proposals.go index 9e096fbaa..ba3d5bba2 100644 --- a/internal/users/proposals.go +++ b/internal/users/proposals.go @@ -237,10 +237,20 @@ func (s *Store) ListProposals(f ProposalFilter) ([]Proposal, error) { return s.queryProposals(q, args...) } -// ProposalsByUser returns the user's proposals, newest first. +// ProposalsByUser returns the user's proposals, newest first, at most +// ProposalListMax. func (s *Store) ProposalsByUser(userID int64) ([]Proposal, error) { + return s.proposalsByUser(userID, ProposalListMax) +} + +// AllProposalsByUser is ProposalsByUser without the cap (the account export). +func (s *Store) AllProposalsByUser(userID int64) ([]Proposal, error) { + return s.proposalsByUser(userID, noLimit) +} + +func (s *Store) proposalsByUser(userID int64, limit int) ([]Proposal, error) { return s.queryProposals(`SELECT `+proposalCols+` FROM proposals WHERE proposer_id = ? - ORDER BY created_at DESC, id DESC LIMIT ?`, userID, ProposalListMax) + ORDER BY created_at DESC, id DESC LIMIT ?`, userID, limit) } func (s *Store) queryProposals(q string, args ...any) ([]Proposal, error) { diff --git a/internal/users/settings.go b/internal/users/settings.go index cd49fcb32..148298abb 100644 --- a/internal/users/settings.go +++ b/internal/users/settings.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "time" ) // ErrSettingsConflict: PutSettings was given a base version that is not @@ -45,6 +46,31 @@ func (s *Store) GetSettings(userID int64) (string, SettingsVersion, error) { return doc, v, nil } +// SettingsRecord is a stored settings document with its version and the +// time of its last write. +type SettingsRecord struct { + Doc string + Version SettingsVersion + UpdatedAt time.Time +} + +// SettingsRecordFor returns the user's stored settings, or nil when the +// user has none. +func (s *Store) SettingsRecordFor(userID int64) (*SettingsRecord, error) { + var r SettingsRecord + var updated int64 + err := s.db.QueryRow(`SELECT doc, revision, generation, updated_at FROM user_settings WHERE user_id = ?`, userID). + Scan(&r.Doc, &r.Version.Revision, &r.Version.Generation, &updated) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + r.UpdatedAt = fromUnix(updated) + return &r, nil +} + // PutSettings stores doc and returns the new version. Without a stored // document base.Revision must be 0, and the write starts a new generation. // With one, base must equal the stored version; the new revision is diff --git a/internal/users/snapshot.go b/internal/users/snapshot.go new file mode 100644 index 000000000..a4c2a0f4f --- /dev/null +++ b/internal/users/snapshot.go @@ -0,0 +1,30 @@ +package users + +import ( + "errors" + "fmt" + "io/fs" + "os" +) + +// Snapshot writes a consistent copy of users.db to path with VACUUM INTO +// (the technique of GET /api/backup) and makes it readable by the owner +// only: it holds password hashes and addresses. path must not exist; an +// empty file there is refused too, although VACUUM INTO would accept it. +// On failure no file is left at path. +func (s *Store) Snapshot(path string) error { + if _, err := os.Lstat(path); err == nil { + return fmt.Errorf("users: snapshot target %s already exists", path) + } else if !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("users: snapshot target %s: %w", path, err) + } + if _, err := s.db.Exec(`VACUUM INTO ?`, path); err != nil { + os.Remove(path) + return fmt.Errorf("users: snapshot: %w", err) + } + if err := os.Chmod(path, 0o600); err != nil { + os.Remove(path) + return fmt.Errorf("users: snapshot mode: %w", err) + } + return nil +} diff --git a/internal/users/snapshot_test.go b/internal/users/snapshot_test.go new file mode 100644 index 000000000..fdf282248 --- /dev/null +++ b/internal/users/snapshot_test.go @@ -0,0 +1,83 @@ +package users + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestSnapshotCopiesRows(t *testing.T) { + st, _ := newTestStore(t) + u := mustCreate(t, st, "a@example.org", "Aaa") + uid := u.ID + if err := st.Audit(&uid, "user.register", &uid, nil); err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "snap.db") + if err := st.Snapshot(path); err != nil { + t.Fatal(err) + } + cp, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer cp.Close() + got, err := cp.GetByEmail("a@example.org") + if err != nil || got.ID != u.ID || got.DisplayName != "Aaa" { + t.Fatalf("user in snapshot = %+v, %v", got, err) + } + if v, err := cp.SchemaVersion(); err != nil || v != len(migrations) { + t.Fatalf("snapshot schema version = %d, %v; want %d", v, err, len(migrations)) + } + if e, err := cp.AuditAllFor(u.ID); err != nil || len(e) != 1 { + t.Fatalf("audit in snapshot = %d, %v", len(e), err) + } +} + +func TestSnapshotFileMode(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Unix file modes") + } + st, _ := newTestStore(t) + path := filepath.Join(t.TempDir(), "snap.db") + if err := st.Snapshot(path); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o600 { + t.Fatalf("mode = %v; want 0600", fi.Mode().Perm()) + } +} + +func TestSnapshotRefusesExistingTarget(t *testing.T) { + st, _ := newTestStore(t) + path := filepath.Join(t.TempDir(), "snap.db") + // An empty file: VACUUM INTO alone would accept and fill it. + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + err := st.Snapshot(path) + if err == nil || !strings.Contains(err.Error(), "already exists") { + t.Fatalf("err = %v; want already exists", err) + } + if fi, _ := os.Stat(path); fi == nil || fi.Size() != 0 { + t.Fatal("the existing file was changed") + } +} + +func TestSnapshotFailureLeavesNoFile(t *testing.T) { + st, _ := newTestStore(t) + st.Close() + path := filepath.Join(t.TempDir(), "snap.db") + if err := st.Snapshot(path); err == nil { + t.Fatal("snapshot of a closed store succeeded") + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("partial snapshot left behind: %v", err) + } +} diff --git a/internal/users/store.go b/internal/users/store.go index 1d0740318..bb7b589dd 100644 --- a/internal/users/store.go +++ b/internal/users/store.go @@ -28,6 +28,9 @@ var ( ErrAccountChanged = errors.New("users: account changed since it was read") ) +// noLimit is SQLite's LIMIT value for "every row" (the account export). +const noLimit = -1 + // Store is the users.db handle. Safe for concurrent use. type Store struct { db *sql.DB diff --git a/public/account.js b/public/account.js index 4110cc0a2..6d8463b28 100644 --- a/public/account.js +++ b/public/account.js @@ -114,8 +114,11 @@ (window.CSProposals && window.CSProposals.enabled() ? '

My proposals

' + msgBox('propMsg') : '') + (window.CSNotify && window.CSNotify.enabled() ? '

Notifications

' + msgBox('notifyMsg') : '') + (window.CSSettingsSync ? '

Settings sync

' : '') + + '

My data

' + + '' + + '' + '

Delete account

'); } diff --git a/public/admin-audit.js b/public/admin-audit.js index 884d3d939..c141b1a5b 100644 --- a/public/admin-audit.js +++ b/public/admin-audit.js @@ -21,6 +21,8 @@ ['user.password.*', 'Password changes and resets'], ['user.email.*', 'Address changes'], ['user.mail.refresh', 'Mail status refreshed'], + ['user.export', 'Data exports'], + ['user.backup', 'users.db downloads'], ['proposal.*', 'Channel proposals'], ['notify.*', 'Notification settings'] ]; diff --git a/tests/e2e/test-user-management-e2e.js b/tests/e2e/test-user-management-e2e.js index a6988aaa1..0d6a1aea7 100644 --- a/tests/e2e/test-user-management-e2e.js +++ b/tests/e2e/test-user-management-e2e.js @@ -416,6 +416,19 @@ async function until(fn, label) { assert(!(await proposer.isChecked('#notifyEnabled')), 'still on after the unsubscribe link'); }); + await step('data export: the account page links to the export, which returns the user\'s own data', async () => { + await proposer.goto(BASE + '/#/account'); + await proposer.waitForSelector('#accountExport'); + assert(await proposer.getAttribute('#accountExport', 'href') === '/api/account/export', 'export link href'); + const r = await proposer.request.get(BASE + '/api/account/export'); + assert(r.ok(), 'export HTTP ' + r.status()); + const cd = r.headers()['content-disposition'] || ''; + assert(/^attachment; filename="corescope-account-\d{4}-\d{2}-\d{2}\.json"$/.test(cd), 'content-disposition ' + cd); + const body = await r.json(); + assert(body.formatVersion === 1, 'formatVersion ' + body.formatVersion); + assert(body.profile && body.profile.email === 'proposer@e2e.test', 'export profile ' + JSON.stringify(body.profile)); + }); + await step('feature off: no userManagement block, so no notifications flag, and no toggle on the node page', async () => { const body = await (await off.request.get(BASE_OFF + '/api/config/client')).json(); assert(!body.userManagement, 'userManagement present while off'); diff --git a/tests/unit/test-admin-dashboard-ui.js b/tests/unit/test-admin-dashboard-ui.js index 5e7289653..71c4d20a6 100644 --- a/tests/unit/test-admin-dashboard-ui.js +++ b/tests/unit/test-admin-dashboard-ui.js @@ -101,6 +101,12 @@ test('the audit action filter offers channel proposals', () => { assert.strictEqual(auditT().readHash('#/admin?tab=audit&action=proposal.*').action, 'proposal.*'); }); +test('the audit action filter offers data export and users.db backup', () => { + const t = auditT(); + assert.strictEqual(t.readHash('#/admin?tab=audit&action=user.export').action, 'user.export'); + assert.strictEqual(t.readHash('#/admin?tab=audit&action=user.backup').action, 'user.backup'); +}); + test('hashFor writes tab=audit and only the set filters', () => { const t = auditT(); assert.strictEqual(t.hashFor({ action: 'user.login.*', user: '12', period: '' }), '#/admin?tab=audit&action=user.login.*&user=12'); diff --git a/tests/unit/test-user-management-ui.js b/tests/unit/test-user-management-ui.js index 8a5105696..4ad62ebe5 100644 --- a/tests/unit/test-user-management-ui.js +++ b/tests/unit/test-user-management-ui.js @@ -600,6 +600,15 @@ test('profile view: Log out button for everyone, Admin link for admins only', () assert(admin.indexOf('href="#/admin">Admin') !== -1, admin); }); +test('profile view: Download my data links to the export route, before Delete account', () => { + const env = loadAccount('#/account', () => ({})); + const html = env.t.profileHtml({ email: 'a', role: 'user', displayName: 'A' }); + const link = ''; + assert(html.indexOf(link) !== -1, html); + assert(html.indexOf('

My data

') !== -1, 'no My data heading'); + assert(html.indexOf('id="accountExport"') < html.indexOf('id="delForm"'), 'export link after the delete form'); +}); + test('profile view follows auth changes: logout redirects, another user re-renders', () => { const env = loadAccount('#/account', () => ({ ok: true, status: 200, data: [] })); env.user.current = { id: 1, email: 'a@b.c', displayName: 'Ann', role: 'user' };