diff --git a/cmd/fox/main.go b/cmd/fox/main.go index f80822e..8fd29f8 100644 --- a/cmd/fox/main.go +++ b/cmd/fox/main.go @@ -106,6 +106,12 @@ Stack: checks: pass, fail, manual or n/a, with the evidence and the fix hardening fix | profile baseline|strict [--branch b] [--yes] Fix a failing check, or every one a profile covers, after a preview (Enterprise) + mask [status] [--branch b] [--json] + Masking personal data: the rules on a branch, and the accounts masked + mask setup [--branch b] [--yes] | account [--off] [--yes] + Set masking up on a branch; mask an account on every branch it does not own (Enterprise) + mask rule <[schema.]table.column> [partial|hash|fake|empty] [--path $.a.b] [--off] [--branch b] [--yes] + Mask a confirmed finding for masked accounts, after saying what it costs (Enterprise) access review [--dormant 90] [--json] [--csv accounts|keys|branches|applications] Who can reach what, with which keys, who may override the guardrails, who has gone quiet, and the applications that assert their users (Enterprise) @@ -406,6 +412,8 @@ func main() { accessCmd(os.Args[2:]) case "hardening": hardeningCmd(os.Args[2:]) + case "mask": + maskCmd(os.Args[2:]) case "app": appCmd(os.Args[2:]) case "check": diff --git a/cmd/fox/maskcmd.go b/cmd/fox/maskcmd.go new file mode 100644 index 0000000..31e7216 --- /dev/null +++ b/cmd/fox/maskcmd.go @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/thefoxbyte/foxbyte/internal/auth" + "github.com/thefoxbyte/foxbyte/internal/branch" + "github.com/thefoxbyte/foxbyte/internal/brand" + "github.com/thefoxbyte/foxbyte/internal/edition" +) + +// maskCmd is `fox mask` (F5, point 4.6): masking personal data a reviewer +// confirmed, for the accounts an admin masks. +// +// fox mask [status] [--branch b] [--json] +// fox mask setup [--branch b] [--yes] (Enterprise) +// fox mask rule <[schema.]table.column> [rule] [--path $.a.b] [--branch b] [--yes] +// fox mask rule <[schema.]table.column> --off [--path $.a.b] [--branch b] +// fox mask account [--off] [--yes] (Enterprise) +// +// Reading where masking stands is in every edition; setting it is Enterprise. +// Each setup, rule and account change is in the security log. +func maskCmd(args []string) { + if len(args) > 0 { + switch args[0] { + case "setup": + maskSetup(args[1:]) + return + case "rule": + maskRule(args[1:]) + return + case "account": + maskAccount(args[1:]) + return + case "status": + args = args[1:] + } + } + st, err := branch.MaskingStatus(maskBranch(args)) + must(err) + if hasFlag(args, "--json") { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + must(enc.Encode(st)) + return + } + printMaskStatus(st) +} + +func maskBranch(args []string) string { + if b := optValue(args, "--branch"); b != "" { + return b + } + return "main" +} + +func printMaskStatus(st branch.MaskStatus) { + if !st.On { + fmt.Printf("Masking: not set up on this install.\n") + if st.Writes { + fmt.Printf(" %s mask setup explains what it does first\n", brand.CLI) + } else { + fmt.Printf(" Setting it up is in FoxByte Enterprise.\n") + } + return + } + fmt.Printf("Masking: on. Branch %s.\n", st.Branch) + if len(st.Pending) > 0 { + fmt.Printf(" Started before masking was set up, so not masking until they start again: %s\n", strings.Join(st.Pending, ", ")) + } + fmt.Printf("\nRules (%d)\n", len(st.Rules)) + ready := map[string]bool{} + for _, t := range st.Ready { + ready[t] = true + } + for _, r := range st.Rules { + view := r.Table + if r.Schema != "public" { + view = r.Schema + "__" + r.Table + } + state := "masked" + if !ready[view] { + state = "NOT MADE: a masked reader is refused the table" + } + fmt.Printf(" %-40s %-8s %-10s %s (%s, %s)\n", r.Where(), r.Rule, r.Category, state, r.By, r.At) + } + if len(st.Rules) == 0 { + fmt.Printf(" none: %s mask rule sets one on a confirmed finding\n", brand.CLI) + } + fmt.Printf("\nMasked accounts (%d), on every branch they do not own\n", len(st.Accounts)) + for _, a := range st.Accounts { + fmt.Printf(" %-40s by %s at %s\n", a.Email, a.By, a.At) + } + if len(st.Accounts) == 0 { + fmt.Printf(" none: nobody is masked until an admin masks them\n") + } +} + +// maskSetup sets masking up on a branch: the key if the install has none, the +// databases restarted to mount it, then the roles and grants. +func maskSetup(args []string) { + requireFeature(edition.Masking) + b := maskBranch(args) + must(branch.MaskHAGuard()) + // Started before the key existed, or on an older image: it starts again. + restart := branch.MaskNeedsRestart(b) + if !hasFlag(args, "--yes") { + fmt.Printf("This sets masking up on %s.\n\n"+ + "It makes the install's masking key if it has none: 32 random bytes in\n"+ + "%s, readable only by the database server and mounted read-only\n"+ + "into each database. FoxByte never reads it back and never shows it.\n\n"+ + "Then it makes, on %s, the masked readers' role, the schema their masks\n"+ + "are read from and their rights on the tables. Nobody is masked yet:\n"+ + "an admin masks accounts with `%s mask account`, and rules are set on\n"+ + "confirmed findings with `%s mask rule`, which says what each costs.\n", + b, branch.MaskKeyDir(), b, brand.CLI, brand.CLI) + if restart { + fmt.Printf("\n%s restarts to mount the key: connections drop for 5–15 seconds\n"+ + "while it comes back, and nothing is lost.\n", b) + } + fmt.Printf("\nRun again with --yes to go ahead.\n") + return + } + must(branch.EnsureMaskImage()) + must(branch.EnsureMaskKey("default")) + if restart { + fmt.Printf("Restarting %s…\n", b) + if err := branch.RestartForMask(b); err != nil { + must(fmt.Errorf("%s did not come back with the key: %w", b, err)) + } + if b == "main" { + // The standby is a primary in waiting: one without the key would + // promote a primary whose masked readers see nothing. + if err := branch.RestartStandby(); err != nil { + fmt.Printf("The HA standby could not be restarted, so it has no key: %v\n"+ + "Rebuild it with `%s ha enable`.\n", err, brand.CLI) + } + } + } + must(branch.SetupMasking(b, cliActor())) + recordMask(auth.EvMaskSetup, b, "masking set up on "+b) + fmt.Printf("Done. `%s mask --branch %s` shows it.\n", brand.CLI, b) +} + +// maskRule sets a rule, or takes one off. Before a rule is set it says what +// it costs (design, decision 9). +func maskRule(args []string) { + requireFeature(edition.Masking) + ps := positionals(args, "--branch", "--path") + if len(ps) < 1 || len(ps) > 2 { + fmt.Fprintf(os.Stderr, "usage: %s mask rule <[schema.]table.column> [%s] [--path $.a.b] [--branch b] [--yes]\n"+ + " %s mask rule <[schema.]table.column> --off [--path $.a.b] [--branch b]\n", + brand.CLI, strings.Join(branch.MaskRules, "|"), brand.CLI) + os.Exit(2) + } + schema, table, column, err := splitColumn(ps[0]) + must(err) + b, path := maskBranch(args), optValue(args, "--path") + where := branch.MaskRule{Schema: schema, Table: table, Column: column, Path: path}.Where() + if hasFlag(args, "--off") { + must(branch.DropMaskRule(b, schema, table, column, path, cliActor())) + recordMask(auth.EvMaskRule, b, fmt.Sprintf("rule taken off %s on %s", where, b)) + fmt.Printf("Done: %s is no longer masked on %s.\n", where, b) + return + } + r := branch.MaskRule{Schema: schema, Table: table, Column: column, Path: path, By: cliActor()} + if len(ps) == 2 { + r.Rule = ps[1] + must(branch.CheckMaskRule(r.Rule, "")) + } + if !hasFlag(args, "--yes") { + rule := r.Rule + if rule == "" { + rule = "the default for its category" + } + fmt.Printf("This masks %s on %s, as %s, for every masked account.\n\n", where, b, rule) + fmt.Print(branch.MaskCost(r.Rule, path)) + fmt.Printf("\nRun again with --yes to set it.\n") + return + } + must(branch.SetMaskRule(b, r)) + recordMask(auth.EvMaskRule, b, fmt.Sprintf("rule %s set on %s on %s", orDefault(r.Rule), where, b)) + fmt.Printf("Done. `%s mask --branch %s` shows it.\n", brand.CLI, b) +} + +func orDefault(rule string) string { + if rule == "" { + return "(the category's default)" + } + return rule +} + +// splitColumn reads [schema.]table.column. +func splitColumn(s string) (schema, table, column string, err error) { + parts := strings.Split(s, ".") + switch len(parts) { + case 2: + schema, table, column = "public", parts[0], parts[1] + case 3: + schema, table, column = parts[0], parts[1], parts[2] + default: + return "", "", "", fmt.Errorf("a column is named [schema.]table.column, not %q", s) + } + for _, p := range []string{schema, table, column} { + if p == "" { + return "", "", "", fmt.Errorf("a column is named [schema.]table.column, not %q", s) + } + } + return schema, table, column, nil +} + +// maskAccount masks an account, or unmasks it. Their sessions on running +// branches end, so each next connection sees the change. +func maskAccount(args []string) { + requireFeature(edition.Masking) + ps := positionals(args) + if len(ps) != 1 { + fmt.Fprintf(os.Stderr, "usage: %s mask account [--off] [--yes]\n", brand.CLI) + os.Exit(2) + } + email, off := strings.ToLower(ps[0]), hasFlag(args, "--off") + if !hasFlag(args, "--yes") { + if off { + fmt.Printf("This unmasks %s: they see real values again on every branch.\n", email) + } else { + fmt.Printf("This masks %s on every branch they do not own: they read the masks of\n"+ + "marked tables, ordinary tables as they are, and cannot write to marked\n"+ + "tables. An admin is never masked.\n", email) + } + fmt.Printf("Their open sessions end, so the next one sees the change.\n\nRun again with --yes to go ahead.\n") + return + } + must(branch.MaskAccount(email, cliActor(), off)) + what := "masked" + if off { + what = "unmasked" + } + recordMask(auth.EvMaskAccount, "", what+" "+email) + fmt.Printf("Done: %s is %s.\n", email, what) +} + +func recordMask(kind, b, detail string) { + if s := openStore(); s != nil { + s.Audit(kind, cliActor(), b, "", detail) + s.Close() + } +} diff --git a/docker/postgres/Dockerfile b/docker/postgres/Dockerfile index 6c96bc3..1785a61 100644 --- a/docker/postgres/Dockerfile +++ b/docker/postgres/Dockerfile @@ -113,8 +113,9 @@ COPY --from=f1-build /out/ / # container (internal/branch/images.go, PGImageRevision). An install keeps the # image it first fetched, so a fox that needs something newer has to be able to # tell: 2 is the first with bb_identity. Raise it with anything a fox comes to -# depend on, and raise PGImageRevision with it. 3 is the first with pgAudit. -LABEL dev.dbengine.image-revision="3" +# depend on, and raise PGImageRevision with it. 3 is the first with pgAudit; +# 4 with bb_identity 0.2, which has the masking hash (F5). +LABEL dev.dbengine.image-revision="4" # Disposable point-in-time restore entrypoint (used by scripts/pitr_restore.sh). COPY restore-entrypoint.sh /usr/local/bin/restore-entrypoint.sh diff --git a/docs/FOX_Checklist.html b/docs/FOX_Checklist.html index 72040bd..b10dd1d 100644 --- a/docs/FOX_Checklist.html +++ b/docs/FOX_Checklist.html @@ -768,6 +768,14 @@

OPersonal data & complianceEach report in the security log; what it cannot claim stated in it
  • Applications: each registration, its account, its keys with age, state and what each asserted by branch, and findings to question; --csv applications (F1)
  • No unmask rights to review until masking (point 7); campaigns are 4.9 (Gate 2)
    • enterprise/accessreview unit; VM §11s; accessreview.spec.ts + O3Masking personal data (point 4.6 and F5, Enterprise; docs/FOX_F5_Masking_Design.md)
      +
    • Gate 0 prototype (make integration-f5, 57/57 on PostgreSQL 18 and 16): the keyed hash in bb_identity 0.2, its key a file the server reads; a masked branch on a ZFS clone left with no real value in its files; the costs measured
      • +
    • Step 1: fox mask setup|rule|account, four API routes and the console’s Masking page; rules (partial, keyed hash, fake, empty) only on confirmed findings, Aadhaar never hashed or faked; masks kept in shadow tables by statement triggers; masked accounts read masks and ordinary tables, never the real marked table, the hash or the key, and cannot write to marked tables; admins and a branch’s owner see real values; the cost said before a rule is set
      • +
    • Step 2: key domains and rotation
      • +
    • Step 3: unmask, the audited lookup, unmask rights in the access review, reads audited
      • +
    • Step 4: every access path tested (COPY, functions, EXPLAIN, statistics, logs)
      • +
    • Step 5: masked branches, closed until masked and verified
      • +
    • One-column primary keys only; a JSON path into an array, or a quoted key, is refused for now; masking stays on once set up; a masked account has no DDL rights
    enterprise/masking unit; internal/branch mask_test.go; VM §11ai; masking.spec.ts
    FOX_Checklist · area O
    @@ -784,7 +792,6 @@

    ZNot started (roadmap)—More import sources: Cassandra, DynamoDBI1–I8 —Release signatures — done 22 Sep 2026A9, M4 - —Masking (point 7, Enterprise; design proposed in docs/FOX_F5_Masking_Design.md)
    • Gate 0 prototype (make integration-f5, 57/57 on PostgreSQL 18 and 16): the keyed hash in bb_identity 0.2, its key a file the server reads; shadow tables, triggers and views; what a masked reader can and cannot reach; the lookup; a masked branch on a ZFS clone left with no real value in its files; the costs measured
    • Masked readers, rules on confirmed marks (partial, keyed hash, fake, empty), kept in shadow tables so no masking function runs at read time
    • The key outside every table; unmask and an audited lookup; every access path tested
    • Masked branches, closed until masked and verified
    O1, O2, H21 —Hosted cloud tier— —Networked storage disaggregation (README roadmap; see the storage engine note on page 1)C1, D1 diff --git a/docs/FOX_Checklist.pdf b/docs/FOX_Checklist.pdf index 76979c0..543a53d 100644 Binary files a/docs/FOX_Checklist.pdf and b/docs/FOX_Checklist.pdf differ diff --git a/docs/FOX_Feature_Implemented.html b/docs/FOX_Feature_Implemented.html index e975595..6f8306f 100644 --- a/docs/FOX_Feature_Implemented.html +++ b/docs/FOX_Feature_Implemented.html @@ -119,6 +119,7 @@

    Change log

    Newest first. Every change to FoxByte adds a row here.

    + @@ -403,6 +404,7 @@

    Feature index

    O · Personal data & compliance
    • O1Personal-data discovery: scan, review, keeping it currentWorking
    • O2Access review reportWorking
      • +
    • O3Masking personal dataPartial
    FOX_Feature Implemented · index
    @@ -1689,6 +1691,16 @@

    Test matrix

    Added in
    feat/access-review.
    +
    O3

    Masking personal data

    Partial
    +
    What it does
    An admin masks accounts; a masked account then reads, on every branch it does not own, the masks of the columns a reviewer confirmed as personal data, and everything else as it is. Each rule is on a confirmed finding (point 4.6), a column or a JSON path in it: partial (all but the last four characters as X), hash (a keyed HMAC-SHA-256, the same value the same mask, so filters, grouping and joins still work), fake (a made-up value of its kind, the same for the same value) or empty. Without a rule named, the category’s default: partial for Aadhaar, PAN, cards, mobiles, GSTIN and IFSC; hash for email; else empty. Aadhaar is never hashed or faked: a hash applications join on would be a reference key, which UIDAI forbids. The cost is said before a rule is set.
    +
    How it works
    Masks are made on write, not read: for a hash, a fake or a JSON path, a shadow table bb_mask.<schema>__<table> holds each row’s masks, kept by statement triggers with transition tables (inserts, changes, a moved key, deletes, TRUNCATE) owned by bb_masker, which nobody logs in as; a view with the table’s own name in bb_mask joins it to the table, and computes partial and empty masks itself, so a table with only those has no shadow and no write cost. Rows already there are filled in committed batches of 10,000 without holding up writers. A masked account’s role is in db_masked rather than db_client, with search_path = bb_mask, public and log_min_error_statement = panic (superuser-only, so it cannot undo them); db_masked has no right on a marked table, its shadow, the hash function or the key. The key is 32 random bytes in the install’s state directory, owned by the database server’s user and readable by nobody else, mounted read-only at /run/bb-mask; bb_identity 0.2’s bbid_mask_hmac reads it, and FoxByte never reads it back. Masked accounts are kept install-wide; the role is made on each branch when the account is used, and changed at once on running ones, with its open sessions ended. Every change is recorded in the Blackbox under the person who asked.
    +
    Who
    Admins only: they set masking up, set rules and mask accounts. An admin is never masked (they can change the masks), and a branch’s owner sees real values on it. Reading where masking stands is in every edition; setting it is Enterprise (masking). Every setup, rule and account change is in the security log (mask.setup, mask.rule, mask.account).
    +
    How to use
    fox mask [status] [--branch b] [--json] · fox mask setup [--branch b] [--yes] (makes the key; the branch restarts to mount it) · fox mask rule <[schema.]table.column> [partial|hash|fake|empty] [--path $.a.b] [--branch b] [--yes] and --off · fox mask account <email> [--off] [--yes] · GET /api/masking, POST /api/masking/setup|rule|account · the console’s Masking page (Compliance): set up, the rules and whether each table’s masks are made, the confirmed findings without a rule with the rules each may have, and the masked accounts. Each change is shown, with its cost, before it is made.
    +
    How to test
    Unit: enterprise/masking/sql_test.go (the shadow, triggers, fill and view for every rule; no shadow for partial alone; rights given back with no rules; what cannot be masked refused, saying why; names outside public), internal/branch/mask_test.go (Aadhaar refused a hash; the defaults; the masked and unmasked role; the accounts’ file and its exemptions; the licence; the mount; the cost). Integration §11ai. Playwright web/tests/masking.spec.ts.
    +
    Status & limits
    Partial: Gate 1 step 1 of the design. Next: key domains and rotation (step 2); unmask, the audited lookup and unmask rights in the access review (step 3); every access path tested (step 4); masked branches (step 5). A table needs a one-column primary key; a JSON path into an array, or a quoted key, is refused for now; masking stays on once set up; a masked account has no DDL rights; a schema made after setup needs setup run again. Measured cost on a table with a hash, fake or JSON mask: a fifth to a third fewer single-row writes, a 10,000-row UPDATE about 3.6 times as long; masked reads about a quarter slower; documents over 64 KB with a JSON path much more (design, decision 9).
    +
    Added in
    feat/f5-mask-step1.
    +
    +
    FOX_Feature Implemented · area O
    diff --git a/docs/FOX_Feature_Implemented.pdf b/docs/FOX_Feature_Implemented.pdf index f78a164..ee20dbb 100644 Binary files a/docs/FOX_Feature_Implemented.pdf and b/docs/FOX_Feature_Implemented.pdf differ diff --git a/enterprise/masking/doc.go b/enterprise/masking/doc.go new file mode 100644 index 0000000..4afab0e --- /dev/null +++ b/enterprise/masking/doc.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +// Package masking makes masks (F5, point 7): rules set on confirmed marks, +// kept in shadow tables by triggers, read through views by masked readers. +package masking + +// Untagged, so `go build ./...` succeeds without `-tags enterprise`. See +// enterprise/doc.go. diff --git a/enterprise/masking/masking.go b/enterprise/masking/masking.go new file mode 100644 index 0000000..9231ccb --- /dev/null +++ b/enterprise/masking/masking.go @@ -0,0 +1,194 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package masking + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/thefoxbyte/foxbyte/internal/branch" + "github.com/thefoxbyte/foxbyte/internal/brand" +) + +func init() { + branch.SetMasker(&branch.Masker{Setup: setup, SetRule: setRule, DropRule: dropRule, MaskTables: maskTables}) +} + +// setup makes the install's default key if it has none, then the branch's +// roles, schema, rules table and a masked reader's rights. It runs again +// safely: a table already masked is masked again from its rules. +func setup(b, actor string) error { + if err := branch.EnsureMaskKey(keyDomain); err != nil { + return fmt.Errorf("making the masking key: %w", err) + } + if err := branch.RunMaskSQL(b, actor, setupSQL(), false); err != nil { + return fmt.Errorf("setting masking up on %s: %w", b, err) + } + rules, err := branch.MaskRulesOn(b) + if err != nil { + return err + } + for _, t := range tablesOf(rules) { + if err := remask(b, t[0], t[1], actor, rules); err != nil { + return err + } + } + return nil +} + +// tablesOf is the tables rules are on, each once. +func tablesOf(rules []branch.MaskRule) [][2]string { + seen := map[[2]string]bool{} + var out [][2]string + for _, r := range rules { + k := [2]string{r.Schema, r.Table} + if !seen[k] { + seen[k] = true + out = append(out, k) + } + } + return out +} + +// confirmedCategory is the category of the confirmed finding at a place: a +// rule is set only on personal data a reviewer confirmed (point 4.6). +func confirmedCategory(b string, r branch.MaskRule) (string, error) { + // A branch never scanned has no findings, and so nothing confirmed. + if ok, err := branch.RelationExists(b, "bb.pii_findings"); err != nil || !ok { + return "", err + } + out, err := branch.QuerySQL(b, fmt.Sprintf(`SELECT coalesce((SELECT category FROM bb.pii_findings WHERE schema_name = %s + AND table_name = %s AND column_name = %s AND json_path = %s AND state = 'confirmed' ORDER BY id LIMIT 1), '')`, + lit(r.Schema), lit(r.Table), lit(r.Column), lit(r.Path))) + if err != nil { + return "", fmt.Errorf("reading the findings on %s: %w (%s)", b, err, strings.TrimSpace(out)) + } + return strings.TrimSpace(out), nil +} + +func setRule(b string, r branch.MaskRule) error { + if !hasSetup(b) { + return fmt.Errorf("%w: masking is not set up on %s: %s mask setup --branch %s", branch.ErrInvalidRequest, b, brand.CLI, b) + } + cat, err := confirmedCategory(b, r) + if err != nil { + return err + } + if cat == "" { + return fmt.Errorf("%w: no confirmed personal data at %s: a rule is set on a finding a reviewer confirmed (%s pii --branch %s, then %s pii mark confirm)", + branch.ErrInvalidRequest, r.Where(), brand.CLI, b, brand.CLI) + } + r.Category = cat + if r.Rule == "" { + r.Rule = branch.DefaultMaskRule(cat) + } + if err := branch.CheckMaskRule(r.Rule, cat); err != nil { + return err + } + // Checked against the table before the rule is kept: a rule that cannot + // be made is refused, not left waiting. + rules, err := branch.MaskRulesOn(b) + if err != nil { + return err + } + rules = replaceRule(rules, r) + cols, err := branch.MaskColumns(b, r.Schema, r.Table) + if err != nil { + return err + } + if _, err := maskTable(r.Schema, r.Table, cols, rulesFor(rules, r.Schema, r.Table)); err != nil { + return err + } + if err := branch.RunMaskSQL(b, r.By, []string{fmt.Sprintf(`INSERT INTO bb.mask_rules (schema_name, table_name, column_name, json_path, rule, category, set_by) + VALUES (%s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (schema_name, table_name, column_name, json_path) DO UPDATE SET rule = EXCLUDED.rule, category = EXCLUDED.category, + set_by = EXCLUDED.set_by, set_at = now()`, lit(r.Schema), lit(r.Table), lit(r.Column), lit(r.Path), lit(r.Rule), lit(cat), lit(r.By))}, false); err != nil { + return err + } + return remask(b, r.Schema, r.Table, r.By, rules) +} + +func dropRule(b, schema, table, column, path, actor string) error { + if !hasSetup(b) { + return fmt.Errorf("%w: masking is not set up on %s", branch.ErrInvalidRequest, b) + } + if err := branch.RunMaskSQL(b, actor, []string{fmt.Sprintf( + "DELETE FROM bb.mask_rules WHERE schema_name = %s AND table_name = %s AND column_name = %s AND json_path = %s", + lit(schema), lit(table), lit(column), lit(path))}, false); err != nil { + return err + } + rules, err := branch.MaskRulesOn(b) + if err != nil { + return err + } + return remask(b, schema, table, actor, rules) +} + +// remask makes a table's masks again from its rules: the old ones and the +// masked reader's rights taken away first, so between the two a masked reader +// is refused the table, never shown it half masked. +func remask(b, schema, table, actor string, all []branch.MaskRule) error { + cols, err := branch.MaskColumns(b, schema, table) + if err != nil { + return err + } + m, err := maskTable(schema, table, cols, rulesFor(all, schema, table)) + if err != nil { + return err + } + if err := branch.RunMaskSQL(b, actor, append(m.Drop, m.Build...), true); err != nil { + return fmt.Errorf("making %s.%s's masks: %w", schema, table, err) + } + if m.Fill != "" { + if err := branch.RunMaskScript(b, m.Fill); err != nil { + return fmt.Errorf("filling %s.%s's masks: %w (a masked reader is refused the table until it is masked again)", schema, table, err) + } + } + if len(m.View) > 0 { + if err := branch.RunMaskSQL(b, actor, m.View, false); err != nil { + return fmt.Errorf("making %s.%s's view: %w", schema, table, err) + } + } + return nil +} + +func replaceRule(rules []branch.MaskRule, r branch.MaskRule) []branch.MaskRule { + out := []branch.MaskRule{r} + for _, x := range rules { + if x.Schema == r.Schema && x.Table == r.Table && x.Column == r.Column && x.Path == r.Path { + continue + } + out = append(out, x) + } + return out +} + +func rulesFor(rules []branch.MaskRule, schema, table string) []branch.MaskRule { + var out []branch.MaskRule + for _, r := range rules { + if r.Schema == schema && r.Table == table { + out = append(out, r) + } + } + return out +} + +func hasSetup(b string) bool { + out, err := branch.QuerySQL(b, "SELECT to_regclass('bb.mask_rules') IS NOT NULL AND to_regrole('"+branch.MaskedRole+"') IS NOT NULL") + return err == nil && strings.TrimSpace(out) == "t" +} + +// maskTables lists the tables whose masks are made: the views in bb_mask. +func maskTables(b string) ([]string, error) { + out, err := branch.QuerySQL(b, `SELECT coalesce(json_agg(c.relname ORDER BY c.relname), '[]'::json) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = '`+branch.MaskSchema+`' AND c.relkind = 'v'`) + if err != nil { + return nil, err + } + var names []string + err = json.Unmarshal([]byte(strings.TrimSpace(out)), &names) + return names, err +} diff --git a/enterprise/masking/sql.go b/enterprise/masking/sql.go new file mode 100644 index 0000000..1430cbb --- /dev/null +++ b/enterprise/masking/sql.go @@ -0,0 +1,350 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package masking + +import ( + "fmt" + "regexp" + "sort" + "strings" + + "github.com/thefoxbyte/foxbyte/internal/branch" +) + +// The SQL that makes a table's masks (docs/FOX_F5_Masking_Design.md, sections +// 6 and 7, as the prototype proved them in section 17), made from its rules and +// its columns by pure functions, so the tests can read it. + +const keyDomain = "default" + +func ident(s string) string { return branch.QuoteIdent(s) } +func lit(s string) string { return branch.QuoteLiteral(s) } + +// names are a table's masking objects, all in bb_mask. The view has the +// table's own name for a table in public, so a masked reader's unqualified +// name reaches it; a table in another schema's view is __
    DateFeature(s)What changedCommit / PR
    6 Nov 2026O3Masking personal data, step 1 (F5 Gate 1, point 4.6, Enterprise; licence feature masking): rules on confirmed findings, and masked accounts. fox mask setup --branch b [--yes] previews, then makes the key (<state>/mask/default.key, 32 bytes, owned by the server’s user, 0400 in a 0700 folder, mounted read-only at /run/bb-mask; never in a table, a log or a message) and restarts the branch to mount it if it must. fox mask rule <[schema.]table.column> [--path $.a] [--rule hash|fake|partial|empty] [--off] [--yes] masks a column the personal-data scan found and a person confirmed, and says what it costs before it is set (design decision 9: the founders did not decide, so the recommendation stands): nothing on writes for partial or empty, about a fifth to a third fewer single-row writes and a 10,000-row UPDATE about 3.6 times as long for a hash or a fake, more for a JSON path. Aadhaar is offered partial and empty only, never hashed or faked. fox mask account <email> [--off] --yes masks an account: on main and any branch it does not own it reads the masks of marked tables and real values elsewhere, cannot write marked tables, and its failing statements are never logged with their text; an admin cannot be masked, and admins and a branch’s owner always see real values (decisions 2–4). Masks sit in shadow tables kept by statement triggers and are read through views, so nothing is computed at read time. The console’s Masking page sets masking up, sets and takes off rules behind a dialog showing the cost, and masks and unmasks accounts; the API is /api/masking (admins); every change is in the security log. Fixed on the way: an account’s role is found by name when it is unmasked (to_regrole never finds an email), and the uninstall removes the key’s folder although the server’s user owns it. Limits: one-column primary keys only; array or quoted JSON paths refused; masking stays on once set up; a new schema needs setup run again; key domains, rotation, lookup, reads audited and masked branches are steps 2–5. Tests: integration 792/792 (§11ai new: the key’s owner, modes and read-only mount, nothing in the logs; rules refused before confirmation; the cost; Aadhaar’s hash refused; each rule’s mask as a masked account reads it; joins on a hash; the real table, shadow and hash out of reach; writes refused; the application’s writes kept up; admins and owners see real values; rules off and accounts unmasked; the security log), v2 403/403, UI 47/47 plus the upgrade test (a rule set after its cost, Aadhaar’s choices, an account masked and unmasked), integration-f5 57/57, f1 70/70, f6 71/71, web unit 256/256, Go unit tests in both editions, vet for macOS, Linux and Windows.feat/f5-mask-step1
    5 Nov 2026O1, O2, H21Masking’s prototype (F5 Gate 0): the design’s claims, proven on PostgreSQL 18 and 16 (docs/FOX_F5_Masking_Design.md, section 17; make integration-f5, 57/57). bb_identity 0.2 adds bbid_mask_hmac(domain, value): HMAC-SHA-256 under a key domain’s key, a 32-byte file at /run/bb-mask/<domain>.key that must be the server’s own and readable by nobody else, never in a table or a message, read again when the file is replaced; never granted to PUBLIC (an upgrade from 0.1 included). The suite builds the image, and on throwaway containers shows: each rule’s mask (fake, hash, partial, empty, a JSON path); the real table, shadows, hash, fake function and key file out of a masked reader’s reach; filtering, grouping, joins on hashes, EXPLAIN, pg_stats and COPY as designed; no writes to marked tables; the lookup masked, 30 a minute, recorded without the value; the application’s writes, key changes, deletes and TRUNCATE kept up; key domains, a missing or loose key refused, rotation seen by a session that read the old key. It found and fixed: a refused lookup’s value reaching the server log (every masked login now has log_min_error_statement = panic), and a masked branch’s steps needing a statement each; with VACUUM FULL of the table and of pg_statistic, and WAL recycling off across three switches and checkpoints, a masked branch on a ZFS clone has no real value left in its files (a fresh cluster of only masked rows: 0.3 s for 20,000). Measured on PostgreSQL 18: single-row writes to a marked table 21–30% fewer, a 10,000-row UPDATE 5.6× (3.6× with a trigger a statement), indexed reads through the view 25% fewer, 1,000,000 masks filled in 14 s without holding up a writer and rotated in 21 s; over the 10% budget, so the cost is said before a rule is set (design decision 9). No product code yet: Gate 1 follows.feat/f5-mask-proto
    5 Nov 2026O1, O2, H21Masking’s design proposed (F5 and point 7; docs/FOX_F5_Masking_Design.md, with the probes in docs/f5/). Checked on a throwaway PostgreSQL 18.6: a view whose masking function the reader may not EXECUTE fails to read at all, so masks computed in a view make the function an oracle (the attack behind PostgreSQL Anonymizer’s CVE); masks kept in a shadow table by a trigger, read through a view with nothing computed at read time, work for WHERE, GROUP BY, ORDER BY and EXPLAIN while the hash stays out of reach; pg_stats shows nothing for a table a reader has no grant on; a unique violation hides the value from a writer who cannot read it, but not its existence. Proposed: FoxByte’s own masks rather than Anonymizer (also not packaged for PostgreSQL 18 in apt.postgresql.org); masked readers who read masks of marked tables and do not write them; nobody masked until an admin says so, admins and owners always seeing real values; the HMAC key in a file the server reads, never a table, its boundary stated; Aadhaar never hashed; masked branches masked in place, vacuumed, analysed, verified and closed until then. No code: the prototype (Gate 0) follows agreement.docs/f5-masking-design
    5 Nov 2026H22Hardening fixes and profiles: what the report finds, fixed after a preview (points 3.3 and 3.4, Enterprise; a new licence feature, hardening). fox hardening fix <check> [--branch b] [--yes] shows what fixing a failing check changes and runs, and with --yes makes it from a plan made again now, then runs the check again and says if something still overrides it. Only fixes safe to make without more than a yes are offered: settings by ALTER SYSTEM and a reload (connections and disconnections logged, log levels, the statement behind an error with the audit log off, verbosity, host names, the line prefix, idle transactions ended, the TLS version, fsync and full-page writes, SCRAM when no MD5 hash is kept), CREATE on the public schema taken from PUBLIC (recorded in the Blackbox under the person who asked; FoxByte’s client role keeps its own grant), BYPASSRLS taken from roles that are not superusers, and the data directory’s mode. What could lock someone out or stop an application (superusers, pg_hba.conf, turning the audit log on) stays a person’s, with the command. fox hardening profile baseline|strict [--branch b] [--yes] previews every fix of its levels and the failing checks it leaves to a person, then makes them one by one, each checked again, and records the profile against the branch, which the report shows. The fixes live in the branch’s own data directory, so a branch made afterwards has them. The console’s Hardening page has a Fix… button per fixable check and a Profiles panel, each behind a confirmation showing the change; the API is POST /api/hardening/fix and POST /api/hardening/profile (admins); every fix and profile is in the security log. Tests: integration 765/765 (11 new in §11ah: a preview that changes nothing; a fix made and checked again; nothing to fix on a passing check, a person’s check named as theirs; the public schema’s CREATE taken while clients still create through the Gateway; both profiles, previewed then applied and recorded; a branch made afterwards has the fixes; the API for admins; the security log), v2 403/403, UI 46/46 plus the upgrade test (a fix made from the page and a profile previewed), web unit 256/256, Go unit tests in both editions (the gate, a profile’s choice, the record, no fix for what is a person’s), Windows builds ok.feat/hardening-fixes
    . +type names struct { + base, shadow, view, fn string +} + +func namesFor(schema, table string) names { + view := table + if schema != "public" { + view = schema + "__" + table + } + return names{ + base: ident(schema) + "." + ident(table), + shadow: branch.MaskSchema + "." + ident(schema+"__"+table), + view: branch.MaskSchema + "." + ident(view), + fn: branch.MaskSchema + "." + ident("keep__"+schema+"__"+table), + } +} + +// A JSON path the masks follow: $.a.b, keys only. An array's elements ($.a[*]) +// are not followed yet, and say so. +var pathRE = regexp.MustCompile(`^\$(\.[A-Za-z_][A-Za-z0-9_]*)+$`) + +func pathKeys(p string) ([]string, error) { + if !pathRE.MatchString(p) { + return nil, fmt.Errorf("%w: the masks follow object keys ($.contact.mobile), not %q: an array's elements are not masked yet", branch.ErrInvalidRequest, p) + } + return strings.Split(strings.TrimPrefix(p, "$."), "."), nil +} + +// hash is the keyed hash of an expression as text. +func hash(expr string) string { + return fmt.Sprintf("bb_f1.bbid_mask_hmac(%s, (%s)::text)", lit(keyDomain), expr) +} + +// partial keeps the last four characters, every other one an X, so the mask +// is as long as the value. +func partial(expr string) string { + return fmt.Sprintf("(repeat('X', greatest(length((%[1]s)::text) - 4, 0)) || right((%[1]s)::text, 4))", expr) +} + +// scalar is a rule applied to a text value. +func scalar(rule, category, expr string) string { + switch rule { + case branch.MaskHash: + return hash(expr) + case branch.MaskFake: + return fmt.Sprintf("%s.fake(%s, %s)", branch.MaskSchema, lit(strings.ToLower(category)), hash(expr)) + case branch.MaskPartial: + return partial(expr) + } + return "NULL" +} + +// tableMask is what masking one table takes. +type tableMask struct { + Drop []string // the masks it replaces, and the masked reader's rights on the table + Build []string // the shadow and its triggers + Fill string // the masks of the rows already there, in batches + View []string // the view, once the shadow is full +} + +// shadowCol is a column the shadow keeps a mask for: hash and fake columns, +// and a JSON column with a path rule, kept as the whole document masked. +type shadowCol struct { + name, typ, expr string // expr over the row alias n +} + +// maskTable makes it, from the table's columns and its rules. A table with no +// rules gets only Drop and its rights back. +func maskTable(schema, table string, cols []branch.MaskColumn, rules []branch.MaskRule) (tableMask, error) { + nm := namesFor(schema, table) + var keys []branch.MaskColumn + byName := map[string]branch.MaskColumn{} + for _, c := range cols { + byName[c.Name] = c + if c.Key { + keys = append(keys, c) + } + } + var m tableMask + m.Drop = []string{ + "REVOKE ALL ON " + nm.base + " FROM " + branch.MaskedRole, + "DROP VIEW IF EXISTS " + nm.view, + "DROP TRIGGER IF EXISTS bb_mask_i ON " + nm.base, + "DROP TRIGGER IF EXISTS bb_mask_u ON " + nm.base, + "DROP TRIGGER IF EXISTS bb_mask_d ON " + nm.base, + "DROP TRIGGER IF EXISTS bb_mask_t ON " + nm.base, + "DROP FUNCTION IF EXISTS " + nm.fn + "()", + "DROP TABLE IF EXISTS " + nm.shadow, + } + if len(rules) == 0 { + m.Drop = append(m.Drop, "GRANT SELECT, INSERT, UPDATE, DELETE ON "+nm.base+" TO "+branch.MaskedRole) + return m, nil + } + if len(keys) != 1 { + return m, fmt.Errorf("%w: %s.%s has %s: masks are kept by a one-column primary key", branch.ErrInvalidRequest, schema, table, + map[bool]string{true: "no primary key", false: "a primary key of several columns"}[len(keys) == 0]) + } + pk := keys[0] + + // The rules by column: the column's own, and its JSON paths. + own := map[string]branch.MaskRule{} + paths := map[string][]branch.MaskRule{} + for _, r := range rules { + c, ok := byName[r.Column] + if !ok { + return m, fmt.Errorf("%w: %s.%s has no column %q", branch.ErrInvalidRequest, schema, table, r.Column) + } + if c.Key { + return m, fmt.Errorf("%w: %s is the table's key, which the masks are kept by: it cannot be masked", branch.ErrInvalidRequest, r.Where()) + } + if r.Path == "" { + own[r.Column] = r + continue + } + if c.Type != "jsonb" { + return m, fmt.Errorf("%w: a JSON path is masked in a jsonb column, and %s is %s", branch.ErrInvalidRequest, r.Column, c.Type) + } + if _, err := pathKeys(r.Path); err != nil { + return m, err + } + paths[r.Column] = append(paths[r.Column], r) + } + + var shadow []shadowCol + for _, c := range cols { + if r, ok := own[c.Name]; ok && (r.Rule == branch.MaskHash || r.Rule == branch.MaskFake) { + shadow = append(shadow, shadowCol{c.Name, "text", scalar(r.Rule, r.Category, "n."+ident(c.Name))}) + continue + } + if ps, ok := paths[c.Name]; ok { + if _, whole := own[c.Name]; whole { + return m, fmt.Errorf("%w: %s has a rule of its own and on its paths: take one off", branch.ErrInvalidRequest, c.Name) + } + sort.Slice(ps, func(i, j int) bool { return ps[i].Path < ps[j].Path }) + doc := "n." + ident(c.Name) + for _, r := range ps { + ks, _ := pathKeys(r.Path) + arr := "'{" + strings.Join(ks, ",") + "}'" + val := fmt.Sprintf("to_jsonb(%s)", scalar(r.Rule, r.Category, "n."+ident(c.Name)+" #>> "+arr)) + if r.Rule == branch.MaskEmpty { + val = "'null'::jsonb" + } + doc = fmt.Sprintf("CASE WHEN n.%[1]s #> %[2]s IS NULL THEN %[3]s ELSE jsonb_set(%[3]s, %[2]s, %[4]s) END", + ident(c.Name), arr, doc, val) + } + shadow = append(shadow, shadowCol{c.Name, "jsonb", doc}) + } + } + + if len(shadow) == 0 { + // Only partial and empty: no secret, so nothing is kept, and writes + // to the table cost nothing more. The view computes the masks. + m.View = viewSQL(nm, cols, own, nil, "") + return m, nil + } + + // The shadow: the key, and the masks kept. + defs := []string{ident(pk.Name) + " " + pk.Type + " PRIMARY KEY"} + var colNames, exprs, sets []string + for _, s := range shadow { + defs = append(defs, ident(s.name)+" "+s.typ) + colNames = append(colNames, ident(s.name)) + exprs = append(exprs, s.expr) + sets = append(sets, ident(s.name)+" = EXCLUDED."+ident(s.name)) + } + k := ident(pk.Name) + m.Build = []string{ + "CREATE TABLE " + nm.shadow + " (" + strings.Join(defs, ", ") + ")", + "ALTER TABLE " + nm.shadow + " OWNER TO " + branch.MaskerRole, + "GRANT SELECT ON " + nm.base + " TO " + branch.MaskerRole, + } + for _, s := range shadow { + if s.typ == "text" { + m.Build = append(m.Build, fmt.Sprintf("CREATE INDEX ON %s (%s)", nm.shadow, ident(s.name))) + } + } + upsert := func(from string) string { + cols, vals, set := "", "", "" + if len(colNames) > 0 { + cols, vals = ", "+strings.Join(colNames, ", "), ", "+strings.Join(exprs, ", ") + set = " ON CONFLICT (" + k + ") DO UPDATE SET " + strings.Join(sets, ", ") + } else { + set = " ON CONFLICT (" + k + ") DO NOTHING" + } + return fmt.Sprintf("INSERT INTO %s (%s%s) SELECT n.%s%s FROM %s%s", nm.shadow, k, cols, k, vals, from, set) + } + // One trigger a statement, from its transition tables (decision 9, and + // section 17: 198 ms rather than 307 for a 10,000-row update). + m.Build = append(m.Build, + fmt.Sprintf(`CREATE FUNCTION %s() RETURNS trigger LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $bbmask$ +BEGIN + IF TG_OP = 'TRUNCATE' THEN + TRUNCATE %[2]s; + RETURN NULL; + END IF; + IF TG_OP = 'DELETE' THEN + DELETE FROM %[2]s s USING o WHERE s.%[3]s = o.%[3]s; + RETURN NULL; + END IF; + IF TG_OP = 'UPDATE' THEN + DELETE FROM %[2]s s USING o WHERE s.%[3]s = o.%[3]s AND NOT EXISTS (SELECT 1 FROM n WHERE n.%[3]s = o.%[3]s); + END IF; + %[4]s; + RETURN NULL; +END $bbmask$`, nm.fn, nm.shadow, k, upsert("n")), + "ALTER FUNCTION "+nm.fn+"() OWNER TO "+branch.MaskerRole, + "REVOKE EXECUTE ON FUNCTION "+nm.fn+"() FROM PUBLIC", + fmt.Sprintf("CREATE TRIGGER bb_mask_i AFTER INSERT ON %s REFERENCING NEW TABLE AS n FOR EACH STATEMENT EXECUTE FUNCTION %s()", nm.base, nm.fn), + fmt.Sprintf("CREATE TRIGGER bb_mask_u AFTER UPDATE ON %s REFERENCING OLD TABLE AS o NEW TABLE AS n FOR EACH STATEMENT EXECUTE FUNCTION %s()", nm.base, nm.fn), + fmt.Sprintf("CREATE TRIGGER bb_mask_d AFTER DELETE ON %s REFERENCING OLD TABLE AS o FOR EACH STATEMENT EXECUTE FUNCTION %s()", nm.base, nm.fn), + fmt.Sprintf("CREATE TRIGGER bb_mask_t AFTER TRUNCATE ON %s FOR EACH STATEMENT EXECUTE FUNCTION %s()", nm.base, nm.fn), + ) + + // The rows already there, 10,000 at a time, each batch committed: a + // writer meanwhile is never held up (section 17: 1,000,000 in 14 s). + m.Fill = fmt.Sprintf(`DO $bbfill$ +DECLARE last %[1]s; nxt %[1]s; +BEGIN + LOOP + SELECT max(%[2]s) INTO nxt FROM (SELECT %[2]s FROM %[3]s WHERE last IS NULL OR %[2]s > last ORDER BY %[2]s LIMIT 10000) b; + EXIT WHEN nxt IS NULL; + %[4]s; + last := nxt; + COMMIT; + END LOOP; +END $bbfill$;`, pk.Type, k, nm.base, + strings.Replace(upsert(nm.base+" n WHERE (last IS NULL OR n."+k+" > last) AND n."+k+" <= nxt"), "DO UPDATE SET "+strings.Join(sets, ", "), "DO NOTHING", 1)) + + m.View = viewSQL(nm, cols, own, shadow, k) + return m, nil +} + +// viewSQL is the view: every column, in order, the masked ones as their +// masks, joined to the shadow on key k when there is one. +func viewSQL(nm names, cols []branch.MaskColumn, own map[string]branch.MaskRule, shadow []shadowCol, k string) []string { + kept := map[string]bool{} + for _, s := range shadow { + kept[s.name] = true + } + var sel []string + for _, c := range cols { + q := ident(c.Name) + switch r, ok := own[c.Name]; { + case kept[c.Name]: + sel = append(sel, "m."+q+" AS "+q) + case ok && r.Rule == branch.MaskPartial: + sel = append(sel, partial("n."+q)+" AS "+q) + case ok && r.Rule == branch.MaskEmpty: + sel = append(sel, "NULL::"+c.Type+" AS "+q) + default: + sel = append(sel, "n."+q) + } + } + from := nm.base + " n" + if len(shadow) > 0 { + from += fmt.Sprintf(" JOIN %s m ON m.%s = n.%s", nm.shadow, k, k) + } + return []string{ + fmt.Sprintf("CREATE VIEW %s AS SELECT %s FROM %s", nm.view, strings.Join(sel, ", "), from), + "ALTER VIEW " + nm.view + " OWNER TO " + branch.MaskerRole, + "GRANT SELECT ON " + nm.base + " TO " + branch.MaskerRole, + "GRANT SELECT ON " + nm.view + " TO " + branch.MaskedRole, + } +} + +// setupSQL is what a branch needs before any table is masked: the hash +// (bb_identity 0.2), the roles, the schema, the rules table, the fake values, +// and a masked reader's rights on every table that is not masked. +func setupSQL() []string { + return []string{ + `DO $bbsetup$ BEGIN + IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'bb_identity') THEN + EXECUTE 'ALTER EXTENSION bb_identity UPDATE'; + ELSE + EXECUTE 'CREATE SCHEMA IF NOT EXISTS bb_f1'; + EXECUTE 'CREATE EXTENSION bb_identity WITH SCHEMA bb_f1'; + END IF; + IF to_regrole('` + branch.MaskerRole + `') IS NULL THEN EXECUTE 'CREATE ROLE ` + branch.MaskerRole + ` NOLOGIN'; END IF; + IF to_regrole('` + branch.MaskedRole + `') IS NULL THEN EXECUTE 'CREATE ROLE ` + branch.MaskedRole + ` NOLOGIN'; END IF; +END $bbsetup$`, + "GRANT USAGE ON SCHEMA bb_f1 TO " + branch.MaskerRole, + "GRANT EXECUTE ON FUNCTION bb_f1.bbid_mask_hmac(text, text) TO " + branch.MaskerRole, + "CREATE SCHEMA IF NOT EXISTS " + branch.MaskSchema + " AUTHORIZATION " + branch.MaskerRole, + "GRANT USAGE ON SCHEMA " + branch.MaskSchema + " TO " + branch.MaskedRole, + `CREATE TABLE IF NOT EXISTS bb.mask_rules ( + schema_name text NOT NULL, table_name text NOT NULL, column_name text NOT NULL, json_path text NOT NULL DEFAULT '', + rule text NOT NULL CHECK (rule IN ('partial', 'hash', 'fake', 'empty')), category text NOT NULL DEFAULT '', + set_by text NOT NULL, set_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (schema_name, table_name, column_name, json_path))`, + "REVOKE ALL ON bb.mask_rules FROM PUBLIC", + // fake: a realistic value, the same for the same input, chosen by the + // hash. Never granted: it takes a hash, and is called by the triggers. + `CREATE OR REPLACE FUNCTION ` + branch.MaskSchema + `.fake(kind text, h text) RETURNS text LANGUAGE sql IMMUTABLE AS $bbfake$ + SELECT CASE kind + WHEN 'name' THEN (ARRAY['Anil Mehta','Bina Das','Chetan Iyer','Divya Nair','Eshan Gupta','Farah Khan','Gopal Rao','Hema Pillai', + 'Imran Sheikh','Jaya Menon','Karan Shah','Lata Joshi','Mohan Reddy','Neha Kapoor','Omar Ali','Priya Sen']) + [1 + ('x' || substr(h, 1, 7))::bit(28)::int % 16] + WHEN 'email' THEN 'user' || substr(h, 1, 10) || '@example.invalid' + WHEN 'mobile' THEN '+91 9' || lpad((('x' || substr(h, 1, 8))::bit(32)::bigint % 1000000000)::text, 9, '0') + WHEN 'city' THEN (ARRAY['Pune','Delhi','Chennai','Kolkata','Jaipur','Lucknow','Bhopal','Kochi'])[1 + ('x' || substr(h, 1, 7))::bit(28)::int % 8] + ELSE 'x' || substr(h, 1, 12) END $bbfake$`, + "ALTER FUNCTION " + branch.MaskSchema + ".fake(text, text) OWNER TO " + branch.MaskerRole, + "REVOKE EXECUTE ON FUNCTION " + branch.MaskSchema + ".fake(text, text) FROM PUBLIC", + // A masked reader reads and writes every table as any account does, + // except the masked ones, whose rights maskTable takes away: on what is + // here now, and on what db_client makes later. + `DO $bbgrant$ +DECLARE s text; +BEGIN + FOR s IN SELECT nspname FROM pg_namespace + WHERE nspname NOT LIKE 'pg\_%' AND nspname NOT IN ('information_schema', 'bb', 'bb_f1', '` + branch.MaskSchema + `') + LOOP + EXECUTE format('GRANT USAGE ON SCHEMA %I TO ` + branch.MaskedRole + `', s); + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA %I TO ` + branch.MaskedRole + `', s); + EXECUTE format('GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA %I TO ` + branch.MaskedRole + `', s); + IF to_regrole('db_client') IS NOT NULL THEN + EXECUTE format('ALTER DEFAULT PRIVILEGES FOR ROLE db_client IN SCHEMA %I GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ` + branch.MaskedRole + `', s); + EXECUTE format('ALTER DEFAULT PRIVILEGES FOR ROLE db_client IN SCHEMA %I GRANT USAGE, SELECT ON SEQUENCES TO ` + branch.MaskedRole + `', s); + END IF; + END LOOP; +END $bbgrant$`, + } +} diff --git a/enterprise/masking/sql_test.go b/enterprise/masking/sql_test.go new file mode 100644 index 0000000..1786569 --- /dev/null +++ b/enterprise/masking/sql_test.go @@ -0,0 +1,146 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package masking + +import ( + "errors" + "os" + "strings" + "testing" + + "github.com/thefoxbyte/foxbyte/internal/branch" +) + +var customers = []branch.MaskColumn{ + {Name: "id", Type: "bigint", Key: true}, + {Name: "name", Type: "text"}, + {Name: "email", Type: "text"}, + {Name: "phone", Type: "text"}, + {Name: "aadhaar", Type: "text"}, + {Name: "notes", Type: "text"}, + {Name: "prefs", Type: "jsonb"}, + {Name: "city", Type: "text"}, +} + +func rule(col, path, r, cat string) branch.MaskRule { + return branch.MaskRule{Schema: "public", Table: "customers", Column: col, Path: path, Rule: r, Category: cat} +} + +var everyRule = []branch.MaskRule{ + rule("name", "", "fake", "name"), + rule("email", "", "hash", "email"), + rule("phone", "", "partial", "mobile"), + rule("aadhaar", "", "partial", "aadhaar"), + rule("notes", "", "empty", "other"), + rule("prefs", "$.contact.mobile", "partial", "mobile"), +} + +// Every rule: a shadow for what needs the key, the view computing the rest, +// the hash never in the view, a trigger a statement, and the rights taken +// before anything is made. +func TestMaskTableEveryRule(t *testing.T) { + m, err := maskTable("public", "customers", customers, everyRule) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(m.Drop[0], `REVOKE ALL ON "public"."customers" FROM db_masked`) { + t.Errorf("the masked reader's rights should go first: %s", m.Drop[0]) + } + build := strings.Join(m.Build, ";\n") + for _, want := range []string{ + `CREATE TABLE bb_mask."public__customers"`, + "REFERENCING OLD TABLE AS o NEW TABLE AS n FOR EACH STATEMENT", + `AFTER TRUNCATE ON "public"."customers" FOR EACH STATEMENT`, + `bb_f1.bbid_mask_hmac('default', (n."email")::text)`, + `bb_mask.fake('name', bb_f1.bbid_mask_hmac('default', (n."name")::text))`, + `jsonb_set(n."prefs", '{contact,mobile}'`, + "SECURITY DEFINER SET search_path = pg_catalog", + `OWNER TO bb_masker`, + } { + if !strings.Contains(build, want) { + t.Errorf("build is missing %q:\n%s", want, build) + } + } + view := m.View[0] + for _, want := range []string{`m."name" AS "name"`, `m."email" AS "email"`, `right((n."phone")::text, 4)) AS "phone"`, + `NULL::text AS "notes"`, `m."prefs" AS "prefs"`, `n."city"`, `JOIN bb_mask."public__customers" m ON m."id" = n."id"`} { + if !strings.Contains(view, want) { + t.Errorf("the view is missing %q:\n%s", want, view) + } + } + if strings.Contains(view, "bbid_mask_hmac") || strings.Contains(view, ".fake(") { + t.Errorf("the view computes a hash, which a masked reader would need to call:\n%s", view) + } + if !strings.Contains(m.Fill, "LIMIT 10000") || !strings.Contains(m.Fill, "COMMIT") || !strings.Contains(m.Fill, "DO NOTHING") { + t.Errorf("the fill should go in committed batches:\n%s", m.Fill) + } + if got := m.View[len(m.View)-1]; got != `GRANT SELECT ON bb_mask."customers" TO db_masked` { + t.Errorf("the last grant: %s", got) + } + if f := os.Getenv("F5_DUMP"); f != "" { + _ = os.WriteFile(f, []byte(strings.Join(setupSQL(), ";\n")+";\n"+strings.Join(append(m.Drop, m.Build...), ";\n")+";\n"+m.Fill+"\n"+strings.Join(m.View, ";\n")+";\n"), 0o644) + } +} + +// Only partial and empty need no key: no shadow, no trigger, nothing more on +// a write. +func TestMaskTableWithoutAShadow(t *testing.T) { + m, err := maskTable("public", "customers", customers, []branch.MaskRule{rule("aadhaar", "", "partial", "aadhaar")}) + if err != nil { + t.Fatal(err) + } + if len(m.Build) != 0 || m.Fill != "" { + t.Fatalf("a shadow for partial alone: %v %q", m.Build, m.Fill) + } + if !strings.Contains(m.View[0], `FROM "public"."customers" n`) || strings.Contains(m.View[0], "JOIN") { + t.Fatalf("the view: %s", m.View[0]) + } +} + +// No rules: the masks go, and the masked reader's rights come back. +func TestMaskTableNoRules(t *testing.T) { + m, err := maskTable("public", "customers", customers, nil) + if err != nil { + t.Fatal(err) + } + if last := m.Drop[len(m.Drop)-1]; last != `GRANT SELECT, INSERT, UPDATE, DELETE ON "public"."customers" TO db_masked` || len(m.View) != 0 { + t.Fatalf("no rules: %v %v", m.Drop, m.View) + } +} + +// What cannot be masked is refused, saying why. +func TestMaskTableRefuses(t *testing.T) { + noKey := []branch.MaskColumn{{Name: "email", Type: "text"}} + two := []branch.MaskColumn{{Name: "a", Type: "int", Key: true}, {Name: "b", Type: "int", Key: true}, {Name: "email", Type: "text"}} + for name, c := range map[string]struct { + cols []branch.MaskColumn + rules []branch.MaskRule + want string + }{ + "no key": {noKey, []branch.MaskRule{rule("email", "", "hash", "email")}, "no primary key"}, + "two keys": {two, []branch.MaskRule{rule("email", "", "hash", "email")}, "several columns"}, + "the key": {customers, []branch.MaskRule{rule("id", "", "hash", "other")}, "the table's key"}, + "no column": {customers, []branch.MaskRule{rule("ssn", "", "hash", "other")}, "no column"}, + "not jsonb": {customers, []branch.MaskRule{rule("city", "$.a", "partial", "other")}, "jsonb column"}, + "array path": {customers, []branch.MaskRule{rule("prefs", "$.orders[*].email", "hash", "email")}, "array's elements"}, + "both levels": {customers, []branch.MaskRule{rule("prefs", "", "empty", "other"), rule("prefs", "$.a", "partial", "other")}, "take one off"}, + } { + _, err := maskTable("public", "customers", c.cols, c.rules) + if err == nil || !errors.Is(err, branch.ErrInvalidRequest) || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: %v", name, err) + } + } +} + +// A table outside public: its view is named for its schema too. +func TestMaskNames(t *testing.T) { + n := namesFor("crm", "leads") + if n.view != `bb_mask."crm__leads"` || n.shadow != `bb_mask."crm__leads"` { + t.Fatalf("%+v", n) + } + if namesFor("public", "orders").view != `bb_mask."orders"` { + t.Fatal(namesFor("public", "orders")) + } +} diff --git a/internal/auth/audit.go b/internal/auth/audit.go index bb25636..ae55b66 100644 --- a/internal/auth/audit.go +++ b/internal/auth/audit.go @@ -127,6 +127,12 @@ const ( // check or profile, on which branch, what changed, and whether it held. EvHardeningFixed = "hardening.fixed" EvHardeningProfile = "hardening.profile" + // Masking (F5): set up on a branch, a rule set or taken off, an account + // masked or unmasked. Who decided what a person sees of personal data is + // something an auditor asks about. + EvMaskSetup = "mask.setup" + EvMaskRule = "mask.rule" + EvMaskAccount = "mask.account" // The databases' audit log (F6) turned on or off: who decided. Whoever // can stop the record of who did what is someone an auditor asks about. EvAuditLogOn = "audit-log.on" diff --git a/internal/branch/branch.go b/internal/branch/branch.go index 41482dd..64b342b 100644 --- a/internal/branch/branch.go +++ b/internal/branch/branch.go @@ -314,6 +314,8 @@ func startContainer(name string, primary bool) error { "-v", mountpoint(name) + ":/var/lib/postgresql/data", } args = append(args, restartPolicy...) + // The masking keys, read-only, once masking is set up (mask.go). + args = append(args, maskMount()...) // The audit log's directory, outside the data directory, which a clone // would copy. Made first: a logging process that cannot write its file // stops the server starting. @@ -615,17 +617,33 @@ func EnsureUserRole(branchName, email string) error { if branchName == "" { branchName = "main" } - k := branchName + "\x00" + email + // Masked or not is part of the key: an admin masking an account changes + // what its role must be, in every process that makes roles (mask.go). + maskedHere := accountMasked(branchName, email) + state := "plain" + if maskedHere { + state = "masked" + } + k := branchName + "\x00" + email + "\x00" + state if _, ok := ensuredRoles.Load(k); ok { return nil } // One at a time per branch and role, and the one that waited finds the // work done (roleguard.go). - defer lockRole("user\x00" + k)() + defer lockRole("user\x00" + branchName + "\x00" + email)() if _, ok := ensuredRoles.Load(k); ok { return nil } - if err := provisionRole(branchName, userRoleSQL(email, UserRolePassword(email))); err != nil { + sql := userRoleSQL(email, UserRolePassword(email)) + switch { + case maskedHere: + sql = maskedRoleSQL(email, UserRolePassword(email)) + case MaskOn(): + // Unmasked, or exempt here: a role once a masked reader's is made an + // ordinary account's again. + sql += "\n" + unmaskedRoleSQL(email) + } + if err := provisionRole(branchName, sql); err != nil { return err } ensuredRoles.Store(k, struct{}{}) diff --git a/internal/branch/ha.go b/internal/branch/ha.go index 6a3bacf..0b2a98e 100644 --- a/internal/branch/ha.go +++ b/internal/branch/ha.go @@ -304,6 +304,8 @@ func standbyRunArgs(dataPath string) []string { } args = append(args, auditMount("standby")...) } + // The masking keys, as main has them: a standby promoted keeps masking. + args = append(args, maskMount()...) // The standby streams from the primary and archives WAL after a failover, so // it is long-lived and must come back on its own like main (see restartPolicy). args = append(args, restartPolicy...) diff --git a/internal/branch/images.go b/internal/branch/images.go index 40efb96..437a4ea 100644 --- a/internal/branch/images.go +++ b/internal/branch/images.go @@ -79,9 +79,9 @@ var SupportedPGMajors = []string{"16", PGMajor} // image it first fetched, so an image older than this is fetched again, or // built, when the engine starts (ensureImageRef). 2 added bb_identity, which // application identity (F1) needs; 3, pgAudit and bb_identity's audit hooks -// (F6), which `fox audit-log setup` needs. An image without the label is -// revision 1. -const PGImageRevision = 3 +// (F6), which `fox audit-log setup` needs; 4, bb_identity 0.2's masking hash +// (F5), which `fox mask setup` needs. An image without the label is revision 1. +const PGImageRevision = 4 // imageRevisionLabel is the label that carries it. const imageRevisionLabel = "dev.dbengine.image-revision" diff --git a/internal/branch/mask.go b/internal/branch/mask.go new file mode 100644 index 0000000..f5c5d30 --- /dev/null +++ b/internal/branch/mask.go @@ -0,0 +1,665 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "strings" + "sync" + "time" + + "github.com/thefoxbyte/foxbyte/internal/auth" + "github.com/thefoxbyte/foxbyte/internal/brand" + "github.com/thefoxbyte/foxbyte/internal/edition" +) + +// Masking (F5, point 7; docs/FOX_F5_Masking_Design.md). Gate 1, step 1: rules +// on confirmed marks, masked readers, and masks kept on a branch. +// +// What is here is what every edition needs once masks exist: the key's mount +// into the containers, and the masked reader's role, which the code that makes +// every account's role (EnsureUserRole) has to know about, and reading the +// rules and the masked accounts. Setting rules up, masking an account and +// making the masks are Enterprise (enterprise/masking). Masks already made +// keep working in every edition, as anchors and findings do: a masked account +// stays masked, and its role says so. + +const ( + // The roles (design, sections 6 and 10). Brand-free, like every stored name. + MaskedRole = "db_masked" // what a masked account's role is in, instead of db_client + MaskerRole = "bb_masker" // owns the shadows, the triggers and the views; nobody logs in as it + // MaskSchema holds the views a masked reader reads, the shadows and the + // lookup; first in a masked reader's search_path. + MaskSchema = "bb_mask" + // maskKeysInContainer is where bb_identity reads the keys (bbid_mask_hmac). + maskKeysInContainer = "/run/bb-mask" +) + +// MaskKeyDir is where the install keeps its masking keys: the state directory, +// owned by the server's user (uid 999 in the image) and readable by nobody +// else, mounted read-only into every database container. +var MaskKeyDir = func() string { return brand.StatePath("mask") } + +// MaskOn reports whether masking has been set up on this install: the default +// key exists. +func MaskOn() bool { + _, err := os.Stat(MaskKeyDir()) + return err == nil +} + +// maskMount is the containers' key mount, once masking is set up. Read-only: +// nothing in a database may change a key. +func maskMount() []string { + if !MaskOn() { + return nil + } + return []string{"-v", MaskKeyDir() + ":" + maskKeysInContainer + ":ro"} +} + +// runsWithMaskKeys reports whether a running database has the key mount: one +// started before masking was set up has not, until it starts again. +func runsWithMaskKeys(name string) bool { + out, err := capture("docker", "inspect", "-f", "{{range .Mounts}}{{.Destination}} {{end}}", container(name)) + return err == nil && strings.Contains(out, maskKeysInContainer) +} + +// MaskPending lists the running databases started before masking was set up, +// which have no key until they next start. +func MaskPending() ([]string, error) { + if !MaskOn() { + return nil, nil + } + names, err := RunningBranches() + if err != nil { + return nil, err + } + var out []string + for _, n := range names { + if !runsWithMaskKeys(n) { + out = append(out, n) + } + } + return out, nil +} + +// MaskNeedsRestart reports whether a running database must start again before +// it can be masked: it was started before masking was set up, so it has no +// key, or on an image older than the masking hash. +func MaskNeedsRestart(name string) bool { + if ContainerState(name) != "running" { + return false + } + if !runsWithMaskKeys(name) { + return true + } + out, err := capture("docker", "inspect", "-f", "{{index .Config.Labels \""+imageRevisionLabel+"\"}}", container(name)) + return err != nil || parseImageRevision(out) < 4 +} + +// RestartForMask starts a database again, on the current image and with the +// key mount. Its connections drop while it does. +func RestartForMask(name string) error { + if name == "main" { + return RestartPrimary() + } + if err := startContainer(name, false); err != nil { + return err + } + return waitReady(name) +} + +// MaskHAGuard refuses a setup that would restart main while the promoted +// standby is serving it. +func MaskHAGuard() error { + if ServingBranch("main") != "standby" { + return nil + } + return fmt.Errorf("refusing `%s mask setup`: 'main' is being served by the promoted standby "+ + "since `%s ha failover`, and this restarts it.\nRun `%s ha failback` first", brand.CLI, brand.CLI, brand.CLI) +} + +// ---- masked accounts ---- + +// MaskedAccount is one account an admin has masked. +type MaskedAccount struct { + Email string `json:"email"` + By string `json:"by"` + At string `json:"at"` // RFC 3339 +} + +var maskedPath = func() string { return brand.StatePath("masked-accounts.json") } + +// The file is read by every process that makes roles (the control plane, the +// Gateway, the CLI), so it is read again when it changes, not cached for good. +var masked struct { + sync.Mutex + mod time.Time + size int64 + byKey map[string]MaskedAccount +} + +func loadMasked() map[string]MaskedAccount { + masked.Lock() + defer masked.Unlock() + st, err := os.Stat(maskedPath()) + if err != nil { + masked.byKey, masked.mod, masked.size = nil, time.Time{}, 0 + return nil + } + if masked.byKey != nil && st.ModTime().Equal(masked.mod) && st.Size() == masked.size { + return masked.byKey + } + m := map[string]MaskedAccount{} + if b, err := os.ReadFile(maskedPath()); err == nil { + var list []MaskedAccount + if json.Unmarshal(b, &list) == nil { + for _, a := range list { + m[strings.ToLower(a.Email)] = a + } + } + } + masked.byKey, masked.mod, masked.size = m, st.ModTime(), st.Size() + return m +} + +// MaskedAccounts lists the masked accounts, by email. Free: reading who is +// masked is reading. +func MaskedAccounts() []MaskedAccount { + m := loadMasked() + out := make([]MaskedAccount, 0, len(m)) + for _, a := range m { + out = append(out, a) + } + sort.Slice(out, func(i, j int) bool { return out[i].Email < out[j].Email }) + return out +} + +// SaveMaskedAccounts writes the list. Enterprise calls it; a Standard install +// never changes it. +func SaveMaskedAccounts(list []MaskedAccount) error { + b, err := json.MarshalIndent(list, "", " ") + if err != nil { + return err + } + tmp := maskedPath() + ".tmp" + if err := os.WriteFile(tmp, b, 0o600); err != nil { + return err + } + return os.Rename(tmp, maskedPath()) +} + +// accountMasked reports whether an account's role on a branch is a masked +// reader's: masked by an admin, and neither an install admin nor the branch's +// owner, who always see real values (design, decision 3). +func accountMasked(branchName, email string) bool { + if _, ok := loadMasked()[strings.ToLower(email)]; !ok { + return false + } + return !maskExempt(branchName, email) +} + +// maskExempt: an install admin, or the branch's owner. A variable so the +// tests need neither a database nor an account store. +var maskExempt = func(branchName, email string) bool { + if ok, err := IsAdmin("main", email); err == nil && ok { + return true + } + st, err := auth.OpenFromEnv() + if err != nil { + return false + } + defer st.Close() + owner, ok := st.BranchOwner(branchName) + if !ok { + return false + } + u, ok := st.UserByEmail(email) + return ok && u.ID == owner +} + +// maskedRoleSQL makes a masked reader's login role, or turns an account's +// role into one: in db_masked and not in db_client (whose rights it would +// inherit, or SET ROLE to), the masks first in its search_path, and its +// failing statements never logged with their text, where a lookup's value or +// a guess would land (design, section 17, item 1). Superuser-only settings, so +// the reader cannot undo them. db_masked is made if this branch has not got it: +// a masked account on a branch with no masks reads nothing there, rather than +// everything. +func maskedRoleSQL(email, password string) string { + return retryingRoleDO(fmt.Sprintf("r text := %s;", quoteLiteral(email)), fmt.Sprintf(` IF to_regrole('%[2]s') IS NULL THEN + EXECUTE 'CREATE ROLE %[2]s NOLOGIN'; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN + EXECUTE format('CREATE ROLE %%I LOGIN NOSUPERUSER NOCREATEROLE NOCREATEDB NOBYPASSRLS INHERIT IN ROLE %[2]s', r); + END IF; + EXECUTE format('ALTER ROLE %%I WITH LOGIN NOSUPERUSER NOCREATEROLE NOCREATEDB NOBYPASSRLS PASSWORD %%L', r, %[1]s); + IF pg_has_role(r, 'db_client', 'MEMBER') THEN + EXECUTE format('REVOKE db_client FROM %%I', r); + END IF; + EXECUTE format('GRANT %[2]s TO %%I', r); + EXECUTE format('ALTER ROLE %%I SET role = %[2]s', r); + EXECUTE format('ALTER ROLE %%I SET search_path = %[3]s, public', r); + EXECUTE format('ALTER ROLE %%I SET log_min_error_statement = panic', r);`, + quoteLiteral(password), MaskedRole, MaskSchema)) +} + +// unmaskedRoleSQL turns a masked reader's role back into an ordinary +// account's, for an account unmasked, or exempt on this branch. +func unmaskedRoleSQL(email string) string { + return fmt.Sprintf(`DO $do$ +DECLARE r text := %s; +BEGIN + -- By name in pg_roles: to_regrole reads an identifier, and an email is not one. + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) AND to_regrole('%[2]s') IS NOT NULL AND pg_has_role(r, '%[2]s', 'MEMBER') THEN + EXECUTE format('REVOKE %[2]s FROM %%I', r); + EXECUTE format('GRANT db_client TO %%I', r); + EXECUTE format('ALTER ROLE %%I SET role = db_client', r); + EXECUTE format('ALTER ROLE %%I RESET search_path', r); + EXECUTE format('ALTER ROLE %%I RESET log_min_error_statement', r); + END IF; +END $do$;`, quoteLiteral(email), MaskedRole) +} + +// endSessionsSQL ends an account's sessions on a branch: a session opened +// before its role changed keeps the rights it logged in with. +func endSessionsSQL(email string) string { + return fmt.Sprintf("SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE usename = %s AND pid <> pg_backend_pid();", + quoteLiteral(email)) +} + +// ApplyAccountMask makes an account's role on a running branch match whether +// it is masked there now, and ends its open sessions when that changed it. A +// branch not running gets it when the role is next used (EnsureUserRole). +func ApplyAccountMask(branchName, email string) error { + if ContainerState(branchName) != "running" { + forgetUserRole(branchName, email) + return nil + } + sql := unmaskedRoleSQL(email) + if accountMasked(branchName, email) { + sql = maskedRoleSQL(email, UserRolePassword(email)) + } + if err := provisionRole(branchName, sql+"\n"+endSessionsSQL(email)); err != nil { + return err + } + forgetUserRole(branchName, email) + return nil +} + +func forgetUserRole(branchName, email string) { + for _, m := range []string{"masked", "plain"} { + ensuredRoles.Delete(branchName + "\x00" + email + "\x00" + m) + } + ensuredRoles.Delete(branchName + "\x00" + email) +} + +// ---- rules ---- + +// The rules (design, section 7). +const ( + MaskPartial = "partial" + MaskHash = "hash" + MaskFake = "fake" + MaskEmpty = "empty" +) + +// MaskRules lists the rules a choice can name. +var MaskRules = []string{MaskPartial, MaskHash, MaskFake, MaskEmpty} + +// MaskRule is one column's rule, or a JSON path's inside it, on a branch. +type MaskRule struct { + Schema string `json:"schema"` + Table string `json:"table"` + Column string `json:"column"` + Path string `json:"path,omitempty"` // a JSON path, $.a.b; "" for the column itself + Rule string `json:"rule"` + Category string `json:"category"` // point 4's, from the confirmed finding it was set on + By string `json:"by"` + At string `json:"at"` +} + +// Where is the rule's place, as point 4 writes it: schema.table.column[path]. +func (r MaskRule) Where() string { + w := r.Schema + "." + r.Table + "." + r.Column + if r.Path != "" { + w += " " + r.Path + } + return w +} + +// MaskCost is what a rule costs, as measured (design, section 17), said +// before a rule is set (decision 9). +func MaskCost(rule, path string) string { + if path == "" && (rule == MaskPartial || rule == MaskEmpty) { + return "What it costs: nothing on writes, as a partial or empty mask is made as\n" + + "it is read. A masked reader's reads of the table are about a quarter slower.\n" + } + s := "What it costs, as measured (PostgreSQL 18, 16 clients):\n" + + " - writes to the table: about a fifth to a third fewer single-row writes a\n" + + " second; a 10,000-row UPDATE takes about 3.6 times as long. Other tables\n" + + " are not touched.\n" + + " - a masked reader's reads of the table: about a quarter slower.\n" + + " - making the masks of rows already there: about 14 seconds a million rows,\n" + + " without holding up writers.\n" + if path != "" { + s += " - a JSON path costs with the size of the documents: in documents over 64 KB\n" + + " a write can take hundreds of times as long (100 documents of 250 KB:\n" + + " 1.4 ms without, 544 ms with).\n" + } + return s +} + +// ErrAadhaarHash is a rule that would hash or fake Aadhaar (design, section 7): +// a keyed hash of Aadhaar that applications join on is a hash used as a +// reference key, which UIDAI forbids. +var ErrAadhaarHash = errors.New("Aadhaar is masked by display only (partial or empty), never hashed or faked: joins on Aadhaar go through the vault's tokens") + +// CheckMaskRule refuses what a rule may not be, before anything is asked of a +// database. +func CheckMaskRule(rule, category string) error { + switch rule { + case MaskPartial, MaskHash, MaskFake, MaskEmpty: + default: + return fmt.Errorf("%w: a rule is %s, not %q", ErrInvalidRequest, strings.Join(MaskRules, ", "), rule) + } + if strings.EqualFold(category, "aadhaar") && (rule == MaskHash || rule == MaskFake) { + return fmt.Errorf("%w: %v", ErrInvalidRequest, ErrAadhaarHash) + } + return nil +} + +// DefaultMaskRule is the rule a category gets when none is named (design, +// section 7). +func DefaultMaskRule(category string) string { + switch strings.ToLower(category) { + case "aadhaar", "pan", "card", "mobile", "gstin", "ifsc": + return MaskPartial + case "email": + return MaskHash + case "name": + return MaskFake + } + return MaskEmpty +} + +// maskRulesSQL reads a branch's rules. +const maskRulesSQL = `SELECT coalesce(json_agg(json_build_object('schema', schema_name, 'table', table_name, 'column', column_name, + 'path', json_path, 'rule', rule, 'category', category, 'by', set_by, 'at', set_at) + ORDER BY schema_name, table_name, column_name, json_path), '[]'::json) FROM bb.mask_rules` + +// RelationExists reports whether a table or view is on a branch. Asked on its +// own: a query naming a missing table fails when it is planned, whatever CASE +// is meant to keep it from running. +func RelationExists(name, rel string) (bool, error) { + out, err := QuerySQL(name, "SELECT to_regclass("+quoteLiteral(rel)+") IS NOT NULL") + if err != nil { + return false, fmt.Errorf("asking %s whether %s is there: %w (%s)", name, rel, err, strings.TrimSpace(out)) + } + return strings.TrimSpace(out) == "t", nil +} + +// MaskRulesOn reads a branch's rules. Free. +func MaskRulesOn(name string) ([]MaskRule, error) { + if err := checkName(name); err != nil { + return nil, err + } + // A branch with masking never set up has no table, and so no rules. + if ok, err := RelationExists(name, "bb.mask_rules"); err != nil || !ok { + return []MaskRule{}, err + } + out, err := QuerySQL(name, maskRulesSQL) + if err != nil { + return nil, fmt.Errorf("reading the masking rules on %s: %w (%s)", name, err, strings.TrimSpace(out)) + } + var rules []MaskRule + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &rules); err != nil { + return nil, fmt.Errorf("reading the masking rules: %w", err) + } + if rules == nil { + rules = []MaskRule{} + } + return rules, nil +} + +// ---- the paid half ---- + +// Masker is what enterprise/masking supplies. +type Masker struct { + Setup func(branch, actor string) error + SetRule func(branch string, r MaskRule) error + DropRule func(branch, schema, table, column, path, actor string) error + MaskTables func(branch string) ([]string, error) // the tables whose masks are ready +} + +var masker *Masker + +// SetMasker installs it. Called from enterprise/masking's init, and from +// nowhere else. +func SetMasker(m *Masker) { masker = m } + +// ErrMaskingNotLicensed is the refusal for masking. +var ErrMaskingNotLicensed = notLicensed{edition.Masking} + +func requireMasker() error { + if err := requireFeature(edition.Masking); err != nil { + return err + } + if masker == nil { + return ErrMaskingNotLicensed + } + return nil +} + +// SetupMasking makes the install's default key, if there is none, and the +// masked reader's roles, schema and grants on a branch. Enterprise. +func SetupMasking(branchName, actor string) error { + if err := requireMasker(); err != nil { + return err + } + return masker.Setup(branchName, actor) +} + +// SetMaskRule sets a rule on a confirmed mark and makes the table's masks. +// Enterprise. +func SetMaskRule(branchName string, r MaskRule) error { + if err := requireMasker(); err != nil { + return err + } + // An unnamed rule is the category's default, chosen once the confirmed + // finding's category is read. + if r.Rule != "" { + if err := CheckMaskRule(r.Rule, r.Category); err != nil { + return err + } + } + return masker.SetRule(branchName, r) +} + +// DropMaskRule takes a rule off; a table left with none is no longer masked. +// Enterprise. +func DropMaskRule(branchName, schema, table, column, path, actor string) error { + if err := requireMasker(); err != nil { + return err + } + return masker.DropRule(branchName, schema, table, column, path, actor) +} + +// MaskAccount masks an account, or with off unmasks it, and makes its role on +// every running branch match. Install admins are refused: they can change the +// masks, so masking them would only pretend (decision 3). Enterprise. +func MaskAccount(email, actor string, off bool) error { + if err := requireMasker(); err != nil { + return err + } + email = strings.ToLower(strings.TrimSpace(email)) + if email == "" { + return fmt.Errorf("%w: an account is named by its email", ErrInvalidRequest) + } + if !off { + if ok, err := IsAdmin("main", email); err == nil && ok { + return fmt.Errorf("%w: %s is an admin, and admins always see real values: they can change the masks", ErrInvalidRequest, email) + } + } + m := loadMasked() + var list []MaskedAccount + for e, a := range m { + if e != email { + list = append(list, a) + } + } + if !off { + list = append(list, MaskedAccount{Email: email, By: actor, At: time.Now().UTC().Format(time.RFC3339)}) + } + sort.Slice(list, func(i, j int) bool { return list[i].Email < list[j].Email }) + if err := SaveMaskedAccounts(list); err != nil { + return err + } + names, err := RunningBranches() + if err != nil { + return err + } + var failed []string + for _, n := range names { + if err := ApplyAccountMask(n, email); err != nil { + failed = append(failed, fmt.Sprintf("%s (%v)", n, err)) + } + } + if len(failed) > 0 { + return fmt.Errorf("the account's role could not be changed on %s: it is changed when next used", strings.Join(failed, ", ")) + } + return nil +} + +// MaskStatus is masking on an install and one branch. +type MaskStatus struct { + On bool `json:"on"` + Pending []string `json:"pending"` // running without the key mount + Branch string `json:"branch"` + Rules []MaskRule `json:"rules"` + Ready []string `json:"ready"` // tables whose masks are made + Accounts []MaskedAccount `json:"accounts"` // install-wide + Writes bool `json:"writes"` // whether this install sets rules (Enterprise) +} + +// MaskingStatus reads it. Free. +func MaskingStatus(branchName string) (MaskStatus, error) { + if branchName == "" { + branchName = "main" + } + st := MaskStatus{On: MaskOn(), Branch: branchName, Pending: []string{}, Rules: []MaskRule{}, Ready: []string{}, + Accounts: MaskedAccounts(), Writes: requireMasker() == nil} + if p, err := MaskPending(); err == nil && p != nil { + st.Pending = p + } + if ContainerState(branchName) != "running" { + return st, nil + } + rules, err := MaskRulesOn(branchName) + if err != nil { + return st, err + } + st.Rules = rules + if masker != nil && masker.MaskTables != nil { + if t, err := masker.MaskTables(branchName); err == nil && t != nil { + st.Ready = t + } + } + return st, nil +} + +// ---- the primitives enterprise/masking makes masks with ---- + +// EnsureMaskKey makes a key domain's key if it has none: 32 random bytes, in a +// file of the server's user (uid 999), readable by nobody else, in a directory +// only that user can enter. The engine never reads it back. +func EnsureMaskKey(domain string) error { + if !maskDomainOK(domain) { + return fmt.Errorf("%w: a key domain is 1 to 32 of a-z, 0-9, _ and -", ErrInvalidRequest) + } + dir := MaskKeyDir() + if err := run("install", "-d", "-o", postgresUID, "-g", postgresUID, "-m", "0700", dir); err != nil { + return err + } + path := dir + "/" + domain + ".key" + if quietOK("test", "-s", path) { + return nil + } + tmp := path + ".new" + script := fmt.Sprintf("set -e; umask 077; head -c 32 /dev/urandom > %[1]s; chown %[2]s:%[2]s %[1]s; chmod 0400 %[1]s; mv %[1]s %[3]s", + shellQuote(tmp), postgresUID, shellQuote(path)) + return run("sh", "-c", script) +} + +func maskDomainOK(d string) bool { + if len(d) == 0 || len(d) > 32 { + return false + } + for _, c := range d { + if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '_' || c == '-') { + return false + } + } + return true +} + +// quietOK runs a privileged command and reports whether it succeeded. +func quietOK(name string, args ...string) bool { + cmd, _, cancel := sudoCmd(execClean, name, args...) + defer cancel() + return cmd.Run() == nil +} + +// RunMaskSQL runs the statements that make or change a table's masks in one +// transaction, recorded in the Blackbox under the person who asked. replace +// lets it drop the masks it is replacing (views, triggers and shadows in +// bb_mask), which the guardrail refuses by default. +func RunMaskSQL(name, actor string, stmts []string, replace bool) error { + body := strings.Join(stmts, ";\n") + ";" + return trackExec(name, trackSQL(actor, brand.CLI+" mask", body, replace)) +} + +// RunMaskScript runs a script as the superuser, a statement at a time, for +// what commits as it goes: filling a table's masks in batches. +func RunMaskScript(name, script string) error { return psqlQuiet(name, script) } + +// EnsureMaskImage makes sure the engine image has the masking hash: revision +// 4 or later, fetched or built if the one in hand is older. +func EnsureMaskImage() error { + img := pgImage() + if err := ensureImageRef(img); err != nil { + return err + } + if ours(img) && imageRevision(img) < 4 { + return fmt.Errorf("the engine image %s is revision %d and has no masking hash; "+ + "revision 4 could not be fetched or built", img, imageRevision(img)) + } + return nil +} + +// MaskColumn is one column of a table, as the masks need it. +type MaskColumn struct { + Name string `json:"name"` + Type string `json:"type"` // format_type + Key bool `json:"key"` // in the primary key +} + +// MaskColumns reads a table's columns in order, and which are its key. +func MaskColumns(name, schema, table string) ([]MaskColumn, error) { + out, err := QuerySQL(name, fmt.Sprintf(`SELECT coalesce(json_agg(json_build_object('name', a.attname, 'type', format_type(a.atttypid, a.atttypmod), + 'key', coalesce(a.attnum = ANY (i.indkey), false)) ORDER BY a.attnum), '[]'::json) + FROM pg_attribute a + LEFT JOIN pg_index i ON i.indrelid = a.attrelid AND i.indisprimary + WHERE a.attrelid = to_regclass(%s) AND a.attnum > 0 AND NOT a.attisdropped`, quoteLiteral(QuoteIdent(schema)+"."+QuoteIdent(table)))) + if err != nil { + return nil, fmt.Errorf("reading %s.%s's columns: %w (%s)", schema, table, err, strings.TrimSpace(out)) + } + var cols []MaskColumn + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &cols); err != nil { + return nil, err + } + return cols, nil +} diff --git a/internal/branch/mask_test.go b/internal/branch/mask_test.go new file mode 100644 index 0000000..1f9b2f2 --- /dev/null +++ b/internal/branch/mask_test.go @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/thefoxbyte/foxbyte/internal/edition" +) + +// Aadhaar is masked by display only: a hash applications join on would be a +// reference key, which UIDAI forbids (design, section 7). +func TestCheckMaskRuleAadhaar(t *testing.T) { + for _, r := range []string{MaskHash, MaskFake} { + for _, c := range []string{"aadhaar", "Aadhaar"} { + err := CheckMaskRule(r, c) + if !errors.Is(err, ErrInvalidRequest) || !strings.Contains(err.Error(), "never hashed or faked") { + t.Errorf("%s on %s: %v", r, c, err) + } + } + } + for _, r := range []string{MaskPartial, MaskEmpty} { + if err := CheckMaskRule(r, "aadhaar"); err != nil { + t.Errorf("%s on aadhaar: %v", r, err) + } + } + if err := CheckMaskRule("redact", "email"); !errors.Is(err, ErrInvalidRequest) { + t.Errorf("an unknown rule: %v", err) + } + if err := CheckMaskRule(MaskHash, "email"); err != nil { + t.Errorf("hash on email: %v", err) + } +} + +func TestDefaultMaskRule(t *testing.T) { + for c, want := range map[string]string{ + "aadhaar": MaskPartial, "pan": MaskPartial, "card": MaskPartial, "mobile": MaskPartial, "gstin": MaskPartial, + "ifsc": MaskPartial, "email": MaskHash, "name": MaskFake, "address": MaskEmpty, "": MaskEmpty, "EMAIL": MaskHash, + } { + if got := DefaultMaskRule(c); got != want { + t.Errorf("%q: %s, want %s", c, got, want) + } + if err := CheckMaskRule(DefaultMaskRule(c), c); err != nil { + t.Errorf("the default for %q is refused: %v", c, err) + } + } +} + +// A masked reader is in db_masked and out of db_client, reads the masks first, +// and its failing statements are never logged with their text. +func TestMaskedRoleSQL(t *testing.T) { + sql := maskedRoleSQL("o'brien@example.com", "pa'ss") + for _, want := range []string{ + "CREATE ROLE db_masked NOLOGIN", + "NOSUPERUSER NOCREATEROLE NOCREATEDB NOBYPASSRLS", + "REVOKE db_client FROM %I", + "GRANT db_masked TO %I", + "SET role = db_masked", + "SET search_path = bb_mask, public", + "SET log_min_error_statement = panic", + "'o''brien@example.com'", + "'pa''ss'", + } { + if !strings.Contains(sql, want) { + t.Errorf("missing %q:\n%s", want, sql) + } + } + un := unmaskedRoleSQL("o'brien@example.com") + for _, want := range []string{"REVOKE db_masked FROM %I", "GRANT db_client TO %I", "SET role = db_client", + "RESET search_path", "RESET log_min_error_statement", "pg_has_role(r, 'db_masked', 'MEMBER')"} { + if !strings.Contains(un, want) { + t.Errorf("unmasking is missing %q:\n%s", want, un) + } + } + // An email is not an identifier, so to_regrole of it is NULL: the role is + // looked up by name. + if strings.Contains(un, "to_regrole(r)") || !strings.Contains(un, "pg_roles WHERE rolname = r") { + t.Errorf("unmasking finds the role with to_regrole, which never finds an email:\n%s", un) + } + if !strings.Contains(endSessionsSQL("a@b.c"), "pg_terminate_backend") { + t.Error("the sessions are not ended") + } +} + +// The masked accounts' file: read again when it changes, and an admin or the +// branch's owner is never masked. +func TestAccountMasked(t *testing.T) { + dir := t.TempDir() + oldPath, oldExempt := maskedPath, maskExempt + t.Cleanup(func() { maskedPath, maskExempt = oldPath, oldExempt; masked.byKey = nil }) + maskedPath = func() string { return filepath.Join(dir, "masked-accounts.json") } + exempt := map[string]bool{} + maskExempt = func(b, e string) bool { return exempt[b+"/"+e] } + + if accountMasked("main", "a@x.io") || len(MaskedAccounts()) != 0 { + t.Fatal("masked with no file") + } + if err := SaveMaskedAccounts([]MaskedAccount{{Email: "A@x.io", By: "cli:root", At: "2026-11-05T10:00:00Z"}}); err != nil { + t.Fatal(err) + } + if !accountMasked("main", "a@x.io") || !accountMasked("dev", "A@X.IO") { + t.Fatal("a listed account is not masked") + } + exempt["dev/a@x.io"] = true + if accountMasked("dev", "a@x.io") || !accountMasked("main", "a@x.io") { + t.Fatal("the owner of dev is masked there, or exempt everywhere") + } + if st, err := os.Stat(maskedPath()); err != nil || st.Mode().Perm() != 0o600 && os.PathSeparator == '/' { + t.Fatalf("the file's mode: %v %v", st, err) + } + // A change made by another process is seen. + time.Sleep(10 * time.Millisecond) + if err := SaveMaskedAccounts([]MaskedAccount{{Email: "b@x.io"}, {Email: "c@x.io"}}); err != nil { + t.Fatal(err) + } + if accountMasked("main", "a@x.io") || !accountMasked("main", "b@x.io") || len(MaskedAccounts()) != 2 { + t.Fatalf("the change is not seen: %v", MaskedAccounts()) + } + _ = os.Remove(maskedPath()) + if accountMasked("main", "b@x.io") { + t.Fatal("still masked with the file gone") + } +} + +// Without Enterprise's masker every change is refused as not licensed, and +// nothing is written. +func TestMaskingNotLicensed(t *testing.T) { + old := masker + t.Cleanup(func() { masker = old }) + masker = nil + oldPath := maskedPath + t.Cleanup(func() { maskedPath = oldPath }) + maskedPath = func() string { return filepath.Join(t.TempDir(), "masked-accounts.json") } + for name, err := range map[string]error{ + "setup": SetupMasking("main", "cli:me"), + "rule": SetMaskRule("main", MaskRule{Schema: "public", Table: "t", Column: "c", Rule: MaskHash}), + "drop": DropMaskRule("main", "public", "t", "c", "", "cli:me"), + "account": MaskAccount("a@x.io", "cli:me", false), + } { + if !errors.Is(err, ErrNotLicensed) { + t.Errorf("%s: %v", name, err) + } + } + if _, err := os.Stat(maskedPath()); err == nil { + t.Error("an account was masked without the licence") + } +} + +// A named rule is checked before Enterprise is asked; an unnamed one is the +// category's default, chosen there. +func TestSetMaskRuleChecksFirst(t *testing.T) { + edition.SetEntitlement(func(edition.Feature) bool { return true }) + t.Cleanup(func() { edition.SetEntitlement(nil) }) + if !edition.Has(edition.Masking) { + t.Skip("the Standard build: the Enterprise run covers it") + } + old := masker + t.Cleanup(func() { masker = old }) + var got []MaskRule + masker = &Masker{SetRule: func(b string, r MaskRule) error { got = append(got, r); return nil }} + if err := SetMaskRule("main", MaskRule{Rule: "hash", Category: "aadhaar"}); !errors.Is(err, ErrInvalidRequest) { + t.Fatalf("hashing Aadhaar: %v", err) + } + if err := SetMaskRule("main", MaskRule{Column: "c"}); err != nil || len(got) != 1 { + t.Fatalf("an unnamed rule: %v %v", err, got) + } +} + +// The key is mounted read-only once masking is set up, and not before. +func TestMaskMount(t *testing.T) { + old := MaskKeyDir + t.Cleanup(func() { MaskKeyDir = old }) + dir := filepath.Join(t.TempDir(), "mask") + MaskKeyDir = func() string { return dir } + if MaskOn() || maskMount() != nil { + t.Fatal("mounted before setup") + } + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + if got := strings.Join(maskMount(), " "); !MaskOn() || got != "-v "+dir+":/run/bb-mask:ro" { + t.Fatalf("the mount: %q", got) + } +} + +// The cost is said before a rule is set (decision 9), from what was measured. +func TestMaskCost(t *testing.T) { + if c := MaskCost(MaskPartial, ""); !strings.Contains(c, "nothing on writes") { + t.Errorf("partial: %s", c) + } + c := MaskCost(MaskHash, "") + if !strings.Contains(c, "3.6 times") || !strings.Contains(c, "a quarter") || strings.Contains(c, "64 KB") { + t.Errorf("hash: %s", c) + } + if c := MaskCost(MaskPartial, "$.a"); !strings.Contains(c, "64 KB") || strings.Contains(c, "nothing on writes") { + t.Errorf("a JSON path: %s", c) + } + if c := MaskCost("", ""); !strings.Contains(c, "3.6 times") { + t.Errorf("an unnamed rule, which may need the key: %s", c) + } +} diff --git a/internal/controlplane/enterprise_on.go b/internal/controlplane/enterprise_on.go index abced84..0d062f3 100644 --- a/internal/controlplane/enterprise_on.go +++ b/internal/controlplane/enterprise_on.go @@ -15,6 +15,7 @@ import ( _ "github.com/thefoxbyte/foxbyte/enterprise/auditevidence" _ "github.com/thefoxbyte/foxbyte/enterprise/hardening" _ "github.com/thefoxbyte/foxbyte/enterprise/impact" + _ "github.com/thefoxbyte/foxbyte/enterprise/masking" _ "github.com/thefoxbyte/foxbyte/enterprise/pii" _ "github.com/thefoxbyte/foxbyte/enterprise/pipeline" _ "github.com/thefoxbyte/foxbyte/enterprise/policy" diff --git a/internal/controlplane/masking.go b/internal/controlplane/masking.go new file mode 100644 index 0000000..9761884 --- /dev/null +++ b/internal/controlplane/masking.go @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controlplane + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/thefoxbyte/foxbyte/internal/auth" + "github.com/thefoxbyte/foxbyte/internal/branch" + "github.com/thefoxbyte/foxbyte/internal/edition" +) + +// Masking over HTTP (F5, point 4.6). Admins only: the status names the +// accounts masked, and what a person sees of personal data is an admin's to +// decide (design, decision 4). +// +// GET /api/masking?branch=main where masking stands: free +// POST /api/masking/setup {"branch", "apply"} what setup does, or with apply does it (Enterprise) +// POST /api/masking/rule {"branch", "schema", "table", "column", "path", "rule", "off", "apply"} +// a rule's cost, or with apply sets it; off takes it off (Enterprise) +// POST /api/masking/account {"email", "off"} masks an account, or unmasks it (Enterprise) +// +// Each setup, rule and account change is in the security log. +func registerMasking(mux *http.ServeMux) { + mux.HandleFunc("GET /api/masking", func(w http.ResponseWriter, r *http.Request) { + if !isAdmin(r) { + writeErr(w, 403, errors.New("only an admin may read where masking stands")) + return + } + st, err := branch.MaskingStatus(r.URL.Query().Get("branch")) + if err != nil { + maskFail(w, err) + return + } + writeJSON(w, 200, st) + }) + + type body struct { + Branch string `json:"branch"` + Schema string `json:"schema"` + Table string `json:"table"` + Column string `json:"column"` + Path string `json:"path"` + Rule string `json:"rule"` + Email string `json:"email"` + Off bool `json:"off"` + Apply bool `json:"apply"` + } + read := func(w http.ResponseWriter, r *http.Request) (body, string, bool) { + var b body + if !requireFeature(w, edition.Masking) { + return b, "", false + } + if !isAdmin(r) { + writeErr(w, 403, errors.New("only an admin may change masking")) + return b, "", false + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&b); err != nil { + writeErr(w, 400, errors.New("the body must be JSON")) + return b, "", false + } + if b.Branch == "" { + b.Branch = "main" + } + u, _ := auth.UserFrom(r.Context()) + return b, u.Email, true + } + record := func(r *http.Request, actor, kind, b, detail string) { + if secLog != nil { + secLog.Audit(kind, actor, b, remoteIP(r), detail) + } + } + + mux.HandleFunc("POST /api/masking/setup", func(w http.ResponseWriter, r *http.Request) { + b, actor, ok := read(w, r) + if !ok { + return + } + if err := branch.MaskHAGuard(); err != nil { + writeErr(w, 409, err) + return + } + restart := branch.MaskNeedsRestart(b.Branch) + if !b.Apply { + writeJSON(w, 200, map[string]any{"branch": b.Branch, "restart": restart, "keyDir": branch.MaskKeyDir()}) + return + } + for _, step := range []func() error{branch.EnsureMaskImage, func() error { return branch.EnsureMaskKey("default") }} { + if err := step(); err != nil { + maskFail(w, err) + return + } + } + if restart { + if err := branch.RestartForMask(b.Branch); err != nil { + maskFail(w, fmt.Errorf("%s did not come back with the key: %w", b.Branch, err)) + return + } + if b.Branch == "main" { + _ = branch.RestartStandby() // as `fox mask setup`: a standby without the key is reported by status + } + } + if err := branch.SetupMasking(b.Branch, actor); err != nil { + maskFail(w, err) + return + } + record(r, actor, auth.EvMaskSetup, b.Branch, "masking set up on "+b.Branch) + writeJSON(w, 200, map[string]any{"branch": b.Branch, "restarted": restart}) + }) + + mux.HandleFunc("POST /api/masking/rule", func(w http.ResponseWriter, r *http.Request) { + b, actor, ok := read(w, r) + if !ok { + return + } + if b.Schema == "" { + b.Schema = "public" + } + if b.Table == "" || b.Column == "" { + writeErr(w, 400, errors.New("a rule is on a table's column: table and column are needed")) + return + } + rule := branch.MaskRule{Schema: b.Schema, Table: b.Table, Column: b.Column, Path: b.Path, Rule: b.Rule, By: actor} + if b.Off { + if err := branch.DropMaskRule(b.Branch, b.Schema, b.Table, b.Column, b.Path, actor); err != nil { + maskFail(w, err) + return + } + record(r, actor, auth.EvMaskRule, b.Branch, fmt.Sprintf("rule taken off %s on %s", rule.Where(), b.Branch)) + writeJSON(w, 200, map[string]any{"off": rule.Where()}) + return + } + if b.Rule != "" { + if err := branch.CheckMaskRule(b.Rule, ""); err != nil { + maskFail(w, err) + return + } + } + if !b.Apply { + writeJSON(w, 200, map[string]any{"where": rule.Where(), "cost": branch.MaskCost(b.Rule, b.Path)}) + return + } + if err := branch.SetMaskRule(b.Branch, rule); err != nil { + maskFail(w, err) + return + } + named := b.Rule + if named == "" { + named = "(the category's default)" + } + record(r, actor, auth.EvMaskRule, b.Branch, fmt.Sprintf("rule %s set on %s on %s", named, rule.Where(), b.Branch)) + writeJSON(w, 200, map[string]any{"set": rule.Where()}) + }) + + mux.HandleFunc("POST /api/masking/account", func(w http.ResponseWriter, r *http.Request) { + b, actor, ok := read(w, r) + if !ok { + return + } + email := strings.ToLower(strings.TrimSpace(b.Email)) + if err := branch.MaskAccount(email, actor, b.Off); err != nil { + maskFail(w, err) + return + } + what := "masked" + if b.Off { + what = "unmasked" + } + record(r, actor, auth.EvMaskAccount, "", what+" "+email) + writeJSON(w, 200, map[string]any{"email": email, "masked": !b.Off}) + }) +} + +func maskFail(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, branch.ErrNotLicensed): + writeErr(w, 403, err) + case errors.Is(err, branch.ErrInvalidRequest), errors.Is(err, branch.ErrBadName): + writeErr(w, 400, err) + default: + writeErr(w, 500, err) + } +} diff --git a/internal/controlplane/openapi.yaml b/internal/controlplane/openapi.yaml index 0dc645d..0c5615c 100644 --- a/internal/controlplane/openapi.yaml +++ b/internal/controlplane/openapi.yaml @@ -2196,6 +2196,107 @@ paths: "400": { description: Not a profile, or not a branch. } "403": { description: Not an admin, or not licensed for hardening fixes and profiles. } "401": { $ref: "#/components/responses/Unauthorized" } + /api/masking: + get: + tags: [audit-log] + summary: Where masking stands on the install and a branch (admins only; in every edition). + description: >- + Whether masking is set up, the databases started before it was (which mask nothing until they start + again), a branch's rules and the tables whose masks are made, and the accounts masked. A masked account + reads the masks of marked tables, ordinary tables as they are, and cannot write to marked tables, on + every branch it does not own. Admins are never masked. + parameters: + - { name: branch, in: query, schema: { type: string, default: main } } + responses: + "200": + description: The status. + content: + application/json: + schema: + type: object + properties: + on: { type: boolean } + pending: { type: array, items: { type: string } } + branch: { type: string } + rules: + type: array + items: + type: object + properties: + schema: { type: string } + table: { type: string } + column: { type: string } + path: { type: string } + rule: { type: string, enum: [partial, hash, fake, empty] } + category: { type: string } + by: { type: string } + at: { type: string } + ready: { type: array, items: { type: string } } + accounts: { type: array, items: { type: object, properties: { email: { type: string }, by: { type: string }, at: { type: string } } } } + writes: { type: boolean, description: Whether this install may set masking (Enterprise). } + "403": { description: Not an admin. } + "401": { $ref: "#/components/responses/Unauthorized" } + /api/masking/setup: + post: + tags: [audit-log] + summary: Set masking up on a branch, after a preview (admins only; Enterprise). + description: >- + Without apply, whether the branch restarts to mount the key. With apply, the install's key is made if it + has none (32 random bytes, readable only by the database server, never read back), the branch restarts if + it must, and the masked readers' role, schema and rights are made. In the security log. + requestBody: + content: + application/json: + schema: { type: object, properties: { branch: { type: string, default: main }, apply: { type: boolean } } } + responses: + "200": { description: '{"branch", "restart", "keyDir"} for a preview, {"branch", "restarted"} once done.', content: { application/json: { schema: { type: object } } } } + "403": { description: Not an admin, or not licensed for masking. } + "409": { description: Main is served by the promoted HA standby. } + "401": { $ref: "#/components/responses/Unauthorized" } + /api/masking/rule: + post: + tags: [audit-log] + summary: Mask a confirmed finding, after saying what it costs, or take a rule off (admins only; Enterprise). + description: >- + A rule is set only where a reviewer confirmed personal data. Without a rule, the category's default + (partial for Aadhaar, PAN, cards and mobiles; hash for email; fake for names; else empty). Aadhaar is never + hashed or faked. Without apply, the cost; with apply, the rule is set and the table's masks made. off takes + it off. In the security log. + requestBody: + content: + application/json: + schema: + type: object + properties: + branch: { type: string, default: main } + schema: { type: string, default: public } + table: { type: string } + column: { type: string } + path: { type: string, description: 'A JSON path in a jsonb column, as $.a.b.' } + rule: { type: string, enum: [partial, hash, fake, empty] } + off: { type: boolean } + apply: { type: boolean } + responses: + "200": { description: '{"where", "cost"} for a preview, {"set"} or {"off"} once done.', content: { application/json: { schema: { type: object } } } } + "400": { description: Not a confirmed finding, a rule the category may not have, or a table that cannot be masked (no one-column primary key, the key itself, an array path). } + "403": { description: Not an admin, or not licensed for masking. } + "401": { $ref: "#/components/responses/Unauthorized" } + /api/masking/account: + post: + tags: [audit-log] + summary: Mask an account on every branch it does not own, or unmask it (admins only; Enterprise). + description: >- + Admins are refused: they can change the masks. The account's open sessions on running branches end, so its + next one sees the change. In the security log. + requestBody: + content: + application/json: + schema: { type: object, properties: { email: { type: string }, off: { type: boolean } } } + responses: + "200": { description: '{"email", "masked"}', content: { application/json: { schema: { type: object } } } } + "400": { description: No email, or an admin. } + "403": { description: Not an admin, or not licensed for masking. } + "401": { $ref: "#/components/responses/Unauthorized" } /api/audit-log/evidence: get: tags: [audit-log] diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index dd0a6fe..b548c1c 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -104,6 +104,7 @@ func Serve(addr string) error { registerAuditEvidence(api) // /api/audit-log/evidence (admins, Enterprise; F6 step 6b) registerHardening(api) // /api/hardening (admins; point 3.2) registerHardeningFixes(api) // /api/hardening/{fix,profile} (admins, Enterprise; points 3.3, 3.4) + registerMasking(api) // /api/masking (admins; setting it is Enterprise; F5, point 4.6) registerRequests(api, store) // /api/branches/{name}/request, /api/requests* registerLicense(api) // /api/license registerHistory(api, store) // /api/history* (the caller's own query history) diff --git a/internal/edition/edition.go b/internal/edition/edition.go index e662d2d..d8d7774 100644 --- a/internal/edition/edition.go +++ b/internal/edition/edition.go @@ -41,6 +41,10 @@ const ( // Hardening is fixing what the hardening report finds, one check at a // time or by a profile (points 3.3 and 3.4). The report itself is free. Hardening Feature = "hardening" + // Masking is masking personal data for the people who should not see it + // (point 7): rules, masked readers, unmask and masked branches. Masks + // already made keep working in every edition. + Masking Feature = "masking" ) // descriptions are used in refusals, so each one says what the user is missing @@ -56,6 +60,7 @@ var descriptions = map[Feature]string{ PersonalData: "personal-data discovery", AccessReview: "the access review report", Hardening: "hardening fixes and profiles", + Masking: "masking personal data", } // Describe names a feature in a sentence. An unknown feature describes itself, diff --git a/internal/host/uninstall.go b/internal/host/uninstall.go index bbf14e6..cd44dea 100644 --- a/internal/host/uninstall.go +++ b/internal/host/uninstall.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "github.com/thefoxbyte/foxbyte/internal/brand" @@ -168,11 +169,28 @@ func hostSteps(o UninstallOptions) []removal { what: "accounts, API keys, secrets and Blackbox anchors (" + dir + ")", data: true, present: func() bool { _, err := os.Stat(dir); return err == nil }, - run: func() error { return os.RemoveAll(dir) }, + run: func() error { return removeStateDir(dir) }, }) return out } +// removeStateDir deletes the state directory. Masking's keys are in a folder +// only the database server's user may enter (uid 999), so what the operator +// cannot delete is deleted with sudo, and only inside the state directory. +func removeStateDir(dir string) error { + err := os.RemoveAll(dir) + // Only a state directory: an absolute path to a dot-folder (.fox, or a + // retired name's), never anything sudo could be pointed at by mistake. + dir = filepath.Clean(dir) + if err == nil || runtime.GOOS == "windows" || !filepath.IsAbs(dir) || !strings.HasPrefix(filepath.Base(dir), ".") || len(filepath.Base(dir)) < 2 { + return err + } + if out, err := exec.Command("sudo", "rm", "-rf", "--", dir).CombinedOutput(); err != nil { + return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out))) + } + return nil +} + // binaryNames is this command and the commands of retired product names, so an // install from before a rename is removed too rather than left on PATH. func binaryNames() []string { diff --git a/scripts/integration_test.sh b/scripts/integration_test.sh index 6622c9a..1d61984 100644 --- a/scripts/integration_test.sh +++ b/scripts/integration_test.sh @@ -85,6 +85,9 @@ rm -rf "${HOME:?}/${BRAND_STATE_DIR:?}/anchors/_audit-log" # dutifully puts the mode back, and the first assertion there ("commits do not # wait by default") fails for a reason that has nothing to do with it. rm -f "$HOME/$BRAND_STATE_DIR/sync-commit" +# Masking's key and masked accounts (F5): §11ai starts from an install that +# has never set masking up. +sudo rm -rf "${HOME:?}/${BRAND_STATE_DIR:?}/mask"; rm -f "${HOME:?}/${BRAND_STATE_DIR:?}/masked-accounts.json" # And the primary pointer, for the same reason: a run cut short after a failover # leaves 'main' pointing at the promoted standby. `stop` above has already # removed every container, so there is no promoted standby left to protect, and @@ -3249,6 +3252,112 @@ assert_eq "every fix and every profile is in the security log" \ "$([ "$($S audit log --limit 200 | grep -c 'hardening.fixed')" -ge 5 ] && echo yes)|$($S audit log --limit 200 | grep -c 'hardening.profile')" "yes|1" $S branch delete hardfix2 >/dev/null 2>&1; $S branch delete hardfix >/dev/null 2>&1 +echo "### 11ai. masking personal data (F5 Gate 1, step 1)" +# Enterprise: rules on confirmed findings; a masked account reads the masks of +# marked tables and ordinary tables as they are, and cannot write to marked +# tables; admins and a branch's owner always see real values. On main, as an +# account reaches main and the branches it owns, and on those it sees real +# values. Its tables and rules are taken off at the end. +$S branch delete maskown >/dev/null 2>&1; $S branch delete maskold >/dev/null 2>&1 +pg pg-main "SET bb.allow_destructive = on; DROP TABLE IF EXISTS m_customers, m_orders, m_kyc, m_plain CASCADE" >/dev/null +$S branch create maskold >/dev/null 2>&1 +gwm() { PGPASSWORD="$1" psql "${GATEWAY}/$2?sslmode=require" -X -q -tA -c "$3" 2>&1; } +pg pg-main "CREATE TABLE m_customers (id bigint PRIMARY KEY, email text, phone text, aadhaar_no text, prefs jsonb, city text, full_name text); + INSERT INTO m_customers VALUES + (1, 'asha@example.com', '+91 98765 43210', '234567890124', '{\"contact\": {\"mobile\": \"+91 98765 43210\"}, \"theme\": \"dark\"}', 'Pune', 'Asha Rao'), + (2, 'ravi@example.org', '9876543211', '456789012341', '{\"contact\": {\"mobile\": \"9876543211\"}}', 'Goa', 'Ravi Kumar'); + CREATE TABLE m_orders (id bigint PRIMARY KEY, customer_email text, total int); + INSERT INTO m_orders VALUES (10, 'asha@example.com', 500), (11, 'ravi@example.org', 700); + CREATE TABLE m_kyc (id int PRIMARY KEY, aadhaar_no text); INSERT INTO m_kyc VALUES (1, '234567890124'); + CREATE TABLE m_plain (id int PRIMARY KEY, v text); INSERT INTO m_plain VALUES (1, 'open');" >/dev/null +printf 'password123\n' | $S user create mask-analyst@foxbyte.dev >/dev/null 2>&1 +MK="$($S apikey create mask-analyst@foxbyte.dev t 2>/dev/null | grep -o 'key_[A-Za-z0-9_-]*')" +assert_eq "before masking, nothing is masked: an account reads real values" \ + "$($S mask | head -1)|$(gwm "$MK" main "SELECT email FROM m_customers WHERE id = 1")" "Masking: not set up on this install.|asha@example.com" +PRE="$($S mask setup 2>&1)" +assert_eq "setup without --yes says what it makes and that the branch restarts, and makes nothing" \ + "$(grep -c 'main restarts to mount the key' <<<"$PRE")|$(grep -c 'Run again with --yes' <<<"$PRE")|$(sudo test -e "${HOME:?}/${BRAND_STATE_DIR:?}/mask" && echo made || echo none)" "1|1|none" +SET="$($S mask setup --yes 2>&1)"; rc=$? +assert_eq "with --yes: the key is made, the server's alone, mounted read-only, and the branch set up" \ + "$rc|$(sudo stat -c '%u %a' "${HOME:?}/${BRAND_STATE_DIR:?}/mask") $(sudo stat -c '%u %a %s' "${HOME:?}/${BRAND_STATE_DIR:?}/mask/default.key")|$(sudo docker inspect -f '{{range .Mounts}}{{if eq .Destination "/run/bb-mask"}}{{.RW}}{{end}}{{end}}' pg-main)|$(pg pg-main "SELECT to_regrole('db_masked') IS NOT NULL AND to_regclass('bb.mask_rules') IS NOT NULL")" \ + "0|999 700 999 400 32|false|t" +[ "$rc" = 0 ] || { echo "--- what it said ---"; tail -8 <<<"$SET"; echo "--- end ---"; } +assert_eq "…and the key is in no table, log or message the setup produced" \ + "$(sudo docker logs pg-main 2>&1 | grep -c "$(sudo od -An -tx1 "${HOME:?}/${BRAND_STATE_DIR:?}/mask/default.key" | tr -d ' \n')")|$(grep -c 'Done.' <<<"$SET")" "0|1" +assert_eq "a branch started before it is listed as masking nothing until it starts again; main restarted with the key" \ + "$($S mask --json | hjson "print('maskold' in d['pending'], 'main' in d['pending'])")" "True False" +$S branch delete maskold >/dev/null 2>&1 +NOPE="$($S mask rule m_customers.email --yes 2>&1)"; rc=$? +assert_eq "a rule is refused where no reviewer confirmed personal data" \ + "$rc|$(grep -c 'no confirmed personal data at public.m_customers.email' <<<"$NOPE")" "1|1" +$S pii scan >/dev/null 2>&1 +MJ="$($S pii --json 2>/dev/null)" +mfid() { hjson "print([f['id'] for f in d['findings'] if f['table']=='$1' and f['column']=='$2' and f['path']=='$3'][0])" <<<"$MJ"; } +for at in "m_customers email" "m_customers phone" "m_customers aadhaar_no" "m_orders customer_email" "m_kyc aadhaar_no"; do + $S pii mark "$(mfid $at '')" confirm >/dev/null 2>&1 +done +$S pii mark "$(mfid m_customers prefs '$.contact.mobile')" confirm >/dev/null 2>&1 +COST="$($S mask rule m_customers.email 2>&1)" +assert_eq "a rule without --yes says what it costs first, and sets nothing" \ + "$(grep -c '3.6 times as long' <<<"$COST")|$(grep -c 'a quarter slower' <<<"$COST")|$(grep -c 'Run again with --yes' <<<"$COST")|$($S mask --json | hjson "print(len(d['rules']))")" "1|1|1|0" +assert_eq "…and a partial mask says it costs nothing on writes" \ + "$($S mask rule m_kyc.aadhaar_no partial 2>&1 | grep -c 'nothing on writes')" "1" +assert_eq "Aadhaar is never hashed: refused, saying why" \ + "$($S mask rule m_customers.aadhaar_no hash --yes 2>&1 | grep -c 'never hashed or faked')" "1" +for c in m_customers.email m_customers.phone m_customers.aadhaar_no m_orders.customer_email m_kyc.aadhaar_no; do + $S mask rule "$c" --yes >/dev/null 2>&1 || bad "setting the rule on $c" +done +$S mask rule m_customers.prefs --path '$.contact.mobile' --yes >/dev/null 2>&1 || bad "setting the rule on a JSON path" +assert_eq "each rule is the category's default, and every table's masks are made" \ + "$($S mask --json | hjson "print(sorted(r['table']+'.'+r['column']+r.get('path','')+':'+r['rule'] for r in d['rules']), sorted(d['ready']))")" \ + "['m_customers.aadhaar_no:partial', 'm_customers.email:hash', 'm_customers.phone:partial', 'm_customers.prefs\$.contact.mobile:partial', 'm_kyc.aadhaar_no:partial', 'm_orders.customer_email:hash'] ['m_customers', 'm_kyc', 'm_orders']" +assert_eq "a table with only partial masks gets no shadow and no trigger: nothing more on a write" \ + "$(pg pg-main "SELECT to_regclass('bb_mask.public__m_kyc') IS NULL, (SELECT count(*) FROM pg_trigger WHERE tgrelid = 'm_kyc'::regclass AND tgname LIKE 'bb_mask%')")" "t|0" +assert_eq "an admin cannot be masked: they can change the masks" \ + "$($S mask account test@foxbyte.dev --yes 2>&1 | grep -c 'is an admin')|$($S mask --json | hjson "print(len(d['accounts']))")" "1|0" +assert_eq "masking an account without --yes says what changes, and masks nobody" \ + "$($S mask account mask-analyst@foxbyte.dev 2>&1 | grep -c 'Run again with --yes')|$(gwm "$MK" main "SELECT email FROM m_customers WHERE id = 1")" "1|asha@example.com" +$S mask account mask-analyst@foxbyte.dev --yes >/dev/null 2>&1 || bad "masking the account" +assert_eq "a masked account reads masks: a keyed hash, partial values, a JSON path, the rest of the row as it is" \ + "$(gwm "$MK" main "SELECT length(email), email ~ '^[0-9a-f]{64}\$', phone, aadhaar_no, prefs->'contact'->>'mobile', prefs->>'theme', city, full_name FROM m_customers WHERE id = 1")" \ + "64|t|XXXXXXXXXXX3210|XXXXXXXX0124|XXXXXXXXXXX3210|dark|Pune|Asha Rao" +assert_eq "…joins on the hash: the same value, the same mask" \ + "$(gwm "$MK" main "SELECT count(*) FROM m_customers c JOIN m_orders o ON o.customer_email = c.email")" "2" +assert_eq "…a table with only partial masks too" "$(gwm "$MK" main "SELECT aadhaar_no FROM m_kyc")" "XXXXXXXX0124" +assert_eq "…is refused the real table, the shadow and the hash" \ + "$(gwm "$MK" main "SELECT email FROM public.m_customers" | grep -c 'permission denied')|$(gwm "$MK" main "SELECT * FROM bb_mask.public__m_customers" | grep -c 'permission denied')|$(gwm "$MK" main "SELECT bb_f1.bbid_mask_hmac('default', 'x')" | grep -c 'permission denied')" "1|1|1" +assert_eq "…cannot write to a marked table, and can to an ordinary one" \ + "$(gwm "$MK" main "UPDATE m_customers SET city = 'x' WHERE id = 1" | grep -cE 'permission denied|cannot update view')|$(gwm "$MK" main "UPDATE m_kyc SET aadhaar_no = 'x'" | grep -cE 'permission denied|cannot update')$(pg pg-main "SELECT aadhaar_no FROM m_kyc" | grep -c '^234567890124$')|$(gwm "$MK" main "INSERT INTO m_plain VALUES (2, 'by the analyst'); SELECT v FROM m_plain WHERE id = 2")" "1|11|by the analyst" +assert_eq "…cannot put its search_path back to the real tables" \ + "$(gwm "$MK" main "SET search_path = public; SELECT email FROM m_customers" | grep -c 'permission denied')" "1" +gwm "$KEY" main "INSERT INTO m_customers VALUES (3, 'neha@example.com', '9123456780', '345678901238', NULL, 'Delhi', 'Neha'); + UPDATE m_customers SET email = 'ravi.k@example.org' WHERE id = 2; UPDATE m_customers SET id = 4 WHERE id = 1; DELETE FROM m_customers WHERE id = 3" >/dev/null +assert_eq "the application's writes keep the masks up: inserts, changes, a moved key and a delete" \ + "$(pg pg-main "SELECT (SELECT count(*) FROM m_customers), (SELECT count(*) FROM bb_mask.public__m_customers), (SELECT count(*) FROM m_customers c JOIN bb_mask.public__m_customers m USING (id))")|$(gwm "$MK" main "SELECT id, email ~ '^[0-9a-f]{64}\$' FROM m_customers ORDER BY id" | tr '\n' ' ')" \ + "2|2|2|2|t 4|t " +assert_eq "…and an admin reads real values throughout" "$(gwm "$KEY" main "SELECT email FROM m_customers WHERE id = 2")" "ravi.k@example.org" +code "$MK" -X POST -H 'Content-Type: application/json' -d '{"name":"maskown","from":"main"}' https://localhost:8080/api/branches >/dev/null +assert_eq "the masked account sees real values on a branch it owns" \ + "$(gwm "$MK" maskown "SELECT email FROM m_customers WHERE id = 2")" "ravi.k@example.org" +assert_eq "the API: an admin reads where masking stands and the cost of a rule; the masked account is refused both" \ + "$(curl -sk -H "$AUTH" 'https://localhost:8080/api/masking?branch=main' | hjson "print(len(d['rules']), [a['email'] for a in d['accounts']])")|$(curl -sk -H "$AUTH" -H 'Content-Type: application/json' -d '{"branch":"main","table":"m_customers","column":"email"}' https://localhost:8080/api/masking/rule | hjson "print('3.6 times' in d['cost'])")|$(code "$MK" 'https://localhost:8080/api/masking?branch=main')|$(code "$MK" -X POST -H 'Content-Type: application/json' -d '{"email":"mask-analyst@foxbyte.dev","off":true}' https://localhost:8080/api/masking/account)" \ + "6 ['mask-analyst@foxbyte.dev']|True|403|403" +$S mask rule m_customers.email --off >/dev/null 2>&1 || bad "taking the email rule off" +assert_eq "a rule taken off: its column is real again for the masked account, the others still masked" \ + "$(gwm "$MK" main "SELECT email, phone FROM m_customers WHERE id = 2")" "ravi.k@example.org|XXXXXX3211" +$S mask account mask-analyst@foxbyte.dev --off --yes >/dev/null 2>&1 || bad "unmasking the account" +assert_eq "unmasked: real values again, and writes as before" \ + "$(gwm "$MK" main "SELECT phone FROM m_customers WHERE id = 2")|$(gwm "$MK" main "UPDATE m_customers SET city = 'Panaji' WHERE id = 2; SELECT city FROM m_customers WHERE id = 2")" "9876543211|Panaji" +assert_eq "every setup, rule and account change is in the security log" \ + "$($S audit log --limit 300 | grep -c 'mask.setup')|$($S audit log --limit 300 | grep -c 'mask.rule')|$($S audit log --limit 300 | grep -c 'mask.account')" "1|7|2" +$S branch delete maskown >/dev/null 2>&1 +for c in m_customers.phone m_customers.aadhaar_no m_orders.customer_email m_kyc.aadhaar_no; do $S mask rule "$c" --off >/dev/null 2>&1; done +$S mask rule m_customers.prefs --path '$.contact.mobile' --off >/dev/null 2>&1 +assert_eq "with every rule taken off, no mask is left behind: no view, shadow, trigger or rule" \ + "$(pg pg-main "SELECT (SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'bb_mask' AND c.relkind IN ('r','v')), (SELECT count(*) FROM pg_trigger WHERE tgname LIKE 'bb_mask%'), (SELECT count(*) FROM bb.mask_rules)")" "0|0|0" +pg pg-main "SET bb.allow_destructive = on; DROP TABLE IF EXISTS m_customers, m_orders, m_kyc, m_plain CASCADE" >/dev/null +$S user delete mask-analyst@foxbyte.dev >/dev/null 2>&1 + echo "### 12. fox uninstall (B1)" # Removal used to be a list of commands to run by hand. This runs last: it takes # the stack apart, so nothing after it has a stack to use. diff --git a/scripts/integration_ui.sh b/scripts/integration_ui.sh index f467801..dbc5145 100644 --- a/scripts/integration_ui.sh +++ b/scripts/integration_ui.sh @@ -113,6 +113,20 @@ rtsql "SET bb.allow_destructive=on; DROP TABLE IF EXISTS public.pii_people" rtsql "CREATE TABLE public.pii_people (id int PRIMARY KEY, aadhaar_no text, email text, card_no text, status text, profile jsonb)" rtsql "INSERT INTO public.pii_people VALUES (1, '234567890124', 'a@example.com', '4111111111111111', 'open', '{\"contact\": {\"phone\": \"9876543210\"}, \"theme\": \"dark\"}'), (2, '456789012341', 'b@example.org', NULL, 'closed', '{\"contact\": {\"phone\": \"+91 98765 43211\"}}')" +# Masking (F5), made here for the same reason: its setup restarts the branch to +# mount the key. A table whose personal data a reviewer confirmed, and an +# account to mask that is not an admin; the page sets the rules. +"$S" branch delete e2emask >/dev/null 2>&1; "$S" branch create e2emask >/dev/null 2>&1 +sudo docker exec -e PGPASSWORD=foxbyte pg-e2emask psql -U dbadmin -d appdb -q -c \ + "CREATE TABLE public.m_people (id int PRIMARY KEY, aadhaar_no text, email text); INSERT INTO public.m_people VALUES (1, '234567890124', 'a@example.com'), (2, '456789012341', 'b@example.org')" >/dev/null 2>&1 +"$S" pii scan --branch e2emask >/dev/null 2>&1 +for id in $("$S" pii --branch e2emask --json 2>/dev/null | python3 -c 'import sys,json; print(" ".join(str(f["id"]) for f in json.load(sys.stdin)["findings"] if f["table"] == "m_people"))'); do + "$S" pii mark "$id" confirm --branch e2emask >/dev/null 2>&1 +done +"$S" mask setup --branch e2emask --yes >/dev/null 2>&1 +printf '%s\n' "$PASSWORD" | "$S" user create e2e-masked@foxbyte.dev >/dev/null 2>&1 +"$S" mask account e2e-masked@foxbyte.dev --off --yes >/dev/null 2>&1 + # Application identity (F1), for the Blackbox's "for" test: a table made on the # branch in a transaction that asserted alice, with a token signed by an # application registered for this account. Through the Gateway, as an @@ -225,7 +239,9 @@ echo "### cleanup" # the feed off would otherwise fail for a reason this suite caused. "$S" realtime teardown --yes >/dev/null 2>&1 "$S" app remove uishop >/dev/null 2>&1 -for b in "$BRANCH" "$NEW_BRANCH" e2eui3; do "$S" branch delete "$b" >/dev/null 2>&1; done +"$S" mask account e2e-masked@foxbyte.dev --off --yes >/dev/null 2>&1 +"$S" user delete e2e-masked@foxbyte.dev >/dev/null 2>&1 +for b in "$BRANCH" "$NEW_BRANCH" e2eui3 e2emask; do "$S" branch delete "$b" >/dev/null 2>&1; done cd "$ROOT" || exit 1 # A failure leaves the screenshots and traces where CI can collect them. if [ "$rc" = 0 ]; then diff --git a/web/src/api.ts b/web/src/api.ts index fb0b330..d16c4df 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -25,6 +25,7 @@ export type Feature = | 'personal-data' | 'access-review' | 'hardening' + | 'masking' export type Status = { mainReady: boolean @@ -937,6 +938,27 @@ export const planHardeningProfile = (branch: string, profile: string) => req('POST', `${API}/api/hardening/profile`, { branch, profile, apply: false }) as Promise<{ fixes: HardeningFix[]; manual: HardeningCheck[] }> export const applyHardeningProfile = (branch: string, profile: string) => req('POST', `${API}/api/hardening/profile`, { branch, profile, apply: true }) as Promise<{ results: HardeningFixResult[] }> +// Masking (F5, point 4.6): admins only. Reading where it stands is free; +// setting it up, rules and masked accounts are Enterprise. +export type MaskRuleName = 'partial' | 'hash' | 'fake' | 'empty' +export type MaskRule = { schema: string; table: string; column: string; path?: string; rule: MaskRuleName; category: string; by: string; at: string } +export type MaskedAccount = { email: string; by: string; at: string } +export type MaskStatus = { on: boolean; pending: string[]; branch: string; rules: MaskRule[]; ready: string[]; accounts: MaskedAccount[]; writes: boolean } +export type MaskPlace = { schema: string; table: string; column: string; path?: string } +export const getMasking = (branch: string) => req('GET', `${API}/api/masking?branch=${encodeURIComponent(branch)}`) as Promise +export const planMaskSetup = (branch: string) => + req('POST', `${API}/api/masking/setup`, { branch, apply: false }) as Promise<{ branch: string; restart: boolean; keyDir: string }> +export const applyMaskSetup = (branch: string) => + req('POST', `${API}/api/masking/setup`, { branch, apply: true }) as Promise<{ branch: string; restarted: boolean }> +export const planMaskRule = (branch: string, at: MaskPlace, rule: MaskRuleName | '') => + req('POST', `${API}/api/masking/rule`, { branch, ...at, rule, apply: false }) as Promise<{ where: string; cost: string }> +export const setMaskRule = (branch: string, at: MaskPlace, rule: MaskRuleName | '') => + req('POST', `${API}/api/masking/rule`, { branch, ...at, rule, apply: true }) as Promise<{ set: string }> +export const dropMaskRule = (branch: string, at: MaskPlace) => + req('POST', `${API}/api/masking/rule`, { branch, ...at, off: true }) as Promise<{ off: string }> +export const maskAccount = (email: string, off: boolean) => + req('POST', `${API}/api/masking/account`, { email, off }) as Promise<{ email: string; masked: boolean }> + const hardeningURL = (branch: string) => `${API}/api/hardening?branch=${encodeURIComponent(branch)}` export const getHardening = (branch: string) => req('GET', hardeningURL(branch)) as Promise export const hardeningJsonUrl = hardeningURL diff --git a/web/src/components/AppLayout.tsx b/web/src/components/AppLayout.tsx index 9b03353..db0fec6 100644 --- a/web/src/components/AppLayout.tsx +++ b/web/src/components/AppLayout.tsx @@ -45,6 +45,7 @@ export const APP_NAV: NavSection[] = [ label: 'Compliance', items: [ { to: '/personal-data', label: 'Personal data', icon: I.IconPersonalData }, + { to: '/masking', label: 'Masking', icon: I.IconPersonalData }, { to: '/access-review', label: 'Access review', icon: I.IconKey }, { to: '/applications', label: 'Applications', icon: I.IconKey }, { to: '/audit-log', label: 'Audit log', icon: I.IconIntegrity }, diff --git a/web/src/main.tsx b/web/src/main.tsx index 9e25135..c16a0a6 100644 --- a/web/src/main.tsx +++ b/web/src/main.tsx @@ -23,6 +23,7 @@ import AccessReview from './pages/AccessReview' import Applications from './pages/Applications' import AuditLog from './pages/AuditLog' import Hardening from './pages/Hardening' +import Masking from './pages/Masking' import PipelineEditor from './pages/PipelineEditor' import { ConfirmProvider } from './confirm' import Dashboard from './pages/Dashboard' @@ -121,6 +122,7 @@ const router = createBrowserRouter([ { path: 'applications', element: }, { path: 'audit-log', element: }, { path: 'hardening', element: }, + { path: 'masking', element: }, { path: 'pipelines/:id', element: }, { path: 'console', element: }, { path: 'keys', element: }, diff --git a/web/src/pages/Masking.tsx b/web/src/pages/Masking.tsx new file mode 100644 index 0000000..3fd0fd2 --- /dev/null +++ b/web/src/pages/Masking.tsx @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +import { useCallback, useEffect, useState } from 'react' +import { + applyMaskSetup, dropMaskRule, getMasking, getPersonalData, maskAccount, planMaskRule, planMaskSetup, setMaskRule, + type MaskPlace, type MaskRule, type MaskRuleName, type MaskStatus, type PIIFinding, +} from '../api' +import { useConfirm } from '../confirm' +import { useFeatures, why } from '../features' +import { useBranches } from '../useBranches' + +// Masking (F5, point 4.6): rules on confirmed personal data, and the accounts +// an admin masks. A masked account reads the masks of marked tables and +// ordinary tables as they are, on every branch it does not own. Admins only; +// reading it is free, setting it is Enterprise. + +const RULES: MaskRuleName[] = ['partial', 'hash', 'fake', 'empty'] +const WHAT: Record = { + partial: 'all but the last four characters as X', + hash: 'a keyed hash: the same value, the same mask, so it still joins', + fake: 'a made-up value of its kind, the same for the same value', + empty: 'nothing', +} + +// The category's default (as the engine's DefaultMaskRule), and what it may be: +// Aadhaar is masked by display only, never hashed or faked. +const defaultRule = (category: string): MaskRuleName => { + const c = category.toLowerCase() + if (['aadhaar', 'pan', 'card', 'mobile', 'gstin', 'ifsc'].includes(c)) return 'partial' + if (c === 'email') return 'hash' + if (c === 'name') return 'fake' + return 'empty' +} +const allowed = (category: string): MaskRuleName[] => + category.toLowerCase() === 'aadhaar' ? ['partial', 'empty'] : RULES + +const where = (p: MaskPlace) => `${p.schema}.${p.table}.${p.column}${p.path ? ' ' + p.path : ''}` +const key = (p: MaskPlace) => `${p.schema}\u0000${p.table}\u0000${p.column}\u0000${p.path ?? ''}` +const viewOf = (r: MaskRule) => (r.schema === 'public' ? r.table : `${r.schema}__${r.table}`) + +export default function Masking() { + const { branches } = useBranches() + const [branch, setBranch] = useState('main') + const [st, setSt] = useState(null) + const [confirmed, setConfirmed] = useState([]) + const [choice, setChoice] = useState>({}) + const [email, setEmail] = useState('') + const [busy, setBusy] = useState(false) + const [err, setErr] = useState('') + const [msg, setMsg] = useState('') + const confirm = useConfirm() + const { can, edition } = useFeatures() + const setting = can('masking') + + const load = useCallback(async (b: string) => { + setBusy(true); setErr('') + try { + const s = await getMasking(b) + setSt(s) + try { + const pd = await getPersonalData(b) + setConfirmed((pd.findings ?? []).filter(f => f.state === 'confirmed')) + } catch { setConfirmed([]) } + } catch (e) { setSt(null); setErr((e as Error).message) } + finally { setBusy(false) } + }, []) + useEffect(() => { load('main') }, [load]) + + const run = async (f: () => Promise) => { + setErr(''); setMsg('') + try { setMsg(await f()); await load(branch) } catch (e) { setErr((e as Error).message) } + } + + const setup = () => run(async () => { + const p = await planMaskSetup(branch) + const ok = await confirm({ + title: `Set masking up on ${branch}`, + message: <> +

    This makes the install’s masking key if it has none: 32 random bytes in {p.keyDir}, readable only + by the database server and mounted read-only into each database. FoxByte never reads it back.

    +

    Then the masked readers’ role, schema and rights on {branch}. Nobody is masked until you mask an account.

    + {p.restart &&

    {branch} restarts to mount the key: connections drop for 5–15 seconds; nothing is lost.

    } + , + confirmText: 'Set it up', + }) + if (!ok) return '' + setBusy(true) + try { await applyMaskSetup(branch) } finally { setBusy(false) } + return `Masking is set up on ${branch}.` + }) + + const mask = (f: PIIFinding) => run(async () => { + const at: MaskPlace = { schema: f.schema, table: f.table, column: f.column, path: f.path || undefined } + const rule = choice[key(at)] ?? defaultRule(f.category) + const p = await planMaskRule(branch, at, rule) + const ok = await confirm({ + title: `Mask ${p.where} as ${rule}`, + message: <> +

    Every masked account sees {WHAT[rule]}, on every branch it does not own.

    + 
    {p.cost}
    + , + confirmText: 'Set the rule', + }) + if (!ok) return '' + await setMaskRule(branch, at, rule) + return `${p.where} is masked as ${rule}.` + }) + + const drop = (r: MaskRule) => run(async () => { + const at: MaskPlace = { schema: r.schema, table: r.table, column: r.column, path: r.path } + const ok = await confirm({ title: `Take the rule off ${where(at)}`, message: 'Masked accounts see its real values again.', confirmText: 'Take it off' }) + if (!ok) return '' + await dropMaskRule(branch, at) + return `${where(at)} is no longer masked.` + }) + + const account = (e: string, off: boolean) => run(async () => { + const ok = await confirm({ + title: off ? `Unmask ${e}` : `Mask ${e}`, + message: off + ? 'They see real values again on every branch. Their open sessions end.' + : 'They read the masks of marked tables, ordinary tables as they are, and cannot write to marked tables, on every branch they do not own. Their open sessions end. An admin is never masked.', + confirmText: off ? 'Unmask' : 'Mask', + }) + if (!ok) return '' + await maskAccount(e, off) + if (!off) setEmail('') + return off ? `${e} is unmasked.` : `${e} is masked.` + }) + + const ruled = new Set((st?.rules ?? []).map(r => key(r))) + const ready = new Set(st?.ready ?? []) + const open = confirmed.filter(f => !ruled.has(key({ schema: f.schema, table: f.table, column: f.column, path: f.path || undefined }))) + + return ( +
    +

    Masking

    +

    + Who sees the real values of personal data. A masked account reads masks of the columns a reviewer confirmed, and + everything else as it is. Admins, and a branch’s owner, always see real values. +

    + +
    + + +
    + + {err &&
    {err}
    } + {msg &&

    {msg}

    } + + {st && ( + <> +
    +

    + {st.on ? <>On. {st.rules.length} rule(s) on {st.branch}; {st.accounts.length} account(s) masked. : <>Not set up on this install.} +

    + {st.pending.length > 0 && ( +

    + Started before masking was set up, so masking nothing until they start again: {st.pending.join(', ')}. +

    + )} + {setting + ? + :

    {why(edition, 'Setting masking up, its rules and masked accounts,')}

    } +
    + +

    Rules

    +
    +
    + + + {st.rules.map(r => ( + + + + + + + + + ))} + {st.rules.length === 0 && } + +
    WhereRuleCategoryMasksSet by
    {where(r)}{r.rule}{r.category}{ready.has(viewOf(r)) ? 'made' : not made: masked accounts are refused the table}{r.by}, {new Date(r.at).toLocaleString()}{setting && }
    No rules on {st.branch}.
    + + +

    Confirmed personal data without a rule

    +

    + A rule is set on what a reviewer confirmed, on the Personal data page. Aadhaar is masked by display only, never hashed or faked. +

    +
    + + + + {open.map(f => { + const at: MaskPlace = { schema: f.schema, table: f.table, column: f.column, path: f.path || undefined } + const r = choice[key(at)] ?? defaultRule(f.category) + return ( + + + + + + + ) + })} + {open.length === 0 && } + +
    WhereCategoryRule
    {where(at)}{f.category} + +
    {WHAT[r]}
    +     		{setting && st.on && }
    None.
    +
    + +

    Masked accounts

    +

    + Install-wide: masked on every branch they do not own. Nobody is masked until an admin masks them. +

    +
    + + + + {st.accounts.map(a => ( + + + + + + ))} + {st.accounts.length === 0 && } + +
    AccountMasked by
    {a.email}{a.by}, {new Date(a.at).toLocaleString()}{setting && }
    None.
    +
    + {setting && st.on && ( +
    { e.preventDefault(); if (email.trim()) account(email.trim().toLowerCase(), false) }}> + + +
    + )} + + )} + + ) +} diff --git a/web/tests/masking.spec.ts b/web/tests/masking.spec.ts new file mode 100644 index 0000000..01c28e4 --- /dev/null +++ b/web/tests/masking.spec.ts @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// Masking (F5, point 4.6) in a browser, against a real engine: a rule set on +// a confirmed finding after the page says what it costs, Aadhaar offered only +// the rules it may have, and an account masked and unmasked. The branch was +// set up by integration_ui.sh before Chromium started, as its setup restarts +// it. integration §11ai checks what a masked account then reads. +import { test, expect, type Page } from '@playwright/test' + +const email = process.env.FOX_E2E_EMAIL ?? 'e2e@foxbyte.dev' +const password = process.env.FOX_E2E_PASSWORD ?? 'password123' +const masked = 'e2e-masked@foxbyte.dev' + +async function signIn(page: Page) { + await page.goto('/login', { waitUntil: 'domcontentloaded' }) + await page.getByPlaceholder('you@example.com').fill(email) + await page.getByPlaceholder('password').fill(password) + await page.getByRole('button', { name: /log in/i }).click() + await expect(page.getByRole('heading', { name: 'Dashboard' })).toBeVisible() +} + +test('a rule is set on a confirmed finding after its cost, and an account is masked and unmasked', async ({ page }) => { + await signIn(page) + await page.getByRole('link', { name: 'Masking' }).click() + await expect(page.getByRole('heading', { name: 'Masking', exact: true })).toBeVisible() + await page.getByLabel('Branch').selectOption('e2emask') + await expect(page.getByTestId('mask-state')).toContainText('On.', { timeout: 30_000 }) + + // Aadhaar: masked by display only, never hashed or faked. + const aadhaar = page.getByTestId('open-m_people-aadhaar_no') + await expect(aadhaar.getByRole('combobox').locator('option')).toHaveText(['partial', 'empty']) + + const row = page.getByTestId('open-m_people-email') + await expect(row.getByRole('combobox')).toHaveValue('hash') + await row.getByRole('button', { name: 'Mask…' }).click() + let dlg = page.getByRole('dialog') + await expect(dlg.getByTestId('mask-cost')).toContainText('3.6 times as long') + await dlg.getByRole('button', { name: 'Set the rule' }).click() + await expect(page.getByTestId('mask-msg')).toHaveText('public.m_people.email is masked as hash.', { timeout: 30_000 }) + await expect(page.getByTestId('rule-m_people-email')).toContainText('made') + await expect(page.getByTestId('open-m_people-email')).toHaveCount(0) + + await aadhaar.getByRole('button', { name: 'Mask…' }).click() + dlg = page.getByRole('dialog') + await expect(dlg.getByTestId('mask-cost')).toContainText('nothing on writes') + await dlg.getByRole('button', { name: 'Set the rule' }).click() + await expect(page.getByTestId('rule-m_people-aadhaar_no')).toContainText('partial', { timeout: 30_000 }) + + // An admin is refused: they can change the masks. + await page.getByLabel('Account to mask').fill(email) + await page.locator('form').getByRole('button', { name: 'Mask…' }).click() + await page.getByRole('dialog').getByRole('button', { name: 'Mask', exact: true }).click() + await expect(page.locator('.err')).toContainText('is an admin') + + await page.getByLabel('Account to mask').fill(masked) + await page.locator('form').getByRole('button', { name: 'Mask…' }).click() + await page.getByRole('dialog').getByRole('button', { name: 'Mask', exact: true }).click() + await expect(page.getByTestId(`account-${masked}`)).toBeVisible({ timeout: 30_000 }) + await page.getByTestId(`account-${masked}`).getByRole('button', { name: 'Unmask…' }).click() + await page.getByRole('dialog').getByRole('button', { name: 'Unmask', exact: true }).click() + await expect(page.getByTestId(`account-${masked}`)).toHaveCount(0, { timeout: 30_000 }) + + // Taken off again, so a rerun starts as this one did. + await page.getByTestId('rule-m_people-email').getByRole('button', { name: 'Take off…' }).click() + await page.getByRole('dialog').getByRole('button', { name: 'Take it off' }).click() + await expect(page.getByTestId('open-m_people-email')).toBeVisible({ timeout: 30_000 }) +})