diff --git a/cmd/fox/main.go b/cmd/fox/main.go index c28784b..f05578b 100644 --- a/cmd/fox/main.go +++ b/cmd/fox/main.go @@ -116,6 +116,10 @@ Stack: The masking keys: domains that never hash alike, and rotating a key (Enterprise) mask key use --table [schema.]table [--branch b] [--yes] Put a table's masks under a key domain (Enterprise) + mask unmask --branch b [--off] [--yes] + Let a masked account see real values on a branch, every read audited (Enterprise) + mask lookups [--branch b] [--limit n] [--json] + Who looked a person up, when, by which column, how many found: never the value access review [--dormant 90] [--json] [--csv accounts|keys|branches|applications] Who can reach what, with which keys, who may override the guardrails, who has gone quiet, and the applications that assert their users (Enterprise) diff --git a/cmd/fox/maskcmd.go b/cmd/fox/maskcmd.go index 9642b5c..e17f976 100644 --- a/cmd/fox/maskcmd.go +++ b/cmd/fox/maskcmd.go @@ -25,6 +25,8 @@ import ( // fox mask key [status] [--json] // fox mask key create|rotate|drop [--yes] (Enterprise) // fox mask key use --table [schema.]table [--branch b] [--yes] (Enterprise) +// fox mask unmask --branch b [--off] [--yes] (Enterprise) +// fox mask lookups [--branch b] [--limit n] [--json] // // Reading where masking stands is in every edition; setting it is Enterprise. // Each setup, rule and account change is in the security log. @@ -43,6 +45,12 @@ func maskCmd(args []string) { case "key": maskKey(args[1:]) return + case "unmask": + maskUnmask(args[1:]) + return + case "lookups": + maskLookups(args[1:]) + return case "status": args = args[1:] } @@ -105,6 +113,14 @@ func printMaskStatus(st branch.MaskStatus) { if len(st.Accounts) == 0 { fmt.Printf(" none: nobody is masked until an admin masks them\n") } + fmt.Printf("\nUnmask on %s (%d): real values, every read of a marked table audited\n", st.Branch, len(st.Unmasks)) + for _, g := range st.Unmasks { + fmt.Printf(" %-40s by %s at %s\n", g.Email, g.By, g.At) + } + if len(st.Unmasks) > 0 && !st.AuditOn { + fmt.Printf(" The audit log is off, so their reads are not being logged: %s audit-log on\n", brand.CLI) + } + fmt.Printf("\nLookups on %s: %d recorded (%s mask lookups --branch %s)\n", st.Branch, len(st.Lookups), brand.CLI, st.Branch) } // maskSetup sets masking up on a branch: the key if the install has none, the @@ -420,3 +436,66 @@ func printMaskKeys(st branch.MaskKeysStatus) { fmt.Printf("\nNot running, so not read: %s\n", strings.Join(st.Stopped, ", ")) } } + +// maskUnmask gives a masked account unmask on a branch, or takes it. There it +// sees real values, and every read it makes of a marked table is audited. +func maskUnmask(args []string) { + requireFeature(edition.Masking) + ps := positionals(args, "--branch") + b := optValue(args, "--branch") + if len(ps) != 1 || b == "" { + fmt.Fprintf(os.Stderr, "usage: %s mask unmask --branch b [--off] [--yes]\n", brand.CLI) + os.Exit(2) + } + email, off := strings.ToLower(ps[0]), hasFlag(args, "--off") + if !hasFlag(args, "--yes") { + if off { + fmt.Printf("This takes unmask on %s from %s: they read masks there again.\n", b, email) + } else { + fmt.Printf("This lets %s see real values on %s, as its owner does.\n\n"+ + "Every read they make there of a marked table is audited: their role's audit\n"+ + "role becomes %s, which only a superuser can change, so the audit log\n"+ + "records those reads, and theirs only. It is listed in the access review.\n", + email, b, branch.AuditReadRole) + if !branch.AuditOn() { + fmt.Printf("\nThe audit log is off, so their reads are not logged until `%s audit-log on`.\n", brand.CLI) + } + } + fmt.Printf("Their open sessions on %s end, so the next one sees the change.\n\nRun again with --yes to go ahead.\n", b) + return + } + must(branch.Unmask(email, b, cliActor(), off)) + what := "given unmask on " + b + if off { + what = "no longer unmasked on " + b + } + recordMask(auth.EvMaskUnmask, b, email+" "+what) + fmt.Printf("Done: %s is %s.\n", email, what) +} + +// maskLookups lists a branch's lookups: who, when, which column, how many +// rows found. Never the value looked up, which is not kept. +func maskLookups(args []string) { + b := maskBranch(args) + n := 100 + if v := optValue(args, "--limit"); v != "" { + if _, err := fmt.Sscan(v, &n); err != nil { + must(fmt.Errorf("--limit is a number, not %q", v)) + } + } + ls, err := branch.Lookups(b, n) + must(err) + if hasFlag(args, "--json") { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + must(enc.Encode(ls)) + return + } + fmt.Printf("Lookups on %s (%d, newest first). The values looked up are not kept.\n", b, len(ls)) + for _, l := range ls { + fmt.Printf(" %s %-36s %s.%s found %d\n", l.At, l.Who, l.Table, l.Column, l.Found) + } + if len(ls) == 0 { + fmt.Printf(" none: a masked account looks a person up with SELECT * FROM %s.find('table', 'column', 'value')\n", branch.MaskSchema) + } +} diff --git a/docs/FOX_Checklist.html b/docs/FOX_Checklist.html index a5e0a7a..b78177e 100644 --- a/docs/FOX_Checklist.html +++ b/docs/FOX_Checklist.html @@ -474,7 +474,7 @@

HBlackbox (record of schema changes)Enterprise: confirming personal data (point 4.6) marks its table’s writes; the tables confirmed and not audited named in every edition, and marked together with table --personal-data
  • SECURITY DEFINER functions naming marked tables named as not audited, in fox check, status and the console
  • Enterprise: the evidence export for a period: coverage and limits as stated, each branch’s marks and gaps, integrity and anchors, delivery, activity and every change; with the records and anchors to check it offline (step 6b)
    • - Unit; f6 suite; §11y–§11af; UI +
  • Reads audited for the logins that may unmask: fox mask unmask (masking step 3, O3) sets the account’s role’s pgaudit.role to bb_audit_read, so its reads of marked tables are logged, and only its
    • Unit; f6 suite; §11y–§11af; UI H22The hardening report (point 3.2; FoxByte’s own checks, not the CIS Benchmark)
    • About 40 checks, basic and strict, across installation, files, logging, auditing, roles, connections, settings and backups: pass, fail, manual or n/a, each with what was found, why it matters and the fix
    • Where FoxByte differs on purpose (TLS at the Gateway, statement text kept out of the log, local trust inside the container), the check says so rather than failing
      • @@ -767,15 +767,16 @@

      OPersonal data & compliancefox access review [--dormant] [--json] [--csv accounts|keys|branches|applications], GET /api/access-review (admins only), and the console’s Access review page with CSV downloads and print for PDF
    • Each report in the security log; what it cannot claim stated in it
    • Applications: each registration, its account, its keys with age, state and what each asserted by branch, and findings to question; --csv applications (F1)
      • -
    • No unmask rights to review until masking (point 7); campaigns are 4.9 (Gate 2)

    enterprise/accessreview unit; VM §11s; accessreview.spec.ts +
  • Sees unmasked: whether each account is an admin, not masked, or masked, with the branches it owns and where it holds unmask, its reads audited (masking step 3)
    • +
  • Campaigns are 4.9 (Gate 2)
    • enterprise/accessreview unit; VM §11s; accessreview.spec.ts O3Masking personal data (point 4.6 and F5, Enterprise; docs/FOX_F5_Masking_Design.md)
    • Gate 0 prototype (make integration-f5, 57/57 on PostgreSQL 18 and 16): the keyed hash in bb_identity 0.2, its key a file the server reads; a masked branch on a ZFS clone left with no real value in its files; the costs measured
    • Step 1: fox mask setup|rule|account, four API routes and the console’s Masking page; rules (partial, keyed hash, fake, empty) only on confirmed findings, Aadhaar never hashed or faked; masks kept in shadow tables by statement triggers; masked accounts read masks and ordinary tables, never the real marked table, the hash or the key, and cannot write to marked tables; admins and a branch’s owner see real values; the cost said before a rule is set
    • Step 2: fox mask key [status|create|rotate|drop|use], two API routes and a Keys panel on the Masking page; key domains whose masks hash apart; a table’s masks put under one; a key rotated on every running branch while masked readers read the old masks until each table is done and writes pause only for a moment, then the old key deleted; a rotation cut short carries on when run again; a branch stopped through a rotation named, and its masks made again by setup; a domain in use, or perhaps in use on a branch stopped since it was made, is not deleted
      • -
    • Step 3: unmask, the audited lookup, unmask rights in the access review, reads audited
      • +
    • Step 3: fox mask unmask <email> --branch b gives a masked account real values on one branch, every read it makes there of a marked table logged by the audit log through bb_audit_read (its role’s pgaudit.role, a superuser’s setting it cannot undo), which closes 3.1; the lookup, bb_mask.find(table, column, value), finds a person by an exact value and returns the rows masked, 30 a minute for each account, each call recorded without the value (fox mask lookups); the access review’s Sees unmasked column; a deleted branch’s unmask goes with it; two API routes and the Masking page’s Unmask and Lookups tables
    • Step 4: every access path tested (COPY, functions, EXPLAIN, statistics, logs)
    • Step 5: masked branches, closed until masked and verified
      • -
    • One-column primary keys only; a JSON path into an array, or a quoted key, is refused for now; masking stays on once set up; a masked account has no DDL rights; a domain’s key is the install’s, shared by every branch, so a branch stopped through a rotation keeps its old masks until set up again
    enterprise/masking unit; internal/branch mask_test.go, maskkey_test.go; VM §11ai, §11aj; masking.spec.ts +
  • One-column primary keys only; a JSON path into an array, or a quoted key, is refused for now; masking stays on once set up; a masked account has no DDL rights; a domain’s key is the install’s, shared by every branch, so a branch stopped through a rotation keeps its old masks until set up again; an unmask holder’s reads are logged only while the audit log is on; the lookup is by a column with a rule of its own, not a JSON path
    • enterprise/masking unit; internal/branch mask_test.go, maskkey_test.go; VM §11ai, §11aj, §11ak; masking.spec.ts
    FOX_Checklist · area O
    diff --git a/docs/FOX_Checklist.pdf b/docs/FOX_Checklist.pdf index fbf606f..431a765 100644 Binary files a/docs/FOX_Checklist.pdf and b/docs/FOX_Checklist.pdf differ diff --git a/docs/FOX_Feature_Implemented.html b/docs/FOX_Feature_Implemented.html index 153f6d7..22f05ca 100644 --- a/docs/FOX_Feature_Implemented.html +++ b/docs/FOX_Feature_Implemented.html @@ -119,6 +119,7 @@

    Change log

    Newest first. Every change to FoxByte adds a row here.

    + @@ -1687,19 +1688,19 @@

    Test matrix

    Who
    Admins only, in the console and the API: it lists every account and credential on the install. At the machine the operator is an admin already, as for fox user list. Each report produced is in the security log (access.reviewed), with its counts.
    How to use
    fox access review [--dormant 90] [--json] [--csv accounts|keys|branches|applications] · GET /api/access-review?dormant=90[&format=csv&section=accounts|keys|branches] · the console’s Access review page (Compliance): a window, the counts and three tables, each table as CSV, and Print or save as PDF (the page prints alone, without the sidebar).
    How to test
    Unit: enterprise/accessreview/review_test.go (admins, reach, ownership, overrides, last active from three signals, dormancy and a new account, key kinds, reach and staleness, branches and the standby, unmasked, the CSV and a formula made safe), AccessReview.test.tsx (five). Integration §11s (a dormant account and a stale key, backdated in the store; ownership and an override granted; no secret in it; CSV with a formula made safe; a suspended branch not woken; the security log; the API for admins only, as JSON and CSV, and its refusals). Playwright web/tests/accessreview.spec.ts.
    -
    Status & limits
    Working, Enterprise (access-review). No masking is in place yet (point 7), so there are no unmask rights to review: every account sees what the branches it reaches hold, and the report says so. Dormant means no activity FoxByte saw: a role made directly in Postgres is not seen, and someone with history off is judged by sign-ins and key use. Who may override the guardrails is read from running branches only; a suspended branch is not woken, and the report says how many. PDF is the browser’s print; review campaigns (approve or revoke on a schedule) are 4.9, in Gate 2.
    +
    Status & limits
    Working, Enterprise (access-review). Sees unmasked comes from masking (point 7, O3): with none set up, every account sees what the branches it reaches hold, and the report says so; with it, whether each account is an admin, not masked, or masked, the branches it owns and where it holds unmask, its reads audited while the audit log is on. Dormant means no activity FoxByte saw: a role made directly in Postgres is not seen, and someone with history off is judged by sign-ins and key use. Who may override the guardrails is read from running branches only; a suspended branch is not woken, and the report says how many. PDF is the browser’s print; review campaigns (approve or revoke on a schedule) are 4.9, in Gate 2.
    Applications (26 Oct 2026)
    The report’s fourth table: the applications that may say whom a change was made for (application identity, H20). For each, the account its tokens are accepted from, its token life and what an invalid token does, and each key with its age, whether it is active, when it was revoked and whether it was reported compromised, and what it asserted: Blackbox records and agent sessions started for a user (H16), counted on each running branch (a branch made from main holds main’s records from before it was made, so two branches’ counts can overlap, which the report says). Findings a reviewer should question, one sentence each: the account no longer exists; it signs in to the console, so it is likely a person’s, who could then assert any user; it is an install admin; there is no active key; an active key is older than the dormant window (rotate it), or has asserted nothing in that time; a key was reported compromised. fox access review prints it, with the findings; --csv applications and section=applications give one row per key; the console’s Access review page shows the table, the findings and a CSV link; the summary counts applications and those with findings. A branch whose Blackbox predates application identity is not counted, and the report says which.
    Added in
    feat/access-review.
    O3

    Masking personal data

    Partial
    What it does
    An admin masks accounts; a masked account then reads, on every branch it does not own, the masks of the columns a reviewer confirmed as personal data, and everything else as it is. Each rule is on a confirmed finding (point 4.6), a column or a JSON path in it: partial (all but the last four characters as X), hash (a keyed HMAC-SHA-256, the same value the same mask, so filters, grouping and joins still work), fake (a made-up value of its kind, the same for the same value) or empty. Without a rule named, the category’s default: partial for Aadhaar, PAN, cards, mobiles, GSTIN and IFSC; hash for email; else empty. Aadhaar is never hashed or faked: a hash applications join on would be a reference key, which UIDAI forbids. The cost is said before a rule is set.
    -
    How it works
    Masks are made on write, not read: for a hash, a fake or a JSON path, a shadow table bb_mask.<schema>__<table> holds each row’s masks, kept by statement triggers with transition tables (inserts, changes, a moved key, deletes, TRUNCATE) owned by bb_masker, which nobody logs in as; a view with the table’s own name in bb_mask joins it to the table, and computes partial and empty masks itself, so a table with only those has no shadow and no write cost. Rows already there are filled in committed batches of 10,000 without holding up writers. A masked account’s role is in db_masked rather than db_client, with search_path = bb_mask, public and log_min_error_statement = panic (superuser-only, so it cannot undo them); db_masked has no right on a marked table, its shadow, the hash function or the key. The key is 32 random bytes in the install’s state directory, owned by the database server’s user and readable by nobody else, mounted read-only at /run/bb-mask; bb_identity 0.2’s bbid_mask_hmac reads it, and FoxByte never reads it back. Key domains (step 2): the install starts with one, default, so a value hashes alike in every table and joins work; a named domain has a key of its own, <domain>.key, so tables under it hash apart from every other domain’s and cannot be joined with them (for data that must never be matched up, such as two tenants’). Each branch keeps its tables’ domain and the key their masks were made under in bb.mask_tables; what is kept about the keys (domain, key number, when made and rotated, by whom) is in mask-keys.json in the state directory, never a key. Rotation makes the new key beside the old as <domain>--next.key (a name a person cannot give a domain), and on each running branch, for each table in the domain with keyed masks: a second shadow kept by the same triggers under the new key beside the old, filled in committed batches, then swapped in, in one transaction, with the view replaced in place (its grants kept) and the old shadow dropped, so masked readers read the old masks until the table is done; a table’s writes pause for a moment as it starts and as it is swapped over (a lock that waits for the writes already running, so none runs triggers changed under it, bounded and tried again behind a long transaction), and reads never wait. Then the new key is copied over the domain’s file, which deletes the old key, the triggers write under the domain’s name again, and the next key is removed. A rotation cut short carries on with the key it made; a branch started during one has its masks made again; one stopped through it keeps masks made under the old key, is named, and is made again by fox mask setup --branch. Masked accounts are kept install-wide; the role is made on each branch when the account is used, and changed at once on running ones, with its open sessions ended. Every change is recorded in the Blackbox under the person who asked.
    -
    Who
    Admins only: they set masking up, set rules and mask accounts. An admin is never masked (they can change the masks), and a branch’s owner sees real values on it. Reading where masking stands is in every edition; setting it is Enterprise (masking). Every setup, rule, account and key change is in the security log (mask.setup, mask.rule, mask.account, mask.key).
    -
    How to use
    fox mask [status] [--branch b] [--json] · fox mask setup [--branch b] [--yes] (makes the key; the branch restarts to mount it) · fox mask rule <[schema.]table.column> [partial|hash|fake|empty] [--path $.a.b] [--branch b] [--yes] and --off · fox mask account <email> [--off] [--yes] · fox mask key [status] [--json], fox mask key create|rotate|drop <domain> [--yes], fox mask key use <domain> --table [schema.]table [--branch b] [--yes] · GET /api/masking, GET /api/masking/keys, POST /api/masking/setup|rule|account|key · the console’s Masking page (Compliance): set up, the rules and whether each table’s masks are made, the confirmed findings without a rule with the rules each may have, the masked accounts, and the keys (each domain, its key number and tables, rotate and delete, a new domain, a table moved). Each change is shown, with its cost, before it is made.
    -
    How to test
    Unit: enterprise/masking/sql_test.go (the shadow, triggers, fill and view for every rule; no shadow for partial alone; rights given back with no rules; what cannot be masked refused, saying why; names outside public), internal/branch/mask_test.go (Aadhaar refused a hash; the defaults; the masked and unmasked role; the accounts’ file and its exemptions; the licence; the mount; the cost), internal/branch/maskkey_test.go (domain names, the records, what a rotation says, the licence) and, in sql_test.go, a table under a domain, rekeying, and repointing. Integration §11ai, §11aj (domains hash apart; a table moved and back; a rotation while the application writes and a masked account reads, every mask under the new key, the old key’s file replaced and the next one removed, nothing left over; a stopped branch named and made again; a domain in use kept; the API; the security log). Playwright web/tests/masking.spec.ts.
    -
    Status & limits
    Partial: Gate 1 steps 1 and 2 of the design. Next: unmask, the audited lookup and unmask rights in the access review (step 3); every access path tested (step 4); masked branches (step 5). A table needs a one-column primary key; a JSON path into an array, or a quoted key, is refused for now; masking stays on once set up; a masked account has no DDL rights; a schema made after setup needs setup run again; a domain is deleted only when no table uses it and no branch has stopped since it was made, whose tables cannot be read. Rotating costs about 21 seconds a million rows, and a table’s writes about twice the usual masking while its masks are written again. Measured cost on a table with a hash, fake or JSON mask: a fifth to a third fewer single-row writes, a 10,000-row UPDATE about 3.6 times as long; masked reads about a quarter slower; documents over 64 KB with a JSON path much more (design, decision 9).
    -
    Added in
    feat/f5-mask-step1; key domains and rotation in feat/f5-mask-step2.
    +
    How it works
    Masks are made on write, not read: for a hash, a fake or a JSON path, a shadow table bb_mask.<schema>__<table> holds each row’s masks, kept by statement triggers with transition tables (inserts, changes, a moved key, deletes, TRUNCATE) owned by bb_masker, which nobody logs in as; a view with the table’s own name in bb_mask joins it to the table, and computes partial and empty masks itself, so a table with only those has no shadow and no write cost. Rows already there are filled in committed batches of 10,000 without holding up writers. A masked account’s role is in db_masked rather than db_client, with search_path = bb_mask, public and log_min_error_statement = panic (superuser-only, so it cannot undo them); db_masked has no right on a marked table, its shadow, the hash function or the key. The key is 32 random bytes in the install’s state directory, owned by the database server’s user and readable by nobody else, mounted read-only at /run/bb-mask; bb_identity 0.2’s bbid_mask_hmac reads it, and FoxByte never reads it back. Key domains (step 2): the install starts with one, default, so a value hashes alike in every table and joins work; a named domain has a key of its own, <domain>.key, so tables under it hash apart from every other domain’s and cannot be joined with them (for data that must never be matched up, such as two tenants’). Each branch keeps its tables’ domain and the key their masks were made under in bb.mask_tables; what is kept about the keys (domain, key number, when made and rotated, by whom) is in mask-keys.json in the state directory, never a key. Rotation makes the new key beside the old as <domain>--next.key (a name a person cannot give a domain), and on each running branch, for each table in the domain with keyed masks: a second shadow kept by the same triggers under the new key beside the old, filled in committed batches, then swapped in, in one transaction, with the view replaced in place (its grants kept) and the old shadow dropped, so masked readers read the old masks until the table is done; a table’s writes pause for a moment as it starts and as it is swapped over (a lock that waits for the writes already running, so none runs triggers changed under it, bounded and tried again behind a long transaction), and reads never wait. Then the new key is copied over the domain’s file, which deletes the old key, the triggers write under the domain’s name again, and the next key is removed. A rotation cut short carries on with the key it made; a branch started during one has its masks made again; one stopped through it keeps masks made under the old key, is named, and is made again by fox mask setup --branch. Unmask (step 3) is a right on one branch an admin gives a masked account: there it sees real values, as an owner does, and its role’s pgaudit.role is bb_audit_read, which holds SELECT on every masked table (and every table marked for audit), so the audit log records its reads of them, and only its; the setting is a superuser’s, so the account cannot undo it. Grants are kept in unmask-grants.json in the state directory; a deleted branch’s go with it, and an account unmasked keeps none. The lookup, bb_mask.find(table, column, value), finds a person by an exact value without unmasking: SECURITY DEFINER as bb_masker, only in a column with a rule of its own, it hashes the value under the table’s domain for a hashed column and compares it in the table for the others, returns the rows as the view has them, refuses a value not of the column’s type without repeating it, allows 30 calls a minute for each account, and records each call in bb_mask.lookups (who, when, which column, how many found), which a masked account cannot read; never the value. Masked accounts are kept install-wide; the role is made on each branch when the account is used, and changed at once on running ones, with its open sessions ended. Every change is recorded in the Blackbox under the person who asked.
    +
    Who
    Admins only: they set masking up, set rules and mask accounts. An admin is never masked (they can change the masks), and a branch’s owner sees real values on it. Reading where masking stands is in every edition; setting it is Enterprise (masking). Every setup, rule, account and key change is in the security log (mask.setup, mask.rule, mask.account, mask.key, mask.unmask). The access review’s Sees unmasked column says, for each account, whether it is an admin, not masked, or masked, with the branches it owns and those where it holds unmask.
    +
    How to use
    fox mask [status] [--branch b] [--json] · fox mask setup [--branch b] [--yes] (makes the key; the branch restarts to mount it) · fox mask rule <[schema.]table.column> [partial|hash|fake|empty] [--path $.a.b] [--branch b] [--yes] and --off · fox mask account <email> [--off] [--yes] · fox mask key [status] [--json], fox mask key create|rotate|drop <domain> [--yes], fox mask key use <domain> --table [schema.]table [--branch b] [--yes] · fox mask unmask <email> --branch b [--off] [--yes] · fox mask lookups [--branch b] [--limit n] [--json] · as a masked account, SELECT * FROM bb_mask.find('table', 'column', 'value') · GET /api/masking, GET /api/masking/keys, GET /api/masking/lookups, POST /api/masking/setup|rule|account|key|unmask · the console’s Masking page (Compliance): set up, the rules and whether each table’s masks are made, the confirmed findings without a rule with the rules each may have, the masked accounts, and the keys (each domain, its key number and tables, rotate and delete, a new domain, a table moved), unmask on the branch, and its lookups. Each change is shown, with its cost, before it is made.
    +
    How to test
    Unit: enterprise/masking/sql_test.go (the shadow, triggers, fill and view for every rule; no shadow for partial alone; rights given back with no rules; what cannot be masked refused, saying why; names outside public), internal/branch/mask_test.go (Aadhaar refused a hash; the defaults; the masked and unmasked role; the accounts’ file and its exemptions; the licence; the mount; the cost), internal/branch/maskkey_test.go (domain names, the records, what a rotation says, the licence) and, in sql_test.go, a table under a domain, rekeying, and repointing. internal/branch/unmask_test.go (unmask on one branch, audited there only, gone with its branch; the audit role’s SQL; the licence); enterprise/accessreview/review_test.go (the column). Integration §11ak (unmask refused to an account not masked; real values on one branch only, the role’s audit role, both reads logged with no value, not undoable; the access review; the lookup by a hash and a partial mask, only by a ruled column, recorded without the value, 30 a minute, no value in the log; a deleted branch’s unmask gone; the security log), §11ai, §11aj (domains hash apart; a table moved and back; a rotation while the application writes and a masked account reads, every mask under the new key, the old key’s file replaced and the next one removed, nothing left over; a stopped branch named and made again; a domain in use kept; the API; the security log). Playwright web/tests/masking.spec.ts.
    +
    Status & limits
    Partial: Gate 1 steps 1 to 3 of the design. Next: every access path tested (step 4); masked branches (step 5). An unmask holder’s reads are logged only while the audit log is on (said when unmask is given); the lookup is by a column with a rule of its own, not a JSON path; reset makes a branch again, so it loses its unmask. The Gateway lets a non-admin reach main and the branches it owns, where it sees real values anyway, so unmask matters on main until a branch can be shared. A table needs a one-column primary key; a JSON path into an array, or a quoted key, is refused for now; masking stays on once set up; a masked account has no DDL rights; a schema made after setup needs setup run again; a domain is deleted only when no table uses it and no branch has stopped since it was made, whose tables cannot be read. Rotating costs about 21 seconds a million rows, and a table’s writes about twice the usual masking while its masks are written again. Measured cost on a table with a hash, fake or JSON mask: a fifth to a third fewer single-row writes, a 10,000-row UPDATE about 3.6 times as long; masked reads about a quarter slower; documents over 64 KB with a JSON path much more (design, decision 9).
    +
    Added in
    feat/f5-mask-step1; key domains and rotation in feat/f5-mask-step2; unmask, the lookup and the access review’s column in feat/f5-mask-step3.
    FOX_Feature Implemented · area O
    diff --git a/docs/FOX_Feature_Implemented.pdf b/docs/FOX_Feature_Implemented.pdf index 0b317ca..b1eb42d 100644 Binary files a/docs/FOX_Feature_Implemented.pdf and b/docs/FOX_Feature_Implemented.pdf differ diff --git a/enterprise/accessreview/review.go b/enterprise/accessreview/review.go index ef3c70e..75e0828 100644 --- a/enterprise/accessreview/review.go +++ b/enterprise/accessreview/review.go @@ -7,6 +7,7 @@ package accessreview import ( "fmt" "sort" + "strings" "time" "github.com/thefoxbyte/foxbyte/internal/access" @@ -30,6 +31,36 @@ type inputs struct { // asserted is, by running branch, what each application key asserted. asserted map[string]map[[2]string]branch.Asserted limits []string // what gathering could not read + // Masking (point 7): whether it is set up, who is masked, and where each + // masked account holds unmask. + maskOn bool + masked map[string]bool + unmasks map[string][]string // email → branches + auditOn bool +} + +// unmaskedText says what an account sees without masking (4.8's column). +func unmaskedText(in inputs, email string, admin bool, owns []string) string { + switch { + case !in.maskOn: + return "everything the branches it reaches hold: no masking is in place (point 7)" + case admin: + return "everything: an admin always sees real values" + case !in.masked[strings.ToLower(email)]: + return "everything the branches it reaches hold: not masked" + } + s := "masks of marked tables" + if len(owns) > 0 { + s += "; real values on the branches it owns (" + strings.Join(owns, ", ") + ")" + } + if u := in.unmasks[strings.ToLower(email)]; len(u) > 0 { + sort.Strings(u) + s += "; unmask on " + strings.Join(u, ", ") + ", its reads of marked tables audited" + if !in.auditOn { + s += " once the audit log is on" + } + } + return s } func review(store *auth.Store, o access.ReviewOptions) (access.Review, error) { @@ -78,6 +109,14 @@ func gather(store *auth.Store) (inputs, error) { in.admins[b] = a } } + in.maskOn, in.auditOn = branch.MaskOn(), branch.AuditOn() + in.masked, in.unmasks = map[string]bool{}, map[string][]string{} + for _, a := range branch.MaskedAccounts() { + in.masked[a.Email] = true + } + for _, g := range branch.UnmaskGrants() { + in.unmasks[g.Email] = append(in.unmasks[g.Email], g.Branch) + } if in.apps, err = store.Apps(); err != nil { return in, fmt.Errorf("listing applications: %w", err) } @@ -188,7 +227,7 @@ func build(in inputs, dormantDays int, now time.Time) access.Review { default: ra.Reaches = "main" } - ra.Unmasked = "everything the branches it reaches hold: no masking is in place (point 7)" + ra.Unmasked = unmaskedText(in, a.Email, ra.Admin, ra.Owns) ra.LastSignIn = ptr(in.signIns[a.Email]) ra.LastKeyUse = ptr(lastKeyUse[a.ID]) ra.LastQuery = ptr(in.queries[a.Email]) @@ -246,8 +285,13 @@ func build(in inputs, dormantDays int, now time.Time) access.Review { r.Summary.Accounts, r.Summary.Keys, r.Summary.Branches = len(r.Accounts), len(r.Keys), len(r.Branches) r.Summary.Applications = len(r.Applications) + masking := "No masking is in place yet (point 7), so there are no unmask rights to review: every account sees, unmasked, what the branches it reaches hold." + if in.maskOn { + masking = "Sees unmasked is from masking (point 7): who an admin masked, and where each masked account owns a branch or holds unmask. " + + "An unmask holder's reads of marked tables are audited while the audit log is on. A role someone made directly in Postgres is not seen." + } r.Limits = append([]string{ - "No masking is in place yet (point 7), so there are no unmask rights to review: every account sees, unmasked, what the branches it reaches hold.", + masking, fmt.Sprintf("Last active is the newest of a console sign-in, a key's use (recorded to the minute) and a query kept in history. Dormant means no activity FoxByte saw in %d days: a role someone made directly in Postgres is not seen.", dormantDays), "Identity is what each credential is: an agent key marks an agent by how it logs in. Whom a change was made for is what a registered application asserted with its key (Applications), which means the application vouched for the user, not that the user was there.", "What an application's key asserted is counted in the Blackbox and the agent sessions of running branches. A branch made from main holds main's records from before it was made, so two branches' counts can overlap; rows whose last_modified_for names an application are not counted.", diff --git a/enterprise/accessreview/review_test.go b/enterprise/accessreview/review_test.go index d38210d..897c46c 100644 --- a/enterprise/accessreview/review_test.go +++ b/enterprise/accessreview/review_test.go @@ -273,3 +273,39 @@ func TestApplicationsEmpty(t *testing.T) { t.Errorf("no applications: %+v", r.Applications) } } + +// With masking: an admin sees real values, an account not masked sees +// everything, a masked one masks, except where it owns a branch or holds +// unmask, whose reads are audited, said to wait for the audit log if it is off. +func TestUnmaskedColumn(t *testing.T) { + in := inputs{maskOn: true, masked: map[string]bool{"m@x.io": true}, unmasks: map[string][]string{"m@x.io": {"dev", "ana"}}} + for _, c := range []struct { + email string + admin bool + owns []string + want []string + not string + }{ + {"a@x.io", true, nil, []string{"an admin always sees real values"}, "masks"}, + {"p@x.io", false, nil, []string{"not masked"}, "masks"}, + {"m@x.io", false, []string{"mine"}, []string{"masks of marked tables", "branches it owns (mine)", "unmask on ana, dev", "audited once the audit log is on"}, "no masking"}, + } { + got := unmaskedText(in, c.email, c.admin, c.owns) + for _, w := range c.want { + if !strings.Contains(got, w) { + t.Errorf("%s: %q lacks %q", c.email, got, w) + } + } + if strings.Contains(got, c.not) { + t.Errorf("%s: %q has %q", c.email, got, c.not) + } + } + in.auditOn = true + if got := unmaskedText(in, "m@x.io", false, nil); strings.Contains(got, "once") || strings.Contains(got, "owns") { + t.Errorf("audit on, owning nothing: %q", got) + } + in.maskOn = false + if got := unmaskedText(in, "m@x.io", false, nil); !strings.Contains(got, "no masking is in place") { + t.Errorf("masking not set up: %q", got) + } +} diff --git a/enterprise/masking/masking.go b/enterprise/masking/masking.go index a4cb5c4..d6076c3 100644 --- a/enterprise/masking/masking.go +++ b/enterprise/masking/masking.go @@ -15,7 +15,7 @@ import ( func init() { branch.SetMasker(&branch.Masker{Setup: setup, SetRule: setRule, DropRule: dropRule, MaskTables: maskTables, - Rotate: rotate, UseDomain: useDomain}) + Rotate: rotate, UseDomain: useDomain, AuditReads: auditReads}) } // setup makes the install's default key if it has none, then the branch's @@ -156,7 +156,9 @@ func remask(b, schema, table, actor string, all []branch.MaskRule) error { } } if len(m.View) > 0 { - if err := branch.RunMaskSQL(b, actor, m.View, false); err != nil { + // The view's grants name bb_audit_read, which a branch set up before + // step 3 may not have. + if err := branch.RunMaskSQL(b, actor, append(lookupSQL()[:1], m.View...), false); err != nil { return fmt.Errorf("making %s.%s's view: %w", schema, table, err) } } @@ -229,3 +231,28 @@ func maskTables(b string) ([]string, error) { err = json.Unmarshal([]byte(strings.TrimSpace(out)), &names) return names, err } + +// auditReads gives bb_audit_read SELECT on a branch's tables with rules, so an +// unmask holder's reads of them are logged: tables masked since step 3 have it +// from their view, older ones get it here. The lookup is made too, if the +// branch was set up before it existed. +func auditReads(b string) error { + if !hasSetup(b) { + return fmt.Errorf("%w: masking is not set up on %s: %s mask setup --branch %s", branch.ErrInvalidRequest, b, brand.CLI, b) + } + if ok, err := branch.RelationExists(b, branch.MaskSchema+".lookups"); err != nil { + return err + } else if !ok { + if err := ensureKeysTable(b); err != nil { + return err + } + if err := branch.RunMaskSQL(b, "system", lookupSQL(), false); err != nil { + return fmt.Errorf("making the lookup on %s: %w", b, err) + } + } + rules, err := branch.MaskRulesOn(b) + if err != nil { + return err + } + return branch.RunMaskSQL(b, "system", auditReadsSQL(tablesOf(rules)), false) +} diff --git a/enterprise/masking/sql.go b/enterprise/masking/sql.go index 430123e..bcd5b07 100644 --- a/enterprise/masking/sql.go +++ b/enterprise/masking/sql.go @@ -413,6 +413,8 @@ func viewSQL(nm names, cols []branch.MaskColumn, own map[string]branch.MaskRule, fmt.Sprintf("CREATE %sVIEW %s AS SELECT %s FROM %s", map[bool]string{true: "OR REPLACE "}[replace], nm.view, strings.Join(sel, ", "), from), "ALTER VIEW " + nm.view + " OWNER TO " + branch.MaskerRole, "GRANT SELECT ON " + nm.base + " TO " + branch.MaskerRole, + // An unmask holder's reads of the table are logged (step 3). + "GRANT SELECT ON " + nm.base + " TO " + branch.AuditReadRole, "GRANT SELECT ON " + nm.view + " TO " + branch.MaskedRole, } } @@ -429,7 +431,7 @@ const maskTablesDDL = `CREATE TABLE IF NOT EXISTS bb.mask_tables ( // (bb_identity 0.2), the roles, the schema, the rules table, the fake values, // and a masked reader's rights on every table that is not masked. func setupSQL() []string { - return []string{ + return append([]string{ `DO $bbsetup$ BEGIN IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'bb_identity') THEN EXECUTE 'ALTER EXTENSION bb_identity UPDATE'; @@ -483,5 +485,73 @@ BEGIN END IF; END LOOP; END $bbgrant$`, + }, lookupSQL()...) +} + +// lookupSQL is the lookup (step 3; design, section 10): a masked reader finds +// a person by an exact value without unmasking. SECURITY DEFINER, as +// bb_masker, which reads the rules, the tables and the hash, and nobody logs +// in as. It looks only in a column with a rule of its own, returns the rows +// masked, as the view has them, allows a number of calls a minute for each +// account, and records each call without the value. A value that is not of +// the column's type is refused without being repeated, so it reaches no log. +func lookupSQL() []string { + return []string{ + `DO $bbaudit$ BEGIN + IF to_regrole('` + branch.AuditReadRole + `') IS NULL THEN EXECUTE 'CREATE ROLE ` + branch.AuditReadRole + ` NOLOGIN'; END IF; +END $bbaudit$`, + "GRANT USAGE ON SCHEMA bb TO " + branch.MaskerRole, + "GRANT SELECT ON bb.mask_rules, bb.mask_tables TO " + branch.MaskerRole, + `CREATE TABLE IF NOT EXISTS ` + branch.MaskSchema + `.lookups (at timestamptz NOT NULL DEFAULT clock_timestamp(), + who text NOT NULL, rel text NOT NULL, col text NOT NULL, found int NOT NULL)`, + "ALTER TABLE " + branch.MaskSchema + ".lookups OWNER TO " + branch.MaskerRole, + "REVOKE ALL ON " + branch.MaskSchema + ".lookups FROM PUBLIC", + `CREATE INDEX IF NOT EXISTS lookups_who_at ON ` + branch.MaskSchema + `.lookups (who, at)`, + fmt.Sprintf(`CREATE OR REPLACE FUNCTION %[1]s.find(tbl text, col text, val text) RETURNS SETOF jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $bbfind$ +DECLARE s text; t text; v text; r text; d text; typ text; k text; rel regclass; n int; ok boolean; +BEGIN + IF position('.' in tbl) > 0 THEN s := split_part(tbl, '.', 1); t := split_part(tbl, '.', 2); ELSE s := 'public'; t := tbl; END IF; + SELECT rule INTO r FROM bb.mask_rules WHERE schema_name = s AND table_name = t AND column_name = col AND json_path = ''; + IF r IS NULL THEN + RAISE EXCEPTION 'a lookup is by a column with a masking rule of its own, and %%.%%.%% has none', s, t, col USING ERRCODE = '42704'; + END IF; + IF (SELECT count(*) FROM %[1]s.lookups WHERE who = session_user AND at > clock_timestamp() - interval '1 minute') >= %[2]d THEN + RAISE EXCEPTION 'too many lookups: %[2]d a minute for each account' USING ERRCODE = '53400'; + END IF; + rel := format('%%I.%%I', s, t)::regclass; + v := CASE WHEN s = 'public' THEN t ELSE s || '__' || t END; + SELECT format_type(a.atttypid, a.atttypmod) INTO typ FROM pg_attribute a WHERE a.attrelid = rel AND a.attname = col; + SELECT a.attname INTO k FROM pg_index i JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = i.indkey[0] + WHERE i.indrelid = rel AND i.indisprimary; + BEGIN + EXECUTE format('SELECT CAST($1 AS %%s) IS NOT NULL', typ) INTO ok USING val; + EXCEPTION WHEN others THEN + RAISE EXCEPTION 'the value is not a %%', typ USING ERRCODE = '22000'; + END; + IF r = 'hash' THEN + -- The value hashed here, under the table's domain, against the kept hashes. + SELECT coalesce((SELECT key_domain FROM bb.mask_tables WHERE schema_name = s AND table_name = t), 'default') INTO d; + RETURN QUERY EXECUTE format('SELECT to_jsonb(m) FROM %[1]s.%%I m WHERE m.%%I = bb_f1.bbid_mask_hmac($2, $1)', v, col) USING val, d; + ELSE + RETURN QUERY EXECUTE format('SELECT to_jsonb(m) FROM %[1]s.%%I m WHERE m.%%I IN (SELECT n.%%I FROM %%I.%%I n WHERE n.%%I = CAST($1 AS %%s))', + v, k, k, s, t, col, typ) USING val; + END IF; + GET DIAGNOSTICS n = ROW_COUNT; + INSERT INTO %[1]s.lookups (who, rel, col, found) VALUES (session_user, s || '.' || t, col, n); +END $bbfind$`, branch.MaskSchema, branch.LookupsPerMinute), + "ALTER FUNCTION " + branch.MaskSchema + ".find(text, text, text) OWNER TO " + branch.MaskerRole, + "REVOKE EXECUTE ON FUNCTION " + branch.MaskSchema + ".find(text, text, text) FROM PUBLIC", + "GRANT EXECUTE ON FUNCTION " + branch.MaskSchema + ".find(text, text, text) TO " + branch.MaskedRole, + } +} + +// auditReadsSQL gives bb_audit_read SELECT on the tables with rules, for a +// branch whose masks were made before step 3. +func auditReadsSQL(tables [][2]string) []string { + out := lookupSQL()[:1] + for _, t := range tables { + out = append(out, "GRANT SELECT ON "+ident(t[0])+"."+ident(t[1])+" TO "+branch.AuditReadRole) } + return out } diff --git a/internal/auth/audit.go b/internal/auth/audit.go index 52b99b9..04203b0 100644 --- a/internal/auth/audit.go +++ b/internal/auth/audit.go @@ -134,6 +134,7 @@ const ( EvMaskRule = "mask.rule" EvMaskAccount = "mask.account" EvMaskKey = "mask.key" + EvMaskUnmask = "mask.unmask" // The databases' audit log (F6) turned on or off: who decided. Whoever // can stop the record of who did what is someone an auditor asks about. EvAuditLogOn = "audit-log.on" diff --git a/internal/branch/branch.go b/internal/branch/branch.go index 64b342b..dcb15e7 100644 --- a/internal/branch/branch.go +++ b/internal/branch/branch.go @@ -620,9 +620,13 @@ func EnsureUserRole(branchName, email string) error { // Masked or not is part of the key: an admin masking an account changes // what its role must be, in every process that makes roles (mask.go). maskedHere := accountMasked(branchName, email) + audited := readsAudited(branchName, email) state := "plain" - if maskedHere { + switch { + case maskedHere: state = "masked" + case audited: + state = "unmasked" // an unmask holder: real values, reads audited } k := branchName + "\x00" + email + "\x00" + state if _, ok := ensuredRoles.Load(k); ok { @@ -640,8 +644,8 @@ func EnsureUserRole(branchName, email string) error { sql = maskedRoleSQL(email, UserRolePassword(email)) case MaskOn(): // Unmasked, or exempt here: a role once a masked reader's is made an - // ordinary account's again. - sql += "\n" + unmaskedRoleSQL(email) + // ordinary account's again, its reads audited if it holds unmask here. + sql += "\n" + unmaskedRoleSQL(email) + "\n" + auditedReadsSQL(email, audited) } if err := provisionRole(branchName, sql); err != nil { return err @@ -880,6 +884,7 @@ func Delete(name string) error { } forgetBranchRoles(name) forgetCatalogScan(name) + forgetBranchUnmasks(name) removeContainer(container(name)) return activeStorage().destroy(name) } diff --git a/internal/branch/mask.go b/internal/branch/mask.go index e44b21f..ce7e39a 100644 --- a/internal/branch/mask.go +++ b/internal/branch/mask.go @@ -199,7 +199,15 @@ func accountMasked(branchName, email string) bool { if _, ok := loadMasked()[strings.ToLower(email)]; !ok { return false } - return !maskExempt(branchName, email) + // Unmask on this branch (step 3): real values, its reads audited. + return !hasUnmask(branchName, email) && !maskExempt(branchName, email) +} + +// readsAudited reports whether an account's reads on a branch are audited as +// an unmask holder's: masked, and given unmask there. +func readsAudited(branchName, email string) bool { + _, ok := loadMasked()[strings.ToLower(email)] + return ok && hasUnmask(branchName, email) } // maskExempt: an install admin, or the branch's owner. A variable so the @@ -283,6 +291,7 @@ func ApplyAccountMask(branchName, email string) error { if accountMasked(branchName, email) { sql = maskedRoleSQL(email, UserRolePassword(email)) } + sql += "\n" + auditedReadsSQL(email, readsAudited(branchName, email)) if err := provisionRole(branchName, sql+"\n"+endSessionsSQL(email)); err != nil { return err } @@ -291,7 +300,7 @@ func ApplyAccountMask(branchName, email string) error { } func forgetUserRole(branchName, email string) { - for _, m := range []string{"masked", "plain"} { + for _, m := range []string{"masked", "plain", "unmasked"} { ensuredRoles.Delete(branchName + "\x00" + email + "\x00" + m) } ensuredRoles.Delete(branchName + "\x00" + email) @@ -437,6 +446,9 @@ type Masker struct { // and a table's masks put under a domain. Rotate func(domain, actor string, say func(string)) error UseDomain func(branch, schema, table, domain, actor string) error + // Step 3 (unmask.go): bb_audit_read given SELECT on a branch's masked + // tables, so an unmask holder's reads of them are logged. + AuditReads func(branch string) error } var masker *Masker @@ -522,6 +534,12 @@ func MaskAccount(email, actor string, off bool) error { if err := SaveMaskedAccounts(list); err != nil { return err } + if off { + // Unmask is a masked account's right: an account unmasked has none. + if err := changeUnmasks(func(g UnmaskGrant) bool { return !strings.EqualFold(g.Email, email) }, nil); err != nil { + return err + } + } names, err := RunningBranches() if err != nil { return err @@ -547,6 +565,10 @@ type MaskStatus struct { Ready []string `json:"ready"` // tables whose masks are made Accounts []MaskedAccount `json:"accounts"` // install-wide Writes bool `json:"writes"` // whether this install sets rules (Enterprise) + Unmasks []UnmaskGrant `json:"unmasks"` // on this branch + Lookups []Lookup `json:"lookups"` // this branch's latest + // AuditOn: whether unmask holders' reads are being logged (the audit log, F6). + AuditOn bool `json:"auditOn"` } // MaskingStatus reads it. Free. @@ -555,7 +577,12 @@ func MaskingStatus(branchName string) (MaskStatus, error) { branchName = "main" } st := MaskStatus{On: MaskOn(), Branch: branchName, Pending: []string{}, Rules: []MaskRule{}, Ready: []string{}, - Accounts: MaskedAccounts(), Writes: requireMasker() == nil} + Accounts: MaskedAccounts(), Writes: requireMasker() == nil, Unmasks: []UnmaskGrant{}, Lookups: []Lookup{}, AuditOn: AuditOn()} + for _, g := range UnmaskGrants() { + if g.Branch == branchName { + st.Unmasks = append(st.Unmasks, g) + } + } if p, err := MaskPending(); err == nil && p != nil { st.Pending = p } @@ -567,6 +594,9 @@ func MaskingStatus(branchName string) (MaskStatus, error) { return st, err } st.Rules = rules + if ls, err := Lookups(branchName, 50); err == nil { + st.Lookups = ls + } if masker != nil && masker.MaskTables != nil { if t, err := masker.MaskTables(branchName); err == nil && t != nil { st.Ready = t diff --git a/internal/branch/unmask.go b/internal/branch/unmask.go new file mode 100644 index 0000000..b0d34c2 --- /dev/null +++ b/internal/branch/unmask.go @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "sync" + "time" + + "github.com/thefoxbyte/foxbyte/internal/brand" +) + +// Unmask and the lookup (F5, Gate 1 step 3; design, section 10). +// +// Unmask is a right on a branch, given by an admin to a masked account: there +// it sees real values, as an owner does, and every read it makes of a marked +// table is audited: its role's pgaudit.role is bb_audit_read, which holds +// SELECT on every masked table (and on every table marked for audit), so +// pgAudit logs those reads, and only that account's. The setting is a +// superuser's, so the account cannot undo it. Reads are logged while the +// audit log is on (F6); the right holds either way, and says so. +// +// The lookup finds a person by an exact value without unmasking: +// bb_mask.find(table, column, value) is SECURITY DEFINER, returns the +// matching rows masked, allows 30 calls a minute for each account, and +// records each call, who, when, which column and how many rows, never the +// value (enterprise/masking). + +// UnmaskGrant is one account's unmask right on one branch. +type UnmaskGrant struct { + Email string `json:"email"` + Branch string `json:"branch"` + By string `json:"by"` + At string `json:"at"` +} + +var unmaskPath = func() string { return brand.StatePath("unmask-grants.json") } + +// Read again when it changes, as the masked accounts' file is: every process +// that makes roles reads it. +var unmasks struct { + sync.Mutex + mod time.Time + size int64 + byKey map[string]UnmaskGrant // branch \x00 email +} + +func unmaskKey(branchName, email string) string { + return branchName + "\x00" + strings.ToLower(email) +} + +func loadUnmasks() map[string]UnmaskGrant { + unmasks.Lock() + defer unmasks.Unlock() + st, err := os.Stat(unmaskPath()) + if err != nil { + unmasks.byKey, unmasks.mod, unmasks.size = nil, time.Time{}, 0 + return nil + } + if unmasks.byKey != nil && st.ModTime().Equal(unmasks.mod) && st.Size() == unmasks.size { + return unmasks.byKey + } + m := map[string]UnmaskGrant{} + if b, err := os.ReadFile(unmaskPath()); err == nil { + var list []UnmaskGrant + if json.Unmarshal(b, &list) == nil { + for _, g := range list { + m[unmaskKey(g.Branch, g.Email)] = g + } + } + } + unmasks.byKey, unmasks.mod, unmasks.size = m, st.ModTime(), st.Size() + return m +} + +// UnmaskGrants lists the unmask rights, by account then branch. Free. +func UnmaskGrants() []UnmaskGrant { + m := loadUnmasks() + out := make([]UnmaskGrant, 0, len(m)) + for _, g := range m { + out = append(out, g) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Email != out[j].Email { + return out[i].Email < out[j].Email + } + return out[i].Branch < out[j].Branch + }) + return out +} + +// UnmaskGrantsOf lists the branches an account may unmask on. +func UnmaskGrantsOf(email string) []string { + var out []string + for _, g := range UnmaskGrants() { + if strings.EqualFold(g.Email, email) { + out = append(out, g.Branch) + } + } + return out +} + +func saveUnmasks(list []UnmaskGrant) error { + if list == nil { + list = []UnmaskGrant{} // [], not null, for whoever reads the file + } + b, err := json.MarshalIndent(list, "", " ") + if err != nil { + return err + } + tmp := unmaskPath() + ".tmp" + if err := os.WriteFile(tmp, b, 0o600); err != nil { + return err + } + return os.Rename(tmp, unmaskPath()) +} + +// hasUnmask reports whether an account holds unmask on a branch. +func hasUnmask(branchName, email string) bool { + _, ok := loadUnmasks()[unmaskKey(branchName, email)] + return ok +} + +// changeUnmasks keeps the grants that keep says to. +func changeUnmasks(keep func(UnmaskGrant) bool, add *UnmaskGrant) error { + var list []UnmaskGrant + for _, g := range loadUnmasks() { + if keep(g) { + list = append(list, g) + } + } + if add != nil { + list = append(list, *add) + } + sort.Slice(list, func(i, j int) bool { + return unmaskKey(list[i].Branch, list[i].Email) < unmaskKey(list[j].Branch, list[j].Email) + }) + return saveUnmasks(list) +} + +// forgetBranchUnmasks drops the unmask rights on a branch being deleted: a +// branch made later with its name is another branch. +func forgetBranchUnmasks(name string) { + if len(loadUnmasks()) == 0 { + return + } + _ = changeUnmasks(func(g UnmaskGrant) bool { return g.Branch != name }, nil) +} + +// auditedReadsSQL sets, or takes off, an account's role's audit role: an +// unmask holder's reads of marked tables are logged through bb_audit_read. +func auditedReadsSQL(email string, on bool) string { + set := "RESET pgaudit.role" + if on { + set = "SET pgaudit.role = " + AuditReadRole + } + return fmt.Sprintf(`DO $do$ +DECLARE r text := %s; +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN + EXECUTE format('ALTER ROLE %%I %s', r); + END IF; +END $do$;`, quoteLiteral(email), set) +} + +// Unmask gives an account unmask on a branch, or with off takes it. Only a +// masked account is given it: any other sees real values already. Its role +// there changes at once, its open sessions there end, and its reads of +// marked tables are audited from then on. Enterprise. +func Unmask(email, branchName, actor string, off bool) error { + if err := requireMasker(); err != nil { + return err + } + email = strings.ToLower(strings.TrimSpace(email)) + if email == "" || branchName == "" { + return fmt.Errorf("%w: unmask is an account's right on a branch: both are needed", ErrInvalidRequest) + } + if err := checkName(branchName); err != nil { + return err + } + if !off { + if _, ok := loadMasked()[email]; !ok { + return fmt.Errorf("%w: %s is not masked, so it sees real values already: unmask is for a masked account", ErrInvalidRequest, email) + } + if !Exists(branchName) { + return fmt.Errorf("%w: there is no branch %q", ErrInvalidRequest, branchName) + } + } + var add *UnmaskGrant + if !off { + add = &UnmaskGrant{Email: email, Branch: branchName, By: actor, At: time.Now().UTC().Format(time.RFC3339)} + } + if err := changeUnmasks(func(g UnmaskGrant) bool { return unmaskKey(g.Branch, g.Email) != unmaskKey(branchName, email) }, add); err != nil { + return err + } + if ContainerState(branchName) != "running" { + forgetUserRole(branchName, email) + return nil + } + if !off && masker.AuditReads != nil { + // Every masked table's reads by an unmask holder are logged. + if err := masker.AuditReads(branchName); err != nil { + return err + } + } + return ApplyAccountMask(branchName, email) +} + +// Lookup is one call of the lookup, as recorded: never the value. +type Lookup struct { + At string `json:"at"` + Who string `json:"who"` + Table string `json:"table"` + Column string `json:"column"` + Found int `json:"found"` +} + +// Lookups reads a running branch's lookup record, newest first. Free. +func Lookups(name string, limit int) ([]Lookup, error) { + if err := checkName(name); err != nil { + return nil, err + } + if limit <= 0 || limit > 10000 { + limit = 100 + } + if ok, err := RelationExists(name, MaskSchema+".lookups"); err != nil || !ok { + return []Lookup{}, err + } + out, err := QuerySQL(name, fmt.Sprintf(`SELECT coalesce(json_agg(json_build_object('at', at, 'who', who, 'table', rel, 'column', col, 'found', found) + ORDER BY at DESC), '[]'::json) FROM (SELECT * FROM %s.lookups ORDER BY at DESC LIMIT %d) l`, MaskSchema, limit)) + if err != nil { + return nil, fmt.Errorf("reading the lookups on %s: %w (%s)", name, err, strings.TrimSpace(out)) + } + var ls []Lookup + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &ls); err != nil { + return nil, err + } + if ls == nil { + ls = []Lookup{} + } + return ls, nil +} + +// LookupsPerMinute is how many lookups an account may make a minute. +const LookupsPerMinute = 30 diff --git a/internal/branch/unmask_test.go b/internal/branch/unmask_test.go new file mode 100644 index 0000000..14d9383 --- /dev/null +++ b/internal/branch/unmask_test.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func tempMaskFiles(t *testing.T) { + t.Helper() + dir := t.TempDir() + oldM, oldU, oldE := maskedPath, unmaskPath, maskExempt + t.Cleanup(func() { + maskedPath, unmaskPath, maskExempt = oldM, oldU, oldE + masked.byKey, unmasks.byKey = nil, nil + }) + maskedPath = func() string { return filepath.Join(dir, "masked-accounts.json") } + unmaskPath = func() string { return filepath.Join(dir, "unmask-grants.json") } + maskExempt = func(string, string) bool { return false } +} + +// Unmask on a branch: real values there, masks elsewhere, its reads audited +// there only; a branch's grants go with it. +func TestUnmaskOnABranch(t *testing.T) { + tempMaskFiles(t) + if err := SaveMaskedAccounts([]MaskedAccount{{Email: "m@x.io"}}); err != nil { + t.Fatal(err) + } + if !accountMasked("dev", "m@x.io") || readsAudited("dev", "m@x.io") { + t.Fatal("masked, with no unmask") + } + if err := changeUnmasks(func(UnmaskGrant) bool { return true }, &UnmaskGrant{Email: "m@x.io", Branch: "dev", By: "cli:a"}); err != nil { + t.Fatal(err) + } + if accountMasked("dev", "M@X.IO") || !readsAudited("dev", "m@x.io") { + t.Error("unmask on dev: real values there, reads audited") + } + if !accountMasked("main", "m@x.io") || readsAudited("main", "m@x.io") { + t.Error("unmask on dev reaches main") + } + if got := strings.Join(UnmaskGrantsOf("m@x.io"), " "); got != "dev" { + t.Errorf("the grants: %q", got) + } + if st, err := os.Stat(unmaskPath()); err != nil || os.PathSeparator == '/' && st.Mode().Perm() != 0o600 { + t.Errorf("the file: %v %v", st, err) + } + forgetBranchUnmasks("dev") + if b, _ := os.ReadFile(unmaskPath()); strings.TrimSpace(string(b)) != "[]" { + t.Errorf("no grants left is [], not %q", b) + } + if hasUnmask("dev", "m@x.io") || !accountMasked("dev", "m@x.io") { + t.Error("a deleted branch's grant stays, and would reach a branch made later with its name") + } + // An account no longer masked is not audited as an unmask holder. + _ = changeUnmasks(func(UnmaskGrant) bool { return true }, &UnmaskGrant{Email: "m@x.io", Branch: "dev"}) + _ = SaveMaskedAccounts(nil) + if readsAudited("dev", "m@x.io") || accountMasked("dev", "m@x.io") { + t.Error("an unmasked account") + } +} + +// The audit role is set by a superuser's ALTER ROLE, so the account cannot +// undo it, and taken off with RESET; a role not there yet is left alone. +func TestAuditedReadsSQL(t *testing.T) { + on, off := auditedReadsSQL("o'k@x.io", true), auditedReadsSQL("o'k@x.io", false) + if !strings.Contains(on, "SET pgaudit.role = bb_audit_read") || !strings.Contains(off, "RESET pgaudit.role") { + t.Errorf("on:\n%s\noff:\n%s", on, off) + } + for _, s := range []string{on, off} { + if !strings.Contains(s, "'o''k@x.io'") || !strings.Contains(s, "pg_roles WHERE rolname = r") { + t.Errorf("the role, quoted and looked up by name:\n%s", s) + } + } +} + +// Without Enterprise, unmask is refused and nothing is written. +func TestUnmaskNotLicensed(t *testing.T) { + tempMaskFiles(t) + old := masker + t.Cleanup(func() { masker = old }) + masker = nil + if err := Unmask("m@x.io", "dev", "cli:a", false); !errors.Is(err, ErrNotLicensed) { + t.Fatalf("unmask: %v", err) + } + if _, err := os.Stat(unmaskPath()); err == nil { + t.Error("a grant was written without the licence") + } +} diff --git a/internal/controlplane/masking.go b/internal/controlplane/masking.go index 9294e4a..ac9d252 100644 --- a/internal/controlplane/masking.go +++ b/internal/controlplane/masking.go @@ -23,6 +23,8 @@ import ( // POST /api/masking/rule {"branch", "schema", "table", "column", "path", "rule", "off", "apply"} // a rule's cost, or with apply sets it; off takes it off (Enterprise) // POST /api/masking/account {"email", "off"} masks an account, or unmasks it (Enterprise) +// POST /api/masking/unmask {"email", "branch", "off"} gives a masked account unmask on a branch, or takes it (Enterprise) +// GET /api/masking/lookups?branch=main who looked a person up, never the value: free // GET /api/masking/keys the key domains and the tables using each: free // POST /api/masking/key {"action": "create"|"rotate"|"drop"|"use", "domain", "branch", "schema", "table", "apply"} // what a key change does, or with apply makes it (Enterprise) @@ -182,6 +184,40 @@ func registerMasking(mux *http.ServeMux) { }) registerMaskKeys(mux, read, record) + + mux.HandleFunc("POST /api/masking/unmask", func(w http.ResponseWriter, r *http.Request) { + b, actor, ok := read(w, r) + if !ok { + return + } + email := strings.ToLower(strings.TrimSpace(b.Email)) + if err := branch.Unmask(email, b.Branch, actor, b.Off); err != nil { + maskFail(w, err) + return + } + what := "given unmask on " + b.Branch + if b.Off { + what = "no longer unmasked on " + b.Branch + } + record(r, actor, auth.EvMaskUnmask, b.Branch, email+" "+what) + writeJSON(w, 200, map[string]any{"email": email, "branch": b.Branch, "unmasked": !b.Off, "auditOn": branch.AuditOn()}) + }) + mux.HandleFunc("GET /api/masking/lookups", func(w http.ResponseWriter, r *http.Request) { + if !isAdmin(r) { + writeErr(w, 403, errors.New("only an admin may read the lookups")) + return + } + b := r.URL.Query().Get("branch") + if b == "" { + b = "main" + } + ls, err := branch.Lookups(b, 500) + if err != nil { + maskFail(w, err) + return + } + writeJSON(w, 200, map[string]any{"branch": b, "lookups": ls}) + }) } // registerMaskKeys is step 2's: the keys. Admins only, as the rest. diff --git a/internal/controlplane/openapi.yaml b/internal/controlplane/openapi.yaml index 428c656..02f261e 100644 --- a/internal/controlplane/openapi.yaml +++ b/internal/controlplane/openapi.yaml @@ -2297,6 +2297,37 @@ paths: "400": { description: No email, or an admin. } "403": { description: Not an admin, or not licensed for masking. } "401": { $ref: "#/components/responses/Unauthorized" } + /api/masking/unmask: + post: + tags: [audit-log] + summary: Give a masked account unmask on a branch, or take it (admins only; Enterprise). + description: >- + There the account sees real values, as the branch's owner does, and every read it makes of a marked table + is audited: its role's pgaudit.role becomes bb_audit_read, which only a superuser can change. Its reads are + logged while the audit log is on. Only a masked account is given it. Its open sessions on the branch end. + Listed in the access review, and in the security log. + requestBody: + content: + application/json: + schema: { type: object, properties: { email: { type: string }, branch: { type: string }, off: { type: boolean } } } + responses: + "200": { description: '{"email", "branch", "unmasked", "auditOn"}', content: { application/json: { schema: { type: object } } } } + "400": { description: No email or branch, a branch that is not there, or an account that is not masked. } + "403": { description: Not an admin, or not licensed for masking. } + "401": { $ref: "#/components/responses/Unauthorized" } + /api/masking/lookups: + get: + tags: [audit-log] + summary: Who looked a person up on a branch, when, by which column and how many rows were found (admins only). + description: >- + A masked account looks a person up by an exact value with bb_mask.find(table, column, value), which returns + the rows masked, 30 a minute for each account. Each call is recorded here; the value never is. + parameters: + - { name: branch, in: query, schema: { type: string, default: main } } + responses: + "200": { description: '{"branch", "lookups": [{"at", "who", "table", "column", "found"}]}', content: { application/json: { schema: { type: object } } } } + "403": { description: Not an admin. } + "401": { $ref: "#/components/responses/Unauthorized" } /api/masking/keys: get: tags: [audit-log] diff --git a/scripts/integration_test.sh b/scripts/integration_test.sh index da5fc6c..bfab3b2 100644 --- a/scripts/integration_test.sh +++ b/scripts/integration_test.sh @@ -87,7 +87,7 @@ rm -rf "${HOME:?}/${BRAND_STATE_DIR:?}/anchors/_audit-log" rm -f "$HOME/$BRAND_STATE_DIR/sync-commit" # Masking's key and masked accounts (F5): §11ai starts from an install that # has never set masking up. -sudo rm -rf "${HOME:?}/${BRAND_STATE_DIR:?}/mask"; rm -f "${HOME:?}/${BRAND_STATE_DIR:?}/masked-accounts.json" "${HOME:?}/${BRAND_STATE_DIR:?}/mask-keys.json" +sudo rm -rf "${HOME:?}/${BRAND_STATE_DIR:?}/mask"; rm -f "${HOME:?}/${BRAND_STATE_DIR:?}/masked-accounts.json" "${HOME:?}/${BRAND_STATE_DIR:?}/mask-keys.json" "${HOME:?}/${BRAND_STATE_DIR:?}/unmask-grants.json" # And the primary pointer, for the same reason: a run cut short after a failover # leaves 'main' pointing at the promoted standby. `stop` above has already # removed every container, so there is no promoted standby left to protect, and @@ -3354,7 +3354,7 @@ $S branch delete maskown >/dev/null 2>&1 for c in m_customers.phone m_customers.aadhaar_no m_orders.customer_email m_kyc.aadhaar_no; do $S mask rule "$c" --off >/dev/null 2>&1; done $S mask rule m_customers.prefs --path '$.contact.mobile' --off >/dev/null 2>&1 assert_eq "with every rule taken off, no mask is left behind: no view, shadow, trigger or rule" \ - "$(pg pg-main "SELECT (SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'bb_mask' AND c.relkind IN ('r','v')), (SELECT count(*) FROM pg_trigger WHERE tgname LIKE 'bb_mask%'), (SELECT count(*) FROM bb.mask_rules)")" "0|0|0" + "$(pg pg-main "SELECT (SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'bb_mask' AND c.relkind IN ('r','v') AND c.relname <> 'lookups'), (SELECT count(*) FROM pg_trigger WHERE tgname LIKE 'bb_mask%'), (SELECT count(*) FROM bb.mask_rules)")" "0|0|0" pg pg-main "SET bb.allow_destructive = on; DROP TABLE IF EXISTS m_customers, m_orders, m_kyc, m_plain CASCADE" >/dev/null $S user delete mask-analyst@foxbyte.dev >/dev/null 2>&1 @@ -3457,6 +3457,86 @@ pg pg-main "SET bb.allow_destructive = on; DROP TABLE IF EXISTS k_people, k_orde $S user delete key-analyst@foxbyte.dev >/dev/null 2>&1 rm -rf "${ROTD:?}" +echo "### 11ak. masking: unmask, the lookup, the access review, reads audited (F5 Gate 1, step 3)" +# Enterprise: a masked account given unmask on a branch sees real values +# there, and its reads of marked tables are logged by the audit log, its only; +# it cannot undo that. The lookup finds a person by an exact value, masked, 30 +# a minute, recorded without the value. The access review says who sees what. +# On main, set up by §11ai. +$S branch delete umother >/dev/null 2>&1 +pg pg-main "SET bb.allow_destructive = on; DROP TABLE IF EXISTS u_people CASCADE" >/dev/null +pg pg-main "CREATE TABLE u_people (id bigint PRIMARY KEY, email text, phone text, aadhaar_no text); + INSERT INTO u_people VALUES (1, 'asha.u@example.com', '+91 98765 43210', '234567890124'), (2, 'ravi.u@example.org', '9876543211', '456789012341')" >/dev/null +$S pii scan >/dev/null 2>&1 +UJ="$($S pii --json 2>/dev/null)" +ufid() { hjson "print([f['id'] for f in d['findings'] if f['table']=='u_people' and f['column']=='$1'][0])" <<<"$UJ"; } +for c in email phone aadhaar_no; do $S pii mark "$(ufid $c)" confirm >/dev/null 2>&1; $S mask rule "u_people.$c" --yes >/dev/null 2>&1 || bad "setting the rule on u_people.$c"; done +for u in unmask-analyst plain-person; do printf 'password123\n' | $S user create "$u@foxbyte.dev" >/dev/null 2>&1; done +UM="$($S apikey create unmask-analyst@foxbyte.dev t 2>/dev/null | grep -o 'key_[A-Za-z0-9_-]*')" +$S mask account unmask-analyst@foxbyte.dev --yes >/dev/null 2>&1 || bad "masking the analyst" +UROLE="SELECT coalesce(array_to_string(rolconfig, ' '), '') FROM pg_roles WHERE rolname = 'unmask-analyst@foxbyte.dev'" +unmasked() { $S access review --json 2>/dev/null | hjson "print([a['unmasked'] for a in d['accounts'] if a['email']=='$1'][0])"; } +assert_eq "the access review says who sees what: an admin real values, an account not masked everything, a masked one masks" \ + "$(unmasked test@foxbyte.dev | grep -c 'an admin always sees real values')|$(unmasked plain-person@foxbyte.dev | grep -c 'not masked')|$(unmasked unmask-analyst@foxbyte.dev | grep -c '^masks of marked tables$')" "1|1|1" +assert_eq "unmask is for a masked account: one that is not is refused, saying why; so is a branch that is not there" \ + "$($S mask unmask plain-person@foxbyte.dev --branch main --yes 2>&1 | grep -c 'is not masked, so it sees real values already')|$($S mask unmask unmask-analyst@foxbyte.dev --branch nosuchbranch --yes 2>&1 | grep -c 'there is no branch')" "1|1" +PRE="$($S mask unmask unmask-analyst@foxbyte.dev --branch main 2>&1)" +assert_eq "without --yes it says what changes, that reads are audited and the audit log is off, and changes nothing" \ + "$(grep -c 'Every read they make there of a marked table is audited' <<<"$PRE")|$(grep -c 'The audit log is off' <<<"$PRE")|$(gwm "$UM" main "SELECT length(email) FROM u_people WHERE id = 1")" "1|1|64" +$S audit-log setup --yes >/dev/null 2>&1; main_up +$S mask unmask unmask-analyst@foxbyte.dev --branch main --yes >/dev/null 2>&1 || bad "unmasking the analyst on main" +assert_eq "with unmask on main, the masked account reads real values there, and its role's audit role is bb_audit_read" \ + "$(gwm "$UM" main "SELECT email, phone FROM u_people WHERE id = 1")|$(pg pg-main "$UROLE" | grep -o 'pgaudit.role=bb_audit_read')" "asha.u@example.com|+91 98765 43210|pgaudit.role=bb_audit_read" +gwm "$UM" main "SELECT count(*) FROM u_people" >/dev/null; sleep 1.5 +assert_eq "…its reads of a marked table are logged (both), under its login, with nothing the table holds" \ + "$(auditlog main | grep '"user":"unmask-analyst@foxbyte.dev"' | grep -c 'AUDIT: OBJECT,[0-9]*,[0-9]*,READ,SELECT,TABLE,public.u_people,')|$(auditlog main | grep -c 'asha.u@example.com')" "2|0" +assert_eq "…and it cannot undo that: the setting is a superuser's" \ + "$(gwm "$UM" main "SET pgaudit.role = 'nobody'" | grep -c 'permission denied')|$(gwm "$UM" main "ALTER ROLE SESSION_USER RESET pgaudit.role" | grep -c 'permission denied')" "1|1" +$S branch create umother >/dev/null 2>&1 +# The Gateway lets a non-admin reach main and the branches it owns only, so +# another branch is read from the record, not by connecting. +assert_eq "unmask is on one branch: another branch lists nobody" \ + "$($S mask --branch umother --json | hjson "print(len(d['unmasks']))")|$($S mask --json | hjson "print(len(d['unmasks']))")" "0|1" +assert_eq "the access review lists the unmask, audited" \ + "$(unmasked unmask-analyst@foxbyte.dev)" "masks of marked tables; unmask on main, its reads of marked tables audited" +assert_eq "status lists it, and the API: an admin reads lookups; the masked account may not unmask itself" \ + "$($S mask --json | hjson "print([g['email'] for g in d['unmasks']], d['auditOn'])")|$(code "$KEY" https://localhost:8080/api/masking/lookups?branch=main)|$(code "$UM" -X POST -H 'Content-Type: application/json' -d '{"email":"unmask-analyst@foxbyte.dev","branch":"umother"}' https://localhost:8080/api/masking/unmask)" \ + "['unmask-analyst@foxbyte.dev'] True|200|403" +$S mask unmask unmask-analyst@foxbyte.dev --branch main --off --yes >/dev/null 2>&1 || bad "taking unmask off" +assert_eq "unmask taken off: masks again, and the audit role is gone from the role" \ + "$(gwm "$UM" main "SELECT length(email) FROM u_people WHERE id = 1")|$(pg pg-main "$UROLE" | grep -c 'pgaudit.role')" "64|0" + +# The lookup: by an exact value, the rows masked; nothing outside a rule. +assert_eq "the lookup finds a person by an exact value and returns the row masked: by a hash, and by a partial mask" \ + "$(gwm "$UM" main "SELECT d->>'id', length(d->>'email'), d->>'phone' FROM bb_mask.find('u_people', 'email', 'asha.u@example.com') d")|$(gwm "$UM" main "SELECT d->>'id', d->>'aadhaar_no' FROM bb_mask.find('public.u_people', 'aadhaar_no', '456789012341') d")|$(gwm "$UM" main "SELECT count(*) FROM bb_mask.find('u_people', 'email', 'nobody@example.com')")" \ + "1|64|XXXXXXXXXXX3210|2|XXXXXXXX2341|0" +assert_eq "…only in a column with a rule of its own, and never a table without one" \ + "$(gwm "$UM" main "SELECT * FROM bb_mask.find('u_people', 'id', '1')" | grep -c 'has none')|$(gwm "$UM" main "SELECT * FROM bb_mask.find('pg_authid', 'rolpassword', 'x')" | grep -c 'has none')|$(gwm "$KEY" main "SELECT * FROM bb_mask.find('u_people', 'email', 'x')" | grep -c 'permission denied')" "1|1|1" +LK="$($S mask lookups --json)" +assert_eq "each lookup is recorded: who, which column, how many found; never the value; the masked account cannot read the record" \ + "$(hjson "print(len(d), sorted(set(l['who'] for l in d)), sorted(set(l['column'] for l in d)), sum(l['found'] for l in d))" <<<"$LK")|$(grep -cE 'asha|456789' <<<"$LK")|$(gwm "$UM" main "SELECT count(*) FROM bb_mask.lookups" | grep -c 'permission denied')" \ + "3 ['unmask-analyst@foxbyte.dev'] ['aadhaar_no', 'email'] 2|0|1" +for _ in $(seq 27); do gwm "$UM" main "SELECT count(*) FROM bb_mask.find('u_people', 'email', 'x@example.com')" >/dev/null; done +assert_eq "30 lookups a minute for each account, then refused" \ + "$(gwm "$UM" main "SELECT count(*) FROM bb_mask.find('u_people', 'email', 'asha.u@example.com')" | grep -c 'too many lookups')" "1" +assert_eq "no value looked up is in the database's log" "$(sudo docker logs pg-main 2>&1 | grep -cE 'asha.u@example.com|456789012341|x@example.com')" "0" + +# A branch's unmask goes with it; an account unmasked keeps none. +$S mask unmask unmask-analyst@foxbyte.dev --branch umother --yes >/dev/null 2>&1 || bad "unmask on umother" +$S branch delete umother >/dev/null 2>&1 +$S mask unmask unmask-analyst@foxbyte.dev --branch main --yes >/dev/null 2>&1 +UG="${HOME:?}/${BRAND_STATE_DIR:?}/unmask-grants.json" +assert_eq "a deleted branch's unmask is gone with it, so a branch made later with its name has none" \ + "$(python3 -c 'import json,sys; print(sorted(g["branch"] for g in json.load(open(sys.argv[1]))))' "$UG")" "['main']" +$S mask account unmask-analyst@foxbyte.dev --off --yes >/dev/null 2>&1 +assert_eq "an account unmasked keeps no unmask, and its audit role is gone" \ + "$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))))' "$UG")|$(pg pg-main "$UROLE" | grep -c 'pgaudit.role')" "0|0" +assert_eq "every unmask given and taken is in the security log" "$($S audit log --limit 300 | grep -c 'mask.unmask')" "4" +$S audit-log off --yes >/dev/null 2>&1; main_up +for c in email phone aadhaar_no; do $S mask rule "u_people.$c" --off >/dev/null 2>&1; done +pg pg-main "SET bb.allow_destructive = on; DROP TABLE IF EXISTS u_people CASCADE" >/dev/null +for u in unmask-analyst plain-person; do $S user delete "$u@foxbyte.dev" >/dev/null 2>&1; done + echo "### 12. fox uninstall (B1)" # Removal used to be a list of commands to run by hand. This runs last: it takes # the stack apart, so nothing after it has a stack to use. diff --git a/web/src/api.ts b/web/src/api.ts index 9764684..7eb9fe2 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -943,7 +943,12 @@ export const applyHardeningProfile = (branch: string, profile: string) => export type MaskRuleName = 'partial' | 'hash' | 'fake' | 'empty' export type MaskRule = { schema: string; table: string; column: string; path?: string; rule: MaskRuleName; category: string; by: string; at: string } export type MaskedAccount = { email: string; by: string; at: string } -export type MaskStatus = { on: boolean; pending: string[]; branch: string; rules: MaskRule[]; ready: string[]; accounts: MaskedAccount[]; writes: boolean } +export type UnmaskGrant = { email: string; branch: string; by: string; at: string } +export type MaskLookup = { at: string; who: string; table: string; column: string; found: number } +export type MaskStatus = { + on: boolean; pending: string[]; branch: string; rules: MaskRule[]; ready: string[]; accounts: MaskedAccount[]; writes: boolean + unmasks: UnmaskGrant[]; lookups: MaskLookup[]; auditOn: boolean +} export type MaskPlace = { schema: string; table: string; column: string; path?: string } export const getMasking = (branch: string) => req('GET', `${API}/api/masking?branch=${encodeURIComponent(branch)}`) as Promise export const planMaskSetup = (branch: string) => @@ -958,6 +963,8 @@ export const dropMaskRule = (branch: string, at: MaskPlace) => req('POST', `${API}/api/masking/rule`, { branch, ...at, off: true }) as Promise<{ off: string }> export const maskAccount = (email: string, off: boolean) => req('POST', `${API}/api/masking/account`, { email, off }) as Promise<{ email: string; masked: boolean }> +export const unmaskAccount = (email: string, branch: string, off: boolean) => + req('POST', `${API}/api/masking/unmask`, { email, branch, off }) as Promise<{ email: string; branch: string; unmasked: boolean; auditOn: boolean }> // Masking keys (F5 step 2): about the keys, never a key. export type MaskTableKey = { branch: string; schema: string; table: string; domain: string; gen: number; keyed: boolean } export type MaskKeyUse = { diff --git a/web/src/pages/AccessReview.tsx b/web/src/pages/AccessReview.tsx index 575f6f4..7f5c8d6 100644 --- a/web/src/pages/AccessReview.tsx +++ b/web/src/pages/AccessReview.tsx @@ -92,12 +92,13 @@ export default function AccessReview() {

    Accounts

    DateFeature(s)What changedCommit / PR
    6 Nov 2026O3, H21, O2Masking personal data, step 3 (F5 Gate 1, Enterprise): unmask, the lookup, the access review’s column, reads audited. fox mask unmask <email> --branch b [--off] [--yes] gives a masked account real values on one branch, as its owner has, and only a masked account is given it. Its role there gets pgaudit.role = bb_audit_read, a superuser’s setting it cannot undo, and bb_audit_read holds SELECT on every masked table, so the audit log records that account’s reads of marked tables, and only its (said when the audit log is off, as they are then not logged). That closes 3.1’s last item (H21). Its open sessions end; a deleted branch’s unmask goes with it, and an account unmasked keeps none. The lookup, bb_mask.find(table, column, value), lets a masked account find a person by an exact value without unmasking: only in a column with a rule of its own, it returns the rows masked, refuses a value not of the column’s type without repeating it, allows 30 calls a minute for each account, and records each call (who, when, which column, how many found) where the account cannot read it, never the value; fox mask lookups lists them. The access review’s Sees unmasked column (O2) says whether each account is an admin, not masked, or masked, with the branches it owns and where it holds unmask, and the report’s note follows masking. The API is POST /api/masking/unmask and GET /api/masking/lookups; the Masking page has Unmask and Lookups tables, and the access review page the column; every unmask is in the security log (mask.unmask). Checked on a throwaway PostgreSQL 18: lookups by a hash, a partial and a fake mask; a column without a rule, a JSON path, a system table and an injected name refused; the record unreadable to the account and without values; the limit; no value in the server’s log. Found and fixed by the suite: the grants’ file, once empty, was written as null. Tests: integration 829/829 (§11ak new), v2 403/403, UI 49/49 plus the upgrade test (an account masked, given unmask, listed in the access review, then taken off), integration-f5 57/57, f1 70/70, f6 71/71, web unit 256/256, Go unit tests in both editions, vet for macOS, Linux and Windows.feat/f5-mask-step3
    6 Nov 2026O3Masking personal data, step 2 (F5 Gate 1, Enterprise): key domains and rotation. fox mask key [status] [--json] lists each key domain, its key number, when it was made and rotated and by whom, the tables under it on running branches and those whose masks were made under an older key; never a key. fox mask key create <domain> [--yes] makes a domain with a key of its own (32 random bytes beside the default, the server’s alone): tables under it hash apart from every other domain’s and cannot be joined with them. fox mask key use <domain> --table [schema.]table [--branch b] [--yes] puts a table’s masks under one, writing only that table’s again. fox mask key rotate <domain> [--yes] says what it does and costs first (about 21 seconds a million rows; every hash changes; no undo), then on every running branch, table by table, keeps a second shadow under the new key beside the old, fills it, and swaps it in with the view replaced in place, so masked readers read the old masks until each table is done; the new key then replaces the old one’s file, which deletes the old key. A rotation cut short carries on when run again; a branch started during one is made again; one stopped through it is named and made again by fox mask setup --branch. fox mask key drop <domain> deletes a domain no table uses; a stopped branch blocks it only if a table was put under the domain there or it was made since the domain was. The console’s Masking page has a Keys panel (make, rotate behind a dialog with the cost, delete, move a table); the API is GET /api/masking/keys and POST /api/masking/key (admins); every change is in the security log (mask.key). Found and fixed by the suite: a write running when a table’s masks were swapped finished with triggers naming a shadow just renamed away, so the start, swap and repoint now hold that table’s writes for a moment (reads go on; bounded, and tried again behind a long transaction). Checked on a throwaway PostgreSQL 18 with 200,000 rows and six writers on new connections: no write refused, every mask under the new key. Tests: integration 811/811 (§11aj new: domains hash apart; a table moved and back; a rotation while the application writes and a masked account reads, every mask under the new key, the old key’s file replaced and the next one removed, nothing left over, the key in no log; a stopped branch named and made again; a domain in use kept, the default never deleted; the API; the security log), v2 403/403, UI 48/48 plus the upgrade test (a domain made, a table moved, its key rotated, a domain in use kept, then deleted), integration-f5 57/57, f1 70/70, f6 71/71, web unit 256/256, Go unit tests in both editions (domain names, the records, which stopped branches may use a domain, the rekey and repoint SQL), vet for macOS, Linux and Windows.feat/f5-mask-step2
    6 Nov 2026O3Masking personal data, step 1 (F5 Gate 1, point 4.6, Enterprise; licence feature masking): rules on confirmed findings, and masked accounts. fox mask setup --branch b [--yes] previews, then makes the key (<state>/mask/default.key, 32 bytes, owned by the server’s user, 0400 in a 0700 folder, mounted read-only at /run/bb-mask; never in a table, a log or a message) and restarts the branch to mount it if it must. fox mask rule <[schema.]table.column> [--path $.a] [--rule hash|fake|partial|empty] [--off] [--yes] masks a column the personal-data scan found and a person confirmed, and says what it costs before it is set (design decision 9: the founders did not decide, so the recommendation stands): nothing on writes for partial or empty, about a fifth to a third fewer single-row writes and a 10,000-row UPDATE about 3.6 times as long for a hash or a fake, more for a JSON path. Aadhaar is offered partial and empty only, never hashed or faked. fox mask account <email> [--off] --yes masks an account: on main and any branch it does not own it reads the masks of marked tables and real values elsewhere, cannot write marked tables, and its failing statements are never logged with their text; an admin cannot be masked, and admins and a branch’s owner always see real values (decisions 2–4). Masks sit in shadow tables kept by statement triggers and are read through views, so nothing is computed at read time. The console’s Masking page sets masking up, sets and takes off rules behind a dialog showing the cost, and masks and unmasks accounts; the API is /api/masking (admins); every change is in the security log. Fixed on the way: an account’s role is found by name when it is unmasked (to_regrole never finds an email), and the uninstall removes the key’s folder although the server’s user owns it. Limits: one-column primary keys only; array or quoted JSON paths refused; masking stays on once set up; a new schema needs setup run again; key domains, rotation, lookup, reads audited and masked branches are steps 2–5. Tests: integration 792/792 (§11ai new: the key’s owner, modes and read-only mount, nothing in the logs; rules refused before confirmation; the cost; Aadhaar’s hash refused; each rule’s mask as a masked account reads it; joins on a hash; the real table, shadow and hash out of reach; writes refused; the application’s writes kept up; admins and owners see real values; rules off and accounts unmasked; the security log), v2 403/403, UI 47/47 plus the upgrade test (a rule set after its cost, Aadhaar’s choices, an account masked and unmasked), integration-f5 57/57, f1 70/70, f6 71/71, web unit 256/256, Go unit tests in both editions, vet for macOS, Linux and Windows.feat/f5-mask-step1
    5 Nov 2026O1, O2, H21Masking’s prototype (F5 Gate 0): the design’s claims, proven on PostgreSQL 18 and 16 (docs/FOX_F5_Masking_Design.md, section 17; make integration-f5, 57/57). bb_identity 0.2 adds bbid_mask_hmac(domain, value): HMAC-SHA-256 under a key domain’s key, a 32-byte file at /run/bb-mask/<domain>.key that must be the server’s own and readable by nobody else, never in a table or a message, read again when the file is replaced; never granted to PUBLIC (an upgrade from 0.1 included). The suite builds the image, and on throwaway containers shows: each rule’s mask (fake, hash, partial, empty, a JSON path); the real table, shadows, hash, fake function and key file out of a masked reader’s reach; filtering, grouping, joins on hashes, EXPLAIN, pg_stats and COPY as designed; no writes to marked tables; the lookup masked, 30 a minute, recorded without the value; the application’s writes, key changes, deletes and TRUNCATE kept up; key domains, a missing or loose key refused, rotation seen by a session that read the old key. It found and fixed: a refused lookup’s value reaching the server log (every masked login now has log_min_error_statement = panic), and a masked branch’s steps needing a statement each; with VACUUM FULL of the table and of pg_statistic, and WAL recycling off across three switches and checkpoints, a masked branch on a ZFS clone has no real value left in its files (a fresh cluster of only masked rows: 0.3 s for 20,000). Measured on PostgreSQL 18: single-row writes to a marked table 21–30% fewer, a 10,000-row UPDATE 5.6× (3.6× with a trigger a statement), indexed reads through the view 25% fewer, 1,000,000 masks filled in 14 s without holding up a writer and rotated in 21 s; over the 10% budget, so the cost is said before a rule is set (design decision 9). No product code yet: Gate 1 follows.feat/f5-mask-proto
    - + {(report.accounts ?? []).map(a => ( + diff --git a/web/src/pages/Masking.tsx b/web/src/pages/Masking.tsx index aa50b2a..03920aa 100644 --- a/web/src/pages/Masking.tsx +++ b/web/src/pages/Masking.tsx @@ -2,7 +2,7 @@ import { useCallback, useEffect, useState } from 'react' import { applyMaskSetup, dropMaskRule, getMaskKeys, getMasking, getPersonalData, maskAccount, maskKey, planMaskRule, planMaskSetup, - setMaskRule, type MaskKeysStatus, type MaskPlace, type MaskRule, type MaskRuleName, type MaskStatus, type PIIFinding, + setMaskRule, unmaskAccount, type MaskKeysStatus, type MaskPlace, type MaskRule, type MaskRuleName, type MaskStatus, type PIIFinding, } from '../api' import { useConfirm } from '../confirm' import { useFeatures, why } from '../features' @@ -48,6 +48,7 @@ export default function Masking() { const [keys, setKeys] = useState(null) const [domain, setDomain] = useState('') const [move, setMove] = useState({ table: '', domain: '' }) + const [unmaskFor, setUnmaskFor] = useState('') const [busy, setBusy] = useState(false) const [err, setErr] = useState('') const [msg, setMsg] = useState('') @@ -190,6 +191,25 @@ export default function Masking() { return `${t} on ${branch} is under key domain ${d}.` }) + const unmask = (e: string, off: boolean) => run(async () => { + const ok = await confirm({ + title: off ? `Take unmask on ${branch} from ${e}` : `Unmask ${e} on ${branch}`, + message: off + ? `They read masks on ${branch} again. Their open sessions there end.` + : <> +

    They see real values on {branch}, as its owner does. Every read they make there of a marked table is audited: their + role’s audit role becomes bb_audit_read, which only a superuser can change. Listed in the access review.

    + {!st?.auditOn &&

    The audit log is off, so their reads are not logged until it is turned on.

    } +

    Their open sessions on {branch} end.

    + , + confirmText: off ? 'Take it off' : 'Unmask', + }) + if (!ok) return '' + await unmaskAccount(e, branch, off) + if (!off) setUnmaskFor('') + return off ? `${e} reads masks on ${branch} again.` : `${e} sees real values on ${branch}, every read audited.` + }) + const ruled = new Set((st?.rules ?? []).map(r => key(r))) const ruleTables = [...new Set((st?.rules ?? []).map(r => `${r.schema}.${r.table}`))] const domainOf = (t: string) => keys?.keys.find(k => k.tables.some(x => x.branch === branch && `${x.schema}.${x.table}` === t))?.domain ?? 'default' @@ -312,6 +332,65 @@ export default function Masking() { )} + {st.on && ( + <> +

    Unmask on {st.branch}

    +

    + A masked account given unmask on a branch sees real values there, and every read it makes of a marked table is audited. + {!st.auditOn && <> The audit log is off, so those reads are not being logged.} +

    +
    +
    AccountReachesMay override onKeysLast activeStatus
    AccountReachesSees unmaskedMay override onKeysLast activeStatus
    {a.email}{a.admin && <> admin} {a.reaches}{(a.owns ?? []).length > 0 &&
    owns {a.owns!.join(', ')}
    }    		{a.unmasked} {(a.overrideOn ?? []).length > 0 ? a.overrideOn!.join(', ') : '–'} {a.keys} {day(a.lastActive)}
    + + + {st.unmasks.map(g => ( + + + + + + ))} + {st.unmasks.length === 0 && } + +
    AccountGiven by
    {g.email}{g.by}, {new Date(g.at).toLocaleString()}{setting && }
    Nobody: masked accounts read masks on {st.branch}.
    + + {setting && st.accounts.length > 0 && ( +
    { e.preventDefault(); if (unmaskFor) unmask(unmaskFor, false) }}> + + +
    + )} + +

    Lookups on {st.branch}

    +

    + A masked account finds a person by an exact value without unmasking: SELECT * FROM bb_mask.find('table', 'column', 'value'), + which returns the rows masked, 30 a minute. Each call is recorded here; the value never is. +

    +
    + + + + {st.lookups.map((l, i) => ( + + + + + + + ))} + {st.lookups.length === 0 && } + +
    WhenWhoColumnFound
    {new Date(l.at).toLocaleString()}{l.who}{l.table}.{l.column}{l.found}
    None.
    +
    + + )} + {keys && keys.keys.length > 0 && ( <>

    Keys

    diff --git a/web/tests/accessreview.spec.ts b/web/tests/accessreview.spec.ts index d6c9b15..7fa2db1 100644 --- a/web/tests/accessreview.spec.ts +++ b/web/tests/accessreview.spec.ts @@ -33,7 +33,8 @@ test('the access review lists accounts, keys and branches, follows a new window, const branches = page.getByRole('table', { name: 'Branches' }) await expect(branches.locator('tr', { has: page.getByText(branch, { exact: true }) })).toContainText(email) await expect(page.getByRole('heading', { name: 'What this report cannot claim' })).toBeVisible() - await expect(page.getByText(/No masking is in place yet/)).toBeVisible() + // Masking is set up on e2emask before Chromium starts (integration_ui.sh). + await expect(page.getByText(/Sees unmasked is from masking/)).toBeVisible() await page.getByLabel('Dormant after (days)').fill('30') await page.getByRole('button', { name: 'Produce report' }).click() diff --git a/web/tests/masking.spec.ts b/web/tests/masking.spec.ts index 3ca4b39..0fcf46f 100644 --- a/web/tests/masking.spec.ts +++ b/web/tests/masking.spec.ts @@ -120,3 +120,38 @@ test('a key domain is made, used, rotated and deleted', async ({ page }) => { await page.getByRole('dialog').getByRole('button', { name: 'Take it off' }).click() await expect(page.getByTestId('open-m_people-email')).toBeVisible({ timeout: 30_000 }) }) + +// Unmask (step 3): a masked account given unmask on a branch, listed, said +// to be audited, shown in the access review, then taken off. +test('a masked account is given unmask on a branch, and the access review says so', async ({ page }) => { + await signIn(page) + await page.getByRole('link', { name: 'Masking' }).click() + await page.getByLabel('Branch').selectOption('e2emask') + await expect(page.getByTestId('mask-state')).toContainText('On.', { timeout: 30_000 }) + + await page.getByLabel('Account to mask').fill(masked) + await page.getByRole('form', { name: 'Mask an account' }).getByRole('button', { name: 'Mask…' }).click() + await page.getByRole('dialog').getByRole('button', { name: 'Mask', exact: true }).click() + await expect(page.getByTestId(`account-${masked}`)).toBeVisible({ timeout: 30_000 }) + + await page.getByLabel('Account to unmask').selectOption(masked) + await page.getByRole('form', { name: 'Unmask an account' }).getByRole('button', { name: 'Unmask…' }).click() + const dlg = page.getByRole('dialog') + await expect(dlg).toContainText('Every read they make there of a marked table is audited') + await dlg.getByRole('button', { name: 'Unmask', exact: true }).click() + await expect(page.getByTestId('mask-msg')).toHaveText(`${masked} sees real values on e2emask, every read audited.`, { timeout: 30_000 }) + await expect(page.getByTestId(`unmask-${masked}`)).toBeVisible() + await expect(page.getByRole('table', { name: 'Lookups' })).toBeVisible() + + await page.getByRole('link', { name: 'Access review' }).click() + await expect(page.getByTestId(`unmasked-${masked}`)).toContainText('unmask on e2emask', { timeout: 30_000 }) + + await page.getByRole('link', { name: 'Masking' }).click() + await page.getByLabel('Branch').selectOption('e2emask') + await page.getByTestId(`unmask-${masked}`).getByRole('button', { name: 'Take off…' }).click() + await page.getByRole('dialog').getByRole('button', { name: 'Take it off' }).click() + await expect(page.getByTestId(`unmask-${masked}`)).toHaveCount(0, { timeout: 30_000 }) + await page.getByTestId(`account-${masked}`).getByRole('button', { name: 'Unmask…' }).click() + await page.getByRole('dialog').getByRole('button', { name: 'Unmask', exact: true }).click() + await expect(page.getByTestId(`account-${masked}`)).toHaveCount(0, { timeout: 30_000 }) +})