From 5840932a7a3116f34fd673c4f5359a3aaef4711f Mon Sep 17 00:00:00 2001 From: walter Date: Mon, 5 Oct 2026 23:15:28 +0330 Subject: [PATCH 1/6] fix(security): minimize persisted merchant data --- .../src/orders/order-payload.mapper.spec.ts | 32 +++- backend/src/orders/order-payload.mapper.ts | 51 +++++-- .../webhook-payload-minimizer.spec.ts | 66 +++++++++ .../src/webhooks/webhook-payload-minimizer.ts | 139 ++++++++++++++++++ ...commerce-webhook-ingestion.service.spec.ts | 23 ++- .../woocommerce-webhook-ingestion.service.ts | 6 +- 6 files changed, 302 insertions(+), 15 deletions(-) create mode 100644 backend/src/webhooks/webhook-payload-minimizer.spec.ts create mode 100644 backend/src/webhooks/webhook-payload-minimizer.ts diff --git a/backend/src/orders/order-payload.mapper.spec.ts b/backend/src/orders/order-payload.mapper.spec.ts index a337fb9..e5e65c0 100644 --- a/backend/src/orders/order-payload.mapper.spec.ts +++ b/backend/src/orders/order-payload.mapper.spec.ts @@ -47,7 +47,8 @@ describe('WooCommerce order payload mapping', () => { total_tax: '2.60', }, customerSnapshot: { - customer_id: 7, + billing: { first_name: 'Jane', last_name: 'Doe' }, + shipping: { city: 'Austin', country: 'US' }, }, paymentSnapshot: { method: 'cod', @@ -60,11 +61,40 @@ describe('WooCommerce order payload mapping', () => { remoteDeletedAt: null, }); expect(mapped.lineItemsSnapshot).toHaveLength(1); + expect(mapped.lineItemsSnapshot).toEqual([ + { name: 'Widget', quantity: 2, total: '20.00' }, + ]); + expect(JSON.stringify(mapped)).not.toContain('customer_id'); expect(mapped.wcCreatedAt.toISOString()).toBe('2026-07-23T10:00:00.000Z'); expect(mapped.wcModifiedAt.toISOString()).toBe('2026-07-23T10:05:00.000Z'); expect(mapped.projectionFingerprint).toMatch(/^[a-f0-9]{64}$/); }); + it('discards contact details, order notes and arbitrary item metadata', () => { + const mapped = mapWooCommerceOrder( + orderPayload({ + billing: { + first_name: 'Jane', + email: 'private@example.test', + phone: 'private-phone', + }, + shipping: { city: 'Austin', phone: 'private-phone' }, + customer_note: 'private-note', + line_items: [ + { + name: 'Widget', + quantity: 1, + total: '10', + meta_data: [{ key: 'private', value: 'private-note' }], + }, + ], + }) + ); + + const stored = JSON.stringify(mapped); + expect(stored).not.toMatch(/private@example|private-phone|private-note/); + }); + it('produces the same fingerprint regardless of source object key order', () => { const left = mapWooCommerceOrder(orderPayload()); const right = mapWooCommerceOrder({ diff --git a/backend/src/orders/order-payload.mapper.ts b/backend/src/orders/order-payload.mapper.ts index 7c684c3..54ea20a 100644 --- a/backend/src/orders/order-payload.mapper.ts +++ b/backend/src/orders/order-payload.mapper.ts @@ -74,12 +74,12 @@ export function mapWooCommerceOrder(payload: unknown): OrderProjection { totals[field] = value; } - const billing = requireRecord( + const billingSource = requireRecord( record['billing'], 'malformed-order-customer', wcOrderId ); - const shipping = requireRecord( + const shippingSource = requireRecord( record['shipping'], 'malformed-order-customer', wcOrderId @@ -101,6 +101,20 @@ export function mapWooCommerceOrder(payload: unknown): OrderProjection { ); } + const billing = pickDisplayFields(billingSource, [ + 'first_name', + 'last_name', + 'company', + ]); + const shipping = pickDisplayFields(shippingSource, [ + 'company', + 'address_1', + 'address_2', + 'city', + 'state', + 'postcode', + 'country', + ]); const projection: Omit = { wcOrderId, orderNumber: requireDisplayString( @@ -120,11 +134,17 @@ export function mapWooCommerceOrder(payload: unknown): OrderProjection { ), totals: canonicalizeJson(totals) as Prisma.InputJsonObject, customerSnapshot: canonicalizeJson({ - customer_id: normalizeCustomerId(record['customer_id']), billing, shipping, }) as Prisma.InputJsonObject, - lineItemsSnapshot: canonicalizeJson(lineItems) as Prisma.InputJsonArray, + lineItemsSnapshot: canonicalizeJson( + lineItems.map((item) => + pickDisplayFields( + requireRecord(item, 'malformed-order-line-items', wcOrderId), + ['name', 'quantity', 'total'] + ) + ) + ) as Prisma.InputJsonArray, paymentSnapshot: canonicalizeJson({ method: normalizeOptionalString(record['payment_method'], wcOrderId), method_title: normalizeOptionalString( @@ -329,16 +349,23 @@ function requireGmtDate(value: unknown, code: string, wcOrderId: string): Date { return parsed; } -function normalizeCustomerId(value: unknown): Prisma.InputJsonValue | null { - if ( - value === null || - (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) || - (typeof value === 'string' && /^\d+$/.test(value)) - ) { - return value; +function pickDisplayFields( + record: Record, + fields: readonly string[] +): Record { + const result: Record = {}; + + for (const field of fields) { + const value = record[field]; + if ( + typeof value === 'string' || + (typeof value === 'number' && Number.isFinite(value)) + ) { + result[field] = value; + } } - throw new OrderPayloadMappingError('malformed-order-customer'); + return result; } function normalizeOptionalString( diff --git a/backend/src/webhooks/webhook-payload-minimizer.spec.ts b/backend/src/webhooks/webhook-payload-minimizer.spec.ts new file mode 100644 index 0000000..e78a33d --- /dev/null +++ b/backend/src/webhooks/webhook-payload-minimizer.spec.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from '@jest/globals'; + +import { minimizeWebhookPayload } from './webhook-payload-minimizer'; + +describe('verified webhook payload minimization', () => { + it('keeps only fields required for order projection and delivery', () => { + const result = minimizeWebhookPayload('order.created', { + id: 101, + status: 'processing', + billing: { + first_name: 'Jane', + email: 'private@example.test', + phone: 'private-phone', + }, + shipping: { + city: 'Austin', + address_1: 'Main St', + phone: 'private-phone', + }, + line_items: [ + { name: 'Widget', quantity: 2, total: '20', metadata: 'private-note' }, + ], + customer_note: 'private-note', + metadata: { secret: 'private-note' }, + }); + + expect(result).toMatchObject({ + id: 101, + billing: { first_name: 'Jane' }, + shipping: { city: 'Austin', address_1: 'Main St' }, + line_items: [{ name: 'Widget', quantity: 2, total: '20' }], + }); + expect(JSON.stringify(result)).not.toMatch( + /private@example|private-phone|private-note/ + ); + }); + + it('reduces delete and unsupported topics to identity', () => { + expect( + minimizeWebhookPayload('order.deleted', { + id: 101, + billing: { email: 'x' }, + }) + ).toEqual({ id: 101 }); + expect( + minimizeWebhookPayload('coupon.created', { id: 44, code: 'sensitive' }) + ).toEqual({ id: 44 }); + }); + + it('keeps only inventory projection and variation scan fields', () => { + const result = minimizeWebhookPayload('product.updated', { + id: 7, + type: 'variable', + variations: [11, 12], + attributes: [{ name: 'Color', option: 'Blue', private: 'secret' }], + description: 'secret', + meta_data: [{ key: 'secret' }], + }); + expect(result).toEqual({ + id: 7, + type: 'variable', + variations: [11, 12], + attributes: [{ name: 'Color', option: 'Blue' }], + }); + }); +}); diff --git a/backend/src/webhooks/webhook-payload-minimizer.ts b/backend/src/webhooks/webhook-payload-minimizer.ts new file mode 100644 index 0000000..f53c622 --- /dev/null +++ b/backend/src/webhooks/webhook-payload-minimizer.ts @@ -0,0 +1,139 @@ +import { Prisma } from '@prisma/client'; + +const ORDER_FIELDS = [ + 'id', + 'number', + 'status', + 'currency', + 'discount_total', + 'discount_tax', + 'shipping_total', + 'shipping_tax', + 'cart_tax', + 'total', + 'total_tax', + 'payment_method', + 'payment_method_title', + 'date_paid_gmt', + 'date_paid', + 'date_created_gmt', + 'date_modified_gmt', +] as const; +const BILLING_FIELDS = ['first_name', 'last_name', 'company'] as const; +const SHIPPING_FIELDS = [ + 'company', + 'address_1', + 'address_2', + 'city', + 'state', + 'postcode', + 'country', +] as const; +const LINE_ITEM_FIELDS = ['name', 'quantity', 'total'] as const; +const SHIPPING_LINE_FIELDS = ['method_id', 'method_title'] as const; +const INVENTORY_FIELDS = [ + 'id', + 'parent_id', + 'type', + 'manage_stock', + 'stock_status', + 'stock_quantity', + 'sku', + 'name', + 'date_modified_gmt', + 'variations', +] as const; +const ATTRIBUTE_FIELDS = ['name', 'option'] as const; + +type JsonRecord = Record; + +function record(value: unknown): JsonRecord | null { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as JsonRecord) + : null; +} + +function scalar(value: unknown): string | number | boolean | null | undefined { + return value === null || + typeof value === 'string' || + typeof value === 'boolean' || + (typeof value === 'number' && Number.isFinite(value)) + ? value + : undefined; +} + +function pick(source: JsonRecord, fields: readonly string[]): JsonRecord { + const result: JsonRecord = {}; + + for (const field of fields) { + const value = scalar(source[field]); + + if (value !== undefined) { + result[field] = value; + } + } + + return result; +} + +function pickedRecord(value: unknown, fields: readonly string[]): JsonRecord { + return pick(record(value) ?? {}, fields); +} + +function pickedArray(value: unknown, fields: readonly string[]): JsonRecord[] { + return Array.isArray(value) + ? value.flatMap((item) => { + const source = record(item); + return source ? [pick(source, fields)] : []; + }) + : []; +} + +/** Data persisted after HMAC verification; this is also used for old-row scrubbing. */ +export function minimizeWebhookPayload( + topic: string, + value: unknown +): Prisma.InputJsonObject { + const source = record(value); + + if (!source) { + return {}; + } + + if (topic.startsWith('order.')) { + const result = pick(source, ORDER_FIELDS); + + if (topic !== 'order.deleted') { + result['billing'] = pickedRecord(source['billing'], BILLING_FIELDS); + result['shipping'] = pickedRecord(source['shipping'], SHIPPING_FIELDS); + result['line_items'] = pickedArray( + source['line_items'], + LINE_ITEM_FIELDS + ); + result['shipping_lines'] = pickedArray( + source['shipping_lines'], + SHIPPING_LINE_FIELDS + ); + } + + return result as Prisma.InputJsonObject; + } + + if (topic.startsWith('product.') || topic.startsWith('variation.')) { + const result = pick(source, INVENTORY_FIELDS); + result['attributes'] = pickedArray(source['attributes'], ATTRIBUTE_FIELDS); + + if (Array.isArray(source['variations'])) { + result['variations'] = source['variations'].flatMap((item) => { + const value = scalar(item); + return typeof value === 'number' || typeof value === 'string' + ? [value] + : []; + }); + } + + return result as Prisma.InputJsonObject; + } + + return pick(source, ['id']) as Prisma.InputJsonObject; +} diff --git a/backend/src/webhooks/woocommerce-webhook-ingestion.service.spec.ts b/backend/src/webhooks/woocommerce-webhook-ingestion.service.spec.ts index 612f3a3..ba5ec69 100644 --- a/backend/src/webhooks/woocommerce-webhook-ingestion.service.spec.ts +++ b/backend/src/webhooks/woocommerce-webhook-ingestion.service.spec.ts @@ -274,6 +274,10 @@ function setup(options: { missingSecret?: boolean } = {}) { preload, publishedJobIds, service, + disconnectStore() { + stores[0]!.status = StoreStatus.DISCONNECTED; + stores[0]!.deletedAt = new Date(); + }, setEnqueueFailure(value: boolean) { enqueueFailure = value; }, @@ -282,6 +286,17 @@ function setup(options: { missingSecret?: boolean } = {}) { } describe('WooCommerce webhook ingestion', () => { + it('rejects signed delivery after verified Store disconnect without persistence', async () => { + const fixture = setup(); + fixture.disconnectStore(); + const body = Buffer.from('{"id":101}'); + await expect( + fixture.service.receive(ENDPOINT_A, headers(body), body) + ).rejects.toBeInstanceOf(NotFoundException); + expect(fixture.eventCreate).not.toHaveBeenCalled(); + expect(fixture.addWooCommerceWebhookJob).not.toHaveBeenCalled(); + }); + it('persists and enqueues one valid signed delivery with deterministic job ID', async () => { const fixture = setup(); const body = Buffer.from(JSON.stringify({ id: 101, status: 'processing' })); @@ -439,7 +454,13 @@ describe('WooCommerce webhook ingestion', () => { expect(fixture.events[0]).toMatchObject({ tenantId: 'ten_a', storeId: 'sto_a', - payload: { tenantId: 'ten_b', storeId: 'sto_b', id: 1 }, + payload: { + id: 1, + billing: {}, + shipping: {}, + line_items: [], + shipping_lines: [], + }, }); expect(fixture.storeFindFirst).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/backend/src/webhooks/woocommerce-webhook-ingestion.service.ts b/backend/src/webhooks/woocommerce-webhook-ingestion.service.ts index 14de3f3..df9fa8b 100644 --- a/backend/src/webhooks/woocommerce-webhook-ingestion.service.ts +++ b/backend/src/webhooks/woocommerce-webhook-ingestion.service.ts @@ -13,6 +13,7 @@ import { StructuredLoggerService } from '../common/logging/structured-logger.ser import { PrismaService } from '../prisma/prisma.service'; import { WooCommerceWebhookJobProducer } from '../queue/woocommerce-webhook-job.producer'; import type { WebhookRequestHeaders } from './woocommerce-webhook.controller'; +import { minimizeWebhookPayload } from './webhook-payload-minimizer'; const GENERIC_NOT_FOUND_MESSAGE = 'Webhook endpoint was not found'; const MALFORMED_REQUEST_MESSAGE = 'Webhook request is malformed'; @@ -87,7 +88,10 @@ export class WooCommerceWebhookIngestionService { throw new UnauthorizedException(INVALID_SIGNATURE_MESSAGE); } - const payload = this.parsePayload(body); + const payload = minimizeWebhookPayload( + requiredHeaders.topic, + this.parsePayload(body) + ); const event = await this.persistOrFind(store, requiredHeaders, payload); if (event.status !== WebhookEventStatus.RECEIVED) { From 87c92b05d338030643b307353ecace1b3380931e Mon Sep 17 00:00:00 2001 From: walter Date: Mon, 5 Oct 2026 23:15:41 +0330 Subject: [PATCH 2/6] feat(security): add pilot data lifecycle controls --- backend/src/app.module.ts | 2 + backend/src/privacy/pilot-data.cli.spec.ts | 89 ++++ backend/src/privacy/pilot-data.cli.ts | 307 ++++++++++++++ .../src/privacy/pilot-data.service.spec.ts | 238 +++++++++++ backend/src/privacy/pilot-data.service.ts | 394 ++++++++++++++++++ .../privacy/pilot-retention.service.spec.ts | 77 ++++ .../src/privacy/pilot-retention.service.ts | 122 ++++++ .../src/privacy/privacy-retention.module.ts | 12 + .../src/privacy/woo-webhook-cleanup.spec.ts | 68 +++ backend/src/privacy/woo-webhook-cleanup.ts | 82 ++++ backend/src/queue/queue-runtime.service.ts | 36 ++ backend/src/queue/queue.constants.ts | 4 + backend/src/queue/queue.spec.ts | 32 ++ .../woocommerce-webhook.processor.spec.ts | 16 +- .../queue/woocommerce-webhook.processor.ts | 26 +- 15 files changed, 1503 insertions(+), 2 deletions(-) create mode 100644 backend/src/privacy/pilot-data.cli.spec.ts create mode 100644 backend/src/privacy/pilot-data.cli.ts create mode 100644 backend/src/privacy/pilot-data.service.spec.ts create mode 100644 backend/src/privacy/pilot-data.service.ts create mode 100644 backend/src/privacy/pilot-retention.service.spec.ts create mode 100644 backend/src/privacy/pilot-retention.service.ts create mode 100644 backend/src/privacy/privacy-retention.module.ts create mode 100644 backend/src/privacy/woo-webhook-cleanup.spec.ts create mode 100644 backend/src/privacy/woo-webhook-cleanup.ts diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index fc8b5cc..f954b2c 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -19,6 +19,7 @@ import { MembershipsModule } from './memberships/memberships.module'; import { OnboardingModule } from './onboarding/onboarding.module'; import { PilotModule } from './pilot/pilot.module'; import { PrismaModule } from './prisma/prisma.module'; +import { PrivacyRetentionModule } from './privacy/privacy-retention.module'; import { StoreModule } from './store/store.module'; import { TenantContextGuard } from './tenant/guards/tenant-context.guard'; import { TenantContextModule } from './tenant/tenant-context.module'; @@ -35,6 +36,7 @@ import { WebhooksModule } from './webhooks/webhooks.module'; EntitlementsModule, AuthModule, PrismaModule, + PrivacyRetentionModule, HealthModule, TenantContextModule, StoreModule, diff --git a/backend/src/privacy/pilot-data.cli.spec.ts b/backend/src/privacy/pilot-data.cli.spec.ts new file mode 100644 index 0000000..929041f --- /dev/null +++ b/backend/src/privacy/pilot-data.cli.spec.ts @@ -0,0 +1,89 @@ +import { afterEach, describe, expect, it, jest } from '@jest/globals'; +import { PrismaClient, StoreStatus } from '@prisma/client'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { appendLedger, replayLedger } from './pilot-data.cli'; +import type { PilotDataService } from './pilot-data.service'; + +const target = { + tenantId: 'ten_a', + storeId: 'sto_a', + baseUrl: 'https://shop.example.test', +}; +const directories: string[] = []; + +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'wctm-erasure-')); + directories.push(directory); + const ledger = join(directory, 'ledger.jsonl'); + await writeFile(ledger, '', { mode: 0o600 }); + return ledger; +} + +afterEach(async () => { + for (const directory of directories.splice(0)) { + await rm(directory, { recursive: true, force: true }); + } +}); + +describe('external erasure ledger replay', () => { + it('records only scoped identity hashes and replays a restored Store before resumption', async () => { + const ledger = await fixture(); + await appendLedger(ledger, target); + const content = await readFile(ledger, 'utf8'); + expect(content).not.toContain(target.baseUrl); + const disconnect = jest.fn(async () => undefined); + const erase = jest.fn(async () => undefined); + const service = { disconnect, erase } as unknown as PilotDataService; + const database = { + store: { + findFirst: jest.fn(async () => ({ + baseUrl: target.baseUrl, + status: StoreStatus.ACTIVE, + })), + }, + } as unknown as PrismaClient; + + await expect( + replayLedger(ledger, service, database, false) + ).resolves.toMatchObject({ + pending: 1, + executed: false, + }); + expect(disconnect).not.toHaveBeenCalled(); + await expect( + replayLedger(ledger, service, database, true) + ).resolves.toMatchObject({ + pending: 1, + executed: true, + }); + expect(disconnect).toHaveBeenCalledWith(target); + expect(erase).toHaveBeenCalledWith(target); + expect(disconnect.mock.invocationCallOrder[0]).toBeLessThan( + erase.mock.invocationCallOrder[0]! + ); + }); + + it('fails closed when restored Store identity differs', async () => { + const ledger = await fixture(); + await appendLedger(ledger, target); + const database = { + store: { + findFirst: jest.fn(async () => ({ + baseUrl: 'https://other.example.test', + status: StoreStatus.ACTIVE, + })), + }, + } as unknown as PrismaClient; + const service = { + disconnect: jest.fn(), + erase: jest.fn(), + } as unknown as PilotDataService; + await expect(replayLedger(ledger, service, database, true)).rejects.toThrow( + 'does not match' + ); + expect(service.disconnect).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/privacy/pilot-data.cli.ts b/backend/src/privacy/pilot-data.cli.ts new file mode 100644 index 0000000..3aec56c --- /dev/null +++ b/backend/src/privacy/pilot-data.cli.ts @@ -0,0 +1,307 @@ +import { PrismaPg } from '@prisma/adapter-pg'; +import { PrismaClient, StoreStatus } from '@prisma/client'; +import { createHash } from 'node:crypto'; +import { open, readFile, stat, writeFile } from 'node:fs/promises'; +import { isAbsolute, resolve } from 'node:path'; + +import { EncryptionService } from '../common/encryption/encryption.service'; +import type { ApplicationConfigService } from '../config/application-config.service'; +import { PilotDataService, type PilotTarget } from './pilot-data.service'; +import { removeOwnedWooWebhooks } from './woo-webhook-cleanup'; + +type Mode = + | 'inspect' + | 'disconnect' + | 'export' + | 'prepare-erasure' + | 'erase' + | 'scrub' + | 'replay'; + +interface Options { + mode: Mode; + target?: PilotTarget; + execute: boolean; + output?: string; + ledger?: string; +} + +function argument(name: string, args: string[]): string | undefined { + const index = args.indexOf(name); + return index < 0 || index + 1 >= args.length ? undefined : args[index + 1]; +} + +function parse(args: string[]): Options { + const mode = args[0] as Mode; + if ( + ![ + 'inspect', + 'disconnect', + 'export', + 'prepare-erasure', + 'erase', + 'scrub', + 'replay', + ].includes(mode) + ) { + throw new Error('privacy mode is invalid'); + } + const execute = args.includes('--execute'); + const tenantId = argument('--tenant-id', args); + const storeId = argument('--store-id', args); + const baseUrl = argument('--base-url', args); + const output = argument('--output', args); + const ledger = argument('--ledger', args); + if ( + mode !== 'replay' && + (!tenantId?.startsWith('ten_') || + !storeId?.startsWith('sto_') || + !baseUrl?.startsWith('https://')) + ) { + throw new Error('exact Tenant, Store and HTTPS base URL are required'); + } + if (['export'].includes(mode) && (!output || !isAbsolute(output))) { + throw new Error('export requires an absolute protected output path'); + } + if ( + ['prepare-erasure', 'erase', 'replay'].includes(mode) && + (!ledger || !isAbsolute(ledger)) + ) { + throw new Error('erasure requires an absolute external ledger path'); + } + return { + mode, + execute, + ...(mode === 'replay' + ? {} + : { + target: { tenantId: tenantId!, storeId: storeId!, baseUrl: baseUrl! }, + }), + ...(output ? { output } : {}), + ...(ledger ? { ledger } : {}), + }; +} + +function fingerprint(value: string): string { + return createHash('sha256').update(value).digest('hex'); +} + +async function verifyLedger(path: string): Promise { + if (resolve(path).startsWith(`${process.cwd()}/`)) { + throw new Error('erasure ledger must be outside the repository'); + } + const details = await stat(path); + if (!details.isFile() || (details.mode & 0o077) !== 0) { + throw new Error('erasure ledger must be a mode-0600 regular file'); + } +} + +export async function appendLedger( + path: string, + target: PilotTarget +): Promise { + await verifyLedger(path); + const handle = await open(path, 'a'); + try { + await handle.writeFile( + `${JSON.stringify({ + tenantId: target.tenantId, + storeId: target.storeId, + baseUrlSha256: fingerprint(target.baseUrl), + requestedAt: new Date().toISOString(), + })}\n` + ); + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function ledgerContains( + path: string, + target: PilotTarget +): Promise { + await verifyLedger(path); + return (await readFile(path, 'utf8')).split('\n').some((line) => { + if (!line) return false; + const entry = JSON.parse(line) as Record; + return ( + entry['tenantId'] === target.tenantId && + entry['storeId'] === target.storeId && + entry['baseUrlSha256'] === fingerprint(target.baseUrl) + ); + }); +} + +export async function replayLedger( + path: string, + service: PilotDataService, + database: PrismaClient, + execute: boolean +) { + await verifyLedger(path); + const lines = (await readFile(path, 'utf8')).split('\n').filter(Boolean); + let pending = 0; + for (const line of lines) { + const entry = JSON.parse(line) as Record; + if ( + typeof entry['tenantId'] !== 'string' || + typeof entry['storeId'] !== 'string' || + typeof entry['baseUrlSha256'] !== 'string' || + !/^[0-9a-f]{64}$/.test(entry['baseUrlSha256']) + ) { + throw new Error('erasure ledger contains an invalid entry'); + } + const store = await database.store.findFirst({ + where: { id: entry['storeId'], tenantId: entry['tenantId'] }, + select: { baseUrl: true, status: true }, + }); + if (!store) continue; + if ( + store.baseUrl === `https://erased.invalid/${entry['storeId']}` && + store.status === StoreStatus.DISCONNECTED + ) + continue; + if (fingerprint(store.baseUrl) !== entry['baseUrlSha256']) { + throw new Error( + 'erasure ledger target identity does not match restored Store' + ); + } + pending++; + if (execute) { + const target = { + tenantId: entry['tenantId'], + storeId: entry['storeId'], + baseUrl: store.baseUrl, + }; + await service.disconnect(target); + await service.erase(target); + } + } + return { ledgerEntries: lines.length, pending, executed: execute }; +} + +async function main(): Promise { + const options = parse(process.argv.slice(2)); + if (process.env['WCTM_PILOT_PRIVACY_OPERATION'] !== 'operator-approved') { + throw new Error('operator-only privacy command is not enabled'); + } + const databaseUrl = process.env['DATABASE_URL']; + const key = process.env['APP_ENCRYPTION_KEY']; + if (!databaseUrl || !key) + throw new Error( + 'protected database and application key configuration is required' + ); + const databaseIdentity = new URL(databaseUrl).username; + if ( + !databaseIdentity || + decodeURIComponent(databaseIdentity) === 'wctm_runtime' + ) { + throw new Error( + 'privacy operation requires a distinct non-runtime database identity' + ); + } + const database = new PrismaClient({ + adapter: new PrismaPg({ connectionString: databaseUrl }), + }); + const encryption = new EncryptionService({ + encryption: { key, previousKey: undefined }, + } as ApplicationConfigService); + const service = new PilotDataService(database, encryption); + try { + if (options.mode === 'replay') { + process.stdout.write( + `${JSON.stringify(await replayLedger(options.ledger!, service, database, options.execute))}\n` + ); + return; + } + const target = options.target!; + if (options.mode === 'inspect') { + process.stdout.write( + `${JSON.stringify(await service.inspect(target))}\n` + ); + } else if (options.mode === 'disconnect') { + let remoteWebhookCleanup = 'not-run'; + if (options.execute) { + const credentials = await database.store.findFirst({ + where: { + id: target.storeId, + tenantId: target.tenantId, + baseUrl: target.baseUrl, + }, + select: { + consumerKeyEncrypted: true, + consumerSecretEncrypted: true, + webhookEndpointKey: true, + }, + }); + if (!credentials) + throw new Error('privacy target changed before disconnect'); + let remoteInput; + try { + remoteInput = { + baseUrl: target.baseUrl, + endpointKey: credentials.webhookEndpointKey, + consumerKey: encryption.decrypt(credentials.consumerKeyEncrypted), + consumerSecret: encryption.decrypt( + credentials.consumerSecretEncrypted + ), + }; + } catch { + remoteInput = undefined; + } + await service.disconnect(target); + remoteWebhookCleanup = remoteInput + ? (await removeOwnedWooWebhooks(remoteInput)).complete + ? 'complete' + : 'manual-required' + : 'manual-required'; + } + process.stdout.write( + `${JSON.stringify({ target: target.storeId, disconnected: options.execute, remoteWebhookCleanup })}\n` + ); + } else if (options.mode === 'export') { + const report = await service.export(target); + await writeFile(options.output!, `${JSON.stringify(report)}\n`, { + flag: 'wx', + mode: 0o600, + }); + process.stdout.write('privacy export: PASS protected-file-created\n'); + } else if (options.mode === 'scrub') { + process.stdout.write( + `${JSON.stringify(await service.scrubHistorical(target, options.execute))}\n` + ); + } else if (options.mode === 'prepare-erasure' || options.mode === 'erase') { + const report = await service.inspect(target); + if (!report.disconnected || report.status !== StoreStatus.DISCONNECTED) { + throw new Error('privacy erasure requires verified disconnect first'); + } + if (options.execute) { + if (options.mode === 'prepare-erasure') { + await appendLedger(options.ledger!, target); + } else { + if (!(await ledgerContains(options.ledger!, target))) { + throw new Error( + 'erasure ledger does not contain the verified target' + ); + } + await service.erase(target); + } + } + process.stdout.write( + `${JSON.stringify({ target: target.storeId, prepared: options.mode === 'prepare-erasure' && options.execute, erased: options.mode === 'erase' && options.execute, counts: report.counts })}\n` + ); + } + } finally { + await database.$disconnect(); + } +} + +if (require.main === module) { + void main().catch(() => { + process.stderr.write( + 'pilot privacy operation failed; inspect protected operator records\n' + ); + process.exitCode = 1; + }); +} diff --git a/backend/src/privacy/pilot-data.service.spec.ts b/backend/src/privacy/pilot-data.service.spec.ts new file mode 100644 index 0000000..57cc14f --- /dev/null +++ b/backend/src/privacy/pilot-data.service.spec.ts @@ -0,0 +1,238 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { PrismaClient, StoreStatus } from '@prisma/client'; + +import type { EncryptionService } from '../common/encryption/encryption.service'; +import { PilotDataService } from './pilot-data.service'; + +const TARGET = { + tenantId: 'ten_a', + storeId: 'sto_a', + baseUrl: 'https://shop.example.test', +}; + +function setup(found = true) { + const store = { + id: TARGET.storeId, + tenantId: TARGET.tenantId, + baseUrl: TARGET.baseUrl, + status: StoreStatus.ACTIVE as StoreStatus, + deletedAt: null as Date | null, + }; + const findMany = jest.fn(async () => (found ? [store] : [])); + const storeUpdate = jest.fn(async () => ({ count: 1 })); + const updateMany = jest.fn(async () => ({ count: 1 })); + const auditCreate = jest.fn(async () => ({ id: 'aud_a' })); + const transaction = { + store: { updateMany: storeUpdate }, + telegramChatAuthorization: { updateMany }, + webhookEvent: { updateMany }, + order: { updateMany }, + inventoryItem: { updateMany }, + telegramCallbackReference: { updateMany }, + telegramSearchReference: { updateMany }, + telegramOrderNoteAction: { updateMany }, + telegramOrderStatusWrite: { updateMany }, + auditLog: { create: auditCreate }, + }; + const database = { + store: { findMany }, + order: { + count: jest.fn(async () => 2), + findMany: jest.fn(async () => []), + updateMany, + }, + webhookEvent: { + count: jest.fn(async () => 3), + findMany: jest.fn(async () => []), + updateMany, + }, + inventoryItem: { + count: jest.fn(async () => 1), + findMany: jest.fn(async () => []), + }, + telegramOrderNotificationDelivery: { count: jest.fn(async () => 1) }, + telegramInventoryNotificationDelivery: { count: jest.fn(async () => 0) }, + $transaction: jest.fn( + async (callback: (tx: typeof transaction) => Promise) => + callback(transaction) + ), + }; + const encryption = { + encrypt: jest.fn(() => 'encrypted-revoked'), + } as unknown as EncryptionService; + return { + service: new PilotDataService( + database as unknown as PrismaClient, + encryption + ), + database, + transaction, + store, + findMany, + storeUpdate, + updateMany, + }; +} + +describe('pilot operator privacy boundary', () => { + it('fails closed when the exact Tenant, Store and URL do not resolve', async () => { + const fixture = setup(false); + await expect(fixture.service.inspect(TARGET)).rejects.toThrow('identity'); + await expect(fixture.service.disconnect(TARGET)).rejects.toThrow( + 'identity' + ); + await expect(fixture.service.erase(TARGET)).rejects.toThrow('identity'); + expect(fixture.database.$transaction).not.toHaveBeenCalled(); + expect(fixture.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'sto_a', tenantId: 'ten_a', baseUrl: TARGET.baseUrl }, + }) + ); + }); + + it('dry-run inspection reports scoped counts without mutating data', async () => { + const fixture = setup(); + await expect(fixture.service.inspect(TARGET)).resolves.toMatchObject({ + tenantId: 'ten_a', + storeId: 'sto_a', + counts: { orders: 2, events: 3, inventory: 1 }, + }); + expect(fixture.database.$transaction).not.toHaveBeenCalled(); + }); + + it('disconnect revokes credentials, ingestion and Telegram Store context', async () => { + const fixture = setup(); + await fixture.service.disconnect(TARGET); + + expect(fixture.storeUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ + tenantId: 'ten_a', + id: 'sto_a', + baseUrl: TARGET.baseUrl, + }), + data: expect.objectContaining({ + status: StoreStatus.DISCONNECTED, + pluginSecretHash: null, + webhookSecretEncrypted: null, + webhookEndpointKey: null, + consumerKeyEncrypted: 'encrypted-revoked', + }), + }) + ); + expect( + fixture.transaction.telegramChatAuthorization.updateMany + ).toHaveBeenCalledWith( + expect.objectContaining({ + where: { activeStoreId: 'sto_a', activeTenantId: 'ten_a' }, + }) + ); + expect(fixture.transaction.webhookEvent.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ tenantId: 'ten_a', storeId: 'sto_a' }), + }) + ); + }); + + it('revokes credentials even when a Store was already soft-deleted', async () => { + const fixture = setup(); + fixture.store.status = StoreStatus.DISCONNECTED; + fixture.store.deletedAt = new Date(); + await fixture.service.disconnect(TARGET); + expect(fixture.storeUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + where: { + id: TARGET.storeId, + tenantId: TARGET.tenantId, + baseUrl: TARGET.baseUrl, + }, + data: expect.objectContaining({ + pluginSecretHash: null, + webhookSecretEncrypted: null, + }), + }) + ); + }); + + it('requires verified disconnect before scoped erasure', async () => { + const fixture = setup(); + await expect(fixture.service.erase(TARGET)).rejects.toThrow('disconnect'); + expect(fixture.database.$transaction).not.toHaveBeenCalled(); + }); + + it('scrubs old terminal payloads in scoped batches, with a non-mutating dry run', async () => { + const fixture = setup(); + const old = { + id: 'evt_a', + topic: 'order.updated', + status: 'COMPLETED', + payload: { + id: 12, + billing: { first_name: 'A', email: 'private@example.test' }, + meta_data: [{ key: 'private' }], + }, + }; + fixture.database.webhookEvent.findMany + .mockResolvedValueOnce([old] as never) + .mockResolvedValueOnce([]); + await expect( + fixture.service.scrubHistorical(TARGET, false) + ).resolves.toEqual({ + changedEvents: 1, + changedOrders: 0, + executed: false, + }); + expect(fixture.database.webhookEvent.updateMany).not.toHaveBeenCalled(); + + fixture.database.webhookEvent.findMany + .mockResolvedValueOnce([old] as never) + .mockResolvedValueOnce([]); + await expect( + fixture.service.scrubHistorical(TARGET, true) + ).resolves.toEqual({ + changedEvents: 1, + changedOrders: 0, + executed: true, + }); + expect(fixture.database.webhookEvent.findMany).toHaveBeenCalledWith( + expect.objectContaining({ take: 100, orderBy: { id: 'asc' } }) + ); + expect(fixture.database.webhookEvent.updateMany).toHaveBeenCalledWith({ + where: { + id: 'evt_a', + tenantId: TARGET.tenantId, + storeId: TARGET.storeId, + status: 'COMPLETED', + }, + data: { + payload: expect.not.objectContaining({ meta_data: expect.anything() }), + }, + }); + }); + + it('anonymizes scoped records while preserving audit identity', async () => { + const fixture = setup(); + fixture.store.status = StoreStatus.DISCONNECTED; + fixture.store.deletedAt = new Date(); + await fixture.service.erase(TARGET); + + expect(fixture.transaction.order.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: { tenantId: 'ten_a', storeId: 'sto_a' }, + data: expect.objectContaining({ + customerSnapshot: {}, + lineItemsSnapshot: [], + }), + }) + ); + expect(fixture.transaction.auditLog.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + tenantId: 'ten_a', + entityId: 'sto_a', + action: 'privacy.store_erased', + }), + }) + ); + }); +}); diff --git a/backend/src/privacy/pilot-data.service.ts b/backend/src/privacy/pilot-data.service.ts new file mode 100644 index 0000000..35a6f0c --- /dev/null +++ b/backend/src/privacy/pilot-data.service.ts @@ -0,0 +1,394 @@ +import { + Prisma, + PrismaClient, + StoreStatus, + WebhookEventStatus, +} from '@prisma/client'; +import { randomBytes, randomUUID } from 'node:crypto'; + +import { EncryptionService } from '../common/encryption/encryption.service'; +import { orderProjectionFingerprint } from '../orders/order-payload.mapper'; +import { minimizeWebhookPayload } from '../webhooks/webhook-payload-minimizer'; + +export interface PilotTarget { + tenantId: string; + storeId: string; + baseUrl: string; +} + +export class PilotDataService { + constructor( + private readonly database: PrismaClient, + private readonly encryption: EncryptionService + ) {} + + async resolve(target: PilotTarget) { + const stores = await this.database.store.findMany({ + where: { + id: target.storeId, + tenantId: target.tenantId, + baseUrl: target.baseUrl, + }, + select: { + id: true, + tenantId: true, + baseUrl: true, + status: true, + deletedAt: true, + }, + take: 2, + }); + if (stores.length !== 1) { + throw new Error('privacy target identity is missing or ambiguous'); + } + return stores[0]!; + } + + async inspect(target: PilotTarget) { + const store = await this.resolve(target); + const where = { tenantId: target.tenantId, storeId: target.storeId }; + const [orders, events, inventory, orderDeliveries, inventoryDeliveries] = + await Promise.all([ + this.database.order.count({ where }), + this.database.webhookEvent.count({ where }), + this.database.inventoryItem.count({ where }), + this.database.telegramOrderNotificationDelivery.count({ where }), + this.database.telegramInventoryNotificationDelivery.count({ where }), + ]); + return { + tenantId: store.tenantId, + storeId: store.id, + status: store.status, + disconnected: store.deletedAt !== null, + counts: { + orders, + events, + inventory, + orderDeliveries, + inventoryDeliveries, + }, + }; + } + + async disconnect(target: PilotTarget): Promise { + await this.resolve(target); + const now = new Date(); + const revoked = this.encryption.encrypt(randomBytes(32).toString('hex')); + + await this.database.$transaction(async (transaction) => { + const result = await transaction.store.updateMany({ + where: { + id: target.storeId, + tenantId: target.tenantId, + baseUrl: target.baseUrl, + }, + data: { + status: StoreStatus.DISCONNECTED, + deletedAt: now, + pluginSecretHash: null, + pluginRegisteredAt: null, + registrationTokenHash: null, + registrationTokenExpiresAt: null, + webhookSecretEncrypted: null, + webhookEndpointKey: null, + consumerKeyEncrypted: revoked, + consumerSecretEncrypted: revoked, + }, + }); + if (result.count !== 1) { + throw new Error('privacy target changed before disconnect'); + } + await transaction.telegramChatAuthorization.updateMany({ + where: { + activeStoreId: target.storeId, + activeTenantId: target.tenantId, + }, + data: { activeStoreId: null, activeTenantId: null, revokedAt: now }, + }); + await transaction.webhookEvent.updateMany({ + where: { + tenantId: target.tenantId, + storeId: target.storeId, + status: { + in: [ + WebhookEventStatus.RECEIVED, + WebhookEventStatus.QUEUED, + WebhookEventStatus.PROCESSING, + ], + }, + }, + data: { + status: WebhookEventStatus.FAILED, + processingStartedAt: null, + failureCategory: 'privacy', + failureMessage: 'store-disconnected', + failedAt: now, + }, + }); + await transaction.auditLog.create({ + data: { + id: `aud_${randomUUID()}`, + tenantId: target.tenantId, + userId: null, + action: 'privacy.store_disconnected', + entityType: 'Store', + entityId: target.storeId, + metadata: { scope: 'operator' }, + }, + }); + }); + } + + async export(target: PilotTarget) { + await this.resolve(target); + const where = { tenantId: target.tenantId, storeId: target.storeId }; + const [orders, inventory, events] = await Promise.all([ + this.database.order.findMany({ + where, + select: { + wcOrderId: true, + orderNumber: true, + status: true, + currency: true, + totals: true, + customerSnapshot: true, + lineItemsSnapshot: true, + paymentSnapshot: true, + shippingLinesSnapshot: true, + wcCreatedAt: true, + }, + }), + this.database.inventoryItem.findMany({ + where, + select: { + wcItemId: true, + displayName: true, + sku: true, + stockQuantity: true, + stockStatus: true, + }, + }), + this.database.webhookEvent.findMany({ + where, + select: { id: true, topic: true, status: true, receivedAt: true }, + }), + ]); + return { + tenantId: target.tenantId, + storeId: target.storeId, + exportedAt: new Date().toISOString(), + orders, + inventory, + events, + }; + } + + async scrubHistorical(target: PilotTarget, execute: boolean) { + await this.resolve(target); + const where = { tenantId: target.tenantId, storeId: target.storeId }; + let changedEvents = 0; + let changedOrders = 0; + let eventCursor: string | undefined; + for (;;) { + const events = await this.database.webhookEvent.findMany({ + where: { + ...where, + status: { + in: [WebhookEventStatus.COMPLETED, WebhookEventStatus.FAILED], + }, + }, + select: { id: true, topic: true, payload: true, status: true }, + orderBy: { id: 'asc' }, + take: 100, + ...(eventCursor ? { cursor: { id: eventCursor }, skip: 1 } : {}), + }); + if (events.length === 0) break; + eventCursor = events[events.length - 1]!.id; + for (const event of events) { + const minimized = minimizeWebhookPayload(event.topic, event.payload); + if (JSON.stringify(event.payload) === JSON.stringify(minimized)) + continue; + changedEvents++; + if (execute) { + const changed = await this.database.webhookEvent.updateMany({ + where: { id: event.id, ...where, status: event.status }, + data: { payload: minimized }, + }); + if (changed.count !== 1) + throw new Error('historical event changed during scrub'); + } + } + } + let orderCursor: string | undefined; + for (;;) { + const orders = await this.database.order.findMany({ + where, + orderBy: { id: 'asc' }, + take: 100, + ...(orderCursor ? { cursor: { id: orderCursor }, skip: 1 } : {}), + }); + if (orders.length === 0) break; + orderCursor = orders[orders.length - 1]!.id; + for (const order of orders) { + const customer = jsonRecord(order.customerSnapshot); + const customerSnapshot = { + billing: pick(jsonRecord(customer['billing']), [ + 'first_name', + 'last_name', + 'company', + ]), + shipping: pick(jsonRecord(customer['shipping']), [ + 'company', + 'address_1', + 'address_2', + 'city', + 'state', + 'postcode', + 'country', + ]), + }; + const lineItemsSnapshot = Array.isArray(order.lineItemsSnapshot) + ? order.lineItemsSnapshot.map((line) => + pick(jsonRecord(line), ['name', 'quantity', 'total']) + ) + : []; + if ( + JSON.stringify(order.customerSnapshot) === + JSON.stringify(customerSnapshot) && + JSON.stringify(order.lineItemsSnapshot) === + JSON.stringify(lineItemsSnapshot) + ) + continue; + changedOrders++; + if (execute) { + const fingerprint = orderProjectionFingerprint({ + wcOrderId: order.wcOrderId, + orderNumber: order.orderNumber, + status: order.status, + currency: order.currency, + totals: order.totals as Prisma.InputJsonObject, + customerSnapshot, + lineItemsSnapshot, + paymentSnapshot: order.paymentSnapshot as Prisma.InputJsonObject, + shippingLinesSnapshot: + order.shippingLinesSnapshot as Prisma.InputJsonArray, + wcCreatedAt: order.wcCreatedAt, + wcModifiedAt: order.wcModifiedAt, + remoteDeletedAt: order.remoteDeletedAt, + }); + const changed = await this.database.order.updateMany({ + where: { + id: order.id, + ...where, + projectionFingerprint: order.projectionFingerprint, + }, + data: { + customerSnapshot, + lineItemsSnapshot, + projectionFingerprint: fingerprint, + }, + }); + if (changed.count !== 1) + throw new Error('historical Order changed during scrub'); + } + } + } + return { changedEvents, changedOrders, executed: execute }; + } + + async erase(target: PilotTarget): Promise { + const store = await this.resolve(target); + if (store.status !== StoreStatus.DISCONNECTED || !store.deletedAt) { + throw new Error('privacy erasure requires verified disconnect first'); + } + const where = { tenantId: target.tenantId, storeId: target.storeId }; + await this.database.$transaction(async (transaction) => { + await transaction.webhookEvent.updateMany({ + where, + data: { payload: {} }, + }); + await transaction.order.updateMany({ + where, + data: { + orderNumber: 'erased', + totals: {}, + customerSnapshot: {}, + lineItemsSnapshot: [], + paymentSnapshot: {}, + shippingLinesSnapshot: [], + projectionFingerprint: '0'.repeat(64), + }, + }); + await transaction.inventoryItem.updateMany({ + where, + data: { + displayName: 'Erased product', + sku: null, + variationContext: [], + }, + }); + await transaction.telegramCallbackReference.updateMany({ + where, + data: { noteBodyEncrypted: null, noteContentFingerprint: null }, + }); + await transaction.telegramSearchReference.updateMany({ + where, + data: { queryEncrypted: null }, + }); + await transaction.telegramOrderNoteAction.updateMany({ + where, + data: { contentFingerprint: '0'.repeat(64), result: Prisma.DbNull }, + }); + await transaction.telegramOrderStatusWrite.updateMany({ + where, + data: { result: Prisma.DbNull }, + }); + const changed = await transaction.store.updateMany({ + where: { + id: target.storeId, + tenantId: target.tenantId, + baseUrl: target.baseUrl, + status: StoreStatus.DISCONNECTED, + deletedAt: { not: null }, + }, + data: { + name: 'Erased store', + baseUrl: `https://erased.invalid/${target.storeId}`, + }, + }); + if (changed.count !== 1) + throw new Error('privacy target changed during erasure'); + await transaction.auditLog.create({ + data: { + id: `aud_${randomUUID()}`, + tenantId: target.tenantId, + userId: null, + action: 'privacy.store_erased', + entityType: 'Store', + entityId: target.storeId, + metadata: { scope: 'operator', preserved: 'security-audit' }, + }, + }); + }); + } +} + +function jsonRecord(value: unknown): Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function pick(record: Record, fields: string[]) { + const value: Record = {}; + for (const field of fields) { + const candidate = record[field]; + if ( + typeof candidate === 'string' || + (typeof candidate === 'number' && Number.isFinite(candidate)) + ) { + value[field] = candidate; + } + } + return value; +} diff --git a/backend/src/privacy/pilot-retention.service.spec.ts b/backend/src/privacy/pilot-retention.service.spec.ts new file mode 100644 index 0000000..ac35c5e --- /dev/null +++ b/backend/src/privacy/pilot-retention.service.spec.ts @@ -0,0 +1,77 @@ +import { describe, expect, it, jest } from '@jest/globals'; + +import type { StructuredLoggerService } from '../common/logging/structured-logger.service'; +import type { ApplicationConfigService } from '../config/application-config.service'; +import type { PrismaService } from '../prisma/prisma.service'; +import { + COMPLETED_WEBHOOK_PAYLOAD_RETENTION_DAYS, + FAILED_WEBHOOK_PAYLOAD_RETENTION_DAYS, + PilotRetentionService, + SECURITY_AUDIT_RETENTION_DAYS, +} from './pilot-retention.service'; + +describe('pilot PostgreSQL retention', () => { + it('uses fixed cutoffs and leaves queued or active webhook payloads untouched', async () => { + const deleteMany = jest.fn(async () => ({ count: 0 })); + const updateMany = jest.fn(async () => ({ count: 0 })); + const webhookUpdate = jest.fn(async () => ({ count: 0 })); + const database = { + telegramLinkToken: { deleteMany }, + store: { updateMany }, + telegramCallbackReference: { updateMany }, + telegramSearchReference: { updateMany }, + webhookEvent: { updateMany: webhookUpdate }, + auditLog: { deleteMany }, + } as unknown as PrismaService; + const service = new PilotRetentionService( + database, + { app: { nodeEnv: 'test' } } as ApplicationConfigService, + { error: jest.fn() } as unknown as StructuredLoggerService + ); + const now = new Date('2026-10-05T12:00:00.000Z'); + + await service.sweep(now); + + expect(database.webhookEvent.updateMany).toHaveBeenCalledTimes(2); + expect(database.webhookEvent.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: { + status: 'COMPLETED', + completedAt: { + lt: new Date( + now.getTime() - + COMPLETED_WEBHOOK_PAYLOAD_RETENTION_DAYS * 86400000 + ), + }, + }, + data: { payload: {} }, + }) + ); + expect(database.webhookEvent.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: { + status: 'FAILED', + failedAt: { + lt: new Date( + now.getTime() - FAILED_WEBHOOK_PAYLOAD_RETENTION_DAYS * 86400000 + ), + }, + }, + }) + ); + expect(database.auditLog.deleteMany).toHaveBeenCalledWith({ + where: { + createdAt: { + lt: new Date( + now.getTime() - SECURITY_AUDIT_RETENTION_DAYS * 86400000 + ), + }, + }, + }); + expect(database.telegramCallbackReference.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + data: { noteBodyEncrypted: null, noteContentFingerprint: null }, + }) + ); + }); +}); diff --git a/backend/src/privacy/pilot-retention.service.ts b/backend/src/privacy/pilot-retention.service.ts new file mode 100644 index 0000000..3d2728e --- /dev/null +++ b/backend/src/privacy/pilot-retention.service.ts @@ -0,0 +1,122 @@ +import { + Injectable, + type OnApplicationShutdown, + type OnModuleInit, +} from '@nestjs/common'; +import { WebhookEventStatus } from '@prisma/client'; + +import { StructuredLoggerService } from '../common/logging/structured-logger.service'; +import { ApplicationConfigService } from '../config/application-config.service'; +import { PrismaService } from '../prisma/prisma.service'; + +const DAY = 24 * 60 * 60 * 1_000; +export const TRANSIENT_TOKEN_RETENTION_DAYS = 1; +export const COMPLETED_WEBHOOK_PAYLOAD_RETENTION_DAYS = 30; +export const FAILED_WEBHOOK_PAYLOAD_RETENTION_DAYS = 90; +export const SECURITY_AUDIT_RETENTION_DAYS = 365; +const RETENTION_INTERVAL_MS = DAY; + +@Injectable() +export class PilotRetentionService + implements OnModuleInit, OnApplicationShutdown +{ + private timer?: NodeJS.Timeout; + private initialTimer?: NodeJS.Timeout; + private running = false; + + constructor( + private readonly database: PrismaService, + private readonly configuration: ApplicationConfigService, + private readonly logger: StructuredLoggerService + ) {} + + onModuleInit(): void { + if (this.configuration.app.nodeEnv === 'test') return; + this.initialTimer = setTimeout(() => this.runSweep(), 60_000); + this.initialTimer.unref(); + this.timer = setInterval(() => this.runSweep(), RETENTION_INTERVAL_MS); + this.timer.unref(); + } + + private runSweep(): void { + void this.sweep().catch(() => { + this.logger.error( + 'Pilot PostgreSQL retention sweep failed', + { policy: 'ppr2' }, + PilotRetentionService.name + ); + }); + } + + async sweep(now = new Date()): Promise { + if (this.running) return; + this.running = true; + try { + const transientCutoff = new Date( + now.getTime() - TRANSIENT_TOKEN_RETENTION_DAYS * DAY + ); + await this.database.telegramLinkToken.deleteMany({ + where: { expiresAt: { lt: transientCutoff } }, + }); + await this.database.store.updateMany({ + where: { registrationTokenExpiresAt: { lt: transientCutoff } }, + data: { + registrationTokenHash: null, + registrationTokenExpiresAt: null, + }, + }); + await this.database.telegramCallbackReference.updateMany({ + where: { + expiresAt: { lt: transientCutoff }, + noteBodyEncrypted: { not: null }, + }, + data: { noteBodyEncrypted: null, noteContentFingerprint: null }, + }); + await this.database.telegramSearchReference.updateMany({ + where: { + expiresAt: { lt: transientCutoff }, + queryEncrypted: { not: null }, + }, + data: { queryEncrypted: null }, + }); + await this.database.webhookEvent.updateMany({ + where: { + status: WebhookEventStatus.COMPLETED, + completedAt: { + lt: new Date( + now.getTime() - COMPLETED_WEBHOOK_PAYLOAD_RETENTION_DAYS * DAY + ), + }, + }, + data: { payload: {} }, + }); + await this.database.webhookEvent.updateMany({ + where: { + status: WebhookEventStatus.FAILED, + failedAt: { + lt: new Date( + now.getTime() - FAILED_WEBHOOK_PAYLOAD_RETENTION_DAYS * DAY + ), + }, + }, + data: { payload: {} }, + }); + await this.database.auditLog.deleteMany({ + where: { + createdAt: { + lt: new Date(now.getTime() - SECURITY_AUDIT_RETENTION_DAYS * DAY), + }, + }, + }); + } finally { + this.running = false; + } + } + + onApplicationShutdown(): void { + if (this.initialTimer) clearTimeout(this.initialTimer); + this.initialTimer = undefined; + if (this.timer) clearInterval(this.timer); + this.timer = undefined; + } +} diff --git a/backend/src/privacy/privacy-retention.module.ts b/backend/src/privacy/privacy-retention.module.ts new file mode 100644 index 0000000..968e749 --- /dev/null +++ b/backend/src/privacy/privacy-retention.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; + +import { StructuredLoggingModule } from '../common/logging/structured-logging.module'; +import { ApplicationConfigModule } from '../config/application-config.module'; +import { PrismaModule } from '../prisma/prisma.module'; +import { PilotRetentionService } from './pilot-retention.service'; + +@Module({ + imports: [PrismaModule, ApplicationConfigModule, StructuredLoggingModule], + providers: [PilotRetentionService], +}) +export class PrivacyRetentionModule {} diff --git a/backend/src/privacy/woo-webhook-cleanup.spec.ts b/backend/src/privacy/woo-webhook-cleanup.spec.ts new file mode 100644 index 0000000..28fb1ea --- /dev/null +++ b/backend/src/privacy/woo-webhook-cleanup.spec.ts @@ -0,0 +1,68 @@ +import { describe, expect, it, jest } from '@jest/globals'; + +import { removeOwnedWooWebhooks } from './woo-webhook-cleanup'; + +const input = { + baseUrl: 'https://shop.example.test/', + endpointKey: 'whk_target', + consumerKey: 'fixture-key', + consumerSecret: 'fixture-secret', +}; + +describe('WooCommerce connector webhook cleanup', () => { + it('deletes only exact connector-owned hooks and makes no unrelated request', async () => { + const request = jest.fn(async (_url, options) => { + if (options?.method === 'DELETE') { + return { ok: true } as Response; + } + return { + ok: true, + json: async () => [ + { + id: 10, + topic: 'order.created', + name: 'WCTM Connector: order.created', + delivery_url: + 'https://wctm.walterbyte.com/api/webhooks/woocommerce/whk_target', + }, + { + id: 11, + topic: 'order.created', + name: 'Accounting', + delivery_url: + 'https://wctm.walterbyte.com/api/webhooks/woocommerce/whk_target', + }, + { + id: 12, + topic: 'order.updated', + name: 'WCTM Connector: order.updated', + delivery_url: + 'https://wctm.walterbyte.com/api/webhooks/woocommerce/other', + }, + ], + } as Response; + }); + + await expect(removeOwnedWooWebhooks(input, request)).resolves.toEqual({ + complete: true, + removed: 1, + }); + expect(request).toHaveBeenCalledTimes(2); + expect(String(request.mock.calls[1]?.[0])).toContain( + '/webhooks/10?force=true' + ); + expect(String(request.mock.calls[1]?.[0])).not.toContain('11'); + expect(String(request.mock.calls[1]?.[0])).not.toContain('12'); + }); + + it('reports manual cleanup when WooCommerce is unavailable', async () => { + const request = jest.fn(async () => { + throw new Error('fixture network error'); + }); + await expect(removeOwnedWooWebhooks(input, request)).resolves.toEqual({ + complete: false, + removed: 0, + }); + expect(request).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/privacy/woo-webhook-cleanup.ts b/backend/src/privacy/woo-webhook-cleanup.ts new file mode 100644 index 0000000..bf8d1d1 --- /dev/null +++ b/backend/src/privacy/woo-webhook-cleanup.ts @@ -0,0 +1,82 @@ +const TOPICS = new Set([ + 'order.created', + 'order.updated', + 'order.deleted', + 'order.restored', + 'product.created', + 'product.updated', + 'product.deleted', + 'product.restored', +]); + +export interface RemoteWebhookCleanupInput { + baseUrl: string; + endpointKey: string | null; + consumerKey: string; + consumerSecret: string; +} + +/** Delete only hooks with the Store's exact WCTM route, name and topic. */ +export async function removeOwnedWooWebhooks( + input: RemoteWebhookCleanupInput, + request: typeof fetch = fetch +): Promise<{ complete: boolean; removed: number }> { + if (!input.endpointKey) return { complete: true, removed: 0 }; + const authorization = `Basic ${Buffer.from(`${input.consumerKey}:${input.consumerSecret}`).toString('base64')}`; + const base = input.baseUrl.endsWith('/') + ? input.baseUrl + : `${input.baseUrl}/`; + let removed = 0; + try { + for (let page = 1; page <= 10; page++) { + const listUrl = new URL( + `wp-json/wc/v3/webhooks?per_page=100&page=${page}`, + base + ); + const response = await request(listUrl, { + headers: { Authorization: authorization }, + redirect: 'error', + signal: AbortSignal.timeout(5_000), + }); + if (!response.ok) return { complete: false, removed }; + const hooks: unknown = await response.json(); + if (!Array.isArray(hooks)) return { complete: false, removed }; + for (const candidate of hooks) { + if (!candidate || typeof candidate !== 'object') continue; + const hook = candidate as Record; + if ( + typeof hook['topic'] !== 'string' || + !TOPICS.has(hook['topic']) || + hook['name'] !== `WCTM Connector: ${hook['topic']}` || + typeof hook['delivery_url'] !== 'string' || + !Number.isSafeInteger(hook['id']) || + Number(hook['id']) <= 0 + ) + continue; + let path: string; + try { + path = new URL(hook['delivery_url']).pathname; + } catch { + continue; + } + if (path !== `/api/webhooks/woocommerce/${input.endpointKey}`) continue; + const deleteUrl = new URL( + `wp-json/wc/v3/webhooks/${hook['id']}?force=true`, + base + ); + const deleted = await request(deleteUrl, { + method: 'DELETE', + headers: { Authorization: authorization }, + redirect: 'error', + signal: AbortSignal.timeout(5_000), + }); + if (!deleted.ok) return { complete: false, removed }; + removed++; + } + if (hooks.length < 100) return { complete: true, removed }; + } + } catch { + return { complete: false, removed }; + } + return { complete: false, removed }; +} diff --git a/backend/src/queue/queue-runtime.service.ts b/backend/src/queue/queue-runtime.service.ts index 6ecbdbc..d44dc67 100644 --- a/backend/src/queue/queue-runtime.service.ts +++ b/backend/src/queue/queue-runtime.service.ts @@ -16,6 +16,10 @@ import { ORDER_NOTIFICATION_JOB_NAME, REFERENCE_JOB_ATTEMPTS, REFERENCE_JOB_BACKOFF_MS, + COMPLETED_JOB_RETENTION_SECONDS, + FAILED_JOB_RETENTION_SECONDS, + QUEUE_RETENTION_SWEEP_MS, + QUEUE_RETENTION_BATCH_SIZE, REFERENCE_JOB_NAME, WOOCOMMERCE_WEBHOOK_JOB_NAME, } from './queue.constants'; @@ -110,6 +114,7 @@ export class QueueRuntimeService OperationsJobResult, OperationsJobName >; + private retentionTimer?: NodeJS.Timeout; constructor( private readonly configuration: ApplicationConfigService, @@ -142,6 +147,8 @@ export class QueueRuntimeService type: 'exponential', delay: REFERENCE_JOB_BACKOFF_MS, }, + removeOnComplete: { age: COMPLETED_JOB_RETENTION_SECONDS }, + removeOnFail: { age: FAILED_JOB_RETENTION_SECONDS }, }, }); this.worker = new Worker( @@ -165,6 +172,31 @@ export class QueueRuntimeService QueueRuntimeService.name ); }); + this.retentionTimer = setInterval(() => { + void this.enforceRetention().catch(() => { + this.logger.error( + 'Operations queue retention sweep failed', + { queue: OPERATIONS_QUEUE_NAME }, + QueueRuntimeService.name + ); + }); + }, QUEUE_RETENTION_SWEEP_MS); + this.retentionTimer.unref(); + } + + /** BullMQ clean only targets terminal states; waiting and active work is retained. */ + async enforceRetention(): Promise { + const queue = this.requiredQueue(); + await queue.clean( + COMPLETED_JOB_RETENTION_SECONDS * 1_000, + QUEUE_RETENTION_BATCH_SIZE, + 'completed' + ); + await queue.clean( + FAILED_JOB_RETENTION_SECONDS * 1_000, + QUEUE_RETENTION_BATCH_SIZE, + 'failed' + ); } addReferenceJob(data: ReferenceJobData): Promise { @@ -368,6 +400,10 @@ export class QueueRuntimeService } async onApplicationShutdown(): Promise { + if (this.retentionTimer) { + clearInterval(this.retentionTimer); + this.retentionTimer = undefined; + } const worker = this.worker; const queue = this.queue; diff --git a/backend/src/queue/queue.constants.ts b/backend/src/queue/queue.constants.ts index 44ee6d5..edb7f10 100644 --- a/backend/src/queue/queue.constants.ts +++ b/backend/src/queue/queue.constants.ts @@ -2,6 +2,10 @@ export const OPERATIONS_QUEUE_NAME = 'operations'; export const REFERENCE_JOB_NAME = 'reference'; export const REFERENCE_JOB_ATTEMPTS = 3; export const REFERENCE_JOB_BACKOFF_MS = 1_000; +export const COMPLETED_JOB_RETENTION_SECONDS = 24 * 60 * 60; +export const FAILED_JOB_RETENTION_SECONDS = 7 * 24 * 60 * 60; +export const QUEUE_RETENTION_SWEEP_MS = 60 * 60 * 1_000; +export const QUEUE_RETENTION_BATCH_SIZE = 1_000; export const WOOCOMMERCE_WEBHOOK_JOB_NAME = 'woocommerce.webhook.process'; export const ORDER_NOTIFICATION_JOB_NAME = 'telegram.order-notification.send'; export const INVENTORY_BOOTSTRAP_JOB_NAME = 'inventory.bootstrap'; diff --git a/backend/src/queue/queue.spec.ts b/backend/src/queue/queue.spec.ts index b2c8cc1..9212cdf 100644 --- a/backend/src/queue/queue.spec.ts +++ b/backend/src/queue/queue.spec.ts @@ -7,6 +7,9 @@ import type { TenantContextService } from '../tenant/tenant-context.service'; import type { InventoryBootstrapProcessor } from './inventory-bootstrap.processor'; import type { InventoryNotificationProcessor } from './inventory-notification.processor'; import { + COMPLETED_JOB_RETENTION_SECONDS, + FAILED_JOB_RETENTION_SECONDS, + QUEUE_RETENTION_BATCH_SIZE, INVENTORY_BOOTSTRAP_JOB_NAME, REFERENCE_JOB_ATTEMPTS, REFERENCE_JOB_NAME, @@ -63,6 +66,35 @@ const inventoryNotificationProcessor = (): InventoryNotificationProcessor => }) as unknown as InventoryNotificationProcessor; describe('M5 operations queue', () => { + it('cleans only terminal jobs after their pilot retention windows', async () => { + const clean = jest.fn().mockResolvedValue([] as never); + const runtime = new QueueRuntimeService( + { app: { nodeEnv: 'test' } } as ApplicationConfigService, + new ReferenceProcessor(), + webhookProcessor(), + notificationProcessor(), + inventoryBootstrapProcessor(), + inventoryNotificationProcessor(), + { error: jest.fn() } as unknown as StructuredLoggerService + ); + Object.assign(runtime, { queue: { clean } }); + + await runtime.enforceRetention(); + + expect(clean).toHaveBeenNthCalledWith( + 1, + COMPLETED_JOB_RETENTION_SECONDS * 1_000, + QUEUE_RETENTION_BATCH_SIZE, + 'completed' + ); + expect(clean).toHaveBeenNthCalledWith( + 2, + FAILED_JOB_RETENTION_SECONDS * 1_000, + QUEUE_RETENTION_BATCH_SIZE, + 'failed' + ); + expect(clean).toHaveBeenCalledTimes(2); + }); it('enqueues a reference job with tenant identity from server context', async () => { const addReferenceJob = jest.fn().mockResolvedValue({ id: '42' } as never); const runtime = { addReferenceJob } as unknown as QueueRuntimeService; diff --git a/backend/src/queue/woocommerce-webhook.processor.spec.ts b/backend/src/queue/woocommerce-webhook.processor.spec.ts index 913a85f..4045e9c 100644 --- a/backend/src/queue/woocommerce-webhook.processor.spec.ts +++ b/backend/src/queue/woocommerce-webhook.processor.spec.ts @@ -54,7 +54,7 @@ function setup( ) { const event = { id: 'evt_a', - tenantId: 'ten_event_untrusted', + tenantId: 'ten_a', storeId: 'sto_a', topic: 'order.created', payload: { id: 101 }, @@ -148,6 +148,7 @@ function setup( } ); const findUnique = jest.fn(async () => event); + const storeCount = jest.fn(async () => 1); const project = jest.fn(async () => undefined); const schedule = jest.fn(async () => undefined); const projectInventory = jest.fn< @@ -157,6 +158,7 @@ function setup( const processor = new WooCommerceWebhookProcessor( { webhookEvent: { updateMany, findUnique }, + store: { count: storeCount }, } as unknown as PrismaService, { project } as unknown as OrderProjectionService, { @@ -174,10 +176,22 @@ function setup( schedule, scheduleInventory, updateMany, + storeCount, }; } describe('WooCommerce order webhook worker lifecycle', () => { + it('stops a claimed event if the Store was disconnected before processing', async () => { + const fixture = setup(); + fixture.storeCount.mockResolvedValue(0); + + await expect(fixture.processor.process(webhookJob())).rejects.toThrow( + 'Store is disconnected' + ); + expect(fixture.project).not.toHaveBeenCalled(); + expect(fixture.schedule).not.toHaveBeenCalled(); + expect(fixture.event.status).toBe(WebhookEventStatus.FAILED); + }); it.each([ [ 'order.created', diff --git a/backend/src/queue/woocommerce-webhook.processor.ts b/backend/src/queue/woocommerce-webhook.processor.ts index f970920..62d6470 100644 --- a/backend/src/queue/woocommerce-webhook.processor.ts +++ b/backend/src/queue/woocommerce-webhook.processor.ts @@ -1,5 +1,5 @@ import { forwardRef, Inject, Injectable } from '@nestjs/common'; -import { WebhookEventStatus } from '@prisma/client'; +import { StoreStatus, WebhookEventStatus } from '@prisma/client'; import { type Job, UnrecoverableError } from 'bullmq'; import { @@ -126,6 +126,13 @@ export class WooCommerceWebhookProcessor { const claimed = await this.prisma.webhookEvent.updateMany({ where: { id: job.data.webhookEventId, + tenantId: job.data.tenantId, + storeId: job.data.storeId, + store: { + status: StoreStatus.ACTIVE, + deletedAt: null, + tenant: { deletedAt: null }, + }, OR: [ // The persisted job may become runnable before ingestion records its // post-enqueue QUEUED acknowledgement. @@ -156,6 +163,23 @@ export class WooCommerceWebhookProcessor { ); } + const stillActive = await this.prisma.store.count({ + where: { + id: job.data.storeId, + tenantId: job.data.tenantId, + status: StoreStatus.ACTIVE, + deletedAt: null, + tenant: { deletedAt: null }, + }, + }); + if (stillActive !== 1) { + await this.failClaimed(event.id, claimedAt, { + category: 'unexpected', + message: 'store-disconnected', + }); + throw new UnrecoverableError('Store is disconnected'); + } + try { if (event.topic.startsWith('order.')) { await this.orderProjection.project(event); From e05a11cb6013d02b97c0433610a161e7c50b5cd6 Mon Sep 17 00:00:00 2001 From: walter Date: Mon, 5 Oct 2026 23:16:02 +0330 Subject: [PATCH 3/6] feat(plugin): encrypt connector credentials --- .../wordpress-plugin-reconciliation.php | 29 ++++-- backend/test/wordpress-plugin.test.cjs | 9 +- wp-content/plugins/wc-telegram-connector.php | 93 +++++++++++++++++-- 3 files changed, 115 insertions(+), 16 deletions(-) diff --git a/backend/test/fixtures/wordpress-plugin-reconciliation.php b/backend/test/fixtures/wordpress-plugin-reconciliation.php index 9f2ffde..80ce028 100644 --- a/backend/test/fixtures/wordpress-plugin-reconciliation.php +++ b/backend/test/fixtures/wordpress-plugin-reconciliation.php @@ -4,11 +4,13 @@ define('ABSPATH', __DIR__); define('WC_TELEGRAM_CONNECTOR_API_BASE_URL', 'https://unused.example.com'); +define('AUTH_KEY', str_repeat('A', 48)); +define('SECURE_AUTH_KEY', str_repeat('B', 48)); $GLOBALS['wctm_options'] = array( - 'wc_telegram_connector_plugin_credential' => 'plg_' . str_repeat('p', 40), + 'wc_telegram_connector_plugin_credential' => 'plg_' . str_repeat('p', 43), 'wc_telegram_connector_store_id' => 'sto_fixture', - 'wc_telegram_connector_webhook_secret' => 'persisted-m8-secret-' . str_repeat('s', 32), + 'wc_telegram_connector_webhook_secret' => str_repeat('s', 43), 'wc_telegram_connector_webhook_endpoint_key' => 'whk_' . str_repeat('e', 32), ); $GLOBALS['wctm_runtime_base_url'] = 'https://connector.wctm.walterbyte.com'; @@ -78,7 +80,7 @@ function wp_safe_remote_post(string $url, array $args): array ++$GLOBALS['wctm_health_calls']; $expected_url = $GLOBALS['wctm_runtime_base_url'] . '/api/webhooks/woocommerce/' . $GLOBALS['wctm_options']['wc_telegram_connector_webhook_endpoint_key']; - $expected_secret = $GLOBALS['wctm_options']['wc_telegram_connector_webhook_secret']; + $expected_secret = wc_telegram_connector_read_secret('webhook_secret'); foreach ($GLOBALS['wctm_topics'] as $topic) { $matches = array_filter( $GLOBALS['wctm_webhooks'], @@ -249,6 +251,13 @@ function wc_get_webhook(int $id): ?WC_Webhook require dirname(__DIR__, 3) . '/wp-content/plugins/wc-telegram-connector.php'; +if (wc_telegram_connector_read_secret('plugin_credential') !== 'plg_' . str_repeat('p', 43) || + wc_telegram_connector_read_secret('webhook_secret') !== str_repeat('s', 43) || + !str_starts_with($GLOBALS['wctm_options']['wc_telegram_connector_plugin_credential'], 'wctm1:') || + !str_starts_with($GLOBALS['wctm_options']['wc_telegram_connector_webhook_secret'], 'wctm1:')) { + throw new RuntimeException('Plaintext credentials did not migrate to authenticated encryption'); +} + if (count(wc_telegram_connector_load_webhooks()) !== 28) { throw new RuntimeException('Connector loader did not enumerate through the WooCommerce proxy'); } @@ -259,7 +268,7 @@ function wc_get_webhook(int $id): ?WC_Webhook $expected_url = 'https://connector.wctm.walterbyte.com/api/webhooks/woocommerce/' . $GLOBALS['wctm_options']['wc_telegram_connector_webhook_endpoint_key']; -$expected_secret = $GLOBALS['wctm_options']['wc_telegram_connector_webhook_secret']; +$expected_secret = wc_telegram_connector_read_secret('webhook_secret'); foreach ($GLOBALS['wctm_topics'] as $topic) { $matches = array_filter( $GLOBALS['wctm_webhooks'], @@ -291,10 +300,18 @@ function wc_get_webhook(int $id): ?WC_Webhook throw new RuntimeException('Second Retry changed webhook count or identity'); } +$saved_credential = $GLOBALS['wctm_options']['wc_telegram_connector_plugin_credential']; +$GLOBALS['wctm_options']['wc_telegram_connector_plugin_credential'] = substr($saved_credential, 0, -2) . 'AA'; +if (wc_telegram_connector_read_secret('plugin_credential') !== '' || + wc_telegram_connector_has_material()) { + throw new RuntimeException('Tampered ciphertext did not fail closed'); +} +$GLOBALS['wctm_options']['wc_telegram_connector_plugin_credential'] = $saved_credential; + $before_privacy_calls = $GLOBALS['wctm_health_calls']; wc_telegram_connector_privacy_policy_content(); -if (!str_contains($GLOBALS['wctm_privacy_text'], 'Telegram is an external platform') || - !str_contains($GLOBALS['wctm_privacy_text'], 'customer names')) { +if (!str_contains($GLOBALS['wctm_privacy_text'], 'Telegram, an external platform') || + !str_contains($GLOBALS['wctm_privacy_text'], 'customer name')) { throw new RuntimeException('Privacy policy text does not disclose external data flow'); } wc_telegram_connector_deactivate(); diff --git a/backend/test/wordpress-plugin.test.cjs b/backend/test/wordpress-plugin.test.cjs index f533864..1d7a383 100644 --- a/backend/test/wordpress-plugin.test.cjs +++ b/backend/test/wordpress-plugin.test.cjs @@ -41,13 +41,20 @@ test('connector stores sensitive material with autoload disabled and never rende 'webhook_secret', 'webhook_endpoint_key', ]) { - assert.match(plugin, new RegExp(`store_option\\('${option}'`)); + assert.match( + plugin, + new RegExp( + `${option === 'webhook_endpoint_key' ? 'store_option' : 'store_secret'}\\('${option}'` + ) + ); } assert.doesNotMatch( plugin, /echo\s+[^;]*(?:plugin_credential|webhook_secret|endpoint_key)/i ); assert.doesNotMatch(plugin, /error_log|trigger_error/); + assert.match(plugin, /openssl_encrypt\(\$value, 'aes-256-gcm'/); + assert.match(plugin, /hash_hkdf\('sha256', AUTH_KEY \./); }); test('connector installs and verifies every required order and inventory webhook before health confirmation', () => { diff --git a/wp-content/plugins/wc-telegram-connector.php b/wp-content/plugins/wc-telegram-connector.php index 2a2c1e3..f844f3b 100644 --- a/wp-content/plugins/wc-telegram-connector.php +++ b/wp-content/plugins/wc-telegram-connector.php @@ -19,6 +19,7 @@ define('WC_TELEGRAM_CONNECTOR_FILE', __FILE__); define('WC_TELEGRAM_CONNECTOR_MENU_SLUG', 'wc-telegram-connector'); define('WC_TELEGRAM_CONNECTOR_OPTION_PREFIX', 'wc_telegram_connector_'); +define('WC_TELEGRAM_CONNECTOR_SECRET_PREFIX', 'wctm1:'); if (!defined('WC_TELEGRAM_CONNECTOR_API_BASE_URL')) { define('WC_TELEGRAM_CONNECTOR_API_BASE_URL', ''); @@ -61,7 +62,7 @@ function wc_telegram_connector_privacy_policy_content(): void wp_add_privacy_policy_content( __('WCTM — Telegram Store Manager for WooCommerce', 'wc-telegram-connector'), '

' . esc_html__( - 'When this store is connected to WCTM by Walterbyte, WooCommerce sends order and product webhook data to the external WCTM service. Order data may include customer names, contact details, addresses, items, totals and notes. WCTM stores operational records and may send selected order and inventory details to authorized store managers through Telegram. Telegram is an external platform. The connector stores its store identifier and connection/webhook credentials in WordPress options and WooCommerce webhooks. Deactivation does not disconnect the store or erase WCTM service data. Contact the store operator for privacy and deletion requests.', + 'When this store is connected to WCTM by Walterbyte, WooCommerce sends order and product webhook data to the external WCTM service. WCTM stores selected order identity, totals, customer name and shipping address, item and stock fields needed for store management; it discards webhook notes, customer phone/email and arbitrary metadata from its new persistence path. Earlier records and backups may retain fuller data until operator cleanup or expiry. WCTM may send selected order and inventory details to authorized store managers through Telegram, an external platform. The connector stores its store identifier and encrypted WCTM-owned connection/webhook credentials in WordPress options; WooCommerce also stores its own webhook secret. Deactivation does not disconnect the store or erase WCTM service data. Contact the store operator for privacy and deletion requests.', 'wc-telegram-connector' ) . '

' ); @@ -71,7 +72,7 @@ function wc_telegram_connector_privacy_policy_content(): void function wc_telegram_connector_uninstall(): void { $endpoint_key = wc_telegram_connector_read_option('webhook_endpoint_key'); - $secret = wc_telegram_connector_read_option('webhook_secret'); + $secret = wc_telegram_connector_read_secret('webhook_secret'); // Match the saved route and secret; never remove another integration's hook. if (is_string($endpoint_key) && $endpoint_key !== '' && is_string($secret) && $secret !== '') { @@ -126,6 +127,78 @@ function wc_telegram_connector_read_option(string $name, $default = '') return get_option(wc_telegram_connector_option_name($name), $default); } +/** Derive a connector-only key from host-side wp-config.php constants, never wp_options. */ +function wc_telegram_connector_secret_key(): ?string +{ + if (!defined('AUTH_KEY') || !defined('SECURE_AUTH_KEY') || + !is_string(AUTH_KEY) || !is_string(SECURE_AUTH_KEY) || + strlen(AUTH_KEY) < 32 || strlen(SECURE_AUTH_KEY) < 32 || + !function_exists('hash_hkdf') || !function_exists('openssl_encrypt')) { + return null; + } + + return hash_hkdf('sha256', AUTH_KEY . SECURE_AUTH_KEY, 32, + 'WCTM WordPress connector option encryption v1'); +} + +function wc_telegram_connector_store_secret(string $name, string $value): bool +{ + if (!in_array($name, array('plugin_credential', 'webhook_secret'), true) || + $value === '' || !function_exists('openssl_encrypt')) { + return false; + } + $key = wc_telegram_connector_secret_key(); + if ($key === null) { + return false; + } + try { + $nonce = random_bytes(12); + } catch (Throwable $error) { + return false; + } + $tag = ''; + $ciphertext = openssl_encrypt($value, 'aes-256-gcm', $key, OPENSSL_RAW_DATA, + $nonce, $tag, 'WCTM:' . $name, 16); + if (!is_string($ciphertext) || strlen($tag) !== 16) { + return false; + } + return wc_telegram_connector_store_option($name, + WC_TELEGRAM_CONNECTOR_SECRET_PREFIX . base64_encode($nonce . $tag . $ciphertext)); +} + +/** Legacy plaintext is accepted only in the exact generated credential shape. */ +function wc_telegram_connector_read_secret(string $name): string +{ + if (!in_array($name, array('plugin_credential', 'webhook_secret'), true)) { + return ''; + } + $stored = wc_telegram_connector_read_option($name); + if (!is_string($stored) || $stored === '') { + return ''; + } + if (str_starts_with($stored, WC_TELEGRAM_CONNECTOR_SECRET_PREFIX)) { + $encoded = substr($stored, strlen(WC_TELEGRAM_CONNECTOR_SECRET_PREFIX)); + $bytes = base64_decode($encoded, true); + $key = wc_telegram_connector_secret_key(); + if (!is_string($bytes) || strlen($bytes) < 29 || $key === null || + !function_exists('openssl_decrypt')) { + return ''; + } + $plaintext = openssl_decrypt(substr($bytes, 28), 'aes-256-gcm', $key, + OPENSSL_RAW_DATA, substr($bytes, 0, 12), substr($bytes, 12, 16), + 'WCTM:' . $name); + return is_string($plaintext) ? $plaintext : ''; + } + $pattern = $name === 'plugin_credential' + ? '/\Aplg_[A-Za-z0-9_-]{43}\z/D' + : '/\A[A-Za-z0-9_-]{43}\z/D'; + if (preg_match($pattern, $stored) !== 1 || + !wc_telegram_connector_store_secret($name, $stored)) { + return ''; + } + return $stored; +} + function wc_telegram_connector_api_base_url(): string { $candidate = apply_filters('wc_telegram_connector_api_base_url', WC_TELEGRAM_CONNECTOR_API_BASE_URL); @@ -229,7 +302,9 @@ function wc_telegram_connector_render_admin_page(): void function wc_telegram_connector_has_material(): bool { foreach (array('plugin_credential', 'store_id', 'webhook_secret', 'webhook_endpoint_key') as $name) { - $value = wc_telegram_connector_read_option($name); + $value = in_array($name, array('plugin_credential', 'webhook_secret'), true) + ? wc_telegram_connector_read_secret($name) + : wc_telegram_connector_read_option($name); if (!is_string($value) || $value === '') { return false; } @@ -263,7 +338,7 @@ function wc_telegram_connector_handle_connect(): void $same_store = is_string($store_id) && is_string($existing_store_id) && hash_equals($existing_store_id, $store_id); $webhook_secret = $result['webhookSecret'] ?? - ($same_store ? wc_telegram_connector_read_option('webhook_secret') : null); + ($same_store ? wc_telegram_connector_read_secret('webhook_secret') : null); $endpoint_key = $result['webhookEndpointKey'] ?? ($same_store ? wc_telegram_connector_read_option('webhook_endpoint_key') : null); @@ -274,9 +349,9 @@ function wc_telegram_connector_handle_connect(): void wc_telegram_connector_safe_redirect('registration_error'); } - $persisted = wc_telegram_connector_store_option('plugin_credential', $plugin_credential) && + $persisted = wc_telegram_connector_store_secret('plugin_credential', $plugin_credential) && wc_telegram_connector_store_option('store_id', $store_id) && - wc_telegram_connector_store_option('webhook_secret', $webhook_secret) && + wc_telegram_connector_store_secret('webhook_secret', $webhook_secret) && wc_telegram_connector_store_option('webhook_endpoint_key', $endpoint_key); $plugin_credential = $webhook_secret = $endpoint_key = ''; @@ -424,7 +499,7 @@ function wc_telegram_connector_webhook_has_expected_state( function wc_telegram_connector_install_and_confirm_webhooks(): bool { - $secret = wc_telegram_connector_read_option('webhook_secret'); + $secret = wc_telegram_connector_read_secret('webhook_secret'); $delivery_url = wc_telegram_connector_delivery_url(); if (!is_string($secret) || $secret === '' || $delivery_url === '') { return false; @@ -477,7 +552,7 @@ function wc_telegram_connector_install_and_confirm_webhooks(): bool function wc_telegram_connector_required_webhooks_are_healthy(): bool { $delivery_url = wc_telegram_connector_delivery_url(); - $secret = wc_telegram_connector_read_option('webhook_secret'); + $secret = wc_telegram_connector_read_secret('webhook_secret'); if ($delivery_url === '' || !is_string($secret) || $secret === '') { return false; } @@ -502,7 +577,7 @@ function wc_telegram_connector_required_webhooks_are_healthy(): bool function wc_telegram_connector_confirm_health(): bool { $base_url = wc_telegram_connector_api_base_url(); - $credential = wc_telegram_connector_read_option('plugin_credential'); + $credential = wc_telegram_connector_read_secret('plugin_credential'); if ($base_url === '' || !is_string($credential) || $credential === '') { return false; } From 04640032ae174ea32dfad506e2859fb420b3a0fe Mon Sep 17 00:00:00 2001 From: walter Date: Mon, 5 Oct 2026 23:16:16 +0330 Subject: [PATCH 4/6] feat(ops): encrypt off-site backups --- ops/systemd/backup.conf.example | 4 + scripts/ops/archive-erasure-ledger.sh | 60 +++++++++++++++ scripts/ops/backup-crypto.mjs | 100 +++++++++++++++++++++++++ scripts/ops/decrypt-offsite.sh | 66 ++++++++++++++++ scripts/ops/offsite-rclone.sh | 43 ++++++++++- scripts/ops/offsite-retention.sh | 36 +++++++++ scripts/ops/pilot-privacy.sh | 98 ++++++++++++++++++++++++ scripts/ops/restore-erasure-ledger.sh | 57 ++++++++++++++ scripts/ops/scheduled-backup.sh | 8 ++ scripts/ops/select-offsite-expired.mjs | 25 +++++++ scripts/ops/test-backup-crypto.sh | 54 +++++++++++++ scripts/ops/test-fixtures/rclone | 11 ++- scripts/ops/test-offsite-retention.sh | 34 +++++++++ scripts/ops/test-operations.sh | 79 +++++++++++++++++-- scripts/ops/test-pilot-privacy.sh | 65 ++++++++++++++++ scripts/ops/test-scheduled-backup.sh | 13 +++- 16 files changed, 739 insertions(+), 14 deletions(-) create mode 100755 scripts/ops/archive-erasure-ledger.sh create mode 100644 scripts/ops/backup-crypto.mjs create mode 100755 scripts/ops/decrypt-offsite.sh create mode 100755 scripts/ops/offsite-retention.sh create mode 100755 scripts/ops/pilot-privacy.sh create mode 100755 scripts/ops/restore-erasure-ledger.sh create mode 100644 scripts/ops/select-offsite-expired.mjs create mode 100755 scripts/ops/test-backup-crypto.sh create mode 100755 scripts/ops/test-offsite-retention.sh create mode 100755 scripts/ops/test-pilot-privacy.sh diff --git a/ops/systemd/backup.conf.example b/ops/systemd/backup.conf.example index ae98a1c..d07765f 100644 --- a/ops/systemd/backup.conf.example +++ b/ops/systemd/backup.conf.example @@ -1,6 +1,7 @@ # Install as /etc/wctm/backup.conf with root:wctm ownership and mode 0640. WCTM_BACKUP_DIRECTORY=/var/backups/wctm WCTM_BACKUP_RETENTION_COUNT=14 +WCTM_OFFSITE_RETENTION_DAYS=30 # Replace with the reviewed com.docker.compose.project label on the running # PostgreSQL container (normally wctm on /srv/wctm). No database secret is needed. WCTM_COMPOSE_PROJECT=SET_REVIEWED_COMPOSE_PROJECT @@ -9,5 +10,8 @@ WCTM_COMPOSE_PROJECT=SET_REVIEWED_COMPOSE_PROJECT # persist token refreshes. The service allows /var/lib/wctm/rclone for this. # Keep credentials in that separate file, not this env file or the unit. RCLONE_CONFIG=SET_ABSOLUTE_READABLE_RCLONE_CONFIG_PATH +# Required for off-site copy: path only. Keep 32 raw key bytes in a separate +# wctm-owned mode-0600 file outside Git and outside OneDrive. +WCTM_BACKUP_CRYPTO_KEY_FILE=SET_ABSOLUTE_PRIVATE_BACKUP_KEY_PATH # Optional provider-neutral rclone remote. Requires the path above when enabled. # WCTM_OFFSITE_DESTINATION=remote-name:wctm-production diff --git a/scripts/ops/archive-erasure-ledger.sh b/scripts/ops/archive-erasure-ledger.sh new file mode 100755 index 0000000..eb41560 --- /dev/null +++ b/scripts/ops/archive-erasure-ledger.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +if (($# != 2)); then + echo >&2 'usage: archive-erasure-ledger.sh LEDGER RCLONE_DESTINATION' + exit 64 +fi +ledger=$1 +destination=${2%/} +key_file=${WCTM_BACKUP_CRYPTO_KEY_FILE:-} +[[ -f "$ledger" && -s "$ledger" && "$key_file" = /* && -f "$key_file" && -r "$key_file" && + -n "${RCLONE_CONFIG:-}" && -f "$RCLONE_CONFIG" && -r "$RCLONE_CONFIG" ]] || { + echo >&2 'erasure archive refused: protected ledger, key and rclone configuration are required' + exit 66 +} +command -v node >/dev/null 2>&1 && command -v rclone >/dev/null 2>&1 || { + echo >&2 'erasure archive failed: required local tool is unavailable' + exit 69 +} +temporary_dir=$(mktemp -d) +trap 'rm -rf -- "$temporary_dir"' EXIT HUP INT TERM +timestamp=$(date -u '+%Y%m%dT%H%M%SZ') +ledger_sha256=$(sha256sum "$ledger" | awk '{print $1}') +name="erasure-ledger-$timestamp-${ledger_sha256:0:12}.jsonl.enc" +encrypted="$temporary_dir/$name" +crypto_script=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)/backup-crypto.mjs +node "$crypto_script" encrypt "$key_file" "$ledger" "$encrypted" || { + echo >&2 'erasure archive failed: encryption failed' + exit 1 +} +cipher_sha256=$(sha256sum "$encrypted" | awk '{print $1}') +key_id=$(sha256sum "$key_file" | awk '{print substr($1, 1, 16)}') +printf '%s %s\n' "$cipher_sha256" "$name" >"$encrypted.sha256" +printf '{"format":"WCTMBACKUPAESG1","keyId":"%s","ciphertextSha256":"%s","ledgerSha256":"%s"}\n' \ + "$key_id" "$cipher_sha256" "$ledger_sha256" >"$encrypted.json" + +for file in "$encrypted" "$encrypted.sha256" "$encrypted.json"; do + artifact=$(basename "$file") + rclone copyto -- "$file" "$destination/$artifact" + local_size=$(wc -c <"$file" | tr -d ' ') + remote_size=$(rclone size --json "$destination/$artifact" | sed -n 's/.*"bytes":\([0-9][0-9]*\).*/\1/p') + [[ "$remote_size" = "$local_size" ]] || { + echo >&2 'erasure archive failed: remote size mismatch' + exit 1 + } +done +remote="$destination/$name" +remote_sha256='' +if hash_output=$(rclone hashsum sha256 "$remote" 2>/dev/null); then + remote_sha256=$(printf '%s\n' "$hash_output" | awk 'NR == 1 {print tolower($1)}') +fi +if [[ ! "$remote_sha256" =~ ^[0-9a-f]{64}$ ]]; then + remote_sha256=$(rclone cat "$remote" | sha256sum | awk '{print $1}') +fi +[[ "$remote_sha256" = "$cipher_sha256" ]] || { + echo >&2 'erasure archive failed: remote content SHA-256 mismatch' + exit 1 +} +echo "erasure archive: PASS file=$name content_sha256=verified" diff --git a/scripts/ops/backup-crypto.mjs b/scripts/ops/backup-crypto.mjs new file mode 100644 index 0000000..e45cb69 --- /dev/null +++ b/scripts/ops/backup-crypto.mjs @@ -0,0 +1,100 @@ +#!/usr/bin/env node +import { createCipheriv, createDecipheriv, hkdfSync, randomBytes } from 'node:crypto'; +import { createReadStream, createWriteStream } from 'node:fs'; +import { appendFile, link, open, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { pipeline } from 'node:stream/promises'; + +const MAGIC = Buffer.from('WCTMBACKUPAESG1\n', 'ascii'); +const NONCE_BYTES = 12; +const TAG_BYTES = 16; + +async function loadKey(path) { + const details = await stat(path); + if (!details.isFile() || (details.mode & 0o077) !== 0) { + throw new Error('backup encryption key must be a private regular file'); + } + const material = await readFile(path); + if (material.length !== 32) { + throw new Error('backup encryption key must contain exactly 32 bytes'); + } + return Buffer.from( + hkdfSync('sha256', material, 'WCTM backup v1', 'off-site database dump', 32) + ); +} + +async function transform(mode, keyFile, input, output) { + if (input === output) { + throw new Error('backup input and output must differ'); + } + const key = await loadKey(keyFile); + const temporary = `${output}.partial-${process.pid}`; + let committed = false; + + try { + if (mode === 'encrypt') { + const nonce = randomBytes(NONCE_BYTES); + const cipher = createCipheriv('aes-256-gcm', key, nonce); + cipher.setAAD(MAGIC); + await writeFile(temporary, Buffer.concat([MAGIC, nonce]), { + flag: 'wx', + mode: 0o600, + }); + await pipeline( + createReadStream(input), + cipher, + createWriteStream(temporary, { flags: 'a', mode: 0o600 }) + ); + await appendFile(temporary, cipher.getAuthTag()); + } else if (mode === 'decrypt') { + const details = await stat(input); + const minimum = MAGIC.length + NONCE_BYTES + TAG_BYTES; + if (!details.isFile() || details.size < minimum) { + throw new Error('encrypted backup format is invalid'); + } + const handle = await open(input, 'r'); + const header = Buffer.alloc(MAGIC.length + NONCE_BYTES); + const tag = Buffer.alloc(TAG_BYTES); + try { + await handle.read(header, 0, header.length, 0); + await handle.read(tag, 0, tag.length, details.size - TAG_BYTES); + } finally { + await handle.close(); + } + if (!header.subarray(0, MAGIC.length).equals(MAGIC)) { + throw new Error('encrypted backup format is invalid'); + } + const nonce = header.subarray(MAGIC.length); + const decipher = createDecipheriv('aes-256-gcm', key, nonce); + decipher.setAAD(MAGIC); + decipher.setAuthTag(tag); + await pipeline( + createReadStream(input, { + start: MAGIC.length + NONCE_BYTES, + end: details.size - TAG_BYTES - 1, + }), + decipher, + createWriteStream(temporary, { flags: 'wx', mode: 0o600 }) + ); + } else { + throw new Error('expected encrypt or decrypt'); + } + await link(temporary, output); + await rm(temporary); + committed = true; + } finally { + key.fill(0); + if (!committed) { + await rm(temporary, { force: true }); + } + } +} + +if (process.argv.length !== 6) { + process.stderr.write('usage: backup-crypto.mjs encrypt|decrypt KEY_FILE INPUT OUTPUT\n'); + process.exitCode = 64; +} else { + transform(...process.argv.slice(2)).catch(() => { + process.stderr.write('backup encryption/decryption failed; check protected key and input\n'); + process.exitCode = 1; + }); +} diff --git a/scripts/ops/decrypt-offsite.sh b/scripts/ops/decrypt-offsite.sh new file mode 100755 index 0000000..f88c275 --- /dev/null +++ b/scripts/ops/decrypt-offsite.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +if (($# != 3)); then + echo >&2 'usage: decrypt-offsite.sh ENCRYPTED_DUMP METADATA OUTPUT_DIRECTORY' + exit 64 +fi +encrypted=$1 +metadata=$2 +directory=$3 +key_file=${WCTM_BACKUP_CRYPTO_KEY_FILE:-} +[[ -f "$encrypted" && -f "$encrypted.sha256" && -f "$metadata" && -d "$directory" && + "$directory" != '/' && "$key_file" = /* && -f "$key_file" && -r "$key_file" ]] || { + echo >&2 'off-site decrypt refused: encrypted set, protected key and explicit output directory are required' + exit 66 +} +command -v node >/dev/null 2>&1 || { + echo >&2 'off-site decrypt failed: local Node runtime is unavailable' + exit 69 +} +encrypted=$(cd "$(dirname "$encrypted")" && pwd -P)/$(basename "$encrypted") +directory=$(cd "$directory" && pwd -P) +(cd "$(dirname "$encrypted")" && sha256sum --check --status "$(basename "$encrypted.sha256")") || { + echo >&2 'off-site decrypt refused: encrypted content checksum mismatch' + exit 1 +} +encrypted_sha256=$(sha256sum "$encrypted" | awk '{print $1}') +key_id=$(sha256sum "$key_file" | awk '{print substr($1, 1, 16)}') +name=$(basename "$encrypted") +[[ "$name" = wctm-postgres-*.dump.enc && "$(basename "$metadata")" = "$name.json" ]] || { + echo >&2 'off-site decrypt refused: backup set names do not match' + exit 65 +} +expected=$(node -e ' +const fs = require("node:fs"); +const data = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); +if (data.format !== "postgresql-custom" || data.offsiteEncryption?.format !== "WCTMBACKUPAESG1" || + data.offsiteEncryption?.keyId !== process.argv[2] || + data.offsiteEncryption?.ciphertextSha256 !== process.argv[3] || + !/^[0-9a-f]{64}$/.test(data.sha256)) process.exit(1); +process.stdout.write(data.sha256); +' "$metadata" "$key_id" "$encrypted_sha256") || { + echo >&2 'off-site decrypt refused: metadata is invalid' + exit 65 +} +dump="$directory/${name%.enc}" +[[ ! -e "$dump" && ! -e "$dump.sha256" && ! -e "$dump.json" ]] || { + echo >&2 'off-site decrypt refused: output backup set already exists' + exit 73 +} +crypto_script=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)/backup-crypto.mjs +node "$crypto_script" decrypt "$key_file" "$encrypted" "$dump" || { + echo >&2 'off-site decrypt failed: authentication failed' + exit 1 +} +actual=$(sha256sum "$dump" | awk '{print $1}') +if [[ "$actual" != "$expected" ]]; then + rm -f -- "$dump" + echo >&2 'off-site decrypt failed: plaintext checksum mismatch' + exit 1 +fi +printf '%s %s\n' "$expected" "$(basename "$dump")" >"$dump.sha256" +cp -- "$metadata" "$dump.json" +chmod 0600 "$dump" "$dump.sha256" "$dump.json" +echo "off-site decrypt: PASS file=$(basename "$dump")" diff --git a/scripts/ops/offsite-rclone.sh b/scripts/ops/offsite-rclone.sh index 60525e8..c20fabe 100755 --- a/scripts/ops/offsite-rclone.sh +++ b/scripts/ops/offsite-rclone.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +umask 077 if (($# != 4)); then echo >&2 'usage: offsite-rclone.sh DUMP CHECKSUM METADATA RCLONE_DESTINATION' @@ -21,6 +22,15 @@ command -v rclone >/dev/null 2>&1 || { echo >&2 'off-site copy failed: rclone is unavailable' exit 69 } +command -v node >/dev/null 2>&1 || { + echo >&2 'off-site copy failed: local Node runtime is unavailable for encryption' + exit 69 +} +key_file=${WCTM_BACKUP_CRYPTO_KEY_FILE:-} +[[ "$key_file" = /* && -f "$key_file" && -r "$key_file" ]] || { + echo >&2 'off-site copy refused: protected backup encryption key file is required' + exit 66 +} read -r expected_dump_sha256 expected_dump_name checksum_extra <"$checksum" [[ "$expected_dump_sha256" =~ ^[0-9a-fA-F]{64}$ && @@ -36,7 +46,32 @@ local_dump_sha256=$(sha256sum "$dump" | awk '{print $1}') exit 1 } -for file in "$dump" "$checksum" "$metadata"; do +crypto_script=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)/backup-crypto.mjs +temporary_dir=$(mktemp -d "$(dirname "$dump")/.wctm-offsite.XXXXXXXX") +trap 'rm -rf -- "$temporary_dir"' EXIT HUP INT TERM +encrypted="$temporary_dir/$(basename "$dump").enc" +node "$crypto_script" encrypt "$key_file" "$dump" "$encrypted" || { + echo >&2 'off-site copy failed: local backup encryption failed' + exit 1 +} +encrypted_checksum="$encrypted.sha256" +encrypted_sha256=$(sha256sum "$encrypted" | awk '{print $1}') +printf '%s %s\n' "$encrypted_sha256" "$(basename "$encrypted")" >"$encrypted_checksum" +encrypted_metadata="$encrypted.json" +key_id=$(sha256sum "$key_file" | awk '{print substr($1, 1, 16)}') +node -e ' +const fs = require("node:fs"); +const [source, target, keyId, ciphertextSha256] = process.argv.slice(1); +const record = JSON.parse(fs.readFileSync(source, "utf8")); +if (record.format !== "postgresql-custom" || !/^[0-9a-f]{64}$/.test(record.sha256)) process.exit(1); +record.offsiteEncryption = { format: "WCTMBACKUPAESG1", keyId, ciphertextSha256 }; +fs.writeFileSync(target, JSON.stringify(record) + "\n", { flag: "wx", mode: 0o600 }); +' "$metadata" "$encrypted_metadata" "$key_id" "$encrypted_sha256" || { + echo >&2 'off-site copy failed: encrypted metadata preparation failed' + exit 1 +} + +for file in "$encrypted" "$encrypted_checksum" "$encrypted_metadata"; do name=$(basename "$file") rclone copyto -- "$file" "$destination/$name" local_size=$(wc -c <"$file" | tr -d ' ') @@ -47,7 +82,7 @@ for file in "$dump" "$checksum" "$metadata"; do } done -remote_dump="$destination/$(basename "$dump")" +remote_dump="$destination/$(basename "$encrypted")" remote_dump_sha256='' verification_method='' if native_hash_output=$(rclone hashsum sha256 "$remote_dump" 2>/dev/null); then @@ -70,8 +105,8 @@ if [[ -z "$remote_dump_sha256" ]]; then verification_method='streamed-sha256' fi -[[ "$remote_dump_sha256" = "$expected_dump_sha256" ]] || { - echo >&2 'off-site copy failed: remote dump SHA-256 mismatch' +[[ "$remote_dump_sha256" = "$encrypted_sha256" ]] || { + echo >&2 'off-site copy failed: remote encrypted dump SHA-256 mismatch' exit 1 } diff --git a/scripts/ops/offsite-retention.sh b/scripts/ops/offsite-retention.sh new file mode 100755 index 0000000..672bdc5 --- /dev/null +++ b/scripts/ops/offsite-retention.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# != 2)); then + echo >&2 'usage: offsite-retention.sh RCLONE_DESTINATION MAX_AGE_DAYS' + exit 64 +fi +destination=${1%/} +days=$2 +[[ "$days" =~ ^[0-9]+$ ]] && ((days >= 1)) || { + echo >&2 'off-site retention refused: max age must be positive whole days' + exit 64 +} +command -v rclone >/dev/null 2>&1 && command -v node >/dev/null 2>&1 || { + echo >&2 'off-site retention failed: required local tool is unavailable' + exit 69 +} +selector=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)/select-offsite-expired.mjs +listing=$(rclone lsf --files-only "$destination") || { + echo >&2 'off-site retention failed: remote listing unavailable' + exit 1 +} +selected=$(printf '%s\n' "$listing" | node "$selector" "$days") || { + echo >&2 'off-site retention failed: selection unavailable' + exit 1 +} +removed=0 +while IFS= read -r name; do + [[ -n "$name" ]] || continue + rclone deletefile "$destination/$name" || { + echo >&2 'off-site retention failed: selected artifact removal failed' + exit 1 + } + removed=$((removed + 1)) +done <<<"$selected" +echo "off-site retention: PASS max_age_days=$days removed_artifacts=$removed legacy_plaintext_untouched=true" diff --git a/scripts/ops/pilot-privacy.sh b/scripts/ops/pilot-privacy.sh new file mode 100755 index 0000000..99f11ba --- /dev/null +++ b/scripts/ops/pilot-privacy.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +if (($# < 6)) || [[ $1 != --credential-file || $3 != --privacy-directory || $5 != -- ]]; then + echo >&2 'usage: pilot-privacy.sh --credential-file PATH --privacy-directory PATH -- MODE [exact target options]' + exit 64 +fi +credential_file=$2 +privacy_directory=$4 +shift 5 +[[ "$credential_file" = /* && -f "$credential_file" && "$privacy_directory" = /* && + "$privacy_directory" != *:* && + -d "$privacy_directory" && "$privacy_directory" != '/' && + "$privacy_directory" != "$PWD" && "$privacy_directory" != "$PWD"/* ]] || { + echo >&2 'privacy operation refused: protected credential and external privacy directory are required' + exit 66 +} +credential_mode=$(stat -c '%a' "$credential_file" 2>/dev/null || stat -f '%Lp' "$credential_file") +directory_mode=$(stat -c '%a' "$privacy_directory" 2>/dev/null || stat -f '%Lp' "$privacy_directory") +[[ "$credential_mode" = 600 || "$credential_mode" = 400 ]] && + [[ "$directory_mode" = 700 ]] || { + echo >&2 'privacy operation refused: credential must be 0600/0400 and directory 0700' + exit 65 +} +database_url='' +count=0 +while IFS= read -r line || [[ -n "$line" ]]; do + case "$line" in + ''|'#'*) continue ;; + DATABASE_URL=*) database_url=${line#DATABASE_URL=}; count=$((count + 1)) ;; + *) echo >&2 'privacy operation refused: credential file format is invalid'; exit 65 ;; + esac +done <"$credential_file" +[[ "$count" = 1 && -n "$database_url" ]] || { + echo >&2 'privacy operation refused: exactly one protected database URL is required' + exit 65 +} + +ledger="$privacy_directory/erasure-ledger.jsonl" +[[ -f "$ledger" ]] || { + echo >&2 'privacy operation refused: protected erasure ledger is missing' + exit 66 +} +ledger_mode=$(stat -c '%a' "$ledger" 2>/dev/null || stat -f '%Lp' "$ledger") +[[ -f "$ledger" && "$ledger_mode" = 600 ]] || { + echo >&2 'privacy operation refused: erasure ledger must be a 0600 regular file' + exit 65 +} + +mode=$1 +shift +case "$mode" in + inspect|disconnect|export|erase|scrub|replay) ;; + *) echo >&2 'privacy operation refused: invalid mode'; exit 64 ;; +esac + +if [[ " $* " = *' --execute '* ]]; then + case "$mode" in + disconnect|scrub|erase|replay) + running=$(docker compose ps --status running -q backend telegram-bot) || { + echo >&2 'privacy operation refused: application quiescence could not be verified' + exit 1 + } + [[ -z "$running" ]] || { + echo >&2 'privacy operation refused: stop backend and bot before execution' + exit 1 + } + ;; + esac +fi + +cleanup() { + database_url='' + unset database_url DATABASE_URL WCTM_PILOT_PRIVACY_OPERATION +} +trap cleanup EXIT HUP INT TERM +export DATABASE_URL=$database_url +export WCTM_PILOT_PRIVACY_OPERATION=operator-approved +run_privacy() { + docker compose run --rm --no-deps \ + --volume "$privacy_directory:$privacy_directory:rw" \ + -e DATABASE_URL -e WCTM_PILOT_PRIVACY_OPERATION \ + --entrypoint node backend \ + dist/privacy/pilot-data.cli.js "$1" "${@:2}" --ledger "$ledger" +} + +if [[ "$mode" = erase && " $* " = *' --execute '* ]]; then + [[ -n "${WCTM_OFFSITE_DESTINATION:-}" ]] || { + echo >&2 'privacy erasure refused: verified off-site ledger destination is required' + exit 66 + } + run_privacy prepare-erasure "$@" + scripts/ops/archive-erasure-ledger.sh "$ledger" "${WCTM_OFFSITE_DESTINATION%/}/privacy-erasure-ledger" + run_privacy erase "$@" +else + run_privacy "$mode" "$@" +fi diff --git a/scripts/ops/restore-erasure-ledger.sh b/scripts/ops/restore-erasure-ledger.sh new file mode 100755 index 0000000..00a7aaf --- /dev/null +++ b/scripts/ops/restore-erasure-ledger.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +if (($# != 2)); then + echo >&2 'usage: restore-erasure-ledger.sh ENCRYPTED_LEDGER OUTPUT_DIRECTORY' + exit 64 +fi +encrypted=$1 +directory=$2 +key_file=${WCTM_BACKUP_CRYPTO_KEY_FILE:-} +[[ -f "$encrypted" && -f "$encrypted.sha256" && -f "$encrypted.json" && + -d "$directory" && "$directory" != '/' && "$key_file" = /* && + -f "$key_file" && -r "$key_file" ]] || { + echo >&2 'erasure ledger restore refused: complete encrypted set, key and output directory are required' + exit 66 +} +name=$(basename "$encrypted") +[[ "$name" =~ ^erasure-ledger-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}\.jsonl\.enc$ ]] || { + echo >&2 'erasure ledger restore refused: encrypted set name is invalid' + exit 65 +} +(cd "$(dirname "$encrypted")" && sha256sum --check --status "$name.sha256") || { + echo >&2 'erasure ledger restore refused: encrypted checksum mismatch' + exit 1 +} +key_id=$(sha256sum "$key_file" | awk '{print substr($1, 1, 16)}') +cipher_sha256=$(sha256sum "$encrypted" | awk '{print $1}') +expected_plaintext=$(node -e ' +const fs = require("node:fs"); +const value = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); +if (value.format !== "WCTMBACKUPAESG1" || value.keyId !== process.argv[2] || + value.ciphertextSha256 !== process.argv[3] || + !/^[0-9a-f]{64}$/.test(value.ledgerSha256)) process.exit(1); +process.stdout.write(value.ledgerSha256); +' "$encrypted.json" "$key_id" "$cipher_sha256") || { + echo >&2 'erasure ledger restore refused: metadata or recovery key mismatch' + exit 1 +} +output="$directory/erasure-ledger.jsonl" +[[ ! -e "$output" ]] || { + echo >&2 'erasure ledger restore refused: output ledger already exists' + exit 73 +} +crypto_script=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)/backup-crypto.mjs +node "$crypto_script" decrypt "$key_file" "$encrypted" "$output" || { + echo >&2 'erasure ledger restore failed: authentication failed' + exit 1 +} +actual=$(sha256sum "$output" | awk '{print $1}') +if [[ "$actual" != "$expected_plaintext" ]]; then + rm -f -- "$output" + echo >&2 'erasure ledger restore failed: plaintext checksum mismatch' + exit 1 +fi +chmod 0600 "$output" +echo 'erasure ledger restore: PASS authenticated=true' diff --git a/scripts/ops/scheduled-backup.sh b/scripts/ops/scheduled-backup.sh index 250ad2f..c4f48b6 100755 --- a/scripts/ops/scheduled-backup.sh +++ b/scripts/ops/scheduled-backup.sh @@ -4,6 +4,7 @@ umask 077 : "${WCTM_BACKUP_DIRECTORY:?WCTM_BACKUP_DIRECTORY is required}" retention_count=${WCTM_BACKUP_RETENTION_COUNT:-14} +offsite_retention_days=${WCTM_OFFSITE_RETENTION_DAYS:-30} compose_project=${WCTM_COMPOSE_PROJECT:-} [[ "$compose_project" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || { echo >&2 'scheduled backup refused: WCTM_COMPOSE_PROJECT must name the reviewed Compose project' @@ -16,6 +17,12 @@ if [[ -n "${WCTM_OFFSITE_DESTINATION:-}" ]]; then exit 66 } export RCLONE_CONFIG=$rclone_config + key_file=${WCTM_BACKUP_CRYPTO_KEY_FILE:-} + [[ "$key_file" = /* && -f "$key_file" && -r "$key_file" ]] || { + echo >&2 'scheduled backup refused: protected backup encryption key file is required for off-site copy' + exit 66 + } + export WCTM_BACKUP_CRYPTO_KEY_FILE=$key_file fi container_id=$(docker ps --no-trunc \ --filter "label=com.docker.compose.project=$compose_project" \ @@ -37,6 +44,7 @@ backup=$(cat "$result_file") if [[ -n "${WCTM_OFFSITE_DESTINATION:-}" ]]; then scripts/ops/offsite-rclone.sh "$backup" "$backup.sha256" "$backup.json" "$WCTM_OFFSITE_DESTINATION" + scripts/ops/offsite-retention.sh "$WCTM_OFFSITE_DESTINATION" "$offsite_retention_days" fi scripts/ops/backup-retention.sh --directory "$WCTM_BACKUP_DIRECTORY" --keep "$retention_count" --apply diff --git a/scripts/ops/select-offsite-expired.mjs b/scripts/ops/select-offsite-expired.mjs new file mode 100644 index 0000000..b03ef54 --- /dev/null +++ b/scripts/ops/select-offsite-expired.mjs @@ -0,0 +1,25 @@ +#!/usr/bin/env node +import { readFileSync } from 'node:fs'; + +const days = Number(process.argv[2]); +if (!Number.isInteger(days) || days < 1) { + process.stderr.write('off-site retention: invalid age policy\n'); + process.exit(64); +} +const cutoff = new Date(Date.now() - days * 86400000) + .toISOString() + .replace(/[-:]/g, '') + .slice(0, 15) + 'Z'; +const names = new Set(readFileSync(0, 'utf8').split('\n').filter(Boolean)); +const dumps = [...names].filter((name) => + /^wctm-postgres-\d{8}T\d{6}Z-[A-Za-z0-9]{1,40}\.dump\.enc$/.test(name) +).sort().reverse(); +let kept = 0; +for (const name of dumps) { + const sidecars = [`${name}.sha256`, `${name}.json`]; + if (!sidecars.every((sidecar) => names.has(sidecar))) continue; + kept++; + const timestamp = name.slice('wctm-postgres-'.length, 'wctm-postgres-'.length + 16); + if (kept <= 2 || timestamp >= cutoff) continue; + process.stdout.write(`${name}\n${sidecars.join('\n')}\n`); +} diff --git a/scripts/ops/test-backup-crypto.sh b/scripts/ops/test-backup-crypto.sh new file mode 100755 index 0000000..28d9fe9 --- /dev/null +++ b/scripts/ops/test-backup-crypto.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +workspace=$(mktemp -d) +trap 'rm -rf -- "$workspace"' EXIT HUP INT TERM +openssl rand 32 >"$workspace/key" +openssl rand 32 >"$workspace/other-key" +printf 'PGDMP synthetic customer marker\n' >"$workspace/source.dump" +node scripts/ops/backup-crypto.mjs encrypt "$workspace/key" \ + "$workspace/source.dump" "$workspace/source.dump.enc" +[[ ! -e "$workspace/remote.dump" ]] +! grep -aq 'synthetic customer marker' "$workspace/source.dump.enc" +node scripts/ops/backup-crypto.mjs decrypt "$workspace/key" \ + "$workspace/source.dump.enc" "$workspace/restored.dump" +cmp "$workspace/source.dump" "$workspace/restored.dump" + +if node scripts/ops/backup-crypto.mjs decrypt "$workspace/other-key" \ + "$workspace/source.dump.enc" "$workspace/wrong.dump" >"$workspace/wrong.log" 2>&1; then + echo >&2 'test failed: wrong recovery key was accepted' + exit 1 +fi +[[ ! -e "$workspace/wrong.dump" ]] + +cp "$workspace/source.dump.enc" "$workspace/tampered.enc" +printf 'X' | dd of="$workspace/tampered.enc" bs=1 seek=25 count=1 conv=notrunc status=none +if node scripts/ops/backup-crypto.mjs decrypt "$workspace/key" \ + "$workspace/tampered.enc" "$workspace/tampered.dump" >"$workspace/tampered.log" 2>&1; then + echo >&2 'test failed: tampered backup was accepted' + exit 1 +fi +[[ ! -e "$workspace/tampered.dump" ]] + +mkdir "$workspace/bin" "$workspace/remote" +cp scripts/ops/test-fixtures/rclone "$workspace/bin/rclone" +chmod 0700 "$workspace/bin/rclone" +printf '[testremote]\ntype = local\n' >"$workspace/rclone.conf" +export PATH="$workspace/bin:$PATH" +export RCLONE_CONFIG="$workspace/rclone.conf" +export WCTM_BACKUP_CRYPTO_KEY_FILE="$workspace/key" +printf '{"tenantId":"ten_fixture","storeId":"sto_fixture","baseUrlSha256":"%064d"}\n' 0 >"$workspace/ledger.jsonl" +scripts/ops/archive-erasure-ledger.sh "$workspace/ledger.jsonl" \ + "testremote:$workspace/remote" >"$workspace/archive.log" +[[ $(find "$workspace/remote" -maxdepth 1 -name '*.jsonl.enc' | wc -l | tr -d ' ') = 1 ]] +! grep -aq 'ten_fixture' "$workspace/remote"/*.jsonl.enc +grep -q 'erasure archive: PASS' "$workspace/archive.log" +mkdir "$workspace/ledger-restored" +encrypted_ledger=$(find "$workspace/remote" -maxdepth 1 -name '*.jsonl.enc' -print) +scripts/ops/restore-erasure-ledger.sh "$encrypted_ledger" "$workspace/ledger-restored" \ + >"$workspace/ledger-restore.log" +cmp "$workspace/ledger.jsonl" "$workspace/ledger-restored/erasure-ledger.jsonl" +grep -q 'erasure ledger restore: PASS' "$workspace/ledger-restore.log" +! grep -q 'synthetic customer marker' "$workspace"/*.log +echo 'backup encryption: PASS roundtrip wrong-key tamper encrypted-ledger-archive no-plaintext-output secret-safe-errors' diff --git a/scripts/ops/test-fixtures/rclone b/scripts/ops/test-fixtures/rclone index 9efcd98..61fe9c9 100755 --- a/scripts/ops/test-fixtures/rclone +++ b/scripts/ops/test-fixtures/rclone @@ -18,7 +18,7 @@ case "$command_name" in target_path=$(remote_path "${2:?}") mkdir -p -- "$(dirname "$target_path")" cp -- "$source_path" "$target_path" - if [[ ${WCTM_TEST_RCLONE_CORRUPT_DUMP:-false} = true && "$target_path" = *.dump ]]; then + if [[ ${WCTM_TEST_RCLONE_CORRUPT_DUMP:-false} = true && "$target_path" = *.dump.enc ]]; then first_byte=$(od -An -tu1 -N1 "$target_path" | tr -d ' ') replacement='X' [[ "$first_byte" = 88 ]] && replacement='Y' @@ -41,6 +41,15 @@ case "$command_name" in target_path=$(remote_path "${1:?}") cat -- "$target_path" ;; + lsf) + [[ ${1:-} = '--files-only' ]] || exit 64 + target_path=$(remote_path "${2:?}") + find "$target_path" -maxdepth 1 -type f -print | sed 's@.*/@@' | LC_ALL=C sort + ;; + deletefile) + target_path=$(remote_path "${1:?}") + rm -- "$target_path" + ;; *) echo >&2 'test rclone received an unsupported command' exit 64 diff --git a/scripts/ops/test-offsite-retention.sh b/scripts/ops/test-offsite-retention.sh new file mode 100755 index 0000000..899b3c3 --- /dev/null +++ b/scripts/ops/test-offsite-retention.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +workspace=$(mktemp -d) +trap 'rm -rf -- "$workspace"' EXIT HUP INT TERM +mkdir "$workspace/bin" "$workspace/remote" +cp scripts/ops/test-fixtures/rclone "$workspace/bin/rclone" +chmod 0700 "$workspace/bin/rclone" +export PATH="$workspace/bin:$PATH" + +stamp() { + node -e 'process.stdout.write(new Date(Date.now() - Number(process.argv[1]) * 86400000).toISOString().replace(/[-:]/g, "").slice(0, 15) + "Z")' "$1" +} +create_set() { + base="wctm-postgres-$(stamp "$1")-deadbeef0000" + : >"$workspace/remote/$base.dump.enc" + : >"$workspace/remote/$base.dump.enc.sha256" + : >"$workspace/remote/$base.dump.enc.json" +} +create_set 0 +create_set 1 +create_set 40 +create_set 50 +legacy="wctm-postgres-$(stamp 70)-deadbeef0000.dump" +: >"$workspace/remote/$legacy" +incomplete="wctm-postgres-$(stamp 80)-deadbeef0000.dump.enc" +: >"$workspace/remote/$incomplete" + +scripts/ops/offsite-retention.sh "testremote:$workspace/remote" 30 >"$workspace/result.log" +[[ $(find "$workspace/remote" -maxdepth 1 -name '*.dump.enc' | wc -l | tr -d ' ') = 3 ]] +[[ -f "$workspace/remote/$legacy" && -f "$workspace/remote/$incomplete" ]] +[[ $(find "$workspace/remote" -maxdepth 1 -name '*.dump.enc.sha256' | wc -l | tr -d ' ') = 2 ]] +grep -q 'removed_artifacts=6 legacy_plaintext_untouched=true' "$workspace/result.log" +echo 'off-site retention: PASS 30-day policy newest-two protected incomplete/legacy preserved' diff --git a/scripts/ops/test-operations.sh b/scripts/ops/test-operations.sh index 34a07f2..4dc2029 100755 --- a/scripts/ops/test-operations.sh +++ b/scripts/ops/test-operations.sh @@ -7,6 +7,7 @@ suffix="$(date -u '+%Y%m%dT%H%M%SZ')-$$" source_container="wctm-ops-test-$suffix" source_volume="wctm_ops_test_$suffix" source_network="wctm_ops_test_$suffix" +privacy_volume="wctm_privacy_test_$suffix" workspace=$(mktemp -d) secret_marker='test-password-must-not-appear' @@ -18,6 +19,9 @@ cleanup() { case "$source_network" in wctm_ops_test_*) docker network rm "$source_network" >/dev/null 2>&1 || true ;; esac + case "$privacy_volume" in + wctm_privacy_test_*) docker volume rm "$privacy_volume" >/dev/null 2>&1 || true ;; + esac rm -rf -- "$workspace" } trap cleanup EXIT HUP INT TERM @@ -31,6 +35,7 @@ done bash -n scripts/ops/test-fixtures/rclone docker compose config --quiet docker compose build migrate >/dev/null +docker compose build backend >/dev/null [[ $(docker run --rm --entrypoint id wctm-migrate:latest -u) = 1000 ]] if WCTM_MIGRATION_OPERATION=P7_2_EXPLICIT_MIGRATION DATABASE_URL='postgresql://wctm_runtime:sentinel@postgres/db' scripts/ops/run-migrations.sh >"$workspace/migration.log" 2>&1; then @@ -99,6 +104,9 @@ backup=$(find "$workspace/backups" -maxdepth 1 -name 'wctm-postgres-*.dump' -pri [[ -n "$backup" && -f "$backup.sha256" && -f "$backup.json" ]] mkdir "$workspace/test-bin" "$workspace/offsite-native" "$workspace/offsite-streamed" "$workspace/offsite-corrupt" +openssl rand 32 >"$workspace/backup.key" +chmod 0600 "$workspace/backup.key" +export WCTM_BACKUP_CRYPTO_KEY_FILE="$workspace/backup.key" cp scripts/ops/test-fixtures/rclone "$workspace/test-bin/rclone" chmod 0700 "$workspace/test-bin/rclone" PATH="$workspace/test-bin:$PATH" scripts/ops/offsite-rclone.sh \ @@ -117,16 +125,24 @@ if WCTM_TEST_RCLONE_NATIVE_HASH=unavailable WCTM_TEST_RCLONE_CORRUPT_DUMP=true \ echo >&2 'test failed: same-size remote corruption was accepted' exit 1 fi -remote_corrupt="$workspace/offsite-corrupt/$(basename "$backup")" -[[ $(wc -c <"$remote_corrupt" | tr -d ' ') = $(wc -c <"$backup" | tr -d ' ') ]] -[[ $(sha256sum "$remote_corrupt" | awk '{print $1}') != $(sha256sum "$backup" | awk '{print $1}') ]] -grep -q 'remote dump SHA-256 mismatch' "$workspace/offsite-corrupt.log" +remote_corrupt="$workspace/offsite-corrupt/$(basename "$backup").enc" +[[ -s "$remote_corrupt" ]] +grep -q 'remote encrypted dump SHA-256 mismatch' "$workspace/offsite-corrupt.log" ! grep -q "$secret_marker" "$workspace/offsite-native.log" "$workspace/offsite-streamed.log" "$workspace/offsite-corrupt.log" if ! scripts/ops/restore-isolated.sh --backup "$backup" --critical-table tenants --critical-table stores --critical-table operations_probe >"$workspace/restore.log" 2>&1; then cat "$workspace/restore.log" >&2 exit 1 fi +mkdir "$workspace/decrypted" +scripts/ops/decrypt-offsite.sh \ + "$workspace/offsite-streamed/$(basename "$backup").enc" \ + "$workspace/offsite-streamed/$(basename "$backup").enc.json" \ + "$workspace/decrypted" >"$workspace/decrypt.log" 2>&1 +cmp "$backup" "$workspace/decrypted/$(basename "$backup")" +scripts/ops/restore-isolated.sh --backup "$workspace/decrypted/$(basename "$backup")" \ + --critical-table operations_probe >"$workspace/restore-offsite.log" 2>&1 +grep -q 'isolated restore: PASS migrations=16' "$workspace/restore-offsite.log" grep -q 'isolated restore: PASS migrations=16' "$workspace/restore.log" grep -q 'critical table: tenants rows=0' "$workspace/restore.log" grep -q 'critical table: stores rows=0' "$workspace/restore.log" @@ -144,4 +160,57 @@ if scripts/ops/restore-isolated.sh --backup "$corrupt_backup" >"$workspace/corru exit 1 fi -echo 'operations integration: PASS migration-twice backup restore data checksum corruption retention offsite-native-sha256 offsite-streamed-sha256 offsite-same-size-corruption-rejected identity-boundary secret-safe-logs' +docker exec -i "$source_container" psql -X -v ON_ERROR_STOP=1 -U postgres -d wctm_ops_test >/dev/null <<'SQL' +INSERT INTO tenants (id, name, updated_at) +VALUES ('ten_privacy_a', 'Privacy A', now()), ('ten_privacy_b', 'Privacy B', now()); +INSERT INTO stores (id, tenant_id, name, base_url, status, consumer_key_encrypted, consumer_secret_encrypted, updated_at) +VALUES ('sto_privacy_a', 'ten_privacy_a', 'Store A', 'https://a.example.test', 'ACTIVE', 'fixture', 'fixture', now()), + ('sto_privacy_b', 'ten_privacy_b', 'Store B', 'https://b.example.test', 'ACTIVE', 'fixture', 'fixture', now()); +INSERT INTO webhook_events (id, tenant_id, store_id, webhook_id, delivery_id, dedupe_key, topic, payload, status, updated_at) +VALUES ('evt_privacy_a', 'ten_privacy_a', 'sto_privacy_a', '42', 'delivery-a', 'delivery-a', + 'order.updated', '{"id":12,"billing":{"email":"fixture@example.test"}}', 'COMPLETED', now()); +SQL +docker volume create "$privacy_volume" >/dev/null +docker run --rm --network none --user root --mount "type=volume,source=$privacy_volume,destination=/privacy" \ + --entrypoint sh wctm-backend:latest -c 'chown node:node /privacy && chmod 0700 /privacy' >/dev/null +docker run --rm --network none --mount "type=volume,source=$privacy_volume,destination=/privacy" \ + --entrypoint sh wctm-backend:latest -c 'umask 077; : > /privacy/erasure-ledger.jsonl' >/dev/null +privacy_database_url="postgresql://postgres:$secret_marker@postgres:5432/wctm_ops_test" +privacy_key=$(openssl rand -base64 32) +privacy_cli() { + docker run --rm --network "$source_network" \ + --mount "type=volume,source=$privacy_volume,destination=/privacy" \ + -e "DATABASE_URL=$privacy_database_url" -e "APP_ENCRYPTION_KEY=$privacy_key" \ + -e WCTM_PILOT_PRIVACY_OPERATION=operator-approved \ + --entrypoint node wctm-backend:latest dist/privacy/pilot-data.cli.js "$@" +} +privacy_target=(--tenant-id ten_privacy_a --store-id sto_privacy_a --base-url https://a.example.test) +privacy_cli inspect "${privacy_target[@]}" >"$workspace/privacy-inspect.log" 2>&1 +privacy_cli scrub "${privacy_target[@]}" --execute >"$workspace/privacy-scrub.log" 2>&1 +[[ $(docker exec "$source_container" psql -XAtq -U postgres -d wctm_ops_test -c "select payload ? 'billing' from webhook_events where id='evt_privacy_a'") = t ]] +[[ $(docker exec "$source_container" psql -XAtq -U postgres -d wctm_ops_test -c "select payload->'billing' ? 'email' from webhook_events where id='evt_privacy_a'") = f ]] +if privacy_cli erase --tenant-id ten_privacy_b --store-id sto_privacy_a \ + --base-url https://a.example.test --ledger /privacy/erasure-ledger.jsonl --execute \ + >"$workspace/privacy-cross-tenant.log" 2>&1; then + echo >&2 'test failed: cross-tenant privacy erasure was accepted' + exit 1 +fi +privacy_cli disconnect "${privacy_target[@]}" --execute >"$workspace/privacy-disconnect.log" 2>&1 +privacy_cli prepare-erasure "${privacy_target[@]}" --ledger /privacy/erasure-ledger.jsonl --execute \ + >"$workspace/privacy-prepare.log" 2>&1 +privacy_cli erase "${privacy_target[@]}" --ledger /privacy/erasure-ledger.jsonl --execute \ + >"$workspace/privacy-erase.log" 2>&1 +[[ $(docker exec "$source_container" psql -XAtq -U postgres -d wctm_ops_test -c "select base_url from stores where id='sto_privacy_a'") = https://erased.invalid/sto_privacy_a ]] +[[ $(docker exec "$source_container" psql -XAtq -U postgres -d wctm_ops_test -c "select base_url from stores where id='sto_privacy_b'") = https://b.example.test ]] +docker exec -i "$source_container" psql -X -v ON_ERROR_STOP=1 -U postgres -d wctm_ops_test >/dev/null <<'SQL' +UPDATE stores SET name='Store A', base_url='https://a.example.test', status='ACTIVE', deleted_at=NULL +WHERE id='sto_privacy_a'; +UPDATE webhook_events SET payload='{"id":12,"billing":{"email":"fixture@example.test"}}' +WHERE id='evt_privacy_a'; +SQL +privacy_cli replay --ledger /privacy/erasure-ledger.jsonl --execute >"$workspace/privacy-replay.log" 2>&1 +[[ $(docker exec "$source_container" psql -XAtq -U postgres -d wctm_ops_test -c "select base_url from stores where id='sto_privacy_a'") = https://erased.invalid/sto_privacy_a ]] +[[ $(docker exec "$source_container" psql -XAtq -U postgres -d wctm_ops_test -c "select payload from webhook_events where id='evt_privacy_a'") = '{}' ]] +! grep -q "$secret_marker" "$workspace"/privacy-*.log + +echo 'operations integration: PASS migration-twice backup restore data checksum corruption retention encrypted-offsite-sha256 cross-tenant-erasure privacy-scrub disconnect erasure ledger-replay identity-boundary secret-safe-logs' diff --git a/scripts/ops/test-pilot-privacy.sh b/scripts/ops/test-pilot-privacy.sh new file mode 100755 index 0000000..abab43b --- /dev/null +++ b/scripts/ops/test-pilot-privacy.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +workspace=$(mktemp -d) +trap 'rm -rf -- "$workspace"' EXIT HUP INT TERM +mkdir -m 0700 "$workspace/privacy" "$workspace/bin" "$workspace/remote" +printf 'DATABASE_URL=postgresql://owner:fixture-only@postgres/wctm\n' >"$workspace/database.conf" +printf '{"tenantId":"ten_fixture","storeId":"sto_fixture","baseUrlSha256":"%064d"}\n' 0 \ + >"$workspace/privacy/erasure-ledger.jsonl" +openssl rand 32 >"$workspace/key" +printf '[testremote]\ntype = local\n' >"$workspace/rclone.conf" +cp scripts/ops/test-fixtures/rclone "$workspace/bin/rclone" +cat >"$workspace/bin/docker" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"$TEST_DOCKER_LOG" +if [[ "$*" = 'compose ps --status running -q backend telegram-bot' && + ${TEST_APP_RUNNING:-false} = true ]]; then + printf 'running-container\n' +fi +SH +chmod 0700 "$workspace/bin/docker" "$workspace/bin/rclone" +export PATH="$workspace/bin:$PATH" +export TEST_DOCKER_LOG="$workspace/docker.log" +export WCTM_BACKUP_CRYPTO_KEY_FILE="$workspace/key" +export RCLONE_CONFIG="$workspace/rclone.conf" +export WCTM_OFFSITE_DESTINATION="testremote:$workspace/remote" + +scripts/ops/pilot-privacy.sh --credential-file "$workspace/database.conf" \ + --privacy-directory "$workspace/privacy" -- inspect \ + --tenant-id ten_fixture --store-id sto_fixture --base-url https://fixture.test \ + >"$workspace/inspect.log" +[[ $(wc -l <"$workspace/docker.log" | tr -d ' ') = 1 ]] + +scripts/ops/pilot-privacy.sh --credential-file "$workspace/database.conf" \ + --privacy-directory "$workspace/privacy" -- erase \ + --tenant-id ten_fixture --store-id sto_fixture --base-url https://fixture.test --execute \ + >"$workspace/erase.log" +grep -q 'pilot-data.cli.js prepare-erasure' "$workspace/docker.log" +grep -q 'pilot-data.cli.js erase' "$workspace/docker.log" +grep -q 'erasure archive: PASS' "$workspace/erase.log" +[[ $(find "$workspace/remote/privacy-erasure-ledger" -name '*.jsonl.enc' | wc -l | tr -d ' ') = 1 ]] +! grep -q 'fixture-only' "$workspace"/*.log + +unset WCTM_OFFSITE_DESTINATION +if scripts/ops/pilot-privacy.sh --credential-file "$workspace/database.conf" \ + --privacy-directory "$workspace/privacy" -- erase \ + --tenant-id ten_fixture --store-id sto_fixture --base-url https://fixture.test --execute \ + >"$workspace/missing.log" 2>&1; then + echo >&2 'test failed: erasure proceeded without off-site ledger destination' + exit 1 +fi +[[ $(wc -l <"$workspace/docker.log" | tr -d ' ') = 5 ]] + +export WCTM_OFFSITE_DESTINATION="testremote:$workspace/remote" +if TEST_APP_RUNNING=true scripts/ops/pilot-privacy.sh \ + --credential-file "$workspace/database.conf" \ + --privacy-directory "$workspace/privacy" -- disconnect \ + --tenant-id ten_fixture --store-id sto_fixture --base-url https://fixture.test --execute \ + >"$workspace/running.log" 2>&1; then + echo >&2 'test failed: disconnect ran while application services were running' + exit 1 +fi +grep -q 'stop backend and bot' "$workspace/running.log" +echo 'pilot privacy wrapper: PASS protected-config dry-run archive-before-erasure fail-closed' diff --git a/scripts/ops/test-scheduled-backup.sh b/scripts/ops/test-scheduled-backup.sh index 4e41d02..b88b9e1 100755 --- a/scripts/ops/test-scheduled-backup.sh +++ b/scripts/ops/test-scheduled-backup.sh @@ -6,7 +6,8 @@ workspace=$(mktemp -d) trap 'chmod 0600 "$workspace/rclone.conf" "$workspace/.env" 2>/dev/null || true; rm -rf -- "$workspace"' EXIT mkdir -p "$workspace/scripts/ops" "$workspace/bin" "$workspace/backups" cp scripts/ops/scheduled-backup.sh scripts/ops/backup-postgres.sh \ - scripts/ops/offsite-rclone.sh "$workspace/scripts/ops/" + scripts/ops/offsite-rclone.sh scripts/ops/offsite-retention.sh \ + scripts/ops/backup-crypto.mjs scripts/ops/select-offsite-expired.mjs "$workspace/scripts/ops/" cat >"$workspace/scripts/ops/backup-retention.sh" <<'SH' #!/usr/bin/env bash @@ -82,6 +83,9 @@ export PATH="$workspace/bin:$PATH" printf '[testremote]\ntype = local\n' >"$TEST_EXPECTED_CONFIG" chmod 0600 "$TEST_EXPECTED_CONFIG" export RCLONE_CONFIG=$TEST_EXPECTED_CONFIG +openssl rand 32 >"$workspace/backup.key" +chmod 0600 "$workspace/backup.key" +export WCTM_BACKUP_CRYPTO_KEY_FILE="$workspace/backup.key" printf 'unreadable production env sentinel\n' >"$workspace/.env" chmod 000 "$workspace/.env" [[ ! -r "$workspace/.env" ]] || { @@ -101,7 +105,8 @@ dump=$(find "$workspace/backups" -maxdepth 1 -name 'wctm-postgres-*.dump' -print (cd "$(dirname "$dump")" && sha256sum -c "$(basename "$dump").sha256") >/dev/null grep -q '"format": "postgresql-custom"' "$dump.json" grep -q '"completedMigrationCount": 16' "$dump.json" -[[ -f "$workspace/native/$(basename "$dump")" && -f "$workspace/native/$(basename "$dump").json" && -f "$workspace/native/$(basename "$dump").sha256" ]] +[[ -f "$workspace/native/$(basename "$dump").enc" && -f "$workspace/native/$(basename "$dump").enc.json" && -f "$workspace/native/$(basename "$dump").enc.sha256" ]] +[[ ! -e "$workspace/native/$(basename "$dump")" ]] if grep -q '^compose ' "$workspace/docker.log"; then echo >&2 'test failed: backup required Docker Compose and its .env' exit 1 @@ -120,8 +125,8 @@ if WCTM_TEST_RCLONE_NATIVE_HASH=unavailable WCTM_TEST_RCLONE_CORRUPT_DUMP=true \ exit 1 fi dump=$(find "$workspace/backups" -maxdepth 1 -name 'wctm-postgres-*.dump' -print) -[[ $(wc -c <"$workspace/corrupt/$(basename "$dump")" | tr -d ' ') = $(wc -c <"$dump" | tr -d ' ') ]] -grep -q 'remote dump SHA-256 mismatch' "$workspace/corrupt.log" +[[ -s "$workspace/corrupt/$(basename "$dump").enc" ]] +grep -q 'remote encrypted dump SHA-256 mismatch' "$workspace/corrupt.log" docker_calls=$(wc -l <"$workspace/docker.log" | tr -d ' ') WCTM_BACKUP_DIRECTORY='' run_scheduled >"$workspace/missing-backup-directory.log" 2>&1 && { From 0199ad28bacd2b2abcbe4af7e5132d2f2461e81a Mon Sep 17 00:00:00 2001 From: walter Date: Mon, 5 Oct 2026 23:16:43 +0330 Subject: [PATCH 5/6] docs: establish pilot data protection controls --- docs/DECISIONS.md | 39 +++++++- docs/HANDOFF.md | 95 +++++++++--------- docs/MASTER-ROADMAP.md | 16 +-- docs/PHASE_BREAKDOWN.md | 6 +- docs/PROJECT_STATE.md | 24 ++--- docs/operations/P7_2_P7_3_RUNBOOK.md | 8 ++ docs/privacy/MERCHANT_PRIVACY.md | 12 +-- docs/privacy/PPR2_DATA_PROTECTION.md | 140 +++++++++++++++++++++++++++ 8 files changed, 266 insertions(+), 74 deletions(-) create mode 100644 docs/privacy/PPR2_DATA_PROTECTION.md diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index f050685..c3cf24f 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -1328,4 +1328,41 @@ deferred/unstarted under D-033. --- -Next decision number: D-035. +## D-035 + +Date + +2026-10-05 + +Decision + +PPR-2 uses explicit order/product webhook field allowlists and a separate +trusted, quiesced, Store-scoped operator process for verified disconnect, +export and erasure. Erasure preserves narrow referential and security evidence; +an external encrypted ledger is replayed after older-backup restore. WCTM-owned +WordPress connector secrets use authenticated encryption derived from host-side +WordPress installation salts. New OneDrive database backups are encrypted +locally with separately held key material before upload; legacy plaintext sets +require a reviewed disposal decision. The pilot retention defaults are 24 hours +for completed BullMQ jobs, 7 days for failed jobs, 1 day after expiry for +transient PostgreSQL tokens/text, 30/90 days for completed/failed webhook +payloads, 365 days for security audit rows, 14 validated local backup sets, +and 30 days for new encrypted off-site sets with the newest two protected. + +Reason + +PPR-1 identified unnecessary persistent customer information, unclear +processing cessation and erasure, plaintext WordPress-owned connector secrets, +and plaintext off-site database dumps. These controls reduce exposure while +preserving existing WooCommerce/Telegram behavior and recovery evidence. The +repository cannot prove host log/AOF physical retention or production execution; +whole-account deletion and legacy plaintext disposal also need separate review. + +Status + +Accepted for PPR-2 implementation. Production validation and real-merchant +pilot acceptance remain pending; see `docs/privacy/PPR2_DATA_PROTECTION.md`. + +--- + +Next decision number: D-036. diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index f322e09..644beeb 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -43,46 +43,47 @@ n8n is **NOT** part of the production architecture (D-008, prototype only). --- -## 3. Architectural Decisions (D-001–D-034) - -| ID | Decision | Status | -| ----- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -| D-001 | Develop as production SaaS | Accepted | -| D-002 | NestJS for backend (alt: Fastify standalone) | Accepted | -| D-003 | PostgreSQL as database | Accepted | -| D-004 | Redis mandatory (cache, queues, sessions, rate-limiting) | Accepted | -| D-005 | BullMQ for async jobs | Accepted | -| D-006 | WooCommerce via REST API + Webhooks | Accepted | -| D-007 | Telegram is primary management UI | Accepted | -| D-008 | n8n excluded from production | Accepted | -| D-009 | WordPress plugin stays lightweight | Accepted | -| D-010 | Simplicity-first; no overengineering, no premature optimization | Accepted | -| D-011 | Prisma ORM + Prisma Migrate; `schema.prisma` is single source of truth; all models have `created_at`/`updated_at`; soft-delete on Tenant, Store, Membership | Accepted | -| D-012 | PrismaService uses Prisma's official PostgreSQL driver adapter | Accepted | -| D-013 | Global typed configuration uses `@nestjs/config` with Joi validation | Accepted | -| D-014 | One in-process BullMQ operations worker with three exponential-backoff attempts | Accepted | -| D-015 | Fail-closed WooCommerce credential validation with bounded REST retries, timeouts, and secret-safe normalized errors | Accepted | -| D-016 | WooCommerce-REST-only plugin registration verification, reissue-and-rotate recovery, and endpoint-scoped Redis limiting | Accepted | -| D-017 | Dedicated encrypted webhook secrets, routing-only endpoint keys, raw-body HMAC authentication, recoverable idempotent persist/enqueue, and OWNER/ADMIN rotation | Accepted | -| D-018 | Store-scoped Order projection with timestamp/fingerprint ordering, processing-lease recovery, bounded single-order reconciliation, and verified delete/restore handling | Accepted | -| D-019 | Backend-owned one-time Telegram linking, bot-key internal API, private-chat-only authorization, update idempotency, exact-one context resolution, and soft unlinking | Accepted | -| D-020 | Read-only M9 Order access with bounded keyset pagination, `lastSyncedAt` freshness, and expiring HMAC-authenticated callback references bound to current Telegram context | Accepted | -| D-021 | OWNER/ADMIN Telegram order-status writes using server-derived targets, single-effect HMAC references, durable idempotency, one WooCommerce dispatch, and authoritative/lost-response reconciliation | Accepted | -| D-022 | Private-pilot setup/readiness tooling with no reset or force path, hidden JWT internals, a public Caddy HTTPS gate, manual synthetic-order creation, and no public onboarding claim | Accepted | -| D-023 | Backend-owned durable new-order notification delivery with existing M10/M11 authorization, M11/M12 actions, deterministic M5 jobs, conservative ambiguous outcomes, and stateless bot-only transport | Accepted | -| D-024 | Tenant-owned timezone/language, Store-owned threshold/category/recipient policy, Membership-selected recipients under M10 authority, M13-only filtering, and stateless bot settings references | Accepted | -| D-025 | WooCommerce-authoritative narrow inventory projection, core product-webhook updates, Store threshold policy, stock-owning item semantics, incident delivery, M18/M10 recipients, and no stock writes | Accepted | -| D-026 | Store-scoped projection-only exact/prefix Order/inventory search and on-demand Tenant-local projected operational daily report, with no live Woo reads, analytics platform, or scheduler | Accepted | -| D-027 | Tenant-authoritative typed Persian/English Telegram presentation, RTL/bidi and Tenant-timezone calendar formatting, and semantic M13/M19 notification rendering without delivery-policy changes | Accepted | -| D-028 | One backend-authoritative Tenant ACTIVE/SUSPENDED lifecycle with derived expiry, operator-only mutation, explicit product gates, continued projections, terminal notification suppression, and no commercial billing scope | Accepted | -| D-029 | Phase 7 production readiness precedes deferred Phase 6; P7.1 establishes the bounded repository security baseline while host changes, credential rotation, deployment, and P7.2+ remain separately controlled | Accepted | -| D-030 | P7.1 uses a temporary decrypt-only previous key, current-key-only writes, and a trusted-shell row-transactional rotation command for the exact five encrypted database fields | Accepted | -| D-031 | P7.2 separates restricted backend runtime from an ephemeral explicit Prisma migration identity and defines revision-locked fail-fast deployment plus volume-preserving immutable data-service reconciliation | Accepted | -| D-032 | P7.3 establishes atomic verified PostgreSQL backups, explicit retention, verified provider-neutral off-host copy, daily systemd scheduling, isolated guarded restore, DR dependencies, and target-only RPO/RTO | Accepted | -| D-033 | Pause incomplete Phase 7 after production-validated P7.2/P7.3; make Private Pilot Readiness active while Phase 6/P7.4–P7.8 remain deferred | Accepted | -| D-034 | Lock WCTM public identity and license only the WordPress connector GPL-2.0-or-later; keep the SaaS service separately owned | Accepted | - -Next decision number: **D-035**, if a future task produces a genuine +## 3. Architectural Decisions (D-001–D-035) + +| ID | Decision | Status | +| ----- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | +| D-001 | Develop as production SaaS | Accepted | +| D-002 | NestJS for backend (alt: Fastify standalone) | Accepted | +| D-003 | PostgreSQL as database | Accepted | +| D-004 | Redis mandatory (cache, queues, sessions, rate-limiting) | Accepted | +| D-005 | BullMQ for async jobs | Accepted | +| D-006 | WooCommerce via REST API + Webhooks | Accepted | +| D-007 | Telegram is primary management UI | Accepted | +| D-008 | n8n excluded from production | Accepted | +| D-009 | WordPress plugin stays lightweight | Accepted | +| D-010 | Simplicity-first; no overengineering, no premature optimization | Accepted | +| D-011 | Prisma ORM + Prisma Migrate; `schema.prisma` is single source of truth; all models have `created_at`/`updated_at`; soft-delete on Tenant, Store, Membership | Accepted | +| D-012 | PrismaService uses Prisma's official PostgreSQL driver adapter | Accepted | +| D-013 | Global typed configuration uses `@nestjs/config` with Joi validation | Accepted | +| D-014 | One in-process BullMQ operations worker with three exponential-backoff attempts | Accepted | +| D-015 | Fail-closed WooCommerce credential validation with bounded REST retries, timeouts, and secret-safe normalized errors | Accepted | +| D-016 | WooCommerce-REST-only plugin registration verification, reissue-and-rotate recovery, and endpoint-scoped Redis limiting | Accepted | +| D-017 | Dedicated encrypted webhook secrets, routing-only endpoint keys, raw-body HMAC authentication, recoverable idempotent persist/enqueue, and OWNER/ADMIN rotation | Accepted | +| D-018 | Store-scoped Order projection with timestamp/fingerprint ordering, processing-lease recovery, bounded single-order reconciliation, and verified delete/restore handling | Accepted | +| D-019 | Backend-owned one-time Telegram linking, bot-key internal API, private-chat-only authorization, update idempotency, exact-one context resolution, and soft unlinking | Accepted | +| D-020 | Read-only M9 Order access with bounded keyset pagination, `lastSyncedAt` freshness, and expiring HMAC-authenticated callback references bound to current Telegram context | Accepted | +| D-021 | OWNER/ADMIN Telegram order-status writes using server-derived targets, single-effect HMAC references, durable idempotency, one WooCommerce dispatch, and authoritative/lost-response reconciliation | Accepted | +| D-022 | Private-pilot setup/readiness tooling with no reset or force path, hidden JWT internals, a public Caddy HTTPS gate, manual synthetic-order creation, and no public onboarding claim | Accepted | +| D-023 | Backend-owned durable new-order notification delivery with existing M10/M11 authorization, M11/M12 actions, deterministic M5 jobs, conservative ambiguous outcomes, and stateless bot-only transport | Accepted | +| D-024 | Tenant-owned timezone/language, Store-owned threshold/category/recipient policy, Membership-selected recipients under M10 authority, M13-only filtering, and stateless bot settings references | Accepted | +| D-025 | WooCommerce-authoritative narrow inventory projection, core product-webhook updates, Store threshold policy, stock-owning item semantics, incident delivery, M18/M10 recipients, and no stock writes | Accepted | +| D-026 | Store-scoped projection-only exact/prefix Order/inventory search and on-demand Tenant-local projected operational daily report, with no live Woo reads, analytics platform, or scheduler | Accepted | +| D-027 | Tenant-authoritative typed Persian/English Telegram presentation, RTL/bidi and Tenant-timezone calendar formatting, and semantic M13/M19 notification rendering without delivery-policy changes | Accepted | +| D-028 | One backend-authoritative Tenant ACTIVE/SUSPENDED lifecycle with derived expiry, operator-only mutation, explicit product gates, continued projections, terminal notification suppression, and no commercial billing scope | Accepted | +| D-029 | Phase 7 production readiness precedes deferred Phase 6; P7.1 establishes the bounded repository security baseline while host changes, credential rotation, deployment, and P7.2+ remain separately controlled | Accepted | +| D-030 | P7.1 uses a temporary decrypt-only previous key, current-key-only writes, and a trusted-shell row-transactional rotation command for the exact five encrypted database fields | Accepted | +| D-031 | P7.2 separates restricted backend runtime from an ephemeral explicit Prisma migration identity and defines revision-locked fail-fast deployment plus volume-preserving immutable data-service reconciliation | Accepted | +| D-032 | P7.3 establishes atomic verified PostgreSQL backups, explicit retention, verified provider-neutral off-host copy, daily systemd scheduling, isolated guarded restore, DR dependencies, and target-only RPO/RTO | Accepted | +| D-033 | Pause incomplete Phase 7 after production-validated P7.2/P7.3; make Private Pilot Readiness active while Phase 6/P7.4–P7.8 remain deferred | Accepted | +| D-034 | Lock WCTM public identity and license only the WordPress connector GPL-2.0-or-later; keep the SaaS service separately owned | Accepted | +| D-035 | Use PPR-2 field minimization, protected Store-scoped lifecycle and erasure replay, explicit retention, WordPress credential encryption and encrypted off-site backups | Accepted for implementation; pilot validation pending | + +Next decision number: **D-036**, if a future task produces a genuine architectural or product decision. --- @@ -1301,13 +1302,13 @@ initiative: privacy/data-flow audit and merchant disclosures, licensing/legal baseline, WordPress connector admin UI polish, `wctm.walterbyte.com` onboarding/product-surface polish, focused security/quality QA, and controlled private merchant pilots. Engineering is frozen except for pilot-blocking -defects. PPR-1 is the current bounded task: `docs/privacy/` inventories the -actual data flow and merchant disclosure; the connector adds Privacy Policy -Guide text, local uninstall cleanup, WCTM/Walterbyte identity and GPL-2.0-or- -later licensing under D-034. Full webhook/order snapshot minimization, -service-side deletion and retention, WordPress-local credential controls, and -off-site backup confidentiality remain explicit pilot blockers. No real- -merchant pilot or broad public launch is authorized by this work. +defects. PPR-1 is complete and merged to canonical `main`; `docs/privacy/` +contains its inventory, disclosure, WordPress privacy integration and GPL +plugin licensing under D-034. PPR-2 is the active data-protection implementation +task from `f5903fd99c999912a1bbbe69f28ecb0c42693581`. See +`docs/privacy/PPR2_DATA_PROTECTION.md` for exact Store-scoped controls, +retention values, backup key/recovery boundaries and remaining pilot gates. +No real-merchant pilot or broad public launch is authorized by this work. ### Last completed product milestone: M22 diff --git a/docs/MASTER-ROADMAP.md b/docs/MASTER-ROADMAP.md index ecb7baf..3f19fcd 100644 --- a/docs/MASTER-ROADMAP.md +++ b/docs/MASTER-ROADMAP.md @@ -317,12 +317,16 @@ admin UI polish, `wctm.walterbyte.com` onboarding/product-surface polish, focused security/quality QA, and controlled private merchant pilots with real users. Engineering remains frozen except for pilot-blocking defects. -**PPR-1 — Privacy, Security & Licensing Foundation:** current bounded task. -The code-backed inventory, merchant notice, WordPress privacy integration and -GPL-2.0-or-later plugin license are in `docs/privacy/`. Full webhook/order -snapshot minimization, service-side deletion and retention, WordPress-host -credential controls, and off-site backup confidentiality remain pilot blockers. -This task does not authorize a merchant pilot or resume Phase 7. +**PPR-1 — Privacy, Security & Licensing Foundation:** complete and merged to +canonical `main`. The code-backed inventory, merchant notice, WordPress privacy +integration and GPL-2.0-or-later plugin license are in `docs/privacy/`. + +**PPR-2 — Pilot Data Protection Controls:** active implementation task. The +branch adds minimized webhook/order persistence, a trusted Store-scoped +disconnect/export/erasure workflow, explicit logical retention, WordPress-owned +credential encryption and encrypted off-site backups. The exact limits and +remaining pilot gates are in `docs/privacy/PPR2_DATA_PROTECTION.md`. This work +does not authorize a real-merchant pilot or resume Phase 7. --- diff --git a/docs/PHASE_BREAKDOWN.md b/docs/PHASE_BREAKDOWN.md index e67ba56..57935cd 100644 --- a/docs/PHASE_BREAKDOWN.md +++ b/docs/PHASE_BREAKDOWN.md @@ -824,8 +824,10 @@ accepted D-033. Historical provenance review remains assigned to P7.8. - Focused security/quality QA - Private merchant pilot with real users and usage/payment evidence -Accepted D-033 freezes engineering except for pilot-blocking defects. Pilot -tasks require separate authorization; none is implemented in this branch. +Accepted D-033 freezes engineering except for pilot-blocking defects. PPR-1 is +complete on canonical `main`. PPR-2 is the active, separately authorized pilot +data-protection task; its implementation and remaining gates are recorded in +`docs/privacy/PPR2_DATA_PROTECTION.md`. Real-merchant pilots have not started. ## Phase 6 — SaaS Platform ⬜ Deferred / unstarted diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index b6843b3..26c43da 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -18,9 +18,10 @@ and production-validated. The corrected scheduled backup passed live systemd execution on `waltpack`, and the daily timer is enabled. D-033 is Accepted: Phase 7 is intentionally paused and incomplete before P7.4; P7.4–P7.8 and Phase 6 remain deferred/unstarted. Private Pilot Readiness is the current active -initiative. PPR-1 audits privacy/security and establishes the WordPress -licensing and disclosure baseline. Unresolved data minimization, deletion, -retention and backup-access findings block real-merchant pilot onboarding. +initiative. PPR-1 is complete on canonical `main`; PPR-2 is the active +implementation task for pilot data protection. New minimization, operator +lifecycle, retention and backup encryption controls are under local validation. +Remaining operational and account-deletion gates block real-merchant onboarding. Engineering remains frozen except for pilot-blocking defects. Unrestricted public launch is not approved. @@ -32,11 +33,10 @@ P7.2 production validation passed on authoritative `waltpack` at reviewed revision `034b2f565fd6b2fc50d6942ff1a40ca16598703c`. P7.3 backup, OneDrive off-site SHA-256, isolated restore, and corrected live scheduled service all passed. `wctm-backup.service` completed as `wctm` with -`SERVICE_RC=0`; the daily timer is enabled and active. PPR-1 is the current task -from `main` `2754958`: code-backed data-flow audit and merchant notice, -WordPress privacy-policy text and local uninstall cleanup, and a -GPL-2.0-or-later plugin under the locked WCTM/Walterbyte identity (D-034). -Findings and pilot blockers are in `docs/privacy/PPR1_DATA_FLOW.md`. The +`SERVICE_RC=0`; the daily timer is enabled and active. PPR-1 is merged in +canonical `main` `f5903fd99c999912a1bbbe69f28ecb0c42693581`. PPR-2 is +implementing the approved protection controls; the exact controls and +remaining pilot gates are in `docs/privacy/PPR2_DATA_PROTECTION.md`. The earlier migration from `hetz` succeeded; its WCTM deployment was removed and is not a rollback host. @@ -50,8 +50,8 @@ Project Version Repository -Current implementation branch: `feat/ppr1-privacy-security-licensing` from -`main` `2754958a6b45fdb7c8ecb2849fd4b4c4e54986ca`. +Current implementation branch: `feat/ppr2-pilot-data-protection` from +canonical `main` `f5903fd99c999912a1bbbe69f28ecb0c42693581`. M22 implementation commit: `35f9e72335c8ed0c6a039497d92bed763dc68fb5 feat(entitlements): add MVP tenant @@ -832,8 +832,8 @@ Next Milestone Private Pilot Readiness is the current active initiative under accepted D-033. P7.4–P7.8 remain deferred/unstarted and require a separate resumption decision. -PPR-1 is in review, with pilot blockers in `docs/privacy/PPR1_DATA_FLOW.md`; -the private merchant pilot has not started. +PPR-1 is complete. PPR-2 is active, with remaining pilot gates in +`docs/privacy/PPR2_DATA_PROTECTION.md`; the private merchant pilot has not started. --- diff --git a/docs/operations/P7_2_P7_3_RUNBOOK.md b/docs/operations/P7_2_P7_3_RUNBOOK.md index 2cb1d0a..01ae291 100644 --- a/docs/operations/P7_2_P7_3_RUNBOOK.md +++ b/docs/operations/P7_2_P7_3_RUNBOOK.md @@ -1,5 +1,13 @@ # P7.2/P7.3 Production Operations Runbook +> PPR-2 implementation note: new off-site sets are encrypted before upload and +> require `WCTM_BACKUP_CRYPTO_KEY_FILE` plus a host Node runtime in the scheduled +> service. This change has **not** been production validated. For any restore of +> an older backup, keep the application stopped and replay the external erasure +> ledger before it becomes authoritative. See +> `docs/privacy/PPR2_DATA_PROTECTION.md` for the key, legacy plaintext and +> erasure boundaries. The P7.3 evidence below remains historical. + Status: P7.2 and P7.3 production validation PASS on authoritative `waltpack` (2026-10-05). P7.2 deployed reviewed revision `034b2f565fd6b2fc50d6942ff1a40ca16598703c`. P7.3 passed backup, diff --git a/docs/privacy/MERCHANT_PRIVACY.md b/docs/privacy/MERCHANT_PRIVACY.md index b9f8915..c5569d0 100644 --- a/docs/privacy/MERCHANT_PRIVACY.md +++ b/docs/privacy/MERCHANT_PRIVACY.md @@ -4,24 +4,24 @@ ## What the connection processes -When you connect a store, WCTM receives store identity/configuration and WooCommerce REST credentials, then installs authenticated order and product webhooks through its WordPress connector. WooCommerce sends order and product webhook payloads to WCTM. These can contain order numbers, items, totals, customer names, email/phone numbers, billing and shipping addresses, notes, and other fields present in WooCommerce's webhook JSON. WCTM also stores account email, team membership/roles, Telegram user/chat IDs, operational audit/delivery records and integration health information. +When you connect a store, WCTM receives store identity/configuration and WooCommerce REST credentials, then installs authenticated order and product webhooks through its WordPress connector. WooCommerce sends order and product webhook payloads to WCTM. WCTM's new processing path stores an explicit subset: order identity/status, totals, selected customer name and shipping address, item summaries, and inventory/stock fields needed for store management. It discards webhook notes, customer email/phone, billing address and arbitrary WooCommerce metadata. Earlier records may still contain fuller data until an operator scrubs them or their backup sets expire. WCTM also stores account email, team membership/roles, Telegram user/chat IDs, operational audit/delivery records and integration health information. -The WordPress connector keeps the store ID, connector credential, webhook secret and route key in WordPress options with autoload disabled. WooCommerce webhook records also hold the webhook configuration. The connector does not install tracking cookies or telemetry and does not keep its own local customer/order database. WordPress/WooCommerce and their normal host/database backups remain under the store operator's control. +The WordPress connector keeps the store ID, connector credential, webhook secret and route key in WordPress options with autoload disabled. The WCTM-owned credentials are authenticated-encrypted with a key derived from WordPress installation secrets outside its database; previously stored plaintext credentials migrate when read. WooCommerce webhook records also hold their signing secret under WooCommerce's own storage rules. The connector does not install tracking cookies or telemetry and does not keep its own local customer/order database. WordPress/WooCommerce and their normal host/database backups remain under the store operator's control. ## Where it goes WCTM stores account, store, event, order and inventory records in its PostgreSQL service. Its Redis queue holds job identifiers/context and rate-limit state; Redis persistence is enabled. The backend makes authorized WooCommerce REST requests. Only authorized private-chat store managers can request operational views/actions. WCTM sends selected order and inventory details to Telegram to deliver those views and notifications; order detail may include customer name and shipping address. Telegram is an external platform with its own message and account retention behavior. Do not put information in manager notes that staff should not share with Telegram or WooCommerce. -WCTM makes custom-format PostgreSQL backups for recovery. Backup sets have SHA-256 checksums and are copied to OneDrive with remote content verification. The operational default retains 14 valid local sets; repository code does not set an automatic deletion period for OneDrive copies or all application records. Backups contain database data, including personal information. The backup workflow verifies integrity; it does not apply its own encryption to the dump before off-site transfer. +WCTM makes custom-format PostgreSQL backups for recovery. The new off-site path encrypts a dump locally before OneDrive transfer and verifies the remote encrypted content by SHA-256. Recovery keys are held separately from OneDrive. The local default retains 14 valid sets; new encrypted off-site sets expire after 30 days while preserving the newest two complete sets. Earlier plaintext off-site sets remain subject to a separate reviewed cleanup decision. A restored older backup must have outstanding erasure requests reapplied before service resumes. ## Security and your responsibilities -WCTM encrypts WooCommerce REST and backend webhook secrets in its database, hashes several connection/link tokens, checks webhook signatures, uses tenant-scoped authorization and restricts production database access. WordPress-side connector and WooCommerce webhook secrets rely on the security of your WordPress database, host, administrators and backups. Keep WordPress/WooCommerce updated, restrict administrator and database access, use HTTPS, and give Telegram manager access only to trusted staff. Telegram messages are outside WCTM's database controls once delivered. +WCTM encrypts WooCommerce REST and backend webhook secrets in its database, hashes several connection/link tokens, checks webhook signatures, uses tenant-scoped authorization and restricts production database access. WordPress-side connector secrecy also depends on the security of your installation salts, host and administrators; WooCommerce's own webhook secret remains within the WooCommerce database trust boundary. Keep WordPress/WooCommerce updated, restrict administrator and database access, use HTTPS, and give Telegram manager access only to trusted staff. Telegram messages are outside WCTM's database controls once delivered. As the store operator, inform customers and staff about the store's use of WCTM and Telegram as appropriate, and handle their access/deletion requests for the WooCommerce data you control. Avoid entering unnecessary personal data in free-form notes. Review your own WordPress/WooCommerce retention and backup settings. ## Disconnecting and requesting deletion -Plugin **deactivation** retains connector settings and WooCommerce webhooks; it does not stop all webhook delivery. The Store `DELETE` operation in WCTM is a **soft delete** that gates active use but retains historical records and encrypted credentials. Confirmed Telegram `/unlink` revokes the active chat link, but prior WCTM and Telegram records remain. Plugin **uninstall** deletes connector options and matching local webhooks when WooCommerce APIs are available; it does not erase WCTM, OneDrive or Telegram data. If WooCommerce was inactive during uninstall, check and remove remaining WCTM webhooks after reactivation. Rotate credentials when a connection is retired or compromised. +Plugin **deactivation** retains connector settings and WooCommerce webhooks; it does not stop all webhook delivery. The normal Store `DELETE` operation is a **soft delete**, not the verified privacy disconnect. Confirmed Telegram `/unlink` revokes the active chat link, but prior WCTM and Telegram records remain. Plugin **uninstall** deletes connector options and matching local webhooks when WooCommerce APIs are available; it does not erase WCTM, OneDrive or Telegram data. If WooCommerce was inactive during uninstall, check and remove remaining WCTM webhooks after reactivation. Rotate credentials when a connection is retired or compromised. -For a private-pilot service-side access or deletion request, contact your Walterbyte pilot contact through the agreed support channel and identify your account/store. The team must verify the requester and agree the scope, legal/operational retention and backup handling before performing deletion. An automated complete-erasure process is not yet available; WCTM will not claim that uninstall or soft deletion erases all data. Pilot onboarding must wait for the retention/deletion and backup-access decisions listed in the internal PPR-1 audit. +For a private-pilot service-side export, disconnect or erasure request, contact your Walterbyte pilot contact through the agreed support channel and identify your account/store. An operator verifies the requester and exact Store, inspects the scope, and can revoke processing access and erase/anonymize Store-scoped live records through a protected maintenance workflow. Narrow security/audit identifiers remain; Telegram messages, merchant WooCommerce records and historical backups are outside live Store erasure. Older backups expire by retention and must have outstanding erasure requests replayed after restore. Whole-account deletion involving shared users/Telegram identities needs separate review. Real-merchant onboarding remains blocked pending the PPR-2 operational and retention gates. diff --git a/docs/privacy/PPR2_DATA_PROTECTION.md b/docs/privacy/PPR2_DATA_PROTECTION.md new file mode 100644 index 0000000..93d14ef --- /dev/null +++ b/docs/privacy/PPR2_DATA_PROTECTION.md @@ -0,0 +1,140 @@ +# PPR-2 pilot data protection controls + +Status: implementation branch, **not production validated**. PPR-1 is complete on +`main`. This document describes the repository behavior and the remaining gates +for a controlled real-merchant pilot. Phase 7 remains paused. + +## Data minimization + +After HMAC verification, order and product webhook bodies are projected through +`minimizeWebhookPayload` before PostgreSQL persistence. The order event retains +order ID/number/status, currency and totals, payment method label, relevant +timestamps, billing name/company, shipping address fields used by Telegram, +line-item name/quantity/total, and shipping method labels. Product events retain +identity, parent/type, stock state/quantity, SKU/name, modification time and +variation attributes needed for inventory. Unknown metadata, order notes, +phone/email, billing address, Woo customer ID and arbitrary line-item fields are +discarded. Deleted events need only identity. The order projection applies the +same customer and line-item limits. Telegram can still receive customer name and +shipping address when an authorized manager requests order detail; Telegram's +own retention is outside WCTM's control. + +Historical rows are **not changed automatically**. The operator must quiesce the +application, back up, dry-run and execute `pilot-privacy.sh scrub` for each +verified Tenant/Store. It processes terminal webhook events and existing order +snapshots in bounded batches, checks scoped identities before each write, and +recomputes order fingerprints. Pending/processing events are left untouched for +normal completion; a later scrub pass is required. Older backups retain the +original data until expiry and require erasure-ledger replay after restore. + +## Verified operator lifecycle + +`scripts/ops/pilot-privacy.sh` is an operator-only, non-HTTP entry point. It +requires an exact Tenant ID, Store ID and HTTPS base URL, a protected database +owner credential file, the application encryption key already in protected +Compose configuration, and a 0700 external privacy directory with a 0600 +erasure ledger. The credential file contains exactly one `DATABASE_URL=` entry, +is 0600/0400, and is never committed. The backend runtime DB role is rejected. +Execution of disconnect/scrub/erase/replay refuses to run while backend or bot +containers are running. This is an intentional maintenance window, not an API +that merchant input can invoke. Inspect/export are read-only with respect to +the database; export creates a new mode-0600 file in the protected directory. + +Disconnect first revokes the connector token, registration token, webhook route +and secret, replaces the encrypted Woo REST key/secret with unusable random +values, marks the Store disconnected/deleted, revokes its active Telegram chat +context, and fails queued/processing Store webhook events. The operator command +then tries to remove only exact WCTM-owned WooCommerce webhook records using the +pre-revocation credentials held in memory. If Woo removal fails, the report says +`manual-required`; the local route and credentials remain revoked. The operator +must reconcile the Woo side before closing the request. Plugin deactivation, +uninstall, normal Store soft delete and Telegram unlink are **not** this workflow. + +Erasure requires a prior verified disconnect. An exact Store-scoped request is +recorded in an external append-only 0600 ledger containing only Tenant/Store IDs, +SHA-256 of the original base URL and time. The ledger is encrypted and remotely +verified **before** live erasure. Erasure clears that Store's webhook payloads, +order customer/line/payment/shipping snapshots and totals, inventory labels/SKU, +encrypted callback/search text and action results. Store name/base URL become a +tombstone. Referential IDs, delivery state and narrow audit events remain for +integrity/security evidence. The operator must inspect the export and confirm +scope/authority before `--execute`. This is **Store-scoped** erasure; it does not +delete shared Tenant/User/Telegram identities or Telegram messages. Whole-account +requests require a separately reviewed procedure. + +Before a restored old backup becomes authoritative, recover the latest external +erasure ledger, verify/decrypt it, keep backend and bot stopped, and execute +`replay`. Replay checks each Tenant/Store and base-URL hash, disconnects and +erases matching restored records, skips already erased stores, and fails closed +on mismatches. Only after replay and normal role/config/readiness checks may +application service resume. Keep the ledger and its encryption key for as long +as any recoverable backup could predate an erasure. Historic backups expire by +retention; they are not rewritten in place. + +## Explicit pilot retention decisions + +| Data | Repository rule | Boundary | +| ----------------------------------------------------------- | ------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| BullMQ completed jobs | 24 hours; terminal cleanup hourly, up to 1000 per sweep | Active/waiting/delayed work is never selected. Heavy backlog may take additional sweeps. | +| BullMQ failed jobs | 7 days; same terminal sweep | Diagnostic result/IDs persist until then. | +| Redis rate-limit and dedupe state | Existing 60-second rate windows; job keys as above | Redis AOF is persistent and old bytes may remain until compaction; physical data expiry needs live proof. | +| Link/registration tokens and encrypted callback/search text | Clear 1 day after their expiry; daily sweep | Existing active TTLs remain unchanged. | +| Completed webhook payload | Clear 30 days after completion; daily sweep | Event metadata/dedupe stays for operational integrity. | +| Failed webhook payload | Clear 90 days after failure; daily sweep | Pending/active events are untouched. | +| Security audit rows | 365 days; daily sweep | Minimal erasure ledger is held separately for restore safety. | +| Local PostgreSQL backups | Existing 14 validated sets | Newer 14 sets always protected; this is count-based, not a calendar age. | +| New encrypted OneDrive sets | 30 days, always keep newest 2 complete sets | Legacy plaintext and incomplete sets are never auto-deleted. | +| Application and scheduler logs | No repository-enforced calendar limit yet | Host Docker/journald policy and live proof are a pilot gate. | + +## WordPress and backup confidentiality + +The WCTM-owned connector credential and WCTM webhook secret in WordPress options +use AES-256-GCM with separate associated-data labels. The key derives by HKDF +from WordPress `AUTH_KEY` and `SECURE_AUTH_KEY`, which should live outside the +WordPress database. Exact legacy plaintext token shapes migrate on first read. +Missing salts or authentication failure yields no credential and no plaintext +fallback. Reconnect/rotation must reprovision secrets with the service operator +if host salts are lost. WordPress administrators, host files and backups remain +within the merchant's trust boundary. WooCommerce itself persists its webhook +signing secret in its own database; this plugin does not alter Woo internals. + +The scheduled off-site path encrypts the locally created custom-format dump +using a separately stored 32-byte mode-0600 key before rclone upload. OneDrive +receives only `.dump.enc`, `.dump.enc.json` and `.dump.enc.sha256` for new sets. +AES-256-GCM authenticates ciphertext; local SHA-256 verifies the original dump, +and remote content SHA-256 verifies uploaded ciphertext. The key must be owned +by the dedicated backup account outside Git and OneDrive. The scheduled service +requires a host Node runtime and a readable key file; these are new deployment +prerequisites requiring live validation. The decrypted dump is only created in +an operator-controlled 0700 directory, then used with the existing isolated +restore procedure. Encryption does not replace checksum or restore tests. + +Retain the current key until every backup encrypted under it has expired and +every archived erasure ledger encrypted under it has been re-encrypted and +verified with the replacement key. Rotation is a reviewed maintenance action: +create a new private key, retain the old key in separate protected recovery +storage, update the key path, make a new encrypted backup, decrypt/restore it in +isolation, and verify ledger recovery before retiring the old key. Losing the +key loses the off-site recovery path. Existing plaintext OneDrive sets are +**not** silently deleted or converted; inventory them and obtain a reviewed +retention/deletion decision. Treat them as sensitive until removed. + +## Remaining gates before real merchants + +1. Review and execute the lifecycle, scrub and encrypted backup/restore paths + against isolated production-like data, including cross-tenant checks and + erasure replay, then validate the scheduled encrypted path on `waltpack` in + a separately authorized production operation. +2. Inventory and decide disposal of pre-PPR-2 plaintext OneDrive sets. No + automatic legacy deletion is authorized by this branch. +3. Set and verify host Docker/journald log rotation/retention and Redis AOF + compaction/physical expiry. The repository controls logical TTLs, not those + host storage lifetimes. +4. Define a reviewed whole-Tenant/User/Telegram-account export and deletion + process, including shared identities and security audit exceptions, before + promising complete account erasure. +5. Verify WordPress salt-loss reconnect/rotation and Woo-owned signing-secret + cleanup on a representative pilot merchant site. + +Until these gates are closed and accepted, PPR-2 remains active and real-merchant +pilot onboarding remains blocked. No public-launch readiness is claimed. From 09eea12b744e43ccc3e3c32661dd87e16a505c13 Mon Sep 17 00:00:00 2001 From: walter Date: Tue, 6 Oct 2026 14:55:23 +0330 Subject: [PATCH 6/6] fix(privacy): bound pilot export retention --- backend/src/privacy/pilot-data.cli.spec.ts | 48 ++++++- backend/src/privacy/pilot-data.cli.ts | 38 +++++- docs/PROJECT_STATE.md | 2 + docs/privacy/MERCHANT_PRIVACY.md | 2 + docs/privacy/PPR2_DATA_PROTECTION.md | 62 +++++++-- .../wctm-privacy-export-retention.service | 16 +++ .../wctm-privacy-export-retention.timer | 11 ++ scripts/ops/pilot-privacy.sh | 36 +++++ scripts/ops/privacy-export-artifacts.mjs | 99 ++++++++++++++ scripts/ops/test-pilot-privacy.sh | 28 +++- scripts/ops/test-privacy-export-artifacts.mjs | 126 ++++++++++++++++++ 11 files changed, 448 insertions(+), 20 deletions(-) create mode 100644 ops/systemd/wctm-privacy-export-retention.service create mode 100644 ops/systemd/wctm-privacy-export-retention.timer create mode 100644 scripts/ops/privacy-export-artifacts.mjs create mode 100644 scripts/ops/test-privacy-export-artifacts.mjs diff --git a/backend/src/privacy/pilot-data.cli.spec.ts b/backend/src/privacy/pilot-data.cli.spec.ts index 929041f..65b2457 100644 --- a/backend/src/privacy/pilot-data.cli.spec.ts +++ b/backend/src/privacy/pilot-data.cli.spec.ts @@ -1,10 +1,15 @@ import { afterEach, describe, expect, it, jest } from '@jest/globals'; import { PrismaClient, StoreStatus } from '@prisma/client'; -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { appendLedger, replayLedger } from './pilot-data.cli'; +import { + appendLedger, + replayLedger, + verifyExportOutput, +} from './pilot-data.cli'; import type { PilotDataService } from './pilot-data.service'; const target = { @@ -87,3 +92,42 @@ describe('external erasure ledger replay', () => { expect(service.disconnect).not.toHaveBeenCalled(); }); }); + +describe('protected privacy export path', () => { + it('accepts only the exact Tenant/Store artifact contract beside the ledger', async () => { + const ledger = await fixture(); + const directory = join(ledger, '..'); + const scope = createHash('sha256') + .update(`${target.tenantId}\0${target.storeId}`) + .digest('hex'); + const name = `wctm-privacy-export-${scope}-${Date.now()}-0123456789abcdef.json`; + await expect( + verifyExportOutput(join(directory, name), ledger, target) + ).resolves.toBeUndefined(); + await expect( + verifyExportOutput( + join(directory, 'erasure-ledger.jsonl'), + ledger, + target + ) + ).rejects.toThrow('scoped artifact contract'); + await expect( + verifyExportOutput(join(directory, '..', name), ledger, target) + ).rejects.toThrow('scoped artifact contract'); + await expect( + verifyExportOutput( + join(directory, name.replace(scope, '0'.repeat(64))), + ledger, + target + ) + ).rejects.toThrow('scoped artifact contract'); + const link = join(directory, 'linked'); + await symlink(directory, link); + await expect( + verifyExportOutput(join(link, name), ledger, target) + ).rejects.toThrow('scoped artifact contract'); + await expect( + verifyExportOutput(`${link}/../${name}`, ledger, target) + ).rejects.toThrow('scoped artifact contract'); + }); +}); diff --git a/backend/src/privacy/pilot-data.cli.ts b/backend/src/privacy/pilot-data.cli.ts index 3aec56c..901e867 100644 --- a/backend/src/privacy/pilot-data.cli.ts +++ b/backend/src/privacy/pilot-data.cli.ts @@ -1,8 +1,8 @@ import { PrismaPg } from '@prisma/adapter-pg'; import { PrismaClient, StoreStatus } from '@prisma/client'; import { createHash } from 'node:crypto'; -import { open, readFile, stat, writeFile } from 'node:fs/promises'; -import { isAbsolute, resolve } from 'node:path'; +import { lstat, open, readFile, stat, writeFile } from 'node:fs/promises'; +import { basename, dirname, isAbsolute, resolve } from 'node:path'; import { EncryptionService } from '../common/encryption/encryption.service'; import type { ApplicationConfigService } from '../config/application-config.service'; @@ -60,9 +60,15 @@ function parse(args: string[]): Options { ) { throw new Error('exact Tenant, Store and HTTPS base URL are required'); } - if (['export'].includes(mode) && (!output || !isAbsolute(output))) { + if (mode === 'export' && (!output || !isAbsolute(output))) { throw new Error('export requires an absolute protected output path'); } + if (mode === 'export' && (!ledger || !isAbsolute(ledger))) { + throw new Error('export requires the protected privacy directory'); + } + if (mode === 'export' && !args.includes('--retain-for-delivery')) { + throw new Error('export requires explicit delivery retention'); + } if ( ['prepare-erasure', 'erase', 'replay'].includes(mode) && (!ledger || !isAbsolute(ledger)) @@ -86,6 +92,31 @@ function fingerprint(value: string): string { return createHash('sha256').update(value).digest('hex'); } +export async function verifyExportOutput( + output: string, + ledger: string, + target: PilotTarget +): Promise { + const scope = createHash('sha256') + .update(`${target.tenantId}\0${target.storeId}`) + .digest('hex'); + if ( + !new RegExp( + `^wctm-privacy-export-${scope}-[0-9]{13}-[0-9a-f]{16}\\.json$` + ).test(basename(output)) || + output !== resolve(output) || + resolve(dirname(output)) !== resolve(dirname(ledger)) + ) { + throw new Error( + 'privacy export path is outside the scoped artifact contract' + ); + } + const directory = await lstat(dirname(ledger)); + if (!directory.isDirectory() || (directory.mode & 0o777) !== 0o700) { + throw new Error('privacy export directory must be protected'); + } +} + async function verifyLedger(path: string): Promise { if (resolve(path).startsWith(`${process.cwd()}/`)) { throw new Error('erasure ledger must be outside the repository'); @@ -261,6 +292,7 @@ async function main(): Promise { `${JSON.stringify({ target: target.storeId, disconnected: options.execute, remoteWebhookCleanup })}\n` ); } else if (options.mode === 'export') { + await verifyExportOutput(options.output!, options.ledger!, target); const report = await service.export(target); await writeFile(options.output!, `${JSON.stringify(report)}\n`, { flag: 'wx', diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index 26c43da..8e8467d 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -21,6 +21,8 @@ Phase 6 remain deferred/unstarted. Private Pilot Readiness is the current active initiative. PPR-1 is complete on canonical `main`; PPR-2 is the active implementation task for pilot data protection. New minimization, operator lifecycle, retention and backup encryption controls are under local validation. +Store privacy exports now have scoped protected files, completion cleanup and +an hourly 24-hour retention bound awaiting operator-host validation. Remaining operational and account-deletion gates block real-merchant onboarding. Engineering remains frozen except for pilot-blocking defects. Unrestricted public launch is not approved. diff --git a/docs/privacy/MERCHANT_PRIVACY.md b/docs/privacy/MERCHANT_PRIVACY.md index c5569d0..ac60e90 100644 --- a/docs/privacy/MERCHANT_PRIVACY.md +++ b/docs/privacy/MERCHANT_PRIVACY.md @@ -25,3 +25,5 @@ As the store operator, inform customers and staff about the store's use of WCTM Plugin **deactivation** retains connector settings and WooCommerce webhooks; it does not stop all webhook delivery. The normal Store `DELETE` operation is a **soft delete**, not the verified privacy disconnect. Confirmed Telegram `/unlink` revokes the active chat link, but prior WCTM and Telegram records remain. Plugin **uninstall** deletes connector options and matching local webhooks when WooCommerce APIs are available; it does not erase WCTM, OneDrive or Telegram data. If WooCommerce was inactive during uninstall, check and remove remaining WCTM webhooks after reactivation. Rotate credentials when a connection is retired or compromised. For a private-pilot service-side export, disconnect or erasure request, contact your Walterbyte pilot contact through the agreed support channel and identify your account/store. An operator verifies the requester and exact Store, inspects the scope, and can revoke processing access and erase/anonymize Store-scoped live records through a protected maintenance workflow. Narrow security/audit identifiers remain; Telegram messages, merchant WooCommerce records and historical backups are outside live Store erasure. Older backups expire by retention and must have outstanding erasure requests replayed after restore. Whole-account deletion involving shared users/Telegram identities needs separate review. Real-merchant onboarding remains blocked pending the PPR-2 operational and retention gates. + +Store export files are created only for an explicit private-pilot request in a protected operator directory. They are removed after confirmed delivery or completed Store erasure; an hourly cleanup is designed to remove abandoned files within 24 hours. A delivered copy is governed by the agreed secure delivery channel, and historical backups follow their separate retention rules. diff --git a/docs/privacy/PPR2_DATA_PROTECTION.md b/docs/privacy/PPR2_DATA_PROTECTION.md index 93d14ef..88d7d74 100644 --- a/docs/privacy/PPR2_DATA_PROTECTION.md +++ b/docs/privacy/PPR2_DATA_PROTECTION.md @@ -38,7 +38,39 @@ is 0600/0400, and is never committed. The backend runtime DB role is rejected. Execution of disconnect/scrub/erase/replay refuses to run while backend or bot containers are running. This is an intentional maintenance window, not an API that merchant input can invoke. Inspect/export are read-only with respect to -the database; export creates a new mode-0600 file in the protected directory. +the database. `export` requires `--retain-for-delivery`; the wrapper generates +an exact Store-scoped `wctm-privacy-export---.json` +name inside the protected directory and the CLI creates it mode 0600. No +operator-supplied output path is accepted by the wrapper. The hash binds the +file to the exact Tenant/Store without putting merchant data in its name. + +After a successful `erase --execute`, the wrapper removes only matching exports +for that Tenant/Store. After a merchant export has been securely delivered, +the operator runs this scoped cleanup to finish that request: + +```sh +node scripts/ops/privacy-export-artifacts.mjs purge /var/lib/wctm/privacy TENANT_ID STORE_ID +``` + +Delivery +outside the protected directory must use an approved protected channel; the +file is not a long-term archive. Every operator entry also sweeps expired +exports. Install and enable the reviewed +`ops/systemd/wctm-privacy-export-retention.service` and `.timer` for the +canonical `/var/lib/wctm/privacy` directory, owned by the `wctm` service +account with mode 0700. The timer runs under that existing account and does not +require another account to receive Docker or ledger access; its unit hides the +Docker socket and production `.env`. Install the two +unit files from `ops/systemd/`, reload systemd, enable the timer and verify the +first one-shot sweep succeeds before using real merchant exports. The hourly +timer deletes exact export artifacts at +23 hours, using the earliest valid name/filesystem creation evidence, so an +operating timer bounds abandoned files to at most 24 hours; +check timer/service failures during pilot operations. The sweep refuses +symlinks and non-0600 matching files and never selects the erasure ledger, +keys, backups or arbitrary operator files. Removal unlinks the file; storage +snapshots and filesystem remnants remain subject to their own retention and +access controls. Disconnect first revokes the connector token, registration token, webhook route and secret, replaces the encrypted Woo REST key/secret with unusable random @@ -73,18 +105,19 @@ retention; they are not rewritten in place. ## Explicit pilot retention decisions -| Data | Repository rule | Boundary | -| ----------------------------------------------------------- | ------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| BullMQ completed jobs | 24 hours; terminal cleanup hourly, up to 1000 per sweep | Active/waiting/delayed work is never selected. Heavy backlog may take additional sweeps. | -| BullMQ failed jobs | 7 days; same terminal sweep | Diagnostic result/IDs persist until then. | -| Redis rate-limit and dedupe state | Existing 60-second rate windows; job keys as above | Redis AOF is persistent and old bytes may remain until compaction; physical data expiry needs live proof. | -| Link/registration tokens and encrypted callback/search text | Clear 1 day after their expiry; daily sweep | Existing active TTLs remain unchanged. | -| Completed webhook payload | Clear 30 days after completion; daily sweep | Event metadata/dedupe stays for operational integrity. | -| Failed webhook payload | Clear 90 days after failure; daily sweep | Pending/active events are untouched. | -| Security audit rows | 365 days; daily sweep | Minimal erasure ledger is held separately for restore safety. | -| Local PostgreSQL backups | Existing 14 validated sets | Newer 14 sets always protected; this is count-based, not a calendar age. | -| New encrypted OneDrive sets | 30 days, always keep newest 2 complete sets | Legacy plaintext and incomplete sets are never auto-deleted. | -| Application and scheduler logs | No repository-enforced calendar limit yet | Host Docker/journald policy and live proof are a pilot gate. | +| Data | Repository rule | Boundary | +| ----------------------------------------------------------- | ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| BullMQ completed jobs | 24 hours; terminal cleanup hourly, up to 1000 per sweep | Active/waiting/delayed work is never selected. Heavy backlog may take additional sweeps. | +| BullMQ failed jobs | 7 days; same terminal sweep | Diagnostic result/IDs persist until then. | +| Redis rate-limit and dedupe state | Existing 60-second rate windows; job keys as above | Redis AOF is persistent and old bytes may remain until compaction; physical data expiry needs live proof. | +| Link/registration tokens and encrypted callback/search text | Clear 1 day after their expiry; daily sweep | Existing active TTLs remain unchanged. | +| Completed webhook payload | Clear 30 days after completion; daily sweep | Event metadata/dedupe stays for operational integrity. | +| Failed webhook payload | Clear 90 days after failure; daily sweep | Pending/active events are untouched. | +| Security audit rows | 365 days; daily sweep | Minimal erasure ledger is held separately for restore safety. | +| Protected Store privacy exports | Remove after erasure or confirmed delivery; 24-hour maximum | Exact scoped names only; hourly sweep begins at 23 hours, with a sweep on every operator entry. Timer installation/live proof remains a pilot gate. | +| Local PostgreSQL backups | Existing 14 validated sets | Newer 14 sets always protected; this is count-based, not a calendar age. | +| New encrypted OneDrive sets | 30 days, always keep newest 2 complete sets | Legacy plaintext and incomplete sets are never auto-deleted. | +| Application and scheduler logs | No repository-enforced calendar limit yet | Host Docker/journald policy and live proof are a pilot gate. | ## WordPress and backup confidentiality @@ -135,6 +168,9 @@ retention/deletion decision. Treat them as sensitive until removed. promising complete account erasure. 5. Verify WordPress salt-loss reconnect/rotation and Woo-owned signing-secret cleanup on a representative pilot merchant site. +6. Install and verify the protected privacy-export retention timer on the + operator host before processing real merchant exports. Review its service + status and confirm the privacy directory ownership and mode. Until these gates are closed and accepted, PPR-2 remains active and real-merchant pilot onboarding remains blocked. No public-launch readiness is claimed. diff --git a/ops/systemd/wctm-privacy-export-retention.service b/ops/systemd/wctm-privacy-export-retention.service new file mode 100644 index 0000000..6996702 --- /dev/null +++ b/ops/systemd/wctm-privacy-export-retention.service @@ -0,0 +1,16 @@ +[Unit] +Description=Remove expired WCTM pilot privacy exports + +[Service] +Type=oneshot +User=wctm +Group=wctm +WorkingDirectory=/srv/wctm +ExecStart=/usr/bin/env node /srv/wctm/scripts/ops/privacy-export-artifacts.mjs sweep /var/lib/wctm/privacy +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/wctm/privacy +InaccessiblePaths=-/run/docker.sock -/srv/wctm/.env diff --git a/ops/systemd/wctm-privacy-export-retention.timer b/ops/systemd/wctm-privacy-export-retention.timer new file mode 100644 index 0000000..6970792 --- /dev/null +++ b/ops/systemd/wctm-privacy-export-retention.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Sweep WCTM pilot privacy exports hourly + +[Timer] +OnCalendar=hourly +Persistent=true +AccuracySec=1s +Unit=wctm-privacy-export-retention.service + +[Install] +WantedBy=timers.target diff --git a/scripts/ops/pilot-privacy.sh b/scripts/ops/pilot-privacy.sh index 99f11ba..126477c 100755 --- a/scripts/ops/pilot-privacy.sh +++ b/scripts/ops/pilot-privacy.sh @@ -23,6 +23,7 @@ directory_mode=$(stat -c '%a' "$privacy_directory" 2>/dev/null || stat -f '%Lp' echo >&2 'privacy operation refused: credential must be 0600/0400 and directory 0700' exit 65 } +privacy_directory=$(cd "$privacy_directory" && pwd -P) database_url='' count=0 while IFS= read -r line || [[ -n "$line" ]]; do @@ -55,6 +56,36 @@ case "$mode" in *) echo >&2 'privacy operation refused: invalid mode'; exit 64 ;; esac +# Expired exports are swept on every operator entry as well as by the hourly timer. +node scripts/ops/privacy-export-artifacts.mjs sweep "$privacy_directory" + +target_ids() { + tenant_id='' + store_id='' + while (($#)); do + case "$1" in + --tenant-id) + [[ -z "$tenant_id" && $# -ge 2 ]] || return 1 + tenant_id=$2; shift 2 ;; + --store-id) + [[ -z "$store_id" && $# -ge 2 ]] || return 1 + store_id=$2; shift 2 ;; + *) shift ;; + esac + done + [[ -n "$tenant_id" && -n "$store_id" ]] +} + +if [[ "$mode" = export ]]; then + [[ " $* " = *' --retain-for-delivery '* && " $* " != *' --output '* ]] || { + echo >&2 'privacy export refused: explicit delivery retention is required; output path is managed' + exit 64 + } + target_ids "$@" || { echo >&2 'privacy export refused: exact target IDs are required'; exit 64; } + output=$(node scripts/ops/privacy-export-artifacts.mjs new-path "$privacy_directory" "$tenant_id" "$store_id") + set -- "$@" --output "$output" +fi + if [[ " $* " = *' --execute '* ]]; then case "$mode" in disconnect|scrub|erase|replay) @@ -93,6 +124,11 @@ if [[ "$mode" = erase && " $* " = *' --execute '* ]]; then run_privacy prepare-erasure "$@" scripts/ops/archive-erasure-ledger.sh "$ledger" "${WCTM_OFFSITE_DESTINATION%/}/privacy-erasure-ledger" run_privacy erase "$@" + target_ids "$@" + node scripts/ops/privacy-export-artifacts.mjs purge "$privacy_directory" "$tenant_id" "$store_id" else run_privacy "$mode" "$@" + if [[ "$mode" = export ]]; then + printf 'privacy export retained for delivery: %s\n' "$output" + fi fi diff --git a/scripts/ops/privacy-export-artifacts.mjs b/scripts/ops/privacy-export-artifacts.mjs new file mode 100644 index 0000000..b6278d2 --- /dev/null +++ b/scripts/ops/privacy-export-artifacts.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node +import { createHash, randomBytes } from 'node:crypto'; +import { lstat, readdir, realpath, unlink } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; + +const namePattern = + /^wctm-privacy-export-([0-9a-f]{64})-([0-9]{13})-([0-9a-f]{16})\.json$/; +// An hourly timer removes files at 23 hours, leaving room before the 24-hour limit. +const sweepAgeMs = 23 * 60 * 60 * 1000; + +function scopeHash(tenantId, storeId) { + if ( + !/^ten_[A-Za-z0-9_-]+$/.test(tenantId) || + !/^sto_[A-Za-z0-9_-]+$/.test(storeId) + ) { + throw new Error('privacy export: exact Tenant and Store IDs are required'); + } + return createHash('sha256').update(`${tenantId}\0${storeId}`).digest('hex'); +} + +async function protectedDirectory(path) { + if ( + !path?.startsWith('/') || + resolve(path) === '/' || + path.split('/').includes('..') + ) { + throw new Error( + 'privacy export: an absolute protected directory is required' + ); + } + const details = await lstat(path); + if (!details.isDirectory() || (details.mode & 0o777) !== 0o700) { + throw new Error('privacy export: directory must be a 0700 real directory'); + } + const directory = await realpath(path); + const ledger = await lstat(join(directory, 'erasure-ledger.jsonl')); + if (!ledger.isFile() || (ledger.mode & 0o777) !== 0o600) { + throw new Error('privacy export: protected erasure ledger is required'); + } + return directory; +} + +async function removeEligible(directory, selectedScope) { + let removed = 0; + const now = Date.now(); + for (const name of await readdir(directory)) { + const match = namePattern.exec(name); + if (!match || (selectedScope && match[1] !== selectedScope)) continue; + const createdAt = Number(match[2]); + if (!Number.isSafeInteger(createdAt)) continue; + const path = join(directory, name); + const details = await lstat(path); + if (!details.isFile() || (details.mode & 0o777) !== 0o600) { + throw new Error( + 'privacy export: matching artifact is not a 0600 regular file' + ); + } + const earliestAgeEvidence = Math.min( + createdAt, + details.mtimeMs, + details.birthtimeMs > 0 ? details.birthtimeMs : Infinity + ); + if (!selectedScope && now - earliestAgeEvidence < sweepAgeMs) continue; + await unlink(path); + removed++; + } + return removed; +} + +async function main() { + const [mode, directoryArg, tenantId, storeId] = process.argv.slice(2); + if ( + !['new-path', 'sweep', 'purge'].includes(mode) || + (mode === 'sweep' && process.argv.length !== 4) || + (mode !== 'sweep' && process.argv.length !== 6) + ) { + throw new Error( + 'usage: privacy-export-artifacts.mjs new-path|purge DIRECTORY TENANT_ID STORE_ID | sweep DIRECTORY' + ); + } + const directory = await protectedDirectory(directoryArg); + if (mode === 'new-path') { + const name = `wctm-privacy-export-${scopeHash(tenantId, storeId)}-${Date.now()}-${randomBytes(8).toString('hex')}.json`; + process.stdout.write(`${join(directory, name)}\n`); + return; + } + const removed = await removeEligible( + directory, + mode === 'purge' ? scopeHash(tenantId, storeId) : undefined + ); + process.stdout.write(`privacy export cleanup: PASS removed=${removed}\n`); +} + +main().catch(() => { + process.stderr.write( + 'privacy export cleanup failed; inspect protected directory and operator configuration\n' + ); + process.exitCode = 1; +}); diff --git a/scripts/ops/test-pilot-privacy.sh b/scripts/ops/test-pilot-privacy.sh index abab43b..c00253f 100755 --- a/scripts/ops/test-pilot-privacy.sh +++ b/scripts/ops/test-pilot-privacy.sh @@ -14,6 +14,14 @@ cp scripts/ops/test-fixtures/rclone "$workspace/bin/rclone" cat >"$workspace/bin/docker" <<'SH' #!/usr/bin/env bash printf '%s\n' "$*" >>"$TEST_DOCKER_LOG" +previous='' +for argument in "$@"; do + if [[ "$previous" = --output ]]; then + printf 'fixture private export\n' >"$argument" + chmod 0600 "$argument" + fi + previous=$argument +done if [[ "$*" = 'compose ps --status running -q backend telegram-bot' && ${TEST_APP_RUNNING:-false} = true ]]; then printf 'running-container\n' @@ -22,6 +30,7 @@ SH chmod 0700 "$workspace/bin/docker" "$workspace/bin/rclone" export PATH="$workspace/bin:$PATH" export TEST_DOCKER_LOG="$workspace/docker.log" +export TEST_PRIVACY_DIRECTORY="$workspace/privacy" export WCTM_BACKUP_CRYPTO_KEY_FILE="$workspace/key" export RCLONE_CONFIG="$workspace/rclone.conf" export WCTM_OFFSITE_DESTINATION="testremote:$workspace/remote" @@ -32,6 +41,20 @@ scripts/ops/pilot-privacy.sh --credential-file "$workspace/database.conf" \ >"$workspace/inspect.log" [[ $(wc -l <"$workspace/docker.log" | tr -d ' ') = 1 ]] +if scripts/ops/pilot-privacy.sh --credential-file "$workspace/database.conf" \ + --privacy-directory "$workspace/privacy" -- export \ + --tenant-id ten_fixture --store-id sto_fixture --base-url https://fixture.test \ + >"$workspace/export-refused.log" 2>&1; then + echo >&2 'test failed: export without explicit delivery retention succeeded' + exit 1 +fi +scripts/ops/pilot-privacy.sh --credential-file "$workspace/database.conf" \ + --privacy-directory "$workspace/privacy" -- export \ + --tenant-id ten_fixture --store-id sto_fixture --base-url https://fixture.test \ + --retain-for-delivery >"$workspace/export.log" +export_path=$(sed -n 's/^privacy export retained for delivery: //p' "$workspace/export.log") +[[ -f "$export_path" && $(stat -c '%a' "$export_path" 2>/dev/null || stat -f '%Lp' "$export_path") = 600 ]] + scripts/ops/pilot-privacy.sh --credential-file "$workspace/database.conf" \ --privacy-directory "$workspace/privacy" -- erase \ --tenant-id ten_fixture --store-id sto_fixture --base-url https://fixture.test --execute \ @@ -40,6 +63,7 @@ grep -q 'pilot-data.cli.js prepare-erasure' "$workspace/docker.log" grep -q 'pilot-data.cli.js erase' "$workspace/docker.log" grep -q 'erasure archive: PASS' "$workspace/erase.log" [[ $(find "$workspace/remote/privacy-erasure-ledger" -name '*.jsonl.enc' | wc -l | tr -d ' ') = 1 ]] +[[ ! -e "$export_path" ]] ! grep -q 'fixture-only' "$workspace"/*.log unset WCTM_OFFSITE_DESTINATION @@ -50,7 +74,7 @@ if scripts/ops/pilot-privacy.sh --credential-file "$workspace/database.conf" \ echo >&2 'test failed: erasure proceeded without off-site ledger destination' exit 1 fi -[[ $(wc -l <"$workspace/docker.log" | tr -d ' ') = 5 ]] +[[ $(wc -l <"$workspace/docker.log" | tr -d ' ') = 6 ]] export WCTM_OFFSITE_DESTINATION="testremote:$workspace/remote" if TEST_APP_RUNNING=true scripts/ops/pilot-privacy.sh \ @@ -62,4 +86,4 @@ if TEST_APP_RUNNING=true scripts/ops/pilot-privacy.sh \ exit 1 fi grep -q 'stop backend and bot' "$workspace/running.log" -echo 'pilot privacy wrapper: PASS protected-config dry-run archive-before-erasure fail-closed' +echo 'pilot privacy wrapper: PASS protected-config explicit export retention erase cleanup archive-before-erasure fail-closed' diff --git a/scripts/ops/test-privacy-export-artifacts.mjs b/scripts/ops/test-privacy-export-artifacts.mjs new file mode 100644 index 0000000..a236e3e --- /dev/null +++ b/scripts/ops/test-privacy-export-artifacts.mjs @@ -0,0 +1,126 @@ +#!/usr/bin/env node +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdtemp, + mkdir, + readFile, + rm, + symlink, + utimes, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const root = await mkdtemp(join(tmpdir(), 'wctm-privacy-export-test-')); +const directory = join(root, 'privacy'); +const outside = join(root, 'outside'); +const helper = 'scripts/ops/privacy-export-artifacts.mjs'; +const run = (...args) => + execFileSync(process.execPath, [helper, ...args], { + encoding: 'utf8', + }).trim(); +const tenantA = 'ten_fixture_a'; +const storeA = 'sto_fixture_a'; +const tenantB = 'ten_fixture_b'; +const storeB = 'sto_fixture_b'; +const create = async (tenant, store, ageHours = 0) => { + let path = run('new-path', directory, tenant, store); + if (ageHours) { + path = path.replace( + /-[0-9]{13}-([0-9a-f]{16}\.json)$/, + `-${Date.now() - ageHours * 3600000}-$1` + ); + } + await writeFile(path, 'fixture private data', { mode: 0o600 }); + return path; +}; +const exists = async (path) => + readFile(path).then( + () => true, + () => false + ); + +try { + await mkdir(directory, { mode: 0o700 }); + await mkdir(outside, { mode: 0o700 }); + const ledger = join(directory, 'erasure-ledger.jsonl'); + const key = join(directory, 'backup.key'); + const unrelated = join(directory, 'operator-created.json'); + const lookalike = join( + directory, + 'wctm-privacy-export-operator-created.json' + ); + const backup = join(directory, 'wctm-postgres-fixture.dump'); + const outsideFile = join(outside, 'outside.json'); + for (const path of [ledger, key, unrelated, lookalike, backup, outsideFile]) { + await writeFile(path, 'must survive', { mode: 0o600 }); + } + const stale = await create(tenantA, storeA, 25); + const fresh = await create(tenantA, storeA); + const otherScope = await create(tenantB, storeB, 25); + const oldMtime = await create(tenantA, storeB); + const yesterday = new Date(Date.now() - 25 * 3600000); + await utimes(oldMtime, yesterday, yesterday); + assert.match(run('sweep', directory), /removed=3$/); + assert.equal(await exists(stale), false); + assert.equal(await exists(otherScope), false); + assert.equal(await exists(oldMtime), false); + assert.equal(await exists(fresh), true); + + const delivered = await create(tenantA, storeA); + const untouchedTenant = await create(tenantB, storeB); + const sameStoreOtherTenant = await create(tenantB, storeA); + const sameTenantOtherStore = await create(tenantA, storeB); + assert.match(run('purge', directory, tenantA, storeA), /removed=2$/); + assert.equal(await exists(fresh), false); + assert.equal(await exists(delivered), false); + assert.equal(await exists(untouchedTenant), true); + assert.equal(await exists(sameStoreOtherTenant), true); + assert.equal(await exists(sameTenantOtherStore), true); + for (const path of [ledger, key, unrelated, lookalike, backup, outsideFile]) { + assert.equal(await exists(path), true); + } + + const symlinkName = run('new-path', directory, tenantA, storeA).replace( + /-[0-9]{13}-([0-9a-f]{16}\.json)$/, + `-${Date.now() - 25 * 3600000}-$1` + ); + await symlink(outsideFile, symlinkName); + const refused = spawnSync(process.execPath, [helper, 'sweep', directory], { + encoding: 'utf8', + }); + assert.notEqual(refused.status, 0); + assert.equal(await exists(outsideFile), true); + const escape = spawnSync( + process.execPath, + [helper, 'sweep', `${directory}/../outside`], + { encoding: 'utf8' } + ); + assert.notEqual(escape.status, 0); + const wrongDirectory = spawnSync( + process.execPath, + [helper, 'sweep', outside], + { encoding: 'utf8' } + ); + assert.notEqual(wrongDirectory.status, 0); + const linkedDirectory = join(root, 'linked-privacy'); + await symlink(directory, linkedDirectory); + const linked = spawnSync( + process.execPath, + [helper, 'sweep', linkedDirectory], + { encoding: 'utf8' } + ); + assert.notEqual(linked.status, 0); + assert.equal(await exists(outsideFile), true); + assert.doesNotMatch( + `${refused.stderr}${escape.stderr}${wrongDirectory.stderr}${linked.stderr}`, + /fixture private data|must survive/ + ); + process.stdout.write( + 'privacy export retention: PASS expired/fresh exact scope protected files symlink traversal secret-safe\n' + ); +} finally { + await rm(root, { recursive: true, force: true }); +}