diff --git a/deploy/README.md b/deploy/README.md index eed8033b..a534c5d5 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -152,6 +152,26 @@ With sign-in on, invite customers from a web pod as delete the Job, and Flux runs it again. A ledger newer than the release refuses the Job, so roll forward rather than back. +## Portal retention purge + +`cronjob-portal-retention.yaml` runs the retention purge +([server/README.md](../server/README.md#retention-purge-job)) every day at 03:17 Copenhagen +time. It deletes the portal records of former clients 3 years after their last membership ended, +as the privacy notice states: + +- **Image and input.** Same as the migration Job. `kustomization.yaml` copies the WebApp's + pinned image into it, and it reads the platform's `business-site-portal-db-migration` Secret + through `--migration-database-projection`, as the migration role. The runtime cannot delete + this history. +- **Network.** Its pods carry the migration Job's `app.kubernetes.io/name: + business-site-portal-migration` label, with `app.kubernetes.io/component: retention-purge`. + The platform's migration policy is therefore its only path to the database. +- **Runs.** `concurrencyPolicy: Forbid`, no retries and a 300-second deadline. A failed run + deletes nothing; a later run tries again. Flux does not wait on a CronJob, so a failed run + never makes the tenant unhealthy. Each run logs only how many former clients lost records. +- **Before the migration.** While a new release's migration Job has not finished, the purge's + schema ownership check refuses to run, so it never deletes against an older schema. + ## Portal billing inputs The portal invoices through Numina and takes cards through Stripe diff --git a/deploy/cronjob-portal-retention.yaml b/deploy/cronjob-portal-retention.yaml new file mode 100644 index 00000000..c896fcd1 --- /dev/null +++ b/deploy/cronjob-portal-retention.yaml @@ -0,0 +1,79 @@ +# Runs the portal retention purge once a day (server/README.md, "Retention purge +# job"), with the release's own image: kustomization.yaml copies the WebApp's +# pinned image into it. It reads the migration Job's Secret as the migration +# role, and its pods carry the migration Job's label, so the platform's migration +# network policy is the only path to the database. A failed run deletes nothing +# and the next day's run tries again. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: business-site-portal-retention + namespace: business-site +spec: + schedule: "17 3 * * *" + timeZone: Europe/Copenhagen + concurrencyPolicy: Forbid + startingDeadlineSeconds: 3600 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 300 + template: + metadata: + labels: + app.kubernetes.io/name: business-site-portal-migration + app.kubernetes.io/component: retention-purge + spec: + restartPolicy: Never + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: purge + args: + - --retention-purge + - --migration-database-projection + - /run/portal-migration/database-url + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 500m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: migration + mountPath: /run/portal-migration + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: migration + secret: + secretName: business-site-portal-db-migration + defaultMode: 0400 + items: + - key: database-url + path: database-url + - key: ca.crt + path: ca.crt + # As for the migration Job: the CA copy needs a sticky temp directory, + # and only a memory emptyDir is 1777. + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 1Mi diff --git a/deploy/kustomization.yaml b/deploy/kustomization.yaml index c0aa3453..a4152f15 100644 --- a/deploy/kustomization.yaml +++ b/deploy/kustomization.yaml @@ -11,7 +11,9 @@ resources: - external-secret-numina.yaml - external-secret-stripe.yaml - job-portal-migration.yaml -# The publisher pins only the WebApp's image; the migration Job runs that same image. + - cronjob-portal-retention.yaml +# The publisher pins only the WebApp's image; the migration Job and the retention +# CronJob run that same image. replacements: - source: kind: WebApp @@ -25,3 +27,10 @@ replacements: - spec.template.spec.containers.[name=migrate].image options: create: true + - select: + kind: CronJob + name: business-site-portal-retention + fieldPaths: + - spec.jobTemplate.spec.template.spec.containers.[name=purge].image + options: + create: true diff --git a/scripts/check-tenant-package.mjs b/scripts/check-tenant-package.mjs index 9550f5c6..60d10621 100644 --- a/scripts/check-tenant-package.mjs +++ b/scripts/check-tenant-package.mjs @@ -13,7 +13,7 @@ function keys(value, allowed) { export function validateTenantPackage(input) { assert.ok(typeof input === "string" && Buffer.byteLength(input) <= 262144, "input size"); const docs = yaml.loadAll(input); - assert.equal(docs.length, 10, "resource count"); + assert.equal(docs.length, 11, "resource count"); const app = docs.find((d) => d?.kind === "WebApp"); const network = docs.find((d) => d?.kind === "CiliumNetworkPolicy"); const budget = docs.find((d) => d?.kind === "PodDisruptionBudget"); @@ -164,6 +164,39 @@ export function validateTenantPackage(input) { }, }, }); + // The one retention CronJob runs this release's image once a day with only + // the purge and the migration Job's Secret, label and pod hardening. + const job = docs.find((d) => d?.kind === "Job"); + assert.deepEqual(docs.find((d) => d?.kind === "CronJob"), { + apiVersion: "batch/v1", + kind: "CronJob", + metadata: { name: "business-site-portal-retention", namespace: "business-site" }, + spec: { + schedule: "17 3 * * *", + timeZone: "Europe/Copenhagen", + concurrencyPolicy: "Forbid", + startingDeadlineSeconds: 3600, + successfulJobsHistoryLimit: 1, + failedJobsHistoryLimit: 3, + jobTemplate: { + spec: { + backoffLimit: 0, + activeDeadlineSeconds: 300, + template: { + metadata: { labels: { "app.kubernetes.io/name": "business-site-portal-migration", "app.kubernetes.io/component": "retention-purge" } }, + spec: { + ...job.spec.template.spec, + containers: [{ + ...job.spec.template.spec.containers[0], + name: "purge", + args: ["--retention-purge", "--migration-database-projection", "/run/portal-migration/database-url"], + }], + }, + }, + }, + }, + }, + }); assert.deepEqual(network.spec, { endpointSelector: { matchLabels: { "app.kubernetes.io/name": "business-site" } }, ingress: [{ fromEntities: ["ingress"], toPorts: [{ ports: [{ port: "8080", protocol: "TCP" }] }] }], diff --git a/scripts/runtime-verification.test.sh b/scripts/runtime-verification.test.sh index a90cf344..97ecacba 100644 --- a/scripts/runtime-verification.test.sh +++ b/scripts/runtime-verification.test.sh @@ -95,6 +95,8 @@ if [[ "${MISSING_STRIPE:-false}" == true ]]; then rm "$5/external-secret-stripe. if [[ "${ALTER_NUMINA:-false}" == true ]]; then yq -i '.spec.data[0].remoteRef.key="apps/other/numina"' "$5/external-secret-numina.yaml"; fi if [[ "${MISSING_MIGRATION:-false}" == true ]]; then rm "$5/job-portal-migration.yaml"; fi if [[ "${ALTER_MIGRATION:-false}" == true ]]; then yq -i '.spec.template.spec.containers[0].args[1]="--migration-database-file"' "$5/job-portal-migration.yaml"; fi +if [[ "${MISSING_RETENTION:-false}" == true ]]; then rm "$5/cronjob-portal-retention.yaml"; fi +if [[ "${ALTER_RETENTION:-false}" == true ]]; then yq -i '.spec.schedule="* * * * *"' "$5/cronjob-portal-retention.yaml"; fi FIXTURE chmod +x "$scratch/bin/gh" "$scratch/bin/cosign" "$scratch/bin/flux" # Restore independent signed-subject fixtures and clear receipts/read counters between controls. @@ -146,7 +148,7 @@ if verify TAG_SOURCE=1123456789abcdef0123456789abcdef01234567; then fail 'a move [[ ! -e "$scratch/runner/runtime-publication.json" ]] || fail 'moved Git tag left a receipt' reset_fixture verify TAG_KIND=tag || fail 'a matching annotated release tag must verify' -for context in VERIFY_FAIL=true PULL_FAIL=true CI_ATTEMPT=3 PUBLISH_ATTEMPT=2 ALTER_MANIFEST=true EXTRA_MEMBER=true MISSING_STORE=true MISSING_DISCLOSURE=true ALTER_STORE=true ALTER_DISCLOSURE=true MISSING_STRIPE=true ALTER_NUMINA=true MISSING_MIGRATION=true ALTER_MIGRATION=true VERSION=01.2.3 RELEASE_DRAFT=true RELEASE_PRERELEASE=true PUBLISHED_AT= WITHDRAW_AFTER_VERIFY=true MOVE_AFTER_VERIFY=true; do +for context in VERIFY_FAIL=true PULL_FAIL=true CI_ATTEMPT=3 PUBLISH_ATTEMPT=2 ALTER_MANIFEST=true EXTRA_MEMBER=true MISSING_STORE=true MISSING_DISCLOSURE=true ALTER_STORE=true ALTER_DISCLOSURE=true MISSING_STRIPE=true ALTER_NUMINA=true MISSING_MIGRATION=true ALTER_MIGRATION=true MISSING_RETENTION=true ALTER_RETENTION=true VERSION=01.2.3 RELEASE_DRAFT=true RELEASE_PRERELEASE=true PUBLISHED_AT= WITHDRAW_AFTER_VERIFY=true MOVE_AFTER_VERIFY=true; do reset_fixture if verify "$context"; then fail 'failed signature/pull, stale attempt or substituted manifests were accepted'; fi [[ ! -e "$scratch/runner/runtime-publication.json" ]] || fail 'refusal left a successful publication receipt' diff --git a/scripts/tenant-package.test.mjs b/scripts/tenant-package.test.mjs index 1c149a8f..a92ba419 100644 --- a/scripts/tenant-package.test.mjs +++ b/scripts/tenant-package.test.mjs @@ -243,6 +243,18 @@ for (const [name, mutate] of [ ["no migration deadline", (d) => { delete d[9].spec.activeDeadlineSeconds; }], ["a migration Job Flux cannot recreate", (d) => { delete d[9].metadata.annotations; }], ["a migration pod labelled as the web workload", (d) => { d[9].spec.template.metadata.labels["app.kubernetes.io/name"] = "business-site"; }], + ["a missing retention CronJob", (d) => { d.splice(10, 1); }], + ["a retention CronJob on another image", (d) => { d[10].spec.jobTemplate.spec.template.spec.containers[0].image = `ghcr.io/devantler-tech/business-site@sha256:${"a".repeat(64)}`; }], + ["a retention CronJob that migrates", (d) => { d[10].spec.jobTemplate.spec.template.spec.containers[0].args = structuredClone(d[9].spec.template.spec.containers[0].args); }], + ["the strict file option in the retention CronJob", (d) => { d[10].spec.jobTemplate.spec.template.spec.containers[0].args[1] = "--migration-database-file"; }], + ["overlapping retention runs", (d) => { d[10].spec.concurrencyPolicy = "Allow"; }], + ["a retention purge more than once a day", (d) => { d[10].spec.schedule = "*/5 * * * *"; }], + ["retention retries", (d) => { d[10].spec.jobTemplate.spec.backoffLimit = 3; }], + ["no retention deadline", (d) => { delete d[10].spec.jobTemplate.spec.activeDeadlineSeconds; }], + ["a retention pod running as root", (d) => { d[10].spec.jobTemplate.spec.template.spec.securityContext.runAsUser = 0; }], + ["a retention pod with a service-account token", (d) => { d[10].spec.jobTemplate.spec.template.spec.automountServiceAccountToken = true; }], + ["the web runtime Secret in the retention CronJob", (d) => { d[10].spec.jobTemplate.spec.template.spec.volumes[0].secret.secretName = "business-site-runtime"; }], + ["a retention pod labelled as the web workload", (d) => { d[10].spec.jobTemplate.spec.template.metadata.labels["app.kubernetes.io/name"] = "business-site"; }], ]) { test(`the actual rendered package rejects ${name}`, () => { const rendered = yaml.loadAll(render()); @@ -255,8 +267,8 @@ for (const [name, mutate] of [ for (const name of ["business-site-portal", "business-site-public"]) docs.push(named("HTTPRoute", name)); // The billing synchronizations follow at indexes 7 (Numina) and 8 (Stripe). for (const name of ["business-site-numina", "business-site-stripe"]) docs.push(named("ExternalSecret", name)); - // The migration Job follows at index 9. - docs.push(named("Job", "business-site-portal-migration")); + // The migration Job follows at index 9 and the retention CronJob at 10. + docs.push(named("Job", "business-site-portal-migration"), named("CronJob", "business-site-portal-retention")); assert.equal(check(docs.map((d) => yaml.dump(d)).join("---\n")).status, 0, "each refusal must start from a complete accepted package"); mutate(docs); @@ -267,9 +279,9 @@ for (const [name, mutate] of [ }); } -test("the migration Job runs whatever image the publisher pins in the WebApp", () => { +test("the migration Job and retention CronJob run whatever image the publisher pins in the WebApp", () => { // The shared publisher rewrites only webapp.yaml's WebApp image; the package - // build copies it into the Job, so the Job never runs another release's image. + // build copies it into both jobs, so neither runs another release's image. const fixture = mkdtempSync(path.join(tmpdir(), "tenant-package-")); try { cpSync(path.join(root, "deploy"), fixture, { recursive: true }); @@ -277,8 +289,13 @@ test("the migration Job runs whatever image the publisher pins in the WebApp", ( const app = yaml.load(readFileSync(path.join(fixture, "webapp.yaml"), "utf8")); app.spec.image = pinned; writeFileSync(path.join(fixture, "webapp.yaml"), yaml.dump(app)); - const job = yaml.loadAll(render(fixture)).find((d) => d?.kind === "Job"); + const rendered = yaml.loadAll(render(fixture)); + const job = rendered.find((d) => d?.kind === "Job"); assert.equal(job.spec.template.spec.containers[0].image, pinned); + const cron = rendered.find((d) => d?.kind === "CronJob"); + assert.equal(cron.spec.jobTemplate.spec.template.spec.containers[0].image, pinned); + const cronSource = yaml.load(readFileSync(path.join(root, "deploy/cronjob-portal-retention.yaml"), "utf8")); + assert.equal(cronSource.spec.jobTemplate.spec.template.spec.containers[0].image, undefined, "the source CronJob names no image of its own"); const source = yaml.load(readFileSync(path.join(root, "deploy/job-portal-migration.yaml"), "utf8")); assert.equal(source.spec.template.spec.containers[0].image, undefined, "the source Job names no image of its own"); } finally { diff --git a/scripts/tenant-runtime-smoke.test.mjs b/scripts/tenant-runtime-smoke.test.mjs index ccd3e040..a59e1555 100644 --- a/scripts/tenant-runtime-smoke.test.mjs +++ b/scripts/tenant-runtime-smoke.test.mjs @@ -4,13 +4,15 @@ import { execFileSync, spawnSync } from "node:child_process"; import { request } from "node:http"; import { fileURLToPath } from "node:url"; import test from "node:test"; +import * as yaml from "js-yaml"; import { validateTenantPackage } from "./check-tenant-package.mjs"; import { containerGroupDisclosureFixture, runtimeSecretFixture } from "./runtime-disclosure-fixture.mjs"; const root = fileURLToPath(new URL("../", import.meta.url)); -const spec = validateTenantPackage(execFileSync("kubectl", ["kustomize", `${root}/deploy`], { +const rendered = execFileSync("kubectl", ["kustomize", `${root}/deploy`], { encoding: "utf8", timeout: 15000, -})); +}); +const spec = validateTenantPackage(rendered); // CI substitutes its newly built production image, just as the later verified // publisher pins a new digest. With no argument, exercise the committed digest. const image = process.argv[2] ?? spec.image; @@ -58,6 +60,21 @@ test("rendered package arguments fail closed without the required disclosure", ( } }); +// The migration Job and the retention CronJob run this same image. With no Secret +// mounted, each must fail on its missing connection, not on its arguments: the +// image accepts exactly the arguments the package gives it. +test("the image accepts the migration Job's and retention CronJob's arguments", () => { + const docs = yaml.loadAll(rendered); + const job = docs.find((d) => d?.kind === "Job").spec.template.spec.containers[0].args; + const cron = docs.find((d) => d?.kind === "CronJob").spec.jobTemplate.spec.template.spec.containers[0].args; + for (const [args, failure] of [[job, "portal schema migration failed\n"], [cron, "portal retention purge failed\n"]]) { + const result = spawnSync("docker", ["run", "--rm", "--network=none", ...restrictions, image, ...args], { encoding: "utf8", timeout: 30000 }); + assert.ifError(result.error); + assert.equal(result.status, 1); + assert.equal(result.stdout + result.stderr, failure, args.join(" ")); + } +}); + // Serving needs the platform's whole runtime Secret, so it must exist before this package. test("rendered package arguments fail closed without the portal settings item", () => { const fixture = containerGroupDisclosureFixture(image, docker); diff --git a/scripts/verify-runtime-publication.sh b/scripts/verify-runtime-publication.sh index fc9efd75..5c898004 100644 --- a/scripts/verify-runtime-publication.sh +++ b/scripts/verify-runtime-publication.sh @@ -116,10 +116,10 @@ flux pull artifact "oci://$artifact@$manifests_digest" --output "$work/manifests entries="$(find "$work/manifests" -mindepth 1 -print)" || fail 'manifest inventory' while IFS= read -r entry; do member=${entry#"$work/manifests/"} - case "$member" in webapp.yaml | http-route-public.yaml | http-route-portal.yaml | network-policy.yaml | disruption-budget.yaml | secret-store.yaml | external-secret-disclosure.yaml | external-secret-numina.yaml | external-secret-stripe.yaml | job-portal-migration.yaml | kustomization.yaml | README.md) ;; *) fail 'unexpected manifest member' ;; esac + case "$member" in webapp.yaml | http-route-public.yaml | http-route-portal.yaml | network-policy.yaml | disruption-budget.yaml | secret-store.yaml | external-secret-disclosure.yaml | external-secret-numina.yaml | external-secret-stripe.yaml | job-portal-migration.yaml | cronjob-portal-retention.yaml | kustomization.yaml | README.md) ;; *) fail 'unexpected manifest member' ;; esac [[ -f "$entry" && ! -L "$entry" && "$(wc -c <"$entry")" -le 1048576 ]] || fail 'manifest member bounds' done <<<"$entries" -for member in webapp.yaml http-route-public.yaml http-route-portal.yaml network-policy.yaml disruption-budget.yaml secret-store.yaml external-secret-disclosure.yaml external-secret-numina.yaml external-secret-stripe.yaml job-portal-migration.yaml kustomization.yaml; do +for member in webapp.yaml http-route-public.yaml http-route-portal.yaml network-policy.yaml disruption-budget.yaml secret-store.yaml external-secret-disclosure.yaml external-secret-numina.yaml external-secret-stripe.yaml job-portal-migration.yaml cronjob-portal-retention.yaml kustomization.yaml; do [[ -f "$work/manifests/$member" && -f "deploy/$member" && ! -L "deploy/$member" ]] || fail 'missing source manifest' yq -o=json -I=0 '.' "deploy/$member" >"$work/source.json" || fail 'source manifest parse' if [[ "$member" == webapp.yaml ]]; then