From 99a6080191db02101d8faac8e1906bb7d6e5611f Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 16 Nov 2026 00:18:28 +0100 Subject: [PATCH 1/2] feat(deploy): run the portal retention purge as a daily CronJob The package ships cronjob-portal-retention.yaml beside the migration Job. It runs the release's own image (kustomization.yaml copies the WebApp's pinned image into it) once a day with --retention-purge --migration-database-projection, reading the platform's migration Secret as the migration role. Its pods carry the migration Job's label, so the platform's migration network policy is their only path to the database. concurrencyPolicy Forbid, no retries, a 300-second deadline, 65532 with fsGroup 65532, a memory /tmp, no service-account token and restricted PSS, as for the migration Job. The package checker, its tests and the publication verifier admit exactly this CronJob. Co-Authored-By: Claude Opus 5.5 --- deploy/README.md | 20 +++++++ deploy/cronjob-portal-retention.yaml | 79 +++++++++++++++++++++++++++ deploy/kustomization.yaml | 11 +++- scripts/check-tenant-package.mjs | 35 +++++++++++- scripts/runtime-verification.test.sh | 4 +- scripts/tenant-package.test.mjs | 27 +++++++-- scripts/verify-runtime-publication.sh | 4 +- 7 files changed, 170 insertions(+), 10 deletions(-) create mode 100644 deploy/cronjob-portal-retention.yaml diff --git a/deploy/README.md b/deploy/README.md index eed8033b..a534c5d5 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -152,6 +152,26 @@ With sign-in on, invite customers from a web pod as delete the Job, and Flux runs it again. A ledger newer than the release refuses the Job, so roll forward rather than back. +## Portal retention purge + +`cronjob-portal-retention.yaml` runs the retention purge +([server/README.md](../server/README.md#retention-purge-job)) every day at 03:17 Copenhagen +time. It deletes the portal records of former clients 3 years after their last membership ended, +as the privacy notice states: + +- **Image and input.** Same as the migration Job. `kustomization.yaml` copies the WebApp's + pinned image into it, and it reads the platform's `business-site-portal-db-migration` Secret + through `--migration-database-projection`, as the migration role. The runtime cannot delete + this history. +- **Network.** Its pods carry the migration Job's `app.kubernetes.io/name: + business-site-portal-migration` label, with `app.kubernetes.io/component: retention-purge`. + The platform's migration policy is therefore its only path to the database. +- **Runs.** `concurrencyPolicy: Forbid`, no retries and a 300-second deadline. A failed run + deletes nothing; a later run tries again. Flux does not wait on a CronJob, so a failed run + never makes the tenant unhealthy. Each run logs only how many former clients lost records. +- **Before the migration.** While a new release's migration Job has not finished, the purge's + schema ownership check refuses to run, so it never deletes against an older schema. + ## Portal billing inputs The portal invoices through Numina and takes cards through Stripe diff --git a/deploy/cronjob-portal-retention.yaml b/deploy/cronjob-portal-retention.yaml new file mode 100644 index 00000000..c896fcd1 --- /dev/null +++ b/deploy/cronjob-portal-retention.yaml @@ -0,0 +1,79 @@ +# Runs the portal retention purge once a day (server/README.md, "Retention purge +# job"), with the release's own image: kustomization.yaml copies the WebApp's +# pinned image into it. It reads the migration Job's Secret as the migration +# role, and its pods carry the migration Job's label, so the platform's migration +# network policy is the only path to the database. A failed run deletes nothing +# and the next day's run tries again. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: business-site-portal-retention + namespace: business-site +spec: + schedule: "17 3 * * *" + timeZone: Europe/Copenhagen + concurrencyPolicy: Forbid + startingDeadlineSeconds: 3600 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 300 + template: + metadata: + labels: + app.kubernetes.io/name: business-site-portal-migration + app.kubernetes.io/component: retention-purge + spec: + restartPolicy: Never + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: purge + args: + - --retention-purge + - --migration-database-projection + - /run/portal-migration/database-url + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 500m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: migration + mountPath: /run/portal-migration + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: migration + secret: + secretName: business-site-portal-db-migration + defaultMode: 0400 + items: + - key: database-url + path: database-url + - key: ca.crt + path: ca.crt + # As for the migration Job: the CA copy needs a sticky temp directory, + # and only a memory emptyDir is 1777. + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 1Mi diff --git a/deploy/kustomization.yaml b/deploy/kustomization.yaml index c0aa3453..a4152f15 100644 --- a/deploy/kustomization.yaml +++ b/deploy/kustomization.yaml @@ -11,7 +11,9 @@ resources: - external-secret-numina.yaml - external-secret-stripe.yaml - job-portal-migration.yaml -# The publisher pins only the WebApp's image; the migration Job runs that same image. + - cronjob-portal-retention.yaml +# The publisher pins only the WebApp's image; the migration Job and the retention +# CronJob run that same image. replacements: - source: kind: WebApp @@ -25,3 +27,10 @@ replacements: - spec.template.spec.containers.[name=migrate].image options: create: true + - select: + kind: CronJob + name: business-site-portal-retention + fieldPaths: + - spec.jobTemplate.spec.template.spec.containers.[name=purge].image + options: + create: true diff --git a/scripts/check-tenant-package.mjs b/scripts/check-tenant-package.mjs index 9550f5c6..60d10621 100644 --- a/scripts/check-tenant-package.mjs +++ b/scripts/check-tenant-package.mjs @@ -13,7 +13,7 @@ function keys(value, allowed) { export function validateTenantPackage(input) { assert.ok(typeof input === "string" && Buffer.byteLength(input) <= 262144, "input size"); const docs = yaml.loadAll(input); - assert.equal(docs.length, 10, "resource count"); + assert.equal(docs.length, 11, "resource count"); const app = docs.find((d) => d?.kind === "WebApp"); const network = docs.find((d) => d?.kind === "CiliumNetworkPolicy"); const budget = docs.find((d) => d?.kind === "PodDisruptionBudget"); @@ -164,6 +164,39 @@ export function validateTenantPackage(input) { }, }, }); + // The one retention CronJob runs this release's image once a day with only + // the purge and the migration Job's Secret, label and pod hardening. + const job = docs.find((d) => d?.kind === "Job"); + assert.deepEqual(docs.find((d) => d?.kind === "CronJob"), { + apiVersion: "batch/v1", + kind: "CronJob", + metadata: { name: "business-site-portal-retention", namespace: "business-site" }, + spec: { + schedule: "17 3 * * *", + timeZone: "Europe/Copenhagen", + concurrencyPolicy: "Forbid", + startingDeadlineSeconds: 3600, + successfulJobsHistoryLimit: 1, + failedJobsHistoryLimit: 3, + jobTemplate: { + spec: { + backoffLimit: 0, + activeDeadlineSeconds: 300, + template: { + metadata: { labels: { "app.kubernetes.io/name": "business-site-portal-migration", "app.kubernetes.io/component": "retention-purge" } }, + spec: { + ...job.spec.template.spec, + containers: [{ + ...job.spec.template.spec.containers[0], + name: "purge", + args: ["--retention-purge", "--migration-database-projection", "/run/portal-migration/database-url"], + }], + }, + }, + }, + }, + }, + }); assert.deepEqual(network.spec, { endpointSelector: { matchLabels: { "app.kubernetes.io/name": "business-site" } }, ingress: [{ fromEntities: ["ingress"], toPorts: [{ ports: [{ port: "8080", protocol: "TCP" }] }] }], diff --git a/scripts/runtime-verification.test.sh b/scripts/runtime-verification.test.sh index a90cf344..97ecacba 100644 --- a/scripts/runtime-verification.test.sh +++ b/scripts/runtime-verification.test.sh @@ -95,6 +95,8 @@ if [[ "${MISSING_STRIPE:-false}" == true ]]; then rm "$5/external-secret-stripe. if [[ "${ALTER_NUMINA:-false}" == true ]]; then yq -i '.spec.data[0].remoteRef.key="apps/other/numina"' "$5/external-secret-numina.yaml"; fi if [[ "${MISSING_MIGRATION:-false}" == true ]]; then rm "$5/job-portal-migration.yaml"; fi if [[ "${ALTER_MIGRATION:-false}" == true ]]; then yq -i '.spec.template.spec.containers[0].args[1]="--migration-database-file"' "$5/job-portal-migration.yaml"; fi +if [[ "${MISSING_RETENTION:-false}" == true ]]; then rm "$5/cronjob-portal-retention.yaml"; fi +if [[ "${ALTER_RETENTION:-false}" == true ]]; then yq -i '.spec.schedule="* * * * *"' "$5/cronjob-portal-retention.yaml"; fi FIXTURE chmod +x "$scratch/bin/gh" "$scratch/bin/cosign" "$scratch/bin/flux" # Restore independent signed-subject fixtures and clear receipts/read counters between controls. @@ -146,7 +148,7 @@ if verify TAG_SOURCE=1123456789abcdef0123456789abcdef01234567; then fail 'a move [[ ! -e "$scratch/runner/runtime-publication.json" ]] || fail 'moved Git tag left a receipt' reset_fixture verify TAG_KIND=tag || fail 'a matching annotated release tag must verify' -for context in VERIFY_FAIL=true PULL_FAIL=true CI_ATTEMPT=3 PUBLISH_ATTEMPT=2 ALTER_MANIFEST=true EXTRA_MEMBER=true MISSING_STORE=true MISSING_DISCLOSURE=true ALTER_STORE=true ALTER_DISCLOSURE=true MISSING_STRIPE=true ALTER_NUMINA=true MISSING_MIGRATION=true ALTER_MIGRATION=true VERSION=01.2.3 RELEASE_DRAFT=true RELEASE_PRERELEASE=true PUBLISHED_AT= WITHDRAW_AFTER_VERIFY=true MOVE_AFTER_VERIFY=true; do +for context in VERIFY_FAIL=true PULL_FAIL=true CI_ATTEMPT=3 PUBLISH_ATTEMPT=2 ALTER_MANIFEST=true EXTRA_MEMBER=true MISSING_STORE=true MISSING_DISCLOSURE=true ALTER_STORE=true ALTER_DISCLOSURE=true MISSING_STRIPE=true ALTER_NUMINA=true MISSING_MIGRATION=true ALTER_MIGRATION=true MISSING_RETENTION=true ALTER_RETENTION=true VERSION=01.2.3 RELEASE_DRAFT=true RELEASE_PRERELEASE=true PUBLISHED_AT= WITHDRAW_AFTER_VERIFY=true MOVE_AFTER_VERIFY=true; do reset_fixture if verify "$context"; then fail 'failed signature/pull, stale attempt or substituted manifests were accepted'; fi [[ ! -e "$scratch/runner/runtime-publication.json" ]] || fail 'refusal left a successful publication receipt' diff --git a/scripts/tenant-package.test.mjs b/scripts/tenant-package.test.mjs index 1c149a8f..a92ba419 100644 --- a/scripts/tenant-package.test.mjs +++ b/scripts/tenant-package.test.mjs @@ -243,6 +243,18 @@ for (const [name, mutate] of [ ["no migration deadline", (d) => { delete d[9].spec.activeDeadlineSeconds; }], ["a migration Job Flux cannot recreate", (d) => { delete d[9].metadata.annotations; }], ["a migration pod labelled as the web workload", (d) => { d[9].spec.template.metadata.labels["app.kubernetes.io/name"] = "business-site"; }], + ["a missing retention CronJob", (d) => { d.splice(10, 1); }], + ["a retention CronJob on another image", (d) => { d[10].spec.jobTemplate.spec.template.spec.containers[0].image = `ghcr.io/devantler-tech/business-site@sha256:${"a".repeat(64)}`; }], + ["a retention CronJob that migrates", (d) => { d[10].spec.jobTemplate.spec.template.spec.containers[0].args = structuredClone(d[9].spec.template.spec.containers[0].args); }], + ["the strict file option in the retention CronJob", (d) => { d[10].spec.jobTemplate.spec.template.spec.containers[0].args[1] = "--migration-database-file"; }], + ["overlapping retention runs", (d) => { d[10].spec.concurrencyPolicy = "Allow"; }], + ["a retention purge more than once a day", (d) => { d[10].spec.schedule = "*/5 * * * *"; }], + ["retention retries", (d) => { d[10].spec.jobTemplate.spec.backoffLimit = 3; }], + ["no retention deadline", (d) => { delete d[10].spec.jobTemplate.spec.activeDeadlineSeconds; }], + ["a retention pod running as root", (d) => { d[10].spec.jobTemplate.spec.template.spec.securityContext.runAsUser = 0; }], + ["a retention pod with a service-account token", (d) => { d[10].spec.jobTemplate.spec.template.spec.automountServiceAccountToken = true; }], + ["the web runtime Secret in the retention CronJob", (d) => { d[10].spec.jobTemplate.spec.template.spec.volumes[0].secret.secretName = "business-site-runtime"; }], + ["a retention pod labelled as the web workload", (d) => { d[10].spec.jobTemplate.spec.template.metadata.labels["app.kubernetes.io/name"] = "business-site"; }], ]) { test(`the actual rendered package rejects ${name}`, () => { const rendered = yaml.loadAll(render()); @@ -255,8 +267,8 @@ for (const [name, mutate] of [ for (const name of ["business-site-portal", "business-site-public"]) docs.push(named("HTTPRoute", name)); // The billing synchronizations follow at indexes 7 (Numina) and 8 (Stripe). for (const name of ["business-site-numina", "business-site-stripe"]) docs.push(named("ExternalSecret", name)); - // The migration Job follows at index 9. - docs.push(named("Job", "business-site-portal-migration")); + // The migration Job follows at index 9 and the retention CronJob at 10. + docs.push(named("Job", "business-site-portal-migration"), named("CronJob", "business-site-portal-retention")); assert.equal(check(docs.map((d) => yaml.dump(d)).join("---\n")).status, 0, "each refusal must start from a complete accepted package"); mutate(docs); @@ -267,9 +279,9 @@ for (const [name, mutate] of [ }); } -test("the migration Job runs whatever image the publisher pins in the WebApp", () => { +test("the migration Job and retention CronJob run whatever image the publisher pins in the WebApp", () => { // The shared publisher rewrites only webapp.yaml's WebApp image; the package - // build copies it into the Job, so the Job never runs another release's image. + // build copies it into both jobs, so neither runs another release's image. const fixture = mkdtempSync(path.join(tmpdir(), "tenant-package-")); try { cpSync(path.join(root, "deploy"), fixture, { recursive: true }); @@ -277,8 +289,13 @@ test("the migration Job runs whatever image the publisher pins in the WebApp", ( const app = yaml.load(readFileSync(path.join(fixture, "webapp.yaml"), "utf8")); app.spec.image = pinned; writeFileSync(path.join(fixture, "webapp.yaml"), yaml.dump(app)); - const job = yaml.loadAll(render(fixture)).find((d) => d?.kind === "Job"); + const rendered = yaml.loadAll(render(fixture)); + const job = rendered.find((d) => d?.kind === "Job"); assert.equal(job.spec.template.spec.containers[0].image, pinned); + const cron = rendered.find((d) => d?.kind === "CronJob"); + assert.equal(cron.spec.jobTemplate.spec.template.spec.containers[0].image, pinned); + const cronSource = yaml.load(readFileSync(path.join(root, "deploy/cronjob-portal-retention.yaml"), "utf8")); + assert.equal(cronSource.spec.jobTemplate.spec.template.spec.containers[0].image, undefined, "the source CronJob names no image of its own"); const source = yaml.load(readFileSync(path.join(root, "deploy/job-portal-migration.yaml"), "utf8")); assert.equal(source.spec.template.spec.containers[0].image, undefined, "the source Job names no image of its own"); } finally { diff --git a/scripts/verify-runtime-publication.sh b/scripts/verify-runtime-publication.sh index fc9efd75..5c898004 100644 --- a/scripts/verify-runtime-publication.sh +++ b/scripts/verify-runtime-publication.sh @@ -116,10 +116,10 @@ flux pull artifact "oci://$artifact@$manifests_digest" --output "$work/manifests entries="$(find "$work/manifests" -mindepth 1 -print)" || fail 'manifest inventory' while IFS= read -r entry; do member=${entry#"$work/manifests/"} - case "$member" in webapp.yaml | http-route-public.yaml | http-route-portal.yaml | network-policy.yaml | disruption-budget.yaml | secret-store.yaml | external-secret-disclosure.yaml | external-secret-numina.yaml | external-secret-stripe.yaml | job-portal-migration.yaml | kustomization.yaml | README.md) ;; *) fail 'unexpected manifest member' ;; esac + case "$member" in webapp.yaml | http-route-public.yaml | http-route-portal.yaml | network-policy.yaml | disruption-budget.yaml | secret-store.yaml | external-secret-disclosure.yaml | external-secret-numina.yaml | external-secret-stripe.yaml | job-portal-migration.yaml | cronjob-portal-retention.yaml | kustomization.yaml | README.md) ;; *) fail 'unexpected manifest member' ;; esac [[ -f "$entry" && ! -L "$entry" && "$(wc -c <"$entry")" -le 1048576 ]] || fail 'manifest member bounds' done <<<"$entries" -for member in webapp.yaml http-route-public.yaml http-route-portal.yaml network-policy.yaml disruption-budget.yaml secret-store.yaml external-secret-disclosure.yaml external-secret-numina.yaml external-secret-stripe.yaml job-portal-migration.yaml kustomization.yaml; do +for member in webapp.yaml http-route-public.yaml http-route-portal.yaml network-policy.yaml disruption-budget.yaml secret-store.yaml external-secret-disclosure.yaml external-secret-numina.yaml external-secret-stripe.yaml job-portal-migration.yaml cronjob-portal-retention.yaml kustomization.yaml; do [[ -f "$work/manifests/$member" && -f "deploy/$member" && ! -L "deploy/$member" ]] || fail 'missing source manifest' yq -o=json -I=0 '.' "deploy/$member" >"$work/source.json" || fail 'source manifest parse' if [[ "$member" == webapp.yaml ]]; then From d483b3102ac3f560f8fd616ef6d31cbfc19ca09c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 16 Nov 2026 00:39:12 +0100 Subject: [PATCH 2/2] test(deploy): run the image with the Job's and CronJob's own arguments Codex review on #620: manifest checks cannot tell whether the image accepts the arguments the package gives it. The runtime smoke test now runs the image CI builds with the rendered migration Job's and retention CronJob's arguments and no Secret mounted. Each must fail on its missing connection, not on its arguments. Co-Authored-By: Claude Opus 5.5 --- scripts/tenant-runtime-smoke.test.mjs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/scripts/tenant-runtime-smoke.test.mjs b/scripts/tenant-runtime-smoke.test.mjs index ccd3e040..a59e1555 100644 --- a/scripts/tenant-runtime-smoke.test.mjs +++ b/scripts/tenant-runtime-smoke.test.mjs @@ -4,13 +4,15 @@ import { execFileSync, spawnSync } from "node:child_process"; import { request } from "node:http"; import { fileURLToPath } from "node:url"; import test from "node:test"; +import * as yaml from "js-yaml"; import { validateTenantPackage } from "./check-tenant-package.mjs"; import { containerGroupDisclosureFixture, runtimeSecretFixture } from "./runtime-disclosure-fixture.mjs"; const root = fileURLToPath(new URL("../", import.meta.url)); -const spec = validateTenantPackage(execFileSync("kubectl", ["kustomize", `${root}/deploy`], { +const rendered = execFileSync("kubectl", ["kustomize", `${root}/deploy`], { encoding: "utf8", timeout: 15000, -})); +}); +const spec = validateTenantPackage(rendered); // CI substitutes its newly built production image, just as the later verified // publisher pins a new digest. With no argument, exercise the committed digest. const image = process.argv[2] ?? spec.image; @@ -58,6 +60,21 @@ test("rendered package arguments fail closed without the required disclosure", ( } }); +// The migration Job and the retention CronJob run this same image. With no Secret +// mounted, each must fail on its missing connection, not on its arguments: the +// image accepts exactly the arguments the package gives it. +test("the image accepts the migration Job's and retention CronJob's arguments", () => { + const docs = yaml.loadAll(rendered); + const job = docs.find((d) => d?.kind === "Job").spec.template.spec.containers[0].args; + const cron = docs.find((d) => d?.kind === "CronJob").spec.jobTemplate.spec.template.spec.containers[0].args; + for (const [args, failure] of [[job, "portal schema migration failed\n"], [cron, "portal retention purge failed\n"]]) { + const result = spawnSync("docker", ["run", "--rm", "--network=none", ...restrictions, image, ...args], { encoding: "utf8", timeout: 30000 }); + assert.ifError(result.error); + assert.equal(result.status, 1); + assert.equal(result.stdout + result.stderr, failure, args.join(" ")); + } +}); + // Serving needs the platform's whole runtime Secret, so it must exist before this package. test("rendered package arguments fail closed without the portal settings item", () => { const fixture = containerGroupDisclosureFixture(image, docker);