diff --git a/convex/crons.ts b/convex/crons.ts new file mode 100644 index 0000000..47baa0b --- /dev/null +++ b/convex/crons.ts @@ -0,0 +1,18 @@ +import { cronJobs } from "convex/server"; +import { internal } from "./_generated/api"; + +const crons = cronJobs(); + +// Daily retention sweep at 03:17 UTC. +crons.cron( + "mission-control-artifact-retention", + "17 3 * * *", + internal.missionControlCore.runArtifactRetentionSweep, + { + maxOwners: 250, + maxRunsPerOwner: 250, + schedulerJobId: "mission-control-artifact-retention", + }, +); + +export default crons; diff --git a/convex/missionControlCore.ts b/convex/missionControlCore.ts index 79d547c..dde68ed 100644 --- a/convex/missionControlCore.ts +++ b/convex/missionControlCore.ts @@ -1,5 +1,5 @@ import { v } from "convex/values"; -import { mutation, query } from "./_generated/server"; +import { internalMutation, mutation, query } from "./_generated/server"; import type { Id } from "./_generated/dataModel"; async function hasListAccess(ctx: any, listId: Id<"lists">, userDid: string) { @@ -217,6 +217,7 @@ export const listApiKeyRotationEvents = query({ }); const DEFAULT_ARTIFACT_RETENTION_DAYS = 30; +const SYSTEM_RETENTION_ACTOR_DID = "system:artifact-retention-job"; export const getMissionControlSettings = query({ args: { ownerDid: v.string() }, @@ -330,6 +331,88 @@ export const listArtifactDeletionLogs = query({ }, }); +export const runArtifactRetentionSweep = internalMutation({ + args: { + ownerDid: v.optional(v.string()), + retentionDays: v.optional(v.number()), + maxRunsPerOwner: v.optional(v.number()), + maxOwners: v.optional(v.number()), + schedulerJobId: v.optional(v.string()), + }, + handler: async (ctx, args) => { + const maxRunsPerOwner = Math.min(Math.max(args.maxRunsPerOwner ?? 250, 1), 1000); + const maxOwners = Math.min(Math.max(args.maxOwners ?? 250, 1), 1000); + + const ownerDidList = args.ownerDid + ? [args.ownerDid] + : (await ctx.db.query("users").take(maxOwners)) + .map((user) => user.did) + .filter((did): did is string => Boolean(did)); + + let totalRunsScanned = 0; + let totalRunsTouched = 0; + let totalDeletedArtifacts = 0; + + for (const ownerDid of ownerDidList) { + const settings = await ctx.db + .query("missionControlSettings") + .withIndex("by_owner", (q) => q.eq("ownerDid", ownerDid)) + .first(); + + const retentionDays = Math.min( + Math.max(Math.floor(args.retentionDays ?? settings?.artifactRetentionDays ?? DEFAULT_ARTIFACT_RETENTION_DAYS), 1), + 365, + ); + const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000; + + const runs = await ctx.db + .query("missionRuns") + .withIndex("by_owner_created", (q) => q.eq("ownerDid", ownerDid)) + .order("desc") + .take(maxRunsPerOwner); + + totalRunsScanned += runs.length; + const now = Date.now(); + + for (const run of runs) { + const artifacts = run.artifactRefs ?? []; + const staleArtifacts = artifacts.filter((a) => a.createdAt < cutoff); + if (!staleArtifacts.length) continue; + + totalRunsTouched += 1; + totalDeletedArtifacts += staleArtifacts.length; + + await ctx.db.insert("missionArtifactDeletionLogs", { + ownerDid, + runId: run._id, + deletedCount: staleArtifacts.length, + dryRun: false, + retentionCutoffAt: cutoff, + actorDid: SYSTEM_RETENTION_ACTOR_DID, + trigger: "system", + schedulerJobId: args.schedulerJobId, + deletedArtifacts: staleArtifacts, + createdAt: now, + }); + + await ctx.db.patch(run._id, { + artifactRefs: artifacts.filter((a) => a.createdAt >= cutoff), + updatedAt: now, + }); + } + } + + return { + ok: true, + ownerCount: ownerDidList.length, + totalRunsScanned, + totalRunsTouched, + totalDeletedArtifacts, + schedulerJobId: args.schedulerJobId, + }; + }, +}); + export const listTasksForList = query({ args: { listId: v.id("lists"), userDid: v.string(), limit: v.optional(v.number()) }, handler: async (ctx, args) => { diff --git a/convex/schema.ts b/convex/schema.ts index 4d36070..29e80d7 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -458,6 +458,7 @@ export default defineSchema({ retentionCutoffAt: v.number(), actorDid: v.string(), trigger: v.union(v.literal("operator"), v.literal("system")), + schedulerJobId: v.optional(v.string()), deletedArtifacts: v.array(v.object({ type: v.union(v.literal("screenshot"), v.literal("log"), v.literal("diff"), v.literal("file"), v.literal("url")), ref: v.string(), @@ -466,7 +467,8 @@ export default defineSchema({ })), createdAt: v.number(), }) - .index("by_owner_created", ["ownerDid", "createdAt"]), + .index("by_owner_created", ["ownerDid", "createdAt"]) + .index("by_run_created", ["runId", "createdAt"]), // Agent memory KV entries for long-lived runtime context agentMemory: defineTable({ diff --git a/docs/mission-control/mission-runs-api.md b/docs/mission-control/mission-runs-api.md index 2b211d8..2800e7b 100644 --- a/docs/mission-control/mission-runs-api.md +++ b/docs/mission-control/mission-runs-api.md @@ -50,6 +50,16 @@ Control endpoints require scope: `runs:control`. - `PUT /api/v1/runs/retention` (update policy) - `POST /api/v1/runs/retention` (apply retention dry-run/live) +Behavior: +- Default policy is `30` days unless overridden per owner. +- A daily scheduler job (`mission-control-artifact-retention`) performs non-dry-run cleanup across owners. +- Every deletion operation writes an auditable row to `missionArtifactDeletionLogs`, including: + - `trigger` (`operator` or `system`) + - `actorDid` + - `retentionCutoffAt` + - `deletedArtifacts[]` + - `schedulerJobId` (when triggered by cron) + ## Dashboard `GET /api/v1/dashboard/runs` diff --git a/package.json b/package.json index 7c7c1c1..7fdf247 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "env:turnkey:prod": "bash ./scripts/sync-convex-turnkey-env.sh .env.local prod", "cap:sync": "npx cap sync", "cap:build": "npm run build && npx cap sync", - "mission-control:readiness-drill": "node scripts/mission-control-readiness-drill.mjs" + "mission-control:readiness-drill": "node scripts/mission-control-readiness-drill.mjs", + "mission-control:validate-retention": "node scripts/validate-mission-control-retention.mjs" }, "dependencies": { "@capacitor/android": "^8.0.2", diff --git a/scripts/validate-mission-control-retention.mjs b/scripts/validate-mission-control-retention.mjs new file mode 100644 index 0000000..67ded25 --- /dev/null +++ b/scripts/validate-mission-control-retention.mjs @@ -0,0 +1,29 @@ +import fs from "node:fs"; + +const checks = [ + { + file: "convex/crons.ts", + mustInclude: ["mission-control-artifact-retention", "runArtifactRetentionSweep"], + }, + { + file: "convex/schema.ts", + mustInclude: ["missionArtifactDeletionLogs", "schedulerJobId", "by_run_created"], + }, + { + file: "convex/missionControlCore.ts", + mustInclude: ["runArtifactRetentionSweep", "SYSTEM_RETENTION_ACTOR_DID", "trigger: \"system\""], + }, +]; + +let failed = false; +for (const check of checks) { + const content = fs.readFileSync(check.file, "utf8"); + for (const token of check.mustInclude) { + if (!content.includes(token)) { + failed = true; + console.error(`[retention-validate] missing '${token}' in ${check.file}`); + } + } +} +if (failed) process.exit(1); +console.log("[retention-validate] OK");