diff --git a/README.md b/README.md index 5c2b697..9a26755 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ cp config/courses.example.json config/courses.json | `courses[].destination` | yes | Where the files land. Absolute, or relative to the repository root. No two courses may share one, or nest one inside another. | | `profilePath` | no | The saved browser session. Defaults to `.data/chrome-profile`. | | `statePath` | no | What has already been downloaded. Defaults to `.data/state.json`. | -| `driveMountPath` | no (watchdog yes) | The Google Drive mount that contains the destinations. The watchdog writes no destination when this directory is absent. | +| `driveMountPath` | no (watchdog yes) | The Google Drive mount that contains the destinations. Before writing, the watchdog requires both this directory and each destination's first Drive root below it to be present. | | `watchdogTimeoutMs` | no | The watchdog's initial timeout in milliseconds. The Owner pins the placeholder `900000` from the first week's logged durations. | | `media.mediaRoot` | required for `active`/`pilot` | The explicit Media store. It must be a directory below `/Volumes/RAID0`; there is no system-disk fallback. | | `media.freeSpaceReserveBytes` | no | Free space retained on the Media store before setup or acquisition. Defaults to 100 GiB. | diff --git a/docs/adr/0013-the-watchdog-is-a-local-scheduled-two-layer-run.md b/docs/adr/0013-the-watchdog-is-a-local-scheduled-two-layer-run.md index f59c09d..648a6d2 100644 --- a/docs/adr/0013-the-watchdog-is-a-local-scheduled-two-layer-run.md +++ b/docs/adr/0013-the-watchdog-is-a-local-scheduled-two-layer-run.md @@ -38,9 +38,11 @@ a second course-reading authority. - **Retrying reds.** Only a crash or timeout with no usable report is retried, three attempts total. A completed red is evidence the run understood and must remain visible under `docs/adr/0012`; retrying it would hammer NTULearn while hiding the signal that needs a person. -- **Pre-checks beyond the Drive mount.** The mount check prevents a 05:00 run from creating a - phantom local destination. More guards can misfire, block a healthy run, and become a second - interpretation of the run's evidence, so they are not added without a concrete failure to solve. +- **Pre-checks beyond the configured Drive roots.** The watchdog requires the mount and each + destination's first root below it to exist, preventing a 05:00 run from creating a phantom local + destination when `My Drive` moves but its former parent remains. More guards can misfire, block a + healthy run, and become a second interpretation of the run's evidence, so they are not added + without a concrete failure to solve. ## Consequences diff --git a/src/watchdog/run.mjs b/src/watchdog/run.mjs index c039fdf..000eec4 100644 --- a/src/watchdog/run.mjs +++ b/src/watchdog/run.mjs @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { mkdir, readFile } from "node:fs/promises"; import { clearTimeout, setTimeout as schedule } from "node:timers"; import { setTimeout as sleep } from "node:timers/promises"; -import { dirname, join, relative, resolve } from "node:path"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { INITIAL_WATCHDOG_TIMEOUT_MS } from "../config.mjs"; import { isDirectoryPresent, writeAtomically } from "../sync/files.mjs"; import { isCrashOrTimeout, sessionLapsed, watchdogVerdict } from "./verdict.mjs"; @@ -15,15 +15,15 @@ export async function runWatchdog({ config, root, runner }) { const paths = watchdogPaths(config.statePath); await mkdir(paths.stateDirectory, { recursive: true }); - const preCheck = await driveMountCheck(config); - if (!preCheck.present) { + const preCheck = await destinationReadinessCheck(config); + if (!preCheck.ready) { const timestamp = new Date(); const run = buildRun({ startedAt: timestamp, finishedAt: timestamp, attempts: 0, timeoutMs: timeoutFor(config), - preChecks: preChecksFor(preCheck, [{ attempt: 0, driveMount: preCheck }]), + preChecks: preChecksFor(preCheck, [{ attempt: 0, ...preCheck }]), attemptResults: [], sync: null, verify: null, @@ -50,11 +50,11 @@ export async function runWatchdogLocked({ config, root, runner, wait = sleep }) let lastAttempt = { sync: null, verify: null }; for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) { - const driveMount = await driveMountCheck(config); - preCheckHistory.push({ attempt, driveMount }); - if (!driveMount.present) break; + const destinationReadiness = await destinationReadinessCheck(config); + preCheckHistory.push({ attempt, ...destinationReadiness }); + if (!destinationReadiness.ready) break; - const result = await runAttempt({ root, runner, timeoutMs: timeoutFor(config) }); + const result = await runAttempt({ config, root, runner, timeoutMs: timeoutFor(config) }); lastAttempt = result; attemptResults.push({ attempt, ...result }); @@ -63,7 +63,7 @@ export async function runWatchdogLocked({ config, root, runner, wait = sleep }) } const finishedAt = new Date(); - const latestPreCheck = preCheckHistory.at(-1)?.driveMount ?? null; + const latestPreCheck = preCheckHistory.at(-1) ?? null; const run = buildRun({ startedAt, finishedAt, @@ -87,24 +87,91 @@ function watchdogPaths(statePath) { }; } -async function runAttempt({ root, runner, timeoutMs }) { +async function runAttempt({ config, root, runner, timeoutMs }) { const timeoutAt = Date.now() + timeoutMs; - const sync = await captureCommand({ root, runner, command: "sync", timeoutAt }); + const sync = classifyDestinationPermission( + await captureCommand({ root, runner, command: "sync", timeoutAt }), + config.courses ?? [], + ); if (isCrashOrTimeout(sync)) return { sync, verify: null }; - const verify = await captureCommand({ root, runner, command: "verify", timeoutAt }); + const verify = classifyDestinationPermission( + await captureCommand({ root, runner, command: "verify", timeoutAt }), + config.courses ?? [], + ); return { sync, verify }; } -async function driveMountCheck(config) { +function classifyDestinationPermission(command, courses) { + const match = command.stderr?.match(/\b(EACCES|EPERM)\b[^\n]*?['"]([^'"]+)['"]/i); + if (!match) return command; + + const deniedPath = resolve(match[2]); + const course = courses.find(({ destination }) => pathsOverlap(destination, deniedPath)); + if (!course) return command; return { - path: config.driveMountPath, - present: Boolean(config.driveMountPath && (await isDirectoryPresent(config.driveMountPath))), + ...command, + destinationPermission: { + code: match[1].toUpperCase(), + destination: course.destination, + path: deniedPath, + }, }; } -function preChecksFor(driveMount, history) { - return { driveMount, history }; +function pathsOverlap(left, right) { + const comparableLeft = resolve(left).toLowerCase(); + const comparableRight = resolve(right).toLowerCase(); + return ( + comparableLeft === comparableRight || + comparableLeft.startsWith(`${comparableRight}${sep}`) || + comparableRight.startsWith(`${comparableLeft}${sep}`) + ); +} + +async function destinationReadinessCheck(config) { + const mountPresent = Boolean( + config.driveMountPath && (await isDirectoryPresent(config.driveMountPath)), + ); + const destinations = mountPresent + ? await Promise.all( + (config.courses ?? []).map(async (course) => { + const root = destinationDriveRoot(config.driveMountPath, course.destination); + return { + path: course.destination, + root, + present: + destinationRelation(config.driveMountPath, course.destination).inside && + (await isDirectoryPresent(root)), + }; + }), + ) + : []; + return { + ready: mountPresent && destinations.every((destination) => destination.present), + driveMount: { path: config.driveMountPath, present: mountPresent }, + destinations, + }; +} + +function destinationDriveRoot(driveMountPath, destination) { + const { inside, pathFromMount } = destinationRelation(driveMountPath, destination); + if (!inside) return driveMountPath; + if (!pathFromMount) return driveMountPath; + return join(driveMountPath, pathFromMount.split(sep)[0]); +} + +function destinationRelation(driveMountPath, destination) { + const pathFromMount = relative(driveMountPath, destination); + const inside = + !isAbsolute(pathFromMount) && pathFromMount !== ".." && !pathFromMount.startsWith(`..${sep}`); + return { inside, pathFromMount }; +} + +function preChecksFor(destinationReadiness, history) { + if (!destinationReadiness) return { history }; + const { driveMount, destinations } = destinationReadiness; + return { driveMount, destinations, history }; } async function runWithLock({ root, runner, lockPath }) { @@ -249,6 +316,7 @@ function timeoutFor(config) { } function shouldRetry({ sync, verify }) { + if ([sync, verify].some((command) => command?.destinationPermission)) return false; if ([sync, verify].some(sessionLapsed)) return false; return [sync, verify].some(isCrashOrTimeout); } diff --git a/src/watchdog/verdict.mjs b/src/watchdog/verdict.mjs index 4ce6d50..c898262 100644 --- a/src/watchdog/verdict.mjs +++ b/src/watchdog/verdict.mjs @@ -15,6 +15,14 @@ export function watchdogVerdict({ return { verdict: "yellow", message: "skipped: a run was already going" }; } + const absentDestination = preChecks.destinations?.find((entry) => !entry.present); + if (absentDestination) { + return { + verdict: "red", + message: `Destination ${absentDestination.path} is unreachable — expected Drive root ${absentDestination.root}; set driveMountPath and the destination to the mounted Drive, then run: npm run watchdog`, + }; + } + if (preChecks.driveMount?.present === false) { return { verdict: "red", @@ -23,6 +31,16 @@ export function watchdogVerdict({ }; } + const destinationPermission = [sync, verify].find( + (command) => command?.destinationPermission, + )?.destinationPermission; + if (destinationPermission) { + return { + verdict: "red", + message: `Destination ${destinationPermission.destination} is unreachable — permission denied at ${destinationPermission.path}; correct driveMountPath or destination permissions, then run: npm run watchdog`, + }; + } + if ([sync, verify].some(sessionLapsed)) { return { verdict: "red", message: "session lapsed — run `npm run login`" }; } diff --git a/test/watchdog-run.test.mjs b/test/watchdog-run.test.mjs index 2bfc06b..d5f79c8 100644 --- a/test/watchdog-run.test.mjs +++ b/test/watchdog-run.test.mjs @@ -59,6 +59,40 @@ test("checks the Drive mount before acquiring the lock", async () => { ); }); +test("refuses before writing when a destination's Drive root is absent", async () => { + const root = await mkdtemp(join(tmpdir(), "ntulearn-watchdog-")); + const driveMountPath = join(root, "Google Drive"); + const driveRoot = join(driveMountPath, "My Drive"); + const destination = join(driveRoot, "Modules", "AB1234", "NTULearn"); + await mkdir(driveMountPath); + const config = { + statePath: join(root, ".data", "state.json"), + driveMountPath, + watchdogTimeoutMs: 1_000, + courses: [{ key: "AB1234", destination }], + }; + const runner = { + spawn() { + throw new Error("the destination pre-check should stop before spawning a command"); + }, + }; + + const digest = await runWatchdogLocked({ config, root, runner }); + + assert.equal(digest.verdict, "red"); + assert.equal( + digest.message, + `Destination ${destination} is unreachable — expected Drive root ${driveRoot}; set driveMountPath and the destination to the mounted Drive, then run: npm run watchdog`, + ); + await assert.rejects(stat(destination), { code: "ENOENT" }); + + const run = JSON.parse(await readFile(join(root, ".data", digest.runLog), "utf8")); + assert.equal(run.attempts, 0); + assert.deepEqual(run.preChecks.destinations, [ + { path: destination, root: driveRoot, present: false }, + ]); +}); + test("kills each timed-out attempt as a process group and retries three times", async () => { const root = await mkdtemp(join(tmpdir(), "ntulearn-watchdog-")); const driveMountPath = join(root, "Google Drive"); @@ -178,6 +212,49 @@ test("does not retry a lapsed session or a completed red run", async () => { assert.deepEqual(waits, []); }); +test("does not retry a destination permission failure", async () => { + const root = await mkdtemp(join(tmpdir(), "ntulearn-watchdog-")); + const driveMountPath = join(root, "Google Drive"); + const driveRoot = join(driveMountPath, "My Drive"); + const destination = join(driveRoot, "Modules", "AB1234", "NTULearn"); + await mkdir(destination, { recursive: true }); + const commands = []; + const waits = []; + + const digest = await runWatchdogLocked({ + config: { + statePath: join(root, ".data", "state.json"), + driveMountPath, + watchdogTimeoutMs: 1_000, + courses: [{ key: "AB1234", destination }], + }, + root, + runner: { + node: process.execPath, + argumentsFor(command) { + return [command]; + }, + spawn(_command, argumentsFor) { + commands.push(argumentsFor[0]); + return completedChild({ + code: 1, + stderr: `EACCES: permission denied, mkdir '${destination}'`, + }); + }, + }, + wait: async (milliseconds) => waits.push(milliseconds), + }); + + assert.deepEqual(commands, ["sync"]); + assert.deepEqual(waits, []); + assert.equal(digest.verdict, "red"); + assert.equal( + digest.message, + `Destination ${destination} is unreachable — permission denied at ${destination}; correct driveMountPath or destination permissions, then run: npm run watchdog`, + ); + assert.doesNotMatch(digest.message, /crash|timeout/i); +}); + function completedChild({ code, stdout = "", stderr = "" }) { const child = new EventEmitter(); child.pid = nextFakePid++; diff --git a/test/watchdog.test.mjs b/test/watchdog.test.mjs index 2509712..4ec473d 100644 --- a/test/watchdog.test.mjs +++ b/test/watchdog.test.mjs @@ -219,6 +219,55 @@ test("refuses a run when the Drive mount is absent", () => { ); }); +test("names an absent destination Drive root", () => { + assert.deepEqual( + watchdogVerdict({ + sync: null, + verify: null, + preChecks: { + driveMount: { path: "/Volumes/Google Drive", present: true }, + destinations: [ + { + path: "/Volumes/Google Drive/My Drive/Modules/AB1234", + root: "/Volumes/Google Drive/My Drive", + present: false, + }, + ], + }, + attempts: 0, + }), + { + verdict: "red", + message: + "Destination /Volumes/Google Drive/My Drive/Modules/AB1234 is unreachable — expected Drive root /Volumes/Google Drive/My Drive; set driveMountPath and the destination to the mounted Drive, then run: npm run watchdog", + }, + ); +}); + +test("reports a destination permission refusal rather than a crash", () => { + assert.deepEqual( + watchdogVerdict({ + sync: { + crashed: true, + report: null, + destinationPermission: { + code: "EACCES", + destination: "/Volumes/Google Drive/My Drive/Modules/AB1234", + path: "/Volumes/Google Drive/My Drive", + }, + }, + verify: null, + preChecks: {}, + attempts: 1, + }), + { + verdict: "red", + message: + "Destination /Volumes/Google Drive/My Drive/Modules/AB1234 is unreachable — permission denied at /Volumes/Google Drive/My Drive; correct driveMountPath or destination permissions, then run: npm run watchdog", + }, + ); +}); + test("reports an exhausted crash or timeout with its attempts and stderr tail", () => { assert.deepEqual( watchdogVerdict({