From d45a4b2c6e42ba13affbac2e2a7e2de0cd6b598a Mon Sep 17 00:00:00 2001 From: karim Date: Sat, 17 Oct 2026 13:20:13 +0200 Subject: [PATCH 1/4] Fix the subdirectory case's long tail (F56 leftovers) - Ask the rules file of the clone's top as well as the connected folder - Follow the full alternates chain, or refuse the folder with a clear message - Replace the F56 entry's false boundary sentence - Give the continueFrom refusal the failed-checkout re-preparation its list omits, with the test extended to assert it --- .../src/__tests__/harnessContinueFrom.test.ts | 10 +- .../src/__tests__/harnessResult.test.ts | 36 +++-- .../src/__tests__/packageRules.test.ts | 145 ++++++++++++++++++ .../__tests__/subdirectoryBoundary.test.ts | 85 ++++++++++ .../src/handlers/harnessHandlers.ts | 79 +++++++--- apps/aflow-executor-host/src/worktree.ts | 114 ++++++++++++-- docs/plans/315-local-first-run-ergonomics.md | 2 +- 7 files changed, 427 insertions(+), 44 deletions(-) create mode 100644 apps/aflow-executor-host/src/__tests__/packageRules.test.ts diff --git a/apps/aflow-executor-host/src/__tests__/harnessContinueFrom.test.ts b/apps/aflow-executor-host/src/__tests__/harnessContinueFrom.test.ts index 10816d93..43115714 100644 --- a/apps/aflow-executor-host/src/__tests__/harnessContinueFrom.test.ts +++ b/apps/aflow-executor-host/src/__tests__/harnessContinueFrom.test.ts @@ -3,7 +3,8 @@ * that decides which sessions it can — a kept session is owned by the run that * made it, and every commission is a new run — and nothing about the session it * named. It also says the ways a kept session ends — once it has been idle for - * an hour, once the executor has restarted since it was made, or once the grant + * an hour, once the executor has restarted since it was made, once a turn + * naming a new base has failed to prepare its checkout again, or once the grant * of its connected folder is withdrawn — because a session this run made and * lost one of those ways reaches the same refusal as another run's. Another * run's session, this run's idled-out one and one that never existed all get @@ -140,6 +141,13 @@ describe('a commission that names a session it cannot continue', () => { expect(message).toContain('once this executor has restarted since it was made'); }); + it('is told that a kept session ends when its checkout cannot be prepared again', async () => { + const { message } = await commission(OTHER_RUNS); + expect(message).toContain( + "once a turn naming a new `base` has failed to prepare the session's checkout again", + ); + }); + it("is told that a kept session ends when its folder's grant is withdrawn", async () => { const { message } = await commission(OTHER_RUNS); expect(message).toContain('or once the grant of its connected folder is withdrawn.'); diff --git a/apps/aflow-executor-host/src/__tests__/harnessResult.test.ts b/apps/aflow-executor-host/src/__tests__/harnessResult.test.ts index 0f80af6c..0807fe44 100644 --- a/apps/aflow-executor-host/src/__tests__/harnessResult.test.ts +++ b/apps/aflow-executor-host/src/__tests__/harnessResult.test.ts @@ -16,7 +16,7 @@ import { compileResultValidator, composeHarnessTask, createHostHarnessHandler, - hasRepositoryRules, + repositoryRulesFiles, readHarnessResult, resultInstruction, retryInstruction, @@ -117,26 +117,44 @@ describe("the task carries the repository's own rules", () => { await rm(checkout, { recursive: true, force: true }); }); - it('names CLAUDE.md before the task when the checkout holds one', async () => { + it('names CLAUDE.md at the root of the checkout before the task when it holds one', async () => { await writeFile(join(checkout, 'CLAUDE.md'), '# Rules\n', 'utf8'); - expect(await hasRepositoryRules(checkout)).toBe(true); - const text = composeHarnessTask('Review the range.', { range: 'main..HEAD' }, true); - expect(text.startsWith("The repository's own instructions are in `CLAUDE.md`")).toBe(true); - expect(text).toContain('treat them as binding'); + const files = await repositoryRulesFiles({ path: checkout, workDir: checkout }); + expect(files).toEqual([{ path: 'CLAUDE.md', where: 'at the root of this checkout' }]); + const text = composeHarnessTask('Review the range.', { range: 'main..HEAD' }, files); + expect( + text.startsWith( + "The repository's own instructions are in `CLAUDE.md` at the root of this checkout; " + + 'read them first and treat them as binding for every change.', + ), + ).toBe(true); expect(text).toContain('Review the range.'); expect(text).toContain('"range":"main..HEAD"'); }); it('says nothing about it when the checkout has none', async () => { - expect(await hasRepositoryRules(checkout)).toBe(false); - expect(composeHarnessTask('Review the range.', undefined, false)).toBe('Review the range.'); + expect(await repositoryRulesFiles({ path: checkout, workDir: checkout })).toEqual([]); + expect(composeHarnessTask('Review the range.', undefined, [])).toBe('Review the range.'); }); it('asks the checkout rather than a directory of the same name', async () => { // A directory called CLAUDE.md is not a file the harness can read, and a // task telling it to read one would send it after nothing. await mkdir(join(checkout, 'CLAUDE.md'), { recursive: true }); - expect(await hasRepositoryRules(checkout)).toBe(false); + expect(await repositoryRulesFiles({ path: checkout, workDir: checkout })).toEqual([]); + }); + + it("names the folder's own and its repository's, each where it is, for a folder inside one", async () => { + const workDir = join(checkout, 'packages', 'app'); + await mkdir(workDir, { recursive: true }); + await writeFile(join(checkout, 'CLAUDE.md'), '# Repository rules\n', 'utf8'); + await writeFile(join(workDir, 'CLAUDE.md'), '# Package rules\n', 'utf8'); + const files = await repositoryRulesFiles({ path: checkout, workDir }); + expect(composeHarnessTask('Fix it.', undefined, files)).toBe( + "The repository's own instructions are in `CLAUDE.md` in this folder and " + + '`../../CLAUDE.md` at the top level of the repository this folder is in; read them ' + + 'first and treat them as binding for every change.\n\nFix it.', + ); }); }); diff --git a/apps/aflow-executor-host/src/__tests__/packageRules.test.ts b/apps/aflow-executor-host/src/__tests__/packageRules.test.ts new file mode 100644 index 00000000..7659be9c --- /dev/null +++ b/apps/aflow-executor-host/src/__tests__/packageRules.test.ts @@ -0,0 +1,145 @@ +/** + * Contract: a folder connected as a package of a repository is briefed with the + * repository's rules as its top level is. The brief names the `CLAUDE.md` at the + * top of the clone by its path from the folder the harness works in, and the + * folder's own beside it when it has one. + * + * The spawn is stood in for and records the task it is handed. + */ +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { SandboxedRunInput, SandboxedRunResult } from '../sandboxedRun.js'; + +const run = promisify(execFile); + +const sandboxed: SandboxedRunInput[] = []; + +vi.mock('../sandboxedRun.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + sandboxReadiness: () => ({ ready: true, missing: [] }), + runSandboxed: (input: SandboxedRunInput): Promise => { + sandboxed.push(input); + return Promise.resolve({ + processId: 'hr_rules', + exitCode: 0, + signal: null, + timedOut: false, + durationMs: 0, + stdout: 'done', + stderr: '', + truncated: false, + }); + }, + }; +}); + +const { createHostHarnessHandler } = await import('../handlers/harnessHandlers.js'); + +const TOP_RULES = + "The repository's own instructions are in `../../CLAUDE.md` at the top level of the " + + 'repository this folder is in; read them first and treat them as binding for every change.'; + +let base: string; +let top: string; +let folder: string; + +async function git(...args: string[]): Promise { + await run('git', ['-C', top, ...args]); +} + +async function commit(files: Record): Promise { + for (const [path, text] of Object.entries(files)) { + await mkdir(join(top, path, '..'), { recursive: true }); + await writeFile(join(top, path), text); + } + await git('add', '-A'); + await git('commit', '-qm', 'the base commit'); +} + +/** The task the harness was handed, from a commission on the connected package. */ +async function briefOf(): Promise { + const policyPath = join(base, 'host-policy.json'); + await writeFile( + policyPath, + JSON.stringify({ + version: 1, + bindings: [ + { + id: 'hb_package', + root: folder, + mode: 'readwrite', + allowsExecution: true, + singleFile: false, + spaceId: 'space-a', + }, + ], + harnesses: [{ id: 'agent', executable: '/bin/sh', args: ['-c', 'true'] }], + }), + ); + const result = await createHostHarnessHandler(policyPath).execute({ + operationId: 'host.harness.run', + spaceId: 'space-a', + runId: 'run-package-rules', + job: { inputRef: 'inline:x' }, + signal: new AbortController().signal, + log: { error: () => undefined, warn: () => undefined, info: () => undefined }, + readPayload: () => + Promise.resolve({ bindingId: 'hb_package', task: 'Fix it.', timeoutMs: 60_000 }), + emitLiveDelta: () => Promise.resolve(), + writePayload: (kind: string) => Promise.resolve(`inline:${kind}`), + } as never); + expect(result.status).toBe('SUCCEEDED'); + expect(sandboxed).toHaveLength(1); + const argv = sandboxed[0]?.argv ?? []; + const task = argv.find((arg) => arg.endsWith('Fix it.')); + expect(task, argv.join(' ')).toBeDefined(); + return task ?? ''; +} + +beforeEach(async () => { + sandboxed.length = 0; + base = await realpath(await mkdtemp(join(tmpdir(), 'aflow-package-rules-'))); + top = join(base, 'mono'); + folder = join(top, 'packages', 'app'); + await mkdir(folder, { recursive: true }); + await git('init', '-q', '--initial-branch=main'); + await git('config', 'user.email', 'test@example.com'); + await git('config', 'user.name', 'Test'); +}); + +afterEach(async () => { + await rm(base, { recursive: true, force: true }); +}); + +describe('a package of a repository with a root CLAUDE.md, connected', () => { + it("is briefed with the repository's rules, named by their path from the package", async () => { + await commit({ 'CLAUDE.md': '# Rules\n', 'packages/app/index.js': '1;\n' }); + expect(await briefOf()).toBe(`${TOP_RULES}\n\nFix it.`); + }); + + it("is briefed with its own rules and the repository's, each where it is", async () => { + await commit({ + 'CLAUDE.md': '# Rules\n', + 'packages/app/CLAUDE.md': '# Package rules\n', + 'packages/app/index.js': '1;\n', + }); + expect(await briefOf()).toBe( + "The repository's own instructions are in `CLAUDE.md` in this folder and " + + '`../../CLAUDE.md` at the top level of the repository this folder is in; read them ' + + 'first and treat them as binding for every change.\n\nFix it.', + ); + }); + + it('is briefed with none when the repository holds none', async () => { + await commit({ 'packages/app/index.js': '1;\n' }); + expect(await briefOf()).toBe('Fix it.'); + }); +}); diff --git a/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts b/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts index 8fff2d0d..cca4d1e8 100644 --- a/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts +++ b/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts @@ -167,6 +167,91 @@ describe('a folder connected inside a repository under a denied home', () => { ); }); +describe('a repository whose store borrows from others, connected under a denied home', () => { + // `chained` borrows from `shared`, which borrows from `top`, the only store + // holding the base commit. + let shared: string; + let chained: string; + const objectsOf = (repository: string): string => join(repository, '.git', 'objects'); + + beforeEach(async () => { + shared = join(home, 'projects', 'shared'); + chained = join(home, 'projects', 'chained'); + await run('git', ['clone', '-q', '--shared', top, shared]); + await run('git', ['clone', '-q', '--shared', shared, chained]); + }); + + it('borrows every store along its alternates, a package and a top level alike', async () => { + expect(await borrowedReads(join(chained, 'packages', 'app'))).toEqual([ + objectsOf(chained), + objectsOf(shared), + objectsOf(top), + ]); + expect(await borrowedReads(chained)).toEqual([objectsOf(shared), objectsOf(top)]); + }); + + it('reads its alternates as git does: relative to the store, quoted, comments skipped', async () => { + await writeFile( + join(objectsOf(chained), 'info', 'alternates'), + `# borrowed\n\n"${join(home, 'projects', 'missing', '.git', 'objects')}"\n` + + `../../../shared/.git/objects\n${objectsOf(shared)}\n`, + ); + expect(await borrowedReads(chained)).toEqual([objectsOf(shared), objectsOf(top)]); + await writeFile( + join(objectsOf(chained), 'info', 'alternates'), + `"${objectsOf(shared).replace('shared', 'sh\\141red')}"\n`, + ); + expect(await borrowedReads(chained)).toEqual([objectsOf(shared), objectsOf(top)]); + }); + + it('is refused, naming the remedy, when its chain is deeper than a clone of it reads', async () => { + // A clone of the folder reads five stores of alternates below the folder's + // own; `shared` is one hop from `top`, and each level here adds one. + let below = shared; + const stores: string[] = []; + for (let level = 0; level < 4; level += 1) { + const next = join(home, 'projects', `level-${String(level)}`); + await run('git', ['clone', '-q', '--shared', below, next]); + stores.unshift(objectsOf(next)); + below = next; + } + expect(await borrowedReads(below)).toEqual([ + ...stores.slice(1), + objectsOf(shared), + objectsOf(top), + ]); + // Made by hand: git itself will not clone past the depth it reads. + const deeper = join(home, 'projects', 'level-4'); + await run('git', ['init', '-q', deeper]); + await writeFile(join(objectsOf(deeper), 'info', 'alternates'), `${objectsOf(below)}\n`); + await expect(borrowedReads(deeper)).rejects.toThrow( + /more levels of `objects\/info\/alternates` than git follows from a clone of it.*`git repack -a -d`/, + ); + }); + + it.skipIf(!sandboxAvailable() || confined.skip)( + confined.title('reads its history inside the run'), + { tags: ['listener'], timeout: 120_000 }, + async () => { + const app = join(chained, 'packages', 'app'); + const sha = await git(chained, 'rev-parse', 'HEAD'); + const checked = await runFolderChecks({ + binding: bindingOf(app), + argv: ['/bin/sh', '-c', 'git log -1 --format=%s'], + timeoutMs: 60_000, + sha, + base: sha, + toolPaths: [], + ownerRunId: 'probe', + signal: new AbortController().signal, + }); + + expect(checked.output).toContain('the base commit'); + expect(checked.result.exitCode).toBe(0); + }, + ); +}); + describe('a linked worktree connected under a denied home', () => { // A top level whose object store is the main repository's, outside it. let linked: string; diff --git a/apps/aflow-executor-host/src/handlers/harnessHandlers.ts b/apps/aflow-executor-host/src/handlers/harnessHandlers.ts index 8337193d..455ff931 100644 --- a/apps/aflow-executor-host/src/handlers/harnessHandlers.ts +++ b/apps/aflow-executor-host/src/handlers/harnessHandlers.ts @@ -18,7 +18,7 @@ */ import { lstat, mkdir, readdir, readFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { basename, join } from 'node:path'; +import { basename, join, relative } from 'node:path'; import AjvModule from 'ajv'; import type { z } from 'zod'; @@ -121,6 +121,7 @@ import { RUN_SCRATCH_DIR, utf8Prefix, WorktreeError, + type Checkout, type PreparedWorktree, } from '../worktree.js'; import { PUBLICATION_SCRATCH_PREFIX } from '../branchCommit.js'; @@ -174,6 +175,42 @@ export function compileResultValidator(schema: Record): ResultV /** The file a repository states its own rules in. */ const REPOSITORY_RULES_FILE = 'CLAUDE.md'; +/** A repository rules file in a checkout, as the harness is told of it. */ +export interface RepositoryRulesFile { + /** Its path from where the harness works, the folder's place in the clone. */ + readonly path: string; + /** Where it is, in the words the brief uses. */ + readonly where: string; +} + +/** + * The rules files in the checkout: at the folder's place in the clone and, for a + * folder inside a repository, at the clone's top as well — a package is bound + * by its repository's rules, and a bare Claude Code discovers neither. + * + * Asked of the isolated checkout, never of the operator's folder: what the + * harness can read is what the checkout holds. + */ +export async function repositoryRulesFiles( + checkout: Pick, +): Promise { + const places = + checkout.workDir === checkout.path + ? [{ dir: checkout.path, where: 'at the root of this checkout' }] + : [ + { dir: checkout.workDir, where: 'in this folder' }, + { dir: checkout.path, where: 'at the top level of the repository this folder is in' }, + ]; + const found: RepositoryRulesFile[] = []; + for (const { dir, where } of places) { + const file = join(dir, REPOSITORY_RULES_FILE); + if ((await stat(file).catch(() => undefined))?.isFile() === true) { + found.push({ path: relative(checkout.workDir, file), where }); + } + } + return found; +} + /** * A harness starts the way its profile starts it, and a bare Claude Code skips * `CLAUDE.md` auto-discovery, so a repository's own rules reach the harness only @@ -181,20 +218,12 @@ const REPOSITORY_RULES_FILE = 'CLAUDE.md'; * skill: the file is a fact about the folder, and every task over that folder is * bound by it, whatever the task is. */ -export const REPOSITORY_RULES_SENTENCE = - `The repository's own instructions are in \`${REPOSITORY_RULES_FILE}\` at the root of this ` + - 'checkout; read them first and treat them as binding for every change.'; - -/** - * Asked of the isolated checkout, never of the operator's folder: what the - * harness can read is what the checkout holds. - */ -export async function hasRepositoryRules(checkoutRoot: string): Promise { - try { - return (await stat(join(checkoutRoot, REPOSITORY_RULES_FILE))).isFile(); - } catch { - return false; - } +export function repositoryRulesSentence(files: readonly RepositoryRulesFile[]): string { + const named = files.map(({ path, where }) => `\`${path}\` ${where}`).join(' and '); + return ( + `The repository's own instructions are in ${named}; read them first and treat them as ` + + 'binding for every change.' + ); } /** The task with its structured inputs attached, so the prose can name them. */ @@ -252,11 +281,11 @@ export function mergeConflictSentence(merge: Pick | undefined, - repositoryRules: boolean, + rulesFiles: readonly RepositoryRulesFile[], merge?: Pick, ): string { return [ - repositoryRules ? REPOSITORY_RULES_SENTENCE : undefined, + rulesFiles.length > 0 ? repositoryRulesSentence(rulesFiles) : undefined, merge !== undefined && merge.conflicts.length > 0 ? mergeConflictSentence(merge) : undefined, taskWithInputs(task, inputs), ] @@ -575,9 +604,10 @@ async function runHarness( if (session === undefined) { // The same answer a session that never existed would get: naming a // session as another run's confirms it exists. This run's own session - // that idled out, was lost to a restart or went with its folder's grant - // reaches this branch too, so the rule states how a kept session ends - // and guesses at none of those for this ref. + // that idled out, was lost to a restart, lost its checkout to a failed + // re-preparation or went with its folder's grant reaches this branch + // too, so the rule states how a kept session ends and guesses at none of + // those for this ref. return await failureWithError( ctx, validationError( @@ -585,8 +615,9 @@ async function runHarness( 'by the run that made it, and every commission is a new run, so `continueFrom` ' + 'can name only a session this same run made; a session made by any other run is ' + 'refused as one that does not exist. A kept session ends once it has been idle ' + - 'for an hour, once this executor has restarted since it was made, or once the ' + - 'grant of its connected folder is withdrawn.', + 'for an hour, once this executor has restarted since it was made, once a turn ' + + "naming a new `base` has failed to prepare the session's checkout again, or once " + + 'the grant of its connected folder is withdrawn.', ), ); } @@ -657,6 +688,7 @@ async function runHarness( mergeFrom === undefined ? undefined : await mergeIdentityArgs(binding.root); const mergeSource = mergeFrom === undefined ? undefined : await fetchMergeSource(binding.root, mergeFrom); + const borrowed = await borrowedReads(binding.root); // A turn that names none keeps the checkout its session's base made, and so // is judged against that base as well. const base = namedRef ?? session?.base; @@ -703,7 +735,6 @@ async function runHarness( ); } worktreePath = worktree.path; - const borrowed = await borrowedReads(binding.root); const keptCheckout = session !== undefined && namedBase === undefined; let merge: BaseMerge | undefined = keptCheckout ? session?.merge : undefined; @@ -886,7 +917,7 @@ async function runHarness( const brief = composeHarnessTask( input.task, input.inputs, - await hasRepositoryRules(worktree.workDir), + await repositoryRulesFiles(worktree), keptCheckout ? undefined : merge, ); const task = continuesAttempt ? continuedAttemptTask(brief) : brief; diff --git a/apps/aflow-executor-host/src/worktree.ts b/apps/aflow-executor-host/src/worktree.ts index 942ae354..e4dc752a 100644 --- a/apps/aflow-executor-host/src/worktree.ts +++ b/apps/aflow-executor-host/src/worktree.ts @@ -32,6 +32,7 @@ import { mkdir, mkdtemp, readdir, + readFile, realpath, rm, rmdir, @@ -101,7 +102,8 @@ export class WorktreeError extends Error { | 'fetch_failed' | 'unresolved_conflict' | 'binary_conflict' - | 'checkout_replaced', + | 'checkout_replaced' + | 'alternates_too_deep', ) { super(message); this.name = 'WorktreeError'; @@ -732,19 +734,111 @@ export async function folderPrefix(root: string): Promise { return (await git(root, ['rev-parse', '--show-prefix'])).trim().replace(/\/$/, ''); } +/** + * How many levels of `info/alternates` below the folder's own store a clone of + * it reads. Git reads alternates six levels down from the store it starts in + * and ignores any deeper, and the clone's own alternates, naming the folder's + * store, are the first of those six. + */ +const FOLDER_ALTERNATES_LEVELS = 5; + +const C_ESCAPES: Readonly> = { + a: 7, + b: 8, + t: 9, + n: 10, + v: 11, + f: 12, + r: 13, + '"': 34, + '\\': 92, +}; + +/** An entry in git's C-style quoting, unquoted; undefined when it is not one. */ +function unquotedC(entry: string): string | undefined { + const bytes: number[] = []; + for (let i = 1; i < entry.length; i += 1) { + const char = entry.charAt(i); + if (char === '"') { + return i === entry.length - 1 ? Buffer.from(bytes).toString('utf8') : undefined; + } + if (char !== '\\') { + bytes.push(...Buffer.from(char, 'utf8')); + continue; + } + const octal = /^[0-3][0-7]{2}/.exec(entry.slice(i + 1)); + const escaped = C_ESCAPES[entry.charAt(i + 1)]; + if (octal !== null) { + bytes.push(Number.parseInt(octal[0], 8)); + i += 3; + } else if (escaped !== undefined) { + bytes.push(escaped); + i += 1; + } else { + return undefined; + } + } + return undefined; +} + +/** The stores `store`'s `info/alternates` names, as git reads them, unresolved. */ +async function alternatesOf(store: string): Promise { + const text = await readFile(join(store, 'info', 'alternates'), 'utf8').catch(() => ''); + return text + .split('\n') + .filter((entry) => entry !== '' && !entry.startsWith('#')) + .map((entry) => (entry.startsWith('"') ? (unquotedC(entry) ?? entry) : entry)) + .map((entry) => resolvePath(store, entry)); +} + +/** + * Appends to `stores` each store `store` borrows from, resolved, following + * `info/alternates` as git does: depth first, an entry relative to the store + * naming it, a store missing or already named skipped. A chain deeper than a + * clone of the folder follows is refused, since a run's checkout would not + * reach the objects at its end. + */ +async function appendAlternateStores( + root: string, + store: string, + level: number, + stores: string[], +): Promise { + const entries = await alternatesOf(store); + if (entries.length === 0) return; + if (level >= FOLDER_ALTERNATES_LEVELS) { + throw new WorktreeError( + `The object store of ${root} borrows through more levels of \`objects/info/alternates\` ` + + "than git follows from a clone of it, so a run's checkout could not read every object " + + `the folder has; the level a clone stops at is ${store}. Copy the borrowed objects ` + + "into the folder's repository with `git repack -a -d`, remove its " + + '`objects/info/alternates`, and run again.', + 'alternates_too_deep', + ); + } + for (const entry of entries) { + const resolved = await realpath(entry).catch(() => undefined); + if (resolved === undefined || stores.includes(resolved)) continue; + stores.push(resolved); + await appendAlternateStores(root, resolved, level + 1, stores); + } +} + /** * What a run in a checkout of the folder at `root` reads outside that folder, * for the sandbox to grant read of (`SandboxWidening.borrowedReads`). * * Decided by where each lies, never by where the folder is in its repository. * The repository's object store, which the clone owns no object of and borrows - * every one from (`clonePrivately`), resolved, whenever it lies outside the - * folder: for a folder inside a repository, and for a top level whose store is - * elsewhere too — a linked worktree, a submodule, a separate git directory. - * And each installation outside the folder of the same set the clone's - * mirrored links lead into (`repositoryInstallations`, `linkDependencies`), so - * those on the folder's way up are readable however deep the folder lies. Read - * from the folder, never from the checkout, which the run writes. + * every one from (`clonePrivately`), and each store it borrows from in turn + * (`appendAlternateStores`), resolved, whenever it lies outside the folder: for + * a folder inside a repository, and for a top level whose store is elsewhere + * too — a linked worktree, a submodule, a separate git directory, a clone made + * with `--shared` or `--reference`. And each installation outside the folder + * of the same set the clone's mirrored links lead into + * (`repositoryInstallations`, `linkDependencies`), so those on the folder's way + * up are readable however deep the folder lies. Read from the folder, never + * from the checkout, which the run writes. */ export async function borrowedReads(root: string): Promise { const rootReal = await realpath(root).catch(() => root); @@ -753,7 +847,9 @@ export async function borrowedReads(root: string): Promise { const reads: string[] = []; const objects = await gitPath(root, 'objects'); const objectsReal = await realpath(objects).catch(() => objects); - if (outsideRoot(objectsReal)) reads.push(objectsReal); + const stores = [objectsReal]; + await appendAlternateStores(root, objectsReal, 0, stores); + reads.push(...stores.filter(outsideRoot)); const top = (await git(root, ['rev-parse', '--show-toplevel'])).trim(); for (const relative of await repositoryInstallations(top, await folderPrefix(root))) { const installation = join(top, relative); diff --git a/docs/plans/315-local-first-run-ergonomics.md b/docs/plans/315-local-first-run-ergonomics.md index 32897027..ac415dfe 100644 --- a/docs/plans/315-local-first-run-ergonomics.md +++ b/docs/plans/315-local-first-run-ergonomics.md @@ -162,7 +162,7 @@ After D13 lands: the chat renders a run's wakeup as a card that says which run s **Then F118 — the start rebuilds the packages the web application reads**: `scripts/dev.mjs`, which `yarn start` and `yarn dev:local` reach through the `local` profile, no longer runs the advisory staleness check that called a stale `dist` safe to ignore. Before any service starts it takes the Next applications among the services it is about to run (a workspace whose `dependencies` name `next`, found through the service's command as `whyUnavailable` finds it) and their dist closure — every workspace package their `dependencies`, `peerDependencies` and `optionalDependencies` reach, transitively, each after what it depends on — from the manifests alone (`scripts/webDists.mjs`); no list of packages is written anywhere. A package in it is stale where a file the `import` condition names is missing or older than the newest file of its `src/`, tests and `.tsbuildinfo` aside; measuring the files the bundler loads rather than all of `dist/` is deliberate, since a type-check writes declarations there without touching the JavaScript and would make a stale build look current. Each stale package is built with `yarn workspace build`, dependencies first, under one line that names them and says which application reads them from `dist/` where every other service reads `src/`, then one line per package with what made it stale and how long it took; a package already current is not built and a start with nothing changed prints nothing. A package the start runs a watcher for — `@aflow/web-product` — is named as stale and left to that watcher's first build. A build that fails does not stop the start, since the Node services read source: its output is printed with what it costs the web app and the command that rebuilds it. A profile with no web application — `core`, `engine`, `mcp` — selects nothing. `yarn check:packages` (`check-package-staleness.mjs`) shares the rule and now names each stale package and which consumer reads `dist/`. Tests: `scripts/webDists.test.mjs` measures the repository's own manifests with injected times — a touched `packages/schemas/src` file selects `@aflow/schemas` alone, a touched orchestrator file selects nothing, a current tree selects nothing, a watched package is left to its watcher, a fresh clone selects the whole closure dependencies first — and two packages on disk, one whose declarations a type-check rewrote after its source changed (stale) and one whose only newer file is a test (current). Not taken: the dev health check that requests the web app's first page, which would add a probe and its failure path to the runner; it stays open on this row, as F58's watch of the imported packages during a session does. -**Then F56 — the private clone is the boundary**: every checkout the host lane makes — a commission's, a check's, a publication's — is a private clone of the folder rather than a worktree added to it. `prepareWorktree` (`worktree.ts`) clones the folder's repository from its top level (`rev-parse --show-toplevel`, since `git clone` takes nothing else) with `--mirror --shared` into a git directory of its own beside the checkout (`git-` in the run's scratch, `checkoutGitDir`), which the sandbox reads and never writes, gives it a working tree whose `.git` is only a gitfile naming that directory — the executor's own git names the directory and the working tree outright (`Checkout`) and never resolves the checkout through what it holds — removes its remote and checks out the base commit detached: the clone reads every ref as the folder names it when it is made, so `origin/main`, a branch and a tag resolve as they do there, and borrows the folder's objects through `objects/info/alternates`, while its refs and config are its own. A `git update-ref`, a `git branch -D`, a call that names its own `core.hooksPath` or a push to `.` in the run moves the clone's refs and nobody else's, and a push to the folder's path meets a `.git` the sandbox does not let it write (D19). So the guard and the record are gone rather than kept beside it: `refGuard.ts` and its `reference-transaction` hook, with the git 2.28 floor a commission was refused under, and `snapshotRefs`/`changedRefs` with the result's `refChanges`, which leaves `HostHarnessRunOutputSchema`, `host.harness.run`'s description and Commission Change's promoted outputs. The publication reads its commit from its clone: `commitPatchOnBranch` (`branchCommit.ts`) applies and commits there, then the executor's git fetches that commit into the folder by sha (`--no-write-fetch-head`, so no FETCH_HEAD another fetch is reading is overwritten; git 2.29 or later) and creates the named branch, or advances it by compare-and-swap on the head it was made on — the one write to the folder's refs the loop makes. A commit's identity is read from the folder's config, since the clone's holds none; a commission's `applies` is judged against a merge commit in the clone, where it exists. `removeWorktree` takes the clone's path and removes the directory, which is the whole of it, and `HarnessSession` no longer carries `bindingRoot`. The boot sweep (`removeOrphanedCheckouts`) takes a scratch directory named as this executor names its own and carrying the marker `makeScratch` (`executorScratch.ts`) writes into every scratch it makes — a commission's, a check's, a publication's and an ephemeral browser profile's — and takes it whole, whatever its checkouts are: a clone killed before its detached checkout, whose HEAD still names a branch, goes with it. The shape of a HEAD decides nothing; a directory without the marker is left whatever it holds; a crashed run's journal entry (`orphans.ts`) removes its scratch, clone and all, with nothing left registered in the folder — for a commission and a check, which hand the sandbox only `agent/` inside it, the journal records the scratch holding the clone (`journalScratchDir`, passed to `recordSpawn` apart from the sandbox's `scratchDir`, `sandboxedRun.ts`), while a run that ends removes `agent/` alone and its caller the rest. Tests: a run whose agent deletes a branch, moves `main` and tags under its own hooks path, pushes to `.` and pushes and deletes into the folder's path leaves every ref of the folder as it was, is handed no `GIT_CONFIG_*` and names no remote, and its publication lands exactly its commit on the named branch with no other ref moved (`harnessResult.test.ts`, sandboxed); the clone's refs, alternates and detached HEAD, every move made inside it reaching only it, and a folder commit made after the clone read through what it borrows (`worktree.test.ts`); the publication's fetch writing the named branch and no other ref and no FETCH_HEAD (`patchApply.test.ts`); a clone removed when its run ends and swept when it is orphaned, by the boot sweep and by the journal, with the operator's own checkouts left (`worktree.test.ts`, `phase2Acceptance.test.ts`). Inside a marked scratch only a real directory is treated as a checkout, read with `lstat`: a link planted there, followed, led the removal's unlinking of mirrored installations into the `node_modules` of whatever the link pointed at, the operator's own project included. A link is now removed with the scratch and nothing it leads to is walked, and `removeWorktree` acts on its checkout and git directory only as real directories. The sweep handles marked scratch only and reads no connected folder, so the detached worktrees the executor before the private clone registered in an operator's repository, in scratch it never marked, are the operator's to prune once by hand (`git worktree remove --force` on each that `git worktree list` shows under the temp root). A folder connected as a subdirectory of a repository — a package of a monorepo, which `rev-parse --is-inside-work-tree` accepts — is cloned from that top level, and the checkout's `workDir` is the folder's path relative to it inside the clone: the harness and a check run there with it as their writable root, the installations of the repository are linked at the same place in the clone — the union of those the scan from the top level reaches, exactly as for a folder that is the top level, so a sibling package's own installation is there too, and those on the walk from the folder up to the top level, its own included, which the scan's depth (`DEPENDENCY_SCAN_DEPTH`) cannot cut short however deep the folder lies (`repositoryInstallations`) — and a workspace link resolving anywhere in the repository — a sibling package included — points at that path in the clone, so resolution in the clone sees what it sees in the folder, and `collectChanges` stages and diffs that directory alone, as paths from the top level, which is what `git apply` reads in the folder and what the publication applies at the clone's top. A session's checkout is found again from its binding's prefix (`folderPrefix`). Inside the run, a clone reads outside the folder wherever what it borrows lies outside it: it owns no object and borrows every one from the repository's store (`rev-parse --git-path objects`, resolved), which is outside the folder for a subdirectory and equally for a top level that is a linked worktree, a submodule or a repository with a separate git directory, and its mirrored installations outside the folder lead into `/node_modules` and the like, all under a home the sandbox denies. So the sandbox is handed exactly those that lie outside the folder, decided by location and never by the folder's prefix, read and never written, for a commission and a check alike (`borrowedReads`, `worktree.ts`, read from the folder rather than the checkout the run writes), and `checkoutWidening` (`sandboxPolicy.ts`) states the whole boundary of a run in a checkout in one place — what it reads and writes for a folder holding its store and installations and for any other — which a test holds the compiled policy to, path for path (`sandboxPolicy.test.ts`). Test: a subdirectory of a repository under a denied home, connected, runs `git log -1` and resolves a hoisted package inside a folder check's sandbox, and reaches neither without the borrowed reads; a linked worktree under a denied home, connected, runs `git log -1` there (`subdirectoryBoundary.test.ts`, sandboxed). Tests: a sweep of a marked scratch holding a half-made clone removes it with no ref or registration of the folder touched, and unmarked directories named like scratch are left; a marked scratch holding links to a project with a linked `node_modules`, as a checkout and as a git directory, is swept with every link in that project left; a folder connected inside a repository gets a checkout whose change is the folder's alone and applies to it (`worktree.test.ts`); a package connected as the folder gets the top level's installation and its own linked, a workspace package resolving into the clone, and a sibling package's own dependency resolving inside the clone, and a folder three levels down resolving its own dependency inside the clone (`worktreeDependencies.test.ts`); `borrowedReads` reads the same set, a folder below the scan's depth borrowing each installation on its way up (`subdirectoryBoundary.test.ts`); a sandboxed run handed `agent/` and killed with the executor is journalled with the scratch holding its clone, which the reap removes clone and all (`worktree.test.ts`, sandboxed), and a commission hands the sandbox that scratch as `journalScratchDir` (`sandboxPosture.test.ts`). The directory holding a checkout and its git directory is never the sandbox's to write. A commission and a check are handed `agent/` inside their scratch (`makeAgentScratch`, `executorScratch.ts`) — their home, temporary directory, harness configuration, browser pipes and check report — with the folder's place in the checkout as their writable root, so a job cannot remove its checkout and put a link to a directory of the operator's in its place. And because git, told `--work-tree`, takes a link there as readily as a directory, every git the executor runs on a checkout first finds the checkout and the folder's place in it real directories whose resolved path is the one `prepareWorktree` recorded, resolved, and refuses otherwise (`assertCheckoutInPlace`, in `gitLocation`) — staging (`collectChanges`), the base merge (`mergeIntoCheckout`) and `checkApplies` among them. Tests: a sandboxed job's attempt to remove or move its checkout or git directory and put a link in its place is refused (sandboxed), and a link put from outside in place of the checkout, or of the folder's place in it, is refused by collection and by the merge with the directory it leads to untouched (`checkoutReplacement.test.ts`); a commission is handed `agent/` beside its checkout (`sandboxPosture.test.ts`). +**Then F56 — the private clone is the boundary**: every checkout the host lane makes — a commission's, a check's, a publication's — is a private clone of the folder rather than a worktree added to it. `prepareWorktree` (`worktree.ts`) clones the folder's repository from its top level (`rev-parse --show-toplevel`, since `git clone` takes nothing else) with `--mirror --shared` into a git directory of its own beside the checkout (`git-` in the run's scratch, `checkoutGitDir`), which the sandbox reads and never writes, gives it a working tree whose `.git` is only a gitfile naming that directory — the executor's own git names the directory and the working tree outright (`Checkout`) and never resolves the checkout through what it holds — removes its remote and checks out the base commit detached: the clone reads every ref as the folder names it when it is made, so `origin/main`, a branch and a tag resolve as they do there, and borrows the folder's objects through `objects/info/alternates`, while its refs and config are its own. A `git update-ref`, a `git branch -D`, a call that names its own `core.hooksPath` or a push to `.` in the run moves no ref at all — the clone's git directory is read-only to the run — and a push to the folder's path meets a `.git` the sandbox does not let it write (D19). So the guard and the record are gone rather than kept beside it: `refGuard.ts` and its `reference-transaction` hook, with the git 2.28 floor a commission was refused under, and `snapshotRefs`/`changedRefs` with the result's `refChanges`, which leaves `HostHarnessRunOutputSchema`, `host.harness.run`'s description and Commission Change's promoted outputs. The publication reads its commit from its clone: `commitPatchOnBranch` (`branchCommit.ts`) applies and commits there, then the executor's git fetches that commit into the folder by sha (`--no-write-fetch-head`, so no FETCH_HEAD another fetch is reading is overwritten; git 2.29 or later) and creates the named branch, or advances it by compare-and-swap on the head it was made on — the one write to the folder's refs the loop makes. A commit's identity is read from the folder's config, since the clone's holds none; a commission's `applies` is judged against a merge commit in the clone, where it exists. `removeWorktree` takes the clone's path and removes the directory, which is the whole of it, and `HarnessSession` no longer carries `bindingRoot`. The boot sweep (`removeOrphanedCheckouts`) takes a scratch directory named as this executor names its own and carrying the marker `makeScratch` (`executorScratch.ts`) writes into every scratch it makes — a commission's, a check's, a publication's and an ephemeral browser profile's — and takes it whole, whatever its checkouts are: a clone killed before its detached checkout, whose HEAD still names a branch, goes with it. The shape of a HEAD decides nothing; a directory without the marker is left whatever it holds; a crashed run's journal entry (`orphans.ts`) removes its scratch, clone and all, with nothing left registered in the folder — for a commission and a check, which hand the sandbox only `agent/` inside it, the journal records the scratch holding the clone (`journalScratchDir`, passed to `recordSpawn` apart from the sandbox's `scratchDir`, `sandboxedRun.ts`), while a run that ends removes `agent/` alone and its caller the rest. Tests: a run whose agent deletes a branch, moves `main` and tags under its own hooks path, pushes to `.` and pushes and deletes into the folder's path leaves every ref of the folder as it was, is handed no `GIT_CONFIG_*` and names no remote, and its publication lands exactly its commit on the named branch with no other ref moved (`harnessResult.test.ts`, sandboxed); the clone's refs, alternates and detached HEAD, every move made inside it reaching only it, and a folder commit made after the clone read through what it borrows (`worktree.test.ts`); the publication's fetch writing the named branch and no other ref and no FETCH_HEAD (`patchApply.test.ts`); a clone removed when its run ends and swept when it is orphaned, by the boot sweep and by the journal, with the operator's own checkouts left (`worktree.test.ts`, `phase2Acceptance.test.ts`). Inside a marked scratch only a real directory is treated as a checkout, read with `lstat`: a link planted there, followed, led the removal's unlinking of mirrored installations into the `node_modules` of whatever the link pointed at, the operator's own project included. A link is now removed with the scratch and nothing it leads to is walked, and `removeWorktree` acts on its checkout and git directory only as real directories. The sweep handles marked scratch only and reads no connected folder, so the detached worktrees the executor before the private clone registered in an operator's repository, in scratch it never marked, are the operator's to prune once by hand (`git worktree remove --force` on each that `git worktree list` shows under the temp root). A folder connected as a subdirectory of a repository — a package of a monorepo, which `rev-parse --is-inside-work-tree` accepts — is cloned from that top level, and the checkout's `workDir` is the folder's path relative to it inside the clone: the harness and a check run there with it as their writable root, the installations of the repository are linked at the same place in the clone — the union of those the scan from the top level reaches, exactly as for a folder that is the top level, so a sibling package's own installation is there too, and those on the walk from the folder up to the top level, its own included, which the scan's depth (`DEPENDENCY_SCAN_DEPTH`) cannot cut short however deep the folder lies (`repositoryInstallations`) — and a workspace link resolving anywhere in the repository — a sibling package included — points at that path in the clone, so resolution in the clone sees what it sees in the folder, and `collectChanges` stages and diffs that directory alone, as paths from the top level, which is what `git apply` reads in the folder and what the publication applies at the clone's top. A session's checkout is found again from its binding's prefix (`folderPrefix`). Inside the run, a clone reads outside the folder wherever what it borrows lies outside it: it owns no object and borrows every one from the repository's store (`rev-parse --git-path objects`, resolved), which is outside the folder for a subdirectory and equally for a top level that is a linked worktree, a submodule or a repository with a separate git directory, and its mirrored installations outside the folder lead into `/node_modules` and the like, all under a home the sandbox denies. So the sandbox is handed exactly those that lie outside the folder, decided by location and never by the folder's prefix, read and never written, for a commission and a check alike (`borrowedReads`, `worktree.ts`, read from the folder rather than the checkout the run writes), and `checkoutWidening` (`sandboxPolicy.ts`) states the whole boundary of a run in a checkout in one place — what it reads and writes for a folder holding its store and installations and for any other — which a test holds the compiled policy to, path for path (`sandboxPolicy.test.ts`). Test: a subdirectory of a repository under a denied home, connected, runs `git log -1` and resolves a hoisted package inside a folder check's sandbox, and reaches neither without the borrowed reads; a linked worktree under a denied home, connected, runs `git log -1` there (`subdirectoryBoundary.test.ts`, sandboxed). Tests: a sweep of a marked scratch holding a half-made clone removes it with no ref or registration of the folder touched, and unmarked directories named like scratch are left; a marked scratch holding links to a project with a linked `node_modules`, as a checkout and as a git directory, is swept with every link in that project left; a folder connected inside a repository gets a checkout whose change is the folder's alone and applies to it (`worktree.test.ts`); a package connected as the folder gets the top level's installation and its own linked, a workspace package resolving into the clone, and a sibling package's own dependency resolving inside the clone, and a folder three levels down resolving its own dependency inside the clone (`worktreeDependencies.test.ts`); `borrowedReads` reads the same set, a folder below the scan's depth borrowing each installation on its way up (`subdirectoryBoundary.test.ts`); a sandboxed run handed `agent/` and killed with the executor is journalled with the scratch holding its clone, which the reap removes clone and all (`worktree.test.ts`, sandboxed), and a commission hands the sandbox that scratch as `journalScratchDir` (`sandboxPosture.test.ts`). The directory holding a checkout and its git directory is never the sandbox's to write. A commission and a check are handed `agent/` inside their scratch (`makeAgentScratch`, `executorScratch.ts`) — their home, temporary directory, harness configuration, browser pipes and check report — with the folder's place in the checkout as their writable root, so a job cannot remove its checkout and put a link to a directory of the operator's in its place. And because git, told `--work-tree`, takes a link there as readily as a directory, every git the executor runs on a checkout first finds the checkout and the folder's place in it real directories whose resolved path is the one `prepareWorktree` recorded, resolved, and refuses otherwise (`assertCheckoutInPlace`, in `gitLocation`) — staging (`collectChanges`), the base merge (`mergeIntoCheckout`) and `checkApplies` among them. Tests: a sandboxed job's attempt to remove or move its checkout or git directory and put a link in its place is refused (sandboxed), and a link put from outside in place of the checkout, or of the folder's place in it, is refused by collection and by the merge with the directory it leads to untouched (`checkoutReplacement.test.ts`); a commission is handed `agent/` beside its checkout (`sandboxPosture.test.ts`). **Then F56's git directory — out of the sandbox's reach**: a clone whose git directory was `/.git` sat inside the root the sandbox writes, where the policy closed only `config` and `hooks`. An agent could write `.git/commondir` naming a repository it built in its scratch with a `filter..clean` defined, add a `.gitattributes` selecting it, and the executor's own unconfined git — `collectChanges`' `git add -A` and `checkApplies` — read that config and ran its program with the executor's PATH and HOME. So the clone's git directory lives beside the checkout in the run's scratch (`checkoutGitDir`: `/git-`), mirrored there and given a working tree whose `.git` is only a gitfile naming it — what `clone --separate-git-dir` makes, which git refuses beside `--mirror`. The scratch around it is writable, so the sandbox is handed it by name (`SandboxWidening.gitDir`) and reads it without writing it, for a commission and for a check. Every git the executor runs in a checkout names that directory itself — `--git-dir` and `--work-tree`, through a `Checkout` that `git()` takes beside a folder path — and never resolves through the gitfile, which the agent may rewrite and which then moves only the agent's own git; the publication fetches its commit from the git directory, not the checkout. The agent's git therefore moves no ref at all, the clone's included. `removeWorktree` takes the git directory with the checkout, and the boot sweep counts a checkout as a directory holding a `.git` or a git directory named for one. A publication is refused below git 2.29, the first whose `git fetch` takes `--no-write-fetch-head`, by `commitPatchOnBranch` before it makes anything (`gitReadiness.ts`); a commission, which never runs that fetch, is not. Commission Change goes to version 4, since its outputs changed. Tests: a sandboxed agent writes a `commondir` and a filter into the clone's git directory (both refused), points the checkout's gitfile at a git directory whose `commondir` names a repository it built with a filter defined, and adds a `.gitattributes` selecting it; collection and the apply check run nothing and read the executor's git directory, while a git resolving through the gitfile with the same safety arguments runs the filter (`checkoutGitDir.test.ts`; unconfined, the executor half in `gitFilterEscape.test.ts`). From 8df9fc596ce55b3f437033a8beefa59db1b5e5d2 Mon Sep 17 00:00:00 2001 From: karim Date: Sat, 17 Oct 2026 13:26:00 +0200 Subject: [PATCH 2/4] Split the oversized new file under the large-files guard's 1,500-line hard limit --- .../src/objectAlternates.ts | 81 ++++++++++++++++ apps/aflow-executor-host/src/worktree.ts | 97 +++---------------- 2 files changed, 94 insertions(+), 84 deletions(-) create mode 100644 apps/aflow-executor-host/src/objectAlternates.ts diff --git a/apps/aflow-executor-host/src/objectAlternates.ts b/apps/aflow-executor-host/src/objectAlternates.ts new file mode 100644 index 00000000..106d0ed2 --- /dev/null +++ b/apps/aflow-executor-host/src/objectAlternates.ts @@ -0,0 +1,81 @@ +/** + * The object stores a git object store borrows from through + * `info/alternates`, read as git reads them. + */ +import { readFile, realpath } from 'node:fs/promises'; +import { join, resolve as resolvePath } from 'node:path'; + +const C_ESCAPES: Readonly> = { + a: 7, + b: 8, + t: 9, + n: 10, + v: 11, + f: 12, + r: 13, + '"': 34, + '\\': 92, +}; + +/** An entry in git's C-style quoting, unquoted; undefined when it is not one. */ +function unquotedC(entry: string): string | undefined { + const bytes: number[] = []; + for (let i = 1; i < entry.length; i += 1) { + const char = entry.charAt(i); + if (char === '"') { + return i === entry.length - 1 ? Buffer.from(bytes).toString('utf8') : undefined; + } + if (char !== '\\') { + bytes.push(...Buffer.from(char, 'utf8')); + continue; + } + const octal = /^[0-3][0-7]{2}/.exec(entry.slice(i + 1)); + const escaped = C_ESCAPES[entry.charAt(i + 1)]; + if (octal !== null) { + bytes.push(Number.parseInt(octal[0], 8)); + i += 3; + } else if (escaped !== undefined) { + bytes.push(escaped); + i += 1; + } else { + return undefined; + } + } + return undefined; +} + +/** The stores `store`'s `info/alternates` names, as git reads them, unresolved. */ +async function alternatesOf(store: string): Promise { + const text = await readFile(join(store, 'info', 'alternates'), 'utf8').catch(() => ''); + return text + .split('\n') + .filter((entry) => entry !== '' && !entry.startsWith('#')) + .map((entry) => (entry.startsWith('"') ? (unquotedC(entry) ?? entry) : entry)) + .map((entry) => resolvePath(store, entry)); +} + +/** + * Appends to `stores` each store `store` borrows from, resolved, following + * `info/alternates` as git does: depth first, an entry relative to the store + * naming it, a store missing or already named skipped. Stops at the first store + * still borrowing `levels` levels below `store`, and resolves to it; resolves + * to undefined once the whole chain is followed. + */ +export async function appendAlternateStores( + store: string, + levels: number, + stores: string[], + level = 0, +): Promise { + const entries = await alternatesOf(store); + if (entries.length === 0) return undefined; + if (level >= levels) return store; + for (const entry of entries) { + const resolved = await realpath(entry).catch(() => undefined); + if (resolved === undefined || stores.includes(resolved)) continue; + stores.push(resolved); + const tooDeep = await appendAlternateStores(resolved, levels, stores, level + 1); + if (tooDeep !== undefined) return tooDeep; + } + return undefined; +} diff --git a/apps/aflow-executor-host/src/worktree.ts b/apps/aflow-executor-host/src/worktree.ts index e4dc752a..28dd53fb 100644 --- a/apps/aflow-executor-host/src/worktree.ts +++ b/apps/aflow-executor-host/src/worktree.ts @@ -32,7 +32,6 @@ import { mkdir, mkdtemp, readdir, - readFile, realpath, rm, rmdir, @@ -51,6 +50,8 @@ import { } from 'node:path'; import { promisify } from 'node:util'; +import { appendAlternateStores } from './objectAlternates.js'; + const run = promisify(execFile); const GIT_TIMEOUT_MS = 30_000; @@ -742,88 +743,6 @@ export async function folderPrefix(root: string): Promise { */ const FOLDER_ALTERNATES_LEVELS = 5; -const C_ESCAPES: Readonly> = { - a: 7, - b: 8, - t: 9, - n: 10, - v: 11, - f: 12, - r: 13, - '"': 34, - '\\': 92, -}; - -/** An entry in git's C-style quoting, unquoted; undefined when it is not one. */ -function unquotedC(entry: string): string | undefined { - const bytes: number[] = []; - for (let i = 1; i < entry.length; i += 1) { - const char = entry.charAt(i); - if (char === '"') { - return i === entry.length - 1 ? Buffer.from(bytes).toString('utf8') : undefined; - } - if (char !== '\\') { - bytes.push(...Buffer.from(char, 'utf8')); - continue; - } - const octal = /^[0-3][0-7]{2}/.exec(entry.slice(i + 1)); - const escaped = C_ESCAPES[entry.charAt(i + 1)]; - if (octal !== null) { - bytes.push(Number.parseInt(octal[0], 8)); - i += 3; - } else if (escaped !== undefined) { - bytes.push(escaped); - i += 1; - } else { - return undefined; - } - } - return undefined; -} - -/** The stores `store`'s `info/alternates` names, as git reads them, unresolved. */ -async function alternatesOf(store: string): Promise { - const text = await readFile(join(store, 'info', 'alternates'), 'utf8').catch(() => ''); - return text - .split('\n') - .filter((entry) => entry !== '' && !entry.startsWith('#')) - .map((entry) => (entry.startsWith('"') ? (unquotedC(entry) ?? entry) : entry)) - .map((entry) => resolvePath(store, entry)); -} - -/** - * Appends to `stores` each store `store` borrows from, resolved, following - * `info/alternates` as git does: depth first, an entry relative to the store - * naming it, a store missing or already named skipped. A chain deeper than a - * clone of the folder follows is refused, since a run's checkout would not - * reach the objects at its end. - */ -async function appendAlternateStores( - root: string, - store: string, - level: number, - stores: string[], -): Promise { - const entries = await alternatesOf(store); - if (entries.length === 0) return; - if (level >= FOLDER_ALTERNATES_LEVELS) { - throw new WorktreeError( - `The object store of ${root} borrows through more levels of \`objects/info/alternates\` ` + - "than git follows from a clone of it, so a run's checkout could not read every object " + - `the folder has; the level a clone stops at is ${store}. Copy the borrowed objects ` + - "into the folder's repository with `git repack -a -d`, remove its " + - '`objects/info/alternates`, and run again.', - 'alternates_too_deep', - ); - } - for (const entry of entries) { - const resolved = await realpath(entry).catch(() => undefined); - if (resolved === undefined || stores.includes(resolved)) continue; - stores.push(resolved); - await appendAlternateStores(root, resolved, level + 1, stores); - } -} - /** * What a run in a checkout of the folder at `root` reads outside that folder, * for the sandbox to grant read of (`SandboxWidening.borrowedReads`). @@ -848,7 +767,17 @@ export async function borrowedReads(root: string): Promise { const objects = await gitPath(root, 'objects'); const objectsReal = await realpath(objects).catch(() => objects); const stores = [objectsReal]; - await appendAlternateStores(root, objectsReal, 0, stores); + const tooDeep = await appendAlternateStores(objectsReal, FOLDER_ALTERNATES_LEVELS, stores); + if (tooDeep !== undefined) { + throw new WorktreeError( + `The object store of ${root} borrows through more levels of \`objects/info/alternates\` ` + + "than git follows from a clone of it, so a run's checkout could not read every object " + + `the folder has; the level a clone stops at is ${tooDeep}. Copy the borrowed objects ` + + "into the folder's repository with `git repack -a -d`, remove its " + + '`objects/info/alternates`, and run again.', + 'alternates_too_deep', + ); + } reads.push(...stores.filter(outsideRoot)); const top = (await git(root, ['rev-parse', '--show-toplevel'])).trim(); for (const relative of await repositoryInstallations(top, await folderPrefix(root))) { From ac0ecce3ec01aee2f9f057d2f7d022ef097c384b Mon Sep 17 00:00:00 2001 From: karim Date: Sat, 17 Oct 2026 13:42:36 +0200 Subject: [PATCH 3/4] Fix round: count only live, new stores toward the alternates depth; state the F56 boundary as built - objectAlternates: the depth check counts only stores that exist and are new to the collected set, so a deepest level of duplicates and missing entries does not refuse a folder a clone would read in full; test walks such a chain and asserts it is not refused - docs/plans/315: the F56 entry says the chain is followed as git follows it, that a chain deeper than a clone would read is refused by name with the operator's remedy, and that the brief names the rules file at the clone's top as well as the folder --- .../__tests__/subdirectoryBoundary.test.ts | 24 +++++++++++++++++++ .../src/objectAlternates.ts | 12 +++++----- docs/plans/315-local-first-run-ergonomics.md | 2 +- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts b/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts index cca4d1e8..3a349ec6 100644 --- a/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts +++ b/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts @@ -229,6 +229,30 @@ describe('a repository whose store borrows from others, connected under a denied ); }); + it('is not refused when its deepest level names only missing and already-collected stores', async () => { + // Only a store that exists and is new to those collected counts toward the + // depth: the rest hold nothing a clone of the folder would miss. + let below = shared; + const stores: string[] = []; + for (let level = 0; level < 4; level += 1) { + const next = join(home, 'projects', `level-${String(level)}`); + await run('git', ['clone', '-q', '--shared', below, next]); + stores.unshift(objectsOf(next)); + below = next; + } + // `top` is five levels below `below`'s store, the deepest a clone reads. + await writeFile( + join(objectsOf(top), 'info', 'alternates'), + `${join(home, 'projects', 'missing', '.git', 'objects')}\n${objectsOf(shared)}\n` + + `${objectsOf(below)}\n`, + ); + expect(await borrowedReads(below)).toEqual([ + ...stores.slice(1), + objectsOf(shared), + objectsOf(top), + ]); + }); + it.skipIf(!sandboxAvailable() || confined.skip)( confined.title('reads its history inside the run'), { tags: ['listener'], timeout: 120_000 }, diff --git a/apps/aflow-executor-host/src/objectAlternates.ts b/apps/aflow-executor-host/src/objectAlternates.ts index 106d0ed2..da4e405d 100644 --- a/apps/aflow-executor-host/src/objectAlternates.ts +++ b/apps/aflow-executor-host/src/objectAlternates.ts @@ -58,8 +58,10 @@ async function alternatesOf(store: string): Promise { * Appends to `stores` each store `store` borrows from, resolved, following * `info/alternates` as git does: depth first, an entry relative to the store * naming it, a store missing or already named skipped. Stops at the first store - * still borrowing `levels` levels below `store`, and resolves to it; resolves - * to undefined once the whole chain is followed. + * `levels` levels below `store` that still borrows a store existing and not yet + * named, and resolves to it; resolves to undefined once the whole chain is + * followed. A store whose entries name only missing or already-named stores + * holds nothing a clone would miss, however deep it lies. */ export async function appendAlternateStores( store: string, @@ -67,12 +69,10 @@ export async function appendAlternateStores( stores: string[], level = 0, ): Promise { - const entries = await alternatesOf(store); - if (entries.length === 0) return undefined; - if (level >= levels) return store; - for (const entry of entries) { + for (const entry of await alternatesOf(store)) { const resolved = await realpath(entry).catch(() => undefined); if (resolved === undefined || stores.includes(resolved)) continue; + if (level >= levels) return store; stores.push(resolved); const tooDeep = await appendAlternateStores(resolved, levels, stores, level + 1); if (tooDeep !== undefined) return tooDeep; diff --git a/docs/plans/315-local-first-run-ergonomics.md b/docs/plans/315-local-first-run-ergonomics.md index ac415dfe..1ea86100 100644 --- a/docs/plans/315-local-first-run-ergonomics.md +++ b/docs/plans/315-local-first-run-ergonomics.md @@ -162,7 +162,7 @@ After D13 lands: the chat renders a run's wakeup as a card that says which run s **Then F118 — the start rebuilds the packages the web application reads**: `scripts/dev.mjs`, which `yarn start` and `yarn dev:local` reach through the `local` profile, no longer runs the advisory staleness check that called a stale `dist` safe to ignore. Before any service starts it takes the Next applications among the services it is about to run (a workspace whose `dependencies` name `next`, found through the service's command as `whyUnavailable` finds it) and their dist closure — every workspace package their `dependencies`, `peerDependencies` and `optionalDependencies` reach, transitively, each after what it depends on — from the manifests alone (`scripts/webDists.mjs`); no list of packages is written anywhere. A package in it is stale where a file the `import` condition names is missing or older than the newest file of its `src/`, tests and `.tsbuildinfo` aside; measuring the files the bundler loads rather than all of `dist/` is deliberate, since a type-check writes declarations there without touching the JavaScript and would make a stale build look current. Each stale package is built with `yarn workspace build`, dependencies first, under one line that names them and says which application reads them from `dist/` where every other service reads `src/`, then one line per package with what made it stale and how long it took; a package already current is not built and a start with nothing changed prints nothing. A package the start runs a watcher for — `@aflow/web-product` — is named as stale and left to that watcher's first build. A build that fails does not stop the start, since the Node services read source: its output is printed with what it costs the web app and the command that rebuilds it. A profile with no web application — `core`, `engine`, `mcp` — selects nothing. `yarn check:packages` (`check-package-staleness.mjs`) shares the rule and now names each stale package and which consumer reads `dist/`. Tests: `scripts/webDists.test.mjs` measures the repository's own manifests with injected times — a touched `packages/schemas/src` file selects `@aflow/schemas` alone, a touched orchestrator file selects nothing, a current tree selects nothing, a watched package is left to its watcher, a fresh clone selects the whole closure dependencies first — and two packages on disk, one whose declarations a type-check rewrote after its source changed (stale) and one whose only newer file is a test (current). Not taken: the dev health check that requests the web app's first page, which would add a probe and its failure path to the runner; it stays open on this row, as F58's watch of the imported packages during a session does. -**Then F56 — the private clone is the boundary**: every checkout the host lane makes — a commission's, a check's, a publication's — is a private clone of the folder rather than a worktree added to it. `prepareWorktree` (`worktree.ts`) clones the folder's repository from its top level (`rev-parse --show-toplevel`, since `git clone` takes nothing else) with `--mirror --shared` into a git directory of its own beside the checkout (`git-` in the run's scratch, `checkoutGitDir`), which the sandbox reads and never writes, gives it a working tree whose `.git` is only a gitfile naming that directory — the executor's own git names the directory and the working tree outright (`Checkout`) and never resolves the checkout through what it holds — removes its remote and checks out the base commit detached: the clone reads every ref as the folder names it when it is made, so `origin/main`, a branch and a tag resolve as they do there, and borrows the folder's objects through `objects/info/alternates`, while its refs and config are its own. A `git update-ref`, a `git branch -D`, a call that names its own `core.hooksPath` or a push to `.` in the run moves no ref at all — the clone's git directory is read-only to the run — and a push to the folder's path meets a `.git` the sandbox does not let it write (D19). So the guard and the record are gone rather than kept beside it: `refGuard.ts` and its `reference-transaction` hook, with the git 2.28 floor a commission was refused under, and `snapshotRefs`/`changedRefs` with the result's `refChanges`, which leaves `HostHarnessRunOutputSchema`, `host.harness.run`'s description and Commission Change's promoted outputs. The publication reads its commit from its clone: `commitPatchOnBranch` (`branchCommit.ts`) applies and commits there, then the executor's git fetches that commit into the folder by sha (`--no-write-fetch-head`, so no FETCH_HEAD another fetch is reading is overwritten; git 2.29 or later) and creates the named branch, or advances it by compare-and-swap on the head it was made on — the one write to the folder's refs the loop makes. A commit's identity is read from the folder's config, since the clone's holds none; a commission's `applies` is judged against a merge commit in the clone, where it exists. `removeWorktree` takes the clone's path and removes the directory, which is the whole of it, and `HarnessSession` no longer carries `bindingRoot`. The boot sweep (`removeOrphanedCheckouts`) takes a scratch directory named as this executor names its own and carrying the marker `makeScratch` (`executorScratch.ts`) writes into every scratch it makes — a commission's, a check's, a publication's and an ephemeral browser profile's — and takes it whole, whatever its checkouts are: a clone killed before its detached checkout, whose HEAD still names a branch, goes with it. The shape of a HEAD decides nothing; a directory without the marker is left whatever it holds; a crashed run's journal entry (`orphans.ts`) removes its scratch, clone and all, with nothing left registered in the folder — for a commission and a check, which hand the sandbox only `agent/` inside it, the journal records the scratch holding the clone (`journalScratchDir`, passed to `recordSpawn` apart from the sandbox's `scratchDir`, `sandboxedRun.ts`), while a run that ends removes `agent/` alone and its caller the rest. Tests: a run whose agent deletes a branch, moves `main` and tags under its own hooks path, pushes to `.` and pushes and deletes into the folder's path leaves every ref of the folder as it was, is handed no `GIT_CONFIG_*` and names no remote, and its publication lands exactly its commit on the named branch with no other ref moved (`harnessResult.test.ts`, sandboxed); the clone's refs, alternates and detached HEAD, every move made inside it reaching only it, and a folder commit made after the clone read through what it borrows (`worktree.test.ts`); the publication's fetch writing the named branch and no other ref and no FETCH_HEAD (`patchApply.test.ts`); a clone removed when its run ends and swept when it is orphaned, by the boot sweep and by the journal, with the operator's own checkouts left (`worktree.test.ts`, `phase2Acceptance.test.ts`). Inside a marked scratch only a real directory is treated as a checkout, read with `lstat`: a link planted there, followed, led the removal's unlinking of mirrored installations into the `node_modules` of whatever the link pointed at, the operator's own project included. A link is now removed with the scratch and nothing it leads to is walked, and `removeWorktree` acts on its checkout and git directory only as real directories. The sweep handles marked scratch only and reads no connected folder, so the detached worktrees the executor before the private clone registered in an operator's repository, in scratch it never marked, are the operator's to prune once by hand (`git worktree remove --force` on each that `git worktree list` shows under the temp root). A folder connected as a subdirectory of a repository — a package of a monorepo, which `rev-parse --is-inside-work-tree` accepts — is cloned from that top level, and the checkout's `workDir` is the folder's path relative to it inside the clone: the harness and a check run there with it as their writable root, the installations of the repository are linked at the same place in the clone — the union of those the scan from the top level reaches, exactly as for a folder that is the top level, so a sibling package's own installation is there too, and those on the walk from the folder up to the top level, its own included, which the scan's depth (`DEPENDENCY_SCAN_DEPTH`) cannot cut short however deep the folder lies (`repositoryInstallations`) — and a workspace link resolving anywhere in the repository — a sibling package included — points at that path in the clone, so resolution in the clone sees what it sees in the folder, and `collectChanges` stages and diffs that directory alone, as paths from the top level, which is what `git apply` reads in the folder and what the publication applies at the clone's top. A session's checkout is found again from its binding's prefix (`folderPrefix`). Inside the run, a clone reads outside the folder wherever what it borrows lies outside it: it owns no object and borrows every one from the repository's store (`rev-parse --git-path objects`, resolved), which is outside the folder for a subdirectory and equally for a top level that is a linked worktree, a submodule or a repository with a separate git directory, and its mirrored installations outside the folder lead into `/node_modules` and the like, all under a home the sandbox denies. So the sandbox is handed exactly those that lie outside the folder, decided by location and never by the folder's prefix, read and never written, for a commission and a check alike (`borrowedReads`, `worktree.ts`, read from the folder rather than the checkout the run writes), and `checkoutWidening` (`sandboxPolicy.ts`) states the whole boundary of a run in a checkout in one place — what it reads and writes for a folder holding its store and installations and for any other — which a test holds the compiled policy to, path for path (`sandboxPolicy.test.ts`). Test: a subdirectory of a repository under a denied home, connected, runs `git log -1` and resolves a hoisted package inside a folder check's sandbox, and reaches neither without the borrowed reads; a linked worktree under a denied home, connected, runs `git log -1` there (`subdirectoryBoundary.test.ts`, sandboxed). Tests: a sweep of a marked scratch holding a half-made clone removes it with no ref or registration of the folder touched, and unmarked directories named like scratch are left; a marked scratch holding links to a project with a linked `node_modules`, as a checkout and as a git directory, is swept with every link in that project left; a folder connected inside a repository gets a checkout whose change is the folder's alone and applies to it (`worktree.test.ts`); a package connected as the folder gets the top level's installation and its own linked, a workspace package resolving into the clone, and a sibling package's own dependency resolving inside the clone, and a folder three levels down resolving its own dependency inside the clone (`worktreeDependencies.test.ts`); `borrowedReads` reads the same set, a folder below the scan's depth borrowing each installation on its way up (`subdirectoryBoundary.test.ts`); a sandboxed run handed `agent/` and killed with the executor is journalled with the scratch holding its clone, which the reap removes clone and all (`worktree.test.ts`, sandboxed), and a commission hands the sandbox that scratch as `journalScratchDir` (`sandboxPosture.test.ts`). The directory holding a checkout and its git directory is never the sandbox's to write. A commission and a check are handed `agent/` inside their scratch (`makeAgentScratch`, `executorScratch.ts`) — their home, temporary directory, harness configuration, browser pipes and check report — with the folder's place in the checkout as their writable root, so a job cannot remove its checkout and put a link to a directory of the operator's in its place. And because git, told `--work-tree`, takes a link there as readily as a directory, every git the executor runs on a checkout first finds the checkout and the folder's place in it real directories whose resolved path is the one `prepareWorktree` recorded, resolved, and refuses otherwise (`assertCheckoutInPlace`, in `gitLocation`) — staging (`collectChanges`), the base merge (`mergeIntoCheckout`) and `checkApplies` among them. Tests: a sandboxed job's attempt to remove or move its checkout or git directory and put a link in its place is refused (sandboxed), and a link put from outside in place of the checkout, or of the folder's place in it, is refused by collection and by the merge with the directory it leads to untouched (`checkoutReplacement.test.ts`); a commission is handed `agent/` beside its checkout (`sandboxPosture.test.ts`). +**Then F56 — the private clone is the boundary**: every checkout the host lane makes — a commission's, a check's, a publication's — is a private clone of the folder rather than a worktree added to it. `prepareWorktree` (`worktree.ts`) clones the folder's repository from its top level (`rev-parse --show-toplevel`, since `git clone` takes nothing else) with `--mirror --shared` into a git directory of its own beside the checkout (`git-` in the run's scratch, `checkoutGitDir`), which the sandbox reads and never writes, gives it a working tree whose `.git` is only a gitfile naming that directory — the executor's own git names the directory and the working tree outright (`Checkout`) and never resolves the checkout through what it holds — removes its remote and checks out the base commit detached: the clone reads every ref as the folder names it when it is made, so `origin/main`, a branch and a tag resolve as they do there, and borrows the folder's objects through `objects/info/alternates`, while its refs and config are its own. A `git update-ref`, a `git branch -D`, a call that names its own `core.hooksPath` or a push to `.` in the run moves no ref at all — the clone's git directory is read-only to the run — and a push to the folder's path meets a `.git` the sandbox does not let it write (D19). So the guard and the record are gone rather than kept beside it: `refGuard.ts` and its `reference-transaction` hook, with the git 2.28 floor a commission was refused under, and `snapshotRefs`/`changedRefs` with the result's `refChanges`, which leaves `HostHarnessRunOutputSchema`, `host.harness.run`'s description and Commission Change's promoted outputs. The publication reads its commit from its clone: `commitPatchOnBranch` (`branchCommit.ts`) applies and commits there, then the executor's git fetches that commit into the folder by sha (`--no-write-fetch-head`, so no FETCH_HEAD another fetch is reading is overwritten; git 2.29 or later) and creates the named branch, or advances it by compare-and-swap on the head it was made on — the one write to the folder's refs the loop makes. A commit's identity is read from the folder's config, since the clone's holds none; a commission's `applies` is judged against a merge commit in the clone, where it exists. `removeWorktree` takes the clone's path and removes the directory, which is the whole of it, and `HarnessSession` no longer carries `bindingRoot`. The boot sweep (`removeOrphanedCheckouts`) takes a scratch directory named as this executor names its own and carrying the marker `makeScratch` (`executorScratch.ts`) writes into every scratch it makes — a commission's, a check's, a publication's and an ephemeral browser profile's — and takes it whole, whatever its checkouts are: a clone killed before its detached checkout, whose HEAD still names a branch, goes with it. The shape of a HEAD decides nothing; a directory without the marker is left whatever it holds; a crashed run's journal entry (`orphans.ts`) removes its scratch, clone and all, with nothing left registered in the folder — for a commission and a check, which hand the sandbox only `agent/` inside it, the journal records the scratch holding the clone (`journalScratchDir`, passed to `recordSpawn` apart from the sandbox's `scratchDir`, `sandboxedRun.ts`), while a run that ends removes `agent/` alone and its caller the rest. Tests: a run whose agent deletes a branch, moves `main` and tags under its own hooks path, pushes to `.` and pushes and deletes into the folder's path leaves every ref of the folder as it was, is handed no `GIT_CONFIG_*` and names no remote, and its publication lands exactly its commit on the named branch with no other ref moved (`harnessResult.test.ts`, sandboxed); the clone's refs, alternates and detached HEAD, every move made inside it reaching only it, and a folder commit made after the clone read through what it borrows (`worktree.test.ts`); the publication's fetch writing the named branch and no other ref and no FETCH_HEAD (`patchApply.test.ts`); a clone removed when its run ends and swept when it is orphaned, by the boot sweep and by the journal, with the operator's own checkouts left (`worktree.test.ts`, `phase2Acceptance.test.ts`). Inside a marked scratch only a real directory is treated as a checkout, read with `lstat`: a link planted there, followed, led the removal's unlinking of mirrored installations into the `node_modules` of whatever the link pointed at, the operator's own project included. A link is now removed with the scratch and nothing it leads to is walked, and `removeWorktree` acts on its checkout and git directory only as real directories. The sweep handles marked scratch only and reads no connected folder, so the detached worktrees the executor before the private clone registered in an operator's repository, in scratch it never marked, are the operator's to prune once by hand (`git worktree remove --force` on each that `git worktree list` shows under the temp root). A folder connected as a subdirectory of a repository — a package of a monorepo, which `rev-parse --is-inside-work-tree` accepts — is cloned from that top level, and the checkout's `workDir` is the folder's path relative to it inside the clone: the harness and a check run there with it as their writable root, the installations of the repository are linked at the same place in the clone — the union of those the scan from the top level reaches, exactly as for a folder that is the top level, so a sibling package's own installation is there too, and those on the walk from the folder up to the top level, its own included, which the scan's depth (`DEPENDENCY_SCAN_DEPTH`) cannot cut short however deep the folder lies (`repositoryInstallations`) — and a workspace link resolving anywhere in the repository — a sibling package included — points at that path in the clone, so resolution in the clone sees what it sees in the folder, and `collectChanges` stages and diffs that directory alone, as paths from the top level, which is what `git apply` reads in the folder and what the publication applies at the clone's top. A session's checkout is found again from its binding's prefix (`folderPrefix`). Inside the run, a clone reads outside the folder wherever what it borrows lies outside it: it owns no object and borrows every one from the repository's store (`rev-parse --git-path objects`, resolved) and from each store that one borrows from in turn, its `objects/info/alternates` followed as git follows it (`appendAlternateStores`, `objectAlternates.ts`): depth first, an entry relative to the store naming it, a C-quoted entry unquoted, a comment, a missing store and a store already collected skipped. Those stores are outside the folder for a subdirectory and equally for a top level that is a linked worktree, a submodule, a repository with a separate git directory or a clone made with `--shared` or `--reference`, and its mirrored installations outside the folder lead into `/node_modules` and the like, all under a home the sandbox denies. So the sandbox is handed exactly those that lie outside the folder, decided by location and never by the folder's prefix, read and never written, for a commission and a check alike (`borrowedReads`, `worktree.ts`, read from the folder rather than the checkout the run writes), and `checkoutWidening` (`sandboxPolicy.ts`) states the whole boundary of a run in a checkout in one place — what it reads and writes for a folder holding its store and installations and for any other — which a test holds the compiled policy to, path for path (`sandboxPolicy.test.ts`). A chain deeper than a clone of the folder reads is refused by name (`alternates_too_deep`): git follows six levels of alternates from the store it starts in, and the clone's own, naming the folder's store, is the first (`FOLDER_ALTERNATES_LEVELS`), so a store five levels below the folder's that still borrows a store refuses the folder where that store exists and is not yet collected, the refusal naming that store and the operator's remedy — copy the borrowed objects into the folder's repository with `git repack -a -d`, remove its `objects/info/alternates`, and run again. A deepest level whose entries name only missing or already-collected stores holds nothing a clone would miss, and the folder is not refused. The brief names the repository's rules file (`CLAUDE.md`) wherever the checkout holds it, by its path from the folder's place in the clone: in the folder and, for a folder inside a repository, at the clone's top as well, each named with where it is (`repositoryRulesFiles`, `harnessHandlers.ts`); a bare Claude Code discovers neither, and the checkout, never the operator's folder, is what is asked. Test: a subdirectory of a repository under a denied home, connected, runs `git log -1` and resolves a hoisted package inside a folder check's sandbox, and reaches neither without the borrowed reads; a linked worktree under a denied home, connected, runs `git log -1` there (`subdirectoryBoundary.test.ts`, sandboxed); a repository borrowing through a chain of `--shared` clones borrows every store along it, read as git reads it, is refused naming the remedy where the chain is deeper than a clone reads, and is not refused where its deepest level names only missing and already-collected stores (`subdirectoryBoundary.test.ts`); a package of a repository is briefed with the repository's rules, with its own and the repository's each where it is, and with none where the repository holds none (`packageRules.test.ts`). Tests: a sweep of a marked scratch holding a half-made clone removes it with no ref or registration of the folder touched, and unmarked directories named like scratch are left; a marked scratch holding links to a project with a linked `node_modules`, as a checkout and as a git directory, is swept with every link in that project left; a folder connected inside a repository gets a checkout whose change is the folder's alone and applies to it (`worktree.test.ts`); a package connected as the folder gets the top level's installation and its own linked, a workspace package resolving into the clone, and a sibling package's own dependency resolving inside the clone, and a folder three levels down resolving its own dependency inside the clone (`worktreeDependencies.test.ts`); `borrowedReads` reads the same set, a folder below the scan's depth borrowing each installation on its way up (`subdirectoryBoundary.test.ts`); a sandboxed run handed `agent/` and killed with the executor is journalled with the scratch holding its clone, which the reap removes clone and all (`worktree.test.ts`, sandboxed), and a commission hands the sandbox that scratch as `journalScratchDir` (`sandboxPosture.test.ts`). The directory holding a checkout and its git directory is never the sandbox's to write. A commission and a check are handed `agent/` inside their scratch (`makeAgentScratch`, `executorScratch.ts`) — their home, temporary directory, harness configuration, browser pipes and check report — with the folder's place in the checkout as their writable root, so a job cannot remove its checkout and put a link to a directory of the operator's in its place. And because git, told `--work-tree`, takes a link there as readily as a directory, every git the executor runs on a checkout first finds the checkout and the folder's place in it real directories whose resolved path is the one `prepareWorktree` recorded, resolved, and refuses otherwise (`assertCheckoutInPlace`, in `gitLocation`) — staging (`collectChanges`), the base merge (`mergeIntoCheckout`) and `checkApplies` among them. Tests: a sandboxed job's attempt to remove or move its checkout or git directory and put a link in its place is refused (sandboxed), and a link put from outside in place of the checkout, or of the folder's place in it, is refused by collection and by the merge with the directory it leads to untouched (`checkoutReplacement.test.ts`); a commission is handed `agent/` beside its checkout (`sandboxPosture.test.ts`). **Then F56's git directory — out of the sandbox's reach**: a clone whose git directory was `/.git` sat inside the root the sandbox writes, where the policy closed only `config` and `hooks`. An agent could write `.git/commondir` naming a repository it built in its scratch with a `filter..clean` defined, add a `.gitattributes` selecting it, and the executor's own unconfined git — `collectChanges`' `git add -A` and `checkApplies` — read that config and ran its program with the executor's PATH and HOME. So the clone's git directory lives beside the checkout in the run's scratch (`checkoutGitDir`: `/git-`), mirrored there and given a working tree whose `.git` is only a gitfile naming it — what `clone --separate-git-dir` makes, which git refuses beside `--mirror`. The scratch around it is writable, so the sandbox is handed it by name (`SandboxWidening.gitDir`) and reads it without writing it, for a commission and for a check. Every git the executor runs in a checkout names that directory itself — `--git-dir` and `--work-tree`, through a `Checkout` that `git()` takes beside a folder path — and never resolves through the gitfile, which the agent may rewrite and which then moves only the agent's own git; the publication fetches its commit from the git directory, not the checkout. The agent's git therefore moves no ref at all, the clone's included. `removeWorktree` takes the git directory with the checkout, and the boot sweep counts a checkout as a directory holding a `.git` or a git directory named for one. A publication is refused below git 2.29, the first whose `git fetch` takes `--no-write-fetch-head`, by `commitPatchOnBranch` before it makes anything (`gitReadiness.ts`); a commission, which never runs that fetch, is not. Commission Change goes to version 4, since its outputs changed. Tests: a sandboxed agent writes a `commondir` and a filter into the clone's git directory (both refused), points the checkout's gitfile at a git directory whose `commondir` names a repository it built with a filter defined, and adds a `.gitattributes` selecting it; collection and the apply check run nothing and read the executor's git directory, while a git resolving through the gitfile with the same safety arguments runs the filter (`checkoutGitDir.test.ts`; unconfined, the executor half in `gitFilterEscape.test.ts`). From 0a90d739ae974bae73776254c99694abf9d36cba Mon Sep 17 00:00:00 2001 From: karim Date: Sat, 17 Oct 2026 14:00:15 +0200 Subject: [PATCH 4/4] Fix round: pin the session-keeping order with a test; decode quoted alternates entries by code point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - harnessHandlers: a test pins borrowedReads before removeWorktree/prepareWorktree — a turn naming a new base on a folder the alternates check refuses leaves the session's checkout in place and the session held - objectAlternates: the quoted-entry decoder walks code points, not UTF-16 code units, so an unescaped character outside the BMP no longer splits into lone surrogates and skips the store; test holds such an entry --- .../sessionCheckoutOnRefusal.test.ts | 194 ++++++++++++++++++ .../__tests__/subdirectoryBoundary.test.ts | 14 ++ .../src/objectAlternates.ts | 5 +- 3 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 apps/aflow-executor-host/src/__tests__/sessionCheckoutOnRefusal.test.ts diff --git a/apps/aflow-executor-host/src/__tests__/sessionCheckoutOnRefusal.test.ts b/apps/aflow-executor-host/src/__tests__/sessionCheckoutOnRefusal.test.ts new file mode 100644 index 00000000..029d94c1 --- /dev/null +++ b/apps/aflow-executor-host/src/__tests__/sessionCheckoutOnRefusal.test.ts @@ -0,0 +1,194 @@ +/** + * Contract: a turn that names a new base on a folder the executor refuses + * leaves its session as it found it — the session's checkout in place, holding + * the earlier turn's work, and the session held for the next turn to continue. + * + * Such a turn moves the session to a fresh checkout, removing the one it had, + * so everything that can refuse the folder is asked before that removal. A + * refusal after it would leave a session naming a checkout that no longer + * exists. The refusal here is the folder's alternates chain running deeper + * than a clone of it reads. + */ +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { SandboxedRunInput, SandboxedRunResult } from '../sandboxedRun.js'; + +const run = promisify(execFile); +const handed: SandboxedRunInput[] = []; + +vi.mock('../sandboxedRun.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + sandboxReadiness: () => ({ ready: true, missing: [] }), + runSandboxed: async (input: SandboxedRunInput): Promise => { + handed.push(input); + const [program, ...args] = input.argv; + const { stdout, stderr } = await run(program ?? '', args, { + cwd: input.cwd, + env: { PATH: process.env['PATH'] ?? '', ...input.env, ...input.trustedEnv }, + }); + return { + processId: 'hr_test', + exitCode: 0, + signal: null, + timedOut: false, + durationMs: 0, + stdout, + stderr, + truncated: false, + }; + }, + }; +}); + +const { createHostHarnessHandler } = await import('../handlers/harnessHandlers.js'); +const { allSessions, discardScratch, forgetSession } = await import('../harnessSessions.js'); +const { removeWorktree } = await import('../worktree.js'); + +let base: string; +let project: string; +let policyPath: string; + +async function git(cwd: string, ...args: string[]): Promise { + return (await run('git', ['-C', cwd, ...args])).stdout; +} + +beforeEach(async () => { + handed.length = 0; + base = await realpath(await mkdtemp(join(tmpdir(), 'aflow-session-refusal-'))); + project = join(base, 'project'); + policyPath = join(base, 'host-policy.json'); + await run('git', ['init', '-q', '-b', 'main', project]); + await git(project, 'config', 'user.email', 't@e.com'); + await git(project, 'config', 'user.name', 'T'); + await writeFile(join(project, 'a.txt'), 'one\n'); + await git(project, 'add', '-A'); + await git(project, 'commit', '-q', '-m', 'initial'); + await writeFile( + policyPath, + JSON.stringify({ + version: 1, + bindings: [ + { + id: 'hb', + root: project, + mode: 'readwrite', + allowsExecution: true, + singleFile: false, + spaceId: 'space-test', + }, + ], + harnesses: [ + { + id: 'converses', + executable: '/bin/sh', + args: ['-c', "printf 'turn\\n' >> work.txt"], + sessionArgs: ['{session}'], + resumeArgs: ['{session}'], + }, + ], + }), + ); +}); + +afterEach(async () => { + for (const session of allSessions()) { + forgetSession(session.id); + await removeWorktree(session.worktreePath).catch(() => {}); + await discardScratch(session); + } + await rm(base, { recursive: true, force: true }); +}); + +interface Outcome { + readonly status: string; + readonly message: string; + readonly output: Record; +} + +async function commission(input: Record): Promise { + const written: Record = {}; + const result = await createHostHarnessHandler(policyPath).execute({ + operationId: 'host.harness.run', + spaceId: 'space-test', + runId: 'run-session-refusal', + job: { inputRef: 'inline:x' }, + signal: new AbortController().signal, + log: { error: () => undefined, warn: () => undefined, info: () => undefined }, + readPayload: () => + Promise.resolve({ + bindingId: 'hb', + harness: 'converses', + task: 'Fix it.', + timeoutMs: 60_000, + ...input, + }), + emitLiveDelta: () => Promise.resolve(), + writePayload: (kind: string, data: unknown) => { + written[kind] = data; + return Promise.resolve(`inline:${kind}`); + }, + } as never); + const error = written['error'] as { message?: string } | undefined; + return { + status: result.status, + message: error?.message ?? result.error?.message ?? '', + output: (written['output'] ?? {}) as Record, + }; +} + +/** + * Makes the folder's store borrow through one level of `info/alternates` more + * than a clone of it reads, every store along the chain existing and new. + * Resolves to what undoes it. + */ +async function borrowTooDeep(): Promise<() => Promise> { + const objects = join(project, '.git', 'objects'); + let borrower = objects; + for (let level = 0; level < 6; level += 1) { + const store = join(base, `store-${String(level)}`); + await mkdir(join(store, 'info'), { recursive: true }); + await writeFile(join(borrower, 'info', 'alternates'), `${store}\n`); + borrower = store; + } + return () => rm(join(objects, 'info', 'alternates'), { force: true }); +} + +describe('a turn naming a new base on a folder the alternates check refuses', () => { + it("leaves the session's checkout in place and the session held for the next turn", async () => { + const first = await commission({ base: 'main' }); + expect(first.status).toBe('SUCCEEDED'); + const sessionRef = String(first.output['sessionRef']); + const [held] = allSessions(); + expect(held?.id).toBe(sessionRef); + const checkout = held?.worktreePath ?? ''; + const checkouts = await git(project, 'worktree', 'list', '--porcelain'); + + const undo = await borrowTooDeep(); + const refused = await commission({ continueFrom: sessionRef, base: 'main' }); + expect(refused.status).toBe('FAILED'); + expect(refused.message).toContain( + 'borrows through more levels of `objects/info/alternates` than git follows', + ); + expect(handed).toHaveLength(1); + + expect(await readFile(join(checkout, 'work.txt'), 'utf8')).toBe('turn\n'); + expect(await git(project, 'worktree', 'list', '--porcelain')).toBe(checkouts); + expect(allSessions()).toEqual([ + expect.objectContaining({ id: sessionRef, worktreePath: checkout, busy: false }), + ]); + + await undo(); + const next = await commission({ continueFrom: sessionRef }); + expect(next.status).toBe('SUCCEEDED'); + expect(next.output['continued']).toBe(true); + expect(await readFile(join(checkout, 'work.txt'), 'utf8')).toBe('turn\nturn\n'); + }, 60_000); +}); diff --git a/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts b/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts index 3a349ec6..bc23d47b 100644 --- a/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts +++ b/apps/aflow-executor-host/src/__tests__/subdirectoryBoundary.test.ts @@ -204,6 +204,20 @@ describe('a repository whose store borrows from others, connected under a denied expect(await borrowedReads(chained)).toEqual([objectsOf(shared), objectsOf(top)]); }); + it('reads a quoted entry holding a character outside the BMP, written out or escaped', async () => { + // U+1D11E is two UTF-16 code units and four bytes of UTF-8; git writes it + // either as itself or as its bytes escaped in octal. + const clef = join(home, 'projects', 'clef-\u{1D11E}'); + await run('git', ['clone', '-q', '--shared', top, clef]); + for (const quoted of [ + objectsOf(clef), + objectsOf(clef).replace('\u{1D11E}', '\\360\\235\\204\\236'), + ]) { + await writeFile(join(objectsOf(chained), 'info', 'alternates'), `"${quoted}"\n`); + expect(await borrowedReads(chained)).toEqual([objectsOf(clef), objectsOf(top)]); + } + }); + it('is refused, naming the remedy, when its chain is deeper than a clone of it reads', async () => { // A clone of the folder reads five stores of alternates below the folder's // own; `shared` is one hop from `top`, and each level here adds one. diff --git a/apps/aflow-executor-host/src/objectAlternates.ts b/apps/aflow-executor-host/src/objectAlternates.ts index da4e405d..fa53eb0c 100644 --- a/apps/aflow-executor-host/src/objectAlternates.ts +++ b/apps/aflow-executor-host/src/objectAlternates.ts @@ -21,12 +21,15 @@ const C_ESCAPES: Readonly> = { function unquotedC(entry: string): string | undefined { const bytes: number[] = []; for (let i = 1; i < entry.length; i += 1) { - const char = entry.charAt(i); + // By code point: a character outside the BMP is two UTF-16 units, and + // either alone encodes as U+FFFD rather than the bytes git wrote. + const char = String.fromCodePoint(entry.codePointAt(i) ?? 0); if (char === '"') { return i === entry.length - 1 ? Buffer.from(bytes).toString('utf8') : undefined; } if (char !== '\\') { bytes.push(...Buffer.from(char, 'utf8')); + i += char.length - 1; continue; } const octal = /^[0-3][0-7]{2}/.exec(entry.slice(i + 1));