From af9ad3b46e652d175c3fc1a3dc873cdea2a25527 Mon Sep 17 00:00:00 2001 From: Roger Chappel Date: Mon, 10 Aug 2026 00:39:44 +1000 Subject: [PATCH 1/3] test: cover concurrent CLI additions --- test/cli.test.ts | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/test/cli.test.ts b/test/cli.test.ts index b81b0ae..dace9ec 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { execFileSync, spawnSync } from 'node:child_process'; +import { execFileSync, spawn, spawnSync } from 'node:child_process'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; @@ -24,6 +24,18 @@ function runResult(args: string[], cwd: string, input?: string) { return spawnSync(process.execPath, [cliPath, ...args], { cwd, encoding: 'utf8', input }); } +function runAsync(args: string[], cwd: string, input: string): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve) => { + const child = spawn(process.execPath, [cliPath, ...args], { cwd, stdio: ['pipe', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8').on('data', (chunk: string) => { stdout += chunk; }); + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { stderr += chunk; }); + child.on('close', (code) => resolve({ code, stdout, stderr })); + child.stdin.end(input); + }); +} + describe('clipcase CLI', () => { it('prints current casefile commands in help output', async () => { const cwd = await tmp(); @@ -66,6 +78,28 @@ describe('clipcase CLI', () => { assert.match(run(['export', 'duplicates'], cwd), /- Entries: 4/); }); + it('preserves every entry added by concurrent CLI processes', async () => { + const cwd = await tmp(); + const inputs = Array.from({ length: 12 }, (_, index) => `parallel capture ${String(index).padStart(2, '0')}\n`); + + run(['init'], cwd); + run(['new', 'concurrent'], cwd); + const results = await Promise.all(inputs.map((input) => runAsync(['add', 'concurrent', '--source', 'parallel'], cwd, input))); + for (const result of results) { + assert.equal(result.code, 0, result.stderr); + assert.match(result.stdout, /Added .* to concurrent/); + } + + const shown = JSON.parse(run(['show', 'concurrent'], cwd)) as { entries: Array<{ id: string; path: string }> }; + assert.equal(shown.entries.length, inputs.length); + assert.equal(new Set(shown.entries.map((entry) => entry.id)).size, inputs.length); + assert.equal(run(['search', 'parallel capture'], cwd).trim().split('\n').length, inputs.length); + const exported = run(['export', 'concurrent'], cwd); + for (const input of inputs) assert.match(exported, new RegExp(input.trim())); + const entryFiles = (await fs.readdir(path.join(cwd, '.clipcase', 'concurrent', 'entries'))).sort(); + assert.deepEqual(entryFiles, shown.entries.map((entry) => path.basename(entry.path)).sort()); + }); + it('rejects case names that resolve outside the configured store', async () => { const cwd = await tmp(); From f0960ab87ceb7c6b990220db511e24834ff16487 Mon Sep 17 00:00:00 2001 From: Roger Chappel Date: Mon, 10 Aug 2026 00:39:44 +1000 Subject: [PATCH 2/3] fix: serialize case index updates --- src/storage.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/storage.ts b/src/storage.ts index 3e76447..86ea750 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -5,14 +5,18 @@ import { findSecrets } from './secrets.js'; import type { AddEntryInput, CaseMetadata, EntryMetadata } from './types.js'; import { escapeMarkdown, formatTags, sha256, shortHash, slugify, toPosix } from './util.js'; const INDEX_FILE = 'index.json'; +const LOCK_DIR = '.index.lock'; +const LOCK_RETRY_MS = 25; +const LOCK_TIMEOUT_MS = 10_000; export async function ensureStore(storageDir: string): Promise { await fs.mkdir(storageDir, { recursive: true }); } export function caseDir(storageDir: string, caseName: string): string { const root = path.resolve(storageDir); const dir = path.resolve(root, slugify(caseName)); if (dir === root || path.dirname(dir) !== root) throw new ClipcaseError(`Invalid case name: ${caseName}`); return dir; } async function indexPath(storageDir: string, caseName: string): Promise { return path.join(caseDir(storageDir, caseName), INDEX_FILE); } export async function loadCase(storageDir: string, caseName: string): Promise { try { return JSON.parse(await fs.readFile(await indexPath(storageDir, caseName), 'utf8')) as CaseMetadata; } catch { throw new ClipcaseError(`Case not found: ${caseName}`, 2); } } -async function saveCase(storageDir: string, meta: CaseMetadata): Promise { meta.entries.sort((a, b) => a.id.localeCompare(b.id)); await fs.writeFile(await indexPath(storageDir, meta.name), JSON.stringify(meta, null, 2) + '\n'); } +async function saveCase(storageDir: string, meta: CaseMetadata): Promise { meta.entries.sort((a, b) => a.id.localeCompare(b.id)); const target = await indexPath(storageDir, meta.name); const temporary = `${target}.${process.pid}.${Date.now()}.tmp`; try { await fs.writeFile(temporary, JSON.stringify(meta, null, 2) + '\n', { flag: 'wx' }); await fs.rename(temporary, target); } finally { await fs.rm(temporary, { force: true }); } } +async function withCaseLock(storageDir: string, caseName: string, operation: () => Promise): Promise { const lock = path.join(caseDir(storageDir, caseName), LOCK_DIR); const deadline = Date.now() + LOCK_TIMEOUT_MS; while (true) { try { await fs.mkdir(lock); break; } catch (error) { if (!isAlreadyExists(error)) throw error; if (Date.now() >= deadline) throw new ClipcaseError(`Timed out waiting to update case: ${slugify(caseName)}`, 4); await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS)); } } try { return await operation(); } finally { await fs.rmdir(lock).catch(() => undefined); } } export async function createCase(storageDir: string, name: string, title?: string, now = new Date()): Promise { await ensureStore(storageDir); const slug = slugify(name); const dir = caseDir(storageDir, slug); await fs.mkdir(path.join(dir, 'entries'), { recursive: true }); const createdAt = now.toISOString(); const meta: CaseMetadata = { name: slug, title: title ?? slug, createdAt, updatedAt: createdAt, entries: [] }; await fs.writeFile(path.join(dir, INDEX_FILE), JSON.stringify(meta, null, 2) + '\n', { flag: 'wx' }); return meta; } export async function listCases(storageDir: string): Promise { await ensureStore(storageDir); const names = await fs.readdir(storageDir).catch(() => [] as string[]); const cases: CaseMetadata[] = []; for (const name of names.sort()) { try { cases.push(await loadCase(storageDir, name)); } catch {} } return cases.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt) || a.name.localeCompare(b.name)); } -export async function addEntry(storageDir: string, input: AddEntryInput): Promise { const findings = findSecrets(input.text); if (findings.length && !input.allowSecret) throw new ClipcaseError(`Refusing to save likely secret(s): ${findings.map((f) => f.label).join(', ')}. Re-run with --allow-secret if this is intentional.`, 3); const meta = await loadCase(storageDir, input.caseName); const now = input.now ?? new Date(); const createdAt = now.toISOString(); const hash = sha256(input.text); const stamp = createdAt.replace(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z'); const baseId = `${stamp}-${shortHash(input.text)}`; let entry: EntryMetadata | undefined; for (let collision = 0; !entry; collision += 1) { const id = collision === 0 ? baseId : `${baseId}-${String(collision).padStart(6, '0')}`; const relPath = toPosix(path.join('entries', `${id}.md`)); const candidate: EntryMetadata = { id, caseName: meta.name, createdAt, source: input.source ?? 'stdin', tags: [...new Set(input.tags ?? [])].sort(), hash, bytes: Buffer.byteLength(input.text), path: relPath }; try { await fs.writeFile(path.join(caseDir(storageDir, meta.name), relPath), renderEntry(candidate, input.text), { flag: 'wx' }); entry = candidate; } catch (error) { if (!isAlreadyExists(error)) throw error; } } meta.entries.push(entry); meta.updatedAt = createdAt; await saveCase(storageDir, meta); return entry; } +export async function addEntry(storageDir: string, input: AddEntryInput): Promise { const findings = findSecrets(input.text); if (findings.length && !input.allowSecret) throw new ClipcaseError(`Refusing to save likely secret(s): ${findings.map((f) => f.label).join(', ')}. Re-run with --allow-secret if this is intentional.`, 3); return withCaseLock(storageDir, input.caseName, async () => { const meta = await loadCase(storageDir, input.caseName); const now = input.now ?? new Date(); const createdAt = now.toISOString(); const hash = sha256(input.text); const stamp = createdAt.replace(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z'); const baseId = `${stamp}-${shortHash(input.text)}`; let entry: EntryMetadata | undefined; for (let collision = 0; !entry; collision += 1) { const id = collision === 0 ? baseId : `${baseId}-${String(collision).padStart(6, '0')}`; const relPath = toPosix(path.join('entries', `${id}.md`)); const candidate: EntryMetadata = { id, caseName: meta.name, createdAt, source: input.source ?? 'stdin', tags: [...new Set(input.tags ?? [])].sort(), hash, bytes: Buffer.byteLength(input.text), path: relPath }; try { await fs.writeFile(path.join(caseDir(storageDir, meta.name), relPath), renderEntry(candidate, input.text), { flag: 'wx' }); entry = candidate; } catch (error) { if (!isAlreadyExists(error)) throw error; } } try { meta.entries.push(entry); meta.updatedAt = createdAt; await saveCase(storageDir, meta); return entry; } catch (error) { await fs.rm(path.join(caseDir(storageDir, meta.name), entry.path), { force: true }); throw error; } }); } function isAlreadyExists(error: unknown): boolean { return error instanceof Error && 'code' in error && error.code === 'EEXIST'; } export function renderEntry(entry: EntryMetadata, text: string): string { return `---\nid: ${entry.id}\ncreatedAt: ${entry.createdAt}\nsource: ${entry.source}\ntags: [${entry.tags.join(', ')}]\nhash: ${entry.hash}\nbytes: ${entry.bytes}\n---\n\n# Entry ${entry.id}\n\n- Source: ${escapeMarkdown(entry.source)}\n- Tags: ${formatTags(entry.tags)}\n- SHA-256: \`${entry.hash}\`\n\n\`\`\`text\n${text.replace(/\n?$/, '\n')}\`\`\`\n`; } export async function readEntryText(storageDir: string, meta: CaseMetadata, entry: EntryMetadata): Promise { const md = await fs.readFile(path.join(caseDir(storageDir, meta.name), entry.path), 'utf8'); const match = md.match(/```text\n([\s\S]*?)```\n?$/); return match ? match[1].replace(/\n$/, '') : md; } From 1ed7809fb81ec7d90878a899d6960446a6da4a0b Mon Sep 17 00:00:00 2001 From: Roger Chappel Date: Mon, 10 Aug 2026 00:39:44 +1000 Subject: [PATCH 3/3] docs: explain concurrent writer behavior --- docs/STORAGE.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/STORAGE.md b/docs/STORAGE.md index 0fdf216..64908da 100644 --- a/docs/STORAGE.md +++ b/docs/STORAGE.md @@ -11,3 +11,11 @@ Entry IDs are timestamp plus content hash prefix: `YYYYMMDDTHHMMSSZ-<12 hex>`. If that ID already exists, ClipCase appends a zero-padded collision counter, starting at `-000001`. This preserves both identical captures made within the same second while keeping filenames and index ordering deterministic. + +## Concurrent writers + +Adds to the same case are serialized with a case-local `.index.lock` directory. +A writer retries every 25 ms for up to 10 seconds, then exits with an error rather +than overwriting another writer's metadata. `index.json` is written to a temporary +file and atomically renamed, so readers see either the previous complete index or +the new complete index, never a partially written JSON document.