From 36d74e6e3e3616927f4dccbe375d81b6865de668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=BE=E5=B1=B1=E8=B2=B4=E8=87=B3?= Date: Sun, 31 May 2026 22:10:44 +0900 Subject: [PATCH] feat(import): make claude-code import idempotent via source.external_id Re-running `basou import claude-code` no longer duplicates sessions: each imported session records the originating Claude session id in `session.source.external_id`, and the importer skips any transcript whose id is already present. - Add optional `source.external_id` to the session and import schemas (additive; absent for live runs). - The transform stores the Claude session id (filename / sessionId) there and in the session label. - Import builds the set of existing external_ids and skips matches. Sessions imported before external_id existed are recognized via their `claude-code import ` label, so prior imports are not duplicated. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 6 +- packages/cli/src/commands/import.test.ts | 42 ++++++++++ packages/cli/src/commands/import.ts | 82 ++++++++++++++++--- .../claude-code/transcript-importer.test.ts | 22 +++++ .../claude-code/transcript-importer.ts | 15 +++- .../core/src/schemas/session-import.schema.ts | 3 + packages/core/src/schemas/session.schema.ts | 4 + 7 files changed, 158 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eaefd164..e0940165 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,14 @@ All notable changes to **basou** are recorded here. The project follows transcript for a project, `--session ` imports one, and `--dry-run` previews without writing. Transcripts with no observable command / file action are skipped. Imports reuse the existing `session import` pipeline, - so path sanitization and id minting are unchanged. + so path sanitization and id minting are unchanged. Re-running an import is + idempotent: a transcript already imported is skipped rather than duplicated. - `claude-code-import` session `source.kind` (additive enum value) to distinguish transcript-derived sessions from live `claude-code-adapter` runs and Basou-format `import`s. +- `session.source.external_id` (additive, optional) — records the originating + session id in the source tool's namespace (e.g. the Claude Code session + UUID), the key that makes re-imports idempotent. ## 0.3.1 — 2026-05-21 diff --git a/packages/cli/src/commands/import.test.ts b/packages/cli/src/commands/import.test.ts index 92957318..996e7f51 100644 --- a/packages/cli/src/commands/import.test.ts +++ b/packages/cli/src/commands/import.test.ts @@ -10,6 +10,7 @@ import { readYamlFile, SessionSchema, writeManifest, + writeYamlFile, } from "@basou/core"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { doRunImportClaudeCode } from "./import.js"; @@ -142,6 +143,47 @@ describe("basou import claude-code", () => { expect(types[types.length - 1]).toBe("session_ended"); }); + it("is idempotent: re-import skips an already-imported session", async () => { + const repo = await setupInitedRepo(); + await writeTranscript(repo, "sess-1", actionTranscript(repo)); + + await doRunImportClaudeCode({ all: true }, { cwd: repo, claudeProjectsDir: getProjectsRoot() }); + expect(await listSessionDirs(repo)).toHaveLength(1); + + // A second run over the same transcript must not create a duplicate. + await doRunImportClaudeCode({ all: true }, { cwd: repo, claudeProjectsDir: getProjectsRoot() }); + expect(await listSessionDirs(repo)).toHaveLength(1); + }); + + it("dedups a pre-external_id import via its label (migration safety)", async () => { + const repo = await setupInitedRepo(); + // Simulate a session imported before source.external_id existed: it carries + // only the `claude-code import ` label. The dedup must still recognize it. + const sid = "ses_01HXABCDEF1234567890ABCDEF"; + const dir = join(basouPaths(repo).sessions, sid); + await mkdir(dir, { recursive: true }); + await writeYamlFile(join(dir, "session.yaml"), { + schema_version: "0.1.0", + session: { + id: sid, + workspace_id: FIXED_WS_ID, + source: { kind: "claude-code-import", version: "0.1.0" }, + started_at: "2026-05-10T00:00:00.000Z", + status: "imported", + working_directory: ".", + invocation: { command: "claude", args: [], exit_code: null }, + related_files: [], + events_log: "events.jsonl", + label: "claude-code import sess-1", + }, + }); + await writeTranscript(repo, "sess-1", actionTranscript(repo)); + + await doRunImportClaudeCode({ all: true }, { cwd: repo, claudeProjectsDir: getProjectsRoot() }); + // sess-1 is recognized via the label and not imported again. + expect(await listSessionDirs(repo)).toHaveLength(1); + }); + it("--session imports a single transcript by id", async () => { const repo = await setupInitedRepo(); await writeTranscript(repo, "sess-1", actionTranscript(repo)); diff --git a/packages/cli/src/commands/import.ts b/packages/cli/src/commands/import.ts index 130066ca..e9b16ecb 100644 --- a/packages/cli/src/commands/import.ts +++ b/packages/cli/src/commands/import.ts @@ -1,16 +1,20 @@ import { readdir, readFile } from "node:fs/promises"; import { homedir } from "node:os"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { assertBasouRootSafe, + type BasouPaths, basouPaths, type ClaudeTranscriptRecord, claudeTranscriptToImportPayload, + enumerateSessionDirs, findErrorCode, type ImportSessionResult, importSessionFromJson, readManifest, + readSessionYaml, resolveRepositoryRoot, + type Session, SessionImportPayloadSchema, } from "@basou/core"; import type { Command } from "commander"; @@ -100,17 +104,29 @@ export async function doRunImportClaudeCode( const transcriptDir = join(projectsRoot, encodeProjectDir(projectPath)); const files = await selectTranscriptFiles(transcriptDir, options); + const alreadyImported = await loadExistingExternalIds(paths); const results: ImportSessionResult[] = []; - let skipped = 0; + let skippedNoAction = 0; + let skippedExisting = 0; let sanitizedPaths = 0; for (const file of files) { + // The transcript filename is the Claude session id. Dedup on it before + // any read so re-imports are idempotent — the same session is never + // written twice under fresh Basou ids. + const externalId = basename(file, ".jsonl"); + if (alreadyImported.has(externalId)) { + skippedExisting++; + continue; + } + const records = await readTranscript(file); const payload = claudeTranscriptToImportPayload(records, { workspaceId: manifest.workspace.id, + externalId, }); if (payload === null) { - skipped++; + skippedNoAction++; continue; } @@ -126,6 +142,8 @@ export async function doRunImportClaudeCode( dryRun: options.dryRun === true, }); results.push(result); + // Guard against a same-run duplicate (two files for one session id). + alreadyImported.add(externalId); sanitizedPaths += result.pathSanitizeReport.relatedFiles + (result.pathSanitizeReport.workingDirectoryRewritten ? 1 : 0); @@ -135,7 +153,7 @@ export async function doRunImportClaudeCode( console.error(`Imported sessions: ${sanitizedPaths} path(s) sanitized`); } - printImportResult(options, results, skipped); + printImportResult(options, results, { skippedNoAction, skippedExisting }); } /** @@ -149,6 +167,40 @@ function encodeProjectDir(projectPath: string): string { return projectPath.replaceAll("/", "-"); } +/** + * Set of source external_ids already present in the workspace, so a re-import + * skips sessions it has already seen. Recognises both the structured + * `source.external_id` (current imports) and the `claude-code import ` + * label form (sessions imported before external_id existed), so existing + * dogfood imports are not duplicated. Unreadable sessions are skipped. + */ +async function loadExistingExternalIds(paths: BasouPaths): Promise> { + const ids = new Set(); + let sessionIds: string[]; + try { + sessionIds = await enumerateSessionDirs(paths); + } catch { + return ids; + } + for (const sessionId of sessionIds) { + let session: Session; + try { + session = await readSessionYaml(paths, sessionId); + } catch { + continue; + } + const ext = session.session.source.external_id; + if (typeof ext === "string" && ext.length > 0) { + ids.add(ext); + continue; + } + const label = session.session.label; + const match = typeof label === "string" ? label.match(/^claude-code import (\S+)$/) : null; + if (match?.[1] !== undefined) ids.add(match[1]); + } + return ids; +} + async function selectTranscriptFiles( transcriptDir: string, options: ImportClaudeCodeOptions, @@ -205,10 +257,11 @@ async function readTranscript(file: string): Promise { function printImportResult( options: ImportClaudeCodeOptions, results: ImportSessionResult[], - skipped: number, + skips: { skippedNoAction: number; skippedExisting: number }, ): void { const isDry = options.dryRun === true; const eventTotal = results.reduce((sum, r) => sum + r.eventCount, 0); + const { skippedNoAction, skippedExisting } = skips; if (options.json === true) { console.log( @@ -220,7 +273,8 @@ function printImportResult( source: { kind: r.finalSourceKind, version: "0.1.0" }, })), imported_count: results.length, - skipped_count: skipped, + skipped_no_action: skippedNoAction, + skipped_already_imported: skippedExisting, event_total: eventTotal, dry_run: isDry, }), @@ -228,28 +282,30 @@ function printImportResult( return; } + const skipParts: string[] = []; + if (skippedNoAction > 0) skipParts.push(`${skippedNoAction} with no actions`); + if (skippedExisting > 0) skipParts.push(`${skippedExisting} already imported`); + const skipSuffix = skipParts.length > 0 ? `; skipped ${skipParts.join(", ")}` : ""; + if (results.length === 0) { console.log( - skipped > 0 - ? `No sessions imported (${skipped} transcript(s) had no importable actions)` + skipParts.length > 0 + ? `No new sessions imported (skipped ${skipParts.join(", ")})` : "No transcripts found to import", ); return; } - const skippedSuffix = skipped > 0 ? `; skipped ${skipped} with no actions` : ""; if (isDry) { console.log( - `Dry run: would import ${results.length} session(s) (${eventTotal} events)${skippedSuffix}`, + `Dry run: would import ${results.length} session(s) (${eventTotal} events)${skipSuffix}`, ); return; } const single = results.length === 1 && results[0] !== undefined ? ` (${shortId(results[0].sessionId)})` : ""; - console.log( - `Imported ${results.length} session(s)${single} (${eventTotal} events)${skippedSuffix}`, - ); + console.log(`Imported ${results.length} session(s)${single} (${eventTotal} events)${skipSuffix}`); } function shortId(id: string): string { diff --git a/packages/core/src/adapters/claude-code/transcript-importer.test.ts b/packages/core/src/adapters/claude-code/transcript-importer.test.ts index 97cdf62a..8efc136b 100644 --- a/packages/core/src/adapters/claude-code/transcript-importer.test.ts +++ b/packages/core/src/adapters/claude-code/transcript-importer.test.ts @@ -69,6 +69,8 @@ describe("claudeTranscriptToImportPayload", () => { expect(payload.session.working_directory).toBe(CWD); expect(payload.session.workspace_id).toBe(WS_ID); expect(payload.session.related_files).toEqual([`${CWD}/a.ts`, `${CWD}/b.ts`]); + // The transcript's own sessionId becomes the source external_id (dedup key). + expect(payload.session.source.external_id).toBe("abc-123"); const types = payload.events.map((e) => e.type); expect(types).toEqual([ @@ -158,6 +160,26 @@ describe("claudeTranscriptToImportPayload", () => { } }); + it("prefers the provided externalId over the records' sessionId", () => { + const records: ClaudeTranscriptRecord[] = [ + { + type: "assistant", + timestamp: "2026-05-10T00:00:01.000Z", + cwd: CWD, + sessionId: "from-records", + message: { content: [{ type: "tool_use", name: "Bash", input: { command: "ls" } }] }, + }, + ]; + const payload = claudeTranscriptToImportPayload(records, { + workspaceId: WS_ID, + externalId: "from-option", + }); + expect(payload).not.toBeNull(); + if (payload === null) return; + expect(payload.session.source.external_id).toBe("from-option"); + expect(payload.session.label).toContain("from-option"); + }); + it("returns null when no observable command / file action exists", () => { const records: ClaudeTranscriptRecord[] = [ { diff --git a/packages/core/src/adapters/claude-code/transcript-importer.ts b/packages/core/src/adapters/claude-code/transcript-importer.ts index 80245a25..80c51480 100644 --- a/packages/core/src/adapters/claude-code/transcript-importer.ts +++ b/packages/core/src/adapters/claude-code/transcript-importer.ts @@ -23,6 +23,12 @@ export type ClaudeTranscriptRecord = Record; export type ClaudeTranscriptToPayloadOptions = { /** Workspace id of the target Basou workspace (from its manifest). */ workspaceId: Manifest["workspace"]["id"]; + /** + * Claude Code session id (transcript filename / `sessionId`). Stored as + * `session.source.external_id` so re-imports can be deduplicated. Falls + * back to the `sessionId` read from the records when omitted. + */ + externalId?: string; }; /** @@ -117,15 +123,20 @@ export function claudeTranscriptToImportPayload( sessionEndedEvent(maxTs, placeholderSessionId), ]; + const externalId = options.externalId ?? claudeSessionId; const label = - claudeSessionId !== undefined ? `claude-code import ${claudeSessionId}` : "claude-code import"; + externalId !== undefined ? `claude-code import ${externalId}` : "claude-code import"; const payload: SessionImportPayload = { schema_version: "0.1.0", session: { label, workspace_id: options.workspaceId, - source: { kind: CLAUDE_IMPORT_SOURCE, version: "0.1.0" }, + source: { + kind: CLAUDE_IMPORT_SOURCE, + version: "0.1.0", + ...(externalId !== undefined ? { external_id: externalId } : {}), + }, started_at: minTs, ended_at: maxTs, // Validated against the canonical enum here; importSessionFromJson diff --git a/packages/core/src/schemas/session-import.schema.ts b/packages/core/src/schemas/session-import.schema.ts index 33e93d14..18e56037 100644 --- a/packages/core/src/schemas/session-import.schema.ts +++ b/packages/core/src/schemas/session-import.schema.ts @@ -40,6 +40,9 @@ export const SessionInnerImportSchema = z source: z.object({ kind: SessionSourceKindSchema, version: z.literal("0.1.0"), + // Source-tool-native id (e.g. Claude Code session UUID), retained so + // re-imports of the same source can be deduplicated. + external_id: z.string().optional(), }), started_at: IsoTimestampSchema, ended_at: IsoTimestampSchema.optional(), diff --git a/packages/core/src/schemas/session.schema.ts b/packages/core/src/schemas/session.schema.ts index 94310565..9b43182c 100644 --- a/packages/core/src/schemas/session.schema.ts +++ b/packages/core/src/schemas/session.schema.ts @@ -43,6 +43,10 @@ export type SessionSourceKind = z.infer; const SessionSourceSchema = z.object({ kind: SessionSourceKindSchema, version: z.literal("0.1.0"), + // Optional id of the originating session in the SOURCE tool's own + // namespace (e.g. the Claude Code session UUID for a `claude-code-import`). + // Lets re-imports of the same source be deduplicated; absent for live runs. + external_id: z.string().optional(), }); const InvocationSchema = z.object({