From 6135dc9e959737aa131c497b7e862b3ef2d7cc09 Mon Sep 17 00:00:00 2001 From: YingqiDuan <141370165+YingqiDuan@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:26:10 -0700 Subject: [PATCH 1/2] feat(sleep): add OpenCode transcript source --- skillopt_sleep/__main__.py | 13 +- skillopt_sleep/config.py | 15 +- skillopt_sleep/harvest_opencode.py | 344 +++++++++ skillopt_sleep/harvest_sources.py | 9 + tests/test_harvest_opencode.py | 1048 +++++++++++++++++++++++++++ tests/test_harvest_opencode_live.py | 80 ++ 6 files changed, 1504 insertions(+), 5 deletions(-) create mode 100644 skillopt_sleep/harvest_opencode.py create mode 100644 tests/test_harvest_opencode.py create mode 100644 tests/test_harvest_opencode_live.py diff --git a/skillopt_sleep/__main__.py b/skillopt_sleep/__main__.py index 044bcd44..4dde10df 100644 --- a/skillopt_sleep/__main__.py +++ b/skillopt_sleep/__main__.py @@ -14,9 +14,10 @@ --target-skill-path PATH explicit live SKILL.md to stage/adopt --tasks-file PATH reviewed TaskRecord JSON file to replay instead of harvesting --backend mock|claude|codex|copilot|cursor|pi|opencode|handoff|azure_openai - --source claude|codex|copilot|copilot_cli|cursor|pi|auto + --source claude|codex|copilot|copilot_cli|cursor|pi|opencode|auto --vscode-workspace-storage PATH --copilot-cli-session-store PATH + --opencode-db PATH --model NAME --lookback-hours N --auto-adopt @@ -85,12 +86,14 @@ def _add_common(p: argparse.ArgumentParser) -> None: p.add_argument("--cursor-home", default="", help="override ~/.cursor for Cursor session harvest") p.add_argument("--pi-home", default="", help="override ~/.pi for Pi session harvest") p.add_argument("--source", default="", - choices=["", "claude", "codex", "copilot", "copilot_cli", "cursor", "pi", "auto"], + choices=["", "claude", "codex", "copilot", "copilot_cli", "cursor", "pi", "opencode", "auto"], help="session transcript source") p.add_argument("--vscode-workspace-storage", default="", help="override VS Code User/workspaceStorage root for copilot source") p.add_argument("--copilot-cli-session-store", default="", help="override ~/.copilot/session-store.db for copilot_cli source") + p.add_argument("--opencode-db", default="", + help="override the local OpenCode transcript database") p.add_argument("--lookback-hours", type=int, default=None, help="harvest window in hours; 0 = scan full history") p.add_argument("--edit-budget", type=int, default=0) @@ -147,6 +150,12 @@ def _cfg_from_args(args, task_meta: Dict[str, Any] | None = None) -> Any: overrides["copilot_cli_session_store"] = os.path.abspath( os.path.expanduser(args.copilot_cli_session_store) ) + if getattr(args, "opencode_db", ""): + overrides["opencode_db"] = ( + ":memory:" + if args.opencode_db == ":memory:" + else os.path.abspath(os.path.expanduser(args.opencode_db)) + ) lh = getattr(args, "lookback_hours", None) if lh is not None: # --lookback-hours was explicitly passed (0 = full history) overrides["lookback_hours"] = lh diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py index f9df3813..5de768e3 100644 --- a/skillopt_sleep/config.py +++ b/skillopt_sleep/config.py @@ -31,9 +31,9 @@ "cursor_home": CURSOR_HOME, "vscode_workspace_storage": "", # "" => auto-detect platform defaults "copilot_cli_session_store": "", # "" => ~/.copilot/session-store.db - # Explicit sources also include copilot, copilot_cli, cursor, and pi. - # ``auto`` keeps - # the established Codex-then-Claude precedence for backward compatibility. + "opencode_db": "", # "" => OPENCODE_DB or the OpenCode XDG data path + # Explicit sources also include copilot, copilot_cli, cursor, pi, and opencode. + # ``auto`` keeps the established Codex-then-Claude precedence. "transcript_source": "claude", "projects": "invoked", # "invoked" | "all" | [list of abs paths] "invoked_project": "", # filled at runtime (cwd) when projects == "invoked" @@ -149,6 +149,15 @@ def copilot_cli_session_store(self) -> str: return "" return os.path.abspath(os.path.expanduser(str(value))) + @property + def opencode_db_path(self) -> str: + value = self.data.get("opencode_db", "") or "" + if not value: + return "" + if str(value) == ":memory:": + return ":memory:" + return os.path.abspath(os.path.expanduser(str(value))) + @property def vscode_workspace_storage(self) -> str: value = self.data.get("vscode_workspace_storage", "") or "" diff --git a/skillopt_sleep/harvest_opencode.py b/skillopt_sleep/harvest_opencode.py new file mode 100644 index 00000000..e2919811 --- /dev/null +++ b/skillopt_sleep/harvest_opencode.py @@ -0,0 +1,344 @@ +"""Read OpenCode session transcripts from its local SQLite database. + +OpenCode normally stores sessions in ``opencode.db`` below its XDG data +directory. +This harvester opens that database read-only and keeps only visible user and +assistant text plus short tool names. Reasoning, tool arguments and results, +file payloads, patches, and provider metadata are never copied into digests. +""" + +from __future__ import annotations + +import json +import os +import re +import sqlite3 +from datetime import datetime, timezone +from typing import Any, Iterable, List, Optional +from urllib.request import pathname2url + +from skillopt_sleep.harvest import ( + _detect_feedback, + _is_agent_session, + _is_headless_replay, + _is_meta_prompt, + _project_matches, +) +from skillopt_sleep.staging import redact_secrets +from skillopt_sleep.types import SessionDigest + +OPENCODE_REPLAY_AGENT_RE = re.compile(r"skillopt-sleep-[0-9a-f]{32}\Z") + +_REQUIRED_COLUMNS = { + "session": { + "id", + "parent_id", + "directory", + "agent", + "time_created", + "time_updated", + }, + "message": {"id", "session_id", "time_created", "data"}, + "part": {"id", "message_id", "session_id", "time_created", "data"}, +} + + +def default_opencode_db() -> str: + """Return the OpenCode database selected by its environment variables.""" + data_home = os.environ.get("XDG_DATA_HOME", "") + if data_home: + data_dir = os.path.abspath(os.path.expanduser(data_home)) + else: + data_dir = os.path.join(os.path.expanduser("~"), ".local", "share") + opencode_data = os.path.join(data_dir, "opencode") + + configured = os.environ.get("OPENCODE_DB", "") + if configured == ":memory:": + return "" + if configured: + if os.path.isabs(configured): + return os.path.abspath(configured) + return os.path.abspath(os.path.join(opencode_data, configured)) + return os.path.abspath(os.path.join(opencode_data, "opencode.db")) + + +def _ro_uri(path: str) -> str: + return "file:" + pathname2url(os.path.abspath(path)) + "?mode=ro" + + +def _open_database(path: str) -> sqlite3.Connection: + connection = sqlite3.connect( + _ro_uri(path), + uri=True, + isolation_level=None, + timeout=5, + ) + try: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA query_only = ON") + connection.execute("PRAGMA busy_timeout = 5000") + return connection + except sqlite3.Error: + connection.close() + raise + + +def _table_columns(connection: sqlite3.Connection, table: str) -> set[str]: + return {row[1] for row in connection.execute(f'PRAGMA table_info("{table}")')} + + +def _has_supported_schema(connection: sqlite3.Connection) -> bool: + table_names = {row[0] for row in connection.execute("SELECT name FROM sqlite_schema WHERE type = 'table'")} + for table, required in _REQUIRED_COLUMNS.items(): + if table not in table_names: + return False + columns = _table_columns(connection, table) + if not required.issubset(columns): + return False + return True + + +def _load_object(raw: Any) -> Optional[dict[str, Any]]: + if not isinstance(raw, str): + return None + try: + value = json.loads(raw) + except (TypeError, ValueError): + return None + return value if isinstance(value, dict) else None + + +def _sanitize_text(parts: Iterable[str]) -> str: + return str(redact_secrets("\n".join(parts))).replace("\x00", "").strip() + + +def _sanitize_tool_name(value: Any) -> str: + if not isinstance(value, str): + return "" + return re.sub(r"[^A-Za-z0-9_.:-]+", "_", value.strip())[:80] + + +def _tool_name(part: dict[str, Any]) -> str: + if part.get("type") == "tool": + return _sanitize_tool_name(part.get("tool")) + # Older OpenCode exports used this shape. It is safe to accept because only + # the short name is read; arguments and results remain ignored. + if part.get("type") == "tool-invocation": + invocation = part.get("toolInvocation") + if isinstance(invocation, dict): + return _sanitize_tool_name(invocation.get("toolName")) + return "" + + +def _millis_to_iso(value: Any) -> str: + if type(value) not in {int, float} or value <= 0: + return "" + try: + return datetime.fromtimestamp(value / 1000, tz=timezone.utc).isoformat().replace("+00:00", "Z") + except (OSError, OverflowError, ValueError): + return "" + + +def _iso_to_millis(value: Optional[str]) -> Optional[int]: + if not value: + return None + try: + normalized = value[:-1] + "+00:00" if value.endswith("Z") else value + parsed = datetime.fromisoformat(normalized) + # Sleep checkpoints without an offset are stored in local time, so let + # timestamp() interpret them in the host timezone. + return int(parsed.timestamp() * 1000) + except (OSError, TypeError, ValueError): + return None + + +def _dedup(values: Iterable[str]) -> List[str]: + return list(dict.fromkeys(value for value in values if value)) + + +def _is_skillopt_session(row: sqlite3.Row) -> bool: + agent = row["agent"] + return isinstance(agent, str) and OPENCODE_REPLAY_AGENT_RE.fullmatch(agent) is not None + + +def _is_harvestable_session( + session: sqlite3.Row, + scope: Any, + invoked_project: str, +) -> bool: + session_id = session["id"] + directory = session["directory"] + created = session["time_created"] + updated = session["time_updated"] + return ( + session["parent_id"] is None + and isinstance(session_id, str) + and bool(session_id) + and isinstance(directory, str) + and os.path.isabs(directory) + and type(created) in {int, float} + and created > 0 + and type(updated) in {int, float} + and updated > 0 + and _project_matches(directory, scope, invoked_project) + and not _is_skillopt_session(session) + ) + + +def _digest_session( + session: sqlite3.Row, + messages: list[sqlite3.Row], + parts_by_message: dict[str, list[sqlite3.Row]], +) -> Optional[SessionDigest]: + prompts: List[str] = [] + finals: List[str] = [] + tools: List[str] = [] + feedback: List[str] = [] + n_user = 0 + n_assistant = 0 + + for message_row in messages: + message = _load_object(message_row["data"]) + if message is None: + return None + role = message.get("role") + if role not in {"user", "assistant"}: + continue + + texts: List[str] = [] + message_tools: List[str] = [] + for part_row in parts_by_message.get(str(message_row["id"]), []): + part = _load_object(part_row["data"]) + if part is None: + return None + if ( + part.get("type") == "text" + and part.get("synthetic") is not True + and part.get("ignored") is not True + and isinstance(part.get("text"), str) + ): + texts.append(part["text"]) + if role == "assistant": + name = _tool_name(part) + if name: + message_tools.append(name) + + text = _sanitize_text(texts) + if role == "user": + if text and not _is_meta_prompt(text): + n_user += 1 + feedback.extend(_detect_feedback(text)) + prompts.append(text) + else: + n_assistant += 1 + tools.extend(message_tools) + if message.get("error"): + feedback.append("neg:opencode_message_error") + if text: + finals.append(text) + if len(finals) > 5: + finals.pop(0) + + if not prompts: + return None + + metadata = _load_object(session["metadata"]) or {} + branch = metadata.get("gitBranch") + digest = SessionDigest( + session_id=str(session["id"]), + project=str(session["directory"]), + git_branch=branch if isinstance(branch, str) else "", + started_at=_millis_to_iso(session["time_created"]), + ended_at=_millis_to_iso(session["time_updated"]), + user_prompts=prompts, + assistant_finals=finals, + tools_used=_dedup(tools), + files_touched=[], + feedback_signals=_dedup(feedback), + n_user_turns=n_user, + n_assistant_turns=n_assistant, + raw_path=f"opencode://{session['id']}", + ) + if _is_headless_replay(digest) or _is_agent_session(digest): + return None + return digest + + +def harvest_opencode( + db_path: str = "", + *, + scope: Any = "all", + invoked_project: str = "", + since_iso: Optional[str] = None, + limit: int = 0, +) -> List[SessionDigest]: + """Read root OpenCode sessions and return matching transcript digests.""" + if db_path == ":memory:": + return [] + path = os.path.abspath(os.path.expanduser(db_path)) if db_path else default_opencode_db() + if not path or not os.path.isfile(path): + return [] + if limit < 0: + limit = 0 + + cutoff = _iso_to_millis(since_iso) + connection: Optional[sqlite3.Connection] = None + try: + connection = _open_database(path) + connection.execute("BEGIN") + if not _has_supported_schema(connection): + connection.execute("ROLLBACK") + return [] + + metadata_column = "metadata" if "metadata" in _table_columns(connection, "session") else "NULL AS metadata" + session_where = "WHERE parent_id IS NULL" + session_params: list[Any] = [] + if cutoff is not None: + session_where += " AND time_updated > ?" + session_params.append(cutoff) + session_rows = connection.execute( + f"SELECT id, parent_id, directory, agent, {metadata_column}, " + "time_created, time_updated FROM session " + f"{session_where} ORDER BY time_updated DESC, id DESC", + session_params, + ) + + digests: List[SessionDigest] = [] + for session in session_rows: + session_id = session["id"] + if not _is_harvestable_session(session, scope, invoked_project): + continue + + messages = connection.execute( + "SELECT id, data FROM message WHERE session_id = ? ORDER BY time_created, id", + (session_id,), + ).fetchall() + parts = connection.execute( + "SELECT message_id, data FROM part WHERE session_id = ? ORDER BY message_id, id", + (session_id,), + ).fetchall() + parts_by_message: dict[str, list[sqlite3.Row]] = {} + for part in parts: + parts_by_message.setdefault(str(part["message_id"]), []).append(part) + digest = _digest_session( + session, + messages, + parts_by_message, + ) + if digest is None: + continue + digests.append(digest) + if limit and len(digests) >= limit: + break + connection.execute("COMMIT") + return digests + except (OSError, sqlite3.Error): + if connection is not None and connection.in_transaction: + try: + connection.execute("ROLLBACK") + except sqlite3.Error: + pass + return [] + finally: + if connection is not None: + connection.close() diff --git a/skillopt_sleep/harvest_sources.py b/skillopt_sleep/harvest_sources.py index ec526c2f..12506e7b 100644 --- a/skillopt_sleep/harvest_sources.py +++ b/skillopt_sleep/harvest_sources.py @@ -8,6 +8,7 @@ from skillopt_sleep.harvest_copilot import harvest_copilot from skillopt_sleep.harvest_copilot_cli import harvest_copilot_cli from skillopt_sleep.harvest_cursor import harvest_cursor +from skillopt_sleep.harvest_opencode import harvest_opencode from skillopt_sleep.harvest_pi import harvest_pi from skillopt_sleep.types import SessionDigest @@ -57,6 +58,14 @@ def harvest_for_config(cfg, *, since_iso: Optional[str] = None, limit: int = 0) since_iso=since_iso, limit=limit, ) + if source == "opencode": + return harvest_opencode( + cfg.opencode_db_path, + scope=scope, + invoked_project=invoked_project, + since_iso=since_iso, + limit=limit, + ) if source == "auto": codex_digests = harvest_codex( cfg.codex_archived_sessions_dir, diff --git a/tests/test_harvest_opencode.py b/tests/test_harvest_opencode.py new file mode 100644 index 00000000..5b66703b --- /dev/null +++ b/tests/test_harvest_opencode.py @@ -0,0 +1,1048 @@ +"""Offline coverage for OpenCode's SQLite transcript source.""" + +from __future__ import annotations + +import argparse +import json +import os +import sqlite3 +from collections.abc import Iterable +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from unittest import mock + +import pytest + +from skillopt_sleep.__main__ import _add_common, _cfg_from_args +from skillopt_sleep.config import load_config +from skillopt_sleep.harvest_opencode import ( + _open_database, + default_opencode_db, + harvest_opencode, +) +from skillopt_sleep.harvest_sources import harvest_for_config +from skillopt_sleep.types import SessionDigest + + +def _schema(*, include_metadata: bool = True) -> str: + metadata_column = ",\n metadata TEXT" if include_metadata else "" + return f""" +CREATE TABLE session ( + id TEXT PRIMARY KEY, + project_id TEXT, + parent_id TEXT, + directory TEXT, + title TEXT, + time_created INTEGER, + time_updated INTEGER, + agent TEXT{metadata_column} +); +CREATE TABLE message ( + id TEXT PRIMARY KEY, + session_id TEXT, + time_created INTEGER, + time_updated INTEGER, + data TEXT +); +CREATE TABLE part ( + id TEXT PRIMARY KEY, + message_id TEXT, + session_id TEXT, + time_created INTEGER, + time_updated INTEGER, + data TEXT +); +""" + + +_BASE_MS = 1_767_225_600_000 # 2026-01-01T00:00:00Z + + +def _new_store( + tmp_path: Path, + name: str = "opencode.db", + *, + wal: bool = False, + include_metadata: bool = True, +) -> tuple[Path, sqlite3.Connection]: + path = tmp_path / name + connection = sqlite3.connect(path) + connection.executescript(_schema(include_metadata=include_metadata)) + if wal: + assert connection.execute("PRAGMA journal_mode = WAL").fetchone()[0] == "wal" + connection.execute("PRAGMA wal_autocheckpoint = 0") + connection.execute("PRAGMA wal_checkpoint(TRUNCATE)") + return path, connection + + +def _add_session( + connection: sqlite3.Connection, + session_id: str, + project: str, + *, + created: int = _BASE_MS, + updated: int = _BASE_MS + 60_000, + parent_id: str | None = None, + title: str = "Interactive session", + agent: str = "build", + metadata: dict[str, Any] | None = None, +) -> None: + connection.execute( + "INSERT INTO session " + "(id, project_id, parent_id, directory, title, time_created, " + "time_updated, agent, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + session_id, + "project-1", + parent_id, + project, + title, + created, + updated, + agent, + json.dumps(metadata if metadata is not None else {}), + ), + ) + + +def _add_message( + connection: sqlite3.Connection, + session_id: str, + message_id: str, + role: str, + *, + at: int, + **extra: Any, +) -> None: + data = {"role": role, **extra} + connection.execute( + "INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?)", + (message_id, session_id, at, at, json.dumps(data)), + ) + + +def _add_part( + connection: sqlite3.Connection, + session_id: str, + message_id: str, + part_id: str, + data: dict[str, Any] | str, + *, + at: int, +) -> None: + raw = data if isinstance(data, str) else json.dumps(data) + connection.execute( + "INSERT INTO part (id, message_id, session_id, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?, ?)", + (part_id, message_id, session_id, at, at, raw), + ) + + +def _add_text_message( + connection: sqlite3.Connection, + session_id: str, + message_id: str, + role: str, + text: str, + *, + at: int, + **message_extra: Any, +) -> None: + _add_message( + connection, + session_id, + message_id, + role, + at=at, + **message_extra, + ) + _add_part( + connection, + session_id, + message_id, + f"{message_id}-text", + {"type": "text", "text": text}, + at=at + 1, + ) + + +def _add_basic_transcript( + connection: sqlite3.Connection, + session_id: str, + project: str, + *, + created: int = _BASE_MS, + updated: int = _BASE_MS + 60_000, + prompt: str | None = None, + answer: str = "The requested change is complete.", + parent_id: str | None = None, + title: str = "Interactive session", + agent: str = "build", +) -> None: + _add_session( + connection, + session_id, + project, + created=created, + updated=updated, + parent_id=parent_id, + title=title, + agent=agent, + ) + if prompt is None: + prompt = f"Please finish the real task recorded in session {session_id}." + user_id = f"{session_id}-user" + assistant_id = f"{session_id}-assistant" + _add_text_message( + connection, + session_id, + user_id, + "user", + prompt, + at=created + 1_000, + ) + _add_text_message( + connection, + session_id, + assistant_id, + "assistant", + answer, + at=updated - 1_000, + ) + + +def _iso_from_millis(value: int) -> str: + return datetime.fromtimestamp(value / 1000, tz=timezone.utc).isoformat().replace("+00:00", "Z") + + +def _ids(digests: Iterable[SessionDigest]) -> list[str]: + return [digest.session_id for digest in digests] + + +def test_open_database_closes_connection_if_read_only_setup_fails(monkeypatch) -> None: + connection = mock.Mock(spec=sqlite3.Connection) + connection.execute.side_effect = sqlite3.OperationalError("setup failed") + monkeypatch.setattr(sqlite3, "connect", mock.Mock(return_value=connection)) + + with pytest.raises(sqlite3.OperationalError, match="setup failed"): + _open_database("opencode.db") + + connection.close.assert_called_once_with() + + +def test_open_database_rejects_writes(tmp_path: Path) -> None: + path, writer = _new_store(tmp_path) + writer.commit() + writer.close() + + connection = _open_database(str(path)) + try: + with pytest.raises(sqlite3.OperationalError): + connection.execute("INSERT INTO session (id) VALUES ('should-fail')") + finally: + connection.close() + + +# Content and privacy + + +def test_maps_visible_text_tools_feedback_and_session_fields(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + path, connection = _new_store(tmp_path) + _add_session( + connection, + "s1", + project, + metadata={"gitBranch": "feature/opencode"}, + ) + _add_message(connection, "s1", "u1", "user", at=_BASE_MS + 1_000) + _add_part( + connection, + "s1", + "u1", + "p1", + {"type": "text", "text": "Please fix the parser."}, + at=_BASE_MS + 1_100, + ) + _add_part( + connection, + "s1", + "u1", + "p2", + {"type": "text", "text": "It is still broken."}, + at=_BASE_MS + 1_200, + ) + _add_message(connection, "s1", "a1", "assistant", at=_BASE_MS + 50_000) + _add_part( + connection, + "s1", + "a1", + "p3", + {"type": "text", "text": "I fixed the parser and ran its tests."}, + at=_BASE_MS + 50_100, + ) + _add_part( + connection, + "s1", + "a1", + "p4", + {"type": "tool", "tool": "read.file", "state": {"input": {}, "output": "hidden"}}, + at=_BASE_MS + 50_200, + ) + _add_part( + connection, + "s1", + "a1", + "p5", + {"type": "tool", "tool": "bad tool/", "state": {"status": "completed"}}, + at=_BASE_MS + 50_300, + ) + _add_part( + connection, + "s1", + "a1", + "p6", + {"type": "tool", "tool": "read.file"}, + at=_BASE_MS + 50_400, + ) + _add_part( + connection, + "s1", + "a1", + "p7", + { + "type": "tool-invocation", + "toolInvocation": {"toolName": "legacy.tool", "arguments": "hidden"}, + }, + at=_BASE_MS + 50_500, + ) + connection.commit() + connection.close() + + [digest] = harvest_opencode(str(path), scope="all") + + assert digest.session_id == "s1" + assert digest.project == project + assert digest.git_branch == "feature/opencode" + assert digest.started_at == _iso_from_millis(_BASE_MS) + assert digest.ended_at == _iso_from_millis(_BASE_MS + 60_000) + assert digest.user_prompts == ["Please fix the parser.\nIt is still broken."] + assert digest.assistant_finals == ["I fixed the parser and ran its tests."] + assert digest.tools_used == ["read.file", "bad_tool_arg_", "legacy.tool"] + assert digest.files_touched == [] + assert digest.n_user_turns == 1 + assert digest.n_assistant_turns == 1 + assert any(signal.startswith("neg:still broken") for signal in digest.feedback_signals) + assert "neg:opencode_message_error" not in digest.feedback_signals + assert digest.raw_path == "opencode://s1" + + +def test_excludes_reasoning_tool_io_files_patches_and_synthetic_text(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + path, connection = _new_store(tmp_path) + _add_session( + connection, + "private", + project, + metadata={"providerSecret": "hidden provider metadata"}, + ) + _add_message(connection, "private", "user", "user", at=_BASE_MS + 1_000) + _add_part( + connection, + "private", + "user", + "visible-user", + {"type": "text", "text": "Keep only this user request."}, + at=_BASE_MS + 1_100, + ) + _add_part( + connection, + "private", + "user", + "synthetic-user", + {"type": "text", "text": "hidden synthetic user text", "synthetic": True}, + at=_BASE_MS + 1_200, + ) + _add_message( + connection, + "private", + "assistant", + "assistant", + at=_BASE_MS + 50_000, + provider="hidden provider id", + model="hidden model id", + account="hidden account id", + ) + private_parts = [ + ("reasoning", {"type": "reasoning", "text": "hidden chain of thought"}), + ( + "tool", + { + "type": "tool", + "tool": "shell", + "state": { + "input": {"command": "echo hidden tool input"}, + "output": "hidden tool output", + }, + }, + ), + ("file", {"type": "file", "text": "hidden file text", "content": "hidden file content"}), + ("patch", {"type": "patch", "text": "hidden patch"}), + ("snapshot", {"type": "snapshot", "snapshot": "hidden snapshot"}), + ("synthetic", {"type": "text", "text": "hidden synthetic answer", "synthetic": True}), + ("ignored", {"type": "text", "text": "hidden ignored answer", "ignored": True}), + ("visible", {"type": "text", "text": "Only this answer is visible."}), + ] + for offset, (part_id, data) in enumerate(private_parts, start=1): + _add_part( + connection, + "private", + "assistant", + part_id, + data, + at=_BASE_MS + 50_000 + offset, + ) + connection.commit() + connection.close() + + [digest] = harvest_opencode(str(path), scope="all") + serialized_digest = json.dumps(digest.to_dict()) + + assert digest.user_prompts == ["Keep only this user request."] + assert digest.assistant_finals == ["Only this answer is visible."] + assert digest.tools_used == ["shell"] + assert digest.files_touched == [] + for hidden in ( + "chain of thought", + "tool input", + "tool output", + "file text", + "file content", + "hidden patch", + "hidden snapshot", + "synthetic", + "ignored", + "hidden provider metadata", + "hidden provider id", + "hidden model id", + "hidden account id", + ): + assert hidden not in serialized_digest + + +def test_redacts_visible_user_and_assistant_secrets(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + user_secret = "sk-abcdefghijklmnopqrstuvwxyz1234567890" + assistant_secret = "super-secret-value-123456" + path, connection = _new_store(tmp_path) + _add_basic_transcript( + connection, + "secrets", + project, + prompt=f"Use Authorization: Bearer {user_secret} for this task.", + answer=f"Configured api_key={assistant_secret}", + ) + connection.commit() + connection.close() + + [digest] = harvest_opencode(str(path), scope="all") + harvested_text = "\n".join(digest.user_prompts + digest.assistant_finals) + + assert user_secret not in harvested_text + assert assistant_secret not in harvested_text + assert "[REDACTED" in harvested_text + + +def test_preserves_long_visible_text_while_redacting_secrets(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + secret = "sk-abcdefghijklmnopqrstuvwxyz1234567890" + tail = "::safe-tail::" + path, connection = _new_store(tmp_path) + _add_basic_transcript( + connection, + "long-redacted-text", + project, + prompt="x" * 5000 + secret + tail, + ) + connection.commit() + connection.close() + + [digest] = harvest_opencode(str(path), scope="all") + + assert secret not in digest.user_prompts[0] + assert "sk-" not in digest.user_prompts[0] + assert len(digest.user_prompts[0]) > 5000 + assert digest.user_prompts[0].endswith(tail) + + +def test_keeps_all_user_prompts_and_last_five_assistant_finals(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + path, connection = _new_store(tmp_path) + _add_session(connection, "many-turns", project) + + prompt_count = 41 # Cross the old Copilot-style 40-prompt boundary. + final_count = 6 + for index in range(prompt_count): + message_id = f"user-{index:02d}" + at = _BASE_MS + 1_000 + index + _add_text_message( + connection, + "many-turns", + message_id, + "user", + f"User prompt {index}", + at=at, + ) + + for index in range(final_count): + message_id = f"assistant-{index:02d}" + at = _BASE_MS + 50_000 + index + _add_text_message( + connection, + "many-turns", + message_id, + "assistant", + f"Assistant final {index}", + at=at, + ) + + connection.commit() + connection.close() + + [digest] = harvest_opencode(str(path), scope="all") + + assert digest.user_prompts == [f"User prompt {index}" for index in range(prompt_count)] + assert digest.assistant_finals == [f"Assistant final {index}" for index in range(final_count - 5, final_count)] + + +def test_records_assistant_errors_and_removes_nul_characters(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + path, connection = _new_store(tmp_path) + _add_session(connection, "assistant-error", project) + _add_text_message( + connection, + "assistant-error", + "user", + "user", + "Fix\x00 the parser.", + at=_BASE_MS + 1_000, + ) + _add_text_message( + connection, + "assistant-error", + "assistant", + "assistant", + "The attempt\x00 failed.", + at=_BASE_MS + 50_000, + error={"name": "ProviderError"}, + ) + _add_text_message( + connection, + "assistant-error", + "assistant-success", + "assistant", + "The retry passed.", + at=_BASE_MS + 51_000, + ) + connection.commit() + connection.close() + + [digest] = harvest_opencode(str(path), scope="all") + + assert digest.user_prompts == ["Fix the parser."] + assert digest.assistant_finals == ["The attempt failed.", "The retry passed."] + assert digest.feedback_signals.count("neg:opencode_message_error") == 1 + + +def test_orders_messages_and_text_parts_independently_of_insert_order(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + path, connection = _new_store(tmp_path) + _add_session(connection, "ordered", project) + + _add_text_message( + connection, + "ordered", + "a-user-late", + "user", + "Second prompt", + at=_BASE_MS + 2_000, + ) + _add_message(connection, "ordered", "z-user-early", "user", at=_BASE_MS + 1_000) + _add_part( + connection, + "ordered", + "z-user-early", + "part-b", + {"type": "text", "text": "Part B"}, + at=_BASE_MS + 1_200, + ) + _add_part( + connection, + "ordered", + "z-user-early", + "part-a", + {"type": "text", "text": "Part A"}, + at=_BASE_MS + 1_100, + ) + connection.commit() + connection.close() + + [digest] = harvest_opencode(str(path), scope="all") + + assert digest.user_prompts == ["Part A\nPart B", "Second prompt"] + + +# Filtering and SQLite behavior + + +def test_applies_scope_since_order_and_limit_after_filtering(tmp_path: Path) -> None: + repo = (tmp_path / "repo").resolve() + child = (repo / "child").resolve() + other = (tmp_path / "other").resolve() + path, connection = _new_store(tmp_path) + + cases = [ + ("old", repo, 100_000), + ("tie-a", repo, 300_000), + ("tie-z", repo, 300_000), + ("new-child", child, 400_000), + ("other", other, 500_000), + ] + for session_id, project, updated_offset in cases: + _add_basic_transcript( + connection, + session_id, + str(project), + created=_BASE_MS + updated_offset - 60_000, + updated=_BASE_MS + updated_offset, + ) + + # The newest row has no visible prompt. It must be skipped before applying + # a result limit, rather than consuming one of the requested slots. + _add_session( + connection, + "empty-newest", + str(repo), + created=_BASE_MS + 540_000, + updated=_BASE_MS + 600_000, + ) + connection.commit() + connection.close() + + assert _ids(harvest_opencode(str(path), scope="all")) == [ + "other", + "new-child", + "tie-z", + "tie-a", + "old", + ] + assert _ids( + harvest_opencode( + str(path), + scope="invoked", + invoked_project=str(repo), + ) + ) == ["new-child", "tie-z", "tie-a", "old"] + assert _ids(harvest_opencode(str(path), scope=[str(other)])) == ["other"] + assert _ids( + harvest_opencode( + str(path), + scope="all", + since_iso=_iso_from_millis(_BASE_MS + 300_000), + ) + ) == ["other", "new-child"] + assert _ids(harvest_opencode(str(path), scope="all", limit=2)) == [ + "other", + "new-child", + ] + + +def test_filters_skillopt_replay_agents_and_all_child_sessions(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + path, connection = _new_store(tmp_path) + replay_agent = "skillopt-sleep-0123456789abcdef0123456789abcdef" + cases = [ + ("exact-self", None, "skillopt-sleep", replay_agent), + ("same-title", None, "skillopt-sleep", "build"), + ("rewritten-title", None, "Generated session title", replay_agent), + ( + "uppercase-agent", + None, + "skillopt-sleep", + "skillopt-sleep-0123456789ABCDEF0123456789ABCDEF", + ), + ("child", "same-title", "Child session", "build"), + ("empty-parent", "", "Child session", "build"), + ] + for index, (session_id, parent_id, title, agent) in enumerate(cases): + _add_basic_transcript( + connection, + session_id, + project, + created=_BASE_MS + index * 100_000, + updated=_BASE_MS + index * 100_000 + 60_000, + parent_id=parent_id, + title=title, + agent=agent, + ) + connection.commit() + connection.close() + + assert set(_ids(harvest_opencode(str(path), scope="all"))) == { + "same-title", + "uppercase-agent", + } + + +def test_missing_corrupt_and_incompatible_databases_fail_soft(tmp_path: Path) -> None: + assert harvest_opencode(str(tmp_path / "missing.db")) == [] + + corrupt = tmp_path / "corrupt.db" + corrupt.write_bytes(b"this is not a sqlite database") + assert harvest_opencode(str(corrupt)) == [] + + incompatible = tmp_path / "incompatible.db" + connection = sqlite3.connect(incompatible) + connection.execute("CREATE TABLE unrelated (id TEXT)") + connection.commit() + connection.close() + assert harvest_opencode(str(incompatible)) == [] + + partial = tmp_path / "partial.db" + connection = sqlite3.connect(partial) + connection.execute("CREATE TABLE session (id TEXT PRIMARY KEY)") + connection.commit() + connection.close() + assert harvest_opencode(str(partial)) == [] + + +def test_database_path_with_spaces_and_unicode(tmp_path: Path) -> None: + store_dir = tmp_path / "OpenCode history ü" + store_dir.mkdir() + project = str((tmp_path / "repo").resolve()) + path, connection = _new_store(store_dir, "session # data.db") + _add_basic_transcript(connection, "unicode-path", project) + connection.commit() + connection.close() + + assert _ids(harvest_opencode(str(path), scope="all")) == ["unicode-path"] + + +def test_optional_session_metadata_column_can_be_absent(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + path, connection = _new_store( + tmp_path, + "without-metadata.db", + include_metadata=False, + ) + connection.execute( + "INSERT INTO session " + "(id, project_id, parent_id, directory, title, time_created, time_updated, agent) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + "without-metadata", + "project-1", + None, + project, + "Interactive session", + _BASE_MS, + _BASE_MS + 60_000, + "build", + ), + ) + _add_message(connection, "without-metadata", "user", "user", at=_BASE_MS + 1_000) + _add_part( + connection, + "without-metadata", + "user", + "user-text", + {"type": "text", "text": "Harvest this session without metadata."}, + at=_BASE_MS + 1_100, + ) + connection.commit() + connection.close() + + [digest] = harvest_opencode(str(path), scope="all") + + assert digest.session_id == "without-metadata" + assert digest.git_branch == "" + + +def test_malformed_message_or_part_json_skips_the_affected_sessions(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + path, connection = _new_store(tmp_path) + _add_session(connection, "bad-message-session", project) + connection.execute( + "INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?)", + ( + "bad-message", + "bad-message-session", + _BASE_MS + 500, + _BASE_MS + 500, + "{not-json", + ), + ) + _add_session(connection, "bad-part-session", project) + _add_message( + connection, + "bad-part-session", + "bad-part-user", + "user", + at=_BASE_MS + 1_000, + ) + _add_part( + connection, + "bad-part-session", + "bad-part-user", + "bad-part", + "{not-json", + at=_BASE_MS + 1_050, + ) + _add_part( + connection, + "bad-part-session", + "bad-part-user", + "good-user", + {"type": "text", "text": "Keep the valid request."}, + at=_BASE_MS + 1_100, + ) + _add_basic_transcript( + connection, + "unaffected", + project, + created=_BASE_MS + 100_000, + updated=_BASE_MS + 160_000, + ) + connection.commit() + connection.close() + + assert _ids(harvest_opencode(str(path), scope="all")) == ["unaffected"] + + +def test_filters_generic_headless_replay_and_agent_sessions(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + path, connection = _new_store(tmp_path) + _add_basic_transcript( + connection, + "headless", + project, + prompt="You are a strict grader. Score this response.", + ) + _add_basic_transcript( + connection, + "agent", + project, + prompt="You are a Claude-Mem observer. Record this context.", + ) + _add_basic_transcript( + connection, + "short-headless", + project, + prompt="Quick automated check", + created=_BASE_MS + 100_000, + updated=_BASE_MS + 102_000, + ) + _add_basic_transcript( + connection, + "interactive", + project, + prompt="Please keep this normal interactive session.", + created=_BASE_MS + 200_000, + updated=_BASE_MS + 260_000, + ) + connection.commit() + connection.close() + + assert _ids(harvest_opencode(str(path), scope="all")) == ["interactive"] + + +def test_reads_committed_session_that_exists_only_in_live_wal(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + path, writer = _new_store(tmp_path, wal=True) + try: + _add_basic_transcript(writer, "wal-session", project) + writer.commit() + wal_path = Path(str(path) + "-wal") + assert wal_path.is_file() + assert wal_path.stat().st_size > 0 + + assert _ids(harvest_opencode(str(path), scope="all")) == ["wal-session"] + finally: + writer.close() + + +def test_harvest_does_not_modify_database_or_live_wal(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + path, writer = _new_store(tmp_path, wal=True) + try: + _add_basic_transcript(writer, "read-only", project) + writer.commit() + wal_path = Path(str(path) + "-wal") + before_db = path.read_bytes() + before_wal = wal_path.read_bytes() + + assert _ids(harvest_opencode(str(path), scope="all")) == ["read-only"] + + assert path.read_bytes() == before_db + assert wal_path.read_bytes() == before_wal + finally: + writer.close() + + +# Path, config, and source routing + + +def test_default_database_honors_xdg_and_opencode_db(monkeypatch, tmp_path: Path) -> None: + data_home = tmp_path / "xdg-data" + monkeypatch.setenv("XDG_DATA_HOME", str(data_home)) + monkeypatch.delenv("OPENCODE_DB", raising=False) + + assert default_opencode_db() == os.path.abspath(data_home / "opencode" / "opencode.db") + + monkeypatch.setenv("OPENCODE_DB", "sessions/custom.db") + assert default_opencode_db() == os.path.abspath(data_home / "opencode" / "sessions" / "custom.db") + + monkeypatch.setenv("OPENCODE_DB", "~/custom.db") + assert default_opencode_db() == os.path.abspath(data_home / "opencode" / "~" / "custom.db") + + absolute = tmp_path / "elsewhere" / "custom.db" + monkeypatch.setenv("OPENCODE_DB", str(absolute)) + assert default_opencode_db() == os.path.abspath(absolute) + + monkeypatch.setenv("OPENCODE_DB", ":memory:") + assert default_opencode_db() == "" + + +def test_default_database_falls_back_to_home_local_share(monkeypatch, tmp_path: Path) -> None: + home = tmp_path / "home" + monkeypatch.delenv("XDG_DATA_HOME", raising=False) + monkeypatch.delenv("OPENCODE_DB", raising=False) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + + assert default_opencode_db() == os.path.abspath(home / ".local" / "share" / "opencode" / "opencode.db") + + +def test_empty_database_path_harvests_from_xdg_default(monkeypatch, tmp_path: Path) -> None: + data_home = tmp_path / "xdg-data" + store_dir = data_home / "opencode" + store_dir.mkdir(parents=True) + project = str((tmp_path / "repo").resolve()) + path, connection = _new_store(store_dir) + _add_basic_transcript(connection, "xdg-default", project) + connection.commit() + connection.close() + monkeypatch.setenv("XDG_DATA_HOME", str(data_home)) + monkeypatch.delenv("OPENCODE_DB", raising=False) + + assert path == store_dir / "opencode.db" + assert _ids(harvest_opencode(scope="all")) == ["xdg-default"] + + +def test_cli_and_config_map_opencode_source_and_explicit_database(monkeypatch) -> None: + parser = argparse.ArgumentParser() + _add_common(parser) + args = parser.parse_args( + [ + "--source", + "opencode", + "--opencode-db", + "~/opencode-test/transcripts.db", + ] + ) + monkeypatch.setattr("skillopt_sleep.config._user_config_path", lambda: None) + + cfg = _cfg_from_args(args) + expected = os.path.abspath(os.path.expanduser("~/opencode-test/transcripts.db")) + + assert cfg.get("transcript_source") == "opencode" + assert cfg.get("opencode_db") == expected + assert cfg.opencode_db_path == expected + + +def test_explicit_memory_database_has_no_persistent_history(monkeypatch) -> None: + parser = argparse.ArgumentParser() + _add_common(parser) + args = parser.parse_args(["--source", "opencode", "--opencode-db", ":memory:"]) + monkeypatch.setattr("skillopt_sleep.config._user_config_path", lambda: None) + + cfg = _cfg_from_args(args) + + assert cfg.opencode_db_path == ":memory:" + assert harvest_opencode(cfg.opencode_db_path) == [] + + +def test_explicit_config_database_wins_over_environment(monkeypatch, tmp_path: Path) -> None: + env_db = tmp_path / "environment.db" + configured_db = tmp_path / "configured.db" + monkeypatch.setenv("OPENCODE_DB", str(env_db)) + monkeypatch.setattr("skillopt_sleep.config._user_config_path", lambda: None) + + cfg = load_config(opencode_db=str(configured_db)) + + assert cfg.opencode_db_path == os.path.abspath(configured_db) + + +def test_empty_config_leaves_default_database_resolution_to_harvester(monkeypatch) -> None: + monkeypatch.setattr("skillopt_sleep.config._user_config_path", lambda: None) + + assert load_config().opencode_db_path == "" + + +def test_explicit_opencode_source_routes_only_to_opencode_harvester(tmp_path: Path) -> None: + db_path = tmp_path / "configured.db" + project = str((tmp_path / "repo").resolve()) + cfg = load_config( + transcript_source="opencode", + projects="invoked", + invoked_project=project, + opencode_db=str(db_path), + ) + expected = [SessionDigest(session_id="opencode-session", project=project)] + with ( + mock.patch("skillopt_sleep.harvest_sources.harvest_opencode", return_value=expected) as opencode, + mock.patch("skillopt_sleep.harvest_sources.harvest_codex") as codex, + mock.patch("skillopt_sleep.harvest_sources.harvest") as claude, + mock.patch("skillopt_sleep.harvest_sources.harvest_copilot") as copilot, + mock.patch("skillopt_sleep.harvest_sources.harvest_copilot_cli") as copilot_cli, + mock.patch("skillopt_sleep.harvest_sources.harvest_cursor") as cursor, + mock.patch("skillopt_sleep.harvest_sources.harvest_pi") as pi, + ): + actual = harvest_for_config(cfg, since_iso="2026-01-01T00:00:00Z", limit=3) + + assert actual == expected + opencode.assert_called_once_with( + os.path.abspath(db_path), + scope="invoked", + invoked_project=project, + since_iso="2026-01-01T00:00:00Z", + limit=3, + ) + codex.assert_not_called() + claude.assert_not_called() + copilot.assert_not_called() + copilot_cli.assert_not_called() + cursor.assert_not_called() + pi.assert_not_called() + + +def test_auto_source_keeps_codex_then_claude_precedence_without_opencode(tmp_path: Path) -> None: + project = str((tmp_path / "repo").resolve()) + cfg = load_config( + transcript_source="auto", + projects="invoked", + invoked_project=project, + opencode_db=str(tmp_path / "opencode.db"), + ) + expected = [SessionDigest(session_id="claude-session", project=project)] + with ( + mock.patch("skillopt_sleep.harvest_sources.harvest_codex", return_value=[]) as codex, + mock.patch("skillopt_sleep.harvest_sources.harvest", return_value=expected) as claude, + mock.patch("skillopt_sleep.harvest_sources.harvest_opencode") as opencode, + ): + actual = harvest_for_config(cfg, since_iso="2026-01-01T00:00:00Z", limit=4) + + assert actual == expected + codex.assert_called_once() + claude.assert_called_once() + opencode.assert_not_called() diff --git a/tests/test_harvest_opencode_live.py b/tests/test_harvest_opencode_live.py new file mode 100644 index 00000000..63b2a706 --- /dev/null +++ b/tests/test_harvest_opencode_live.py @@ -0,0 +1,80 @@ +"""Opt-in smoke test for harvesting a real OpenCode transcript database. + +This test only reads the local SQLite database. It does not launch the +OpenCode CLI, call a model, or use the network. It is skipped unless +``SKILLOPT_TEST_REAL_OPENCODE_SOURCE=1`` is set. Set +``SKILLOPT_TEST_OPENCODE_DB`` to test a specific database; otherwise the +harvester uses OpenCode's normal ``OPENCODE_DB``/XDG path resolution. +""" + +from __future__ import annotations + +import os + +import pytest + +from skillopt_sleep.harvest_opencode import ( + default_opencode_db, + harvest_opencode, +) +from skillopt_sleep.types import SessionDigest + +_LIVE_ENABLED = os.environ.get("SKILLOPT_TEST_REAL_OPENCODE_SOURCE", "").strip() == "1" + +pytestmark = pytest.mark.skipif( + not _LIVE_ENABLED, + reason=("set SKILLOPT_TEST_REAL_OPENCODE_SOURCE=1 to read a real OpenCode transcript database"), +) + + +def _live_database_path() -> str: + configured = os.environ.get("SKILLOPT_TEST_OPENCODE_DB", "").strip() + path = os.path.abspath(os.path.expanduser(configured)) if configured else default_opencode_db() + if not path or not os.path.isfile(path): + pytest.fail( + "no OpenCode database was found for the opted-in source test; " + "set SKILLOPT_TEST_OPENCODE_DB to an existing database", + pytrace=False, + ) + return path + + +def _invalid_digest_field(digest: SessionDigest) -> str: + checks = ( + ("session_id", isinstance(digest.session_id, str) and bool(digest.session_id)), + ("project", isinstance(digest.project, str) and os.path.isabs(digest.project)), + ("started_at", isinstance(digest.started_at, str) and bool(digest.started_at)), + ("ended_at", isinstance(digest.ended_at, str) and bool(digest.ended_at)), + ("user_prompts", isinstance(digest.user_prompts, list) and bool(digest.user_prompts)), + ("n_user_turns", type(digest.n_user_turns) is int and digest.n_user_turns >= 1), + ( + "n_assistant_turns", + type(digest.n_assistant_turns) is int and digest.n_assistant_turns >= 0, + ), + ( + "raw_path", + isinstance(digest.raw_path, str) and digest.raw_path == f"opencode://{digest.session_id}", + ), + ) + return next((field for field, valid in checks if not valid), "") + + +def test_real_opencode_harvest_smoke() -> None: + """Read real sessions without exposing transcript content in test output.""" + path = _live_database_path() + digests = harvest_opencode(path, scope="all", limit=20) + if not digests: + pytest.fail( + "the real OpenCode database contained no harvestable root session", + pytrace=False, + ) + + invalid_field = next( + (field for digest in digests if (field := _invalid_digest_field(digest))), + "", + ) + if invalid_field: + pytest.fail( + f"the real OpenCode database produced an invalid {invalid_field} field", + pytrace=False, + ) From 29cb7295de8b52b80672b18a9171f08ddd3237fd Mon Sep 17 00:00:00 2001 From: YingqiDuan <141370165+YingqiDuan@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:26:13 -0700 Subject: [PATCH 2/2] docs(sleep): document OpenCode transcript harvesting --- CHANGELOG.md | 6 +++-- docs/reference/cli.md | 61 +++++++++++++++++++++++++++++++------------ docs/sleep/README.md | 27 ++++++++++++------- plugins/README.md | 16 +++++++----- 4 files changed, 76 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ccc528e..29c8fa30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,14 @@ All notable changes to SkillOpt are documented here. This project adheres to ## [Unreleased] ### Added +- **OpenCode transcript source** (`--source opencode`) for SkillOpt-Sleep. It + reads visible user/assistant text and tool names from OpenCode's local SQLite + history without requiring its CLI, login, or a provider connection. - **OpenCode CLI backend** (`--backend opencode`) for SkillOpt-Sleep model calls, including plain task replay, using an installed OpenCode CLI with the user's existing login and file-based global configuration. Calls parse OpenCode's JSONL output and disable project configuration, tool use, external plugins, - and configured MCP servers. Transcript harvesting and tool-aware replay - remain follow-up work. + and configured MCP servers. Tool-aware replay remains follow-up work. - **GitHub Copilot CLI backend**, in two forms: `copilot_chat` (usable as both optimizer and target) and `copilot_exec` (target-only execution harness). Because the Copilot CLI carries its own sign-in, `--backend copilot` selects diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 1ff2e574..0ec5e82b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -4,9 +4,9 @@ > include the generic research `openai_compatible` backend, Sleep handoff, > Sleep support for non-Azure OpenAI-compatible endpoints, the Sleep > `--preferences` flag, the research `cursor_exec` target harness, or Cursor -> source/backend/plugin support, Pi source/backend support, the OpenCode Sleep -> backend, or VS Code Copilot transcript harvesting; use a source install from -> `main` for those features until the next release. +> source/backend/plugin support, Pi source/backend support, OpenCode Sleep +> source/backend support, or VS Code Copilot transcript harvesting; use a source +> install from `main` for those features until the next release. ## Training @@ -129,7 +129,7 @@ Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, and |---|---| | `--project PATH` | Project used for transcript scope, targets, state, and staging (default: current directory) | | `--scope invoked\|all` | Harvest this project or all projects | -| `--source claude\|codex\|copilot\|cursor\|pi\|auto` | Transcript source; `auto` keeps Codex-then-Claude precedence and does not select Copilot, Cursor, or Pi | +| `--source claude\|codex\|copilot\|cursor\|pi\|opencode\|auto` | Transcript source; `auto` keeps Codex-then-Claude precedence and does not select Copilot, Cursor, Pi, or OpenCode | | `--backend mock\|claude\|codex\|copilot\|cursor\|pi\|opencode\|handoff\|azure_openai` | Replay/optimizer backend | | `--model NAME` | Backend-specific model override | | `--cursor-home PATH` | Override `~/.cursor` for Cursor transcript harvesting | @@ -138,6 +138,7 @@ Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, and | `--cursor-path PATH` | Path to the installed Cursor Agent CLI | | `--pi-path PATH` | Path to the installed Pi coding-agent CLI | | `--opencode-path PATH` | Path to the installed OpenCode CLI | +| `--opencode-db PATH` | Path to the OpenCode SQLite history database | | `--preferences TEXT` | House rules supplied to reflection | | `--lookback-hours N` | Initial transcript lookback; `0` scans all history | | `--max-sessions N` / `--max-tasks N` | Bound the harvested workload | @@ -214,18 +215,46 @@ The managed `schedule` command preserves the backend but not `--source`, `~/.skillopt-sleep/config.json`; use an absolute `pi_path` and verify authentication for the scheduled account. -### OpenCode backend +### OpenCode source and backend -Install and configure OpenCode using its -[official documentation](https://opencode.ai/docs/), then confirm the CLI is -available with `opencode --version`. +`--source opencode` reads OpenCode's local SQLite history directly in read-only +mode. The source does not launch OpenCode, require the OpenCode CLI, use its +login, or contact a model provider. Source selection remains explicit: +`--source auto` keeps Codex-then-Claude precedence and does not select OpenCode. + +The database path is selected from `--opencode-db` or the `opencode_db` config +key, then `OPENCODE_DB`, then +`${XDG_DATA_HOME:-~/.local/share}/opencode/opencode.db`. A relative +`OPENCODE_DB` value is resolved below OpenCode's data directory; +`OPENCODE_DB=:memory:` has no persistent history to harvest. + +The harvester keeps visible user and assistant text, short tool names, the +recorded project directory, Git branch, and session timestamps. It excludes +reasoning, tool arguments and results, file contents, patches, and +provider/model/account metadata. Only root sessions are considered, and +sessions produced by SkillOpt's own OpenCode backend are excluded. Known +secret-shaped strings in retained text are redacted as defense in depth; +inspect harvested tasks before sending them to a real backend. The database is +opened read-only, although SQLite may still update its transient `-shm` file +while coordinating an active WAL database. + +The transcript source and model backend are independent. For example, export +OpenCode-derived tasks for review before using any configured backend: + +```bash +skillopt-sleep harvest --project "$(pwd)" \ + --source opencode --output reviewed-tasks.json --progress +``` `--backend opencode` runs SkillOpt's model calls for mining, plain task replay, judging, and reflection through an installed OpenCode CLI. It uses the user's existing OpenCode login, provider environment variables, and file-based global configuration; SkillOpt does not manage OpenCode accounts or provider -credentials. Transcript sources remain independent, and there is not yet a -`--source opencode` harvester. +credentials. + +Install and configure OpenCode using its +[official documentation](https://opencode.ai/docs/), then confirm the CLI is +available with `opencode --version`. If OpenCode is on `PATH`, no path option is needed. Otherwise use `--opencode-path`, the `opencode_path` config key, or @@ -234,7 +263,7 @@ If OpenCode is on `PATH`, no path option is needed. Otherwise use ```bash skillopt-sleep run --project "$(pwd)" \ - --source codex --backend opencode \ + --source opencode --backend opencode \ --opencode-path /absolute/path/to/opencode \ --model provider/model --max-sessions 5 --max-tasks 3 --progress ``` @@ -256,11 +285,11 @@ unavailable. Calls may appear in the user's normal OpenCode session history; these controls are invocation settings, not complete account or process isolation. -The managed scheduler stores the backend but not `--opencode-path`, `--model`, -or the transcript source. Before scheduling OpenCode, put `opencode_path`, -`model`, and `transcript_source` in `~/.skillopt-sleep/config.json` as needed. -Prefer an absolute executable path and verify OpenCode access for the account -that runs the scheduled job. +The managed scheduler stores the backend but not `--source`, `--opencode-db`, +`--opencode-path`, or `--model`. Put `transcript_source`, `opencode_db`, +`opencode_path`, and `model` in `~/.skillopt-sleep/config.json` as needed. Use +absolute database and executable paths, and verify OpenCode access when the +scheduled run uses the backend. ### Cursor source and backend diff --git a/docs/sleep/README.md b/docs/sleep/README.md index 8a976a5e..0d649b9f 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -17,7 +17,7 @@ normal agent requests. One "night": ``` -harvest Claude Code / Codex / VS Code Copilot / Cursor / Pi transcripts → mine recurring tasks → replay via the configured backend (isolation varies by backend; mock/handoff make no network calls) +harvest Claude Code / Codex / VS Code Copilot / Cursor / Pi / OpenCode transcripts → mine recurring tasks → replay via the configured backend (isolation varies by backend; mock/handoff make no network calls) → consolidate (reflect → bounded edit → GATE on real held-out tasks) → stage proposal → (you) adopt ``` @@ -90,8 +90,9 @@ skillopt-sleep schedule # install a nightly cron entry for this project > **Version note.** This page tracks `main`. PyPI 0.2.0 provides the base > commands above. Cursor source/backend/plugin support, VS Code Copilot > transcript harvesting, Pi source/backend support, Sleep handoff, non-Azure -> OpenAI-compatible endpoints, the OpenCode Sleep backend, and `--preferences` -> landed later and require a source install from `main` until the next release. +> OpenAI-compatible endpoints, OpenCode Sleep source/backend support, and +> `--preferences` landed later and require a source install from `main` until +> the next release. The per-agent integrations below still come from the repo; the CLI above is the standalone, pip-only way to run a cycle. Claude Code, Codex, Cursor, Copilot, and @@ -172,7 +173,13 @@ scheduled account's Pi authentication. ### OpenCode -Install and configure OpenCode using its +Use `--source opencode` to read local OpenCode SQLite history without launching +the CLI or requiring login or provider access. It is not selected by +`--source auto`. See the +[CLI reference](../reference/cli.md#opencode-source-and-backend) for database +selection and the retained-data boundary. + +For model calls, install and configure OpenCode using its [official documentation](https://opencode.ai/docs/), then confirm the CLI is available with `opencode --version`. @@ -184,7 +191,7 @@ not suitable: ```bash skillopt-sleep run --project "$(pwd)" \ - --source codex --backend opencode \ + --source opencode --backend opencode \ --opencode-path /absolute/path/to/opencode --model provider/model ``` @@ -198,11 +205,11 @@ the user's existing value in that child process, so settings supplied only through that value are unavailable; use file-based global configuration or provider environment variables instead. -OpenCode transcript harvesting and tool-aware replay are not implemented yet. -For scheduling, put `opencode_path`, `model`, and the desired -`transcript_source` in `~/.skillopt-sleep/config.json` as needed, and verify the -scheduled account can run OpenCode. See the -[CLI reference](../reference/cli.md#opencode-backend) for full details. +Tool-aware replay and a native OpenCode plugin or command are not implemented +yet. For scheduled runs, configure the source, database, executable, and model +in `~/.skillopt-sleep/config.json` as needed; the +[CLI reference](../reference/cli.md#opencode-source-and-backend) has the full +scheduler details. ### Cursor diff --git a/plugins/README.md b/plugins/README.md index 95283eb9..81b4b03e 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -48,8 +48,8 @@ an importable `skillopt_sleep` module. Install with `uv tool install skillopt` o > **Version note.** This integration reference tracks `main`. PyPI 0.2.0 > supports the base Sleep CLI, while Cursor source/backend/plugin support, > Pi source/backend support, handoff, Sleep support for non-Azure -> OpenAI-compatible endpoints, the OpenCode Sleep backend, and `--preferences` -> require a source checkout from `main` until the next release. +> OpenAI-compatible endpoints, OpenCode Sleep source/backend support, and +> `--preferences` require a source checkout from `main` until the next release. ## One sleep cycle @@ -93,12 +93,15 @@ optimization. retained for scope filtering and may appear in miner prompts sent to a real backend and its provider. Known secret-shaped strings in retained message text are redacted only as defense in depth. +- The core `opencode` source reads local OpenCode SQLite history without the + CLI, authentication, or provider access. See + [the CLI reference](../docs/reference/cli.md#opencode-source-and-backend) for + its retained-data boundary. - The core `opencode` backend uses the installed OpenCode CLI for plain model calls. It keeps the user's login and file-based global configuration while disabling project configuration, tool use, external plugins, and - configured MCP servers for those calls. OpenCode transcript harvesting, - tool-aware replay, and a native OpenCode plugin or command are not included - yet. + configured MCP servers for those calls. Tool-aware replay and a native + OpenCode plugin or command are not included yet. - Outbound prompts are not currently guaranteed to be free of secrets. Do not use a third-party provider on sensitive transcripts without reviewing the data source and the provider's retention policy. @@ -136,12 +139,13 @@ Common implemented flags include: |---|---|---| | `--backend mock\|claude\|codex\|cursor\|copilot\|pi\|opencode\|handoff\|azure_openai` | `mock` | select who performs model calls | | `--model NAME` | backend default | select a backend-specific model | -| `--source claude\|codex\|copilot\|cursor\|pi\|auto` | `claude` | select the transcript source; `auto` retains Codex-then-Claude precedence and does not select Copilot, Cursor, or Pi | +| `--source claude\|codex\|copilot\|cursor\|pi\|opencode\|auto` | `claude` | select the transcript source; `auto` retains Codex-then-Claude precedence and does not select Copilot, Cursor, Pi, or OpenCode | | `--cursor-home PATH` | `~/.cursor` | override the Cursor transcript home | | `--cursor-path PATH` | auto-detect `cursor-agent` | select the Cursor Agent CLI executable | | `--pi-home PATH` | `~/.pi` | select the parent directory containing `agent/sessions` | | `--pi-path PATH` | auto-detect `pi` | select the Pi coding-agent CLI executable | | `--opencode-path PATH` | `SKILLOPT_SLEEP_OPENCODE_PATH`, then `opencode` on `PATH`/`PATHEXT` | select the OpenCode CLI executable | +| `--opencode-db PATH` | `OPENCODE_DB`, then `${XDG_DATA_HOME:-~/.local/share}/opencode/opencode.db` | select the OpenCode SQLite history database | | `--project PATH` | current directory | select the project and invoked harvest scope | | `--scope invoked\|all` | `invoked` | limit transcript harvesting | | `--target-skill-path PATH` | managed skill | select a specific `SKILL.md` to stage/adopt |