diff --git a/spanweave/cli/commands/harvest.py b/spanweave/cli/commands/harvest.py index 6f13036..26990e0 100644 --- a/spanweave/cli/commands/harvest.py +++ b/spanweave/cli/commands/harvest.py @@ -8,6 +8,7 @@ from __future__ import annotations +from datetime import date from pathlib import Path import click @@ -16,6 +17,26 @@ from spanweave.harvest import SUPPORTED_TOOLS, available_tools, harvest_tool +def _validate_since( + ctx: click.Context, param: click.Parameter, value: str | None +) -> str | None: + """Validate ``--since`` is a parseable ISO date (``YYYY-MM-DD``). + + Returns the value unchanged when valid (or ``None`` when omitted); raises a + clean :class:`click.BadParameter` (usage error, non-zero exit, no traceback) + on bad input. + """ + if value is None: + return None + try: + date.fromisoformat(value) + except ValueError: + raise click.BadParameter( + f"{value!r} is not a valid date — expected YYYY-MM-DD." + ) from None + return value + + @click.command( "harvest", context_settings={"help_option_names": ["-h", "--help"]}, @@ -33,6 +54,7 @@ "spanweave harvest --tool claude-code --repo .", "spanweave harvest -t claude-code -t codex --repo .", "spanweave harvest --tool codex --repo . --all # all projects", + "spanweave harvest --tool codex --repo . --since 2026-05-20 # recent only", ), ), ) @@ -62,7 +84,20 @@ "whose cwd matches --repo. (No effect on claude-code, already per-repo.)" ), ) -def harvest_command(tools: tuple[str, ...], repo: Path, all_projects: bool) -> None: +@click.option( + "--since", + "since", + type=str, + default=None, + callback=_validate_since, + help=( + "Only harvest Codex sessions updated on/after this date (YYYY-MM-DD). " + "Codex only." + ), +) +def harvest_command( + tools: tuple[str, ...], repo: Path, all_projects: bool, since: str | None +) -> None: """Import native memory from coding agents into .spanweave/memory/pending/.""" repo_path = repo.resolve() @@ -78,7 +113,9 @@ def harvest_command(tools: tuple[str, ...], repo: Path, all_projects: bool) -> N pending_dir = repo_path / ".spanweave" / "memory" / "pending" total = 0 for tool in selected: - result = harvest_tool(tool, repo_path, all_projects=all_projects) + result = harvest_tool( + tool, repo_path, all_projects=all_projects, since=since + ) total += result.count if not result.store_present: click.echo(f"No {tool} native memory found — skipped.") diff --git a/spanweave/harvest/__init__.py b/spanweave/harvest/__init__.py index 287b843..7f9518e 100644 --- a/spanweave/harvest/__init__.py +++ b/spanweave/harvest/__init__.py @@ -70,7 +70,11 @@ def available_tools(repo_root: Path) -> list[str]: def harvest_tool( - tool: str, repo_root: Path, *, all_projects: bool = False + tool: str, + repo_root: Path, + *, + all_projects: bool = False, + since: str | None = None, ) -> HarvestResult: """Harvest a single tool's native memory into the pending queue. @@ -80,11 +84,18 @@ def harvest_tool( project's Codex memory. The Claude harvester is already per-repo and ignores the flag. + ``since`` (an ISO ``YYYY-MM-DD`` date) likewise only affects ``codex``: it + restricts the harvest to sessions updated on/after that date (recency + scoping). The Claude harvester has no comparable per-session timestamp, so + ``since`` is ignored for ``claude-code``. + Raises ``KeyError`` for an unknown tool name; the CLI validates the name via a Click ``Choice`` before calling this. """ if tool == "codex": - staged = harvest_codex_memory(repo_root, all_projects=all_projects) + staged = harvest_codex_memory( + repo_root, all_projects=all_projects, since=since + ) else: staged = _HARVESTERS[tool](repo_root) return HarvestResult( diff --git a/spanweave/harvest/codex.py b/spanweave/harvest/codex.py index 33dfcd1..3b4c132 100644 --- a/spanweave/harvest/codex.py +++ b/spanweave/harvest/codex.py @@ -22,10 +22,22 @@ flag) to import the entire global store unscoped. The consolidated ``memory_summary.md`` has no single ``cwd`` (it is cross-project) and is included only in the unscoped ``--all`` view. + +**Recency scoping.** cwd-scoping is by FOLDER, not topic — a Codex session that +ran in this repo's folder but was about a different/old subject still matches the +``cwd``. The ``since`` parameter (an ISO ``YYYY-MM-DD`` date) further restricts +rollout summaries to those updated on/after that date (inclusive), since the most +recent work is usually the most relevant. The date is taken from the frontmatter +``updated_at`` (ISO-8601 datetime), falling back to the leading ``YYYY-MM-DD`` in +the filename. A summary with no resolvable date can't be proven recent, so it is +excluded under ``since``; the cross-project ``memory_summary.md`` has no single +date and is likewise excluded whenever ``since`` is set. ``since=None`` (the +default) applies no date filtering and preserves the prior behavior. """ from __future__ import annotations +from datetime import date, datetime from pathlib import Path from typing import Any @@ -92,6 +104,39 @@ def _parse_frontmatter(text: str) -> tuple[dict[str, str], str]: return meta, body +def _parse_since(since: str | None) -> date | None: + """Parse an optional ISO ``YYYY-MM-DD`` string into a :class:`date`. + + Returns ``None`` when ``since`` is ``None`` (no filtering). Raises + :class:`ValueError` on an unparseable string; the CLI validates earlier, so + this is a defensive backstop for direct API callers. + """ + if since is None: + return None + return date.fromisoformat(since) + + +def _summary_date(meta: dict[str, str], path: Path) -> date | None: + """Resolve a rollout summary's date for ``since`` comparison. + + Prefers the frontmatter ``updated_at`` (ISO-8601 datetime); falls back to + the leading ``YYYY-MM-DD`` in the filename. Returns ``None`` when neither + yields a parseable date (such a record can't be proven recent, so callers + exclude it under ``since``). + """ + updated_at = meta.get("updated_at", "").strip() + if updated_at: + try: + return datetime.fromisoformat(updated_at).date() + except ValueError: + pass + # Filename fallback: the leading 10 chars are the ``YYYY-MM-DD`` prefix. + try: + return date.fromisoformat(path.stem[:10]) + except ValueError: + return None + + def _record_from_file( path: Path, *, kind: str, meta: dict[str, str] | None = None ) -> dict[str, Any] | None: @@ -133,7 +178,9 @@ def _record_from_file( } -def read_codex_records(repo_root: Path | None = None) -> list[dict[str, Any]]: +def read_codex_records( + repo_root: Path | None = None, *, since: str | None = None +) -> list[dict[str, Any]]: """Parse Codex native-memory summaries into stage-able decision dicts. Codex's memory store is GLOBAL — a single ``~/.codex/memories`` tree shared @@ -147,6 +194,13 @@ def read_codex_records(repo_root: Path | None = None) -> list[dict[str, Any]]: * When ``repo_root`` is ``None`` (the ``--all`` case), include the entire global store unscoped, ``memory_summary.md`` included. + When ``since`` (an ISO ``YYYY-MM-DD`` date) is given, additionally include a + rollout summary only if its date (frontmatter ``updated_at``, else the + leading filename date) is on/after ``since`` (inclusive). Summaries with no + resolvable date are excluded. Because ``memory_summary.md`` has no single + date, it is excluded whenever ``since`` is set. ``since=None`` (default) + applies no date filtering. + Returns an empty list when the Codex memory store is absent. """ memory_dir = codex_memory_dir() @@ -154,12 +208,14 @@ def read_codex_records(repo_root: Path | None = None) -> list[dict[str, Any]]: return [] scoped_cwd = str(repo_root.resolve()) if repo_root is not None else None + since_date = _parse_since(since) records: list[dict[str, Any]] = [] - # The consolidated summary is cross-project; include it only when unscoped. + # The consolidated summary is cross-project; include it only when unscoped + # AND no recency filter is active (it carries no single date to compare). summary_path = memory_dir / _SUMMARY_FILENAME - if scoped_cwd is None and summary_path.is_file(): + if scoped_cwd is None and since_date is None and summary_path.is_file(): record = _record_from_file(summary_path, kind="summary") if record is not None: records.append(record) @@ -171,6 +227,11 @@ def read_codex_records(repo_root: Path | None = None) -> list[dict[str, Any]]: if scoped_cwd is not None and meta.get("cwd") != scoped_cwd: # Different project, or no cwd at all → not in the scoped view. continue + if since_date is not None: + rec_date = _summary_date(meta, path) + if rec_date is None or rec_date < since_date: + # Older than --since, or no resolvable date → excluded. + continue record = _record_from_file(path, kind="rollout", meta=meta) if record is not None: records.append(record) @@ -179,7 +240,7 @@ def read_codex_records(repo_root: Path | None = None) -> list[dict[str, Any]]: def harvest_codex_memory( - repo_root: Path, *, all_projects: bool = False + repo_root: Path, *, all_projects: bool = False, since: str | None = None ) -> list[Path]: """Read Codex native memory and stage it into the pending review queue. @@ -190,12 +251,16 @@ def harvest_codex_memory( flag) to import the entire global store unscoped, including ``memory_summary.md``. + When ``since`` (an ISO ``YYYY-MM-DD`` date) is given, only sessions updated + on/after that date are imported — recency scoping on top of cwd scoping. See + :func:`read_codex_records`. + Records are written with ``source: native-codex``. Records whose body already exists in pending or confirmed memory are skipped, so repeated runs are idempotent. Returns the paths of the records actually written. """ scope = None if all_projects else repo_root - records = read_codex_records(scope) + records = read_codex_records(scope, since=since) fresh = dedupe_against_pending_and_confirmed(records, repo_root=repo_root) if not fresh: return [] diff --git a/tests/test_harvest_codex.py b/tests/test_harvest_codex.py index 3732ac4..57fc13e 100644 --- a/tests/test_harvest_codex.py +++ b/tests/test_harvest_codex.py @@ -306,3 +306,185 @@ def test_harvest_all_stages_everything(fake_home: Path, tmp_path: Path) -> None: assert "pydantic-settings" in staged assert "billing webhook" in staged assert "tool-agnostic memory" in staged + + +# --- --since recency filter (#116) ------------------------------------------ + + +def _rollout_dated(cwd: str, body: str, *, updated_at: str | None) -> str: + """A rollout summary with an optional ``updated_at`` frontmatter line. + + When ``updated_at`` is None the line is omitted entirely, so the reader + must fall back to the date-prefixed filename. + """ + lines = ["thread_id: 019deadbeef"] + if updated_at is not None: + lines.append(f"updated_at: {updated_at}") + lines += [f"cwd: {cwd}", "git_branch: main", "", body, ""] + return "\n".join(lines) + + +def _write_dated_tree(home: Path) -> None: + """Three same-cwd (/repo/A) rollouts dated 05-07, 05-15, 05-25. + + Both the ``updated_at`` frontmatter and the date-prefixed filename carry + the date, so either source resolves to the same day. Distinct prose keeps + the word-overlap dedup from collapsing them. + """ + _write_codex_memory( + home, + summary=None, + rollouts={ + "2026-05-07T09-14-22-old.md": _rollout_dated( + "/repo/A", + "Adopted pydantic-settings for config loading.", + updated_at="2026-05-07T09:14:30+00:00", + ), + "2026-05-15T12-15-13-mid.md": _rollout_dated( + "/repo/A", + "Migrated the CI pipeline onto GitHub Actions runners.", + updated_at="2026-05-15T12:15:35+00:00", + ), + "2026-05-25T08-00-00-new.md": _rollout_dated( + "/repo/A", + "Refactored the billing webhook retry handler thoroughly.", + updated_at="2026-05-25T08:00:01+00:00", + ), + }, + ) + + +def test_since_filters_out_older_than_date(fake_home: Path) -> None: + """since=2026-05-20 keeps only the 05-25 session.""" + _write_dated_tree(fake_home) + + records = read_codex_records(Path("/repo/A"), since="2026-05-20") + + bodies = "\n".join(r["body"] for r in records) + assert "billing webhook" in bodies + assert "pydantic-settings" not in bodies + assert "GitHub Actions" not in bodies + assert len(records) == 1 + + +def test_since_is_inclusive_on_boundary(fake_home: Path) -> None: + """since=2026-05-15 keeps the 05-15 (boundary) and 05-25 sessions.""" + _write_dated_tree(fake_home) + + records = read_codex_records(Path("/repo/A"), since="2026-05-15") + + bodies = "\n".join(r["body"] for r in records) + assert "GitHub Actions" in bodies # 05-15 boundary is inclusive + assert "billing webhook" in bodies # 05-25 + assert "pydantic-settings" not in bodies # 05-07 excluded + assert len(records) == 2 + + +def test_since_none_returns_all(fake_home: Path) -> None: + """since=None is the current behavior: no date filtering.""" + _write_dated_tree(fake_home) + + records = read_codex_records(Path("/repo/A"), since=None) + + assert len(records) == 3 + + +def test_since_falls_back_to_filename_date(fake_home: Path) -> None: + """With no updated_at frontmatter, the leading filename date is used.""" + _write_codex_memory( + fake_home, + summary=None, + rollouts={ + "2026-05-25T08-00-00-fname.md": _rollout_dated( + "/repo/A", + "Refactored the billing webhook retry handler thoroughly.", + updated_at=None, + ), + "2026-05-01T00-00-00-old.md": _rollout_dated( + "/repo/A", + "Adopted pydantic-settings for config loading.", + updated_at=None, + ), + }, + ) + + records = read_codex_records(Path("/repo/A"), since="2026-05-20") + + bodies = "\n".join(r["body"] for r in records) + assert "billing webhook" in bodies # 05-25 from filename + assert "pydantic-settings" not in bodies # 05-01 from filename, excluded + assert len(records) == 1 + + +def test_since_excludes_undateable_record(fake_home: Path) -> None: + """A summary with neither updated_at nor a date-prefixed filename is + excluded once since is set (cannot prove it is recent).""" + _write_codex_memory( + fake_home, + summary=None, + rollouts={ + "2026-05-25T08-00-00-new.md": _rollout_dated( + "/repo/A", + "Refactored the billing webhook retry handler thoroughly.", + updated_at="2026-05-25T08:00:01+00:00", + ), + "no-date-here.md": _rollout_dated( + "/repo/A", + "Adopted pydantic-settings for config loading.", + updated_at=None, + ), + }, + ) + + records = read_codex_records(Path("/repo/A"), since="2026-05-20") + + bodies = "\n".join(r["body"] for r in records) + assert "billing webhook" in bodies + assert "pydantic-settings" not in bodies # undateable → excluded + assert len(records) == 1 + + +def test_since_excludes_cross_project_summary(fake_home: Path) -> None: + """The cross-project memory_summary.md has no single date → excluded + whenever since is set, even in all-projects (repo_root=None) mode.""" + _write_dated_tree(fake_home) + memories = fake_home / ".codex" / "memories" + (memories / "memory_summary.md").write_text(_MEMORY_SUMMARY, encoding="utf-8") + + records = read_codex_records(None, since="2026-05-01") + + bodies = "\n".join(r["body"] for r in records) + assert "tool-agnostic memory" not in bodies # summary excluded + # The three dated rollouts are all on/after 05-01. + assert len(records) == 3 + + +def test_harvest_codex_memory_threads_since( + fake_home: Path, tmp_path: Path +) -> None: + """harvest_codex_memory(repo, since=...) only stages recent sessions.""" + repo = tmp_path / "repo" + repo.mkdir() + _write_codex_memory( + fake_home, + summary=None, + rollouts={ + "2026-05-07T00-00-00-old.md": _rollout_dated( + str(repo.resolve()), + "Adopted pydantic-settings for config loading.", + updated_at="2026-05-07T00:00:00+00:00", + ), + "2026-05-25T00-00-00-new.md": _rollout_dated( + str(repo.resolve()), + "Refactored the billing webhook retry handler thoroughly.", + updated_at="2026-05-25T00:00:00+00:00", + ), + }, + ) + + paths = harvest_codex_memory(repo, since="2026-05-20") + + assert len(paths) == 1 + staged = paths[0].read_text(encoding="utf-8") + assert "billing webhook" in staged + assert "pydantic-settings" not in staged diff --git a/tests/test_harvest_command.py b/tests/test_harvest_command.py index f488d76..72e133a 100644 --- a/tests/test_harvest_command.py +++ b/tests/test_harvest_command.py @@ -230,3 +230,66 @@ def test_harvest_requested_tool_without_store_is_graceful( assert result.exit_code == 0, result.output assert "No codex native memory found" in result.output + + +def _codex_dated(cwd: str, body: str, *, updated_at: str) -> str: + """Codex rollout with bare frontmatter incl. cwd and updated_at.""" + return ( + f"thread_id: 019abc\nupdated_at: {updated_at}\n" + f"cwd: {cwd}\ngit_branch: main\n\n{body}\n" + ) + + +def test_harvest_codex_since_stages_only_recent( + runner: CliRunner, fake_home: Path, tmp_path: Path +) -> None: + """--since stages only sessions updated on/after the date (Codex).""" + repo = tmp_path / "repo" + repo.mkdir() + rollout_dir = fake_home / ".codex" / "memories" / "rollout_summaries" + rollout_dir.mkdir(parents=True, exist_ok=True) + (rollout_dir / "2026-05-07T00-00-00-old.md").write_text( + _codex_dated( + str(repo.resolve()), + "Old throwaway demo login rate-limiting note.", + updated_at="2026-05-07T00:00:00+00:00", + ), + encoding="utf-8", + ) + (rollout_dir / "2026-05-25T00-00-00-new.md").write_text( + _codex_dated( + str(repo.resolve()), + "Recent spanweave harvest recency-filter work.", + updated_at="2026-05-25T00:00:00+00:00", + ), + encoding="utf-8", + ) + + result = runner.invoke( + harvest_command, + ["--tool", "codex", "--repo", str(repo), "--since", "2026-05-20"], + ) + + assert result.exit_code == 0, result.output + bodies = "\n".join(_pending_bodies(repo)) + assert "harvest recency-filter" in bodies + assert "login rate-limiting" not in bodies + + +def test_harvest_since_bad_date_is_clean_error( + runner: CliRunner, fake_home: Path, tmp_path: Path +) -> None: + """A non-date --since fails cleanly (non-zero exit, no traceback).""" + repo = tmp_path / "repo" + repo.mkdir() + + result = runner.invoke( + harvest_command, + ["--tool", "codex", "--repo", str(repo), "--since", "notadate"], + ) + + assert result.exit_code != 0 + # Click emits a BadParameter usage error, not an uncaught exception. + assert result.exception is None or isinstance(result.exception, SystemExit) + out = result.output.lower() + assert "notadate" in result.output or "date" in out