perf: scale readable ref diffs with changed notes

.mise/tasks/diff

@@ -2,7 +2,7 @@
 #MISE description="Show readable note diffs for the working tree, refs, or a PR"
 #USAGE flag "--dir <dir>" default="notes" help="Notes directory relative to repo root (default: notes)"
 #USAGE flag "--pr <number>" default="" help="Diff a GitHub PR from the current repo's origin remote"
-#USAGE flag "--out <dir>" default="" help="Write readable base/head trees and readable.patch to this directory"
+#USAGE flag "--out <dir>" default="" help="Write changed-note base/head artifacts and readable.patch to this directory"
 #USAGE arg "[refs]" var=#true default="" help="Optional ref/range: BASE HEAD, BASE..HEAD, or BASE...HEAD. Omit for working-tree diff against HEAD."
 set -euo pipefail
 
@@ -146,8 +146,13 @@ cleanup_out_dir() {
 trap 'cleanup_temp_refs; cleanup_out_dir' EXIT
 
 _prepare_diff_workspace "$out_dir"
-materialize_readable_notes_ref "$repo_root" "$notes_dir" "$base_ref" "$out_dir/base"
-materialize_readable_notes_ref "$repo_root" "$notes_dir" "$head_ref" "$out_dir/head"
+materialize_changed_readable_notes_refs \
+  "$repo_root" \
+  "$notes_dir" \
+  "$base_ref" \
+  "$head_ref" \
+  "$out_dir/base" \
+  "$out_dir/head"
 generate_readable_notes_patch "$out_dir/base" "$out_dir/head" "$out_dir/readable.patch"
 
 if [ -s "$out_dir/readable.patch" ]; then
@@ -157,7 +162,7 @@ else
 fi
 
 if [ -n "$out_arg" ]; then
-  echo "Wrote readable base: $out_dir/base" >&2
-  echo "Wrote readable head: $out_dir/head" >&2
+  echo "Wrote changed-note base artifacts: $out_dir/base" >&2
+  echo "Wrote changed-note head artifacts: $out_dir/head" >&2
   echo "Wrote readable patch: $out_dir/readable.patch" >&2
 fi

CONTRIBUTING.md

@@ -67,11 +67,11 @@ classification and execution, while `test/setup_suite.bash` establishes the
 repo-local tool environment and deterministic fixture Git identity once per
 BATS invocation.
 
-Notes intentionally uses eight BATS workers, including within-file
-parallelism. Several suites contain many independent Git fixtures, so the
-starter template's four-job files-only default is materially slower here.
-Keep mutable fixtures under `$BATS_TEST_TMPDIR`; do not introduce shared test
-repositories, fixed temporary paths, or ambient signing dependencies.
+Notes uses eight BATS workers across test files. BATS does not parallelize the
+tests within one file, so split large suites along coherent fixture and behavior
+boundaries when that exposes useful independent work. Keep mutable fixtures
+under `$BATS_TEST_TMPDIR`; do not introduce shared test repositories, fixed
+temporary paths, or ambient signing dependencies.
 
 ## Encryption and Git safety
 

README.md

@@ -6,7 +6,7 @@
 
 **Collective memory, encrypted.**
 
-[![tests: 408](https://img.shields.io/badge/tests-408-brightgreen?style=flat)](test/)
+[![tests: 412](https://img.shields.io/badge/tests-412-brightgreen?style=flat)](test/)
 ![lints: 8](https://img.shields.io/badge/lints-8-blue?style=flat)
 [![license: MIT](https://img.shields.io/badge/license-MIT-blue?style=flat)](LICENSE)
 

lib/diff.sh

@@ -1,11 +1,6 @@
 #!/usr/bin/env bash
 # diff.sh — Materialize readable note trees from refs and diff them.
 
-_ref_has_path() {
-  local repo_root="$1" ref="$2" path="$3"
-  git -C "$repo_root" cat-file -e "$ref:$path" 2>/dev/null
-}
-
 _guard_manifest_path() {
   local kind="$1" value="$2"
 
@@ -27,87 +22,222 @@ _guard_manifest_path() {
   esac
 }
 
-_materialize_manifest_for_ref() {
-  local repo_root="$1" notes_dir="$2" ref="$3" out="$4"
-  : > "$out"
-
-  if ! _ref_has_path "$repo_root" "$ref" "$notes_dir/.manifest"; then
-    return 0
-  fi
-
-  git -C "$repo_root" cat-file --filters "$ref:$notes_dir/.manifest" > "$out"
-}
-
-# Materialize a ref's obfuscated notes into readable filenames under dest.
-# Usage: materialize_readable_notes_ref <repo_root> <notes_dir> <ref> <dest>
-materialize_readable_notes_ref() {
-  local repo_root="${1:?usage: materialize_readable_notes_ref <repo_root> <notes_dir> <ref> <dest>}"
-  local notes_dir="${2:?usage: materialize_readable_notes_ref <repo_root> <notes_dir> <ref> <dest>}"
-  local ref="${3:?usage: materialize_readable_notes_ref <repo_root> <notes_dir> <ref> <dest>}"
-  local dest="${4:?usage: materialize_readable_notes_ref <repo_root> <notes_dir> <ref> <dest>}"
-  local manifest
-  manifest=$(mktemp) || return 1
+# Build and validate one ref's readable-name index without materializing note blobs.
+_prepare_readable_notes_ref_index() {
+  local repo_root="$1" notes_dir="$2" ref="$3" manifest_out="$4"
+  local tree_paths tree_ids raw_manifest validated_manifest missing_manifest unmapped_file
+  tree_paths=$(mktemp) || return 1
+  tree_ids=$(mktemp) || { rm -f "$tree_paths"; return 1; }
+  raw_manifest=$(mktemp) || { rm -f "$tree_paths" "$tree_ids"; return 1; }
+  validated_manifest=$(mktemp) || { rm -f "$tree_paths" "$tree_ids" "$raw_manifest"; return 1; }
+  missing_manifest=$(mktemp) || { rm -f "$tree_paths" "$tree_ids" "$raw_manifest" "$validated_manifest"; return 1; }
+  unmapped_file=$(mktemp) || {
+    rm -f "$tree_paths" "$tree_ids" "$raw_manifest" "$validated_manifest" "$missing_manifest"
+    return 1
+  }
+  : > "$manifest_out"
 
   if ! git -C "$repo_root" cat-file -e "$ref^{tree}" 2>/dev/null; then
     echo "Error: not a tree-ish ref: $ref" >&2
-    rm -f "$manifest"
+    rm -f "$tree_paths" "$tree_ids" "$raw_manifest" "$validated_manifest" "$missing_manifest" "$unmapped_file"
+    return 1
+  fi
+  if ! git -C "$repo_root" ls-tree -r -z --name-only "$ref" -- "$notes_dir" > "$tree_paths"; then
+    rm -f "$tree_paths" "$tree_ids" "$raw_manifest" "$validated_manifest" "$missing_manifest" "$unmapped_file"
     return 1
   fi
 
-  mkdir -p "$dest/$notes_dir"
-  local has_manifest=false
-  if _ref_has_path "$repo_root" "$ref" "$notes_dir/.manifest"; then
-    has_manifest=true
+  local tree_path relpath has_manifest=false
+  while IFS= read -r -d '' tree_path; do
+    relpath="${tree_path#"$notes_dir/"}"
+    if [ "$relpath" = ".manifest" ]; then
+      has_manifest=true
+    else
+      printf '%s\n' "$relpath" >> "$tree_ids"
+    fi
+  done < "$tree_paths"
+
+  if ! $has_manifest; then
+    local tree_count=0
+    while IFS= read -r relpath; do
+      [ -z "$relpath" ] && continue
+      tree_count=$((tree_count + 1))
+    done < "$tree_ids"
+    rm -f "$tree_paths" "$tree_ids" "$raw_manifest" "$validated_manifest" "$missing_manifest" "$unmapped_file"
+    if [ "$tree_count" -gt 0 ]; then
+      echo "Error: $ref has $tree_count note file(s) but no $notes_dir/.manifest" >&2
+      return 1
+    fi
+    return 0
   fi
-  if ! _materialize_manifest_for_ref "$repo_root" "$notes_dir" "$ref" "$manifest"; then
-    rm -f "$manifest"
+
+  if ! git -C "$repo_root" cat-file --filters "$ref:$notes_dir/.manifest" > "$raw_manifest"; then
+    rm -f "$tree_paths" "$tree_ids" "$raw_manifest" "$validated_manifest" "$missing_manifest" "$unmapped_file"
     return 1
   fi
 
-  local manifest_ids
-  manifest_ids=$(mktemp) || { rm -f "$manifest"; return 1; }
-  # Failure paths may unlink these temp files before returning; open readers stay valid.
-  # shellcheck disable=SC2094
+  local id manifest_valid=true
   while IFS=$'\t' read -r id relpath; do
     [ -z "$id" ] && continue
-    _guard_manifest_path "id" "$id" || { rm -f "$manifest" "$manifest_ids"; return 1; }
-    _guard_manifest_path "path" "$relpath" || { rm -f "$manifest" "$manifest_ids"; return 1; }
-    printf '%s\n' "$id" >> "$manifest_ids"
-
-    if ! _ref_has_path "$repo_root" "$ref" "$notes_dir/$id"; then
-      echo "Warning: $ref manifest maps $id to $relpath, but $notes_dir/$id is missing" >&2
-      continue
+    if ! _guard_manifest_path "id" "$id" || ! _guard_manifest_path "path" "$relpath"; then
+      manifest_valid=false
+      break
     fi
+    printf '%s\t%s\n' "$id" "$relpath" >> "$validated_manifest"
+  done < "$raw_manifest"
+  if ! $manifest_valid; then
+    rm -f "$tree_paths" "$tree_ids" "$raw_manifest" "$validated_manifest" "$missing_manifest" "$unmapped_file"
+    return 1
+  fi
 
-    mkdir -p "$(dirname "$dest/$notes_dir/$relpath")"
-    if ! git -C "$repo_root" cat-file --filters "$ref:$notes_dir/$id" > "$dest/$notes_dir/$relpath"; then
-      rm -f "$manifest" "$manifest_ids"
-      return 1
-    fi
-  done < "$manifest"
+  awk -F '\t' \
+    -v valid="$manifest_out" \
+    -v missing="$missing_manifest" \
+    -v unmapped="$unmapped_file" '
+      FILENAME == ARGV[1] { tree[$0] = 1; next }
+      {
+        mapped[$1] = 1
+        if ($1 in tree) print $0 > valid
+        else print $0 > missing
+      }
+      END {
+        count = 0
+        for (id in tree) if (!(id in mapped)) count++
+        print count > unmapped
+      }
+    ' "$tree_ids" "$validated_manifest"
 
-  local tree_path relpath unmapped_count=0
-  while IFS= read -r -d '' tree_path; do
-    relpath="${tree_path#"$notes_dir/"}"
-    [ "$relpath" = ".manifest" ] && continue
-    if ! $has_manifest || ! grep -Fxq -- "$relpath" "$manifest_ids"; then
-      unmapped_count=$((unmapped_count + 1))
-    fi
-  done < <(
-    if ! git -C "$repo_root" ls-tree -r -z --name-only "$ref" -- "$notes_dir" 2>/dev/null; then
-      :
-    fi
-  )
+  while IFS=$'\t' read -r id relpath; do
+    [ -z "$id" ] && continue
+    echo "Warning: $ref manifest maps $id to $relpath, but $notes_dir/$id is missing" >&2
+  done < "$missing_manifest"
 
-  rm -f "$manifest" "$manifest_ids"
+  local unmapped_count=0
+  IFS= read -r unmapped_count < "$unmapped_file" || unmapped_count=0
+  rm -f "$tree_paths" "$tree_ids" "$raw_manifest" "$validated_manifest" "$missing_manifest" "$unmapped_file"
   if [ "$unmapped_count" -gt 0 ]; then
-    if $has_manifest; then
-      echo "Error: $ref has $unmapped_count note file(s) not listed in $notes_dir/.manifest" >&2
-    else
-      echo "Error: $ref has $unmapped_count note file(s) but no $notes_dir/.manifest" >&2
+    echo "Error: $ref has $unmapped_count note file(s) not listed in $notes_dir/.manifest" >&2
+    return 1
+  fi
+}
+
+_changed_note_ids_between_refs() {
+  local repo_root="$1" notes_dir="$2" base_ref="$3" head_ref="$4"
+  local base_manifest="$5" head_manifest="$6" out="$7"
+  local changed_paths candidates
+  changed_paths=$(mktemp) || return 1
+  candidates=$(mktemp) || { rm -f "$changed_paths"; return 1; }
+
+  if ! git -C "$repo_root" diff --name-only -z "$base_ref" "$head_ref" -- "$notes_dir" > "$changed_paths"; then
+    rm -f "$changed_paths" "$candidates"
+    return 1
+  fi
+
+  local path id
+  while IFS= read -r -d '' path; do
+    id="${path#"$notes_dir/"}"
+    case "$id" in
+      .manifest|*/*) continue ;;
+    esac
+    [ -n "$id" ] && printf '%s\n' "$id" >> "$candidates"
+  done < "$changed_paths"
+
+  awk -F '\t' '
+    FILENAME == ARGV[1] { selected_id[$1] = 1; next }
+    FILENAME == ARGV[2] {
+      base_row[$0] = 1
+      row_id[$0] = $1
+      edge_id[++edge_count] = $1
+      edge_path[edge_count] = $2
+      next
+    }
+    {
+      head_row[$0] = 1
+      row_id[$0] = $1
+      edge_id[++edge_count] = $1
+      edge_path[edge_count] = $2
+    }
+    END {
+      for (row in base_row) if (!(row in head_row)) selected_id[row_id[row]] = 1
+      for (row in head_row) if (!(row in base_row)) selected_id[row_id[row]] = 1
+
+      # A malformed or merge-produced manifest can alias one ID to multiple
+      # paths, or multiple IDs to one path. Pull in the entire affected
+      # ID/path component so selection preserves full-tree diff semantics.
+      do {
+        expanded = 0
+        for (i = 1; i <= edge_count; i++) {
+          if ((edge_id[i] in selected_id) || (edge_path[i] in selected_path)) {
+            if (!(edge_id[i] in selected_id)) {
+              selected_id[edge_id[i]] = 1
+              expanded = 1
+            }
+            if (!(edge_path[i] in selected_path)) {
+              selected_path[edge_path[i]] = 1
+              expanded = 1
+            }
+          }
+        }
+      } while (expanded)
+
+      for (id in selected_id) print id
+    }
+  ' "$candidates" "$base_manifest" "$head_manifest" > "$out"
+
+  LC_ALL=C sort -u -o "$out" "$out"
+  rm -f "$changed_paths" "$candidates"
+}
+
+_materialize_selected_notes_ref() {
+  local repo_root="$1" notes_dir="$2" ref="$3" manifest="$4" ids="$5" dest="$6"
+  local selected
+  selected=$(mktemp) || return 1
+  mkdir -p "$dest/$notes_dir"
+
+  awk -F '\t' '
+    FILENAME == ARGV[1] { wanted[$1] = 1; next }
+    $1 in wanted { print }
+  ' "$ids" "$manifest" > "$selected"
+
+  local id relpath output_path output_parent materialize_ok=true
+  while IFS=$'\t' read -r id relpath; do
+    [ -z "$id" ] && continue
+    output_path="$dest/$notes_dir/$relpath"
+    output_parent="${output_path%/*}"
+    mkdir -p "$output_parent"
+    if ! git -C "$repo_root" cat-file --filters "$ref:$notes_dir/$id" > "$output_path"; then
+      materialize_ok=false
+      break
     fi
+  done < "$selected"
+
+  rm -f "$selected"
+  $materialize_ok
+}
+
+# Validate both refs and materialize only notes that affect their readable diff.
+materialize_changed_readable_notes_refs() {
+  local repo_root="${1:?usage: materialize_changed_readable_notes_refs <repo_root> <notes_dir> <base_ref> <head_ref> <base_dest> <head_dest>}"
+  local notes_dir="${2:?usage: materialize_changed_readable_notes_refs <repo_root> <notes_dir> <base_ref> <head_ref> <base_dest> <head_dest>}"
+  local base_ref="${3:?usage: materialize_changed_readable_notes_refs <repo_root> <notes_dir> <base_ref> <head_ref> <base_dest> <head_dest>}"
+  local head_ref="${4:?usage: materialize_changed_readable_notes_refs <repo_root> <notes_dir> <base_ref> <head_ref> <base_dest> <head_dest>}"
+  local base_dest="${5:?usage: materialize_changed_readable_notes_refs <repo_root> <notes_dir> <base_ref> <head_ref> <base_dest> <head_dest>}"
+  local head_dest="${6:?usage: materialize_changed_readable_notes_refs <repo_root> <notes_dir> <base_ref> <head_ref> <base_dest> <head_dest>}"
+  local base_manifest head_manifest changed_ids
+  base_manifest=$(mktemp) || return 1
+  head_manifest=$(mktemp) || { rm -f "$base_manifest"; return 1; }
+  changed_ids=$(mktemp) || { rm -f "$base_manifest" "$head_manifest"; return 1; }
+
+  if ! _prepare_readable_notes_ref_index "$repo_root" "$notes_dir" "$base_ref" "$base_manifest" ||
+     ! _prepare_readable_notes_ref_index "$repo_root" "$notes_dir" "$head_ref" "$head_manifest" ||
+     ! _changed_note_ids_between_refs "$repo_root" "$notes_dir" "$base_ref" "$head_ref" "$base_manifest" "$head_manifest" "$changed_ids" ||
+     ! _materialize_selected_notes_ref "$repo_root" "$notes_dir" "$base_ref" "$base_manifest" "$changed_ids" "$base_dest" ||
+     ! _materialize_selected_notes_ref "$repo_root" "$notes_dir" "$head_ref" "$head_manifest" "$changed_ids" "$head_dest"; then
+    rm -f "$base_manifest" "$head_manifest" "$changed_ids"
     return 1
   fi
+
+  rm -f "$base_manifest" "$head_manifest" "$changed_ids"
 }
 
 _copy_tree_contents() {

libexec/test

@@ -183,7 +183,7 @@ run_bats() {
 
   if [ "$effective_jobs" -gt 1 ]; then
     call_arguments+=(--parallel-binary-name "$effective_parallel_command")
-    printf 'BATS parallelism: %s jobs within files via %s\n' \
+    printf 'BATS parallelism: %s jobs across files via %s\n' \
       "$effective_jobs" "$effective_parallel_command"
   else
     printf 'BATS parallelism: serial\n'