feat(ui): Analyze/Refine top-bar flow with incremental refinement

crates/diffcore-core/src/llm/refinement.rs

@@ -713,6 +713,97 @@ pub fn has_refinements(response: &RefinementResponse) -> bool {
         || !response.reclassifications.is_empty()
 }
 
+/// Carry a previously refined grouping onto a fresh analysis for incremental
+/// refinement.
+///
+/// Rebuilds the fresh analysis's groups from the previous refinement's
+/// name+files layout (via the manifest import path, which preserves fresh file
+/// metadata and intra-group edges), dropping files that no longer exist and
+/// routing files new since the previous refinement into the unassigned set —
+/// where the refinement prompt already asks the LLM to place them.
+///
+/// Returns the carried-over analysis and a delta summary for the prompt's
+/// `diff_summary` slot.
+// ponytail: delta is path add/remove only; add per-file change stats if
+// refinement quality on modified-only pushes demands it.
+pub fn carry_over_grouping(
+    fresh: &crate::types::AnalysisOutput,
+    prev_groups: &[FlowGroup],
+    prev_infra: Option<&InfrastructureGroup>,
+) -> (crate::types::AnalysisOutput, String) {
+    let fresh_paths: HashSet<&str> = fresh
+        .groups
+        .iter()
+        .flat_map(|g| g.files.iter().map(|f| f.path.as_str()))
+        .chain(
+            fresh
+                .infrastructure_group
+                .iter()
+                .flat_map(|ig| ig.files.iter().map(String::as_str)),
+        )
+        .collect();
+    let fresh_grouped: HashSet<&str> = fresh
+        .groups
+        .iter()
+        .flat_map(|g| g.files.iter().map(|f| f.path.as_str()))
+        .collect();
+    let prev_paths: HashSet<&str> = prev_groups
+        .iter()
+        .flat_map(|g| g.files.iter().map(|f| f.path.as_str()))
+        .chain(
+            prev_infra
+                .iter()
+                .flat_map(|ig| ig.files.iter().map(String::as_str)),
+        )
+        .collect();
+
+    let added = fresh_paths.difference(&prev_paths).count();
+    let removed = prev_paths.difference(&fresh_paths).count();
+
+    let groups: Vec<crate::manifest::ManifestGroup> = prev_groups
+        .iter()
+        .map(|g| crate::manifest::ManifestGroup {
+            name: g.name.clone(),
+            files: g
+                .files
+                .iter()
+                .map(|f| f.path.clone())
+                .filter(|p| fresh_grouped.contains(p.as_str()))
+                .collect(),
+            review_order: g.review_order,
+            description: None,
+        })
+        .filter(|mg| !mg.files.is_empty())
+        .collect();
+
+    let assigned: HashSet<&str> = groups
+        .iter()
+        .flat_map(|mg| mg.files.iter().map(String::as_str))
+        .collect();
+    let mut unassigned_files: Vec<String> = fresh_paths
+        .difference(&assigned)
+        .map(|p| p.to_string())
+        .collect();
+    unassigned_files.sort();
+
+    let manifest = crate::manifest::GroupsManifest {
+        version: crate::manifest::GroupsManifest::VERSION.to_string(),
+        groups,
+        unassigned_files,
+    };
+    let carried = crate::manifest::import_manifest(fresh, &manifest);
+
+    let delta_summary = format!(
+        "Incremental update to a previously AI-refined grouping. Since the last \
+         refinement: {} files added (listed under Ungrouped), {} files removed. \
+         The groups below already reflect the previous refinement — only adjust \
+         where the new files or removals warrant; otherwise return empty arrays.",
+        added, removed,
+    );
+
+    (carried, delta_summary)
+}
+
 // ── Internal helpers ──
 
 fn remove_file_from_group_or_infra(
@@ -2476,4 +2567,135 @@ mod tests {
         assert!(prompt.contains("infrastructure]"));
         assert!(prompt.contains("NEVER substitute"));
     }
+
+    // ── carry_over_grouping tests ──
+
+    fn make_carry_analysis(
+        groups: Vec<FlowGroup>,
+        infra_files: Vec<&str>,
+    ) -> crate::types::AnalysisOutput {
+        crate::types::AnalysisOutput {
+            version: "1.0.0".to_string(),
+            diff_source: crate::types::DiffSource {
+                diff_type: crate::types::DiffType::BranchComparison,
+                base: None,
+                head: None,
+                base_sha: None,
+                head_sha: Some("head2".to_string()),
+            },
+            summary: crate::types::AnalysisSummary {
+                total_files_changed: 0,
+                total_groups: groups.len() as u32,
+                languages_detected: vec![],
+                frameworks_detected: vec![],
+            },
+            groups,
+            infrastructure_group: if infra_files.is_empty() {
+                None
+            } else {
+                Some(InfrastructureGroup {
+                    files: infra_files.iter().map(|s| s.to_string()).collect(),
+                    sub_groups: vec![],
+                    reason: "test".to_string(),
+                })
+            },
+            annotations: None,
+        }
+    }
+
+    #[test]
+    fn carry_over_keeps_previous_groups_and_routes_new_files_to_unassigned() {
+        let fresh = make_carry_analysis(
+            vec![make_group(
+                "group_1",
+                "fresh grouping",
+                vec![
+                    make_file("a.ts", 0),
+                    make_file("b.ts", 1),
+                    make_file("d.ts", 2),
+                ],
+            )],
+            vec![],
+        );
+        let prev = vec![
+            make_group("group_refined_1", "auth", vec![make_file("a.ts", 0)]),
+            make_group("group_refined_2", "billing", vec![make_file("b.ts", 0)]),
+        ];
+
+        let (carried, delta) = carry_over_grouping(&fresh, &prev, None);
+
+        let names: Vec<&str> = carried.groups.iter().map(|g| g.name.as_str()).collect();
+        assert_eq!(names, vec!["auth", "billing"]);
+        let infra = carried.infrastructure_group.expect("new file goes unassigned");
+        assert_eq!(infra.files, vec!["d.ts"]);
+        assert!(delta.contains("1 files added"));
+        assert!(delta.contains("0 files removed"));
+    }
+
+    #[test]
+    fn carry_over_drops_removed_files_and_prunes_empty_groups() {
+        let fresh = make_carry_analysis(
+            vec![make_group("group_1", "g", vec![make_file("a.ts", 0)])],
+            vec![],
+        );
+        let prev = vec![
+            make_group(
+                "r1",
+                "keep",
+                vec![make_file("a.ts", 0), make_file("gone.ts", 1)],
+            ),
+            make_group("r2", "all gone", vec![make_file("gone2.ts", 0)]),
+        ];
+
+        let (carried, delta) = carry_over_grouping(&fresh, &prev, None);
+
+        assert_eq!(carried.groups.len(), 1);
+        assert_eq!(carried.groups[0].name, "keep");
+        let paths: Vec<&str> = carried.groups[0].files.iter().map(|f| f.path.as_str()).collect();
+        assert_eq!(paths, vec!["a.ts"]);
+        assert!(carried.infrastructure_group.is_none());
+        assert!(delta.contains("2 files removed"));
+    }
+
+    #[test]
+    fn carry_over_preserves_review_order_and_previous_infra_stays_unassigned() {
+        let fresh = make_carry_analysis(
+            vec![make_group(
+                "group_1",
+                "g",
+                vec![make_file("a.ts", 0), make_file("b.ts", 1)],
+            )],
+            vec!["infra.toml"],
+        );
+        let prev_groups = vec![
+            FlowGroup {
+                review_order: 2,
+                ..make_group("r1", "second", vec![make_file("a.ts", 0)])
+            },
+            FlowGroup {
+                review_order: 1,
+                ..make_group("r2", "first", vec![make_file("b.ts", 0)])
+            },
+        ];
+        let prev_infra = InfrastructureGroup {
+            files: vec!["infra.toml".to_string()],
+            sub_groups: vec![],
+            reason: "prev".to_string(),
+        };
+
+        let (carried, delta) =
+            carry_over_grouping(&fresh, &prev_groups, Some(&prev_infra));
+
+        let orders: Vec<(String, u32)> = carried
+            .groups
+            .iter()
+            .map(|g| (g.name.clone(), g.review_order))
+            .collect();
+        assert!(orders.contains(&("second".to_string(), 2)));
+        assert!(orders.contains(&("first".to_string(), 1)));
+        let infra = carried.infrastructure_group.expect("infra file stays unassigned");
+        assert_eq!(infra.files, vec!["infra.toml"]);
+        assert!(delta.contains("0 files added"));
+        assert!(delta.contains("0 files removed"));
+    }
 }

crates/diffcore-core/src/pr_url.rs

@@ -533,6 +533,31 @@ pub fn resolve(pr: &PrUrl) -> Result<ResolvedPr, PrUrlError> {
     resolve_in(pr, &root)
 }
 
+/// SHA of the PR's tip on the remote, via a single `ls-remote` — no clone, no
+/// checkout. Providers that publish only a merge ref (Azure DevOps) report that
+/// ref's SHA instead; it moves whenever either side of the PR does, which is
+/// all a change watcher needs.
+pub fn remote_head_sha(pr: &PrUrl) -> Result<String, PrUrlError> {
+    let glob = pr
+        .ref_glob()
+        .ok_or(PrUrlError::NoGitRefs(pr.provider.name()))?;
+    let listing = git(Path::new("."), &["ls-remote", &pr.clone_url, &glob])?;
+    let pairs: Vec<(&str, &str)> = listing
+        .lines()
+        .filter_map(|l| l.split_once('\t').map(|(sha, r)| (sha.trim(), r.trim())))
+        .collect();
+    let refs: Vec<String> = pairs.iter().map(|(_, r)| r.to_string()).collect();
+    let not_found =
+        || PrUrlError::NotFound(pr.provider.unit(), pr.number, pr.clone_url.clone());
+    let (head_ref, merge_ref) = pr.pick_refs(&refs).ok_or_else(not_found)?;
+    let want = head_ref.or(merge_ref).ok_or_else(not_found)?;
+    pairs
+        .iter()
+        .find(|(_, r)| *r == want)
+        .map(|(sha, _)| sha.to_string())
+        .ok_or_else(not_found)
+}
+
 /// `resolve`, against an explicit cache root. Tests use this so they never have
 /// to mutate the process environment, which races every other thread's getenv.
 pub(crate) fn resolve_in(pr: &PrUrl, root: &Path) -> Result<ResolvedPr, PrUrlError> {
@@ -910,6 +935,24 @@ mod tests {
         }
     }
 
+    #[test]
+    fn remote_head_sha_tracks_the_pr_tip_without_cloning() {
+        with_cache(|root, _cache| {
+            let o = Origin::new(root, "origin");
+            let tip = o.open_pr(1, "feature.txt");
+            let pr = local_pr(&o, 1);
+            assert_eq!(remote_head_sha(&pr).unwrap(), tip);
+
+            // New commits land on the PR branch and the provider republishes
+            // the head ref — the watcher must see the new SHA.
+            o.run(&["checkout", "-q", "pr1"]);
+            let new_tip = o.commit("feature.txt", "more work\n", "more work");
+            o.run(&["checkout", "-q", "main"]);
+            o.run(&["update-ref", "refs/pull/1/head", &new_tip]);
+            assert_eq!(remote_head_sha(&pr).unwrap(), new_tip);
+        });
+    }
+
     #[test]
     fn resolve_uses_the_merge_ref_parent_as_the_target_tip() {
         with_cache(|root, cache| {

crates/diffcore-core/tests/adversarial_refinement.rs

@@ -1187,3 +1187,105 @@ fn adversarial_fixtures_deterministic_consistency() {
         );
     }
 }
+
+/// Incremental refinement: refine, land a new commit, carry the refined
+/// grouping onto the fresh analysis, refine again with the delta summary.
+/// The incremental result must not score materially worse than refining
+/// the updated diff from scratch would have started from.
+#[test]
+fn incremental_refinement_carries_previous_grouping() {
+    if !should_run_live() {
+        eprintln!("Skipping incremental refinement test (set DIFFCORE_RUN_LIVE_LLM_TESTS=1)");
+        return;
+    }
+
+    use diffcore_core::llm::refinement::carry_over_grouping;
+
+    let provider = create_vcr_provider();
+    let (name, builder) = ADVERSARIAL_FIXTURES[0];
+    let (rb, baseline) = builder();
+    let actual_branch = {
+        let repo = git2::Repository::open(rb.path()).unwrap();
+        let branches = repo.branches(Some(git2::BranchType::Local)).unwrap();
+        let mut found = None;
+        for branch in branches {
+            let (b, _) = branch.unwrap();
+            let bname = b.name().unwrap().unwrap().to_string();
+            if bname != "main" {
+                found = Some(bname);
+                break;
+            }
+        }
+        found.expect("Should have a feature branch")
+    };
+
+    // 1. Deterministic analysis + from-scratch refinement.
+    let det = run_full_pipeline(&rb, "main", &actual_branch);
+    let (scratch, _) = apply_refinement_to_output(&det, provider.as_ref(), &vcr_cache_dir());
+    let scratch_scores = score_output(&scratch, &baseline);
+    eprintln!("  {}: from-scratch refined overall={:.4}", name, scratch_scores.overall);
+
+    // 2. A new commit lands on the feature branch after the refinement.
+    rb.checkout(&actual_branch);
+    rb.write_file("src/modules/late_addition.ts", "export const late = 1;\n");
+    rb.commit("late addition");
+
+    // 3. Fresh deterministic analysis, previous refinement carried onto it.
+    let fresh = run_full_pipeline(&rb, "main", &actual_branch);
+    let (carried, delta_summary) =
+        carry_over_grouping(&fresh, &scratch.groups, scratch.infrastructure_group.as_ref());
+    assert!(
+        carried
+            .infrastructure_group
+            .as_ref()
+            .is_some_and(|ig| ig.files.iter().any(|f| f.contains("late_addition"))),
+        "new file must land in the unassigned set for the LLM to place"
+    );
+    assert!(delta_summary.contains("Incremental update"));
+
+    // 4. Incremental refine: same op machinery, delta summary in diff_summary.
+    let analysis_json = serde_json::to_string_pretty(&carried).unwrap();
+    let request = build_refinement_request(
+        &carried.groups,
+        carried.infrastructure_group.as_ref(),
+        &analysis_json,
+        &delta_summary,
+    );
+    let rt = tokio::runtime::Runtime::new().unwrap();
+    let incremental = match rt.block_on(provider.refine_groups(&request)) {
+        Ok(ref r) if has_refinements(r) => {
+            let (groups, infra, warnings) = apply_refinement_lenient(
+                &carried.groups,
+                carried.infrastructure_group.as_ref(),
+                r,
+            );
+            for w in &warnings {
+                eprintln!("  Refinement repair: {}", w.message);
+            }
+            let mut out = carried.clone();
+            out.groups = groups;
+            out.infrastructure_group = infra;
+            out.summary.total_groups = out.groups.len() as u32;
+            out
+        }
+        Ok(_) => carried.clone(),
+        Err(e) => {
+            eprintln!("  LLM incremental refinement call failed: {}", e);
+            carried.clone()
+        }
+    };
+
+    let inc_scores = score_output(&incremental, &baseline);
+    eprintln!(
+        "  {}: incremental refined overall={:.4} (delta {:+.4})",
+        name,
+        inc_scores.overall,
+        inc_scores.overall - scratch_scores.overall
+    );
+    assert!(
+        inc_scores.overall >= scratch_scores.overall - 0.20,
+        "incremental refinement degraded too far: incremental={:.4}, from-scratch={:.4}",
+        inc_scores.overall,
+        scratch_scores.overall
+    );
+}