From 1e4834373bb9aada8a4af4f80188b4e4d6a41a01 Mon Sep 17 00:00:00 2001 From: jsture Date: Wed, 10 Jun 2026 11:41:27 +0200 Subject: [PATCH 1/7] Fix breakage left behind by spectral/ann/dists simplification The last refactor commits removed topo.base.dists.pairwise_distances, topo.spectral.eigen.spectral_layout, kNN's return_instance/random_state kwargs and the graph_kernel assignment, but left callers and state checks behind, so import and fit() were broken. - kernels.py: delegate to sklearn.metrics.pairwise_distances - spectral/eigen.py: import Kernel lazily inside fit() to break the kernels <-> spectral.eigen circular import - projector.py: spectral init via EigenDecomposition with random fallback; silence pyright on the vendored pymde/torch glue - _pipeline + uom + eval + intrinsic_dim: drop removed kNN kwargs - topograph.py: remove the dead graph_kernel attribute and its fitted-state check (operators are already exposed as P_Z_/K_Z_/P_msZ_/K_msZ_) - uom.py: set P_Z_/P_msZ_ so the UoM path passes the fitted-state check; annotate uom_eigenvalues_*_list for mypy/pyright - update tests for the current APIs (LE instead of spectral_layout, standalone find_ideal_projection, new error messages, mixin contracts) Co-Authored-By: Claude Fable 5 --- src/topo/_pipeline/eigen.py | 3 -- src/topo/_pipeline/graph.py | 1 - src/topo/_pipeline/layout.py | 2 - src/topo/eval/local_scores.py | 2 - src/topo/layouts/projector.py | 43 ++++++++------------ src/topo/spectral/eigen.py | 5 ++- src/topo/topograph.py | 4 -- src/topo/tpgraph/intrinsic_dim.py | 2 - src/topo/tpgraph/kernels.py | 6 +-- src/topo/uom.py | 9 ++-- tests/topo/_pipeline/test_pipeline_mixins.py | 16 ++++++-- tests/topo/test_analysis.py | 14 ++++--- tests/topo/test_topograph.py | 13 ++++-- tests/topo/utils/test_utils.py | 2 +- 14 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/topo/_pipeline/eigen.py b/src/topo/_pipeline/eigen.py index 6a9aa86f..50956c76 100644 --- a/src/topo/_pipeline/eigen.py +++ b/src/topo/_pipeline/eigen.py @@ -83,7 +83,6 @@ class EigenBuildMixin: graph_metric: str graph_kernel_version: str low_memory: bool - graph_kernel: Kernel | None knn_Z_: csr_matrix | None knn_msZ_: csr_matrix | None P_Z_: csr_matrix | None @@ -273,7 +272,6 @@ def _build_scaffold_graphs( metric=self.graph_metric, n_jobs=self._n_jobs_effective, backend=self._backend_resolved, - return_instance=False, verbose=self.bases_graph_verbose, ) self.runtimes["kNN_msZ"] = time.time() - t0 @@ -289,7 +287,6 @@ def _build_scaffold_graphs( metric=self.graph_metric, n_jobs=self._n_jobs_effective, backend=self._backend_resolved, - return_instance=False, verbose=self.bases_graph_verbose, ) self.runtimes["kNN_Z"] = time.time() - t0 diff --git a/src/topo/_pipeline/graph.py b/src/topo/_pipeline/graph.py index 4ef6913a..7997b5b5 100644 --- a/src/topo/_pipeline/graph.py +++ b/src/topo/_pipeline/graph.py @@ -104,7 +104,6 @@ def _build_base_graph(self, X: np.ndarray | csr_matrix | None) -> None: metric=self.base_metric, n_jobs=self._n_jobs_effective, backend=self._backend_resolved, - return_instance=False, verbose=self.bases_graph_verbose, ) self.runtimes["kNN_X"] = time.time() - t0 diff --git a/src/topo/_pipeline/layout.py b/src/topo/_pipeline/layout.py index 37be60ab..de23675b 100644 --- a/src/topo/_pipeline/layout.py +++ b/src/topo/_pipeline/layout.py @@ -64,8 +64,6 @@ class LayoutBuildMixin: msTopoMAP_snapshots: list[dict[str, Any]] TopoMAP_snapshots: list[dict[str, Any]] - uom_eigenvalues_ms_list: list[np.ndarray] - uom_eigenvalues_dm_list: list[np.ndarray] uom_components_: list[np.ndarray] | None _uom_active_mode: str uom_enabled: bool diff --git a/src/topo/eval/local_scores.py b/src/topo/eval/local_scores.py index 83a662cc..4beeacf7 100755 --- a/src/topo/eval/local_scores.py +++ b/src/topo/eval/local_scores.py @@ -308,7 +308,6 @@ def geodesic_correlation( n_neighbors=n_neighbors, metric=metric, n_jobs=n_jobs, - return_instance=False, verbose=False, **kwargs, ) @@ -320,7 +319,6 @@ def geodesic_correlation( n_neighbors=n_neighbors, metric=metric, n_jobs=n_jobs, - return_instance=False, verbose=False, **kwargs, ) diff --git a/src/topo/layouts/projector.py b/src/topo/layouts/projector.py index d6546134..05767c11 100755 --- a/src/topo/layouts/projector.py +++ b/src/topo/layouts/projector.py @@ -23,7 +23,7 @@ from topo.base.ann import kNN from topo.layouts.isomap import Isomap from topo.layouts.map import fuzzy_embedding -from topo.spectral.eigen import spectral_layout +from topo.spectral.eigen import EigenDecomposition from topo.tpgraph.kernels import Kernel from topo.utils._utils import get_landmark_indices @@ -317,25 +317,16 @@ def fit(self, X: np.ndarray | csr_matrix | Kernel, **kwargs: Any) -> "Projector" if self.init == "spectral": try: self.init_Y_ = np.asarray( - spectral_layout( - K, - self.n_components, - self.random_state, - laplacian_type="random_walk", - eigen_tol=10e-4, - return_evals=False, - ) + EigenDecomposition( + n_components=self.n_components + ).fit_transform(K) ) except Exception: warnings.warn( - "Multicomponent spectral layout initialization failed, falling back to simple spectral layout..." + "Spectral layout initialization failed, falling back to random initialization..." ) - from topo.spectral.eigen import EigenDecomposition - - self.init_Y_ = np.asarray( - EigenDecomposition( - n_components=self.n_components - ).fit_transform(K) + self.init_Y_ = self.random_state.randn( + _n_rows(K, "projection graph"), self.n_components ) else: self.init_Y_ = self.random_state.randn( @@ -771,7 +762,7 @@ def IsomorphicMDE( f"Computing {n_neighbors}-nearest neighbors, with " f"max_distance={max_distance}" ) - knn_graph = preprocess.generic.k_nearest_neighbors( + knn_graph = preprocess.generic.k_nearest_neighbors( # pyright: ignore[reportAttributeAccessIssue] data, k=n_neighbors, max_distance=max_distance, @@ -796,7 +787,7 @@ def IsomorphicMDE( if not isinstance( constraint, (constraints._Centered, constraints._Standardized) ): - constraint.project_onto_constraint(X_init, inplace=True) + constraint.project_onto_constraint(X_init, inplace=True) # pyright: ignore[reportArgumentType] elif init == "random": X_init = constraint.initialization(n, embedding_dim, device) else: @@ -823,8 +814,10 @@ def IsomorphicMDE( device ) - negative_weights = -torch.ones( - negative_edges.shape[0], dtype=X_init.dtype, device=device + negative_weights = -torch.ones( # pyright: ignore[reportCallIssue] + negative_edges.shape[0], + dtype=X_init.dtype, # pyright: ignore[reportArgumentType] + device=device, ) if isinstance(constraint, constraints.Anchored): @@ -838,7 +831,7 @@ def IsomorphicMDE( f = penalties.PushAndPull( weights, attractive_penalty=attractive_penalty, - repulsive_penalty=repulsive_penalty, + repulsive_penalty=repulsive_penalty, # pyright: ignore[reportArgumentType] ) else: f = attractive_penalty(weights) @@ -853,7 +846,7 @@ def IsomorphicMDE( constraint=constraint, device=device, ) - mde._X_init = X_init + mde._X_init = X_init # pyright: ignore[reportArgumentType] # Won't need to cache the graph - we have already computed it and cached with TopoMetry @@ -863,10 +856,10 @@ def IsomorphicMDE( # non-differentiable average distortion. perturb the initialization to # mitigate. x_init = mde._X_init - mde._X_init += 1e-4 * torch.randn( + mde._X_init += 1e-4 * torch.randn( # pyright: ignore[reportCallIssue] x_init.shape, device=x_init.device, - dtype=x_init.dtype, + dtype=x_init.dtype, # pyright: ignore[reportArgumentType] ) return mde @@ -965,7 +958,7 @@ def IsometricMDE( edges = data.edges.to(device) deviations = data.distances.to(device) else: - graph = preprocess.generic.distances( + graph = preprocess.generic.distances( # pyright: ignore[reportAttributeAccessIssue] data, retain_fraction=retain_fraction, verbose=verbose ) edges = graph.edges.to(device) diff --git a/src/topo/spectral/eigen.py b/src/topo/spectral/eigen.py index 4d02c8a8..a5c5d0c3 100755 --- a/src/topo/spectral/eigen.py +++ b/src/topo/spectral/eigen.py @@ -14,7 +14,6 @@ from scipy.sparse.linalg import ArpackError, eigsh from topo.spectral._spectral import diffusion_operator, graph_laplacian -from topo.tpgraph.kernels import Kernel EIGEN_SOLVERS = {"auto", "dense", "arpack"} @@ -337,6 +336,10 @@ def fit(self, X): largest = True else: largest = False + # Imported here to avoid a circular import: kernels.py depends on + # topo.spectral at module level. + from topo.tpgraph.kernels import Kernel + target: Any = None if isinstance(X, Kernel): self.N = X.N diff --git a/src/topo/topograph.py b/src/topo/topograph.py index f2097dc6..01f6f826 100644 --- a/src/topo/topograph.py +++ b/src/topo/topograph.py @@ -318,7 +318,6 @@ def __init__( self.K_Z_: csr_matrix | None = None self.K_msZ_: csr_matrix | None = None self.eigenbasis: Any = None - self.graph_kernel: Kernel | None = None self.SpecLayout: np.ndarray | None = None self.global_dimensionality: int | float | None = None self.local_dimensionality: np.ndarray | None = None @@ -689,9 +688,6 @@ def _check_fitted_pipeline_state(self) -> None: if self.eigenbasis is None and not self.uom_enabled: raise RuntimeError("fit() completed without an active eigenbasis.") - if self.graph_kernel is None and not self.uom_enabled: - raise RuntimeError("fit() completed without an active graph_kernel.") - # ------------------------------------------------------------------ # Scaffold access # ------------------------------------------------------------------ diff --git a/src/topo/tpgraph/intrinsic_dim.py b/src/topo/tpgraph/intrinsic_dim.py index c23c9864..eb6309ba 100644 --- a/src/topo/tpgraph/intrinsic_dim.py +++ b/src/topo/tpgraph/intrinsic_dim.py @@ -464,7 +464,6 @@ def _cap_int(v, lo, hi): n_neighbors=k_eff, metric=metric, backend=backend, - random_state=random_state, **knn_kwargs, ) d_local = fsa_local(K, n_neighbors=k_eff) @@ -506,7 +505,6 @@ def _cap_int(v, lo, hi): n_neighbors=k_int, metric=metric, backend=backend, - random_state=random_state, **knn_kwargs, ) local = np.asarray(mle_local(K, n_neighbors=k_int), dtype=float) diff --git a/src/topo/tpgraph/kernels.py b/src/topo/tpgraph/kernels.py index eb25e6c4..a47f3322 100755 --- a/src/topo/tpgraph/kernels.py +++ b/src/topo/tpgraph/kernels.py @@ -33,12 +33,12 @@ ) from scipy.spatial import procrustes from sklearn.base import BaseEstimator, TransformerMixin +from sklearn.metrics import pairwise_distances from sklearn.preprocessing import normalize as _l2_normalize_rows from sklearn.utils import check_random_state from topo._compat.umap import fuzzy_graph_from_knn from topo.base.ann import kNN -from topo.base.dists import pairwise_distances from topo.base.graph_matrix import get_indices_distances_from_sparse_matrix from topo.spectral._spectral import degree as compute_degree from topo.spectral._spectral import diffusion_operator, graph_laplacian @@ -246,7 +246,7 @@ def _compute_knn_distance_graph( ) -> csr_matrix: """Compute KNN or pairwise distance graph.""" if pairwise: - K_dense = pairwise_distances(X_prep, metric) + K_dense = pairwise_distances(X_prep, metric=metric) K = csr_matrix(K_dense) else: K = kNN( @@ -572,7 +572,7 @@ def compute_kernel( K = _as_csr_matrix(X) K = _sanitize_sparse_data(K) expand_nbr_search = False - dens_dict = {} + dens_dict: dict = {} else: X_for_knn = _prepare_knn_input(X, metric, backend, pairwise) K = _compute_knn_distance_graph( diff --git a/src/topo/uom.py b/src/topo/uom.py index 2cd6493c..f3df1ed9 100644 --- a/src/topo/uom.py +++ b/src/topo/uom.py @@ -577,8 +577,8 @@ def _init_uom_state(self) -> None: self.uom_BaseKernel_list = None self.uom_DMEig_list = None self.uom_msDMEig_list = None - self.uom_eigenvalues_dm_list = None - self.uom_eigenvalues_ms_list = None + self.uom_eigenvalues_dm_list: list[np.ndarray] | None = None + self.uom_eigenvalues_ms_list: list[np.ndarray] | None = None self._uom_active_mode = "msDM" self.uom_Z_list = None self.uom_msZ_list = None @@ -698,7 +698,6 @@ def _fit_uom(self, X): metric=self.base_metric, n_jobs=self._n_jobs_effective, backend=self._backend_resolved, - return_instance=False, verbose=False, ) knn_i = as_float32_csr(knn_i, "knn_i") @@ -783,7 +782,6 @@ def _fit_uom(self, X): metric=self.graph_metric, n_jobs=self._n_jobs_effective, backend=self._backend_resolved, - return_instance=False, verbose=False, ) ) @@ -794,7 +792,6 @@ def _fit_uom(self, X): metric=self.graph_metric, n_jobs=self._n_jobs_effective, backend=self._backend_resolved, - return_instance=False, verbose=False, ) ) @@ -1113,6 +1110,8 @@ def _aggregate_uom_blocks(self) -> None: self.knn_Z_ = self.knn_Z_uom self.knn_msZ_ = self.knn_msZ_uom self.P_X_ = csr_matrix(self.P_of_X_uom) + self.P_Z_ = csr_matrix(self.P_of_Z_uom) + self.P_msZ_ = csr_matrix(self.P_of_msZ_uom) self.K_Z_ = csr_matrix(self.P_of_Z_uom) self.K_msZ_ = csr_matrix(self.P_of_msZ_uom) diff --git a/tests/topo/_pipeline/test_pipeline_mixins.py b/tests/topo/_pipeline/test_pipeline_mixins.py index f708fb20..8a31b4e4 100644 --- a/tests/topo/_pipeline/test_pipeline_mixins.py +++ b/tests/topo/_pipeline/test_pipeline_mixins.py @@ -37,6 +37,10 @@ def __init__(self): self.build_kernel_calls = [] self.dummy_kernel = cast(Kernel, DummyFittedKernel()) + def _build_kernel(self, graph, n_neighbors, version, **kwargs): + self.build_kernel_calls.append((graph, n_neighbors, version, {})) + return self.dummy_kernel + class DummyEigenBuilder(EigenBuildMixin): def __init__(self): @@ -95,6 +99,10 @@ def __init__(self): self.uom_components_ = None self.eigenbasis = None self.base_kernel = None + self.K_Z_ = None + self.K_msZ_ = None + self.P_Z_ = None + self.P_msZ_ = None def test_graph_build_base_graph_accepts_precomputed_matrix(): @@ -160,13 +168,13 @@ def fake_sizing(*args, **kwargs): def test_layout_get_projection_standard_and_uom_keys(): layout = DummyLayoutBuilder() - standard_key = "MAP of gk from msDM with bk" + standard_key = "MAP of msDM" layout.ProjectionDict[standard_key] = np.ones((3, 2)) assert layout._get_projection("MAP", multiscale=True).shape == (3, 2) layout.ProjectionDict.clear() - uom_key = "t-SNE of UoM DM with bk" - layout.ProjectionDict[uom_key] = np.zeros((3, 2)) + dm_key = "t-SNE of DM" + layout.ProjectionDict[dm_key] = np.zeros((3, 2)) assert layout._get_projection("t-SNE", multiscale=False).shape == (3, 2) with pytest.raises(AttributeError, match="embedding unavailable"): @@ -175,7 +183,7 @@ def test_layout_get_projection_standard_and_uom_keys(): def test_layout_spectral_layout_requires_graph(): layout = DummyLayoutBuilder() - with pytest.raises(ValueError, match="No graph kernel"): + with pytest.raises(AttributeError, match="refined affinity unavailable"): layout.spectral_layout() diff --git a/tests/topo/test_analysis.py b/tests/topo/test_analysis.py index d3883021..19f144dc 100644 --- a/tests/topo/test_analysis.py +++ b/tests/topo/test_analysis.py @@ -4,7 +4,7 @@ from scipy import sparse from topo import analysis -from topo.spectral.eigen import spectral_layout +from topo.spectral import LE class TestFilterSignal: @@ -103,10 +103,14 @@ def test_disconnected_graph_returns_one_row_per_vertex(self): format="csr", ) - embedding = spectral_layout( - graph=graph, - dim=2, - random_state=42, + embedding = np.asarray( + LE( + graph, + n_eigs=2, + laplacian_type="normalized", + drop_first=True, + return_evals=False, + ) ) assert isinstance(embedding, np.ndarray) diff --git a/tests/topo/test_topograph.py b/tests/topo/test_topograph.py index 682468a3..01baa37e 100644 --- a/tests/topo/test_topograph.py +++ b/tests/topo/test_topograph.py @@ -82,8 +82,11 @@ def test_scaffolds_shape(self, fitted_topograph, swiss_roll_data): def test_base_kernel_exists(self, fitted_topograph): assert fitted_topograph.base_kernel is not None - def test_graph_kernel_exists(self, fitted_topograph): - assert fitted_topograph.graph_kernel is not None + def test_graph_kernel_operators_exist(self, fitted_topograph): + assert fitted_topograph.P_msZ_ is not None + assert fitted_topograph.K_msZ_ is not None + assert fitted_topograph.P_Z_ is not None + assert fitted_topograph.K_Z_ is not None def test_intrinsic_dim_estimated(self, fitted_topograph): assert fitted_topograph.global_id is not None @@ -244,13 +247,15 @@ def test_spectral_selectivity_smooths_with_named_operator(self, fitted_topograph assert "smoothed_EAS" in fitted_topograph.LocalScoresDict def test_find_ideal_projection_runs(self, fitted_topograph): + from topo.layouts.diagnostics import find_ideal_projection + # A very minimal grid search to test the machinery - res = fitted_topograph.find_ideal_projection( + res = find_ideal_projection( + fitted_topograph, min_dist_grid=[0.1], spread_grid=[1.0], initial_alpha_grid=[1.0], num_iters=10, - verbosity=0, ) assert "best_params" in res assert "best_score" in res diff --git a/tests/topo/utils/test_utils.py b/tests/topo/utils/test_utils.py index f0f1cc51..80ee5dad 100644 --- a/tests/topo/utils/test_utils.py +++ b/tests/topo/utils/test_utils.py @@ -67,7 +67,7 @@ def test_sparse_knn_matrix_roundtrip_helpers(self): def test_sparse_knn_matrix_requires_enough_neighbors(self): graph = sparse.csr_matrix([[0.0, 1.0], [0.0, 0.0]]) - with pytest.raises(ValueError, match="fewer than n_neighbors"): + with pytest.raises(ValueError, match="expected at least"): get_indices_distances_from_sparse_matrix(graph, n_neighbors=2) From 2ed36fae1b43447c77938a771b02612bfb5f0c02 Mon Sep 17 00:00:00 2001 From: jsture Date: Wed, 10 Jun 2026 11:42:06 +0200 Subject: [PATCH 2/7] Keep zero-distance duplicate neighbors in kNN graphs Duplicate (or float32-identical) points produce zero-distance edges that eliminate_zeros() silently dropped from CSR kNN graphs, leaving rows with fewer than k neighbors and breaking downstream consumers, e.g. the CkNN candidate search ("Row 1 contains 45 distances, expected at least 46"). Clamp genuine off-diagonal zero distances to float32 tiny so they survive sparse storage and later float32 casts, while self-loops are still eliminated. Applies to both the sklearn and HNSWlib graph builders. Also update the kNN tests for the simplified API (return_instance is gone) and add a duplicate-point regression test for both backends. Co-Authored-By: Claude Fable 5 --- src/topo/base/ann.py | 26 +++++++++++++++++++++++++- tests/topo/base/test_ann.py | 37 +++++++++++++++++++++++++++++++++---- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/topo/base/ann.py b/src/topo/base/ann.py index 06150281..3ccef6ac 100755 --- a/src/topo/base/ann.py +++ b/src/topo/base/ann.py @@ -31,6 +31,11 @@ logger = logging.getLogger(__name__) +# Clamp value for genuine zero-distance neighbor edges (duplicate points). +# float32 tiny survives both float32 and float64 sparse-data round-trips, +# unlike float64 tiny which underflows to 0.0 when graphs are cast to float32. +_ZERO_DISTANCE_TINY = float(np.finfo(np.float32).tiny) + def _resolve_n_jobs(n_jobs: int | str | None) -> int: """Resolve sklearn/joblib-style n_jobs.""" @@ -191,8 +196,24 @@ def _build_sparse_knn_graph( k = int(indices.shape[1]) indptr = np.arange(0, n_query_samples * k + 1, k, dtype=np.int64) + vals = distances.ravel() + if not np.issubdtype(vals.dtype, np.floating): + vals = vals.astype(np.float64) + + # Genuine zero-distance neighbors (duplicate points) must survive CSR + # storage; clamp off-diagonal zeros to the smallest positive float so + # eliminate_zeros() below only drops self-loops. + cols = indices.ravel() + rows = np.repeat(np.arange(n_query_samples), k) + off_diagonal = ( + rows != cols + if n_query_samples == n_fit_samples + else np.ones(vals.shape, dtype=bool) + ) + vals = np.where((vals <= 0) & off_diagonal, _ZERO_DISTANCE_TINY, vals) + graph = csr_matrix( - (distances.ravel(), indices.ravel(), indptr), + (vals, cols, indptr), shape=(n_query_samples, n_fit_samples), ) graph.eliminate_zeros() @@ -235,6 +256,9 @@ def _sklearn_knn_graph( ), "sklearn kneighbors_graph output", ) + # Keep genuine zero-distance neighbors (duplicate points): clamp to a + # tiny positive float so eliminate_zeros() only drops self-loops. + graph.data = np.where(graph.data <= 0, _ZERO_DISTANCE_TINY, graph.data) graph.setdiag(0.0) graph.eliminate_zeros() return graph diff --git a/tests/topo/base/test_ann.py b/tests/topo/base/test_ann.py index 9419a2b9..a296af1d 100644 --- a/tests/topo/base/test_ann.py +++ b/tests/topo/base/test_ann.py @@ -3,6 +3,7 @@ import numpy as np import pytest from scipy.sparse import csr_matrix +from sklearn.metrics import pairwise_distances from topo.base.ann import HNSWlibTransformer, _resolve_n_jobs, kNN @@ -16,21 +17,49 @@ def test_kNN_sklearn(): def test_kNN_sklearn_direct_kneighbors_graph_path(): X = np.random.default_rng(0).normal(size=(30, 3)) - graph = kNN(X, n_neighbors=5, backend="sklearn", return_instance=False) + graph = kNN(X, n_neighbors=5, backend="sklearn") assert isinstance(graph, csr_matrix) assert graph.shape == (30, 30) assert graph.nnz == 30 * 5 assert graph.format == "csr" -def test_kNN_sklearn_return_instance_with_precomputed(): +def test_kNN_sklearn_precomputed(): X = np.random.default_rng(1).normal(size=(20, 3)) - nbrs, graph = kNN(X, n_neighbors=5, backend="sklearn", return_instance=True) - assert hasattr(nbrs, "kneighbors") + D = pairwise_distances(X) + graph = kNN(D, n_neighbors=5, metric="precomputed", backend="sklearn") assert isinstance(graph, csr_matrix) assert graph.shape == (20, 20) +def test_kNN_rejects_unknown_kwargs(): + X = np.random.default_rng(2).normal(size=(20, 3)) + with pytest.raises(TypeError, match="Unexpected kNN keyword"): + kNN(X, n_neighbors=5, backend="sklearn", return_instance=True) + + +def test_knn_keeps_zero_distance_duplicate_neighbors(): + """Duplicate points must not shorten kNN graph rows (regression). + + Zero-distance edges used to be dropped by eliminate_zeros(), which broke + downstream consumers that require exactly k neighbors per row. + """ + rng = np.random.default_rng(7) + X = rng.normal(size=(20, 3)) + X[1] = X[0] # exact duplicate -> zero distance between rows 0 and 1 + k = 4 + + for backend in ["sklearn", "hnswlib"]: + if backend == "hnswlib": + pytest.importorskip("hnswlib") + G = kNN(X, n_neighbors=k, backend=backend) + row_counts = np.diff(G.indptr) + assert np.all(row_counts == k), ( + f"{backend}: expected {k} neighbors per row, got {np.unique(row_counts)}" + ) + assert np.all(G.data > 0) + + def test_resolve_n_jobs(): assert _resolve_n_jobs(None) == 1 assert _resolve_n_jobs("2") == 2 From d284215f607eee9267fb82e8e8138852d3fcb017 Mon Sep 17 00:00:00 2001 From: jsture Date: Wed, 10 Jun 2026 11:42:19 +0200 Subject: [PATCH 3/7] Delegate euclidean_grad to umap-learn and drop dists.py The only remaining consumer of topo.base.dists was the euclidean_grad default in graph_utils.simplicial_set_embedding. umap-learn ships an identical numba-jitted implementation (umap.distances.euclidean_grad), so use that and delete the module. Co-Authored-By: Claude Fable 5 --- src/topo/base/dists.py | 26 -------------------------- src/topo/layouts/graph_utils.py | 4 ++-- 2 files changed, 2 insertions(+), 28 deletions(-) delete mode 100644 src/topo/base/dists.py diff --git a/src/topo/base/dists.py b/src/topo/base/dists.py deleted file mode 100644 index c8838687..00000000 --- a/src/topo/base/dists.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Distance gradients used by layout optimization.""" - -import importlib.util - -import numpy as np - -_have_numba = importlib.util.find_spec("numba") is not None - -if _have_numba: - from numba import njit # type: ignore[reportMissingImports] -else: - - def njit(*_args, **_kwargs): # noqa: D103 - def _decorator(func): - return func - - return _decorator - - -@njit(fastmath=True) -def euclidean_grad(x: np.ndarray, y: np.ndarray) -> tuple[float, np.ndarray]: - """Euclidean distance and gradient with respect to x.""" - diff = x - y - dist = np.sqrt(np.dot(diff, diff)) - grad = diff / (1e-8 + dist) - return dist, grad diff --git a/src/topo/layouts/graph_utils.py b/src/topo/layouts/graph_utils.py index 1f6ab87b..760629a5 100755 --- a/src/topo/layouts/graph_utils.py +++ b/src/topo/layouts/graph_utils.py @@ -10,11 +10,11 @@ import numpy as np from sklearn.neighbors import KDTree +from umap.distances import euclidean_grad from umap.layouts import optimize_layout_euclidean, optimize_layout_generic from umap.umap_ import make_epochs_per_sample from topo._compat.umap import find_umap_ab_params, fuzzy_graph_from_data -from topo.base import dists as dist from topo.spectral import LE find_ab_params = find_umap_ab_params @@ -79,7 +79,7 @@ def simplicial_set_embedding( densmap, densmap_kwds=None, output_dens=False, - output_metric=dist.euclidean_grad, + output_metric=euclidean_grad, output_metric_kwds=None, euclidean_output=True, parallel=True, From d464f3e0b07e4b1f981ff34d39c138099330bc45 Mon Sep 17 00:00:00 2001 From: jsture Date: Wed, 10 Jun 2026 11:42:30 +0200 Subject: [PATCH 4/7] Clean up packaging: runtime deps, extras and stale config - move ipykernel/ipywidgets/jupyterlab from runtime dependencies to the dev group; the library itself never imports them - stop requiring hnswlib and pacmap at runtime; they were both required and offered as extras, and the code already guards them via _optional - fix the optional-dependency hint for hnswlib to name the actual extra ([ann], not the nonexistent [hnswlib]) - drop ruff per-file-ignores pointing at deleted files - remove the unused uv-init main.py stub (no console script references it) Co-Authored-By: Claude Fable 5 --- main.py | 10 ---------- pyproject.toml | 16 ++++------------ src/topo/_optional.py | 4 ++-- uv.lock | 20 ++++++++------------ 4 files changed, 14 insertions(+), 36 deletions(-) delete mode 100644 main.py diff --git a/main.py b/main.py deleted file mode 100644 index 1b8d09c6..00000000 --- a/main.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Command-line entry point for topometry-nosc.""" - - -def main(): - """Run the topometry-nosc command-line entry point.""" - print("Hello from topometry-nosc!") - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index 7f7698c8..a2a39172 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,18 +55,13 @@ dependencies = [ "umap-learn>=0.5.8", "matplotlib>=3.6", "pandas>=1.5", - "ipykernel", - "ipywidgets", - "jupyterlab", - "hnswlib>=0.8.0", - "pacmap>=0.9.1", ] [project.optional-dependencies] ann = ["hnswlib>=0.8.0"] amg = ["pyamg>=5.3.0"] layouts = [ - "pacmap>=0.7", + "pacmap>=0.9.1", "pymde>=0.1", "trimap>=1.1", ] @@ -81,6 +76,9 @@ Changelog = "https://github.com/HauserGroup/topometryNoSC/blob/master/CHANGELOG. [dependency-groups] dev = [ + "ipykernel", + "ipywidgets", + "jupyterlab", "jupytext>=1.19.3", "mypy>=1.13", "nbstripout>=0.9.1", @@ -147,12 +145,6 @@ convention = "numpy" [tool.ruff.lint.per-file-ignores] "tests/*" = ["B", "D"] -# numba-jitted internals: agreed bar is a concise one-line summary, not full -# NumPy sections. Keep presence rules (D1xx); drop the section-content rules. -"src/topo/base/sparse.py" = ["D205", "D400", "D401", "D415"] -"src/topo/base/dists.py" = ["D205", "D400", "D401", "D415"] -"src/topo/spectral/umap_layouts.py" = ["D205", "D400", "D401", "D415"] -"src/topo/utils/umap_utils.py" = ["D205", "D400", "D401", "D415"] [tool.mypy] python_version = "3.10" diff --git a/src/topo/_optional.py b/src/topo/_optional.py index 8abf1d9e..b46af74b 100644 --- a/src/topo/_optional.py +++ b/src/topo/_optional.py @@ -1,7 +1,7 @@ """Centralised handling of optional dependencies. The core package depends on the numerical stack, plotting (matplotlib), -dataframe I/O (pandas), Jupyter, and ``umap-learn``. Everything else—the AMG +dataframe I/O (pandas), and ``umap-learn``. Everything else—the AMG eigensolver, the HNSWlib approximate-nearest-neighbour backend, and third-party layout libraries—is optional and gated through the helpers in this module so that: @@ -23,7 +23,7 @@ _EXTRA_FOR: dict[str, str] = { "pyamg": "amg", - "hnswlib": "hnswlib", + "hnswlib": "ann", "pacmap": "layouts", "pymde": "layouts", "trimap": "layouts", diff --git a/uv.lock b/uv.lock index a4774839..41e43d78 100644 --- a/uv.lock +++ b/uv.lock @@ -4187,16 +4187,11 @@ wheels = [ name = "topometry-nosc" source = { editable = "." } dependencies = [ - { name = "hnswlib" }, - { name = "ipykernel" }, - { name = "ipywidgets" }, { name = "joblib" }, - { name = "jupyterlab" }, { name = "matplotlib" }, { name = "numba" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "pacmap" }, { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "pandas", version = "3.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -4228,6 +4223,9 @@ layouts = [ [package.dev-dependencies] dev = [ + { name = "ipykernel" }, + { name = "ipywidgets" }, + { name = "jupyterlab" }, { name = "jupytext" }, { name = "mypy" }, { name = "nbstripout" }, @@ -4244,19 +4242,14 @@ docs = [ [package.metadata] requires-dist = [ - { name = "hnswlib", specifier = ">=0.8.0" }, { name = "hnswlib", marker = "extra == 'all'", specifier = ">=0.8.0" }, { name = "hnswlib", marker = "extra == 'ann'", specifier = ">=0.8.0" }, - { name = "ipykernel" }, - { name = "ipywidgets" }, { name = "joblib", specifier = ">=1.3" }, - { name = "jupyterlab" }, { name = "matplotlib", specifier = ">=3.6" }, { name = "numba", specifier = ">=0.57" }, { name = "numpy", specifier = ">=1.23" }, - { name = "pacmap", specifier = ">=0.9.1" }, - { name = "pacmap", marker = "extra == 'all'", specifier = ">=0.7" }, - { name = "pacmap", marker = "extra == 'layouts'", specifier = ">=0.7" }, + { name = "pacmap", marker = "extra == 'all'", specifier = ">=0.9.1" }, + { name = "pacmap", marker = "extra == 'layouts'", specifier = ">=0.9.1" }, { name = "pandas", specifier = ">=1.5" }, { name = "pyamg", marker = "extra == 'all'", specifier = ">=5.3.0" }, { name = "pyamg", marker = "extra == 'amg'", specifier = ">=5.3.0" }, @@ -4272,6 +4265,9 @@ provides-extras = ["all", "amg", "ann", "layouts"] [package.metadata.requires-dev] dev = [ + { name = "ipykernel" }, + { name = "ipywidgets" }, + { name = "jupyterlab" }, { name = "jupytext", specifier = ">=1.19.3" }, { name = "mypy", specifier = ">=1.13" }, { name = "nbstripout", specifier = ">=0.9.1" }, From d601f7218156130ecd6977b6be07e0244d895242 Mon Sep 17 00:00:00 2001 From: jsture Date: Wed, 10 Jun 2026 11:42:57 +0200 Subject: [PATCH 5/7] Expand API reference and cknn_graph documentation - add the standalone building blocks to the API reference: compute_kernel, eigendecompose, LE, graph_laplacian, diffusion_operator, automated_scaffold_sizing, find_ideal_projection, run_best_projection - give cknn_graph a full NumPy-style docstring with parameters, returns and the Berry & Sauer reference Co-Authored-By: Claude Fable 5 --- docs/api/advanced.md | 48 ++++++++++++++++++++++++++++++++++++++++ src/topo/tpgraph/cknn.py | 39 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/docs/api/advanced.md b/docs/api/advanced.md index 8ba1c13b..ebfdc980 100644 --- a/docs/api/advanced.md +++ b/docs/api/advanced.md @@ -32,6 +32,12 @@ unweighted graph construction, not a weighted adaptive kernel; use show_root_heading: true show_root_toc_entry: true +::: topo.tpgraph.kernels.compute_kernel + options: + heading_level: 3 + show_root_heading: true + show_root_toc_entry: true + ::: topo.tpgraph.kernels.Kernel options: heading_level: 3 @@ -55,6 +61,30 @@ unweighted graph construction, not a weighted adaptive kernel; use - "!^__" - "!^_" +::: topo.spectral.eigen.eigendecompose + options: + heading_level: 3 + show_root_heading: true + show_root_toc_entry: true + +::: topo.spectral.LE + options: + heading_level: 3 + show_root_heading: true + show_root_toc_entry: true + +::: topo.spectral.graph_laplacian + options: + heading_level: 3 + show_root_heading: true + show_root_toc_entry: true + +::: topo.spectral.diffusion_operator + options: + heading_level: 3 + show_root_heading: true + show_root_toc_entry: true + ::: topo.layouts.projector.Projector options: heading_level: 3 @@ -74,3 +104,21 @@ unweighted graph construction, not a weighted adaptive kernel; use filters: - "!^__" - "!^_" + +::: topo.tpgraph.intrinsic_dim.automated_scaffold_sizing + options: + heading_level: 3 + show_root_heading: true + show_root_toc_entry: true + +::: topo.layouts.diagnostics.find_ideal_projection + options: + heading_level: 3 + show_root_heading: true + show_root_toc_entry: true + +::: topo.layouts.diagnostics.run_best_projection + options: + heading_level: 3 + show_root_heading: true + show_root_toc_entry: true diff --git a/src/topo/tpgraph/cknn.py b/src/topo/tpgraph/cknn.py index a6434149..1c537b05 100644 --- a/src/topo/tpgraph/cknn.py +++ b/src/topo/tpgraph/cknn.py @@ -365,6 +365,45 @@ def cknn_graph( from sample ``i`` to its ``scale_k``-th nearest neighbor. Exact mode thresholds all pairs. Candidate-neighbor mode is scalable but may miss edges if ``candidate_k`` is too small. + + Introduced by Berry & Sauer, "Consistent manifold representation for + topological data analysis" (https://arxiv.org/abs/1606.02353). The binary + CkNN graph with the unnormalized Laplacian gives a consistent estimate of + the Laplace-Beltrami operator regardless of sampling density. + + Parameters + ---------- + X : array-like or sparse matrix of shape (n_samples, n_features) + Input data, or a square distance matrix when ``metric='precomputed'``. + scale_k : int, default=10 + Neighbor rank used for the local scale ``rho_i``. + delta : float, default=1.0 + Connection threshold; larger values produce denser graphs. + metric : str, default='euclidean' + Distance metric, or ``'precomputed'``. + candidate_k : int, optional + Number of candidate neighbors searched per sample. Defaults to + ``max(3 * scale_k, scale_k + 15)``. + exact : bool, default=False + Threshold all pairwise distances instead of candidate neighbors only. + Quadratic in memory; use for small datasets or validation. + include_self : bool, default=False + Whether to keep self-loops. + symmetrize : {'or', 'and', 'none'}, default='or' + How to symmetrize the directed candidate relation. + backend : {'sklearn', 'hnswlib'}, default='sklearn' + Neighbor-search backend for candidate retrieval. + n_jobs : int, default=-1 + Number of threads. ``-1`` uses all available CPUs. + verbose : bool, default=False + Emit search diagnostics through logging. + **kwargs + Forwarded to :func:`topo.base.ann.kNN`. + + Returns + ------- + scipy.sparse.csr_matrix + Binary (0/1) adjacency matrix of shape ``(n_samples, n_samples)``. """ n_samples = int(X.shape[0]) _validate_cknn_inputs(n_samples, scale_k, delta) From 6b548be576aa9e20f259fb745f385b96c79d93ee Mon Sep 17 00:00:00 2001 From: jsture Date: Wed, 10 Jun 2026 11:43:21 +0200 Subject: [PATCH 6/7] Update example notebooks to the current API - _example_utils.py: use topo.spectral.LE for the spectral layout initialization (spectral_layout was removed) and drop the removed weight= argument from EigenDecomposition - re-sync the .ipynb files from their jupytext .py sources, which had drifted (the .py side carries the Colab setup cells and doc links) The demo pipeline (kNN -> cknn Kernel -> msDM -> LE init -> PaCMAP -> metrics) was run end-to-end to verify. Co-Authored-By: Claude Fable 5 --- notebooks/_example_utils.py | 15 ++-- notebooks/example.ipynb | 2 + notebooks/example_explained.ipynb | 126 +++++++++++++++++++++++++----- notebooks/example_extended.ipynb | 4 +- 4 files changed, 121 insertions(+), 26 deletions(-) diff --git a/notebooks/_example_utils.py b/notebooks/_example_utils.py index c34da4a7..86a81e7e 100644 --- a/notebooks/_example_utils.py +++ b/notebooks/_example_utils.py @@ -32,7 +32,8 @@ topo_preserve_score, ) from topo.layouts.projector import Projector -from topo.spectral.eigen import EigenDecomposition, spectral_layout +from topo.spectral import LE +from topo.spectral.eigen import EigenDecomposition from topo.topograph import _KERNEL_CONFIGS from topo.tpgraph.kernels import Kernel @@ -307,7 +308,7 @@ def graph_layout_array(value: Any, n_samples: int) -> FloatArray: value = value[0] return checked_embedding( value, - "spectral_layout()", + "LE()", n_samples=n_samples, min_columns=2, ) @@ -357,7 +358,6 @@ def run_pipeline(data: DemoData, config: DemoConfig) -> PipelineResult: method=config.dm_method, eigensolver=config.eigensolver, drop_first=True, - weight=True, t=config.diffusion_time, ) eigen.fit(kernel_X) @@ -387,9 +387,12 @@ def run_pipeline(data: DemoData, config: DemoConfig) -> PipelineResult: K_Z = as_csr_matrix(kernel_Z.K, "kernel_Z.K") L_Z = as_csr_matrix(kernel_Z.L, "kernel_Z.L") - init_raw = spectral_layout( - graph=K_Z, - dim=config.n_components_2d, + init_raw = LE( + K_Z, + n_eigs=config.n_components_2d, + laplacian_type="normalized", + drop_first=True, + return_evals=False, random_state=config.random_state, ) init_Y = graph_layout_array(init_raw, n_samples=X.shape[0]) diff --git a/notebooks/example.ipynb b/notebooks/example.ipynb index 89f92b73..b3cba897 100644 --- a/notebooks/example.ipynb +++ b/notebooks/example.ipynb @@ -31,6 +31,8 @@ "source": [ "# # TopOMetry demo\n", "#\n", + "# **Purpose:** a compact first-run path through the full TopOMetry workflow.\n", + "#\n", "# This notebook gives a compact tour of the TopOMetry workflow:\n", "#\n", "# 1. load or generate data;\n", diff --git a/notebooks/example_explained.ipynb b/notebooks/example_explained.ipynb index 71a963e3..90dbe287 100644 --- a/notebooks/example_explained.ipynb +++ b/notebooks/example_explained.ipynb @@ -31,6 +31,8 @@ "source": [ "# # TopOMetry explained example\n", "#\n", + "# **Purpose:** a documented interpretation guide for every major output.\n", + "#\n", "# This notebook is the documented version of the basic and extended examples.\n", "# It runs the same helper-driven workflow, but explains each reported object,\n", "# plot, and metric next to the cell that produces it.\n", @@ -47,7 +49,40 @@ "#\n", "# The implementation details live in `_example_utils.py`. This notebook is\n", "# intentionally descriptive: it should help you understand the output, not hide\n", - "# the method choices." + "# the method choices.\n", + "#\n", + "# For the full theory, see the docs on\n", + "# [concepts](../docs/concepts.md),\n", + "# [mathematical details](../docs/math_details.md), and\n", + "# [background reading](../docs/background.md)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c08f635", + "metadata": {}, + "outputs": [], + "source": [ + "# ## Colab Setup\n", + "# This cell automatically sets up the environment if running in Google Colab.\n", + "import sys\n", + "\n", + "if \"google.colab\" in sys.modules:\n", + " import subprocess\n", + "\n", + " print(\"Colab detected: installing topometry-nosc and downloading utils...\")\n", + " subprocess.run([\"pip\", \"install\", \"-q\", \"topometry-nosc[all]\"], check=True)\n", + " subprocess.run(\n", + " [\n", + " \"wget\",\n", + " \"-q\",\n", + " \"https://raw.githubusercontent.com/HauserGroup/topometryNoSC/master/notebooks/_example_utils.py\",\n", + " ],\n", + " check=True,\n", + " )\n", + " subprocess.run([\"mkdir\", \"-p\", \"../data\", \"../figures\"], check=True)\n", + " print(\"Setup complete.\")" ] }, { @@ -102,10 +137,15 @@ "# larger values make the graph more connected and more global.\n", "# - `metric` is the distance used for nearest-neighbor search.\n", "# - `backend` chooses the neighbor-search implementation.\n", - "# - `kernel_version` controls how distances are converted into affinities.\n", + "# - `kernel_version` controls how distances become a graph or affinities.\n", "# - `sigma` is used by the fixed-bandwidth Gaussian kernel.\n", - "# - `anisotropy` controls diffusion normalization. Values near 1 reduce the\n", - "# influence of sampling density; values near 0 preserve more density signal.\n", + "# - `anisotropy` is the diffusion-map alpha normalization parameter. Values near\n", + "# 1 reduce sampling-density bias; values near 0 preserve more density signal.\n", + "#\n", + "# See the docs on [kernel graphs](../docs/concepts.md#kernel-graph),\n", + "# [nearest-neighbor graphs](../docs/background.md#1-nearest-neighbor-graphs),\n", + "# [kernels and the Laplace-Beltrami operator](../docs/background.md#2-kernels-and-the-laplacebeltrami-operator),\n", + "# and [density-bias correction](../docs/math_details.md#6-removing-the-density-bias-two-approaches).\n", "#\n", "# **Spectral scaffold parameters**\n", "#\n", @@ -116,10 +156,17 @@ "# - `diffusion_time=0` gives the multiscale diffusion representation when using\n", "# diffusion maps. Positive values give a fixed diffusion time.\n", "#\n", + "# See the docs on the [diffusion operator](../docs/concepts.md#diffusion-operator),\n", + "# [diffusion eigenvalues](../docs/math_details.md#7-the-diffusion-operator-and-its-eigenvalues),\n", + "# [spectral scaffolds](../docs/concepts.md#spectral-scaffold), and\n", + "# [DM/msDM](../docs/math_details.md#8-the-spectral-scaffold-dm-and-msdm).\n", + "#\n", "# **Layout parameters**\n", "#\n", "# - `projection_method` chooses the final 2-D optimizer or projection method.\n", "# - `num_iters` affects iterative methods such as MAP, UMAP, and PaCMAP.\n", + "#\n", + "# See the docs on [layouts and projections](../docs/background.md#6-layouts-and-projections).\n", "\n", "config = DemoConfig(\n", " # -- Data source ---------------------------------------------------------\n", @@ -149,7 +196,7 @@ " # \"hnswlib\" is fast and approximate; \"sklearn\" is exact and dependency-light;\n", " backend=\"hnswlib\",\n", " # -- Kernel --------------------------------------------------------------\n", - " # Converts neighbor distances into affinities. Try \"cknn\" and \"fuzzy\" first.\n", + " # Converts neighbor distances into a graph/affinities. Try \"cknn\" and \"fuzzy\" first.\n", " # Other options: \"bw_adaptive\", \"bw_adaptive_alpha_decaying\",\n", " # \"bw_adaptive_nbr_expansion\", \"bw_adaptive_alpha_decaying_nbr_expansion\",\n", " # and \"gaussian\".\n", @@ -203,14 +250,16 @@ "#\n", "# **Kernel versions**\n", "#\n", - "# - `bw_adaptive`: adaptive-bandwidth Gaussian affinities.\n", + "# - `bw_adaptive`: adaptive-bandwidth Gaussian affinities with density\n", + "# correction.\n", "# - `bw_adaptive_alpha_decaying`: adaptive bandwidth with alpha-decaying\n", "# exponent behavior.\n", "# - `bw_adaptive_nbr_expansion`: adaptive bandwidth with expanded neighbor\n", "# search.\n", "# - `bw_adaptive_alpha_decaying_nbr_expansion`: combines both extensions.\n", "# - `fuzzy`: UMAP-style fuzzy simplicial set affinities.\n", - "# - `cknn`: continuous k-nearest-neighbor affinities.\n", + "# - `cknn`: binary continuous k-nearest-neighbor graph construction for\n", + "# density-aware geometry.\n", "# - `gaussian`: fixed-bandwidth Gaussian affinities; tune `sigma`.\n", "#\n", "# **Spectral scaffold methods**\n", @@ -222,7 +271,11 @@ "# **Projection methods**\n", "#\n", "# `MAP`, `PaCMAP`, `Isomap`, `UMAP`, `t-SNE`, `TriMAP`, `IsomorphicMDE`, and\n", - "# `IsometricMDE` are common choices. Some require optional layout dependencies.\n", + "# `IsometricMDE` are common choices. They trade off local, global, and distortion\n", + "# objectives differently. Some require optional layout dependencies.\n", + "#\n", + "# See [background: layouts and projections](../docs/background.md#6-layouts-and-projections)\n", + "# for method references.\n", "\n", "print_options()" ] @@ -276,6 +329,9 @@ "# The printed lines summarize the kernel matrix, scaffold, and final projection\n", "# shapes. Sparse matrix `nnz` values tell you how many nonzero graph or kernel\n", "# entries were retained.\n", + "#\n", + "# See [math details: full pipeline](../docs/math_details.md#10-topometrys-pipeline-putting-it-all-together)\n", + "# for the operator-level view of these steps.\n", "\n", "result = run_pipeline(data, config)" ] @@ -307,6 +363,11 @@ "# - Very flat leading eigenvalues can indicate weak spectral separation.\n", "# - If `Z` has fewer useful dimensions than expected, increase data quality,\n", "# adjust `n_neighbors`, or try a different kernel.\n", + "#\n", + "# Related docs:\n", + "# [diffusion operator](../docs/concepts.md#diffusion-operator),\n", + "# [diffusion eigenvalues](../docs/math_details.md#7-the-diffusion-operator-and-its-eigenvalues),\n", + "# and [spectral scaffold](../docs/concepts.md#spectral-scaffold).\n", "\n", "print(\"Input\")\n", "print(f\" X {data.X.shape}\")\n", @@ -343,12 +404,16 @@ "# overview.\n", "#\n", "# - **Eigenvalue spectrum**: shows the strength of the first spectral\n", - "# components. Large gaps suggest natural low-dimensional structure.\n", + "# components. For DM/msDM, large values near 1 indicate smooth, persistent\n", + "# modes. Gaps can suggest natural geometric scales.\n", "# - **Spectral scaffold**: plots the first two coordinates of `Z`, colored by\n", "# `data.color`. This is not the final embedding; it is the geometry-aware\n", "# coordinate system used before optimization.\n", "# - **Final layout**: plots `Y`, the 2-D output that most users inspect or use\n", "# downstream.\n", + "#\n", + "# See [math details: DM and msDM](../docs/math_details.md#8-the-spectral-scaffold-dm-and-msdm)\n", + "# for how diffusion eigenvectors become scaffold coordinates.\n", "\n", "fig, axes = plt.subplots(1, 3, figsize=(15, 4))\n", "\n", @@ -397,6 +462,9 @@ "#\n", "# The 3x3 overview explains how the data move through the pipeline.\n", "#\n", + "# See [math details: TopoMetry's pipeline](../docs/math_details.md#10-topometrys-pipeline-putting-it-all-together)\n", + "# for the corresponding mathematical pipeline.\n", + "#\n", "# Row 1: input space\n", "#\n", "# - **(a) Input data**: raw input coordinates. For the default Swiss roll this\n", @@ -405,12 +473,15 @@ "# long tail can indicate uneven sampling density, outliers, or too many\n", "# neighbors.\n", "# - **(c) Distance to affinity**: shows how graph distances become kernel\n", - "# weights. Nearby points should generally receive stronger affinity.\n", + "# weights. Nearby points should generally receive stronger affinity, with the\n", + "# exact curve determined by the selected kernel.\n", "#\n", "# Row 2: spectral scaffold\n", "#\n", "# - **(d) Spectrum**: leading eigenvalues. A visible gap can suggest an\n", - "# intrinsic scale or dimensionality.\n", + "# intrinsic scale or dimensionality. DM/msDM eigenvalues are ordered\n", + "# decreasingly; LE eigenvalues follow the Laplacian convention and should not\n", + "# be interpreted on the same scale.\n", "# - **(e) First scaffold dimensions**: first two spectral coordinates.\n", "# - **(f) Higher harmonics**: later spectral coordinates, useful for seeing\n", "# structure that is not captured by the first pair.\n", @@ -440,11 +511,12 @@ "#\n", "# **Global scores**\n", "#\n", - "# - `global_score_pca(X, Y)` returns a value in `[0, 1]`. Higher means `Y`\n", - "# preserves global variance structure better relative to a PCA baseline.\n", - "# - `global_score_laplacian(X, Y)` also returns a value in `[0, 1]`. Higher\n", - "# means `Y` preserves graph-aware global structure better relative to a\n", - "# Laplacian Eigenmaps baseline.\n", + "# - `global_score_pca(X, Y)` returns a clipped value in `(0, 1]`. A value near\n", + "# 1 means `Y` preserves at least as much global linear reconstruction\n", + "# structure as a same-dimensional PCA baseline.\n", + "# - `global_score_laplacian(X, Y)` returns a clipped value in `(0, 1]`. A value\n", + "# near 1 means `Y` preserves at least as much graph-aware reconstruction\n", + "# structure as a same-dimensional Laplacian Eigenmaps baseline.\n", "#\n", "# **Local and geodesic score**\n", "#\n", @@ -466,6 +538,9 @@ "# - `Rank diffusion corr.`: rank correlation between diffusion similarities.\n", "# - `Spectral similarity`: eigen-spectrum similarity between operators.\n", "#\n", + "# See [concepts: topology-preservation metrics](../docs/concepts.md#topology-preservation-metrics)\n", + "# and [background: evaluation diagnostics](../docs/background.md#7-evaluation-and-geometry-diagnostics).\n", + "#\n", "# **Riemannian deformation**\n", "#\n", "# The deformation values come from a local Riemannian metric estimated on the\n", @@ -476,6 +551,8 @@ "# - A narrow distribution around 0 indicates more uniform local area\n", "# preservation.\n", "# - Large absolute values identify regions to inspect in the deep-dive plots.\n", + "#\n", + "# See [math details: measuring distortion](../docs/math_details.md#9-measuring-distortion-the-riemannian-metric).\n", "\n", "metrics = compute_metrics(data, result, config)\n", "print_metric_summary(metrics)" @@ -492,6 +569,9 @@ "#\n", "# This 2x2 diagnostic figure is a compact dashboard for the metric output.\n", "#\n", + "# See [background: evaluation and geometry diagnostics](../docs/background.md#7-evaluation-and-geometry-diagnostics)\n", + "# and [math details: Riemannian metric](../docs/math_details.md#9-measuring-distortion-the-riemannian-metric).\n", + "#\n", "# - **(a) Local metric ellipses**: each ellipse represents local stretching\n", "# estimated by the Riemannian metric. Nearly circular ellipses mean isotropic\n", "# local scaling; elongated ellipses mean direction-dependent distortion.\n", @@ -501,8 +581,8 @@ "# in a few regions or spread across the layout. Narrow and centered near zero\n", "# is preferable.\n", "# - **(d) Metrics at a glance**: bar chart of the printed scores. Higher is\n", - "# better for all displayed scores except deformation is summarized separately\n", - "# in panel (c).\n", + "# better for these displayed score metrics. Deformation is summarized\n", + "# separately in panel (c), where smaller spread around zero is usually better.\n", "\n", "plot_metric_overview(data, result, metrics, config)" ] @@ -520,6 +600,9 @@ "# pairwise distortion summaries. Run it when you need to understand where and\n", "# how an embedding is distorted.\n", "#\n", + "# See [background: evaluation and geometry diagnostics](../docs/background.md#7-evaluation-and-geometry-diagnostics)\n", + "# for the motivation and references behind these diagnostics.\n", + "#\n", "# Row 1: Euclidean and rank distortion\n", "#\n", "# - **(a) Per-point rank distortion**: for landmark distances, compares the\n", @@ -576,6 +659,11 @@ "# - try `kernel_version=\"gaussian\"` and tune `sigma`;\n", "# - compare `dm_method=\"DM\"`, `\"msDM\"`, and `\"LE\"`;\n", "# - compare final layouts such as `\"MAP\"`, `\"PaCMAP\"`, `\"UMAP\"`, and `\"Isomap\"`.\n", + "#\n", + "# Method background:\n", + "# [kernels](../docs/background.md#2-kernels-and-the-laplacebeltrami-operator),\n", + "# [spectral scaffolds](../docs/background.md#3-spectral-scaffolds-the-eigenbasis),\n", + "# and [layouts](../docs/background.md#6-layouts-and-projections).\n", "\n", "RUN_VARIANT = False\n", "\n", @@ -667,7 +755,7 @@ "#\n", "# The `delta` column is `variant - baseline`. Positive values are better for\n", "# all scores shown here. A useful variant usually improves several metrics\n", - "# without causing an obvious visual artifact or a large deformation increase.\n", + "# without causing an obvious visual artifact or a larger deformation spread.\n", "\n", "if variant_metrics is None:\n", " print(\"Run the variant cell with RUN_VARIANT = True first.\")\n", diff --git a/notebooks/example_extended.ipynb b/notebooks/example_extended.ipynb index 4cd3bdc8..4f867a23 100644 --- a/notebooks/example_extended.ipynb +++ b/notebooks/example_extended.ipynb @@ -31,6 +31,8 @@ "source": [ "# # TopOMetry extended exploration\n", "#\n", + "# **Purpose:** hands-on parameter exploration and method comparison.\n", + "#\n", "# This notebook keeps the same high-level architecture as `example.py`, but adds\n", "# tuning cells for comparing graph, kernel, spectral, and layout choices.\n", "#\n", @@ -92,7 +94,7 @@ " # Graph/kernel\n", " n_neighbors=15,\n", " metric=\"euclidean\", # any metric accepted by the chosen backend\n", - " backend=\"hnswlib\", # \"hnswlib\" | \"nmslib\" | \"sklearn\"\n", + " backend=\"hnswlib\", # \"hnswlib\" | \"sklearn\"\n", " kernel_version=\"cknn\", # \"bw_adaptive\" | \"fuzzy\" | \"cknn\" | \"gaussian\"\n", " sigma=1.0, # bandwidth for \"gaussian\" only\n", " anisotropy=1.0, # alpha for the diffusion operator (0-1)\n", From 550233e8184152f331844d2ea1e1fd0c78dc73c5 Mon Sep 17 00:00:00 2001 From: jsture Date: Wed, 10 Jun 2026 11:43:44 +0200 Subject: [PATCH 7/7] Add integration tests composing standalone estimators The TopOGraph tests kept passing while the standalone building blocks (kNN, Kernel, EigenDecomposition, LE, Projector, cknn_graph) were broken, because nothing exercised them composed outside the orchestrator. Add two integration tests mirroring the documented custom-pipeline path so that class of breakage is caught. Co-Authored-By: Claude Fable 5 --- tests/topo/test_standalone_pipeline.py | 85 ++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 tests/topo/test_standalone_pipeline.py diff --git a/tests/topo/test_standalone_pipeline.py b/tests/topo/test_standalone_pipeline.py new file mode 100644 index 00000000..920b5601 --- /dev/null +++ b/tests/topo/test_standalone_pipeline.py @@ -0,0 +1,85 @@ +"""Integration test composing standalone estimators outside TopOGraph. + +Mirrors the documented "custom pipeline" path (see docs/api/advanced.md and +notebooks/_example_utils.py): each building block must work on its own and +compose into a full embedding pipeline without the TopOGraph orchestrator. +This guards against breakage in standalone APIs that the TopOGraph tests do +not exercise. +""" + +import numpy as np +import pytest +from scipy.sparse import csr_matrix +from sklearn.datasets import make_swiss_roll + +from topo.base.ann import kNN +from topo.layouts.projector import Projector +from topo.spectral import LE +from topo.spectral.eigen import EigenDecomposition +from topo.tpgraph.cknn import cknn_graph +from topo.tpgraph.kernels import Kernel + + +@pytest.fixture(scope="module") +def data(): + X, _ = make_swiss_roll(n_samples=200, noise=0.05, random_state=0) + return np.asarray(X, dtype=np.float64) + + +def test_standalone_composition_knn_kernel_eigen_projection(data): + """kNN -> Kernel -> EigenDecomposition -> LE init -> Projector.""" + n = data.shape[0] + + knn = kNN(data, n_neighbors=15, backend="sklearn") + assert isinstance(knn, csr_matrix) + assert knn.shape == (n, n) + + kernel = Kernel(n_neighbors=15, metric="euclidean", backend="sklearn") + kernel.fit(data) + assert kernel.P is not None + + eigen = EigenDecomposition(n_components=10, method="msDM", drop_first=True) + eigen.fit(kernel) + Z = np.asarray(eigen.transform()) + assert Z.shape[0] == n + assert Z.shape[1] >= 2 + assert np.isfinite(Z).all() + + kernel_Z = Kernel(n_neighbors=15, metric="euclidean", backend="sklearn") + kernel_Z.fit(Z) + K_Z = csr_matrix(kernel_Z.K) + + init = np.asarray(LE(K_Z, n_eigs=2, laplacian_type="normalized", drop_first=True)) + assert init.shape == (n, 2) + assert np.isfinite(init).all() + + projector = Projector( + projection_method="MAP", + n_components=2, + n_neighbors=15, + num_iters=30, + init=init, + random_state=42, + ) + Y = np.asarray(projector.fit_transform(csr_matrix(kernel_Z.P))) + assert Y.shape == (n, 2) + assert np.isfinite(Y).all() + + +def test_standalone_cknn_kernel_pipeline(data): + """CkNN kernel path: Kernel(cknn) on data -> eigenbasis on binary graph.""" + n = data.shape[0] + + W = cknn_graph(data, scale_k=10, delta=1.0, backend="sklearn") + assert W.shape == (n, n) + assert set(np.unique(W.data)).issubset({1.0}) + # symmetrize='or' default yields a symmetric adjacency + assert (W != W.T).nnz == 0 + + kernel = Kernel(n_neighbors=10, cknn=True, backend="sklearn") + kernel.fit(data) + + eigen = EigenDecomposition(n_components=5, method="DM", drop_first=True) + Z = np.asarray(eigen.fit_transform(kernel)) + assert Z.shape[0] == n + assert np.isfinite(Z).all()