From 389d40c65e82c55b96d875276e5658a5c9d6b0e3 Mon Sep 17 00:00:00 2001 From: afloresep Date: Thu, 29 Oct 2026 12:17:07 +0100 Subject: [PATCH 01/10] Fix MQN create_tmap, serial fingerprints, rename PQKMeans iterations - create_tmap: use euclidean metric for MQN (jaccard rejected count vectors with "Binary matrix must contain only 0/1 values"). Also fixes chelombus-tmap --fingerprint mqn. - FingerprintFromSmiles: nprocesses=1 now computes in-process instead of starting a one-worker Pool. - PQKMeans: rename `iteration` to `iterations` to match PQEncoder. `iteration=` still works with a DeprecationWarning, and models saved with the old attribute are migrated on load. - Add tests for all three. Reported by Dan Kovari while following the tutorial. --- chelombus/clustering/PyQKmeans.py | 27 +++++++++--- chelombus/utils/fingerprints.py | 11 ++--- chelombus/utils/visualization.py | 7 +-- scripts/benchmark_1B_pipeline.py | 2 +- scripts/fit_pqkmeans.py | 2 +- scripts/k_selection_gpu.py | 2 +- scripts/pipeline/01_train_models.py | 2 +- scripts/select_k.py | 2 +- tests/test_clustering.py | 67 ++++++++++++++++------------- tests/test_fingerprints.py | 17 +++++++- tests/test_integration.py | 6 +-- tests/test_visualization.py | 18 ++++++++ 12 files changed, 109 insertions(+), 54 deletions(-) create mode 100644 tests/test_visualization.py diff --git a/chelombus/clustering/PyQKmeans.py b/chelombus/clustering/PyQKmeans.py index b8de301..6c180fe 100644 --- a/chelombus/clustering/PyQKmeans.py +++ b/chelombus/clustering/PyQKmeans.py @@ -10,6 +10,7 @@ """ from typing import Literal, overload +import warnings import joblib from pathlib import Path import numpy as np @@ -126,11 +127,12 @@ class PQKMeans: Args: encoder: A trained PQEncoder instance k: Number of clusters - iteration: Maximum number of k-means iterations (default: 20) + iterations: Maximum number of k-means iterations (default: 20) tol: Early-stopping tolerance for the GPU path. Training stops when the fraction of changed center coordinates drops below *tol*. 0 means stop only on exact convergence. (default: 1e-3) verbose: Whether to print progress information (default: False) + iteration: Deprecated alias for ``iterations``. Example: >>> encoder = PQEncoder(k=256, m=6) @@ -144,16 +146,24 @@ def __init__( self, encoder: PQEncoder, k: int, - iteration: int = 20, + iterations: int = 20, tol: float = 1e-3, - verbose: bool = False + verbose: bool = False, + iteration: int | None = None, ): + if iteration is not None: + warnings.warn( + "'iteration' is deprecated, use 'iterations' instead", + DeprecationWarning, + stacklevel=2, + ) + iterations = iteration if not encoder.encoder_is_trained: # type: ignore raise ValueError("Encoder must be trained before clustering") self.encoder = encoder self.k = k - self.iteration = iteration + self.iterations = iterations self.tol = tol self.verbose = verbose self.trained = False @@ -163,7 +173,7 @@ def __init__( self._cluster = pqkmeans.clustering.PQKMeans( encoder=encoder, k=k, - iteration=iteration, + iteration=iterations, verbose=verbose, ) @@ -249,7 +259,7 @@ def _fit_gpu(self, X_train: np.ndarray, return_labels: bool = False) -> np.ndarr labels = None prev_centers = None final_labels_match_centers = False - for it in range(self.iteration): + for it in range(self.iterations): t0 = time.time() # assignment (GPU) @@ -274,7 +284,7 @@ def _fit_gpu(self, X_train: np.ndarray, return_labels: bool = False) -> np.ndarr if self.verbose: print( - f" iter {it + 1}/{self.iteration} " + f" iter {it + 1}/{self.iterations} " f"assign={t1 - t0:.1f}s update={t2 - t1:.1f}s " f"changed={changed}/{total_coords} ({frac:.4%})" ) @@ -392,6 +402,9 @@ def load(cls, path: str | Path) -> "PQKMeans": obj = joblib.load(path) if not isinstance(obj, cls): raise TypeError(f"Expected {cls.__name__}, got {type(obj).__name__}") + if not hasattr(obj, 'iterations'): + # Models saved before the 'iteration' -> 'iterations' rename + obj.iterations = obj.__dict__.pop('iteration') if not hasattr(obj, '_dtables'): obj._dtables = None if not hasattr(obj, '_centers_u8'): diff --git a/chelombus/utils/fingerprints.py b/chelombus/utils/fingerprints.py index 77180f1..2d46495 100644 --- a/chelombus/utils/fingerprints.py +++ b/chelombus/utils/fingerprints.py @@ -111,7 +111,8 @@ def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None Args: smiles_list (list): A list of SMILES strings. fp (str): The fingerprint type to compute (e.g., 'morgan'). - nprocesses (int): Number of processes for multithreaded fingerprint calculation. Default to cpu cores + nprocesses (int): Number of worker processes. Defaults to all CPU cores. + Use 1 to compute in the current process (no multiprocessing). return_valid_idx (bool): If True, return ``(valid_idx, fingerprints)`` where ``valid_idx`` is an int64 array of the input positions that parsed OK. If False (default), return only the fingerprints. @@ -136,12 +137,12 @@ def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None if isinstance(smiles, str): smiles = [smiles] - try: + if nprocesses == 1: + # No worker processes: avoids pool startup cost and spawn issues + fingerprints = [part_func(s) for s in smiles] + else: with Pool(processes=nprocesses) as pool: fingerprints = pool.map(part_func, smiles) - finally: - pool.close() - pool.join() if return_valid_idx: # Keep the input positions that produced a fingerprint, so the diff --git a/chelombus/utils/visualization.py b/chelombus/utils/visualization.py index 7a156eb..d7099c0 100644 --- a/chelombus/utils/visualization.py +++ b/chelombus/utils/visualization.py @@ -47,8 +47,9 @@ def create_tmap( fps = fingerprints_from_smiles(smiles, fp_type=fingerprint, **fp_kwargs) - # Fit TMAP - model = TMAP(metric='jaccard', n_neighbors=k_neighbors).fit(fps) + # Fit TMAP, use euclidean for dense descriptors like MQN + metric = 'euclidean' if fingerprint == 'mqn' else 'jaccard' + model = TMAP(metric=metric, n_neighbors=k_neighbors).fit(fps) # Build visualization viz = model.to_tmapviz(include_edges=True) @@ -104,7 +105,7 @@ def representative_tmap( fps = fingerprints_from_smiles(smiles, fp_type=fingerprint, **fp_kwargs) - # Fit TMAP — use euclidean for dense descriptors like MQN + # Fit TMAP, use euclidean for dense descriptors like MQN metric = 'euclidean' if fingerprint == 'mqn' else 'jaccard' model = TMAP(metric=metric, n_neighbors=k_neighbors).fit(fps) diff --git a/scripts/benchmark_1B_pipeline.py b/scripts/benchmark_1B_pipeline.py index b58174b..5cc3d02 100644 --- a/scripts/benchmark_1B_pipeline.py +++ b/scripts/benchmark_1B_pipeline.py @@ -156,7 +156,7 @@ def main(): # ── 3. Cluster training ────────────────────────────────────────────── print(f"\n{sep}\n STEP 3: Cluster training ({N:,}, K={args.k:,}, {args.fit_iters} iters, tol=0)\n{sep}", flush=True) - clusterer = PQKMeans(encoder, k=args.k, iteration=args.fit_iters, tol=0, verbose=True) + clusterer = PQKMeans(encoder, k=args.k, iterations=args.fit_iters, tol=0, verbose=True) t_fit, _ = bench("GPU", lambda: clusterer.fit(pq_codes, device='gpu')) results["Cluster training"] = t_fit diff --git a/scripts/fit_pqkmeans.py b/scripts/fit_pqkmeans.py index ca2aa54..8187ddc 100644 --- a/scripts/fit_pqkmeans.py +++ b/scripts/fit_pqkmeans.py @@ -225,7 +225,7 @@ def main(): pq_encoder = joblib.load(args.encoder) kmeans = PQKMeans(encoder=pq_encoder, k=args.k, - iteration=args.iterations, + iterations=args.iterations, verbose=args.kmeans_verbose) train_start = time.time() diff --git a/scripts/k_selection_gpu.py b/scripts/k_selection_gpu.py index 7867a9a..d534fe1 100644 --- a/scripts/k_selection_gpu.py +++ b/scripts/k_selection_gpu.py @@ -125,7 +125,7 @@ def main(): continue # Fit on GPU - clusterer = PQKMeans(encoder, k=k, iteration=args.iterations, verbose=False) + clusterer = PQKMeans(encoder, k=k, iterations=args.iterations, verbose=False) t0 = time.perf_counter() clusterer.fit(pq_codes, device='gpu') t_fit = time.perf_counter() - t0 diff --git a/scripts/pipeline/01_train_models.py b/scripts/pipeline/01_train_models.py index f843c7f..bbd5117 100644 --- a/scripts/pipeline/01_train_models.py +++ b/scripts/pipeline/01_train_models.py @@ -468,7 +468,7 @@ def stage4_train_clusterer( clusterer = PQKMeans( encoder=encoder, k=config.n_clusters, - iteration=config.clusterer_iterations, + iterations=config.clusterer_iterations, verbose=verbose > 0 ) diff --git a/scripts/select_k.py b/scripts/select_k.py index 945bfaa..9e3886e 100644 --- a/scripts/select_k.py +++ b/scripts/select_k.py @@ -253,7 +253,7 @@ def main(): continue # Fit - clusterer = PQKMeans(encoder, k=k, iteration=args.iterations, verbose=False) + clusterer = PQKMeans(encoder, k=k, iterations=args.iterations, verbose=False) t0 = time.time() clusterer.fit(pq_codes) t_fit = time.time() - t0 diff --git a/tests/test_clustering.py b/tests/test_clustering.py index 8d3bfa3..c98a56d 100644 --- a/tests/test_clustering.py +++ b/tests/test_clustering.py @@ -60,11 +60,11 @@ class TestPQKMeansInit: def test_init_with_trained_encoder(self, trained_encoder): """Test that PQKMeans can be initialized with a trained encoder.""" - clusterer = PQKMeans(trained_encoder, k=10, iteration=5, verbose=False) + clusterer = PQKMeans(trained_encoder, k=10, iterations=5, verbose=False) assert clusterer.encoder == trained_encoder assert clusterer.k == 10 - assert clusterer.iteration == 5 + assert clusterer.iterations == 5 assert clusterer.verbose is False assert clusterer.encoder.is_trained is True # Encoder is trained @@ -78,23 +78,30 @@ def test_init_default_parameters(self, trained_encoder): clusterer = PQKMeans(trained_encoder, k=100) assert clusterer.k == 100 - assert clusterer.iteration == 20 # default + assert clusterer.iterations == 20 # default assert clusterer.verbose is False # default + def test_init_deprecated_iteration_alias(self, trained_encoder): + """The old 'iteration' keyword still works but warns.""" + with pytest.warns(DeprecationWarning, match="iterations"): + clusterer = PQKMeans(trained_encoder, k=10, iteration=7) + + assert clusterer.iterations == 7 + class TestPQKMeansFit: """Tests for PQKMeans fit method.""" def test_fit_returns_self(self, trained_encoder, sample_pq_codes): """Test that fit() returns self for method chaining.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) result = clusterer.fit(sample_pq_codes) assert result is clusterer def test_fit_with_valid_pq_codes(self, trained_encoder, sample_pq_codes): """Test fitting with valid PQ codes.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) # Should complete without error @@ -102,7 +109,7 @@ def test_fit_with_valid_pq_codes(self, trained_encoder, sample_pq_codes): def test_fit_with_small_k(self, trained_encoder, sample_pq_codes): """Test fitting with small number of clusters.""" - clusterer = PQKMeans(trained_encoder, k=3, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=3, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) # Predict should return labels in range [0, k) @@ -116,7 +123,7 @@ class TestPQKMeansPredict: def test_predict_shape(self, trained_encoder, sample_pq_codes): """Test that predict returns correct shape.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) labels = clusterer.predict(sample_pq_codes) @@ -126,7 +133,7 @@ def test_predict_shape(self, trained_encoder, sample_pq_codes): def test_predict_label_range(self, trained_encoder, sample_pq_codes): """Test that predicted labels are in valid range.""" k = 5 - clusterer = PQKMeans(trained_encoder, k=k, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=k, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) labels = clusterer.predict(sample_pq_codes) @@ -136,7 +143,7 @@ def test_predict_label_range(self, trained_encoder, sample_pq_codes): def test_predict_dtype(self, trained_encoder, sample_pq_codes): """Test that predicted labels are integers.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) labels = clusterer.predict(sample_pq_codes) @@ -149,7 +156,7 @@ class TestPQKMeansFitPredict: def test_fit_predict_shape(self, trained_encoder, sample_pq_codes): """Test that fit_predict returns correct shape.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) labels = clusterer.fit_predict(sample_pq_codes) @@ -159,7 +166,7 @@ def test_fit_predict_equivalence(self, trained_encoder, sample_pq_codes): """Test that fit_predict gives same result as fit then predict.""" np.random.seed(42) - clusterer1 = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer1 = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) labels1 = clusterer1.fit_predict(sample_pq_codes) # Note: Due to randomness in k-means, we can't guarantee exact equality @@ -173,7 +180,7 @@ class TestPQKMeansSaveLoad: def test_save_creates_file(self, trained_encoder, sample_pq_codes, tmp_path): """Test that save creates a file.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) save_path = tmp_path / "clusterer.pkl" @@ -183,7 +190,7 @@ def test_save_creates_file(self, trained_encoder, sample_pq_codes, tmp_path): def test_load_restores_clusterer(self, trained_encoder, sample_pq_codes, tmp_path): """Test that load restores a clusterer correctly.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) labels_before = clusterer.predict(sample_pq_codes) @@ -208,7 +215,7 @@ def test_load_wrong_type_raises(self, tmp_path): def test_load_with_string_path(self, trained_encoder, sample_pq_codes, tmp_path): """Test that load works with string path.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) save_path = str(tmp_path / "clusterer.pkl") @@ -253,7 +260,7 @@ def test_cluster_mqn_like_data(self, trained_encoder): # Encode and cluster pq_codes = trained_encoder.transform(X, verbose=0) - clusterer = PQKMeans(trained_encoder, k=n_true_clusters, iteration=10, verbose=False) + clusterer = PQKMeans(trained_encoder, k=n_true_clusters, iterations=10, verbose=False) labels = clusterer.fit_predict(pq_codes) # Check basic properties @@ -269,7 +276,7 @@ def test_large_k_clustering(self, trained_encoder): pq_codes = trained_encoder.transform(X, verbose=0) k = 50 - clusterer = PQKMeans(trained_encoder, k=k, iteration=5, verbose=False) + clusterer = PQKMeans(trained_encoder, k=k, iterations=5, verbose=False) labels = clusterer.fit_predict(pq_codes) # Should have multiple clusters represented @@ -319,7 +326,7 @@ def test_mocked_gpu_fit_predict_matches_final_centers(self, monkeypatch, trained lambda *args, **kwargs: np.random.Generator(np.random.PCG64(0)), ) - clusterer = PQKMeans(trained_encoder, k=10, iteration=20, tol=1e-3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=10, iterations=20, tol=1e-3, verbose=False) labels = clusterer.fit_predict(sample_pq_codes, device='gpu') final_labels = clusterer.predict(sample_pq_codes, device='cpu') @@ -341,7 +348,7 @@ def test_mocked_gpu_fit_clears_transient_labels(self, monkeypatch, trained_encod lambda *args, **kwargs: np.random.Generator(np.random.PCG64(1)), ) - clusterer = PQKMeans(trained_encoder, k=5, iteration=5, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=5, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') assert clusterer._fit_labels is None @@ -356,7 +363,7 @@ def fail_predict_gpu(*args, **kwargs): monkeypatch.setattr(pyqkmeans_module, "predict_gpu", fail_predict_gpu, raising=False) - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) # Pretend the encoder has k > 256 to trigger the guard monkeypatch.setattr(trained_encoder, "k", 512) @@ -412,7 +419,7 @@ class TestGPUPredict: @gpu def test_gpu_predict_matches_cpu(self, trained_encoder, sample_pq_codes): """GPU and CPU predict must return identical labels.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes, device='cpu') labels_cpu = clusterer.predict(sample_pq_codes, device='cpu') @@ -424,7 +431,7 @@ def test_gpu_predict_matches_cpu(self, trained_encoder, sample_pq_codes): def test_gpu_predict_shape_and_range(self, trained_encoder, sample_pq_codes): """GPU predict returns correct shape and valid label range.""" k = 5 - clusterer = PQKMeans(trained_encoder, k=k, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=k, iterations=3, verbose=False) clusterer.fit(sample_pq_codes, device='cpu') labels = clusterer.predict(sample_pq_codes, device='gpu') @@ -437,7 +444,7 @@ def test_gpu_predict_shape_and_range(self, trained_encoder, sample_pq_codes): @gpu def test_gpu_predict_deterministic(self, trained_encoder, sample_pq_codes): """Repeated GPU predict calls return the same labels.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes, device='cpu') labels1 = clusterer.predict(sample_pq_codes, device='gpu') @@ -453,7 +460,7 @@ class TestGPUFit: def test_gpu_fit_produces_valid_clusters(self, trained_encoder, sample_pq_codes): """GPU fit must produce a trained model with valid labels.""" k = 5 - clusterer = PQKMeans(trained_encoder, k=k, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=k, iterations=3, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') assert clusterer.is_trained @@ -465,7 +472,7 @@ def test_gpu_fit_produces_valid_clusters(self, trained_encoder, sample_pq_codes) @gpu def test_gpu_fit_returns_self(self, trained_encoder, sample_pq_codes): """GPU fit returns self for method chaining.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) result = clusterer.fit(sample_pq_codes, device='gpu') assert result is clusterer @@ -473,7 +480,7 @@ def test_gpu_fit_returns_self(self, trained_encoder, sample_pq_codes): @gpu def test_gpu_fit_centers_are_valid_codes(self, trained_encoder, sample_pq_codes): """Cluster centers after GPU fit must be valid codebook indices.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') centers = clusterer.cluster_centers_ @@ -485,7 +492,7 @@ def test_gpu_fit_centers_are_valid_codes(self, trained_encoder, sample_pq_codes) def test_gpu_fit_predict(self, trained_encoder, sample_pq_codes): """GPU fit_predict returns labels of correct shape and range.""" k = 5 - clusterer = PQKMeans(trained_encoder, k=k, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=k, iterations=3, verbose=False) labels = clusterer.fit_predict(sample_pq_codes, device='gpu') assert labels.shape == (len(sample_pq_codes),) @@ -499,7 +506,7 @@ class TestGPUSaveLoad: @gpu def test_gpu_save_load_labels_match(self, trained_encoder, sample_pq_codes, tmp_path): """Labels must match after save/load of a GPU-trained model.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, tol=0, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, tol=0, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') labels_before = clusterer.predict(sample_pq_codes) @@ -513,7 +520,7 @@ def test_gpu_save_load_labels_match(self, trained_encoder, sample_pq_codes, tmp_ @gpu def test_gpu_save_load_gpu_predict(self, trained_encoder, sample_pq_codes, tmp_path): """GPU predict works correctly on a loaded model.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, tol=0, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, tol=0, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') labels_before = clusterer.predict(sample_pq_codes, device='gpu') @@ -531,7 +538,7 @@ class TestEarlyStopping: @gpu def test_tol_stops_before_max_iterations(self, trained_encoder, sample_pq_codes): """With default tol, training should stop before iteration limit.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=50, tol=1e-3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=50, tol=1e-3, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') # If early stopping works, it should finish quickly (not run 50 iters). @@ -544,7 +551,7 @@ def test_tol_stops_before_max_iterations(self, trained_encoder, sample_pq_codes) @gpu def test_tol_zero_only_stops_on_exact_convergence(self, trained_encoder, sample_pq_codes): """tol=0 should only stop when no centers change at all.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, tol=0, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, tol=0, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') assert clusterer.is_trained diff --git a/tests/test_fingerprints.py b/tests/test_fingerprints.py index d34b781..94fd53c 100644 --- a/tests/test_fingerprints.py +++ b/tests/test_fingerprints.py @@ -112,4 +112,19 @@ def test_fingerprint_with_data_parser(tmp_path): assert len(result) == 4 assert result[0].shape == (2, 1024) - assert result[3].shape == (1, 1024) \ No newline at end of file + assert result[3].shape == (1, 1024) + +def test_fingerprint_calculator_serial_matches_pool(): + """nprocesses=1 runs in-process and gives the same result as the pool.""" + smiles = ['CCO', 'c1ccccc1O', 'not_a_smiles', 'CC(=O)Nc1ccc(O)cc1'] + calculator = FingerprintCalculator() + + idx_serial, fps_serial = calculator.FingerprintFromSmiles( + smiles, 'mqn', nprocesses=1, return_valid_idx=True) + idx_pool, fps_pool = calculator.FingerprintFromSmiles( + smiles, 'mqn', nprocesses=2, return_valid_idx=True) + + np.testing.assert_array_equal(idx_serial, [0, 1, 3]) + np.testing.assert_array_equal(idx_serial, idx_pool) + np.testing.assert_array_equal(fps_serial, fps_pool) + assert fps_serial.shape == (3, 42) diff --git a/tests/test_integration.py b/tests/test_integration.py index b59ca11..ca13242 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -172,7 +172,7 @@ def test_full_pipeline_small_dataset(self, smiles_file, results_dir): # Step 4: Train and apply clustering n_clusters = 5 - clusterer = PQKMeans(encoder, k=n_clusters, iteration=5, verbose=False) + clusterer = PQKMeans(encoder, k=n_clusters, iterations=5, verbose=False) labels = clusterer.fit_predict(pq_codes) assert labels.shape == (n_valid,) @@ -214,7 +214,7 @@ def test_chunked_processing_pipeline(self, smiles_file, results_dir): # Step 2: Train clusterer on PQ codes training_pq_codes = encoder.transform(training_fps.astype(np.float32), verbose=0) n_clusters = 3 - clusterer = PQKMeans(encoder, k=n_clusters, iteration=5, verbose=False) + clusterer = PQKMeans(encoder, k=n_clusters, iterations=5, verbose=False) clusterer.fit(training_pq_codes) # Step 3: Process in chunks @@ -266,7 +266,7 @@ def test_save_load_models(self, tmp_path): # Train clusterer pq_codes = encoder.transform(fps.astype(np.float32), verbose=0) - clusterer = PQKMeans(encoder, k=3, iteration=5, verbose=False) + clusterer = PQKMeans(encoder, k=3, iterations=5, verbose=False) clusterer.fit(pq_codes) # Save clusterer diff --git a/tests/test_visualization.py b/tests/test_visualization.py new file mode 100644 index 0000000..68a6d40 --- /dev/null +++ b/tests/test_visualization.py @@ -0,0 +1,18 @@ +import pytest + +pytest.importorskip("tmap") + +from chelombus.utils.visualization import create_tmap + +SMILES = ["C" * i + "O" for i in range(1, 30)] + ["c1ccccc1" + "C" * i for i in range(1, 20)] + + +@pytest.mark.parametrize("fingerprint", ["mqn", "morgan"]) +def test_create_tmap_writes_html(fingerprint, tmp_path, monkeypatch): + """create_tmap works for both count (MQN) and binary (Morgan) fingerprints.""" + monkeypatch.chdir(tmp_path) + + create_tmap(SMILES, fingerprint=fingerprint, properties=["mw", "logp"], + tmap_name="test_map", k_neighbors=5) + + assert (tmp_path / "test_map.html").exists() From 27100a2d7aaaf51ac896649497daa1487ae699e9 Mon Sep 17 00:00:00 2001 From: afloresep Date: Thu, 29 Oct 2026 12:32:09 +0100 Subject: [PATCH 02/10] Fail fast instead of hanging when fingerprint workers crash On Windows and macOS (spawn start method), a script calling FingerprintFromSmiles without an `if __name__ == "__main__":` guard made multiprocessing.Pool respawn crashing workers forever. Use ProcessPoolExecutor, which raises BrokenProcessPool, and re-raise it as a RuntimeError that explains the missing guard. - Default nprocesses is now None (all cores, capped at 61 on Windows, where ProcessPoolExecutor rejects more). - Shut the executor down without waiting so idle workers exiting does not add ~1s, keeping speed on par with the old Pool. - Add a subprocess test that runs an unguarded script under spawn. --- chelombus/utils/fingerprints.py | 35 ++++++++++++++++++++++++++------- tests/test_fingerprints.py | 25 +++++++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/chelombus/utils/fingerprints.py b/chelombus/utils/fingerprints.py index 2d46495..6399978 100644 --- a/chelombus/utils/fingerprints.py +++ b/chelombus/utils/fingerprints.py @@ -1,5 +1,6 @@ #This module provides functionality to compute molecular fingerprints from SMILES strings -from multiprocessing import Pool +from concurrent.futures import ProcessPoolExecutor +from concurrent.futures.process import BrokenProcessPool from functools import partial from rdkit import Chem, DataStructs import os @@ -88,14 +89,18 @@ def __init__(self): 'mqn': _calculate_mqn_fp, } - def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None = os.cpu_count(), + def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None = None, return_valid_idx: bool = False, **params): """ Generate fingerprints for a list of SMILES strings in parallel. The method selects the appropriate fingerprint function based on the 'fp' parameter, binds additional keyword parameters using functools.partial, and then applies the function - across the SMILES list using multiprocessing.Pool.map. + across the SMILES list in a pool of worker processes. + + On Windows and macOS, worker processes re-import the calling script, so + a script that calls this must do so under ``if __name__ == "__main__":``. + Notebooks are not affected. Some SMILES may fail to parse; those rows are dropped. By default you get back only the surviving fingerprints, and you cannot tell *which* inputs @@ -111,8 +116,8 @@ def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None Args: smiles_list (list): A list of SMILES strings. fp (str): The fingerprint type to compute (e.g., 'morgan'). - nprocesses (int): Number of worker processes. Defaults to all CPU cores. - Use 1 to compute in the current process (no multiprocessing). + nprocesses (int): Number of worker processes. Defaults to all CPU cores + (at most 61 on Windows). Use 1 to compute in the current process. return_valid_idx (bool): If True, return ``(valid_idx, fingerprints)`` where ``valid_idx`` is an int64 array of the input positions that parsed OK. If False (default), return only the fingerprints. @@ -127,6 +132,8 @@ def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None Raises: ValueError: If an unsupported fingerprint type is requested. + RuntimeError: If a worker process dies, e.g. a script on Windows or + macOS that is missing the ``__main__`` guard. """ func = self.fingerprint_function_map.get(fp) if func is None: @@ -141,8 +148,22 @@ def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None # No worker processes: avoids pool startup cost and spawn issues fingerprints = [part_func(s) for s in smiles] else: - with Pool(processes=nprocesses) as pool: - fingerprints = pool.map(part_func, smiles) + # A few chunks per worker keeps them busy without per-item overhead + n_workers = nprocesses or os.cpu_count() or 1 + chunksize = max(1, len(smiles) // (n_workers * 4)) + executor = ProcessPoolExecutor(max_workers=nprocesses) + try: + fingerprints = list(executor.map(part_func, smiles, chunksize=chunksize)) + except BrokenProcessPool as e: + # multiprocessing.Pool would respawn crashed workers forever here + raise RuntimeError( + "A fingerprint worker process crashed. On Windows and macOS this " + "usually means the script calling FingerprintFromSmiles is missing " + "an `if __name__ == \"__main__\":` guard around its code." + ) from e + finally: + # Results are collected; don't block while idle workers exit + executor.shutdown(wait=False, cancel_futures=True) if return_valid_idx: # Keep the input positions that produced a fingerprint, so the diff --git a/tests/test_fingerprints.py b/tests/test_fingerprints.py index 94fd53c..66e79f9 100644 --- a/tests/test_fingerprints.py +++ b/tests/test_fingerprints.py @@ -128,3 +128,28 @@ def test_fingerprint_calculator_serial_matches_pool(): np.testing.assert_array_equal(idx_serial, idx_pool) np.testing.assert_array_equal(fps_serial, fps_pool) assert fps_serial.shape == (3, 42) + + +def test_fingerprint_calculator_unguarded_script_errors_instead_of_hanging(tmp_path): + """A script without a __main__ guard under spawn (Windows default) must + fail fast with a helpful error, not respawn workers forever.""" + import os + import subprocess + import sys + from pathlib import Path + + script = tmp_path / "unguarded.py" + script.write_text( + "import multiprocessing as mp\n" + "mp.set_start_method('spawn', force=True)\n" + "from chelombus.utils.fingerprints import FingerprintCalculator\n" + "FingerprintCalculator().FingerprintFromSmiles(['CCO'] * 20, 'mqn', nprocesses=2)\n" + ) + repo_root = str(Path(__file__).resolve().parents[1]) + env = {**os.environ, "PYTHONPATH": repo_root + os.pathsep + os.environ.get("PYTHONPATH", "")} + + result = subprocess.run([sys.executable, str(script)], cwd=tmp_path, env=env, + capture_output=True, text=True, timeout=120) + + assert result.returncode != 0 + assert "__main__" in result.stderr and "guard" in result.stderr From cecb2d84089090ed46f48c33d9620ca6f2ab3b3c Mon Sep 17 00:00:00 2001 From: afloresep Date: Thu, 29 Oct 2026 12:53:40 +0100 Subject: [PATCH 03/10] cluster_io: run queries with duckdb.execute for DuckDB 2.0 DuckDB 2.0 relations no longer have fetchone(), which broke every cluster_io function that counts rows (and CI, which installs the latest DuckDB). duckdb.execute() returns a cursor with fetchone/fetchall/df on both 1.x and 2.0. --- chelombus/utils/cluster_io.py | 20 ++++++++++---------- tests/test_cluster_io.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/chelombus/utils/cluster_io.py b/chelombus/utils/cluster_io.py index 25f690c..ebf17ea 100644 --- a/chelombus/utils/cluster_io.py +++ b/chelombus/utils/cluster_io.py @@ -129,7 +129,7 @@ def query_cluster( WHERE {quoted_cluster_column} = {cluster_id} """ - result = duckdb.query(query).df() + result = duckdb.execute(query).df() if result.empty: logger.warning(f"No molecules found for cluster_id={cluster_id}") @@ -184,7 +184,7 @@ def export_cluster( FROM '{glob_pattern}' WHERE {quoted_cluster_column} = {cluster_id} """ - count = duckdb.query(count_query).fetchone()[0] # type: ignore + count = duckdb.execute(count_query).fetchone()[0] # type: ignore if count == 0: logger.warning(f"No molecules found for cluster_id={cluster_id}") @@ -226,7 +226,7 @@ def export_cluster( "Use .parquet, .csv, .smi, or .txt" ) - duckdb.query(export_query) + duckdb.execute(export_query) logger.info(f"Exported {count} molecules to {output_path}") return count @@ -319,7 +319,7 @@ def export_all_clusters( """ logger.info("Starting partitioned export (streaming, memory-efficient)...") - duckdb.query(export_query) + duckdb.execute(export_query) # Get cluster statistics after export logger.info("Calculating cluster statistics...") @@ -373,7 +373,7 @@ def get_cluster_stats( ORDER BY {quoted_column} """ - return duckdb.query(query).df() + return duckdb.execute(query).df() def get_cluster_ids( @@ -404,7 +404,7 @@ def get_cluster_ids( ORDER BY {_quote_identifier(cluster_id_column_name)} """ - result = duckdb.query(query).fetchall() + result = duckdb.execute(query).fetchall() return [row[0] for row in result] @@ -426,7 +426,7 @@ def get_total_molecules( glob_pattern = str(path / "*.parquet") query = f"SELECT COUNT(*) FROM '{glob_pattern}'" - return duckdb.query(query).fetchone()[0] # type: ignore + return duckdb.execute(query).fetchone()[0] # type: ignore def query_clusters_batch( @@ -468,7 +468,7 @@ def query_clusters_batch( WHERE {quoted_cluster_column} IN ({ids_str}) """ - return duckdb.query(query).df() + return duckdb.execute(query).df() def sample_from_cluster( @@ -502,7 +502,7 @@ def sample_from_cluster( # Use setseed() for reproducibility when random_state is provided if random_state is not None: - duckdb.query(f"SELECT setseed({random_state / 2**31})") + duckdb.execute(f"SELECT setseed({random_state / 2**31})") # Use ORDER BY RANDOM() for sampling, which is more portable query = f""" @@ -513,4 +513,4 @@ def sample_from_cluster( LIMIT {n} """ - return duckdb.query(query).df() + return duckdb.execute(query).df() diff --git a/tests/test_cluster_io.py b/tests/test_cluster_io.py index 3829317..652fdaa 100644 --- a/tests/test_cluster_io.py +++ b/tests/test_cluster_io.py @@ -453,7 +453,7 @@ def test_export_all_and_query_back(self, sample_results_dir, tmp_path): stats = export_all_clusters(sample_results_dir, output_dir) # Query from partitioned output using DuckDB directly - df = duckdb.query(f""" + df = duckdb.execute(f""" SELECT * FROM '{output_dir}/*/*.parquet' WHERE cluster_id = 0 """).df() From cb45c09ea02fd7c3d97e49702993ae43e52195f7 Mon Sep 17 00:00:00 2001 From: afloresep Date: Thu, 29 Oct 2026 12:47:20 +0100 Subject: [PATCH 04/10] Drop invalid SMILES in TMAP builders, fix chelombus-tmap --cluster-file create_tmap and representative_tmap crashed on any unparseable SMILES because tmap2 skips it when fingerprinting while the property layers, SMILES and cluster ids still had one entry per input. Both now share one helper that filters with tmap2's return_valid flags. chelombus-tmap --cluster-file no longer also requires --smiles. --- chelombus/utils/visualization.py | 89 +++++++++++++++----------------- tests/test_visualization.py | 39 +++++++++++++- 2 files changed, 80 insertions(+), 48 deletions(-) diff --git a/chelombus/utils/visualization.py b/chelombus/utils/visualization.py index d7099c0..829caa9 100644 --- a/chelombus/utils/visualization.py +++ b/chelombus/utils/visualization.py @@ -26,7 +26,7 @@ def create_tmap( """Create a TMAP visualization from SMILES strings. Args: - smiles: List of SMILES strings to visualize. + smiles: List of SMILES strings to visualize. Unparseable SMILES are skipped. fingerprint: Fingerprint type ('morgan' or 'mqn'). fingerprint_bits: Number of bits for Morgan fingerprint (default: 2048). fingerprint_radius: Radius for Morgan fingerprint (default: 2). @@ -35,35 +35,8 @@ def create_tmap( tmap_name: Name for the output TMAP HTML file. k_neighbors: Number of neighbors for layout (default: 20). """ - if properties is None: - properties = DEFAULT_PROPERTIES - - _validate_properties(properties) - - # Compute fingerprints via tmap2 - fp_kwargs = {} - if fingerprint == 'morgan': - fp_kwargs = {'n_bits': fingerprint_bits, 'radius': fingerprint_radius} - - fps = fingerprints_from_smiles(smiles, fp_type=fingerprint, **fp_kwargs) - - # Fit TMAP, use euclidean for dense descriptors like MQN - metric = 'euclidean' if fingerprint == 'mqn' else 'jaccard' - model = TMAP(metric=metric, n_neighbors=k_neighbors).fit(fps) - - # Build visualization - viz = model.to_tmapviz(include_edges=True) - viz.title = tmap_name - viz.background_color = "#FFFFFF" - - # Add molecular properties as color layers - props = molecular_properties(smiles, properties=properties) - for prop_name, values in props.items(): - viz.add_color_layout(prop_name, values, categorical=False, color='rainbow') - - # Add SMILES for structure rendering in tooltips - viz.add_smiles(smiles) - + viz, _ = _build_viz(smiles, fingerprint, fingerprint_bits, fingerprint_radius, + properties, tmap_name, k_neighbors) viz.write_html(f"{tmap_name}.html") @@ -93,6 +66,35 @@ def representative_tmap( tmap_name: Name for the output TMAP HTML file. k_neighbors: Number of neighbors for KNN graph (default: 21). """ + viz, valid = _build_viz(smiles, fingerprint, fingerprint_bits, fingerprint_radius, + properties, tmap_name, k_neighbors) + + # Add cluster ID as categorical color channel + if cluster_ids is not None: + viz.add_color_layout( + 'Cluster ID', + [str(cid) for cid, ok in zip(cluster_ids, valid) if ok], + categorical=True, + color='tab20', + ) + + viz.write_html(f"{tmap_name}.html") + + +def _build_viz( + smiles: list[str], + fingerprint: str, + fingerprint_bits: int, + fingerprint_radius: int, + properties: list[str] | None, + tmap_name: str, + k_neighbors: int, +): + """Fit a TMAP and return ``(viz, valid)`` with property layers and SMILES added. + + SMILES that RDKit cannot parse are dropped, so every layer lines up with the + points. ``valid`` holds one flag per input SMILES, for aligning extra layers. + """ if properties is None: properties = DEFAULT_PROPERTIES @@ -103,7 +105,8 @@ def representative_tmap( if fingerprint == 'morgan': fp_kwargs = {'n_bits': fingerprint_bits, 'radius': fingerprint_radius} - fps = fingerprints_from_smiles(smiles, fp_type=fingerprint, **fp_kwargs) + fps, valid = fingerprints_from_smiles(smiles, fp_type=fingerprint, return_valid=True, **fp_kwargs) + smiles = [smi for smi, ok in zip(smiles, valid) if ok] # Fit TMAP, use euclidean for dense descriptors like MQN metric = 'euclidean' if fingerprint == 'mqn' else 'jaccard' @@ -119,19 +122,10 @@ def representative_tmap( for prop_name, values in props.items(): viz.add_color_layout(prop_name, values, categorical=False, color='rainbow') - # Add cluster ID as categorical color channel - if cluster_ids is not None: - viz.add_color_layout( - 'Cluster ID', - [str(cid) for cid in cluster_ids], - categorical=True, - color='tab20', - ) - # Add SMILES for structure rendering in tooltips viz.add_smiles(smiles) - viz.write_html(f"{tmap_name}.html") + return viz, valid def _validate_properties(properties: list[str]) -> None: @@ -154,11 +148,14 @@ def main(): Examples: chelombus-tmap --smiles molecules.smi chelombus-tmap --smiles molecules.smi --fingerprint mqn --properties mw,logp,qed - chelombus-tmap --smiles molecules.smi --cluster-file clusters.csv + chelombus-tmap --cluster-file clusters.csv """ ) - parser.add_argument('--smiles', type=str, required=True, - help="Path to file containing SMILES (one per line)") + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument('--smiles', type=str, + help="Path to file containing SMILES (one per line)") + source.add_argument('--cluster-file', type=str, + help="CSV/parquet file with 'smiles' and 'cluster_id' columns for representative TMAP") parser.add_argument('--fingerprint', type=str, default='morgan', choices=['morgan', 'mqn'], help="Fingerprint type (default: morgan)") @@ -170,8 +167,6 @@ def main(): help=f"Comma-separated properties (default: all). Available: {', '.join(AVAILABLE_PROPERTIES)}") parser.add_argument('--output', type=str, default='my_tmap', help="Output TMAP name (default: my_tmap)") - parser.add_argument('--cluster-file', type=str, default=None, - help="CSV/parquet file with 'smiles' and 'cluster_id' columns for representative TMAP") parser.add_argument('--representative', action='store_true', help="Use representative_tmap layout (edge list based)") @@ -204,7 +199,7 @@ def main(): tmap_name=args.output, ) else: - with open(args.smiles, 'r') as file: + with open(args.smiles, 'r', encoding='utf-8') as file: smiles = [line.strip() for line in file.readlines() if line.strip()] if args.representative: diff --git a/tests/test_visualization.py b/tests/test_visualization.py index 68a6d40..19efa74 100644 --- a/tests/test_visualization.py +++ b/tests/test_visualization.py @@ -2,7 +2,7 @@ pytest.importorskip("tmap") -from chelombus.utils.visualization import create_tmap +from chelombus.utils.visualization import create_tmap, representative_tmap SMILES = ["C" * i + "O" for i in range(1, 30)] + ["c1ccccc1" + "C" * i for i in range(1, 20)] @@ -16,3 +16,40 @@ def test_create_tmap_writes_html(fingerprint, tmp_path, monkeypatch): tmap_name="test_map", k_neighbors=5) assert (tmp_path / "test_map.html").exists() + + +@pytest.mark.parametrize("fingerprint", ["mqn", "morgan"]) +def test_create_tmap_skips_invalid_smiles(fingerprint, tmp_path, monkeypatch): + """Unparseable SMILES are dropped instead of misaligning property layers.""" + monkeypatch.chdir(tmp_path) + + create_tmap(SMILES + ["not_a_smiles"], fingerprint=fingerprint, properties=["mw"], + tmap_name="test_map", k_neighbors=5) + + assert (tmp_path / "test_map.html").exists() + + +def test_representative_tmap_aligns_cluster_ids(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + smiles = SMILES[:10] + ["not_a_smiles"] + SMILES[10:] + representative_tmap(smiles, cluster_ids=list(range(len(smiles))), + properties=["mw"], tmap_name="rep", k_neighbors=5) + + assert (tmp_path / "rep.html").exists() + + +def test_cli_cluster_file_without_smiles(tmp_path, monkeypatch): + """--cluster-file alone is enough (it used to also require --smiles).""" + import pandas as pd + from chelombus.utils import visualization + + monkeypatch.chdir(tmp_path) + pd.DataFrame({"smiles": SMILES, "cluster_id": [i % 3 for i in range(len(SMILES))]}).to_csv( + tmp_path / "reps.csv", index=False) + monkeypatch.setattr("sys.argv", ["chelombus-tmap", "--cluster-file", "reps.csv", + "--properties", "mw", "--output", "cli_map"]) + + visualization.main() + + assert (tmp_path / "cli_map.html").exists() From e3a9f0686385b04e7439a840bcf5715a7f8a2854 Mon Sep 17 00:00:00 2001 From: afloresep Date: Thu, 29 Oct 2026 12:47:20 +0100 Subject: [PATCH 05/10] cluster_io: reproducible sampling, configurable columns, safe paths - sample_from_cluster(random_state=) now returns the same sample every time (setseed ran in a separate query and had no effect). Orders by a seeded row hash, independent of DuckDB threading. - sample_from_cluster and export_all_clusters take cluster/smiles column names; export_cluster takes smiles_column for .smi output. - Quote column lists and escape paths containing a quote. - Remove references to a nonexistent chelombus[io] extra and a nonexistent 'chelombus visualize' command. --- chelombus/utils/cluster_io.py | 114 +++++++++++++++++++--------------- tests/test_cluster_io.py | 41 +++++++++++- 2 files changed, 101 insertions(+), 54 deletions(-) diff --git a/chelombus/utils/cluster_io.py b/chelombus/utils/cluster_io.py index ebf17ea..08f7ced 100644 --- a/chelombus/utils/cluster_io.py +++ b/chelombus/utils/cluster_io.py @@ -42,7 +42,7 @@ def _check_duckdb_available() -> None: except ImportError: raise ImportError( "duckdb is required for cluster I/O operations. " - "Install it with: pip install chelombus[io] or pip install duckdb" + "Install it with: pip install duckdb" ) @@ -72,6 +72,17 @@ def _validate_results_dir(results_dir: Union[str, Path]) -> Path: return path +def _sql_string(value: Union[str, Path]) -> str: + """Quote a path as a SQL string literal (handles paths containing ').""" + escaped = str(value).replace("'", "''") + return f"'{escaped}'" + + +def _parquet_source(path: Path) -> str: + """SQL source for all chunked parquet files in a results directory.""" + return _sql_string(path / "*.parquet") + + def _quote_identifier(identifier: str) -> str: if identifier.startswith('"') and identifier.endswith('"'): return identifier @@ -114,7 +125,7 @@ def query_cluster( import duckdb path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) quoted_cluster_column = _quote_identifier(cluster_column) # Build column selection @@ -125,7 +136,7 @@ def query_cluster( query = f""" SELECT {col_str} - FROM '{glob_pattern}' + FROM {source} WHERE {quoted_cluster_column} = {cluster_id} """ @@ -143,6 +154,7 @@ def export_cluster( output_path: Union[str, Path], cluster_column: str = "cluster_id", columns: Optional[List[str]] = None, + smiles_column: str = "smiles", ) -> int: """Export a single cluster to a file. @@ -155,6 +167,7 @@ def export_cluster( output_path: Output file path. Must end with .parquet or .csv. cluster_column: Name of the column containing cluster IDs. columns: Optional list of columns to export. Default exports all columns. + smiles_column: Column written to .smi/.txt outputs. Returns: Number of molecules exported. @@ -172,16 +185,16 @@ def export_cluster( path = _validate_results_dir(results_dir) output_path = Path(output_path) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) quoted_cluster_column = _quote_identifier(cluster_column) # Build column selection - col_str = ", ".join(columns) if columns else "*" + col_str = ", ".join(_quote_identifier(col) for col in columns) if columns else "*" # First get the count count_query = f""" SELECT COUNT(*) as cnt - FROM '{glob_pattern}' + FROM {source} WHERE {quoted_cluster_column} = {cluster_id} """ count = duckdb.execute(count_query).fetchone()[0] # type: ignore @@ -199,26 +212,26 @@ def export_cluster( export_query = f""" COPY ( SELECT {col_str} - FROM '{glob_pattern}' + FROM {source} WHERE {quoted_cluster_column} = {cluster_id} - ) TO '{output_path}' (FORMAT PARQUET) + ) TO {_sql_string(output_path)} (FORMAT PARQUET) """ elif suffix == ".csv": export_query = f""" COPY ( SELECT {col_str} - FROM '{glob_pattern}' + FROM {source} WHERE {quoted_cluster_column} = {cluster_id} - ) TO '{output_path}' (FORMAT CSV, HEADER) + ) TO {_sql_string(output_path)} (FORMAT CSV, HEADER) """ elif suffix in (".smi", ".txt"): # For SMILES files, export only the smiles column without header export_query = f""" COPY ( - SELECT smiles - FROM '{glob_pattern}' + SELECT {_quote_identifier(smiles_column)} + FROM {source} WHERE {quoted_cluster_column} = {cluster_id} - ) TO '{output_path}' (FORMAT CSV, HEADER FALSE) + ) TO {_sql_string(output_path)} (FORMAT CSV, HEADER FALSE) """ else: raise ValueError( @@ -236,6 +249,8 @@ def export_all_clusters( results_dir: Union[str, Path], output_dir: Union[str, Path], format: str = "parquet", + cluster_column: str = "cluster_id", + smiles_column: str = "smiles", ) -> dict[int, int]: """Export all clusters to individual files using partitioned writes. @@ -260,8 +275,8 @@ def export_all_clusters( results_dir: Path to directory containing chunked parquet files. output_dir: Output directory for partitioned cluster files. format: Output format, either 'parquet' or 'csv'. Default 'parquet'. - batch_size: Number of clusters to process per batch for progress reporting. - Does not affect memory usage (DuckDB handles streaming internally). + cluster_column: Name of the column containing cluster IDs. + smiles_column: Name of the column containing SMILES. Returns: Dictionary mapping cluster_id to molecule count. @@ -276,24 +291,14 @@ def export_all_clusters( >>> print(f"Largest cluster: {max(stats.values())} molecules") Note: - For HPC/SLURM usage, you can process clusters in parallel: - - ```bash - # SLURM array job - #SBATCH --array=0-99999 - - chelombus visualize \\ - --input clusters/cluster_id=$SLURM_ARRAY_TASK_ID/ \\ - --output tmaps/cluster_$SLURM_ARRAY_TASK_ID.html - ``` + For HPC/SLURM usage, each array task can read its own partition, e.g. + ``clusters/cluster_id=$SLURM_ARRAY_TASK_ID/``. """ _check_duckdb_available() import duckdb - from tqdm import tqdm - path = _validate_results_dir(results_dir) output_path = Path(output_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) if format not in ("parquet", "csv"): raise ValueError(f"Unsupported format: {format}. Use 'parquet' or 'csv'") @@ -309,11 +314,11 @@ def export_all_clusters( export_query = f""" COPY ( - SELECT smiles, cluster_id - FROM '{glob_pattern}' - ) TO '{output_path}' ( + SELECT {_quote_identifier(smiles_column)}, {_quote_identifier(cluster_column)} + FROM {source} + ) TO {_sql_string(output_path)} ( FORMAT {format_str}, - PARTITION_BY (cluster_id), + PARTITION_BY ({_quote_identifier(cluster_column)}), OVERWRITE_OR_IGNORE ) """ @@ -323,7 +328,7 @@ def export_all_clusters( # Get cluster statistics after export logger.info("Calculating cluster statistics...") - stats = get_cluster_stats(results_dir) + stats = get_cluster_stats(results_dir, column=cluster_column) # Convert to dictionary result = dict(zip(stats["cluster_id"], stats["molecule_count"])) @@ -361,14 +366,14 @@ def get_cluster_stats( import duckdb path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) quoted_column = _quote_identifier(column) query = f""" SELECT {quoted_column} as cluster_id, COUNT(*) as molecule_count - FROM '{glob_pattern}' + FROM {source} GROUP BY {quoted_column} ORDER BY {quoted_column} """ @@ -384,6 +389,7 @@ def get_cluster_ids( Args: results_dir: Path to directory containing chunked parquet files. + cluster_id_column_name: Name of the column containing cluster IDs. Returns: Sorted list of cluster IDs. @@ -396,11 +402,11 @@ def get_cluster_ids( import duckdb path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) query = f""" SELECT DISTINCT {_quote_identifier(cluster_id_column_name)} - FROM '{glob_pattern}' + FROM {source} ORDER BY {_quote_identifier(cluster_id_column_name)} """ @@ -423,9 +429,9 @@ def get_total_molecules( import duckdb path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) - query = f"SELECT COUNT(*) FROM '{glob_pattern}'" + query = f"SELECT COUNT(*) FROM {source}" return duckdb.execute(query).fetchone()[0] # type: ignore @@ -444,6 +450,7 @@ def query_clusters_batch( results_dir: Path to directory containing chunked parquet files. cluster_ids: List of cluster IDs to query. columns: Optional list of columns to return. + cluster_column: Name of the column containing cluster IDs. Returns: DataFrame containing molecules from all specified clusters. @@ -457,14 +464,14 @@ def query_clusters_batch( quoted_cluster_column = _quote_identifier(cluster_column) path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) - col_str = ", ".join(columns) if columns else "*" + col_str = ", ".join(_quote_identifier(col) for col in columns) if columns else "*" ids_str = ", ".join(str(cid) for cid in cluster_ids) query = f""" SELECT {col_str} - FROM '{glob_pattern}' + FROM {source} WHERE {quoted_cluster_column} IN ({ids_str}) """ @@ -476,6 +483,7 @@ def sample_from_cluster( cluster_id: int, n: int = 100, random_state: Optional[int] = None, + cluster_column: str = "cluster_id", ) -> "pd.DataFrame": """Get a random sample of molecules from a cluster. @@ -485,7 +493,9 @@ def sample_from_cluster( results_dir: Path to directory containing chunked parquet files. cluster_id: The cluster ID to sample from. n: Number of molecules to sample. - random_state: Random seed for reproducibility. + random_state: Random seed. The same seed on the same data returns the + same sample. + cluster_column: Name of the column containing cluster IDs. Returns: DataFrame with sampled molecules. @@ -498,18 +508,20 @@ def sample_from_cluster( import duckdb path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) - # Use setseed() for reproducibility when random_state is provided - if random_state is not None: - duckdb.execute(f"SELECT setseed({random_state / 2**31})") + # A seeded hash of each row gives the same order regardless of how DuckDB + # splits the scan across threads (setseed + RANDOM() does not) + if random_state is None: + order_by = "RANDOM()" + else: + order_by = f"hash(t, {int(random_state)})" - # Use ORDER BY RANDOM() for sampling, which is more portable query = f""" SELECT * - FROM '{glob_pattern}' - WHERE cluster_id = {cluster_id} - ORDER BY RANDOM() + FROM {source} AS t + WHERE {_quote_identifier(cluster_column)} = {cluster_id} + ORDER BY {order_by} LIMIT {n} """ diff --git a/tests/test_cluster_io.py b/tests/test_cluster_io.py index 652fdaa..e7567e9 100644 --- a/tests/test_cluster_io.py +++ b/tests/test_cluster_io.py @@ -415,9 +415,10 @@ def test_sample_with_random_state(self, large_results_dir): df1 = sample_from_cluster(large_results_dir, cluster_id=0, n=5, random_state=42) df2 = sample_from_cluster(large_results_dir, cluster_id=0, n=5, random_state=42) - # Note: DuckDB's SAMPLE REPEATABLE should give same results - # but may not be exactly identical in all versions - assert len(df1) == len(df2) + df3 = sample_from_cluster(large_results_dir, cluster_id=0, n=5, random_state=7) + + assert df1["smiles"].tolist() == df2["smiles"].tolist() + assert df1["smiles"].tolist() != df3["smiles"].tolist() def test_sample_empty_cluster(self, sample_results_dir): """Test sampling from non-existent cluster.""" @@ -459,3 +460,37 @@ def test_export_all_and_query_back(self, sample_results_dir, tmp_path): """).df() assert len(df) == stats[0] + + +class TestPathsAndColumnNames: + """Paths with quotes and non-default column names.""" + + @pytest.fixture + def custom_dir(self, tmp_path): + results_dir = tmp_path / "O'Brien results" + results_dir.mkdir() + pd.DataFrame({ + "SMILES": ["CCO", "CC", "CCC", "CCCC"], + "label": [0, 1, 0, 1], + }).to_parquet(results_dir / "part.parquet", index=False) + return results_dir + + def test_query_path_with_quote(self, custom_dir): + df = query_cluster(custom_dir, cluster_id=0, cluster_column="label") + assert sorted(df["SMILES"]) == ["CCC", "CCO"] + + def test_sample_custom_cluster_column(self, custom_dir): + df = sample_from_cluster(custom_dir, cluster_id=1, n=5, cluster_column="label") + assert sorted(df["SMILES"]) == ["CC", "CCCC"] + + def test_export_smi_custom_smiles_column(self, custom_dir, tmp_path): + out = tmp_path / "it's.smi" + count = export_cluster(custom_dir, 0, out, cluster_column="label", smiles_column="SMILES") + assert count == 2 + assert sorted(out.read_text().split()) == ["CCC", "CCO"] + + def test_export_all_custom_columns(self, custom_dir, tmp_path): + stats = export_all_clusters(custom_dir, tmp_path / "out", cluster_column="label", + smiles_column="SMILES") + assert stats == {0: 2, 1: 2} + assert (tmp_path / "out" / "label=0").is_dir() From 13100172118a4b78f4ab22320fb88da5a415d65f Mon Sep 17 00:00:00 2001 From: afloresep Date: Thu, 29 Oct 2026 12:47:20 +0100 Subject: [PATCH 06/10] DataStreamer: read .gz/.bz2, globs, CSV; stable file order - Decompress .gz and .bz2 on the fly (including .sdf.gz). - Read text as UTF-8 regardless of platform default. - Accept glob patterns, as the pipeline --input help already promised. - New sep argument for comma separated input (CSV was silently dropped). - Read directories in sorted order and skip hidden files. - Raise on an invalid input type instead of logging and continuing. --- chelombus/streamer/data_streamer.py | 60 ++++++++++++++++++++++------- chelombus/utils/helper_functions.py | 9 +++-- tests/test_data_streamer.py | 51 ++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 18 deletions(-) diff --git a/chelombus/streamer/data_streamer.py b/chelombus/streamer/data_streamer.py index 225a30d..30d6bd6 100644 --- a/chelombus/streamer/data_streamer.py +++ b/chelombus/streamer/data_streamer.py @@ -1,3 +1,6 @@ +import bz2 +import glob +import gzip import os from typing import Iterator, List, Optional @@ -9,13 +12,15 @@ class DataStreamer: """ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose: int = 0, smiles_col: int = 0, id_col: Optional[int] = None, - with_ids: bool = False) -> Iterator[List]: + with_ids: bool = False, sep: Optional[str] = None) -> Iterator[List]: """ Read input data from a file or a directory of files and yield it in chunks. - The input path can be a single file or a directory of files. Plain-text + The input path can be a single file, a directory of files, or a glob + pattern like "data/*.smi". Files ending in .gz or .bz2 are decompressed + on the fly, and text is read as UTF-8. Plain-text files are read line by line; the whitespace-separated column at - `smiles_col` (0-based) is taken as the SMILES. So a file with ONE SMILES + `smiles_col` (0-based) is taken as the SMILES (pass `sep=","` for CSV). So a file with ONE SMILES per line works as-is (smiles_col=0), and a file like "idsmiles" works with smiles_col=1. SDF/SD files are streamed with RDKit and turned into SMILES on the fly. Items are buffered until `chunksize` is reached, @@ -34,7 +39,8 @@ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose: Ids are always returned as strings. Args: - input_path (str): Path to a file or directory of input files. + input_path (str): Path to a file, a directory of input files, or a + glob pattern. Directory and glob matches are read in sorted order. chunksize (Optional[int]): Items per yielded chunk. None -> one chunk per file. verbose (int): 0 silent, 1 prints each file name. @@ -43,28 +49,35 @@ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose: id_col (Optional[int]): 0-based column holding the id, or None to make one up. Passing it turns on (id, smiles) output. with_ids (bool): Yield (id, smiles) tuples instead of bare SMILES. + sep (Optional[str]): Column separator for text inputs, e.g. "," for + CSV. None splits on any whitespace. Yields: List: a chunk of SMILES strings, or of (id, smiles) tuples when ids are requested. """ # Sanity check - if not os.path.exists(input_path): + if glob.has_magic(input_path): + input_paths = sorted(glob.glob(input_path)) + if not input_paths: + raise FileNotFoundError(f"No files match {input_path}.") + elif os.path.exists(input_path): + input_paths = input_path + else: raise FileNotFoundError(f"Data source {input_path} not found.") want_ids = with_ids or id_col is not None buffer: List = [] next_id = 0 # running counter for made-up ids (all files, in order) - for file_path in _process_input(input_paths=input_path): + for file_path in _process_input(input_paths=input_paths): if verbose == 1: print("Processing file:", file_path) - file_ext = os.path.splitext(file_path)[1].lower() - if file_ext in {".sdf", ".sd"}: + if _file_ext(file_path) in {".sdf", ".sd"}: records = self._stream_sdf(file_path) else: - records = self._stream_text(file_path, smiles_col, id_col) + records = self._stream_text(file_path, smiles_col, id_col, sep) for raw_id, smiles in records: if want_ids: @@ -83,14 +96,14 @@ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose: buffer.clear() def _stream_text(self, file_path: str, smiles_col: int, - id_col: Optional[int] = None) -> Iterator[tuple]: + id_col: Optional[int] = None, sep: Optional[str] = None) -> Iterator[tuple]: """Yield (raw_id, smiles) per line. raw_id is the value in column `id_col`, or None when there is no id column. Blank / too-short lines (no column at `smiles_col`) are skipped.""" - with open(file_path, "r") as file: + with _open(file_path) as file: for line in file: - parts = line.split() - if smiles_col >= len(parts): + parts = [part.strip() for part in line.split(sep)] + if smiles_col >= len(parts) or not parts[smiles_col]: continue smiles = parts[smiles_col] raw_id = parts[id_col] if (id_col is not None and id_col < len(parts)) else None @@ -105,7 +118,7 @@ def _stream_sdf(self, file_path: str) -> Iterator[tuple]: ) from exc # ForwardSDMolSupplier streams molecules without loading the full file in memory - with open(file_path, "rb") as file_handle: + with _open(file_path, binary=True) as file_handle: supplier = Chem.ForwardSDMolSupplier(file_handle) for mol in supplier: if mol is None: @@ -118,3 +131,22 @@ def _stream_sdf(self, file_path: str) -> Iterator[tuple]: # Use the SD record title as the id when present. name = mol.GetProp("_Name").strip() if mol.HasProp("_Name") else "" yield (name or None), smiles + + +_COMPRESSED = {".gz": gzip.open, ".bz2": bz2.open} + + +def _file_ext(file_path: str) -> str: + """Lowercase extension, ignoring a compression suffix (x.sdf.gz -> .sdf).""" + root, ext = os.path.splitext(file_path.lower()) + if ext in _COMPRESSED: + ext = os.path.splitext(root)[1] + return ext + + +def _open(file_path: str, binary: bool = False): + """Open a plain, .gz or .bz2 file as UTF-8 text (or bytes).""" + opener = _COMPRESSED.get(os.path.splitext(file_path)[1].lower(), open) + if binary: + return opener(file_path, "rb") + return opener(file_path, "rt", encoding="utf-8") diff --git a/chelombus/utils/helper_functions.py b/chelombus/utils/helper_functions.py index e0ba58a..ed3fe60 100644 --- a/chelombus/utils/helper_functions.py +++ b/chelombus/utils/helper_functions.py @@ -20,14 +20,15 @@ def _process_input(input_paths): # Single path (file or folder) input_paths = [input_paths] elif not isinstance(input_paths, (list, tuple)): - logging.error(f"Invalid input type: {type(input_paths)}. Provide a string, list, or tuple. Skipping") + raise TypeError(f"Invalid input type: {type(input_paths)}. Provide a string, list, or tuple.") for input_path in input_paths: if os.path.isdir(input_path): - # Process all files in the directory - for file in os.listdir(input_path): + # Process all files in the directory, in a stable order, skipping + # hidden files like .DS_Store + for file in sorted(os.listdir(input_path)): file_path = os.path.join(input_path, file) - if os.path.isfile(file_path): # Optional: filter by file extensions + if os.path.isfile(file_path) and not file.startswith("."): yield file_path elif os.path.isfile(input_path): # Process a single file diff --git a/tests/test_data_streamer.py b/tests/test_data_streamer.py index ad02124..a4ed692 100644 --- a/tests/test_data_streamer.py +++ b/tests/test_data_streamer.py @@ -66,3 +66,54 @@ def test_parse_input_omits_invalid_line_without_chunksize(tmp_path, monkeypatch) # Expected: one chunk containing only the valid lines. assert result == [["valid_line1", "valid_line2"]] + + +def test_parse_input_gzip(tmp_path: Path): + import gzip + with gzip.open(tmp_path / "mols.smi.gz", "wt") as f: + f.write("CCO\nc1ccccc1\n") + + result = list(DataStreamer().parse_input(str(tmp_path / "mols.smi.gz"))) + assert result == [["CCO", "c1ccccc1"]] + + +def test_parse_input_gzip_sdf(tmp_path: Path): + import gzip + from rdkit import Chem + with gzip.open(tmp_path / "mols.sdf.gz", "wt") as f: + for smi in ("CCO", "CCN"): + f.write(Chem.MolToMolBlock(Chem.MolFromSmiles(smi)) + "$$$$\n") + + result = list(DataStreamer().parse_input(str(tmp_path / "mols.sdf.gz"))) + assert result == [["CCO", "CCN"]] + + +def test_parse_input_glob_sorted(tmp_path: Path): + (tmp_path / "b.smi").write_text("CCN\n") + (tmp_path / "a.smi").write_text("CCO\n") + (tmp_path / "notes.md").write_text("not smiles\n") + + result = list(DataStreamer().parse_input(str(tmp_path / "*.smi"))) + assert result == [["CCO"], ["CCN"]] + + +def test_parse_input_glob_no_match(tmp_path: Path): + with pytest.raises(FileNotFoundError): + list(DataStreamer().parse_input(str(tmp_path / "*.smi"))) + + +def test_parse_input_directory_sorted_skips_hidden(tmp_path: Path): + (tmp_path / "b.smi").write_text("CCN\n") + (tmp_path / "a.smi").write_text("CCO\n") + (tmp_path / ".DS_Store").write_bytes(b"\x00\x01") + + result = list(DataStreamer().parse_input(str(tmp_path))) + assert result == [["CCO"], ["CCN"]] + + +def test_parse_input_csv_sep(tmp_path: Path): + (tmp_path / "mols.csv").write_text("id1, CCO\nid2,CCN\n\n") + + result = list(DataStreamer().parse_input(str(tmp_path / "mols.csv"), smiles_col=1, + id_col=0, sep=",")) + assert result == [[("id1", "CCO"), ("id2", "CCN")]] From 857230cf5f11dbee54f820896bfe211ac155cbbe Mon Sep 17 00:00:00 2001 From: afloresep Date: Thu, 29 Oct 2026 12:47:20 +0100 Subject: [PATCH 07/10] Load PQEncoder/PQKMeans lazily, tolerate broken torch installs import chelombus no longer imports torch, numba and pqkmeans up front (0.23s instead of ~1.3s). This matters most for fingerprint worker processes on spawn platforms, which re-import the package. A torch install that fails with something other than ImportError (e.g. a missing DLL) now falls back to CPU instead of breaking the import. FingerprintFromSmiles warns when no SMILES in a batch parse, which usually means the wrong column or separator. --- chelombus/__init__.py | 28 +++++++++++++++++++--------- chelombus/clustering/PyQKmeans.py | 2 +- chelombus/encoder/encoder.py | 2 +- chelombus/utils/fingerprints.py | 7 +++++++ tests/test_fingerprints.py | 8 ++++++++ 5 files changed, 36 insertions(+), 11 deletions(-) diff --git a/chelombus/__init__.py b/chelombus/__init__.py index 9524403..425c65c 100644 --- a/chelombus/__init__.py +++ b/chelombus/__init__.py @@ -5,17 +5,27 @@ """ from .streamer.data_streamer import DataStreamer -from .encoder.encoder import PQEncoder from .utils.fingerprints import FingerprintCalculator from .utils.helper_functions import save_chunk, format_time -# Optional imports - only available if dependencies are installed -try: - from .clustering.PyQKmeans import PQKMeans -except ImportError as e: - import warnings - warnings.warn(f"PQKMeans not available: {e}") - PQKMeans = None # pqkmeans or numba not installed + +def __getattr__(name): + # PQEncoder and PQKMeans pull in torch, numba and pqkmeans, so they load on + # first use. This keeps `import chelombus` fast, which matters most for + # fingerprint worker processes on Windows and macOS (each re-imports it). + if name == "PQEncoder": + from .encoder.encoder import PQEncoder as value + elif name == "PQKMeans": + try: + from .clustering.PyQKmeans import PQKMeans as value + except ImportError as e: + import warnings + warnings.warn(f"PQKMeans not available: {e}") + value = None # pqkmeans or numba not installed + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + globals()[name] = value + return value # Cluster I/O utilities (require duckdb) try: @@ -36,7 +46,7 @@ def _cluster_io_not_available(*args, **kwargs): raise ImportError( "Cluster I/O functions require duckdb. " - "Install with: pip install chelombus[io] or pip install duckdb" + "Install with: pip install duckdb" ) query_cluster = _cluster_io_not_available export_cluster = _cluster_io_not_available diff --git a/chelombus/clustering/PyQKmeans.py b/chelombus/clustering/PyQKmeans.py index 6c180fe..77341a2 100644 --- a/chelombus/clustering/PyQKmeans.py +++ b/chelombus/clustering/PyQKmeans.py @@ -24,7 +24,7 @@ if torch.cuda.is_available(): from chelombus.clustering._gpu_predict import predict_gpu _GPU_AVAILABLE = True -except ImportError: +except Exception: # not installed, or a broken install (e.g. missing DLL) pass diff --git a/chelombus/encoder/encoder.py b/chelombus/encoder/encoder.py index f331121..a70ec04 100644 --- a/chelombus/encoder/encoder.py +++ b/chelombus/encoder/encoder.py @@ -11,7 +11,7 @@ import torch if torch.cuda.is_available(): _GPU_AVAILABLE = True -except ImportError: +except Exception: # not installed, or a broken install (e.g. missing DLL) pass diff --git a/chelombus/utils/fingerprints.py b/chelombus/utils/fingerprints.py index 6399978..aae99f3 100644 --- a/chelombus/utils/fingerprints.py +++ b/chelombus/utils/fingerprints.py @@ -165,6 +165,13 @@ def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None # Results are collected; don't block while idle workers exit executor.shutdown(wait=False, cancel_futures=True) + if smiles and all(fp_ is None for fp_ in fingerprints): + # Usually a reading problem (wrong column or separator), not bad molecules + logger.warning( + "None of the %d SMILES could be parsed (first: %r). If reading with " + "DataStreamer, check smiles_col and sep.", len(smiles), smiles[0] + ) + if return_valid_idx: # Keep the input positions that produced a fingerprint, so the # caller can re-align ids / SMILES that travel with the molecules. diff --git a/tests/test_fingerprints.py b/tests/test_fingerprints.py index 66e79f9..d0dae0f 100644 --- a/tests/test_fingerprints.py +++ b/tests/test_fingerprints.py @@ -153,3 +153,11 @@ def test_fingerprint_calculator_unguarded_script_errors_instead_of_hanging(tmp_p assert result.returncode != 0 assert "__main__" in result.stderr and "guard" in result.stderr + + +def test_fingerprint_calculator_warns_when_nothing_parses(caplog): + with caplog.at_level(logging.WARNING): + fps = FingerprintCalculator().FingerprintFromSmiles(["CCO,1", "CCN,2"], 'mqn', nprocesses=1) + + assert len(fps) == 0 + assert "None of the 2 SMILES could be parsed" in caplog.text From 5ba4be4a9e0b3f5d9be8568806ee8ae7466e4b81 Mon Sep 17 00:00:00 2001 From: afloresep Date: Thu, 29 Oct 2026 12:47:20 +0100 Subject: [PATCH 08/10] pipeline: keep smiles when some fail to parse; portable run_all.sh - 01_train_models wrote chunks without the smiles column whenever any SMILES failed, breaking DuckDB globs over mixed schemas. Align with return_valid_idx like 02_assign_clusters. - run_all.sh derives REPO from its own location and treats an empty ID_COL as no id column. - README: move an inline comment that broke a line continuation. --- scripts/pipeline/01_train_models.py | 22 ++++++++-------------- scripts/pipeline/README.md | 7 +++++-- scripts/pipeline/run_all.sh | 12 ++++++------ 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/scripts/pipeline/01_train_models.py b/scripts/pipeline/01_train_models.py index bbd5117..1866bc3 100644 --- a/scripts/pipeline/01_train_models.py +++ b/scripts/pipeline/01_train_models.py @@ -39,7 +39,6 @@ import argparse import gc -import glob import json import os import sys @@ -534,8 +533,10 @@ def stage5_cluster_all_data( chunk_idx += 1 continue - # Calculate fingerprints - fps = fp_calc.FingerprintFromSmiles(smiles_chunk, fp="mqn") + # Calculate fingerprints. valid_idx keeps SMILES and labels aligned + # when some SMILES fail to parse. + valid_idx, fps = fp_calc.FingerprintFromSmiles(smiles_chunk, fp="mqn", + return_valid_idx=True) if len(fps) == 0: chunk_idx += 1 @@ -547,17 +548,10 @@ def stage5_cluster_all_data( # Predict clusters cluster_ids = clusterer.predict(pq_codes) - # Create output DataFrame - if len(fps) == len(smiles_chunk): - output_df = pd.DataFrame({ - "smiles": smiles_chunk, - "cluster_id": cluster_ids, - }) - else: - # Some SMILES failed - can't reliably associate - output_df = pd.DataFrame({ - "cluster_id": cluster_ids, - }) + output_df = pd.DataFrame({ + "smiles": [smiles_chunk[i] for i in valid_idx], + "cluster_id": cluster_ids, + }) # Save chunk output_file = os.path.join(results_dir, f"chunk_{chunk_idx:05d}.parquet") diff --git a/scripts/pipeline/README.md b/scripts/pipeline/README.md index b979338..ce52ba7 100644 --- a/scripts/pipeline/README.md +++ b/scripts/pipeline/README.md @@ -120,6 +120,9 @@ Result: `OUT/models/encoder.joblib` and `OUT/models/clusterer.joblib`. This is the one pass over **all** your molecules. It uses the GPU. It writes parquet files with three columns: `id`, `smiles`, `cluster_id`. +If your file has no id column, leave out `--id-col 0` (molecules are then +numbered 0, 1, 2, ...). `--resume` skips chunks already done if you restart. + ```bash $PQ scripts/pipeline/02_assign_clusters.py \ --input all_molecules.smi \ @@ -127,9 +130,9 @@ $PQ scripts/pipeline/02_assign_clusters.py \ --encoder OUT/models/encoder.joblib \ --clusterer OUT/models/clusterer.joblib \ --smiles-col 0 \ - --id-col 0 \ # DELETE this line if your file has no id column + --id-col 0 \ --device gpu \ - --resume # skip chunks already done if you restart + --resume ``` ### Step 3 — Group (put each cluster together) diff --git a/scripts/pipeline/run_all.sh b/scripts/pipeline/run_all.sh index 08d9fc3..b720e2e 100644 --- a/scripts/pipeline/run_all.sh +++ b/scripts/pipeline/run_all.sh @@ -25,22 +25,22 @@ PQ=/home/afloresep/miniforge3/envs/pqkmeans/bin/python # compute / GPU TM=/home/afloresep/miniforge3/envs/tmap2/bin/python # tmap2 viz # --- Where the code lives ---------------------------------------------------- -REPO=/mnt/10tb_hdd/work/Chelombus +REPO=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) # this checkout P=$REPO/scripts/pipeline # all pipeline scripts (01..06 + helpers) live here # --- Your data and where results go (EDIT THESE) ----------------------------- SRC=/mnt/10tb_hdd/cleaned_enamine_10b/output_file_0.cxsmiles # input file(s) ROOT=/mnt/10tb_hdd/enamine_1b # all output here SMILES_COL=1 # 0-based column that holds the SMILES in $SRC -ID_COL=0 # 0-based column that holds the molecule id (see note below) +ID_COL=0 # 0-based column that holds the molecule id, or empty (see note below) K=50000 # number of clusters TRAIN_SAMPLE=100000000 # molecules used to TRAIN (you don't need all 1B) MAX_CLUSTERS=1000 # how many drill-down maps to build in this run # # NOTE on columns: if your file is just ONE SMILES per line with nothing else, -# set SMILES_COL=0 and delete the "--id-col $ID_COL" text in STEP 2 below; the -# molecules will then be auto-numbered (0,1,2,...). Set ID_COL only when your -# file really has an id column you want to keep. +# set SMILES_COL=0 and ID_COL= (empty); the molecules will then be +# auto-numbered (0,1,2,...). Set ID_COL only when your file really has an id +# column you want to keep. # ---------------------------------------------------------------------------- SAMPLE=$ROOT/train_sample.smi @@ -72,7 +72,7 @@ say "STEP 2: assigning all molecules to clusters (full pass, --resume)" "$PQ" "$P/02_assign_clusters.py" --input "$SRC" --output "$ROOT/labels" \ --encoder "$ROOT/models/encoder.joblib" \ --clusterer "$ROOT/models/clusterer.joblib" \ - --smiles-col "$SMILES_COL" --id-col "$ID_COL" \ + --smiles-col "$SMILES_COL" ${ID_COL:+--id-col "$ID_COL"} \ --device gpu --resume || die "assign" # ── STEP 3: GROUP molecules by cluster (DuckDB) ────────────────────────────── From 6e8a9b143dd5504182971d430f1099492fac8b85 Mon Sep 17 00:00:00 2001 From: afloresep Date: Thu, 29 Oct 2026 12:47:20 +0100 Subject: [PATCH 09/10] Docs and notebooks: fix Quick Start, Windows note, stale API - README Quick Start used a DataStreamer signature that does not exist. - Note that Windows is not supported (use WSL2) and that scripts need a __main__ guard on macOS/Windows. - Triton ships with torch on Linux only. - tutorial.ipynb: drop a cell loading a local model, use iterations=, point cluster stats at the tutorial's own results, note alignment. - enamine_1B_clustering.ipynb: iterations=. --- README.md | 11 ++++++++--- docs/getting-started.md | 10 ++++++++-- examples/enamine_1B_clustering.ipynb | 2 +- examples/tutorial.ipynb | 27 +++++++++------------------ 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 0f74d90..0e8b3d5 100644 --- a/README.md +++ b/README.md @@ -45,11 +45,15 @@ pip install -e . **Apple Silicon (M1/M2/M3)**: The `pqkmeans` library is not currently supported on Apple Silicon Macs. My plan is to rewrite pqkmeans with Silicon and GPU support but that's for a future release... For now, clustering functionality requires an x86_64 system. +**Windows**: Not supported natively. `pqkmeans` has no Windows wheels (building it needs MSVC and CMake), and the GPU kernels need Triton, which is Linux only. Use [WSL2](https://learn.microsoft.com/windows/wsl/install) instead. + +**Scripts on macOS (and Windows)**: `FingerprintFromSmiles` runs in parallel worker processes, which re-import the calling script on these systems. Put a script's code under `if __name__ == "__main__":`. Notebooks are not affected. + ## GPU Acceleration Every step in the pipeline supports optional GPU acceleration via the `device` parameter: encoder training, PQ encoding, cluster training, and label assignment. When a CUDA GPU is available, `device='auto'` (the default) uses the GPU transparently; otherwise it falls back to CPU. -**Requirements**: `torch` and `triton` (both installed with `pip install torch`). +**Requirements**: `torch` and `triton` (on Linux, `pip install torch` installs both). ```python encoder = PQEncoder(k=256, m=6, iterations=20) @@ -90,11 +94,12 @@ The GPU implementation uses a custom Triton kernel for cluster assignment that t from chelombus import DataStreamer, FingerprintCalculator, PQEncoder, PQKMeans # 1. Stream SMILES in chunks -streamer = DataStreamer(path='molecules.smi', chunksize=100000) +streamer = DataStreamer() +smiles_gen = streamer.parse_input('molecules.smi', chunksize=100000) # 2. Calculate MQN fingerprints fp_calc = FingerprintCalculator() -for smiles_chunk in streamer.parse_input(): +for smiles_chunk in smiles_gen: fingerprints = fp_calc.FingerprintFromSmiles(smiles_chunk, fp='mqn') # Save fingerprints... diff --git a/docs/getting-started.md b/docs/getting-started.md index fdf6bca..bb41365 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -10,10 +10,10 @@ pip install chelombus ### GPU Support (optional) -GPU acceleration requires PyTorch and Triton. Install them separately for your CUDA version: +GPU acceleration requires PyTorch and Triton (Linux only). Install PyTorch for your CUDA version; on Linux it includes Triton: ```bash -pip install torch triton +pip install torch ``` When CUDA is available, `PQEncoder.fit()`, `.transform()`, `PQKMeans.fit()`, and `.predict()` automatically use the GPU. No code changes needed — the library falls back to CPU transparently if CUDA is not detected. @@ -29,6 +29,12 @@ pip install -e . !!! warning "Apple Silicon (M1/M2/M3)" The `pqkmeans` library is not currently supported on Apple Silicon Macs. Clustering functionality requires an x86_64 system. +!!! warning "Windows" + Not supported natively. `pqkmeans` has no Windows wheels (building it needs MSVC and CMake), and the GPU kernels need Triton, which is Linux only. Use [WSL2](https://learn.microsoft.com/windows/wsl/install) instead. + +!!! note "Scripts on macOS (and Windows)" + `FingerprintFromSmiles` runs in parallel worker processes, which re-import the calling script on these systems. Put a script's code under `if __name__ == "__main__":`. Notebooks are not affected. + ## Quick Start ```python diff --git a/examples/enamine_1B_clustering.ipynb b/examples/enamine_1B_clustering.ipynb index ea25073..42a884b 100644 --- a/examples/enamine_1B_clustering.ipynb +++ b/examples/enamine_1B_clustering.ipynb @@ -253,7 +253,7 @@ "from chelombus import PQKMeans\n", "import time\n", "\n", - "kmeans = PQKMeans(encoder=pq_encoder, k=10_000, iteration=20)\n", + "kmeans = PQKMeans(encoder=pq_encoder, k=10_000, iterations=20)\n", "s = time.time()\n", "kmeans.fit(pq_codes)\n", "e = time.time()\n", diff --git a/examples/tutorial.ipynb b/examples/tutorial.ipynb index a1e7b58..c29aa7f 100644 --- a/examples/tutorial.ipynb +++ b/examples/tutorial.ipynb @@ -211,18 +211,6 @@ "Since we have a PQ-code of 4 digits and each digit can take value {0,255} then the number of theoretical unique PQ-codes we can get is therefore $256^4 = 4,294,967,296$. However we can test that this is much less in reality. " ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from chelombus import PQEncoder\n", - "\n", - "encoder = PQEncoder.load('/home/afloresep/work/Chelombus/models/paper_encoder.joblib')\n", - "encoder.transform(np.random.randint(0, 200, (10_000_000, 42)))\n" - ] - }, { "cell_type": "code", "execution_count": null, @@ -281,9 +269,9 @@ "\n", "# Create a PQKMeans clusterer\n", "# k: number of clusters\n", - "# iteration: number of k-means iterations\n", + "# iterations: number of k-means iterations\n", "n_clusters = 100\n", - "clusterer = PQKMeans(encoder=pq_encoder, k=n_clusters, iteration=20, verbose=False)\n", + "clusterer = PQKMeans(encoder=pq_encoder, k=n_clusters, iterations=20, verbose=False)\n", "\n", "# Fit and predict in one step\n", "labels = clusterer.fit_predict(X_pq_code)\n", @@ -330,7 +318,7 @@ "outputs": [], "source": [ "# Fit on training data\n", - "clusterer2 = PQKMeans(encoder=pq_encoder, k=50, iteration=15)\n", + "clusterer2 = PQKMeans(encoder=pq_encoder, k=50, iterations=15)\n", "clusterer2.fit(X_pq_code)\n", "\n", "# Predict on new data (using same data for demo)\n", @@ -384,7 +372,10 @@ "\n", "os.makedirs(\"data/results\", exist_ok=True)\n", "\n", - "# Create a dataframe with smiles and cluster_id\n", + "# Create a dataframe with smiles and cluster_id.\n", + "# labels come from the first fingerprint chunk, so they line up with the first\n", + "# len(labels) SMILES only because every SMILES in it parses. With your own data,\n", + "# use FingerprintFromSmiles(..., return_valid_idx=True) to keep them aligned.\n", "cluster_df = pd.DataFrame({\n", " \"smiles\": smiles[:len(labels)],\n", " \"cluster_id\": labels\n", @@ -408,7 +399,7 @@ "from chelombus import get_cluster_stats, query_cluster, sample_from_cluster\n", "\n", "# Get statistics for all clusters\n", - "stats = get_cluster_stats(\"/mnt/10tb_hdd/enamine_clustered_october_2025/build_clickhouse_db_enamine_files\", \"2B_cluster_id\")\n", + "stats = get_cluster_stats(\"data/results/\")\n", "print(f\"Number of clusters: {len(stats)}\")\n", "print(f\"Total molecules: {stats['molecule_count'].sum():,}\")\n", "print(f\"\\nTop 5 largest clusters:\")\n", @@ -530,4 +521,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} From bffb4908b2796664010ca376450a04f834008c1b Mon Sep 17 00:00:00 2001 From: afloresep Date: Thu, 29 Oct 2026 13:01:56 +0100 Subject: [PATCH 10/10] Fix CSV quoting, duplicate-row sampling, array input, bracket paths - DataStreamer: read sep-delimited input with csv.reader, so quoted fields ("CCO", "a,b") parse correctly. - DataStreamer: an existing path is taken literally even if it contains glob characters (batch[1]/), glob expansion only when it does not exist. - sample_from_cluster: seed the hash on each row's file name and row number, so duplicate rows are sampled independently. Still reproducible across DuckDB threading and if the folder is moved. - FingerprintFromSmiles: the empty-result warning used `if smiles`, which raised for numpy arrays and pandas Series. --- chelombus/streamer/data_streamer.py | 26 ++++++++++++++++---------- chelombus/utils/cluster_io.py | 24 ++++++++++++------------ chelombus/utils/fingerprints.py | 4 ++-- tests/test_cluster_io.py | 12 ++++++++++++ tests/test_data_streamer.py | 16 ++++++++++++++++ tests/test_fingerprints.py | 12 ++++++++++++ 6 files changed, 70 insertions(+), 24 deletions(-) diff --git a/chelombus/streamer/data_streamer.py b/chelombus/streamer/data_streamer.py index 30d6bd6..be55b7f 100644 --- a/chelombus/streamer/data_streamer.py +++ b/chelombus/streamer/data_streamer.py @@ -1,4 +1,5 @@ import bz2 +import csv import glob import gzip import os @@ -56,13 +57,14 @@ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose: List: a chunk of SMILES strings, or of (id, smiles) tuples when ids are requested. """ - # Sanity check - if glob.has_magic(input_path): + # Sanity check. A path that exists is taken literally, even if it + # contains glob characters like "batch[1]". + if os.path.exists(input_path): + input_paths = input_path + elif glob.has_magic(input_path): input_paths = sorted(glob.glob(input_path)) if not input_paths: raise FileNotFoundError(f"No files match {input_path}.") - elif os.path.exists(input_path): - input_paths = input_path else: raise FileNotFoundError(f"Data source {input_path} not found.") @@ -101,12 +103,16 @@ def _stream_text(self, file_path: str, smiles_col: int, `id_col`, or None when there is no id column. Blank / too-short lines (no column at `smiles_col`) are skipped.""" with _open(file_path) as file: - for line in file: - parts = [part.strip() for part in line.split(sep)] - if smiles_col >= len(parts) or not parts[smiles_col]: + if sep is None: + rows = (line.split() for line in file) + else: + # csv handles quoted fields, e.g. "C,C" or "CCO" + rows = csv.reader(file, delimiter=sep, skipinitialspace=True) + for parts in rows: + if smiles_col >= len(parts) or not parts[smiles_col].strip(): continue - smiles = parts[smiles_col] - raw_id = parts[id_col] if (id_col is not None and id_col < len(parts)) else None + smiles = parts[smiles_col].strip() + raw_id = parts[id_col].strip() if (id_col is not None and id_col < len(parts)) else None yield raw_id, smiles def _stream_sdf(self, file_path: str) -> Iterator[tuple]: @@ -149,4 +155,4 @@ def _open(file_path: str, binary: bool = False): opener = _COMPRESSED.get(os.path.splitext(file_path)[1].lower(), open) if binary: return opener(file_path, "rb") - return opener(file_path, "rt", encoding="utf-8") + return opener(file_path, "rt", encoding="utf-8", newline="") diff --git a/chelombus/utils/cluster_io.py b/chelombus/utils/cluster_io.py index 08f7ced..4344329 100644 --- a/chelombus/utils/cluster_io.py +++ b/chelombus/utils/cluster_io.py @@ -510,19 +510,19 @@ def sample_from_cluster( path = _validate_results_dir(results_dir) source = _parquet_source(path) - # A seeded hash of each row gives the same order regardless of how DuckDB - # splits the scan across threads (setseed + RANDOM() does not) + where = f"{_quote_identifier(cluster_column)} = {cluster_id}" if random_state is None: - order_by = "RANDOM()" + query = f"SELECT * FROM {source} WHERE {where} ORDER BY RANDOM() LIMIT {n}" else: - order_by = f"hash(t, {int(random_state)})" - - query = f""" - SELECT * - FROM {source} AS t - WHERE {_quote_identifier(cluster_column)} = {cluster_id} - ORDER BY {order_by} - LIMIT {n} - """ + # Hash each row's position (file name, row number) with the seed. The + # order is the same however DuckDB splits the scan across threads + # (setseed + RANDOM() is not), and duplicate rows get distinct keys. + query = f""" + SELECT * EXCLUDE (filename, file_row_number) + FROM read_parquet({source}, filename = true, file_row_number = true) + WHERE {where} + ORDER BY hash(parse_filename(filename), file_row_number, {int(random_state)}) + LIMIT {n} + """ return duckdb.execute(query).df() diff --git a/chelombus/utils/fingerprints.py b/chelombus/utils/fingerprints.py index aae99f3..067eeb2 100644 --- a/chelombus/utils/fingerprints.py +++ b/chelombus/utils/fingerprints.py @@ -165,11 +165,11 @@ def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None # Results are collected; don't block while idle workers exit executor.shutdown(wait=False, cancel_futures=True) - if smiles and all(fp_ is None for fp_ in fingerprints): + if len(smiles) > 0 and all(fp_ is None for fp_ in fingerprints): # Usually a reading problem (wrong column or separator), not bad molecules logger.warning( "None of the %d SMILES could be parsed (first: %r). If reading with " - "DataStreamer, check smiles_col and sep.", len(smiles), smiles[0] + "DataStreamer, check smiles_col and sep.", len(smiles), next(iter(smiles)) ) if return_valid_idx: diff --git a/tests/test_cluster_io.py b/tests/test_cluster_io.py index e7567e9..2623af6 100644 --- a/tests/test_cluster_io.py +++ b/tests/test_cluster_io.py @@ -420,6 +420,18 @@ def test_sample_with_random_state(self, large_results_dir): assert df1["smiles"].tolist() == df2["smiles"].tolist() assert df1["smiles"].tolist() != df3["smiles"].tolist() + def test_sample_with_random_state_duplicate_rows(self, tmp_path): + """Duplicate rows are sampled independently, not grouped together.""" + results_dir = tmp_path / "dup" + results_dir.mkdir() + pd.DataFrame({"smiles": ["A"] * 100 + ["B"] * 100, "cluster_id": 0}).to_parquet( + results_dir / "chunk_00000.parquet", index=False) + + for seed in range(5): + df = sample_from_cluster(results_dir, cluster_id=0, n=20, random_state=seed) + assert list(df.columns) == ["smiles", "cluster_id"] + assert set(df["smiles"]) == {"A", "B"} + def test_sample_empty_cluster(self, sample_results_dir): """Test sampling from non-existent cluster.""" df = sample_from_cluster(sample_results_dir, cluster_id=999, n=5) diff --git a/tests/test_data_streamer.py b/tests/test_data_streamer.py index a4ed692..6cedc7c 100644 --- a/tests/test_data_streamer.py +++ b/tests/test_data_streamer.py @@ -117,3 +117,19 @@ def test_parse_input_csv_sep(tmp_path: Path): result = list(DataStreamer().parse_input(str(tmp_path / "mols.csv"), smiles_col=1, id_col=0, sep=",")) assert result == [[("id1", "CCO"), ("id2", "CCN")]] + + +def test_parse_input_csv_quoted_fields(tmp_path: Path): + (tmp_path / "mols.csv").write_text('"CCO",id1\n"C(C)O","a,b"\n') + + result = list(DataStreamer().parse_input(str(tmp_path / "mols.csv"), id_col=1, sep=",")) + assert result == [[("id1", "CCO"), ("a,b", "C(C)O")]] + + +def test_parse_input_existing_path_with_glob_characters(tmp_path: Path): + folder = tmp_path / "batch[1]" + folder.mkdir() + (folder / "a.smi").write_text("CCO\n") + + result = list(DataStreamer().parse_input(str(folder))) + assert result == [["CCO"]] diff --git a/tests/test_fingerprints.py b/tests/test_fingerprints.py index d0dae0f..30344f0 100644 --- a/tests/test_fingerprints.py +++ b/tests/test_fingerprints.py @@ -161,3 +161,15 @@ def test_fingerprint_calculator_warns_when_nothing_parses(caplog): assert len(fps) == 0 assert "None of the 2 SMILES could be parsed" in caplog.text + + +@pytest.mark.parametrize("container", [np.array, list, tuple]) +def test_fingerprint_calculator_accepts_array_like(container): + fps = FingerprintCalculator().FingerprintFromSmiles(container(["CCO", "CCN"]), 'mqn', nprocesses=1) + assert fps.shape == (2, 42) + + +def test_fingerprint_calculator_accepts_pandas_series(): + pd = pytest.importorskip("pandas") + fps = FingerprintCalculator().FingerprintFromSmiles(pd.Series(["CCO", "CCN"]), 'mqn', nprocesses=1) + assert fps.shape == (2, 42)