diff --git a/README.md b/README.md index 0f74d90..0e8b3d5 100644 --- a/README.md +++ b/README.md @@ -45,11 +45,15 @@ pip install -e . **Apple Silicon (M1/M2/M3)**: The `pqkmeans` library is not currently supported on Apple Silicon Macs. My plan is to rewrite pqkmeans with Silicon and GPU support but that's for a future release... For now, clustering functionality requires an x86_64 system. +**Windows**: Not supported natively. `pqkmeans` has no Windows wheels (building it needs MSVC and CMake), and the GPU kernels need Triton, which is Linux only. Use [WSL2](https://learn.microsoft.com/windows/wsl/install) instead. + +**Scripts on macOS (and Windows)**: `FingerprintFromSmiles` runs in parallel worker processes, which re-import the calling script on these systems. Put a script's code under `if __name__ == "__main__":`. Notebooks are not affected. + ## GPU Acceleration Every step in the pipeline supports optional GPU acceleration via the `device` parameter: encoder training, PQ encoding, cluster training, and label assignment. When a CUDA GPU is available, `device='auto'` (the default) uses the GPU transparently; otherwise it falls back to CPU. -**Requirements**: `torch` and `triton` (both installed with `pip install torch`). +**Requirements**: `torch` and `triton` (on Linux, `pip install torch` installs both). ```python encoder = PQEncoder(k=256, m=6, iterations=20) @@ -90,11 +94,12 @@ The GPU implementation uses a custom Triton kernel for cluster assignment that t from chelombus import DataStreamer, FingerprintCalculator, PQEncoder, PQKMeans # 1. Stream SMILES in chunks -streamer = DataStreamer(path='molecules.smi', chunksize=100000) +streamer = DataStreamer() +smiles_gen = streamer.parse_input('molecules.smi', chunksize=100000) # 2. Calculate MQN fingerprints fp_calc = FingerprintCalculator() -for smiles_chunk in streamer.parse_input(): +for smiles_chunk in smiles_gen: fingerprints = fp_calc.FingerprintFromSmiles(smiles_chunk, fp='mqn') # Save fingerprints... diff --git a/chelombus/__init__.py b/chelombus/__init__.py index 9524403..425c65c 100644 --- a/chelombus/__init__.py +++ b/chelombus/__init__.py @@ -5,17 +5,27 @@ """ from .streamer.data_streamer import DataStreamer -from .encoder.encoder import PQEncoder from .utils.fingerprints import FingerprintCalculator from .utils.helper_functions import save_chunk, format_time -# Optional imports - only available if dependencies are installed -try: - from .clustering.PyQKmeans import PQKMeans -except ImportError as e: - import warnings - warnings.warn(f"PQKMeans not available: {e}") - PQKMeans = None # pqkmeans or numba not installed + +def __getattr__(name): + # PQEncoder and PQKMeans pull in torch, numba and pqkmeans, so they load on + # first use. This keeps `import chelombus` fast, which matters most for + # fingerprint worker processes on Windows and macOS (each re-imports it). + if name == "PQEncoder": + from .encoder.encoder import PQEncoder as value + elif name == "PQKMeans": + try: + from .clustering.PyQKmeans import PQKMeans as value + except ImportError as e: + import warnings + warnings.warn(f"PQKMeans not available: {e}") + value = None # pqkmeans or numba not installed + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + globals()[name] = value + return value # Cluster I/O utilities (require duckdb) try: @@ -36,7 +46,7 @@ def _cluster_io_not_available(*args, **kwargs): raise ImportError( "Cluster I/O functions require duckdb. " - "Install with: pip install chelombus[io] or pip install duckdb" + "Install with: pip install duckdb" ) query_cluster = _cluster_io_not_available export_cluster = _cluster_io_not_available diff --git a/chelombus/clustering/PyQKmeans.py b/chelombus/clustering/PyQKmeans.py index b8de301..77341a2 100644 --- a/chelombus/clustering/PyQKmeans.py +++ b/chelombus/clustering/PyQKmeans.py @@ -10,6 +10,7 @@ """ from typing import Literal, overload +import warnings import joblib from pathlib import Path import numpy as np @@ -23,7 +24,7 @@ if torch.cuda.is_available(): from chelombus.clustering._gpu_predict import predict_gpu _GPU_AVAILABLE = True -except ImportError: +except Exception: # not installed, or a broken install (e.g. missing DLL) pass @@ -126,11 +127,12 @@ class PQKMeans: Args: encoder: A trained PQEncoder instance k: Number of clusters - iteration: Maximum number of k-means iterations (default: 20) + iterations: Maximum number of k-means iterations (default: 20) tol: Early-stopping tolerance for the GPU path. Training stops when the fraction of changed center coordinates drops below *tol*. 0 means stop only on exact convergence. (default: 1e-3) verbose: Whether to print progress information (default: False) + iteration: Deprecated alias for ``iterations``. Example: >>> encoder = PQEncoder(k=256, m=6) @@ -144,16 +146,24 @@ def __init__( self, encoder: PQEncoder, k: int, - iteration: int = 20, + iterations: int = 20, tol: float = 1e-3, - verbose: bool = False + verbose: bool = False, + iteration: int | None = None, ): + if iteration is not None: + warnings.warn( + "'iteration' is deprecated, use 'iterations' instead", + DeprecationWarning, + stacklevel=2, + ) + iterations = iteration if not encoder.encoder_is_trained: # type: ignore raise ValueError("Encoder must be trained before clustering") self.encoder = encoder self.k = k - self.iteration = iteration + self.iterations = iterations self.tol = tol self.verbose = verbose self.trained = False @@ -163,7 +173,7 @@ def __init__( self._cluster = pqkmeans.clustering.PQKMeans( encoder=encoder, k=k, - iteration=iteration, + iteration=iterations, verbose=verbose, ) @@ -249,7 +259,7 @@ def _fit_gpu(self, X_train: np.ndarray, return_labels: bool = False) -> np.ndarr labels = None prev_centers = None final_labels_match_centers = False - for it in range(self.iteration): + for it in range(self.iterations): t0 = time.time() # assignment (GPU) @@ -274,7 +284,7 @@ def _fit_gpu(self, X_train: np.ndarray, return_labels: bool = False) -> np.ndarr if self.verbose: print( - f" iter {it + 1}/{self.iteration} " + f" iter {it + 1}/{self.iterations} " f"assign={t1 - t0:.1f}s update={t2 - t1:.1f}s " f"changed={changed}/{total_coords} ({frac:.4%})" ) @@ -392,6 +402,9 @@ def load(cls, path: str | Path) -> "PQKMeans": obj = joblib.load(path) if not isinstance(obj, cls): raise TypeError(f"Expected {cls.__name__}, got {type(obj).__name__}") + if not hasattr(obj, 'iterations'): + # Models saved before the 'iteration' -> 'iterations' rename + obj.iterations = obj.__dict__.pop('iteration') if not hasattr(obj, '_dtables'): obj._dtables = None if not hasattr(obj, '_centers_u8'): diff --git a/chelombus/encoder/encoder.py b/chelombus/encoder/encoder.py index f331121..a70ec04 100644 --- a/chelombus/encoder/encoder.py +++ b/chelombus/encoder/encoder.py @@ -11,7 +11,7 @@ import torch if torch.cuda.is_available(): _GPU_AVAILABLE = True -except ImportError: +except Exception: # not installed, or a broken install (e.g. missing DLL) pass diff --git a/chelombus/streamer/data_streamer.py b/chelombus/streamer/data_streamer.py index 225a30d..be55b7f 100644 --- a/chelombus/streamer/data_streamer.py +++ b/chelombus/streamer/data_streamer.py @@ -1,3 +1,7 @@ +import bz2 +import csv +import glob +import gzip import os from typing import Iterator, List, Optional @@ -9,13 +13,15 @@ class DataStreamer: """ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose: int = 0, smiles_col: int = 0, id_col: Optional[int] = None, - with_ids: bool = False) -> Iterator[List]: + with_ids: bool = False, sep: Optional[str] = None) -> Iterator[List]: """ Read input data from a file or a directory of files and yield it in chunks. - The input path can be a single file or a directory of files. Plain-text + The input path can be a single file, a directory of files, or a glob + pattern like "data/*.smi". Files ending in .gz or .bz2 are decompressed + on the fly, and text is read as UTF-8. Plain-text files are read line by line; the whitespace-separated column at - `smiles_col` (0-based) is taken as the SMILES. So a file with ONE SMILES + `smiles_col` (0-based) is taken as the SMILES (pass `sep=","` for CSV). So a file with ONE SMILES per line works as-is (smiles_col=0), and a file like "idsmiles" works with smiles_col=1. SDF/SD files are streamed with RDKit and turned into SMILES on the fly. Items are buffered until `chunksize` is reached, @@ -34,7 +40,8 @@ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose: Ids are always returned as strings. Args: - input_path (str): Path to a file or directory of input files. + input_path (str): Path to a file, a directory of input files, or a + glob pattern. Directory and glob matches are read in sorted order. chunksize (Optional[int]): Items per yielded chunk. None -> one chunk per file. verbose (int): 0 silent, 1 prints each file name. @@ -43,28 +50,36 @@ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose: id_col (Optional[int]): 0-based column holding the id, or None to make one up. Passing it turns on (id, smiles) output. with_ids (bool): Yield (id, smiles) tuples instead of bare SMILES. + sep (Optional[str]): Column separator for text inputs, e.g. "," for + CSV. None splits on any whitespace. Yields: List: a chunk of SMILES strings, or of (id, smiles) tuples when ids are requested. """ - # Sanity check - if not os.path.exists(input_path): + # Sanity check. A path that exists is taken literally, even if it + # contains glob characters like "batch[1]". + if os.path.exists(input_path): + input_paths = input_path + elif glob.has_magic(input_path): + input_paths = sorted(glob.glob(input_path)) + if not input_paths: + raise FileNotFoundError(f"No files match {input_path}.") + else: raise FileNotFoundError(f"Data source {input_path} not found.") want_ids = with_ids or id_col is not None buffer: List = [] next_id = 0 # running counter for made-up ids (all files, in order) - for file_path in _process_input(input_paths=input_path): + for file_path in _process_input(input_paths=input_paths): if verbose == 1: print("Processing file:", file_path) - file_ext = os.path.splitext(file_path)[1].lower() - if file_ext in {".sdf", ".sd"}: + if _file_ext(file_path) in {".sdf", ".sd"}: records = self._stream_sdf(file_path) else: - records = self._stream_text(file_path, smiles_col, id_col) + records = self._stream_text(file_path, smiles_col, id_col, sep) for raw_id, smiles in records: if want_ids: @@ -83,17 +98,21 @@ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose: buffer.clear() def _stream_text(self, file_path: str, smiles_col: int, - id_col: Optional[int] = None) -> Iterator[tuple]: + id_col: Optional[int] = None, sep: Optional[str] = None) -> Iterator[tuple]: """Yield (raw_id, smiles) per line. raw_id is the value in column `id_col`, or None when there is no id column. Blank / too-short lines (no column at `smiles_col`) are skipped.""" - with open(file_path, "r") as file: - for line in file: - parts = line.split() - if smiles_col >= len(parts): + with _open(file_path) as file: + if sep is None: + rows = (line.split() for line in file) + else: + # csv handles quoted fields, e.g. "C,C" or "CCO" + rows = csv.reader(file, delimiter=sep, skipinitialspace=True) + for parts in rows: + if smiles_col >= len(parts) or not parts[smiles_col].strip(): continue - smiles = parts[smiles_col] - raw_id = parts[id_col] if (id_col is not None and id_col < len(parts)) else None + smiles = parts[smiles_col].strip() + raw_id = parts[id_col].strip() if (id_col is not None and id_col < len(parts)) else None yield raw_id, smiles def _stream_sdf(self, file_path: str) -> Iterator[tuple]: @@ -105,7 +124,7 @@ def _stream_sdf(self, file_path: str) -> Iterator[tuple]: ) from exc # ForwardSDMolSupplier streams molecules without loading the full file in memory - with open(file_path, "rb") as file_handle: + with _open(file_path, binary=True) as file_handle: supplier = Chem.ForwardSDMolSupplier(file_handle) for mol in supplier: if mol is None: @@ -118,3 +137,22 @@ def _stream_sdf(self, file_path: str) -> Iterator[tuple]: # Use the SD record title as the id when present. name = mol.GetProp("_Name").strip() if mol.HasProp("_Name") else "" yield (name or None), smiles + + +_COMPRESSED = {".gz": gzip.open, ".bz2": bz2.open} + + +def _file_ext(file_path: str) -> str: + """Lowercase extension, ignoring a compression suffix (x.sdf.gz -> .sdf).""" + root, ext = os.path.splitext(file_path.lower()) + if ext in _COMPRESSED: + ext = os.path.splitext(root)[1] + return ext + + +def _open(file_path: str, binary: bool = False): + """Open a plain, .gz or .bz2 file as UTF-8 text (or bytes).""" + opener = _COMPRESSED.get(os.path.splitext(file_path)[1].lower(), open) + if binary: + return opener(file_path, "rb") + return opener(file_path, "rt", encoding="utf-8", newline="") diff --git a/chelombus/utils/cluster_io.py b/chelombus/utils/cluster_io.py index 25f690c..4344329 100644 --- a/chelombus/utils/cluster_io.py +++ b/chelombus/utils/cluster_io.py @@ -42,7 +42,7 @@ def _check_duckdb_available() -> None: except ImportError: raise ImportError( "duckdb is required for cluster I/O operations. " - "Install it with: pip install chelombus[io] or pip install duckdb" + "Install it with: pip install duckdb" ) @@ -72,6 +72,17 @@ def _validate_results_dir(results_dir: Union[str, Path]) -> Path: return path +def _sql_string(value: Union[str, Path]) -> str: + """Quote a path as a SQL string literal (handles paths containing ').""" + escaped = str(value).replace("'", "''") + return f"'{escaped}'" + + +def _parquet_source(path: Path) -> str: + """SQL source for all chunked parquet files in a results directory.""" + return _sql_string(path / "*.parquet") + + def _quote_identifier(identifier: str) -> str: if identifier.startswith('"') and identifier.endswith('"'): return identifier @@ -114,7 +125,7 @@ def query_cluster( import duckdb path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) quoted_cluster_column = _quote_identifier(cluster_column) # Build column selection @@ -125,11 +136,11 @@ def query_cluster( query = f""" SELECT {col_str} - FROM '{glob_pattern}' + FROM {source} WHERE {quoted_cluster_column} = {cluster_id} """ - result = duckdb.query(query).df() + result = duckdb.execute(query).df() if result.empty: logger.warning(f"No molecules found for cluster_id={cluster_id}") @@ -143,6 +154,7 @@ def export_cluster( output_path: Union[str, Path], cluster_column: str = "cluster_id", columns: Optional[List[str]] = None, + smiles_column: str = "smiles", ) -> int: """Export a single cluster to a file. @@ -155,6 +167,7 @@ def export_cluster( output_path: Output file path. Must end with .parquet or .csv. cluster_column: Name of the column containing cluster IDs. columns: Optional list of columns to export. Default exports all columns. + smiles_column: Column written to .smi/.txt outputs. Returns: Number of molecules exported. @@ -172,19 +185,19 @@ def export_cluster( path = _validate_results_dir(results_dir) output_path = Path(output_path) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) quoted_cluster_column = _quote_identifier(cluster_column) # Build column selection - col_str = ", ".join(columns) if columns else "*" + col_str = ", ".join(_quote_identifier(col) for col in columns) if columns else "*" # First get the count count_query = f""" SELECT COUNT(*) as cnt - FROM '{glob_pattern}' + FROM {source} WHERE {quoted_cluster_column} = {cluster_id} """ - count = duckdb.query(count_query).fetchone()[0] # type: ignore + count = duckdb.execute(count_query).fetchone()[0] # type: ignore if count == 0: logger.warning(f"No molecules found for cluster_id={cluster_id}") @@ -199,26 +212,26 @@ def export_cluster( export_query = f""" COPY ( SELECT {col_str} - FROM '{glob_pattern}' + FROM {source} WHERE {quoted_cluster_column} = {cluster_id} - ) TO '{output_path}' (FORMAT PARQUET) + ) TO {_sql_string(output_path)} (FORMAT PARQUET) """ elif suffix == ".csv": export_query = f""" COPY ( SELECT {col_str} - FROM '{glob_pattern}' + FROM {source} WHERE {quoted_cluster_column} = {cluster_id} - ) TO '{output_path}' (FORMAT CSV, HEADER) + ) TO {_sql_string(output_path)} (FORMAT CSV, HEADER) """ elif suffix in (".smi", ".txt"): # For SMILES files, export only the smiles column without header export_query = f""" COPY ( - SELECT smiles - FROM '{glob_pattern}' + SELECT {_quote_identifier(smiles_column)} + FROM {source} WHERE {quoted_cluster_column} = {cluster_id} - ) TO '{output_path}' (FORMAT CSV, HEADER FALSE) + ) TO {_sql_string(output_path)} (FORMAT CSV, HEADER FALSE) """ else: raise ValueError( @@ -226,7 +239,7 @@ def export_cluster( "Use .parquet, .csv, .smi, or .txt" ) - duckdb.query(export_query) + duckdb.execute(export_query) logger.info(f"Exported {count} molecules to {output_path}") return count @@ -236,6 +249,8 @@ def export_all_clusters( results_dir: Union[str, Path], output_dir: Union[str, Path], format: str = "parquet", + cluster_column: str = "cluster_id", + smiles_column: str = "smiles", ) -> dict[int, int]: """Export all clusters to individual files using partitioned writes. @@ -260,8 +275,8 @@ def export_all_clusters( results_dir: Path to directory containing chunked parquet files. output_dir: Output directory for partitioned cluster files. format: Output format, either 'parquet' or 'csv'. Default 'parquet'. - batch_size: Number of clusters to process per batch for progress reporting. - Does not affect memory usage (DuckDB handles streaming internally). + cluster_column: Name of the column containing cluster IDs. + smiles_column: Name of the column containing SMILES. Returns: Dictionary mapping cluster_id to molecule count. @@ -276,24 +291,14 @@ def export_all_clusters( >>> print(f"Largest cluster: {max(stats.values())} molecules") Note: - For HPC/SLURM usage, you can process clusters in parallel: - - ```bash - # SLURM array job - #SBATCH --array=0-99999 - - chelombus visualize \\ - --input clusters/cluster_id=$SLURM_ARRAY_TASK_ID/ \\ - --output tmaps/cluster_$SLURM_ARRAY_TASK_ID.html - ``` + For HPC/SLURM usage, each array task can read its own partition, e.g. + ``clusters/cluster_id=$SLURM_ARRAY_TASK_ID/``. """ _check_duckdb_available() import duckdb - from tqdm import tqdm - path = _validate_results_dir(results_dir) output_path = Path(output_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) if format not in ("parquet", "csv"): raise ValueError(f"Unsupported format: {format}. Use 'parquet' or 'csv'") @@ -309,21 +314,21 @@ def export_all_clusters( export_query = f""" COPY ( - SELECT smiles, cluster_id - FROM '{glob_pattern}' - ) TO '{output_path}' ( + SELECT {_quote_identifier(smiles_column)}, {_quote_identifier(cluster_column)} + FROM {source} + ) TO {_sql_string(output_path)} ( FORMAT {format_str}, - PARTITION_BY (cluster_id), + PARTITION_BY ({_quote_identifier(cluster_column)}), OVERWRITE_OR_IGNORE ) """ logger.info("Starting partitioned export (streaming, memory-efficient)...") - duckdb.query(export_query) + duckdb.execute(export_query) # Get cluster statistics after export logger.info("Calculating cluster statistics...") - stats = get_cluster_stats(results_dir) + stats = get_cluster_stats(results_dir, column=cluster_column) # Convert to dictionary result = dict(zip(stats["cluster_id"], stats["molecule_count"])) @@ -361,19 +366,19 @@ def get_cluster_stats( import duckdb path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) quoted_column = _quote_identifier(column) query = f""" SELECT {quoted_column} as cluster_id, COUNT(*) as molecule_count - FROM '{glob_pattern}' + FROM {source} GROUP BY {quoted_column} ORDER BY {quoted_column} """ - return duckdb.query(query).df() + return duckdb.execute(query).df() def get_cluster_ids( @@ -384,6 +389,7 @@ def get_cluster_ids( Args: results_dir: Path to directory containing chunked parquet files. + cluster_id_column_name: Name of the column containing cluster IDs. Returns: Sorted list of cluster IDs. @@ -396,15 +402,15 @@ def get_cluster_ids( import duckdb path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) query = f""" SELECT DISTINCT {_quote_identifier(cluster_id_column_name)} - FROM '{glob_pattern}' + FROM {source} ORDER BY {_quote_identifier(cluster_id_column_name)} """ - result = duckdb.query(query).fetchall() + result = duckdb.execute(query).fetchall() return [row[0] for row in result] @@ -423,10 +429,10 @@ def get_total_molecules( import duckdb path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) - query = f"SELECT COUNT(*) FROM '{glob_pattern}'" - return duckdb.query(query).fetchone()[0] # type: ignore + query = f"SELECT COUNT(*) FROM {source}" + return duckdb.execute(query).fetchone()[0] # type: ignore def query_clusters_batch( @@ -444,6 +450,7 @@ def query_clusters_batch( results_dir: Path to directory containing chunked parquet files. cluster_ids: List of cluster IDs to query. columns: Optional list of columns to return. + cluster_column: Name of the column containing cluster IDs. Returns: DataFrame containing molecules from all specified clusters. @@ -457,18 +464,18 @@ def query_clusters_batch( quoted_cluster_column = _quote_identifier(cluster_column) path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") + source = _parquet_source(path) - col_str = ", ".join(columns) if columns else "*" + col_str = ", ".join(_quote_identifier(col) for col in columns) if columns else "*" ids_str = ", ".join(str(cid) for cid in cluster_ids) query = f""" SELECT {col_str} - FROM '{glob_pattern}' + FROM {source} WHERE {quoted_cluster_column} IN ({ids_str}) """ - return duckdb.query(query).df() + return duckdb.execute(query).df() def sample_from_cluster( @@ -476,6 +483,7 @@ def sample_from_cluster( cluster_id: int, n: int = 100, random_state: Optional[int] = None, + cluster_column: str = "cluster_id", ) -> "pd.DataFrame": """Get a random sample of molecules from a cluster. @@ -485,7 +493,9 @@ def sample_from_cluster( results_dir: Path to directory containing chunked parquet files. cluster_id: The cluster ID to sample from. n: Number of molecules to sample. - random_state: Random seed for reproducibility. + random_state: Random seed. The same seed on the same data returns the + same sample. + cluster_column: Name of the column containing cluster IDs. Returns: DataFrame with sampled molecules. @@ -498,19 +508,21 @@ def sample_from_cluster( import duckdb path = _validate_results_dir(results_dir) - glob_pattern = str(path / "*.parquet") - - # Use setseed() for reproducibility when random_state is provided - if random_state is not None: - duckdb.query(f"SELECT setseed({random_state / 2**31})") + source = _parquet_source(path) - # Use ORDER BY RANDOM() for sampling, which is more portable - query = f""" - SELECT * - FROM '{glob_pattern}' - WHERE cluster_id = {cluster_id} - ORDER BY RANDOM() - LIMIT {n} - """ + where = f"{_quote_identifier(cluster_column)} = {cluster_id}" + if random_state is None: + query = f"SELECT * FROM {source} WHERE {where} ORDER BY RANDOM() LIMIT {n}" + else: + # Hash each row's position (file name, row number) with the seed. The + # order is the same however DuckDB splits the scan across threads + # (setseed + RANDOM() is not), and duplicate rows get distinct keys. + query = f""" + SELECT * EXCLUDE (filename, file_row_number) + FROM read_parquet({source}, filename = true, file_row_number = true) + WHERE {where} + ORDER BY hash(parse_filename(filename), file_row_number, {int(random_state)}) + LIMIT {n} + """ - return duckdb.query(query).df() + return duckdb.execute(query).df() diff --git a/chelombus/utils/fingerprints.py b/chelombus/utils/fingerprints.py index 77180f1..067eeb2 100644 --- a/chelombus/utils/fingerprints.py +++ b/chelombus/utils/fingerprints.py @@ -1,5 +1,6 @@ #This module provides functionality to compute molecular fingerprints from SMILES strings -from multiprocessing import Pool +from concurrent.futures import ProcessPoolExecutor +from concurrent.futures.process import BrokenProcessPool from functools import partial from rdkit import Chem, DataStructs import os @@ -88,14 +89,18 @@ def __init__(self): 'mqn': _calculate_mqn_fp, } - def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None = os.cpu_count(), + def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None = None, return_valid_idx: bool = False, **params): """ Generate fingerprints for a list of SMILES strings in parallel. The method selects the appropriate fingerprint function based on the 'fp' parameter, binds additional keyword parameters using functools.partial, and then applies the function - across the SMILES list using multiprocessing.Pool.map. + across the SMILES list in a pool of worker processes. + + On Windows and macOS, worker processes re-import the calling script, so + a script that calls this must do so under ``if __name__ == "__main__":``. + Notebooks are not affected. Some SMILES may fail to parse; those rows are dropped. By default you get back only the surviving fingerprints, and you cannot tell *which* inputs @@ -111,7 +116,8 @@ def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None Args: smiles_list (list): A list of SMILES strings. fp (str): The fingerprint type to compute (e.g., 'morgan'). - nprocesses (int): Number of processes for multithreaded fingerprint calculation. Default to cpu cores + nprocesses (int): Number of worker processes. Defaults to all CPU cores + (at most 61 on Windows). Use 1 to compute in the current process. return_valid_idx (bool): If True, return ``(valid_idx, fingerprints)`` where ``valid_idx`` is an int64 array of the input positions that parsed OK. If False (default), return only the fingerprints. @@ -126,6 +132,8 @@ def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None Raises: ValueError: If an unsupported fingerprint type is requested. + RuntimeError: If a worker process dies, e.g. a script on Windows or + macOS that is missing the ``__main__`` guard. """ func = self.fingerprint_function_map.get(fp) if func is None: @@ -136,12 +144,33 @@ def FingerprintFromSmiles(self, smiles:List | str, fp:str, nprocesses:int | None if isinstance(smiles, str): smiles = [smiles] - try: - with Pool(processes=nprocesses) as pool: - fingerprints = pool.map(part_func, smiles) - finally: - pool.close() - pool.join() + if nprocesses == 1: + # No worker processes: avoids pool startup cost and spawn issues + fingerprints = [part_func(s) for s in smiles] + else: + # A few chunks per worker keeps them busy without per-item overhead + n_workers = nprocesses or os.cpu_count() or 1 + chunksize = max(1, len(smiles) // (n_workers * 4)) + executor = ProcessPoolExecutor(max_workers=nprocesses) + try: + fingerprints = list(executor.map(part_func, smiles, chunksize=chunksize)) + except BrokenProcessPool as e: + # multiprocessing.Pool would respawn crashed workers forever here + raise RuntimeError( + "A fingerprint worker process crashed. On Windows and macOS this " + "usually means the script calling FingerprintFromSmiles is missing " + "an `if __name__ == \"__main__\":` guard around its code." + ) from e + finally: + # Results are collected; don't block while idle workers exit + executor.shutdown(wait=False, cancel_futures=True) + + if len(smiles) > 0 and all(fp_ is None for fp_ in fingerprints): + # Usually a reading problem (wrong column or separator), not bad molecules + logger.warning( + "None of the %d SMILES could be parsed (first: %r). If reading with " + "DataStreamer, check smiles_col and sep.", len(smiles), next(iter(smiles)) + ) if return_valid_idx: # Keep the input positions that produced a fingerprint, so the diff --git a/chelombus/utils/helper_functions.py b/chelombus/utils/helper_functions.py index e0ba58a..ed3fe60 100644 --- a/chelombus/utils/helper_functions.py +++ b/chelombus/utils/helper_functions.py @@ -20,14 +20,15 @@ def _process_input(input_paths): # Single path (file or folder) input_paths = [input_paths] elif not isinstance(input_paths, (list, tuple)): - logging.error(f"Invalid input type: {type(input_paths)}. Provide a string, list, or tuple. Skipping") + raise TypeError(f"Invalid input type: {type(input_paths)}. Provide a string, list, or tuple.") for input_path in input_paths: if os.path.isdir(input_path): - # Process all files in the directory - for file in os.listdir(input_path): + # Process all files in the directory, in a stable order, skipping + # hidden files like .DS_Store + for file in sorted(os.listdir(input_path)): file_path = os.path.join(input_path, file) - if os.path.isfile(file_path): # Optional: filter by file extensions + if os.path.isfile(file_path) and not file.startswith("."): yield file_path elif os.path.isfile(input_path): # Process a single file diff --git a/chelombus/utils/visualization.py b/chelombus/utils/visualization.py index 7a156eb..829caa9 100644 --- a/chelombus/utils/visualization.py +++ b/chelombus/utils/visualization.py @@ -26,7 +26,7 @@ def create_tmap( """Create a TMAP visualization from SMILES strings. Args: - smiles: List of SMILES strings to visualize. + smiles: List of SMILES strings to visualize. Unparseable SMILES are skipped. fingerprint: Fingerprint type ('morgan' or 'mqn'). fingerprint_bits: Number of bits for Morgan fingerprint (default: 2048). fingerprint_radius: Radius for Morgan fingerprint (default: 2). @@ -35,34 +35,8 @@ def create_tmap( tmap_name: Name for the output TMAP HTML file. k_neighbors: Number of neighbors for layout (default: 20). """ - if properties is None: - properties = DEFAULT_PROPERTIES - - _validate_properties(properties) - - # Compute fingerprints via tmap2 - fp_kwargs = {} - if fingerprint == 'morgan': - fp_kwargs = {'n_bits': fingerprint_bits, 'radius': fingerprint_radius} - - fps = fingerprints_from_smiles(smiles, fp_type=fingerprint, **fp_kwargs) - - # Fit TMAP - model = TMAP(metric='jaccard', n_neighbors=k_neighbors).fit(fps) - - # Build visualization - viz = model.to_tmapviz(include_edges=True) - viz.title = tmap_name - viz.background_color = "#FFFFFF" - - # Add molecular properties as color layers - props = molecular_properties(smiles, properties=properties) - for prop_name, values in props.items(): - viz.add_color_layout(prop_name, values, categorical=False, color='rainbow') - - # Add SMILES for structure rendering in tooltips - viz.add_smiles(smiles) - + viz, _ = _build_viz(smiles, fingerprint, fingerprint_bits, fingerprint_radius, + properties, tmap_name, k_neighbors) viz.write_html(f"{tmap_name}.html") @@ -92,6 +66,35 @@ def representative_tmap( tmap_name: Name for the output TMAP HTML file. k_neighbors: Number of neighbors for KNN graph (default: 21). """ + viz, valid = _build_viz(smiles, fingerprint, fingerprint_bits, fingerprint_radius, + properties, tmap_name, k_neighbors) + + # Add cluster ID as categorical color channel + if cluster_ids is not None: + viz.add_color_layout( + 'Cluster ID', + [str(cid) for cid, ok in zip(cluster_ids, valid) if ok], + categorical=True, + color='tab20', + ) + + viz.write_html(f"{tmap_name}.html") + + +def _build_viz( + smiles: list[str], + fingerprint: str, + fingerprint_bits: int, + fingerprint_radius: int, + properties: list[str] | None, + tmap_name: str, + k_neighbors: int, +): + """Fit a TMAP and return ``(viz, valid)`` with property layers and SMILES added. + + SMILES that RDKit cannot parse are dropped, so every layer lines up with the + points. ``valid`` holds one flag per input SMILES, for aligning extra layers. + """ if properties is None: properties = DEFAULT_PROPERTIES @@ -102,9 +105,10 @@ def representative_tmap( if fingerprint == 'morgan': fp_kwargs = {'n_bits': fingerprint_bits, 'radius': fingerprint_radius} - fps = fingerprints_from_smiles(smiles, fp_type=fingerprint, **fp_kwargs) + fps, valid = fingerprints_from_smiles(smiles, fp_type=fingerprint, return_valid=True, **fp_kwargs) + smiles = [smi for smi, ok in zip(smiles, valid) if ok] - # Fit TMAP — use euclidean for dense descriptors like MQN + # Fit TMAP, use euclidean for dense descriptors like MQN metric = 'euclidean' if fingerprint == 'mqn' else 'jaccard' model = TMAP(metric=metric, n_neighbors=k_neighbors).fit(fps) @@ -118,19 +122,10 @@ def representative_tmap( for prop_name, values in props.items(): viz.add_color_layout(prop_name, values, categorical=False, color='rainbow') - # Add cluster ID as categorical color channel - if cluster_ids is not None: - viz.add_color_layout( - 'Cluster ID', - [str(cid) for cid in cluster_ids], - categorical=True, - color='tab20', - ) - # Add SMILES for structure rendering in tooltips viz.add_smiles(smiles) - viz.write_html(f"{tmap_name}.html") + return viz, valid def _validate_properties(properties: list[str]) -> None: @@ -153,11 +148,14 @@ def main(): Examples: chelombus-tmap --smiles molecules.smi chelombus-tmap --smiles molecules.smi --fingerprint mqn --properties mw,logp,qed - chelombus-tmap --smiles molecules.smi --cluster-file clusters.csv + chelombus-tmap --cluster-file clusters.csv """ ) - parser.add_argument('--smiles', type=str, required=True, - help="Path to file containing SMILES (one per line)") + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument('--smiles', type=str, + help="Path to file containing SMILES (one per line)") + source.add_argument('--cluster-file', type=str, + help="CSV/parquet file with 'smiles' and 'cluster_id' columns for representative TMAP") parser.add_argument('--fingerprint', type=str, default='morgan', choices=['morgan', 'mqn'], help="Fingerprint type (default: morgan)") @@ -169,8 +167,6 @@ def main(): help=f"Comma-separated properties (default: all). Available: {', '.join(AVAILABLE_PROPERTIES)}") parser.add_argument('--output', type=str, default='my_tmap', help="Output TMAP name (default: my_tmap)") - parser.add_argument('--cluster-file', type=str, default=None, - help="CSV/parquet file with 'smiles' and 'cluster_id' columns for representative TMAP") parser.add_argument('--representative', action='store_true', help="Use representative_tmap layout (edge list based)") @@ -203,7 +199,7 @@ def main(): tmap_name=args.output, ) else: - with open(args.smiles, 'r') as file: + with open(args.smiles, 'r', encoding='utf-8') as file: smiles = [line.strip() for line in file.readlines() if line.strip()] if args.representative: diff --git a/docs/getting-started.md b/docs/getting-started.md index fdf6bca..bb41365 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -10,10 +10,10 @@ pip install chelombus ### GPU Support (optional) -GPU acceleration requires PyTorch and Triton. Install them separately for your CUDA version: +GPU acceleration requires PyTorch and Triton (Linux only). Install PyTorch for your CUDA version; on Linux it includes Triton: ```bash -pip install torch triton +pip install torch ``` When CUDA is available, `PQEncoder.fit()`, `.transform()`, `PQKMeans.fit()`, and `.predict()` automatically use the GPU. No code changes needed — the library falls back to CPU transparently if CUDA is not detected. @@ -29,6 +29,12 @@ pip install -e . !!! warning "Apple Silicon (M1/M2/M3)" The `pqkmeans` library is not currently supported on Apple Silicon Macs. Clustering functionality requires an x86_64 system. +!!! warning "Windows" + Not supported natively. `pqkmeans` has no Windows wheels (building it needs MSVC and CMake), and the GPU kernels need Triton, which is Linux only. Use [WSL2](https://learn.microsoft.com/windows/wsl/install) instead. + +!!! note "Scripts on macOS (and Windows)" + `FingerprintFromSmiles` runs in parallel worker processes, which re-import the calling script on these systems. Put a script's code under `if __name__ == "__main__":`. Notebooks are not affected. + ## Quick Start ```python diff --git a/examples/enamine_1B_clustering.ipynb b/examples/enamine_1B_clustering.ipynb index ea25073..42a884b 100644 --- a/examples/enamine_1B_clustering.ipynb +++ b/examples/enamine_1B_clustering.ipynb @@ -253,7 +253,7 @@ "from chelombus import PQKMeans\n", "import time\n", "\n", - "kmeans = PQKMeans(encoder=pq_encoder, k=10_000, iteration=20)\n", + "kmeans = PQKMeans(encoder=pq_encoder, k=10_000, iterations=20)\n", "s = time.time()\n", "kmeans.fit(pq_codes)\n", "e = time.time()\n", diff --git a/examples/tutorial.ipynb b/examples/tutorial.ipynb index a1e7b58..c29aa7f 100644 --- a/examples/tutorial.ipynb +++ b/examples/tutorial.ipynb @@ -211,18 +211,6 @@ "Since we have a PQ-code of 4 digits and each digit can take value {0,255} then the number of theoretical unique PQ-codes we can get is therefore $256^4 = 4,294,967,296$. However we can test that this is much less in reality. " ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from chelombus import PQEncoder\n", - "\n", - "encoder = PQEncoder.load('/home/afloresep/work/Chelombus/models/paper_encoder.joblib')\n", - "encoder.transform(np.random.randint(0, 200, (10_000_000, 42)))\n" - ] - }, { "cell_type": "code", "execution_count": null, @@ -281,9 +269,9 @@ "\n", "# Create a PQKMeans clusterer\n", "# k: number of clusters\n", - "# iteration: number of k-means iterations\n", + "# iterations: number of k-means iterations\n", "n_clusters = 100\n", - "clusterer = PQKMeans(encoder=pq_encoder, k=n_clusters, iteration=20, verbose=False)\n", + "clusterer = PQKMeans(encoder=pq_encoder, k=n_clusters, iterations=20, verbose=False)\n", "\n", "# Fit and predict in one step\n", "labels = clusterer.fit_predict(X_pq_code)\n", @@ -330,7 +318,7 @@ "outputs": [], "source": [ "# Fit on training data\n", - "clusterer2 = PQKMeans(encoder=pq_encoder, k=50, iteration=15)\n", + "clusterer2 = PQKMeans(encoder=pq_encoder, k=50, iterations=15)\n", "clusterer2.fit(X_pq_code)\n", "\n", "# Predict on new data (using same data for demo)\n", @@ -384,7 +372,10 @@ "\n", "os.makedirs(\"data/results\", exist_ok=True)\n", "\n", - "# Create a dataframe with smiles and cluster_id\n", + "# Create a dataframe with smiles and cluster_id.\n", + "# labels come from the first fingerprint chunk, so they line up with the first\n", + "# len(labels) SMILES only because every SMILES in it parses. With your own data,\n", + "# use FingerprintFromSmiles(..., return_valid_idx=True) to keep them aligned.\n", "cluster_df = pd.DataFrame({\n", " \"smiles\": smiles[:len(labels)],\n", " \"cluster_id\": labels\n", @@ -408,7 +399,7 @@ "from chelombus import get_cluster_stats, query_cluster, sample_from_cluster\n", "\n", "# Get statistics for all clusters\n", - "stats = get_cluster_stats(\"/mnt/10tb_hdd/enamine_clustered_october_2025/build_clickhouse_db_enamine_files\", \"2B_cluster_id\")\n", + "stats = get_cluster_stats(\"data/results/\")\n", "print(f\"Number of clusters: {len(stats)}\")\n", "print(f\"Total molecules: {stats['molecule_count'].sum():,}\")\n", "print(f\"\\nTop 5 largest clusters:\")\n", @@ -530,4 +521,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/scripts/benchmark_1B_pipeline.py b/scripts/benchmark_1B_pipeline.py index b58174b..5cc3d02 100644 --- a/scripts/benchmark_1B_pipeline.py +++ b/scripts/benchmark_1B_pipeline.py @@ -156,7 +156,7 @@ def main(): # ── 3. Cluster training ────────────────────────────────────────────── print(f"\n{sep}\n STEP 3: Cluster training ({N:,}, K={args.k:,}, {args.fit_iters} iters, tol=0)\n{sep}", flush=True) - clusterer = PQKMeans(encoder, k=args.k, iteration=args.fit_iters, tol=0, verbose=True) + clusterer = PQKMeans(encoder, k=args.k, iterations=args.fit_iters, tol=0, verbose=True) t_fit, _ = bench("GPU", lambda: clusterer.fit(pq_codes, device='gpu')) results["Cluster training"] = t_fit diff --git a/scripts/fit_pqkmeans.py b/scripts/fit_pqkmeans.py index ca2aa54..8187ddc 100644 --- a/scripts/fit_pqkmeans.py +++ b/scripts/fit_pqkmeans.py @@ -225,7 +225,7 @@ def main(): pq_encoder = joblib.load(args.encoder) kmeans = PQKMeans(encoder=pq_encoder, k=args.k, - iteration=args.iterations, + iterations=args.iterations, verbose=args.kmeans_verbose) train_start = time.time() diff --git a/scripts/k_selection_gpu.py b/scripts/k_selection_gpu.py index 7867a9a..d534fe1 100644 --- a/scripts/k_selection_gpu.py +++ b/scripts/k_selection_gpu.py @@ -125,7 +125,7 @@ def main(): continue # Fit on GPU - clusterer = PQKMeans(encoder, k=k, iteration=args.iterations, verbose=False) + clusterer = PQKMeans(encoder, k=k, iterations=args.iterations, verbose=False) t0 = time.perf_counter() clusterer.fit(pq_codes, device='gpu') t_fit = time.perf_counter() - t0 diff --git a/scripts/pipeline/01_train_models.py b/scripts/pipeline/01_train_models.py index f843c7f..1866bc3 100644 --- a/scripts/pipeline/01_train_models.py +++ b/scripts/pipeline/01_train_models.py @@ -39,7 +39,6 @@ import argparse import gc -import glob import json import os import sys @@ -468,7 +467,7 @@ def stage4_train_clusterer( clusterer = PQKMeans( encoder=encoder, k=config.n_clusters, - iteration=config.clusterer_iterations, + iterations=config.clusterer_iterations, verbose=verbose > 0 ) @@ -534,8 +533,10 @@ def stage5_cluster_all_data( chunk_idx += 1 continue - # Calculate fingerprints - fps = fp_calc.FingerprintFromSmiles(smiles_chunk, fp="mqn") + # Calculate fingerprints. valid_idx keeps SMILES and labels aligned + # when some SMILES fail to parse. + valid_idx, fps = fp_calc.FingerprintFromSmiles(smiles_chunk, fp="mqn", + return_valid_idx=True) if len(fps) == 0: chunk_idx += 1 @@ -547,17 +548,10 @@ def stage5_cluster_all_data( # Predict clusters cluster_ids = clusterer.predict(pq_codes) - # Create output DataFrame - if len(fps) == len(smiles_chunk): - output_df = pd.DataFrame({ - "smiles": smiles_chunk, - "cluster_id": cluster_ids, - }) - else: - # Some SMILES failed - can't reliably associate - output_df = pd.DataFrame({ - "cluster_id": cluster_ids, - }) + output_df = pd.DataFrame({ + "smiles": [smiles_chunk[i] for i in valid_idx], + "cluster_id": cluster_ids, + }) # Save chunk output_file = os.path.join(results_dir, f"chunk_{chunk_idx:05d}.parquet") diff --git a/scripts/pipeline/README.md b/scripts/pipeline/README.md index b979338..ce52ba7 100644 --- a/scripts/pipeline/README.md +++ b/scripts/pipeline/README.md @@ -120,6 +120,9 @@ Result: `OUT/models/encoder.joblib` and `OUT/models/clusterer.joblib`. This is the one pass over **all** your molecules. It uses the GPU. It writes parquet files with three columns: `id`, `smiles`, `cluster_id`. +If your file has no id column, leave out `--id-col 0` (molecules are then +numbered 0, 1, 2, ...). `--resume` skips chunks already done if you restart. + ```bash $PQ scripts/pipeline/02_assign_clusters.py \ --input all_molecules.smi \ @@ -127,9 +130,9 @@ $PQ scripts/pipeline/02_assign_clusters.py \ --encoder OUT/models/encoder.joblib \ --clusterer OUT/models/clusterer.joblib \ --smiles-col 0 \ - --id-col 0 \ # DELETE this line if your file has no id column + --id-col 0 \ --device gpu \ - --resume # skip chunks already done if you restart + --resume ``` ### Step 3 — Group (put each cluster together) diff --git a/scripts/pipeline/run_all.sh b/scripts/pipeline/run_all.sh index 08d9fc3..b720e2e 100644 --- a/scripts/pipeline/run_all.sh +++ b/scripts/pipeline/run_all.sh @@ -25,22 +25,22 @@ PQ=/home/afloresep/miniforge3/envs/pqkmeans/bin/python # compute / GPU TM=/home/afloresep/miniforge3/envs/tmap2/bin/python # tmap2 viz # --- Where the code lives ---------------------------------------------------- -REPO=/mnt/10tb_hdd/work/Chelombus +REPO=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) # this checkout P=$REPO/scripts/pipeline # all pipeline scripts (01..06 + helpers) live here # --- Your data and where results go (EDIT THESE) ----------------------------- SRC=/mnt/10tb_hdd/cleaned_enamine_10b/output_file_0.cxsmiles # input file(s) ROOT=/mnt/10tb_hdd/enamine_1b # all output here SMILES_COL=1 # 0-based column that holds the SMILES in $SRC -ID_COL=0 # 0-based column that holds the molecule id (see note below) +ID_COL=0 # 0-based column that holds the molecule id, or empty (see note below) K=50000 # number of clusters TRAIN_SAMPLE=100000000 # molecules used to TRAIN (you don't need all 1B) MAX_CLUSTERS=1000 # how many drill-down maps to build in this run # # NOTE on columns: if your file is just ONE SMILES per line with nothing else, -# set SMILES_COL=0 and delete the "--id-col $ID_COL" text in STEP 2 below; the -# molecules will then be auto-numbered (0,1,2,...). Set ID_COL only when your -# file really has an id column you want to keep. +# set SMILES_COL=0 and ID_COL= (empty); the molecules will then be +# auto-numbered (0,1,2,...). Set ID_COL only when your file really has an id +# column you want to keep. # ---------------------------------------------------------------------------- SAMPLE=$ROOT/train_sample.smi @@ -72,7 +72,7 @@ say "STEP 2: assigning all molecules to clusters (full pass, --resume)" "$PQ" "$P/02_assign_clusters.py" --input "$SRC" --output "$ROOT/labels" \ --encoder "$ROOT/models/encoder.joblib" \ --clusterer "$ROOT/models/clusterer.joblib" \ - --smiles-col "$SMILES_COL" --id-col "$ID_COL" \ + --smiles-col "$SMILES_COL" ${ID_COL:+--id-col "$ID_COL"} \ --device gpu --resume || die "assign" # ── STEP 3: GROUP molecules by cluster (DuckDB) ────────────────────────────── diff --git a/scripts/select_k.py b/scripts/select_k.py index 945bfaa..9e3886e 100644 --- a/scripts/select_k.py +++ b/scripts/select_k.py @@ -253,7 +253,7 @@ def main(): continue # Fit - clusterer = PQKMeans(encoder, k=k, iteration=args.iterations, verbose=False) + clusterer = PQKMeans(encoder, k=k, iterations=args.iterations, verbose=False) t0 = time.time() clusterer.fit(pq_codes) t_fit = time.time() - t0 diff --git a/tests/test_cluster_io.py b/tests/test_cluster_io.py index 3829317..2623af6 100644 --- a/tests/test_cluster_io.py +++ b/tests/test_cluster_io.py @@ -415,9 +415,22 @@ def test_sample_with_random_state(self, large_results_dir): df1 = sample_from_cluster(large_results_dir, cluster_id=0, n=5, random_state=42) df2 = sample_from_cluster(large_results_dir, cluster_id=0, n=5, random_state=42) - # Note: DuckDB's SAMPLE REPEATABLE should give same results - # but may not be exactly identical in all versions - assert len(df1) == len(df2) + df3 = sample_from_cluster(large_results_dir, cluster_id=0, n=5, random_state=7) + + assert df1["smiles"].tolist() == df2["smiles"].tolist() + assert df1["smiles"].tolist() != df3["smiles"].tolist() + + def test_sample_with_random_state_duplicate_rows(self, tmp_path): + """Duplicate rows are sampled independently, not grouped together.""" + results_dir = tmp_path / "dup" + results_dir.mkdir() + pd.DataFrame({"smiles": ["A"] * 100 + ["B"] * 100, "cluster_id": 0}).to_parquet( + results_dir / "chunk_00000.parquet", index=False) + + for seed in range(5): + df = sample_from_cluster(results_dir, cluster_id=0, n=20, random_state=seed) + assert list(df.columns) == ["smiles", "cluster_id"] + assert set(df["smiles"]) == {"A", "B"} def test_sample_empty_cluster(self, sample_results_dir): """Test sampling from non-existent cluster.""" @@ -453,9 +466,43 @@ def test_export_all_and_query_back(self, sample_results_dir, tmp_path): stats = export_all_clusters(sample_results_dir, output_dir) # Query from partitioned output using DuckDB directly - df = duckdb.query(f""" + df = duckdb.execute(f""" SELECT * FROM '{output_dir}/*/*.parquet' WHERE cluster_id = 0 """).df() assert len(df) == stats[0] + + +class TestPathsAndColumnNames: + """Paths with quotes and non-default column names.""" + + @pytest.fixture + def custom_dir(self, tmp_path): + results_dir = tmp_path / "O'Brien results" + results_dir.mkdir() + pd.DataFrame({ + "SMILES": ["CCO", "CC", "CCC", "CCCC"], + "label": [0, 1, 0, 1], + }).to_parquet(results_dir / "part.parquet", index=False) + return results_dir + + def test_query_path_with_quote(self, custom_dir): + df = query_cluster(custom_dir, cluster_id=0, cluster_column="label") + assert sorted(df["SMILES"]) == ["CCC", "CCO"] + + def test_sample_custom_cluster_column(self, custom_dir): + df = sample_from_cluster(custom_dir, cluster_id=1, n=5, cluster_column="label") + assert sorted(df["SMILES"]) == ["CC", "CCCC"] + + def test_export_smi_custom_smiles_column(self, custom_dir, tmp_path): + out = tmp_path / "it's.smi" + count = export_cluster(custom_dir, 0, out, cluster_column="label", smiles_column="SMILES") + assert count == 2 + assert sorted(out.read_text().split()) == ["CCC", "CCO"] + + def test_export_all_custom_columns(self, custom_dir, tmp_path): + stats = export_all_clusters(custom_dir, tmp_path / "out", cluster_column="label", + smiles_column="SMILES") + assert stats == {0: 2, 1: 2} + assert (tmp_path / "out" / "label=0").is_dir() diff --git a/tests/test_clustering.py b/tests/test_clustering.py index 8d3bfa3..c98a56d 100644 --- a/tests/test_clustering.py +++ b/tests/test_clustering.py @@ -60,11 +60,11 @@ class TestPQKMeansInit: def test_init_with_trained_encoder(self, trained_encoder): """Test that PQKMeans can be initialized with a trained encoder.""" - clusterer = PQKMeans(trained_encoder, k=10, iteration=5, verbose=False) + clusterer = PQKMeans(trained_encoder, k=10, iterations=5, verbose=False) assert clusterer.encoder == trained_encoder assert clusterer.k == 10 - assert clusterer.iteration == 5 + assert clusterer.iterations == 5 assert clusterer.verbose is False assert clusterer.encoder.is_trained is True # Encoder is trained @@ -78,23 +78,30 @@ def test_init_default_parameters(self, trained_encoder): clusterer = PQKMeans(trained_encoder, k=100) assert clusterer.k == 100 - assert clusterer.iteration == 20 # default + assert clusterer.iterations == 20 # default assert clusterer.verbose is False # default + def test_init_deprecated_iteration_alias(self, trained_encoder): + """The old 'iteration' keyword still works but warns.""" + with pytest.warns(DeprecationWarning, match="iterations"): + clusterer = PQKMeans(trained_encoder, k=10, iteration=7) + + assert clusterer.iterations == 7 + class TestPQKMeansFit: """Tests for PQKMeans fit method.""" def test_fit_returns_self(self, trained_encoder, sample_pq_codes): """Test that fit() returns self for method chaining.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) result = clusterer.fit(sample_pq_codes) assert result is clusterer def test_fit_with_valid_pq_codes(self, trained_encoder, sample_pq_codes): """Test fitting with valid PQ codes.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) # Should complete without error @@ -102,7 +109,7 @@ def test_fit_with_valid_pq_codes(self, trained_encoder, sample_pq_codes): def test_fit_with_small_k(self, trained_encoder, sample_pq_codes): """Test fitting with small number of clusters.""" - clusterer = PQKMeans(trained_encoder, k=3, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=3, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) # Predict should return labels in range [0, k) @@ -116,7 +123,7 @@ class TestPQKMeansPredict: def test_predict_shape(self, trained_encoder, sample_pq_codes): """Test that predict returns correct shape.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) labels = clusterer.predict(sample_pq_codes) @@ -126,7 +133,7 @@ def test_predict_shape(self, trained_encoder, sample_pq_codes): def test_predict_label_range(self, trained_encoder, sample_pq_codes): """Test that predicted labels are in valid range.""" k = 5 - clusterer = PQKMeans(trained_encoder, k=k, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=k, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) labels = clusterer.predict(sample_pq_codes) @@ -136,7 +143,7 @@ def test_predict_label_range(self, trained_encoder, sample_pq_codes): def test_predict_dtype(self, trained_encoder, sample_pq_codes): """Test that predicted labels are integers.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) labels = clusterer.predict(sample_pq_codes) @@ -149,7 +156,7 @@ class TestPQKMeansFitPredict: def test_fit_predict_shape(self, trained_encoder, sample_pq_codes): """Test that fit_predict returns correct shape.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) labels = clusterer.fit_predict(sample_pq_codes) @@ -159,7 +166,7 @@ def test_fit_predict_equivalence(self, trained_encoder, sample_pq_codes): """Test that fit_predict gives same result as fit then predict.""" np.random.seed(42) - clusterer1 = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer1 = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) labels1 = clusterer1.fit_predict(sample_pq_codes) # Note: Due to randomness in k-means, we can't guarantee exact equality @@ -173,7 +180,7 @@ class TestPQKMeansSaveLoad: def test_save_creates_file(self, trained_encoder, sample_pq_codes, tmp_path): """Test that save creates a file.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) save_path = tmp_path / "clusterer.pkl" @@ -183,7 +190,7 @@ def test_save_creates_file(self, trained_encoder, sample_pq_codes, tmp_path): def test_load_restores_clusterer(self, trained_encoder, sample_pq_codes, tmp_path): """Test that load restores a clusterer correctly.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) labels_before = clusterer.predict(sample_pq_codes) @@ -208,7 +215,7 @@ def test_load_wrong_type_raises(self, tmp_path): def test_load_with_string_path(self, trained_encoder, sample_pq_codes, tmp_path): """Test that load works with string path.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes) save_path = str(tmp_path / "clusterer.pkl") @@ -253,7 +260,7 @@ def test_cluster_mqn_like_data(self, trained_encoder): # Encode and cluster pq_codes = trained_encoder.transform(X, verbose=0) - clusterer = PQKMeans(trained_encoder, k=n_true_clusters, iteration=10, verbose=False) + clusterer = PQKMeans(trained_encoder, k=n_true_clusters, iterations=10, verbose=False) labels = clusterer.fit_predict(pq_codes) # Check basic properties @@ -269,7 +276,7 @@ def test_large_k_clustering(self, trained_encoder): pq_codes = trained_encoder.transform(X, verbose=0) k = 50 - clusterer = PQKMeans(trained_encoder, k=k, iteration=5, verbose=False) + clusterer = PQKMeans(trained_encoder, k=k, iterations=5, verbose=False) labels = clusterer.fit_predict(pq_codes) # Should have multiple clusters represented @@ -319,7 +326,7 @@ def test_mocked_gpu_fit_predict_matches_final_centers(self, monkeypatch, trained lambda *args, **kwargs: np.random.Generator(np.random.PCG64(0)), ) - clusterer = PQKMeans(trained_encoder, k=10, iteration=20, tol=1e-3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=10, iterations=20, tol=1e-3, verbose=False) labels = clusterer.fit_predict(sample_pq_codes, device='gpu') final_labels = clusterer.predict(sample_pq_codes, device='cpu') @@ -341,7 +348,7 @@ def test_mocked_gpu_fit_clears_transient_labels(self, monkeypatch, trained_encod lambda *args, **kwargs: np.random.Generator(np.random.PCG64(1)), ) - clusterer = PQKMeans(trained_encoder, k=5, iteration=5, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=5, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') assert clusterer._fit_labels is None @@ -356,7 +363,7 @@ def fail_predict_gpu(*args, **kwargs): monkeypatch.setattr(pyqkmeans_module, "predict_gpu", fail_predict_gpu, raising=False) - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) # Pretend the encoder has k > 256 to trigger the guard monkeypatch.setattr(trained_encoder, "k", 512) @@ -412,7 +419,7 @@ class TestGPUPredict: @gpu def test_gpu_predict_matches_cpu(self, trained_encoder, sample_pq_codes): """GPU and CPU predict must return identical labels.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes, device='cpu') labels_cpu = clusterer.predict(sample_pq_codes, device='cpu') @@ -424,7 +431,7 @@ def test_gpu_predict_matches_cpu(self, trained_encoder, sample_pq_codes): def test_gpu_predict_shape_and_range(self, trained_encoder, sample_pq_codes): """GPU predict returns correct shape and valid label range.""" k = 5 - clusterer = PQKMeans(trained_encoder, k=k, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=k, iterations=3, verbose=False) clusterer.fit(sample_pq_codes, device='cpu') labels = clusterer.predict(sample_pq_codes, device='gpu') @@ -437,7 +444,7 @@ def test_gpu_predict_shape_and_range(self, trained_encoder, sample_pq_codes): @gpu def test_gpu_predict_deterministic(self, trained_encoder, sample_pq_codes): """Repeated GPU predict calls return the same labels.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes, device='cpu') labels1 = clusterer.predict(sample_pq_codes, device='gpu') @@ -453,7 +460,7 @@ class TestGPUFit: def test_gpu_fit_produces_valid_clusters(self, trained_encoder, sample_pq_codes): """GPU fit must produce a trained model with valid labels.""" k = 5 - clusterer = PQKMeans(trained_encoder, k=k, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=k, iterations=3, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') assert clusterer.is_trained @@ -465,7 +472,7 @@ def test_gpu_fit_produces_valid_clusters(self, trained_encoder, sample_pq_codes) @gpu def test_gpu_fit_returns_self(self, trained_encoder, sample_pq_codes): """GPU fit returns self for method chaining.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) result = clusterer.fit(sample_pq_codes, device='gpu') assert result is clusterer @@ -473,7 +480,7 @@ def test_gpu_fit_returns_self(self, trained_encoder, sample_pq_codes): @gpu def test_gpu_fit_centers_are_valid_codes(self, trained_encoder, sample_pq_codes): """Cluster centers after GPU fit must be valid codebook indices.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') centers = clusterer.cluster_centers_ @@ -485,7 +492,7 @@ def test_gpu_fit_centers_are_valid_codes(self, trained_encoder, sample_pq_codes) def test_gpu_fit_predict(self, trained_encoder, sample_pq_codes): """GPU fit_predict returns labels of correct shape and range.""" k = 5 - clusterer = PQKMeans(trained_encoder, k=k, iteration=3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=k, iterations=3, verbose=False) labels = clusterer.fit_predict(sample_pq_codes, device='gpu') assert labels.shape == (len(sample_pq_codes),) @@ -499,7 +506,7 @@ class TestGPUSaveLoad: @gpu def test_gpu_save_load_labels_match(self, trained_encoder, sample_pq_codes, tmp_path): """Labels must match after save/load of a GPU-trained model.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, tol=0, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, tol=0, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') labels_before = clusterer.predict(sample_pq_codes) @@ -513,7 +520,7 @@ def test_gpu_save_load_labels_match(self, trained_encoder, sample_pq_codes, tmp_ @gpu def test_gpu_save_load_gpu_predict(self, trained_encoder, sample_pq_codes, tmp_path): """GPU predict works correctly on a loaded model.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, tol=0, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, tol=0, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') labels_before = clusterer.predict(sample_pq_codes, device='gpu') @@ -531,7 +538,7 @@ class TestEarlyStopping: @gpu def test_tol_stops_before_max_iterations(self, trained_encoder, sample_pq_codes): """With default tol, training should stop before iteration limit.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=50, tol=1e-3, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=50, tol=1e-3, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') # If early stopping works, it should finish quickly (not run 50 iters). @@ -544,7 +551,7 @@ def test_tol_stops_before_max_iterations(self, trained_encoder, sample_pq_codes) @gpu def test_tol_zero_only_stops_on_exact_convergence(self, trained_encoder, sample_pq_codes): """tol=0 should only stop when no centers change at all.""" - clusterer = PQKMeans(trained_encoder, k=5, iteration=3, tol=0, verbose=False) + clusterer = PQKMeans(trained_encoder, k=5, iterations=3, tol=0, verbose=False) clusterer.fit(sample_pq_codes, device='gpu') assert clusterer.is_trained diff --git a/tests/test_data_streamer.py b/tests/test_data_streamer.py index ad02124..6cedc7c 100644 --- a/tests/test_data_streamer.py +++ b/tests/test_data_streamer.py @@ -66,3 +66,70 @@ def test_parse_input_omits_invalid_line_without_chunksize(tmp_path, monkeypatch) # Expected: one chunk containing only the valid lines. assert result == [["valid_line1", "valid_line2"]] + + +def test_parse_input_gzip(tmp_path: Path): + import gzip + with gzip.open(tmp_path / "mols.smi.gz", "wt") as f: + f.write("CCO\nc1ccccc1\n") + + result = list(DataStreamer().parse_input(str(tmp_path / "mols.smi.gz"))) + assert result == [["CCO", "c1ccccc1"]] + + +def test_parse_input_gzip_sdf(tmp_path: Path): + import gzip + from rdkit import Chem + with gzip.open(tmp_path / "mols.sdf.gz", "wt") as f: + for smi in ("CCO", "CCN"): + f.write(Chem.MolToMolBlock(Chem.MolFromSmiles(smi)) + "$$$$\n") + + result = list(DataStreamer().parse_input(str(tmp_path / "mols.sdf.gz"))) + assert result == [["CCO", "CCN"]] + + +def test_parse_input_glob_sorted(tmp_path: Path): + (tmp_path / "b.smi").write_text("CCN\n") + (tmp_path / "a.smi").write_text("CCO\n") + (tmp_path / "notes.md").write_text("not smiles\n") + + result = list(DataStreamer().parse_input(str(tmp_path / "*.smi"))) + assert result == [["CCO"], ["CCN"]] + + +def test_parse_input_glob_no_match(tmp_path: Path): + with pytest.raises(FileNotFoundError): + list(DataStreamer().parse_input(str(tmp_path / "*.smi"))) + + +def test_parse_input_directory_sorted_skips_hidden(tmp_path: Path): + (tmp_path / "b.smi").write_text("CCN\n") + (tmp_path / "a.smi").write_text("CCO\n") + (tmp_path / ".DS_Store").write_bytes(b"\x00\x01") + + result = list(DataStreamer().parse_input(str(tmp_path))) + assert result == [["CCO"], ["CCN"]] + + +def test_parse_input_csv_sep(tmp_path: Path): + (tmp_path / "mols.csv").write_text("id1, CCO\nid2,CCN\n\n") + + result = list(DataStreamer().parse_input(str(tmp_path / "mols.csv"), smiles_col=1, + id_col=0, sep=",")) + assert result == [[("id1", "CCO"), ("id2", "CCN")]] + + +def test_parse_input_csv_quoted_fields(tmp_path: Path): + (tmp_path / "mols.csv").write_text('"CCO",id1\n"C(C)O","a,b"\n') + + result = list(DataStreamer().parse_input(str(tmp_path / "mols.csv"), id_col=1, sep=",")) + assert result == [[("id1", "CCO"), ("a,b", "C(C)O")]] + + +def test_parse_input_existing_path_with_glob_characters(tmp_path: Path): + folder = tmp_path / "batch[1]" + folder.mkdir() + (folder / "a.smi").write_text("CCO\n") + + result = list(DataStreamer().parse_input(str(folder))) + assert result == [["CCO"]] diff --git a/tests/test_fingerprints.py b/tests/test_fingerprints.py index d34b781..30344f0 100644 --- a/tests/test_fingerprints.py +++ b/tests/test_fingerprints.py @@ -112,4 +112,64 @@ def test_fingerprint_with_data_parser(tmp_path): assert len(result) == 4 assert result[0].shape == (2, 1024) - assert result[3].shape == (1, 1024) \ No newline at end of file + assert result[3].shape == (1, 1024) + +def test_fingerprint_calculator_serial_matches_pool(): + """nprocesses=1 runs in-process and gives the same result as the pool.""" + smiles = ['CCO', 'c1ccccc1O', 'not_a_smiles', 'CC(=O)Nc1ccc(O)cc1'] + calculator = FingerprintCalculator() + + idx_serial, fps_serial = calculator.FingerprintFromSmiles( + smiles, 'mqn', nprocesses=1, return_valid_idx=True) + idx_pool, fps_pool = calculator.FingerprintFromSmiles( + smiles, 'mqn', nprocesses=2, return_valid_idx=True) + + np.testing.assert_array_equal(idx_serial, [0, 1, 3]) + np.testing.assert_array_equal(idx_serial, idx_pool) + np.testing.assert_array_equal(fps_serial, fps_pool) + assert fps_serial.shape == (3, 42) + + +def test_fingerprint_calculator_unguarded_script_errors_instead_of_hanging(tmp_path): + """A script without a __main__ guard under spawn (Windows default) must + fail fast with a helpful error, not respawn workers forever.""" + import os + import subprocess + import sys + from pathlib import Path + + script = tmp_path / "unguarded.py" + script.write_text( + "import multiprocessing as mp\n" + "mp.set_start_method('spawn', force=True)\n" + "from chelombus.utils.fingerprints import FingerprintCalculator\n" + "FingerprintCalculator().FingerprintFromSmiles(['CCO'] * 20, 'mqn', nprocesses=2)\n" + ) + repo_root = str(Path(__file__).resolve().parents[1]) + env = {**os.environ, "PYTHONPATH": repo_root + os.pathsep + os.environ.get("PYTHONPATH", "")} + + result = subprocess.run([sys.executable, str(script)], cwd=tmp_path, env=env, + capture_output=True, text=True, timeout=120) + + assert result.returncode != 0 + assert "__main__" in result.stderr and "guard" in result.stderr + + +def test_fingerprint_calculator_warns_when_nothing_parses(caplog): + with caplog.at_level(logging.WARNING): + fps = FingerprintCalculator().FingerprintFromSmiles(["CCO,1", "CCN,2"], 'mqn', nprocesses=1) + + assert len(fps) == 0 + assert "None of the 2 SMILES could be parsed" in caplog.text + + +@pytest.mark.parametrize("container", [np.array, list, tuple]) +def test_fingerprint_calculator_accepts_array_like(container): + fps = FingerprintCalculator().FingerprintFromSmiles(container(["CCO", "CCN"]), 'mqn', nprocesses=1) + assert fps.shape == (2, 42) + + +def test_fingerprint_calculator_accepts_pandas_series(): + pd = pytest.importorskip("pandas") + fps = FingerprintCalculator().FingerprintFromSmiles(pd.Series(["CCO", "CCN"]), 'mqn', nprocesses=1) + assert fps.shape == (2, 42) diff --git a/tests/test_integration.py b/tests/test_integration.py index b59ca11..ca13242 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -172,7 +172,7 @@ def test_full_pipeline_small_dataset(self, smiles_file, results_dir): # Step 4: Train and apply clustering n_clusters = 5 - clusterer = PQKMeans(encoder, k=n_clusters, iteration=5, verbose=False) + clusterer = PQKMeans(encoder, k=n_clusters, iterations=5, verbose=False) labels = clusterer.fit_predict(pq_codes) assert labels.shape == (n_valid,) @@ -214,7 +214,7 @@ def test_chunked_processing_pipeline(self, smiles_file, results_dir): # Step 2: Train clusterer on PQ codes training_pq_codes = encoder.transform(training_fps.astype(np.float32), verbose=0) n_clusters = 3 - clusterer = PQKMeans(encoder, k=n_clusters, iteration=5, verbose=False) + clusterer = PQKMeans(encoder, k=n_clusters, iterations=5, verbose=False) clusterer.fit(training_pq_codes) # Step 3: Process in chunks @@ -266,7 +266,7 @@ def test_save_load_models(self, tmp_path): # Train clusterer pq_codes = encoder.transform(fps.astype(np.float32), verbose=0) - clusterer = PQKMeans(encoder, k=3, iteration=5, verbose=False) + clusterer = PQKMeans(encoder, k=3, iterations=5, verbose=False) clusterer.fit(pq_codes) # Save clusterer diff --git a/tests/test_visualization.py b/tests/test_visualization.py new file mode 100644 index 0000000..19efa74 --- /dev/null +++ b/tests/test_visualization.py @@ -0,0 +1,55 @@ +import pytest + +pytest.importorskip("tmap") + +from chelombus.utils.visualization import create_tmap, representative_tmap + +SMILES = ["C" * i + "O" for i in range(1, 30)] + ["c1ccccc1" + "C" * i for i in range(1, 20)] + + +@pytest.mark.parametrize("fingerprint", ["mqn", "morgan"]) +def test_create_tmap_writes_html(fingerprint, tmp_path, monkeypatch): + """create_tmap works for both count (MQN) and binary (Morgan) fingerprints.""" + monkeypatch.chdir(tmp_path) + + create_tmap(SMILES, fingerprint=fingerprint, properties=["mw", "logp"], + tmap_name="test_map", k_neighbors=5) + + assert (tmp_path / "test_map.html").exists() + + +@pytest.mark.parametrize("fingerprint", ["mqn", "morgan"]) +def test_create_tmap_skips_invalid_smiles(fingerprint, tmp_path, monkeypatch): + """Unparseable SMILES are dropped instead of misaligning property layers.""" + monkeypatch.chdir(tmp_path) + + create_tmap(SMILES + ["not_a_smiles"], fingerprint=fingerprint, properties=["mw"], + tmap_name="test_map", k_neighbors=5) + + assert (tmp_path / "test_map.html").exists() + + +def test_representative_tmap_aligns_cluster_ids(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + smiles = SMILES[:10] + ["not_a_smiles"] + SMILES[10:] + representative_tmap(smiles, cluster_ids=list(range(len(smiles))), + properties=["mw"], tmap_name="rep", k_neighbors=5) + + assert (tmp_path / "rep.html").exists() + + +def test_cli_cluster_file_without_smiles(tmp_path, monkeypatch): + """--cluster-file alone is enough (it used to also require --smiles).""" + import pandas as pd + from chelombus.utils import visualization + + monkeypatch.chdir(tmp_path) + pd.DataFrame({"smiles": SMILES, "cluster_id": [i % 3 for i in range(len(SMILES))]}).to_csv( + tmp_path / "reps.csv", index=False) + monkeypatch.setattr("sys.argv", ["chelombus-tmap", "--cluster-file", "reps.csv", + "--properties", "mw", "--output", "cli_map"]) + + visualization.main() + + assert (tmp_path / "cli_map.html").exists()