Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,15 @@ pip install -e .

**Apple Silicon (M1/M2/M3)**: The `pqkmeans` library is not currently supported on Apple Silicon Macs. My plan is to rewrite pqkmeans with Silicon and GPU support but that's for a future release... For now, clustering functionality requires an x86_64 system.

**Windows**: Not supported natively. `pqkmeans` has no Windows wheels (building it needs MSVC and CMake), and the GPU kernels need Triton, which is Linux only. Use [WSL2](https://learn.microsoft.com/windows/wsl/install) instead.

**Scripts on macOS (and Windows)**: `FingerprintFromSmiles` runs in parallel worker processes, which re-import the calling script on these systems. Put a script's code under `if __name__ == "__main__":`. Notebooks are not affected.

## GPU Acceleration

Every step in the pipeline supports optional GPU acceleration via the `device` parameter: encoder training, PQ encoding, cluster training, and label assignment. When a CUDA GPU is available, `device='auto'` (the default) uses the GPU transparently; otherwise it falls back to CPU.

**Requirements**: `torch` and `triton` (both installed with `pip install torch`).
**Requirements**: `torch` and `triton` (on Linux, `pip install torch` installs both).

```python
encoder = PQEncoder(k=256, m=6, iterations=20)
Expand Down Expand Up @@ -90,11 +94,12 @@ The GPU implementation uses a custom Triton kernel for cluster assignment that t
from chelombus import DataStreamer, FingerprintCalculator, PQEncoder, PQKMeans

# 1. Stream SMILES in chunks
streamer = DataStreamer(path='molecules.smi', chunksize=100000)
streamer = DataStreamer()
smiles_gen = streamer.parse_input('molecules.smi', chunksize=100000)

# 2. Calculate MQN fingerprints
fp_calc = FingerprintCalculator()
for smiles_chunk in streamer.parse_input():
for smiles_chunk in smiles_gen:
fingerprints = fp_calc.FingerprintFromSmiles(smiles_chunk, fp='mqn')
# Save fingerprints...

Expand Down
28 changes: 19 additions & 9 deletions chelombus/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,27 @@
"""

from .streamer.data_streamer import DataStreamer
from .encoder.encoder import PQEncoder
from .utils.fingerprints import FingerprintCalculator
from .utils.helper_functions import save_chunk, format_time

# Optional imports - only available if dependencies are installed
try:
from .clustering.PyQKmeans import PQKMeans
except ImportError as e:
import warnings
warnings.warn(f"PQKMeans not available: {e}")
PQKMeans = None # pqkmeans or numba not installed

def __getattr__(name):
# PQEncoder and PQKMeans pull in torch, numba and pqkmeans, so they load on
# first use. This keeps `import chelombus` fast, which matters most for
# fingerprint worker processes on Windows and macOS (each re-imports it).
if name == "PQEncoder":
from .encoder.encoder import PQEncoder as value
elif name == "PQKMeans":
try:
from .clustering.PyQKmeans import PQKMeans as value
except ImportError as e:
import warnings
warnings.warn(f"PQKMeans not available: {e}")
value = None # pqkmeans or numba not installed
else:
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
globals()[name] = value
return value

# Cluster I/O utilities (require duckdb)
try:
Expand All @@ -36,7 +46,7 @@
def _cluster_io_not_available(*args, **kwargs):
raise ImportError(
"Cluster I/O functions require duckdb. "
"Install with: pip install chelombus[io] or pip install duckdb"
"Install with: pip install duckdb"
)
query_cluster = _cluster_io_not_available
export_cluster = _cluster_io_not_available
Expand Down
29 changes: 21 additions & 8 deletions chelombus/clustering/PyQKmeans.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"""
from typing import Literal, overload

import warnings
import joblib
from pathlib import Path
import numpy as np
Expand All @@ -23,7 +24,7 @@
if torch.cuda.is_available():
from chelombus.clustering._gpu_predict import predict_gpu
_GPU_AVAILABLE = True
except ImportError:
except Exception: # not installed, or a broken install (e.g. missing DLL)
pass


Expand Down Expand Up @@ -126,11 +127,12 @@ class PQKMeans:
Args:
encoder: A trained PQEncoder instance
k: Number of clusters
iteration: Maximum number of k-means iterations (default: 20)
iterations: Maximum number of k-means iterations (default: 20)
tol: Early-stopping tolerance for the GPU path. Training stops when
the fraction of changed center coordinates drops below *tol*.
0 means stop only on exact convergence. (default: 1e-3)
verbose: Whether to print progress information (default: False)
iteration: Deprecated alias for ``iterations``.

Example:
>>> encoder = PQEncoder(k=256, m=6)
Expand All @@ -144,16 +146,24 @@ def __init__(
self,
encoder: PQEncoder,
k: int,
iteration: int = 20,
iterations: int = 20,
tol: float = 1e-3,
verbose: bool = False
verbose: bool = False,
iteration: int | None = None,
):
if iteration is not None:
warnings.warn(
"'iteration' is deprecated, use 'iterations' instead",
DeprecationWarning,
stacklevel=2,
)
iterations = iteration
if not encoder.encoder_is_trained: # type: ignore
raise ValueError("Encoder must be trained before clustering")

self.encoder = encoder
self.k = k
self.iteration = iteration
self.iterations = iterations
self.tol = tol
self.verbose = verbose
self.trained = False
Expand All @@ -163,7 +173,7 @@ def __init__(
self._cluster = pqkmeans.clustering.PQKMeans(
encoder=encoder,
k=k,
iteration=iteration,
iteration=iterations,
verbose=verbose,
)

Expand Down Expand Up @@ -249,7 +259,7 @@ def _fit_gpu(self, X_train: np.ndarray, return_labels: bool = False) -> np.ndarr
labels = None
prev_centers = None
final_labels_match_centers = False
for it in range(self.iteration):
for it in range(self.iterations):
t0 = time.time()

# assignment (GPU)
Expand All @@ -274,7 +284,7 @@ def _fit_gpu(self, X_train: np.ndarray, return_labels: bool = False) -> np.ndarr

if self.verbose:
print(
f" iter {it + 1}/{self.iteration} "
f" iter {it + 1}/{self.iterations} "
f"assign={t1 - t0:.1f}s update={t2 - t1:.1f}s "
f"changed={changed}/{total_coords} ({frac:.4%})"
)
Expand Down Expand Up @@ -392,6 +402,9 @@ def load(cls, path: str | Path) -> "PQKMeans":
obj = joblib.load(path)
if not isinstance(obj, cls):
raise TypeError(f"Expected {cls.__name__}, got {type(obj).__name__}")
if not hasattr(obj, 'iterations'):
# Models saved before the 'iteration' -> 'iterations' rename
obj.iterations = obj.__dict__.pop('iteration')
if not hasattr(obj, '_dtables'):
obj._dtables = None
if not hasattr(obj, '_centers_u8'):
Expand Down
2 changes: 1 addition & 1 deletion chelombus/encoder/encoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import torch
if torch.cuda.is_available():
_GPU_AVAILABLE = True
except ImportError:
except Exception: # not installed, or a broken install (e.g. missing DLL)
pass


Expand Down
74 changes: 56 additions & 18 deletions chelombus/streamer/data_streamer.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
import bz2
import csv
import glob
import gzip
import os
from typing import Iterator, List, Optional

Expand All @@ -9,13 +13,15 @@ class DataStreamer:
"""
def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose: int = 0,
smiles_col: int = 0, id_col: Optional[int] = None,
with_ids: bool = False) -> Iterator[List]:
with_ids: bool = False, sep: Optional[str] = None) -> Iterator[List]:
"""
Read input data from a file or a directory of files and yield it in chunks.

The input path can be a single file or a directory of files. Plain-text
The input path can be a single file, a directory of files, or a glob
pattern like "data/*.smi". Files ending in .gz or .bz2 are decompressed
on the fly, and text is read as UTF-8. Plain-text
files are read line by line; the whitespace-separated column at
`smiles_col` (0-based) is taken as the SMILES. So a file with ONE SMILES
`smiles_col` (0-based) is taken as the SMILES (pass `sep=","` for CSV). So a file with ONE SMILES
per line works as-is (smiles_col=0), and a file like "id<TAB>smiles"
works with smiles_col=1. SDF/SD files are streamed with RDKit and turned
into SMILES on the fly. Items are buffered until `chunksize` is reached,
Expand All @@ -34,7 +40,8 @@ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose:
Ids are always returned as strings.

Args:
input_path (str): Path to a file or directory of input files.
input_path (str): Path to a file, a directory of input files, or a
glob pattern. Directory and glob matches are read in sorted order.
chunksize (Optional[int]): Items per yielded chunk. None -> one chunk
per file.
verbose (int): 0 silent, 1 prints each file name.
Expand All @@ -43,28 +50,36 @@ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose:
id_col (Optional[int]): 0-based column holding the id, or None to make
one up. Passing it turns on (id, smiles) output.
with_ids (bool): Yield (id, smiles) tuples instead of bare SMILES.
sep (Optional[str]): Column separator for text inputs, e.g. "," for
CSV. None splits on any whitespace.

Yields:
List: a chunk of SMILES strings, or of (id, smiles) tuples when ids
are requested.
"""
# Sanity check
if not os.path.exists(input_path):
# Sanity check. A path that exists is taken literally, even if it
# contains glob characters like "batch[1]".
if os.path.exists(input_path):
input_paths = input_path
elif glob.has_magic(input_path):
input_paths = sorted(glob.glob(input_path))
if not input_paths:
raise FileNotFoundError(f"No files match {input_path}.")
else:
raise FileNotFoundError(f"Data source {input_path} not found.")

want_ids = with_ids or id_col is not None
buffer: List = []
next_id = 0 # running counter for made-up ids (all files, in order)

for file_path in _process_input(input_paths=input_path):
for file_path in _process_input(input_paths=input_paths):
if verbose == 1:
print("Processing file:", file_path)

file_ext = os.path.splitext(file_path)[1].lower()
if file_ext in {".sdf", ".sd"}:
if _file_ext(file_path) in {".sdf", ".sd"}:
records = self._stream_sdf(file_path)
else:
records = self._stream_text(file_path, smiles_col, id_col)
records = self._stream_text(file_path, smiles_col, id_col, sep)

for raw_id, smiles in records:
if want_ids:
Expand All @@ -83,17 +98,21 @@ def parse_input(self, input_path: str, chunksize: Optional[int] = None, verbose:
buffer.clear()

def _stream_text(self, file_path: str, smiles_col: int,
id_col: Optional[int] = None) -> Iterator[tuple]:
id_col: Optional[int] = None, sep: Optional[str] = None) -> Iterator[tuple]:
"""Yield (raw_id, smiles) per line. raw_id is the value in column
`id_col`, or None when there is no id column. Blank / too-short lines
(no column at `smiles_col`) are skipped."""
with open(file_path, "r") as file:
for line in file:
parts = line.split()
if smiles_col >= len(parts):
with _open(file_path) as file:
if sep is None:
rows = (line.split() for line in file)
else:
# csv handles quoted fields, e.g. "C,C" or "CCO"
rows = csv.reader(file, delimiter=sep, skipinitialspace=True)
for parts in rows:
if smiles_col >= len(parts) or not parts[smiles_col].strip():
continue
smiles = parts[smiles_col]
raw_id = parts[id_col] if (id_col is not None and id_col < len(parts)) else None
smiles = parts[smiles_col].strip()
raw_id = parts[id_col].strip() if (id_col is not None and id_col < len(parts)) else None
yield raw_id, smiles

def _stream_sdf(self, file_path: str) -> Iterator[tuple]:
Expand All @@ -105,7 +124,7 @@ def _stream_sdf(self, file_path: str) -> Iterator[tuple]:
) from exc

# ForwardSDMolSupplier streams molecules without loading the full file in memory
with open(file_path, "rb") as file_handle:
with _open(file_path, binary=True) as file_handle:
supplier = Chem.ForwardSDMolSupplier(file_handle)
for mol in supplier:
if mol is None:
Expand All @@ -118,3 +137,22 @@ def _stream_sdf(self, file_path: str) -> Iterator[tuple]:
# Use the SD record title as the id when present.
name = mol.GetProp("_Name").strip() if mol.HasProp("_Name") else ""
yield (name or None), smiles


_COMPRESSED = {".gz": gzip.open, ".bz2": bz2.open}


def _file_ext(file_path: str) -> str:
"""Lowercase extension, ignoring a compression suffix (x.sdf.gz -> .sdf)."""
root, ext = os.path.splitext(file_path.lower())
if ext in _COMPRESSED:
ext = os.path.splitext(root)[1]
return ext


def _open(file_path: str, binary: bool = False):
"""Open a plain, .gz or .bz2 file as UTF-8 text (or bytes)."""
opener = _COMPRESSED.get(os.path.splitext(file_path)[1].lower(), open)
if binary:
return opener(file_path, "rb")
return opener(file_path, "rt", encoding="utf-8", newline="")
Loading
Loading