feat: add core functionalities

.gitignore

@@ -30,6 +30,7 @@ env/
 # Testing
 .pytest_cache/
 .coverage
+coverage.xml
 htmlcov/
 .tox/
 
@@ -53,6 +54,7 @@ dist/
 
 # Notebooks
 .ipynb_checkpoints/
+*.ipynb
 
 # OS
 .DS_Store

.pre-commit-config.yaml

@@ -43,13 +43,13 @@ repos:
         language: system
         types: [python]
         pass_filenames: false
-        entry: uv run ruff check src/ configs/ scripts/ --fix --exclude src/leap/_version.py
+        entry: uv run ruff check src/leap/ configs/ scripts/ --fix --exclude src/leap/_version.py
       - id: mypy
         name: Static type checking using mypy
         language: system
         types: [python]
         pass_filenames: false
-        entry: uv run mypy src/ configs/ --exclude src/leap/_version.py
+        entry: uv run mypy src/leap/ configs/ --exclude src/leap/_version.py
       - id: pydoclint
         name: Docstring linting with pydoclint
         language: system

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install checks testing clean
+.PHONY: help install checks tests clean
 
 UV_VERSION := 0.8.23
 
@@ -38,7 +38,7 @@ checks: ## Run pre-commit checks on all files
 	@echo "🔍 Running checks..."
 	@PIP_INDEX_URL=https://pypi.org/simple PIP_EXTRA_INDEX_URL="" uv run pre-commit run --all-files
 
-testing: ## Run tests with coverage
+tests: ## Run tests with coverage
 	@echo "🧪 Running tests..."
 	@uv run pytest src/tests/ -vv
 

README.md

@@ -24,7 +24,7 @@ Run quality checks before committing:
 
 ```bash
 make checks   # Run pre-commit hooks (linting, formatting, type checking)
-make testing  # Run tests with coverage
+make tests  # Run tests with coverage
 ```
 
 ## Usage

data/README.md

@@ -2,4 +2,13 @@
 
 Directory for storing datasets.
 
+To run experiments, please download the following files from [DepMap](https://depmap.org/portal/data_page/?tab=allData):
+- CRISPRGeneDependency.csv
+- OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv
+- Model.csv
+
+And [this file](https://www.gsea-msigdb.org/gsea/msigdb/download_file.jsp?filePath=/msigdb/release/2025.1.Hs/c2.all.v2025.1.Hs.json) from MsigDB. This file is the JSON bundle associated with the GCP (chemical and genetic perturbations) gene set.
+
+Save all those file in this directory.
+
 **Note:** Data files are gitignored. Only this README is tracked.

pyproject.toml

@@ -3,10 +3,10 @@ build-backend = "hatchling.build"
 requires = ["hatchling", "hatch-vcs"]
 
 [tool.hatch.build.targets.sdist]
-include = ["README.md", "src/leap"]
+include = ["README.md", "src/leap", "configs"]
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/leap"]
+packages = ["src/leap", "configs"]
 
 [tool.hatch.version]
 source = "vcs"
@@ -57,7 +57,15 @@ dependencies = [
     "numpy>=2.0.0",
     "pandas>=2.0.0",
     "scikit-learn>=1.3.0",
+    "scipy>=1.16.0",
     "torch>=2.0.0",
+    "loguru>=0.7.0",
+    "skglm<0.4", # Breaking changes introduced in the 0.4 version in Apr 25
+    "ray>=2.20.0",
+    "tqdm>=4.0.0",
+    "lightgbm>=4.1.0",
+    "pyarrow>=19.0.0",
+    "ml-collections>=1.1.0",
 ]
 
 [project.urls]
@@ -172,7 +180,7 @@ convention = "numpy"
 python_version = "3.11"
 ignore_errors = false
 files = ["src/", "configs/", "scripts/"]
-mypy_path = ["src", "configs", "scripts"]
+mypy_path = ["."]
 
 # Enforce typing on public functions
 disallow_incomplete_defs = true
@@ -188,15 +196,28 @@ strict_equality = true
 
 [tool.coverage.run]
 branch = true
-source_pkgs = ["leap"]
+source_pkgs = ["leap", "configs"]
 
 [tool.coverage.report]
 omit = []
 exclude_lines = [
     "pragma: no cover",
     "if TYPE_CHECKING:",
+    "import ",
+    "from .* import ",
+    "@abstractmethod",
+    "@abc.abstractmethod",
+    "if __name__ == .__main__.:",
+    "raise NotImplementedError",
+    "raise AssertionError",
+    "def __repr__",
+    "def __str__",
+    "__all__",
 ]
 
+[tool.pydoclint]
+should-document-star-arguments=false
+
 [tool.pytest.ini_options]
 testpaths = ["src/tests"]
 python_files = ["test_*.py", "*_test.py"]
@@ -209,6 +230,7 @@ addopts = [
     "--cov-report=term-missing",
     "--cov-report=html",
     "--cov-report=xml",
+    "-p", "no:threadexception",  # Prevent segfaults with PyTorch tests
 ]
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",

src/leap/data/__init__.py

@@ -0,0 +1 @@
+"""Data module for LEAP."""

src/leap/data/load_depmap.py

@@ -0,0 +1,61 @@
+"""Loading functions for depmap data."""
+
+import re
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from loguru import logger
+
+
+DATA_PATH = Path(__file__).parent.parent.parent.parent / "data"
+
+
+def load_expression() -> pd.DataFrame:
+    """Load DepMap RNASeq data with tpm normalization.
+
+    It is important to note that RNAseq tpm data from DepMap is already log-scaled with log2(X+1), therefore here we
+    convert the tpm data with exp2(x) - 1 so that we can access the TPM values.
+    """
+    path_processed = DATA_PATH / "processed" / "tpm_rnaseq_processed.parquet"
+    if path_processed.exists():
+        depmap_expr = pd.read_parquet(path_processed)
+    else:
+        logger.info("Preprocessing DepMap RNASeq data with tpm normalization...")
+        path = DATA_PATH / "OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv"
+        depmap_expr = (
+            pd.read_csv(path, index_col=0)
+            .drop(columns=["SequencingID", "IsDefaultEntryForModel", "ModelConditionID", "IsDefaultEntryForMC"])
+            .rename(columns={"ModelID": "DepMap_ID"})
+            .set_index("DepMap_ID")
+            .apply(lambda x: np.exp2(x) - 1)
+            .astype("float32")
+        )
+        # Clean column names to have gene symbol only
+        depmap_expr.rename(columns=lambda x: re.sub(r"[\(\[ ].*?[\)\]]", "", x), inplace=True)
+        depmap_expr.to_parquet(path_processed, engine="pyarrow", compression="brotli")
+    return depmap_expr
+
+
+def load_essentiality() -> pd.DataFrame:
+    """Load DeepDEP essentiality scores."""
+    labels_path_processed = DATA_PATH / "processed" / "dependencies_processed.parquet"
+    labels_path_raw = DATA_PATH / "CRISPRGeneDependency.csv"
+
+    if labels_path_processed.exists():
+        gene_dependencies = pd.read_parquet(labels_path_processed)
+    else:
+        logger.info("Preprocessing DepMap CRISPR data...")
+        gene_dependencies = pd.read_csv(labels_path_raw, index_col=0).rename(
+            columns=lambda x: re.sub(r"[\(\[ ].*?[\)\]]", "", x)
+        )
+        gene_dependencies.index.names = ["DepMap_ID"]
+        gene_dependencies.to_parquet(labels_path_processed, engine="pyarrow", compression="brotli")
+    return gene_dependencies
+
+
+def load_metadata() -> pd.DataFrame:
+    """Load metadata for all of DepMap's cancer cell lines."""
+    df_sample_info = pd.read_csv(DATA_PATH / "Model.csv", index_col=0)
+    df_sample_info.index.names = ["DepMap_ID"]
+    return df_sample_info

src/leap/data/load_gene_sets.py

@@ -0,0 +1,73 @@
+"""Loading functions for MSigDB gene set data."""
+
+import json
+from pathlib import Path
+
+import pandas as pd
+from loguru import logger
+
+
+DATA_PATH = Path(__file__).parent.parent.parent.parent / "data"
+
+
+def load_fingerprints() -> pd.DataFrame:
+    """Load MSigDB gene set membership matrix as fingerprints.
+
+    Returns a binary DataFrame where rows are genes, columns are gene set names,
+    and values are 1 if the gene belongs to the gene set, 0 otherwise.
+
+    The data is cached as a processed parquet file for faster loading.
+
+    Returns
+    -------
+    pd.DataFrame
+        Binary membership matrix with shape (n_genes, n_gene_sets).
+        Index: gene symbols
+        Columns: gene set names
+        Values: 1 (gene in set) or 0 (gene not in set)
+    """
+    path_processed = DATA_PATH / "processed" / "gene_set_fingerprints_processed.parquet"
+
+    if path_processed.exists():
+        fingerprints = pd.read_parquet(path_processed)
+    else:
+        logger.info("Processing MSigDB gene sets to create fingerprint matrix...")
+        path = DATA_PATH / "c2.cgp.v2025.1.Hs.json"
+
+        # Load raw JSON data
+        with open(path) as f:
+            raw_data = json.load(f)
+
+        logger.info(f"Loaded {len(raw_data)} gene sets")
+
+        # Collect all unique genes
+        all_genes: set[str] = set()
+        for gene_set_data in raw_data.values():
+            all_genes.update(gene_set_data["geneSymbols"])
+
+        all_genes_list = sorted(all_genes)  # Sort for consistent ordering
+        gene_set_names = list(raw_data.keys())
+
+        logger.info(f"Creating membership matrix with {len(all_genes_list)} genes and {len(gene_set_names)} gene sets")
+
+        # Create binary matrix efficiently
+        data = {gene_set_name: [0] * len(all_genes_list) for gene_set_name in gene_set_names}
+        gene_to_idx = {gene: idx for idx, gene in enumerate(all_genes_list)}
+
+        for gene_set_name, gene_set_data in raw_data.items():
+            for gene in gene_set_data["geneSymbols"]:
+                idx = gene_to_idx[gene]
+                data[gene_set_name][idx] = 1
+
+        # Create DataFrame
+        fingerprints = pd.DataFrame(data, index=all_genes_list)
+        fingerprints.index.name = "gene"
+
+        # Save to parquet for faster loading next time
+        path_processed.parent.mkdir(parents=True, exist_ok=True)
+        fingerprints.to_parquet(path_processed, engine="pyarrow", compression="brotli")
+
+        logger.info(f"Created and cached fingerprint matrix with shape {fingerprints.shape}")
+        logger.info(f"Matrix density: {fingerprints.sum().sum() / (fingerprints.shape[0] * fingerprints.shape[1]):.2%}")
+
+    return fingerprints