sentence-transformers as an optional dependency (lose: sem_join, sem_topk)

README.md

@@ -15,19 +15,19 @@ Our [full documentation](https://palimpzest.org) is the definitive resource for
 ## 🚀 Getting started
 You can find a stable version of the PZ package on PyPI [here](https://pypi.org/project/palimpzest/). To install the package, run:
 ```bash
-$ pip install palimpzest
+$ pip install palimpzest --extra join-topk
 ```
 
 You can also install PZ with [uv](https://docs.astral.sh/uv/) for a faster installation:
 ```bash
-$ uv pip install palimpzest
+$ uv pip install palimpzest --extra join-topk
 ```
 
 Alternatively, to install the latest version of the package from this repository, you can clone this repository and run the following commands:
 ```bash
 $ git clone git@github.com:mitdbg/palimpzest.git
 $ cd palimpzest
-$ pip install .
+$ pip install . --extra join-topk
 ```
 
 ## 🙋🏽 Join the PZ Community

pyproject.toml

@@ -33,7 +33,6 @@ dependencies = [
     "pyyaml>=6.0.1",
     "requests>=2.25",
     "ruff>=0.9.0",
-    "sentence-transformers==5.0.0",
     "setuptools>=70.1.1",
     "smolagents[toolkit]",
     "tabulate>=0.9.0",
@@ -54,6 +53,9 @@ classifiers=[
 vllm = [
     "vllm>=0.10.1.1",
 ]
+join-topk = [
+    "sentence-transformers>=5.0.0",
+]
 
 [tool.setuptools]
 package-dir = {"" = "src"}

src/palimpzest/query/operators/join.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import threading
 import time
 from abc import ABC, abstractmethod
@@ -10,7 +11,11 @@
 from numpy.linalg import norm
 from PIL import Image
 from pydantic.fields import FieldInfo
-from sentence_transformers import SentenceTransformer
+
+try:
+    from sentence_transformers import SentenceTransformer
+except ModuleNotFoundError:
+    logging.getLogger().info("sentence-transformers not found; sem_join, sem_topk unavailable!")
 
 from palimpzest.constants import (
     NAIVE_EST_JOIN_SELECTIVITY,

src/palimpzest/query/operators/topk.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import os
 import threading
 import time
@@ -10,7 +11,11 @@
 from chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction
 from openai import OpenAI
 from pydantic import BaseModel
-from sentence_transformers import SentenceTransformer
+
+try:
+    from sentence_transformers import SentenceTransformer
+except ModuleNotFoundError:
+    logging.getLogger().info("sentence-transformers not found; sem_join, sem_topk unavailable!")
 
 from palimpzest.constants import Model
 from palimpzest.core.elements.records import DataRecord, DataRecordSet