From de1ed599b7ea13f187394dda39fd6b9666c82a37 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Thu, 7 May 2026 23:42:29 +1000 Subject: [PATCH 1/2] docs: add Databricks dialect design spec Co-Authored-By: Claude Opus 4.6 (1M context) --- .../2026-05-07-databricks-dialect-design.md | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 docs/superpowers/specs/2026-05-07-databricks-dialect-design.md diff --git a/docs/superpowers/specs/2026-05-07-databricks-dialect-design.md b/docs/superpowers/specs/2026-05-07-databricks-dialect-design.md new file mode 100644 index 0000000..7bc057e --- /dev/null +++ b/docs/superpowers/specs/2026-05-07-databricks-dialect-design.md @@ -0,0 +1,102 @@ +# Databricks Dialect Design + +**Date:** 2026-05-07 +**Issue:** mountainash-io/mountainash#108 (follow-on from ClickHouse dialect) +**Scope:** Add Databricks as the 14th supported dialect in `IbisBackend` + +## Context + +Databricks is a cloud-based SQL analytics platform. Its ibis backend +(`ibis.databricks.connect()`) uses kwargs-only connection with no connection +string or port — instead it takes `server_hostname` + `http_path` to identify +a SQL warehouse or cluster, plus auth credentials. + +This follows the same `DialectSpec` + `BackendDescriptor` pattern used by +ClickHouse (PR #81) and the 12 original dialects. + +## Connection Model + +Databricks connects via `ibis.databricks.connect(**kwargs)`: + +- `server_hostname` — workspace URL (e.g. `adb-123.12.azuredatabricks.net`) +- `http_path` — SQL warehouse path (e.g. `/sql/1.0/warehouses/abc123`) +- `access_token` — PAT token +- `catalog` + `schema` — three-level namespace (catalog.schema.table) +- `use_cloud_fetch` — performance optimisation for large result sets + +No `host`/`port`/`database` — this is the key difference from standard dialects. + +## Auth Modes + +| Mode | Mapping | Use case | +|------|---------|----------| +| `TokenAuth` | `token` → `access_token` | Primary — PAT tokens | +| `PasswordAuth` | `username` + `password` | Rare — basic auth | +| `NoAuth` | nothing | Env-var-driven (`DATABRICKS_TOKEN`) | + +Custom `credentials_provider` (OAuth M2M) is a callable, not a credential +pair — better handled as a passthrough kwarg than a first-class auth mode. + +## Changes + +### 1. Constants (`core/constants.py`) + +Add `DATABRICKS = auto()` to `CONST_DB_PROVIDER_TYPE`. The `CONST_DB_BACKEND` +and `CONST_DB_BACKEND_IBIS_PREFIX` entries already exist. + +### 2. Settings class (`core/settings/databricks.py`) + +```python +DATABRICKS_DESCRIPTOR = BackendDescriptor( + name="databricks", + provider_type=CONST_DB_PROVIDER_TYPE.DATABRICKS, + ibis_dialect="databricks", + auth_modes=[TokenAuth, PasswordAuth, NoAuth], + parameters=[ + ParameterSpec(name="SERVER_HOSTNAME", type=str, tier="core", + driver_key="server_hostname"), + ParameterSpec(name="HTTP_PATH", type=str, tier="core", + driver_key="http_path"), + ParameterSpec(name="CATALOG", type=Optional[str], tier="core", + default=None, driver_key="catalog"), + ParameterSpec(name="SCHEMA", type=str, tier="core", + default="default", driver_key="schema"), + ParameterSpec(name="USE_CLOUD_FETCH", type=bool, tier="advanced", + default=False, driver_key="use_cloud_fetch"), + ], +) +``` + +No `default_port` or `connection_string_scheme` — Databricks uses neither. + +### 3. Adapter (`core/settings/adapters/databricks.py`) + +Custom adapter following Snowflake's pattern. Handles auth dispatch: + +- `TokenAuth` → `access_token = token.get_secret_value()` +- `PasswordAuth` → `username`, `password` +- `NoAuth` → no auth kwargs (driver falls back to env vars) + +### 4. Dialect registration (`backends/ibis/dialects/_registry.py`) + +- `_build_databricks_connection(**config)` — extracts known params, calls + `ibis.databricks.connect(**kwargs)` +- `DialectSpec` entry: `connection_mode=KWARGS`, `connection_string_scheme=""` + +### 5. Optional dependency (`pyproject.toml`) + +```toml +databricks = ["databricks-sql-connector>=4", "ibis-framework[databricks]>=11.0.0"] +``` + +### 6. Exports (`core/settings/__init__.py`) + +Import and export `DatabricksAuthSettings`. + +### 7. Tests + +- `test_unit/core/settings/backends/test_databricks.py` — provider type, + defaults, token auth kwargs plumbing, password auth kwargs, no-auth, schema + default +- `test_dialect_spec.py` — registry count 13 → 14, add `"databricks"` to + expected set From 4384bece07b03f4bf2a85778bcf5e63207dc4a63 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Fri, 8 May 2026 00:01:21 +1000 Subject: [PATCH 2/2] feat(databricks): add Databricks dialect to IbisBackend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registers Databricks as the 14th supported dialect. Uses a custom adapter for auth dispatch (TokenAuth → access_token, PasswordAuth → username/password) following the Snowflake adapter pattern. Co-Authored-By: Claude Opus 4.6 (1M context) --- pyproject.toml | 1 + .../backends/ibis/dialects/_registry.py | 49 +++++++++++++ src/mountainash_data/core/constants.py | 1 + .../core/settings/__init__.py | 3 +- .../core/settings/adapters/databricks.py | 22 ++++++ .../core/settings/databricks.py | 43 +++++++++++ .../backends/ibis/test_dialect_spec.py | 4 +- .../core/settings/backends/test_databricks.py | 73 +++++++++++++++++++ 8 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 src/mountainash_data/core/settings/adapters/databricks.py create mode 100644 src/mountainash_data/core/settings/databricks.py create mode 100644 tests/test_unit/core/settings/backends/test_databricks.py diff --git a/pyproject.toml b/pyproject.toml index 161120d..4e98505 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ postgres = ["psycopg2-binary==2.9.9", "ibis-framework[postgres]>=11.0.0"] bigquery = ["ibis-framework[bigquery]>=11.0.0"] pyspark = ["setuptools", "ibis-framework[pyspark]>=11.0.0"] clickhouse = ["ibis-framework[clickhouse]>=11.0.0"] +databricks = ["databricks-sql-connector>=4", "ibis-framework[databricks]>=11.0.0"] trino = ["ibis-framework[trino]>=11.0.0"] diff --git a/src/mountainash_data/backends/ibis/dialects/_registry.py b/src/mountainash_data/backends/ibis/dialects/_registry.py index cdf9efe..229297f 100644 --- a/src/mountainash_data/backends/ibis/dialects/_registry.py +++ b/src/mountainash_data/backends/ibis/dialects/_registry.py @@ -387,6 +387,49 @@ def _build_clickhouse_connection(**config: t.Any) -> t.Any: ) +def _build_databricks_connection(**config: t.Any) -> t.Any: + """Build a Databricks ibis connection. + + Uses ibis.databricks.connect() with kwargs: server_hostname, http_path, + access_token, catalog, schema, use_cloud_fetch. + """ + import ibis + + server_hostname = config.get("server_hostname", None) + http_path = config.get("http_path", None) + access_token = config.get("access_token", None) + catalog = config.get("catalog", None) + schema = config.get("schema", "default") + use_cloud_fetch = config.get("use_cloud_fetch", False) + + known = {"server_hostname", "http_path", "access_token", "catalog", + "schema", "use_cloud_fetch", "username", "password", + "connection_string"} + extra = {k: v for k, v in config.items() if k not in known} + + kwargs: dict[str, t.Any] = {} + if server_hostname is not None: + kwargs["server_hostname"] = server_hostname + if http_path is not None: + kwargs["http_path"] = http_path + if access_token is not None: + kwargs["access_token"] = access_token + if catalog is not None: + kwargs["catalog"] = catalog + kwargs["schema"] = schema + kwargs["use_cloud_fetch"] = use_cloud_fetch + + username = config.get("username", None) + password = config.get("password", None) + if username is not None: + kwargs["username"] = username + if password is not None: + kwargs["password"] = password + + kwargs.update(extra) + return ibis.databricks.connect(**kwargs) + + def _build_pyspark_connection(**config: t.Any) -> t.Any: """Build a PySpark ibis connection. @@ -518,6 +561,12 @@ def _build_pyspark_connection(**config: t.Any) -> t.Any: connection_string_scheme="clickhouse://", connection_builder=_build_clickhouse_connection, ), + "databricks": DialectSpec( + ibis_backend_name="databricks", + connection_mode=_KWARGS, + connection_string_scheme="", + connection_builder=_build_databricks_connection, + ), "pyspark": DialectSpec( ibis_backend_name="pyspark", connection_mode=_CONNECTION_STRING, diff --git a/src/mountainash_data/core/constants.py b/src/mountainash_data/core/constants.py index 5b7aefd..0134209 100644 --- a/src/mountainash_data/core/constants.py +++ b/src/mountainash_data/core/constants.py @@ -24,6 +24,7 @@ class CONST_DB_PROVIDER_TYPE(Enum): PYICEBERG_REST = auto() ORACLE = auto() CLICKHOUSE = auto() + DATABRICKS = auto() PYSPARK = auto() diff --git a/src/mountainash_data/core/settings/__init__.py b/src/mountainash_data/core/settings/__init__.py index 59fa64a..c4b7bcf 100644 --- a/src/mountainash_data/core/settings/__init__.py +++ b/src/mountainash_data/core/settings/__init__.py @@ -39,6 +39,7 @@ class body is a two-line shell (``__descriptor__`` + ``__adapter__``). from .motherduck import MotherDuckAuthSettings from .postgresql import PostgreSQLAuthSettings from .clickhouse import ClickHouseAuthSettings +from .databricks import DatabricksAuthSettings from .mysql import MySQLAuthSettings from .mssql import MSSQLAuthSettings from .snowflake import SnowflakeAuthSettings @@ -60,7 +61,7 @@ class body is a two-line shell (``__descriptor__`` + ``__adapter__``). # backends "SQLiteAuthSettings", "DuckDBAuthSettings", "MotherDuckAuthSettings", "PostgreSQLAuthSettings", "ClickHouseAuthSettings", - "MySQLAuthSettings", "MSSQLAuthSettings", + "DatabricksAuthSettings", "MySQLAuthSettings", "MSSQLAuthSettings", "SnowflakeAuthSettings", "BigQueryAuthSettings", "RedshiftAuthSettings", "PySparkAuthSettings", "TrinoAuthSettings", "PyIcebergRestAuthSettings", ] diff --git a/src/mountainash_data/core/settings/adapters/databricks.py b/src/mountainash_data/core/settings/adapters/databricks.py new file mode 100644 index 0000000..f23986c --- /dev/null +++ b/src/mountainash_data/core/settings/adapters/databricks.py @@ -0,0 +1,22 @@ +"""Databricks adapter: maps TokenAuth → access_token, PasswordAuth → user/pass.""" + +from __future__ import annotations + +import typing as t + +from mountainash_settings.auth import PasswordAuth, TokenAuth + +if t.TYPE_CHECKING: + from mountainash_data.core.settings.databricks import DatabricksAuthSettings + + +def build_driver_kwargs(profile: "DatabricksAuthSettings") -> dict[str, t.Any]: + kwargs = profile._default_kwargs() + + auth = profile.auth + if isinstance(auth, TokenAuth): + kwargs["access_token"] = auth.token.get_secret_value() + elif isinstance(auth, PasswordAuth): + kwargs["username"] = auth.username + kwargs["password"] = auth.password.get_secret_value() + return kwargs diff --git a/src/mountainash_data/core/settings/databricks.py b/src/mountainash_data/core/settings/databricks.py new file mode 100644 index 0000000..3a6579b --- /dev/null +++ b/src/mountainash_data/core/settings/databricks.py @@ -0,0 +1,43 @@ +"""Databricks backend settings. + +Driver: https://docs.databricks.com/en/dev-tools/python-sql-connector.html +Ibis: ``ibis.databricks.connect(server_hostname, http_path, access_token, + catalog, schema, ...)`` +""" + +from __future__ import annotations + +import typing as t + +from ..constants import CONST_DB_PROVIDER_TYPE +from .adapters import databricks as _adapter +from mountainash_settings.auth import NoAuth, PasswordAuth, TokenAuth +from .descriptor import BackendDescriptor, ParameterSpec +from .profile import ConnectionProfile +from .registry import register + + +DATABRICKS_DESCRIPTOR = BackendDescriptor( + name="databricks", + provider_type=CONST_DB_PROVIDER_TYPE.DATABRICKS, + ibis_dialect="databricks", + auth_modes=[TokenAuth, PasswordAuth, NoAuth], + parameters=[ + ParameterSpec(name="SERVER_HOSTNAME", type=str, tier="core", + driver_key="server_hostname"), + ParameterSpec(name="HTTP_PATH", type=str, tier="core", + driver_key="http_path"), + ParameterSpec(name="CATALOG", type=t.Optional[str], tier="core", + default=None, driver_key="catalog"), + ParameterSpec(name="SCHEMA", type=str, tier="core", + default="default", driver_key="schema"), + ParameterSpec(name="USE_CLOUD_FETCH", type=bool, tier="advanced", + default=False, driver_key="use_cloud_fetch"), + ], +) + + +@register(DATABRICKS_DESCRIPTOR) +class DatabricksAuthSettings(ConnectionProfile): + __descriptor__ = DATABRICKS_DESCRIPTOR + __adapter__ = staticmethod(_adapter.build_driver_kwargs) diff --git a/tests/test_unit/backends/ibis/test_dialect_spec.py b/tests/test_unit/backends/ibis/test_dialect_spec.py index 573d821..7040c60 100644 --- a/tests/test_unit/backends/ibis/test_dialect_spec.py +++ b/tests/test_unit/backends/ibis/test_dialect_spec.py @@ -33,11 +33,11 @@ def fake_index_sql(table_name, index_name): assert spec.get_index_exists_sql("users", "idx_users_id") == "SELECT 1 FROM users" -def test_registry_contains_all_13_backends(): +def test_registry_contains_all_14_backends(): expected = { "sqlite", "duckdb", "motherduck", "postgres", "mysql", "mssql", "oracle", "snowflake", "bigquery", "redshift", "trino", "pyspark", - "clickhouse", + "clickhouse", "databricks", } assert set(DIALECTS.keys()) == expected diff --git a/tests/test_unit/core/settings/backends/test_databricks.py b/tests/test_unit/core/settings/backends/test_databricks.py new file mode 100644 index 0000000..58af3ad --- /dev/null +++ b/tests/test_unit/core/settings/backends/test_databricks.py @@ -0,0 +1,73 @@ +# tests/test_unit/core/settings/backends/test_databricks.py +from __future__ import annotations + +import pytest +from pydantic import SecretStr + +from mountainash_data.core.constants import CONST_DB_PROVIDER_TYPE +from mountainash_data.core.settings.auth import NoAuth, PasswordAuth, TokenAuth +from mountainash_data.core.settings.databricks import DatabricksAuthSettings + + +@pytest.mark.unit +class TestDatabricksAuthSettings: + def _token(self, **extra): + return DatabricksAuthSettings( + SERVER_HOSTNAME="adb-123.12.azuredatabricks.net", + HTTP_PATH="/sql/1.0/warehouses/abc123", + auth=TokenAuth(token=SecretStr("dapi-xyz")), + **extra, + ) + + def test_provider_type_is_databricks(self): + s = self._token() + assert s.provider_type == CONST_DB_PROVIDER_TYPE.DATABRICKS + + def test_schema_default(self): + s = self._token() + assert s.SCHEMA == "default" + + def test_use_cloud_fetch_default_false(self): + s = self._token() + assert s.USE_CLOUD_FETCH is False + + def test_token_auth_kwargs(self): + s = self._token(CATALOG="analytics", SCHEMA="gold") + kwargs = s.to_driver_kwargs() + assert kwargs["server_hostname"] == "adb-123.12.azuredatabricks.net" + assert kwargs["http_path"] == "/sql/1.0/warehouses/abc123" + assert kwargs["access_token"] == "dapi-xyz" + assert kwargs["catalog"] == "analytics" + assert kwargs["schema"] == "gold" + assert "username" not in kwargs + assert "password" not in kwargs + + def test_password_auth_kwargs(self): + s = DatabricksAuthSettings( + SERVER_HOSTNAME="adb-123.12.azuredatabricks.net", + HTTP_PATH="/sql/1.0/warehouses/abc123", + auth=PasswordAuth(username="user", password=SecretStr("pass")), + ) + kwargs = s.to_driver_kwargs() + assert kwargs["username"] == "user" + assert kwargs["password"] == "pass" + assert "access_token" not in kwargs + + def test_no_auth(self): + s = DatabricksAuthSettings( + SERVER_HOSTNAME="adb-123.12.azuredatabricks.net", + HTTP_PATH="/sql/1.0/warehouses/abc123", + auth=NoAuth(), + ) + kwargs = s.to_driver_kwargs() + assert "access_token" not in kwargs + assert "username" not in kwargs + + def test_ibis_dialect(self): + s = self._token() + assert s.backend == "databricks" + + def test_use_cloud_fetch_plumbed(self): + s = self._token(USE_CLOUD_FETCH=True) + kwargs = s.to_driver_kwargs() + assert kwargs["use_cloud_fetch"] is True