From 7b56f7a28d67feeff84a02e0faad71e1f0bbf086 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 13:47:21 +1000 Subject: [PATCH 01/12] feat(backend): new IbisBackend constructor with dispatch and error validation Add _SCHEME_TO_DIALECT map, three-way dispatch (settings, URL, dialect), empty-list normalization in connect(), and URL-direct via ibis.connect(). Error cases tested: no args, both args, unknown scheme. Co-Authored-By: Claude Sonnet 4.6 --- src/mountainash_data/backends/ibis/backend.py | 159 ++++++++++++++++-- tests/test_unit/backends/ibis/test_backend.py | 18 ++ 2 files changed, 161 insertions(+), 16 deletions(-) diff --git a/src/mountainash_data/backends/ibis/backend.py b/src/mountainash_data/backends/ibis/backend.py index 8c6ddd9..e1bf456 100644 --- a/src/mountainash_data/backends/ibis/backend.py +++ b/src/mountainash_data/backends/ibis/backend.py @@ -103,39 +103,166 @@ def __exit__(self, *args): self.close() +# --------------------------------------------------------------------------- +# Scheme → dialect reverse lookup (built once from the DIALECTS registry) +# --------------------------------------------------------------------------- +def _build_scheme_to_dialect() -> dict[str, str]: + """Build a map from URL scheme (e.g. 'sqlite', 'postgres') to dialect name.""" + result: dict[str, str] = {} + for dialect_name, spec in DIALECTS.items(): + # connection_string_scheme is e.g. "postgres://", "duckdb://md:" + scheme = spec.connection_string_scheme.split("://")[0].lower() + # First dialect wins — e.g. "postgres" maps to "postgres", not "redshift" + if scheme not in result: + result[scheme] = dialect_name + # Common aliases + result.setdefault("postgresql", result.get("postgres", "postgres")) + return result + + +_SCHEME_TO_DIALECT: dict[str, str] = _build_scheme_to_dialect() + + class IbisBackend: - """Ibis backend factory. + """Ibis backend — single entry point for all Ibis connections. - Construction takes a dialect name (e.g. 'postgres') and config. - connect() returns a live connection that satisfies - core.protocol.Connection. + Three input forms, all producing IbisConnection via connect(): - Usage: + # Settings object (deployment, env-driven config) + backend = IbisBackend(settings_params) + + # Connection URL (universal connection strings) + backend = IbisBackend("postgresql://user:pass@host:5432/db") + + # Dialect keyword + kwargs (tests, scripts) backend = IbisBackend(dialect="sqlite", database=":memory:") - conn = backend.connect() - try: - tables = conn.list_tables() - finally: - conn.close() """ name = "ibis" - def __init__(self, dialect: str, **config: t.Any): - if dialect not in DIALECTS: + def __init__( + self, + settings_or_connection_string: str | t.Any | None = None, + /, + *, + dialect: str | None = None, + **config: t.Any, + ): + if settings_or_connection_string is not None and dialect is not None: + raise ValueError( + "Cannot specify both a positional settings/URL argument " + "and dialect= keyword" + ) + + if settings_or_connection_string is not None: + self._init_from_positional(settings_or_connection_string, config) + elif dialect is not None: + self._init_from_dialect(dialect, config) + else: + raise ValueError( + "Either a SettingsParameters/URL positional argument " + "or a dialect= keyword is required" + ) + + def _init_from_positional( + self, value: str | t.Any, config: dict[str, t.Any] + ) -> None: + # Lazy import — only pay for it on the settings/URL paths + from mountainash_settings import SettingsParameters + + if isinstance(value, SettingsParameters): + self._init_from_settings(value, config) + elif isinstance(value, str): + if "://" in value: + self._init_from_url(value, config) + else: + # Plain string — treat as dialect name + self._init_from_dialect(value, config) + else: + raise TypeError( + f"Expected SettingsParameters or str, got {type(value).__name__}" + ) + + def _init_from_dialect( + self, dialect_name: str, config: dict[str, t.Any] + ) -> None: + if dialect_name not in DIALECTS: raise KeyError( - f"Unknown ibis dialect {dialect!r}. " + f"Unknown ibis dialect {dialect_name!r}. " f"Available: {sorted(DIALECTS)}" ) - self.dialect = dialect - self._spec: DialectSpec = DIALECTS[dialect] + self.dialect = dialect_name + self._spec: DialectSpec = DIALECTS[dialect_name] + self._url: str | None = None self._config = config + def _init_from_url( + self, url: str, config: dict[str, t.Any] + ) -> None: + from urllib.parse import urlparse + + scheme = urlparse(url).scheme.lower() + + # Special case: MotherDuck URLs are "duckdb://md:..." + if scheme == "duckdb" and url.startswith("duckdb://md:"): + resolved_dialect = "motherduck" + else: + resolved_dialect = _SCHEME_TO_DIALECT.get(scheme) + + if resolved_dialect is None: + raise ValueError( + f"Cannot detect ibis dialect from URL scheme: {scheme!r}" + ) + + self.dialect = resolved_dialect + self._spec = DIALECTS[resolved_dialect] + self._url = url + self._config = config + + def _init_from_settings( + self, settings_params: t.Any, config: dict[str, t.Any] + ) -> None: + obj_settings = settings_params.settings_class.get_settings( + settings_parameters=settings_params + ) + descriptor = getattr(obj_settings, "__descriptor__", None) + if descriptor is None or getattr(descriptor, "ibis_dialect", None) is None: + raise ValueError( + f"Settings class {type(obj_settings).__name__} has no " + f"ibis_dialect on its descriptor" + ) + resolved_dialect = descriptor.ibis_dialect + if resolved_dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {resolved_dialect!r} from descriptor. " + f"Available: {sorted(DIALECTS)}" + ) + driver_kwargs = obj_settings.to_driver_kwargs() + driver_kwargs.update(config) + + self.dialect = resolved_dialect + self._spec = DIALECTS[resolved_dialect] + self._url = None + self._config = driver_kwargs + def connect(self) -> IbisConnection: """Build and return a live ibis connection.""" if self._spec.connection_builder is None: raise NotImplementedError( f"Dialect {self.dialect!r} has no connection_builder configured" ) - ibis_conn = self._spec.connection_builder(**self._config) + if self._url is not None: + # URL path: delegate directly to ibis.connect() which + # natively handles all URL forms and preserves all URL + # components (host, port, credentials, database, query params). + import ibis + ibis_conn = ibis.connect(self._url, **self._config) + else: + # Settings/dialect path: go through the dialect builder + # with empty-list normalization (e.g. DuckDB extensions=[]). + cleaned_config = { + k: v for k, v in self._config.items() + if not (isinstance(v, (list, tuple)) and len(v) == 0) + } + ibis_conn = self._spec.connection_builder(**cleaned_config) return IbisConnection(ibis_conn, self._spec) diff --git a/tests/test_unit/backends/ibis/test_backend.py b/tests/test_unit/backends/ibis/test_backend.py index 9dee07d..7dfe819 100644 --- a/tests/test_unit/backends/ibis/test_backend.py +++ b/tests/test_unit/backends/ibis/test_backend.py @@ -33,3 +33,21 @@ def test_in_memory_sqlite_connect_and_inspect(): assert conn.list_tables() == [] finally: conn.close() + + +def test_neither_positional_nor_dialect_raises(): + """Constructor with no arguments must raise ValueError.""" + with pytest.raises(ValueError, match="Either.*or.*dialect"): + IbisBackend() + + +def test_both_positional_and_dialect_raises(): + """Cannot supply both a positional arg and dialect= keyword.""" + with pytest.raises(ValueError, match="Cannot specify both"): + IbisBackend("sqlite://", dialect="sqlite") + + +def test_unknown_url_scheme_raises(): + """URL with unrecognised scheme must raise ValueError.""" + with pytest.raises(ValueError, match="Cannot detect ibis dialect"): + IbisBackend("nosuch://localhost/db") From 00189729bd133947007911f7a38467edc36eeab2 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 13:49:07 +1000 Subject: [PATCH 02/12] feat(backend): make IbisBackend settings-aware with URL and SettingsParameters support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend IbisBackend constructor to accept three input forms: - SettingsParameters (positional) — resolves via descriptor.ibis_dialect + to_driver_kwargs() - Connection URL (positional) — detects dialect from scheme, delegates to ibis.connect() - dialect= keyword + **config — existing path, unchanged Adds _SCHEME_TO_DIALECT reverse lookup, empty-list normalization in connect(), and 8 new tests covering settings, URL, and error paths. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/test_unit/backends/ibis/test_backend.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/test_unit/backends/ibis/test_backend.py b/tests/test_unit/backends/ibis/test_backend.py index 7dfe819..f07db8b 100644 --- a/tests/test_unit/backends/ibis/test_backend.py +++ b/tests/test_unit/backends/ibis/test_backend.py @@ -51,3 +51,84 @@ def test_unknown_url_scheme_raises(): """URL with unrecognised scheme must raise ValueError.""" with pytest.raises(ValueError, match="Cannot detect ibis dialect"): IbisBackend("nosuch://localhost/db") + + +# --------------------------------------------------------------------------- +# Settings path +# --------------------------------------------------------------------------- + +def test_settings_path_sqlite(): + """Construct IbisBackend from SQLite SettingsParameters and connect.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import SQLiteAuthSettings, NoAuth + from mountainash_data.backends.ibis.backend import IbisConnection + + params = SettingsParameters.create( + settings_class=SQLiteAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "sqlite" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + tables = conn.list_tables() + assert isinstance(tables, list) + conn.close() + + +def test_settings_path_duckdb_empty_extensions(): + """DuckDB settings with default EXTENSIONS=[] must not crash ibis.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + from mountainash_data.backends.ibis.backend import IbisConnection + + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "duckdb" + conn = backend.connect() # Must not raise — empty-list filter active + assert isinstance(conn, IbisConnection) + conn.close() + + +# --------------------------------------------------------------------------- +# URL path +# --------------------------------------------------------------------------- + +def test_url_path_sqlite(): + """Construct IbisBackend from sqlite:// URL and connect.""" + from mountainash_data.backends.ibis.backend import IbisConnection + + backend = IbisBackend("sqlite://") + assert backend.dialect == "sqlite" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + + +def test_url_path_duckdb(): + """Construct IbisBackend from duckdb:// URL and connect.""" + from mountainash_data.backends.ibis.backend import IbisConnection + + backend = IbisBackend("duckdb://") + assert backend.dialect == "duckdb" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + + +def test_url_path_preserves_database(tmp_path): + """URL database component must reach the driver, not be discarded.""" + from mountainash_data.backends.ibis.backend import IbisConnection + + db_file = tmp_path / "test.db" + backend = IbisBackend(f"sqlite:///{db_file}") + assert backend.dialect == "sqlite" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + assert db_file.exists() From 7852a0e27317c9699ccc4e70e84d1ae19afbbeab Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 13:50:27 +1000 Subject: [PATCH 03/12] chore(specs): abandon old to-relation spec/plan, add settings-aware spec/plan Old spec bundled to_relation() which has been descoped per revised principle. New spec focuses solely on settings-aware IbisBackend constructor (Phase 1 of connection consolidation). Co-Authored-By: Claude Opus 4.6 (1M context) --- ...-27-settings-aware-backends-to-relation.md | 432 +++++++++++++++++ .../2026-04-27-settings-aware-ibis-backend.md | 452 ++++++++++++++++++ .../specs/2026-04-26-to-relation-design.md | 210 ++++++++ ...4-27-settings-aware-ibis-backend-design.md | 262 ++++++++++ 4 files changed, 1356 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md create mode 100644 docs/superpowers/plans/2026-04-27-settings-aware-ibis-backend.md create mode 100644 docs/superpowers/specs/2026-04-26-to-relation-design.md create mode 100644 docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md diff --git a/docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md b/docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md new file mode 100644 index 0000000..b22f9c8 --- /dev/null +++ b/docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md @@ -0,0 +1,432 @@ +# Settings-Aware Backends + to_relation() Implementation Plan + +> **Status:** ABANDONED -- superseded by `docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md`. to_relation() descoped; constructor design revised. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `IbisBackend` accept `SettingsParameters` as an alternative constructor, add `to_relation()` to the Connection protocol and both Ibis connection paths, and declare `mountainash` as an optional dependency. + +**Architecture:** `IbisBackend.__init__` detects whether it received a `str` (dialect) or `SettingsParameters` and resolves config accordingly. `to_relation()` is added to the `Connection` protocol and implemented on `IbisConnection` (new-style) and `BaseIbisConnection` (factory path), both using an import-guarded call to `mountainash.relations.relation()`. Iceberg gets a `NotImplementedError` stub. + +**Tech Stack:** Python, ibis-framework, mountainash-settings, mountainash (optional) + +**Spec:** `docs/superpowers/specs/2026-04-26-to-relation-design.md` + +--- + +## File Map + +| File | Action | What changes | +|------|--------|--------------| +| `src/mountainash_data/backends/ibis/backend.py` | Modify | Settings-aware `__init__` + `to_relation()` on `IbisConnection` | +| `src/mountainash_data/core/protocol.py` | Modify | Add `to_relation()` to `Connection` protocol | +| `src/mountainash_data/backends/ibis/connection.py` | Modify | Add `to_relation()` to `BaseIbisConnection` | +| `src/mountainash_data/backends/iceberg/connection.py` | Modify | Add `to_relation()` stub to `IcebergConnectionBase` | +| `pyproject.toml` | Modify | Add `relations` optional extra | +| `tests/test_unit/core/test_protocol.py` | Modify | Add `to_relation` to `_FakeConnection` | +| `tests/test_unit/backends/ibis/test_backend_settings.py` | Create | Tests for settings-aware `IbisBackend` | +| `tests/test_unit/backends/ibis/test_to_relation.py` | Create | Tests for `to_relation()` on both Ibis paths | + +--- + +### Task 1: Settings-aware IbisBackend + +**Files:** +- Modify: `src/mountainash_data/backends/ibis/backend.py:106-142` +- Create: `tests/test_unit/backends/ibis/test_backend_settings.py` + +- [ ] **Step 1: Create test directory and write failing tests** + +Create `tests/test_unit/backends/__init__.py` and `tests/test_unit/backends/ibis/__init__.py` if they don't exist, then create the test file: + +```python +"""Tests for settings-aware IbisBackend constructor.""" + +import pytest +from mountainash_data.backends.ibis.backend import IbisBackend, IbisConnection +from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings, NoAuth +from mountainash_settings import SettingsParameters + + +class TestIbisBackendFromSettings: + """Test IbisBackend constructed from SettingsParameters.""" + + def test_sqlite_settings_creates_backend(self): + params = SettingsParameters.create( + settings_class=SQLiteAuthSettings, + kwargs={"DATABASE": ":memory:", "auth": NoAuth()}, + ) + backend = IbisBackend(params) + assert backend.dialect == "sqlite" + + def test_duckdb_settings_creates_backend(self): + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + kwargs={"DATABASE": ":memory:", "auth": NoAuth()}, + ) + backend = IbisBackend(params) + assert backend.dialect == "duckdb" + + def test_settings_backend_connects(self): + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + kwargs={"DATABASE": ":memory:", "auth": NoAuth()}, + ) + backend = IbisBackend(params) + conn = backend.connect() + try: + assert isinstance(conn, IbisConnection) + tables = conn.list_tables() + assert isinstance(tables, list) + finally: + conn.close() + + def test_direct_dialect_still_works(self): + backend = IbisBackend(dialect="duckdb", database=":memory:") + conn = backend.connect() + try: + assert isinstance(conn, IbisConnection) + finally: + conn.close() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend_settings.py -v` +Expected: FAIL — `IbisBackend` does not accept `SettingsParameters` + +- [ ] **Step 3: Implement settings-aware constructor** + +In `src/mountainash_data/backends/ibis/backend.py`, replace the `IbisBackend` class (lines 106-142) with: + +```python +class IbisBackend: + """Ibis backend factory. + + Construction takes either a dialect name with config kwargs, or a + SettingsParameters object that resolves both automatically. + + Usage: + # Direct config + backend = IbisBackend(dialect="sqlite", database=":memory:") + + # Settings-driven + backend = IbisBackend(settings_params) + + conn = backend.connect() + try: + tables = conn.list_tables() + finally: + conn.close() + """ + + name = "ibis" + + def __init__(self, dialect: str | t.Any = "", **config: t.Any): + from mountainash_settings import SettingsParameters + + if isinstance(dialect, SettingsParameters): + settings_params = dialect + settings = settings_params.settings_class.get_settings(settings_params) + descriptor = settings.__descriptor__ + ibis_dialect = descriptor.ibis_dialect + if ibis_dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {ibis_dialect!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = ibis_dialect + self._spec: DialectSpec = DIALECTS[ibis_dialect] + driver_kwargs = settings.to_driver_kwargs() + self._config = { + k: v for k, v in driver_kwargs.items() + if not (isinstance(v, (list, tuple)) and len(v) == 0) + } + else: + if dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {dialect!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = dialect + self._spec = DIALECTS[dialect] + self._config = config + + def connect(self) -> IbisConnection: + """Build and return a live ibis connection.""" + if self._spec.connection_builder is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} has no connection_builder configured" + ) + ibis_conn = self._spec.connection_builder(**self._config) + return IbisConnection(ibis_conn, self._spec) +``` + +Note: The parameter is named `dialect` (not `dialect_or_settings`) to +preserve backward compatibility with `IbisBackend(dialect="sqlite")`. +Type detection via `isinstance` distinguishes `SettingsParameters` from a +string. The `SettingsParameters` import is inside `__init__` to avoid a +top-level dependency for the direct-config path. + +Empty list/tuple values are filtered from settings-derived kwargs (e.g. +`extensions=[]` from DuckDB) because some ibis drivers reject them. This +matches the normalization in `BaseIbisConnection.connect_default()`. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend_settings.py -v` +Expected: PASS — all 4 tests + +- [ ] **Step 5: Run full test suite** + +Run: `hatch run test:test-quick` +Expected: All existing tests still pass + +- [ ] **Step 6: Commit** + +```bash +git add src/mountainash_data/backends/ibis/backend.py tests/test_unit/backends/ +git commit -m "feat(backend): make IbisBackend settings-aware" +``` + +--- + +### Task 2: Add to_relation() to protocol and all implementations + +**Files:** +- Modify: `src/mountainash_data/core/protocol.py:19-52` +- Modify: `src/mountainash_data/backends/ibis/backend.py:21-103` (IbisConnection) +- Modify: `src/mountainash_data/backends/ibis/connection.py:20-99` (BaseIbisConnection) +- Modify: `src/mountainash_data/backends/iceberg/connection.py:50-114` (IcebergConnectionBase) +- Modify: `tests/test_unit/core/test_protocol.py` (_FakeConnection) +- Create: `tests/test_unit/backends/ibis/test_to_relation.py` + +- [ ] **Step 1: Add to_relation() to the Connection protocol** + +In `src/mountainash_data/core/protocol.py`, add after `close()` (after line 52): + +```python + def to_relation( + self, name: str, namespace: str | None = None + ) -> t.Any: + """Return a mountainash Relation for the named table. + + Requires the mountainash package. Raises ImportError if not installed, + or NotImplementedError if the backend does not support it. + """ + ... +``` + +- [ ] **Step 2: Update _FakeConnection in test_protocol.py** + +In `tests/test_unit/core/test_protocol.py`, add to `_FakeConnection` (after `close`): + +```python + def to_relation(self, name: str, namespace: str | None = None) -> t.Any: + return f"relation:{name}" +``` + +- [ ] **Step 3: Run protocol tests to verify they still pass** + +Run: `hatch run test:test-target tests/test_unit/core/test_protocol.py -v` +Expected: PASS + +- [ ] **Step 4: Add to_relation() to IbisConnection** + +In `src/mountainash_data/backends/ibis/backend.py`, add to `IbisConnection` +(after `inspect_catalog`, before `close`): + +```python + def to_relation( + self, name: str, namespace: str | None = None + ) -> t.Any: + """Return a mountainash Relation wrapping the named ibis table.""" + try: + from mountainash.relations import relation + except ImportError: + raise ImportError( + "mountainash package is required for to_relation(). " + "Install it with: pip install mountainash" + ) + ibis_table = self._ibis_conn.table(name, database=namespace) + return relation(ibis_table) +``` + +- [ ] **Step 5: Add to_relation() to BaseIbisConnection** + +In `src/mountainash_data/backends/ibis/connection.py`, add to +`BaseIbisConnection` (after `connect_default`, before `_connect` — after +line 137): + +```python + def to_relation( + self, name: str, namespace: str | None = None + ) -> t.Any: + """Return a mountainash Relation wrapping the named ibis table.""" + try: + from mountainash.relations import relation + except ImportError: + raise ImportError( + "mountainash package is required for to_relation(). " + "Install it with: pip install mountainash" + ) + self.connect() + ibis_table = self.ibis_backend.table(name, database=namespace) + return relation(ibis_table) +``` + +- [ ] **Step 6: Add to_relation() stub to IcebergConnectionBase** + +In `src/mountainash_data/backends/iceberg/connection.py`, add to +`IcebergConnectionBase` (after `is_connected`, before the schema cache +section — after line 138): + +```python + def to_relation( + self, name: str, namespace: str | None = None + ) -> t.Any: + """Not yet supported for Iceberg connections.""" + raise NotImplementedError( + "to_relation() is not yet supported for Iceberg connections. " + "Use table() to get the native pyiceberg Table object." + ) +``` + +- [ ] **Step 7: Write tests for to_relation()** + +Create `tests/test_unit/backends/ibis/test_to_relation.py`: + +```python +"""Tests for to_relation() on Ibis connection paths.""" + +import pytest +import typing as t +from unittest.mock import patch + +from mountainash_data.backends.ibis.backend import IbisBackend, IbisConnection + + +class TestIbisConnectionToRelation: + """Test to_relation() on the new-style IbisConnection path.""" + + @pytest.fixture + def conn(self): + backend = IbisBackend(dialect="duckdb", database=":memory:") + conn = backend.connect() + yield conn + conn.close() + + def test_to_relation_returns_relation(self, conn): + try: + from mountainash.relations import Relation + except ImportError: + pytest.skip("mountainash package not installed") + + conn._ibis_conn.raw_sql("CREATE TABLE test_tbl (id INTEGER, name VARCHAR)") + result = conn.to_relation("test_tbl") + assert isinstance(result, Relation) + + def test_to_relation_import_error(self, conn): + conn._ibis_conn.raw_sql("CREATE TABLE test_tbl2 (id INTEGER)") + with patch.dict("sys.modules", {"mountainash": None, "mountainash.relations": None}): + with pytest.raises(ImportError, match="mountainash package is required"): + conn.to_relation("test_tbl2") + + +class TestBaseIbisConnectionToRelation: + """Test to_relation() on the settings/factory path.""" + + @pytest.fixture + def conn(self): + from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + from mountainash_settings import SettingsParameters + from mountainash_data.core.factories import ConnectionFactory + + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + kwargs={"DATABASE": ":memory:", "auth": NoAuth()}, + ) + connection = ConnectionFactory.get_connection(params) + connection.connect() + yield connection + connection.disconnect() + + def test_to_relation_returns_relation(self, conn): + try: + from mountainash.relations import Relation + except ImportError: + pytest.skip("mountainash package not installed") + + conn.ibis_backend.raw_sql("CREATE TABLE test_tbl3 (id INTEGER, name VARCHAR)") + result = conn.to_relation("test_tbl3") + assert isinstance(result, Relation) + + +class TestIcebergToRelationStub: + """Test that Iceberg raises NotImplementedError.""" + + def test_raises_not_implemented(self): + from mountainash_data.backends.iceberg.connection import IcebergConnectionBase + with pytest.raises(NotImplementedError, match="not yet supported"): + IcebergConnectionBase.to_relation(None, "some_table") +``` + +- [ ] **Step 8: Run tests** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_to_relation.py -v` +Expected: Tests pass (or skip if mountainash not installed) + +Run: `hatch run test:test-quick` +Expected: Full suite passes + +- [ ] **Step 9: Commit** + +```bash +git add src/mountainash_data/core/protocol.py \ + src/mountainash_data/backends/ibis/backend.py \ + src/mountainash_data/backends/ibis/connection.py \ + src/mountainash_data/backends/iceberg/connection.py \ + tests/test_unit/core/test_protocol.py \ + tests/test_unit/backends/ibis/test_to_relation.py +git commit -m "feat(protocol): add to_relation() to Connection protocol and Ibis implementations" +``` + +--- + +### Task 3: Add mountainash as optional dependency + +**Files:** +- Modify: `pyproject.toml:51-68` + +- [ ] **Step 1: Add relations extra to pyproject.toml** + +In `pyproject.toml`, after the `trino` optional dependency line (line 68), add: + +```toml +relations = ["mountainash"] +``` + +- [ ] **Step 2: Run tests** + +Run: `hatch run test:test-quick` +Expected: All tests pass + +- [ ] **Step 3: Commit** + +```bash +git add pyproject.toml +git commit -m "feat(deps): add mountainash as optional relations extra" +``` + +--- + +## Follow-up (not in this plan) + +- **Phase 2:** Migrate `DatabaseUtils` consumers to `IbisBackend(settings_params)` +- **Phase 3:** Deprecate `ConnectionFactory`, `OperationsFactory`, 12 concrete subclasses +- **Dead code candidate:** After the legacy branch removal in PR #78, + `BaseIbisConnection.ibis_connection_mode` and `connection_string_scheme` + abstract properties are unused by `connect_default()` but still declared + on all 12 subclasses. Track separately. + +Tracked in: `mountainash-central/01.principles/mountainash-data/f.backlog/settings-aware-backends.md` diff --git a/docs/superpowers/plans/2026-04-27-settings-aware-ibis-backend.md b/docs/superpowers/plans/2026-04-27-settings-aware-ibis-backend.md new file mode 100644 index 0000000..2bc0a8f --- /dev/null +++ b/docs/superpowers/plans/2026-04-27-settings-aware-ibis-backend.md @@ -0,0 +1,452 @@ +# Settings-Aware IbisBackend Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Extend `IbisBackend` to accept `SettingsParameters` and connection URLs alongside the existing dialect keyword, producing `IbisConnection` from all three input forms. + +**Architecture:** The `IbisBackend.__init__` constructor gains a positional-only first parameter that accepts `SettingsParameters` or a URL string. A `dialect=` keyword preserves the existing path. All forms resolve to `(self.dialect, self._spec, self._config)` so `connect()` stays simple. A module-level `_SCHEME_TO_DIALECT` map (built from the `DIALECTS` registry) handles URL scheme detection. + +**Tech Stack:** ibis-framework, mountainash-settings (`SettingsParameters`), mountainash-data settings (`ConnectionProfile`, `BackendDescriptor`) + +**Spec:** `docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md` + +--- + +## File Map + +| File | Responsibility | Change | +|------|---------------|--------| +| `src/mountainash_data/backends/ibis/backend.py` | `IbisBackend` + `IbisConnection` | New constructor, `_SCHEME_TO_DIALECT` map, empty-list filter in `connect()` | +| `tests/test_unit/backends/ibis/test_backend.py` | Unit tests for `IbisBackend` | New tests for settings, URL, error paths | + +No new files. `IbisConnection`, `DialectSpec` registry, and builders are untouched. + +--- + +### Task 1: Error-case tests and `_SCHEME_TO_DIALECT` map + +Establishes the constructor signature, dispatch validation, and the scheme lookup — all tested before the happy paths. + +**Files:** +- Modify: `src/mountainash_data/backends/ibis/backend.py:106-142` +- Modify: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing tests for constructor validation** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +import pytest +from mountainash_data.backends.ibis.backend import IbisBackend + + +def test_neither_positional_nor_dialect_raises(): + """Constructor with no arguments must raise ValueError.""" + with pytest.raises(ValueError, match="Either.*or.*dialect"): + IbisBackend() + + +def test_both_positional_and_dialect_raises(): + """Cannot supply both a positional arg and dialect= keyword.""" + with pytest.raises(ValueError, match="Cannot specify both"): + IbisBackend("sqlite://", dialect="sqlite") + + +def test_unknown_url_scheme_raises(): + """URL with unrecognised scheme must raise ValueError.""" + with pytest.raises(ValueError, match="Cannot detect ibis dialect"): + IbisBackend("nosuch://localhost/db") +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_neither_positional_nor_dialect_raises tests/test_unit/backends/ibis/test_backend.py::test_both_positional_and_dialect_raises tests/test_unit/backends/ibis/test_backend.py::test_unknown_url_scheme_raises -v` + +Expected: FAIL — the current constructor signature `__init__(self, dialect: str, **config)` doesn't match these call forms. + +- [ ] **Step 3: Implement `_SCHEME_TO_DIALECT` map and new constructor** + +Replace the `IbisBackend` class in `src/mountainash_data/backends/ibis/backend.py` (lines 106–142) with: + +```python +# --------------------------------------------------------------------------- +# Scheme → dialect reverse lookup (built once from the DIALECTS registry) +# --------------------------------------------------------------------------- +def _build_scheme_to_dialect() -> dict[str, str]: + """Build a map from URL scheme (e.g. 'sqlite', 'postgres') to dialect name.""" + result: dict[str, str] = {} + for dialect_name, spec in DIALECTS.items(): + # connection_string_scheme is e.g. "postgres://", "duckdb://md:" + scheme = spec.connection_string_scheme.split("://")[0].lower() + # First dialect wins — e.g. "postgres" maps to "postgres", not "redshift" + if scheme not in result: + result[scheme] = dialect_name + # Common aliases + result.setdefault("postgresql", result.get("postgres", "postgres")) + return result + + +_SCHEME_TO_DIALECT: dict[str, str] = _build_scheme_to_dialect() + + +class IbisBackend: + """Ibis backend — single entry point for all Ibis connections. + + Three input forms, all producing IbisConnection via connect(): + + # Settings object (deployment, env-driven config) + backend = IbisBackend(settings_params) + + # Connection URL (universal connection strings) + backend = IbisBackend("postgresql://user:pass@host:5432/db") + + # Dialect keyword + kwargs (tests, scripts) + backend = IbisBackend(dialect="sqlite", database=":memory:") + """ + + name = "ibis" + + def __init__( + self, + settings_or_connection_string: str | t.Any | None = None, + /, + *, + dialect: str | None = None, + **config: t.Any, + ): + if settings_or_connection_string is not None and dialect is not None: + raise ValueError( + "Cannot specify both a positional settings/URL argument " + "and dialect= keyword" + ) + + if settings_or_connection_string is not None: + self._init_from_positional(settings_or_connection_string, config) + elif dialect is not None: + self._init_from_dialect(dialect, config) + else: + raise ValueError( + "Either a SettingsParameters/URL positional argument " + "or a dialect= keyword is required" + ) + + def _init_from_positional( + self, value: str | t.Any, config: dict[str, t.Any] + ) -> None: + # Lazy import — only pay for it on the settings/URL paths + from mountainash_settings import SettingsParameters + + if isinstance(value, SettingsParameters): + self._init_from_settings(value, config) + elif isinstance(value, str): + if "://" in value: + self._init_from_url(value, config) + else: + # Plain string — treat as dialect name + self._init_from_dialect(value, config) + else: + raise TypeError( + f"Expected SettingsParameters or str, got {type(value).__name__}" + ) + + def _init_from_dialect( + self, dialect_name: str, config: dict[str, t.Any] + ) -> None: + if dialect_name not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {dialect_name!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = dialect_name + self._spec: DialectSpec = DIALECTS[dialect_name] + self._config = config + + def _init_from_dialect( + self, dialect_name: str, config: dict[str, t.Any] + ) -> None: + if dialect_name not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {dialect_name!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = dialect_name + self._spec: DialectSpec = DIALECTS[dialect_name] + self._url: str | None = None + self._config = config + + def _init_from_url( + self, url: str, config: dict[str, t.Any] + ) -> None: + from urllib.parse import urlparse + + scheme = urlparse(url).scheme.lower() + + # Special case: MotherDuck URLs are "duckdb://md:..." + if scheme == "duckdb" and url.startswith("duckdb://md:"): + resolved_dialect = "motherduck" + else: + resolved_dialect = _SCHEME_TO_DIALECT.get(scheme) + + if resolved_dialect is None: + raise ValueError( + f"Cannot detect ibis dialect from URL scheme: {scheme!r}" + ) + + self.dialect = resolved_dialect + self._spec = DIALECTS[resolved_dialect] + self._url = url + self._config = config + + def _init_from_settings( + self, settings_params: t.Any, config: dict[str, t.Any] + ) -> None: + obj_settings = settings_params.settings_class.get_settings( + settings_parameters=settings_params + ) + descriptor = getattr(obj_settings, "__descriptor__", None) + if descriptor is None or getattr(descriptor, "ibis_dialect", None) is None: + raise ValueError( + f"Settings class {type(obj_settings).__name__} has no " + f"ibis_dialect on its descriptor" + ) + resolved_dialect = descriptor.ibis_dialect + if resolved_dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {resolved_dialect!r} from descriptor. " + f"Available: {sorted(DIALECTS)}" + ) + driver_kwargs = obj_settings.to_driver_kwargs() + driver_kwargs.update(config) + + self.dialect = resolved_dialect + self._spec = DIALECTS[resolved_dialect] + self._url = None + self._config = driver_kwargs + + def connect(self) -> IbisConnection: + """Build and return a live ibis connection.""" + if self._spec.connection_builder is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} has no connection_builder configured" + ) + if self._url is not None: + # URL path: delegate directly to ibis.connect() which + # natively handles all URL forms and preserves all URL + # components (host, port, credentials, database, query params). + import ibis + ibis_conn = ibis.connect(self._url, **self._config) + else: + # Settings/dialect path: go through the dialect builder + # with empty-list normalization (e.g. DuckDB extensions=[]). + cleaned_config = { + k: v for k, v in self._config.items() + if not (isinstance(v, (list, tuple)) and len(v) == 0) + } + ibis_conn = self._spec.connection_builder(**cleaned_config) + return IbisConnection(ibis_conn, self._spec) +``` + +Key details: +- The type annotation for `settings_or_connection_string` uses `str | t.Any | None` rather than `str | SettingsParameters | None` to avoid a top-level import of `mountainash_settings`. The `SettingsParameters` isinstance check happens inside `_init_from_positional` with a lazy import. +- `_init_from_url` stores the raw URL on `self._url`. In `connect()`, when `_url` is set, the URL is passed directly to `ibis.connect(url)` — this natively handles all URL forms for all backends and preserves all URL components. The dialect builders are NOT used for the URL path (they don't all handle `connection_string` kwargs). +- `_init_from_settings` resolves the settings object, extracts `ibis_dialect` from the descriptor, and calls `to_driver_kwargs()`. Sets `_url = None`. +- `_init_from_dialect` is the existing path, unchanged except for initialising `_url = None`. +- `connect()` branches on `self._url`: URL path uses `ibis.connect(url)`, settings/dialect path uses the builder with empty-list filtering. The empty-list filter is essential because `DuckDBAuthSettings.to_driver_kwargs()` returns `extensions: []` by default, which `ibis.duckdb.connect()` rejects. + +- [ ] **Step 4: Run the three error tests** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_neither_positional_nor_dialect_raises tests/test_unit/backends/ibis/test_backend.py::test_both_positional_and_dialect_raises tests/test_unit/backends/ibis/test_backend.py::test_unknown_url_scheme_raises -v` + +Expected: PASS — all three error cases now handled by the new constructor. + +- [ ] **Step 5: Run existing tests to verify no regressions** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py -v` + +Expected: All 4 existing tests + 3 new tests PASS. The existing tests use `IbisBackend(dialect="sqlite", ...)` which hits `_init_from_dialect` — the unchanged path. + +- [ ] **Step 6: Commit** + +```bash +git add src/mountainash_data/backends/ibis/backend.py tests/test_unit/backends/ibis/test_backend.py +git commit -m "feat(backend): new IbisBackend constructor with dispatch and error validation + +Add _SCHEME_TO_DIALECT map, three-way dispatch (settings, URL, dialect), +empty-list normalization in connect(), and URL-direct via ibis.connect(). +Error cases tested: no args, both args, unknown scheme." +``` + +--- + +### Task 2: Settings path — SQLite and DuckDB + +Wire and test the `SettingsParameters` input form. + +**Files:** +- Modify: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing tests for settings path** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +from mountainash_data.backends.ibis.backend import IbisBackend, IbisConnection + + +def test_settings_path_sqlite(): + """Construct IbisBackend from SQLite SettingsParameters and connect.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import SQLiteAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=SQLiteAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "sqlite" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + tables = conn.list_tables() + assert isinstance(tables, list) + conn.close() + + +def test_settings_path_duckdb_empty_extensions(): + """DuckDB settings with default EXTENSIONS=[] must not crash ibis.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "duckdb" + conn = backend.connect() # Must not raise — empty-list filter active + assert isinstance(conn, IbisConnection) + conn.close() +``` + +- [ ] **Step 2: Run tests to verify they pass** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_settings_path_sqlite tests/test_unit/backends/ibis/test_backend.py::test_settings_path_duckdb_empty_extensions -v` + +Expected: PASS — the constructor's `_init_from_settings` path was implemented in Task 1. + +If the DuckDB test fails with an error about `extensions=[]`, verify that `connect()` properly filters empty lists before calling the builder. + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_unit/backends/ibis/test_backend.py +git commit -m "test(backend): settings path — SQLite and DuckDB SettingsParameters + +Verifies constructor resolves SettingsParameters via descriptor.ibis_dialect +and to_driver_kwargs(). DuckDB test confirms empty-list normalization +filters extensions=[] before reaching ibis." +``` + +--- + +### Task 3: URL path — SQLite and DuckDB + +Wire and test the connection URL input form. + +**Files:** +- Modify: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing tests for URL path** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +def test_url_path_sqlite(): + """Construct IbisBackend from sqlite:// URL and connect.""" + backend = IbisBackend("sqlite://") + assert backend.dialect == "sqlite" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + + +def test_url_path_duckdb(): + """Construct IbisBackend from duckdb:// URL and connect.""" + backend = IbisBackend("duckdb://") + assert backend.dialect == "duckdb" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + + +def test_url_path_preserves_database(tmp_path): + """URL database component must reach the driver, not be discarded.""" + db_file = tmp_path / "test.db" + backend = IbisBackend(f"sqlite:///{db_file}") + assert backend.dialect == "sqlite" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + assert db_file.exists() +``` + +- [ ] **Step 2: Run tests to verify they pass** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_url_path_sqlite tests/test_unit/backends/ibis/test_backend.py::test_url_path_duckdb tests/test_unit/backends/ibis/test_backend.py::test_url_path_preserves_database -v` + +Expected: PASS — the URL path calls `ibis.connect(url)` directly, which handles all URL forms natively. + +The `test_url_path_preserves_database` test is the critical regression test identified by Codex review: it proves the URL's database path actually reaches the driver (the file must exist on disk after connect). + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_unit/backends/ibis/test_backend.py +git commit -m "test(backend): URL path — sqlite://, duckdb://, file path preservation + +Verifies URL dispatch via _SCHEME_TO_DIALECT, ibis.connect(url) delegation. +Critical regression: sqlite:///path creates the file on disk, proving +URL components are not discarded." +``` + +--- + +### Task 4: Full test suite and mark old docs abandoned + +Run the entire test suite, then commit the abandoned old spec/plan. + +**Files:** +- Modify: `docs/superpowers/specs/2026-04-26-to-relation-design.md` (already marked) +- Modify: `docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md` (already marked) + +- [ ] **Step 1: Run full test suite** + +Run: `hatch run test:test-quick` + +Expected: All tests PASS (441+ existing + 8 new). No regressions. + +If any existing tests fail, investigate — the existing `test_ibis_backend_satisfies_protocol`, `test_unknown_dialect_raises`, `test_all_registered_dialects_construct`, and `test_in_memory_sqlite_connect_and_inspect` should all still pass because they use `dialect=` keyword. + +- [ ] **Step 2: Verify old spec and plan are marked abandoned** + +Check that these files already have ABANDONED status (done earlier in the brainstorming session): +- `docs/superpowers/specs/2026-04-26-to-relation-design.md` — line 4 should say `ABANDONED` +- `docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md` — line 3 should say `ABANDONED` + +- [ ] **Step 3: Commit abandoned docs** + +```bash +git add docs/superpowers/specs/2026-04-26-to-relation-design.md docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md docs/superpowers/plans/2026-04-27-settings-aware-ibis-backend.md +git commit -m "chore(specs): abandon old to-relation spec/plan, add new settings-aware spec/plan + +Old spec bundled to_relation() which has been descoped per revised +principle. New spec focuses solely on settings-aware IbisBackend +constructor (Phase 1 of connection consolidation)." +``` + +- [ ] **Step 4: Run full test suite one final time** + +Run: `hatch run test:test-quick` + +Expected: All tests PASS. Clean working tree (no uncommitted changes to source). diff --git a/docs/superpowers/specs/2026-04-26-to-relation-design.md b/docs/superpowers/specs/2026-04-26-to-relation-design.md new file mode 100644 index 0000000..c11e8c8 --- /dev/null +++ b/docs/superpowers/specs/2026-04-26-to-relation-design.md @@ -0,0 +1,210 @@ +# Settings-Aware Backends + to_relation() + +> **Date:** 2026-04-26 (updated 2026-04-27) +> **Status:** ABANDONED -- superseded by `2026-04-27-settings-aware-ibis-backend-design.md`. to_relation() descoped per revised principle. +> **Backlog refs:** +> - `mountainash-central/01.principles/mountainash-data/f.backlog/to-relation-gap.md` +> - `mountainash-central/01.principles/mountainash-data/f.backlog/settings-aware-backends.md` (Phase 1) + +## Context + +mountainash-data has two parallel connection paths that produce the same +result with different indirection: + +1. **New-style:** `IbisBackend(dialect="sqlite", **config).connect()` → + `IbisConnection` (protocol-compliant) +2. **Settings-driven:** `ConnectionFactory.get_connection(settings_params)` → + `BaseIbisConnection` subclass (old hierarchy) + +The gap: `IbisBackend` only accepts direct config. Consumers with a +`SettingsParameters` must use the old factory path or manually resolve +settings. This keeps the factory alive despite adding no value post-refactor. + +Separately, `to_relation()` is defined on the `Connection` protocol concept +but not yet implemented. The `mountainash` package (formerly +mountainash-expressions) provides `relation()` which accepts ibis tables +and wraps them in a `Relation` AST node. + +This spec covers: +- Making `IbisBackend` settings-aware (Phase 1 of settings-aware-backends backlog) +- Wiring `to_relation()` on both connection paths + +## Design + +### 1. Settings-aware IbisBackend + +`IbisBackend.__init__` accepts either a dialect string + kwargs (existing) +or a `SettingsParameters` object (new): + +```python +class IbisBackend: + name = "ibis" + + def __init__(self, dialect_or_settings: str | SettingsParameters, **config: Any): + if isinstance(dialect_or_settings, SettingsParameters): + settings_params = dialect_or_settings + settings = settings_params.settings_class.get_settings(settings_params) + descriptor = settings.__descriptor__ + ibis_dialect = descriptor.ibis_dialect + if ibis_dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {ibis_dialect!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = ibis_dialect + self._spec = DIALECTS[ibis_dialect] + self._config = settings.to_driver_kwargs() + else: + dialect = dialect_or_settings + if dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {dialect!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = dialect + self._spec = DIALECTS[dialect] + self._config = config +``` + +Usage: + +```python +from mountainash_data import IbisBackend + +# Direct +backend = IbisBackend(dialect="sqlite", database=":memory:") + +# Settings-driven +backend = IbisBackend(settings_params) + +# Both produce the same IbisConnection +conn = backend.connect() +``` + +### 2. Protocol — to_relation() + +Add `to_relation()` to the `Connection` protocol in `core/protocol.py`: + +```python +def to_relation(self, name: str, namespace: str | None = None) -> "Relation": + """Return a mountainash Relation for the named table. + + Requires the mountainash package. Raises ImportError if not installed. + """ + ... +``` + +Signature matches `inspect_table(name, namespace)` for consistency. Return +type uses a string forward reference to avoid importing mountainash at module +level. + +### 3. Ibis implementation — new-style path (IbisConnection) + +In `backends/ibis/backend.py`, add to `IbisConnection`: + +```python +def to_relation(self, name: str, namespace: str | None = None) -> "Relation": + try: + from mountainash.relations import relation + except ImportError: + raise ImportError( + "mountainash package is required for to_relation(). " + "Install it with: pip install mountainash" + ) + ibis_table = self._ibis_conn.table(name, database=namespace) + return relation(ibis_table) +``` + +### 4. Ibis implementation — settings/factory path (BaseIbisConnection) + +In `backends/ibis/connection.py`, add to `BaseIbisConnection`: + +```python +def to_relation(self, name: str, namespace: str | None = None) -> "Relation": + try: + from mountainash.relations import relation + except ImportError: + raise ImportError( + "mountainash package is required for to_relation(). " + "Install it with: pip install mountainash" + ) + self.connect() + ibis_table = self.ibis_backend.table(name, database=namespace) + return relation(ibis_table) +``` + +This covers connections obtained via `ConnectionFactory` and `DatabaseUtils`. +The factory path gets `to_relation()` now; it will be deprecated in Phase 3 +of the settings-aware-backends backlog. + +### 5. Iceberg stub + +In `backends/iceberg/connection.py`, add to `IcebergConnectionBase`: + +```python +def to_relation(self, name: str, namespace: str | None = None) -> "Relation": + raise NotImplementedError( + "to_relation() is not yet supported for Iceberg connections. " + "Use table() to get the native pyiceberg Table object." + ) +``` + +### 6. Dependency + +Add `mountainash` as an optional extra in `pyproject.toml`: + +```toml +[project.optional-dependencies] +relations = ["mountainash"] +``` + +The core package does NOT depend on mountainash — the import is guarded at +call time in `to_relation()`. + +### 7. Extensibility pattern + +Future backends (DataFusion, etc.) follow the same two-step pattern: + +1. Make the backend class settings-aware (`__init__` accepts `SettingsParameters`) +2. Add `to_relation()` — get native table handle, pass to `relation()` + +The only prerequisite is that `identify_backend()` in mountainash recognises +the native table type and a corresponding relation system backend exists. + +## Testing + +### Settings-aware IbisBackend +- Construct `IbisBackend(settings_params)` with SQLite/DuckDB settings, + verify `.connect()` returns working `IbisConnection` +- Verify `IbisBackend(settings_params).dialect` matches expected dialect +- Verify invalid settings raise `KeyError` +- Verify existing direct path still works unchanged + +### to_relation() +- `IbisConnection.to_relation()` (new-style) with in-memory DuckDB: create + table, call `to_relation()`, verify returns `Relation` instance +- `BaseIbisConnection.to_relation()` (factory path) via + `ConnectionFactory.get_connection()`: same verification +- Round-trip test: `to_relation()` → `.collect()` returns expected data +- `ImportError` path: mock the import to verify clear error message +- `IcebergConnectionBase.to_relation()` raises `NotImplementedError` + +## Commit strategy + +Three commits, single branch, single PR targeting `develop`: + +1. `feat(backend): make IbisBackend settings-aware` +2. `feat(protocol): add to_relation() to Connection protocol and Ibis implementations` +3. `feat(deps): add mountainash as optional relations extra` + +## Follow-up (not in this PR) + +- **Phase 2:** Migrate `DatabaseUtils` consumers to use `IbisBackend(settings_params)` directly +- **Phase 3:** Deprecate `ConnectionFactory`, `OperationsFactory`, `DatabaseUtils.create_connection()`/`create_operations()`, and the 12 concrete `BaseIbisConnection` subclasses + +Tracked in: `mountainash-central/01.principles/mountainash-data/f.backlog/settings-aware-backends.md` + +## Backlog updates after merge + +- `to-relation-gap.md` → **RESOLVED** (Ibis wired; Iceberg stub) +- `settings-aware-backends.md` → Phase 1 **RESOLVED** diff --git a/docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md b/docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md new file mode 100644 index 0000000..e914601 --- /dev/null +++ b/docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md @@ -0,0 +1,262 @@ +# Settings-Aware IbisBackend + +> **Date:** 2026-04-27 +> **Status:** Approved +> **Principle:** `mountainash-central/01.principles/mountainash-data/b.connection-management/connection-consolidation.md` (Phase 1) +> **Backlog ref:** `mountainash-central/01.principles/mountainash-data/f.backlog/settings-aware-backends.md` +> **Supersedes:** `docs/superpowers/specs/2026-04-26-to-relation-design.md` (ABANDONED) + +## Goal + +Extend `IbisBackend` to accept `SettingsParameters` and connection URLs +alongside the existing dialect keyword. All three input forms resolve to +the same internal state and produce `IbisConnection` via `connect()`. +This is Phase 1 of connection consolidation. + +## Constructor Signature + +```python +def __init__( + self, + settings_or_connection_string: str | SettingsParameters | None = None, + /, + *, + dialect: str | None = None, + **config: t.Any, +): +``` + +### Input Forms + +```python +from mountainash_data import IbisBackend +from mountainash_settings import SettingsParameters +from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + +# Form 1: Settings object (deployment, env-driven config) +settings_params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), +) +backend = IbisBackend(settings_params) + +# Form 2: Connection URL (universal connection strings) +backend = IbisBackend("postgresql://user:pass@host:5432/db") + +# Form 3: Dialect keyword + kwargs (tests, scripts) +backend = IbisBackend(dialect="sqlite", database=":memory:") + +# All produce the same IbisConnection +conn = backend.connect() +``` + +## Dispatch Logic + +The constructor resolves all input forms to `(self.dialect, self._spec, +self._config)` so that `connect()` requires no changes beyond empty-list +normalization. + +1. **Both positional and `dialect=` provided** -> `ValueError`. +2. **Positional is `SettingsParameters`** -> settings path: + - `settings_class.get_settings(settings_parameters)` to resolve. + - `descriptor.ibis_dialect` to get dialect name. + - `to_driver_kwargs()` for config dict. + - `**config` kwargs merged on top (caller overrides). +3. **Positional is `str` containing `://`** -> URL path: + - Detect dialect from URL scheme using a reverse lookup on + `DialectSpec.connection_string_scheme`. + - Store the raw URL as `connection_string` in config — the dialect + builder passes it straight to `ibis.connect(url)`. + - `**config` kwargs merged on top (caller overrides). +4. **Positional is a plain `str`** -> treat as dialect name (same as + `dialect=` keyword). +5. **`dialect` keyword provided** -> existing dialect path (unchanged). +6. **Neither provided** -> `ValueError`. + +### Settings Resolution Detail + +```python +# Inside __init__, settings path: +from mountainash_settings import SettingsParameters + +obj_settings = settings_or_connection_string.settings_class.get_settings( + settings_parameters=settings_or_connection_string +) +descriptor = getattr(obj_settings, "__descriptor__", None) +if descriptor is None or descriptor.ibis_dialect is None: + raise ValueError( + f"Settings class {type(obj_settings).__name__} has no ibis_dialect on its descriptor" + ) +resolved_dialect = descriptor.ibis_dialect +driver_kwargs = obj_settings.to_driver_kwargs() +driver_kwargs.update(config) # caller overrides +``` + +### URL Resolution Detail + +The URL path bypasses settings entirely. The raw URL is passed to the +dialect builder as `connection_string`, which forwards it to +`ibis.connect(url)`. This preserves all URL components (host, port, +credentials, database, query params) without lossy round-tripping +through settings fields. + +Dialect detection uses a reverse lookup built from the `DIALECTS` +registry — each `DialectSpec` already carries `connection_string_scheme`. + +```python +# Inside __init__, URL path: +from urllib.parse import urlparse + +# Build reverse scheme -> dialect map from registry +# e.g. {"sqlite": "sqlite", "duckdb": "duckdb", "postgres": "postgres", ...} +scheme = urlparse(settings_or_connection_string).scheme.lower() + +# Special cases: "postgresql" -> "postgres", "md" -> "motherduck" +# MotherDuck also detected by "duckdb://md:" prefix +resolved_dialect = _SCHEME_TO_DIALECT.get(scheme) +if resolved_dialect is None: + raise ValueError( + f"Cannot detect ibis dialect from URL scheme: {scheme!r}" + ) + +# Store raw URL as connection_string — builders pass it to ibis.connect() +driver_kwargs = {"connection_string": settings_or_connection_string} +driver_kwargs.update(config) # caller overrides +``` + +The `_SCHEME_TO_DIALECT` map is built once at module level from the +`DIALECTS` registry, with additional aliases for common scheme variants +(e.g. `postgresql` -> `postgres`). + +## Empty-List Normalization + +Happens in `connect()`, not `__init__`. Some ibis drivers reject empty +sequences (e.g. `ibis.duckdb.connect(extensions=[])` fails). + +```python +def connect(self) -> IbisConnection: + cleaned_config = { + k: v for k, v in self._config.items() + if not (isinstance(v, (list, tuple)) and len(v) == 0) + } + ibis_conn = self._spec.connection_builder(**cleaned_config) + return IbisConnection(ibis_conn, self._spec) +``` + +## Files Changed + +| File | Change | +|------|--------| +| `src/mountainash_data/backends/ibis/backend.py` | New constructor signature, dispatch logic, empty-list normalization in `connect()`, `_SCHEME_TO_DIALECT` map | +| `tests/test_unit/backends/ibis/test_backend.py` | New tests for settings, URL, and error paths | + +## Files NOT Changed + +- `IbisConnection` -- untouched +- `DialectSpec` registry / builders -- untouched (URL path uses existing `connection_string` kwarg) +- `ConnectionFactory`, `DatabaseUtils`, `BaseIbisConnection` -- no deprecation yet (Phase 2) +- `core/protocol.py` -- untouched +- No new files + +## Testing + +All tests use SQLite and DuckDB (in-memory, no external deps). + +### 1. Dialect Path (existing, unchanged) + +```python +def test_dialect_path(): + backend = IbisBackend(dialect="sqlite", database=":memory:") + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() +``` + +### 2. Settings Path + +```python +def test_settings_path_sqlite(): + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import SQLiteAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=SQLiteAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + conn = backend.connect() + assert isinstance(conn, IbisConnection) + tables = conn.list_tables() + assert isinstance(tables, list) + conn.close() + +def test_settings_path_duckdb_empty_extensions(): + """DuckDB settings with default EXTENSIONS=[] must not reach ibis.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + conn = backend.connect() # Must not raise + assert isinstance(conn, IbisConnection) + conn.close() +``` + +### 3. URL Path + +```python +def test_url_path_sqlite(): + backend = IbisBackend("sqlite://") + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + +def test_url_path_duckdb(): + backend = IbisBackend("duckdb://") + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + +def test_url_path_preserves_database(tmp_path): + """URL database component must reach the driver, not be discarded.""" + db_file = tmp_path / "test.db" + backend = IbisBackend(f"sqlite:///{db_file}") + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + assert db_file.exists() + +def test_url_path_unknown_scheme_raises(): + with pytest.raises(ValueError, match="Cannot detect ibis dialect"): + IbisBackend("nosuch://localhost/db") +``` + +### 4. Error Cases + +```python +def test_both_positional_and_dialect_raises(): + with pytest.raises(ValueError): + IbisBackend("sqlite://", dialect="sqlite") + +def test_neither_provided_raises(): + with pytest.raises(ValueError): + IbisBackend() + +def test_unknown_dialect_raises(): + with pytest.raises(KeyError): + IbisBackend(dialect="nosuch") +``` + +## Commit Strategy + +Single branch (`feature/settings-aware-ibis-backend`) targeting `develop`. +Two commits: + +1. `feat(backend): make IbisBackend settings-aware` -- constructor + tests +2. `chore(specs): abandon old to-relation spec and plan` -- mark old docs From 9badcec2fdf6e001894f00c9e9ab7428106d1ff0 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 16:44:10 +1000 Subject: [PATCH 04/12] chore(specs): add backend-as-single-handle design spec Collapses connection consolidation Phases 2+3 into a single phase: make IbisBackend the single public handle for all ibis interaction by absorbing operations via DialectSpec hooks, adding fluent API, and deleting factories/DatabaseUtils/legacy class hierarchies. Co-Authored-By: Claude Opus 4.6 (1M context) --- ...6-04-27-backend-as-single-handle-design.md | 360 ++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md diff --git a/docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md b/docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md new file mode 100644 index 0000000..52bd406 --- /dev/null +++ b/docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md @@ -0,0 +1,360 @@ +# Backend as Single Handle — Design Spec + +> **Status:** APPROVED +> **Date:** 2026-04-27 +> **Branch:** `feature/settings-aware-ibis-backend` (continues Phase 1 work) +> **Supersedes:** Connection consolidation Phases 2+3 (collapsed into one) + +## Goal + +Make `IbisBackend` the single public handle for all ibis interaction — +construction, connection lifecycle, inspection, and operations. Delete +`ConnectionFactory`, `OperationsFactory`, `DatabaseUtils`, the 12 concrete +`BaseIbisConnection` subclasses, and the separate `BaseIbisOperations` class +hierarchy. + +## Architecture + +`IbisBackend` composes three internal concerns: + +1. **Config resolution** (existing, unchanged) — three-way constructor + dispatch: `SettingsParameters`, URL, or `dialect=` keyword. +2. **Connection lifecycle** — `connect()` returns `self`, `close()` returns + `self`, context manager support. +3. **Operations dispatch** — instance methods on `IbisBackend` that delegate + to the internal `IbisConnection` (inspection), the raw ibis connection + (thin wrappers), or `DialectSpec` callable hooks (per-dialect operations). + +`IbisConnection` stays as an **internal** class. The consumer never imports +or interacts with it directly. `IbisBackend` exposes two accessor methods: + +- `ibis_connection()` → raw ibis backend object (for the seam with + mountainash-expressions: `backend.ibis_connection().table("users")`) +- `get_connection()` → our `IbisConnection` wrapper (for internal use) + +Both raise `RuntimeError` if not connected. + +### What Gets Deleted + +| Target | Location | Reason | +|--------|----------|--------| +| `ConnectionFactory` | `core/factories/connection_factory.py` | Replaced by `IbisBackend` constructor | +| `OperationsFactory` | `core/factories/operations_factory.py` | Replaced by operations on `IbisBackend` | +| `SettingsFactory` | `core/factories/settings_factory.py` | URL detection now in `_SCHEME_TO_DIALECT`; settings creation via `IbisBackend(settings_params)` | +| `DatabaseUtils` | `core/utils.py` | Entire class — `IbisBackend` is the API | +| `BaseDBConnection` | `core/connection.py` | Abstract base no longer needed | +| `BaseIbisConnection` + 12 subclasses | `backends/ibis/connection.py` | Replaced by `IbisConnection` (internal) + `DialectSpec` registry | +| `BaseIbisOperations` + concrete subclasses | `backends/ibis/operations.py` | Replaced by operations on `IbisBackend` + `DialectSpec` hooks | +| `_DuckDBFamilyOperationsMixin` | `backends/ibis/operations.py` | Implementations become `DialectSpec` hooks | +| `_BaseIbisMixin` | `backends/ibis/operations.py` | Already a deprecated shim | + +### What Stays + +| Component | Location | Role | +|-----------|----------|------| +| `IbisBackend` | `backends/ibis/backend.py` | Single public handle (expanded) | +| `IbisConnection` | `backends/ibis/backend.py` | Internal wrapper — inspection delegation | +| `DialectSpec` registry | `backends/ibis/dialects/_registry.py` | Expanded with operation hooks | +| Module-level operation functions | `backends/ibis/operations.py` | Wired as `DialectSpec` hooks | +| `IcebergBackend` | `backends/iceberg/backend.py` | Unchanged (separate backend) | +| `Backend` protocol | `core/protocol.py` | Updated — `connect()` returns `Self` | +| Inspection model | `core/inspection.py` | Unchanged | +| Settings classes | `core/settings/` | Unchanged | + +## Protocol Changes + +### `Backend` protocol (updated) + +```python +@t.runtime_checkable +class Backend(t.Protocol): + name: str + + def connect(self) -> Self: ... + def close(self) -> Self: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args) -> None: ... + + # Inspection (terminal — return data) + def list_tables(self, namespace: str | None = None) -> list[str]: ... + def list_namespaces(self) -> list[str]: ... + def inspect_table(self, name: str, namespace: str | None = None) -> TableInfo: ... + def inspect_namespace(self, name: str) -> NamespaceInfo: ... + def inspect_catalog(self) -> CatalogInfo: ... +``` + +### `Connection` protocol — REMOVED + +No longer part of the public API. `IbisConnection` is internal. + +## Connection Lifecycle + +```python +# Explicit connect/close +backend = IbisBackend(dialect="sqlite", database=":memory:") +backend.connect() +tables = backend.list_tables() +backend.close() + +# Context manager (recommended) +with IbisBackend(dialect="sqlite", database=":memory:") as backend: + tables = backend.list_tables() + +# Fluent chaining +with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("users", df).create_index("users", ["email"], unique=True) + rows = backend.list_tables() +``` + +`__enter__` calls `connect()` and returns `self`. +`__exit__` calls `close()`. + +Calling any method before `connect()` or after `close()` raises `RuntimeError`. + +## Fluent API + +Methods are categorised into two groups: + +### Fluent (return `self` — chainable mutations) + +| Method | Signature | +|--------|-----------| +| `connect()` | `() -> Self` | +| `close()` | `() -> Self` | +| `create_table()` | `(name, obj, *, schema?, database?, temp?, overwrite?) -> Self` | +| `drop_table()` | `(name, *, database?, force?) -> Self` | +| `create_view()` | `(name, obj, *, database?, overwrite?) -> Self` | +| `drop_view()` | `(name, *, database?, force?) -> Self` | +| `insert()` | `(name, obj, *, database?, overwrite?) -> Self` | +| `upsert()` | `(name, obj, *, conflict_columns, update_columns?, conflict_action?, ...) -> Self` | +| `truncate()` | `(name, *, database?, schema?) -> Self` | +| `rename_table()` | `(old_name, new_name) -> Self` | +| `create_index()` | `(table, columns, *, index_name?, unique?, ...) -> Self` | +| `create_unique_index()` | `(table, columns, *, index_name?, ...) -> Self` | +| `drop_index()` | `(index_name, *, table_name?, database?, if_exists?) -> Self` | + +### Terminal (return data — end of chain) + +| Method | Returns | +|--------|---------| +| `list_tables(namespace?)` | `list[str]` | +| `list_namespaces()` | `list[str]` | +| `table_exists(name, database?)` | `bool` | +| `inspect_table(name, namespace?)` | `TableInfo` | +| `inspect_namespace(name)` | `NamespaceInfo` | +| `inspect_catalog()` | `CatalogInfo` | +| `table(name, *, database?)` | `ir.Table` | +| `run_sql(query, *, schema?, dialect?)` | `ir.Table | None` | +| `run_expr(expr, *, params?, limit?)` | `Any` | +| `to_sql(expr, *, params?, limit?, pretty?)` | `str | None` | +| `index_exists(index_name, *, table_name?, database?)` | `bool` | +| `list_indexes(table_name, *, database?)` | `list[dict]` | +| `ibis_connection()` | raw ibis backend object | +| `get_connection()` | `IbisConnection` (internal wrapper) | + +## DialectSpec Expansion + +`DialectSpec` gains operation hook fields: + +```python +@dataclass(frozen=True) +class DialectSpec: + # Existing + ibis_backend_name: str + connection_mode: str + connection_string_scheme: str + connection_builder: Callable | None = None + get_index_exists_sql: Callable | None = None # existing + get_list_indexes_sql: Callable | None = None # existing + + # New operation hooks + upsert_hook: Callable | None = None + create_index_hook: Callable | None = None + drop_index_hook: Callable | None = None + rename_table_hook: Callable | None = None + + extras: Mapping[str, Any] = field(default_factory=dict) +``` + +### Hook wiring + +The existing `_DuckDBFamilyOperationsMixin` methods become standalone +functions and are wired as hooks: + +| Hook | DuckDB/MotherDuck | SQLite | Others | +|------|-------------------|--------|--------| +| `upsert_hook` | `duckdb_family_upsert` | `duckdb_family_upsert` | `None` | +| `create_index_hook` | `duckdb_family_create_index` | `duckdb_family_create_index` | `None` | +| `drop_index_hook` | `duckdb_family_drop_index` | `duckdb_family_drop_index` | `None` | +| `rename_table_hook` | `None` (not implemented) | `None` (not implemented) | `None` | +| `get_index_exists_sql` | `duckdb_get_index_exists_sql` | `sqlite_get_index_exists_sql` | `None` | +| `get_list_indexes_sql` | `duckdb_get_list_indexes_sql` | `sqlite_get_list_indexes_sql` | `None` | + +`IbisBackend.upsert()` checks `self._spec.upsert_hook`; if `None`, raises +`NotImplementedError(f"Dialect {self.dialect!r} does not support upsert")`. + +### Hook function signatures + +Hooks receive the raw ibis connection as their first argument (not the +`IbisBackend` instance), plus the method's arguments: + +```python +# upsert_hook signature +def duckdb_family_upsert( + ibis_conn: Any, + table_name: str, + df: Any, + *, + conflict_columns: list[str] | str, + update_columns: list[str] | str | None = None, + conflict_action: str = "UPDATE", + update_condition: str | None = None, + database: str | None = None, + schema: str | None = None, +) -> None: ... + +# create_index_hook signature +def duckdb_family_create_index( + ibis_conn: Any, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + unique: bool = False, + index_type: str | None = None, + where_condition: str | None = None, + database: str | None = None, + if_not_exists: bool = True, +) -> None: ... +``` + +Note: hooks return `None`, not `bool`. The `IbisBackend` wrapper converts +the call to fluent `return self`. Errors raise exceptions (no silent +`False` returns). + +### Thin wrapper methods + +Methods that are simple ibis delegations don't need hooks — they work the +same across all dialects: + +- `create_table` → `ibis_conn.create_table(...)` +- `drop_table` → `ibis_conn.drop_table(...)` +- `create_view` → `ibis_conn.create_view(...)` +- `drop_view` → `ibis_conn.drop_view(...)` +- `insert` → `ibis_conn.insert(...)` +- `truncate` → `ibis_conn.truncate_table(...)` +- `table` → `ibis_conn.table(...)` +- `run_sql` → `ibis_conn.sql(...)` +- `run_expr` → `ibis_conn.execute(...)` +- `to_sql` → `ibis_conn.compile(...)` +- `list_tables` → delegates to `IbisConnection.list_tables()` +- `list_namespaces` → delegates to `IbisConnection.list_namespaces()` +- `inspect_*` → delegates to `IbisConnection.inspect_*()` + +## Public API Changes (`__init__.py`) + +### Removed exports + +- `Connection` (protocol removed) +- `ConnectionFactory` +- `OperationsFactory` +- `SettingsFactory` +- `DatabaseUtils` + +### Retained exports + +- `Backend` (protocol, updated) +- `IbisBackend` (expanded) +- `IcebergBackend` +- `CatalogInfo`, `ColumnInfo`, `NamespaceInfo`, `TableInfo` + +## IcebergBackend Impact + +`IcebergBackend` must also satisfy the updated `Backend` protocol +(`connect()` returns `Self`, context manager). This is a minor change — +same pattern as `IbisBackend`. Operations that don't apply (upsert, indexes) +are simply not on the protocol. + +## Test Strategy + +### New tests for `IbisBackend` operations + +Test with SQLite and DuckDB (in-memory, no external services): + +- **Lifecycle**: `connect()` → use → `close()`, context manager, double-close idempotent, use-before-connect raises +- **Fluent API**: chain `create_table().create_index()`, verify returns `self` +- **Thin wrappers**: `create_table`, `drop_table`, `insert`, `list_tables`, `table`, `run_sql` +- **Hook-based operations**: `upsert` (DuckDB), `create_index`/`drop_index`/`index_exists`/`list_indexes` (SQLite + DuckDB) +- **Unsupported operations**: `upsert` on Trino dialect raises `NotImplementedError` +- **Accessor methods**: `ibis_connection()` returns raw ibis, `get_connection()` returns `IbisConnection` + +### Existing tests — rewrite + +Factory test files (`test_connection_factory.py`, `test_operations_factory.py`) +are deleted. `test_database_utils.py` is deleted. Operations tests +(`test_base_ibis_operations.py`, `test_upsert_and_indexes.py`) are rewritten +to use `IbisBackend` directly. `test_backend.py` is expanded. + +### Existing tests — verify unchanged + +- Inspection tests remain valid +- Settings tests remain valid +- Iceberg tests need minor update for protocol change + +## Files Modified or Deleted + +### Modified + +| File | Change | +|------|--------| +| `backends/ibis/backend.py` | Add lifecycle, operations methods, accessor methods | +| `backends/ibis/dialects/_registry.py` | Add operation hook fields to `DialectSpec`, wire hooks | +| `backends/ibis/operations.py` | Extract mixin methods into standalone hook functions; delete class hierarchy | +| `backends/iceberg/backend.py` | Update to return `Self` from `connect()`, add context manager | +| `core/protocol.py` | Update `Backend` protocol, remove `Connection` protocol | +| `__init__.py` | Remove factory/utils exports, remove `Connection` | +| `tests/test_unit/backends/ibis/test_backend.py` | Expand with operations + lifecycle tests | + +### Deleted + +| File | Reason | +|------|--------| +| `core/factories/` (entire directory) | All factories replaced by `IbisBackend` | +| `core/utils.py` | `DatabaseUtils` replaced by `IbisBackend` | +| `core/connection.py` | `BaseDBConnection` no longer needed | +| `backends/ibis/connection.py` | `BaseIbisConnection` + 12 subclasses replaced | +| `tests/test_unit/factories/` | All factory tests | +| `tests/test_unit/test_database_utils.py` | `DatabaseUtils` tests | +| `tests/test_unit/databases/test_database_connections.py` | Legacy connection tests | +| `tests/test_unit/databases/connections/` | Legacy connection lifecycle tests | + +## Error Handling + +- Methods that previously returned `bool` or `None` now raise on failure + (no silent swallowing). The `print(f"Error: ...")` pattern throughout + `BaseIbisOperations` is replaced with proper exception propagation. +- Unsupported operations (hook is `None`) raise `NotImplementedError` + with dialect name in the message. +- Use-before-connect and use-after-close raise `RuntimeError`. + +## Consumer Migration + +Before: +```python +from mountainash_data import DatabaseUtils, ConnectionFactory +conn = DatabaseUtils.create_connection(settings_params) +backend = conn.connect() +ops = DatabaseUtils.create_operations(settings_params) +ops.create_table(backend, "users", df) +ops.upsert(backend, "users", new_df, conflict_columns=["id"]) +tables = ops.list_tables(backend) +``` + +After: +```python +from mountainash_data import IbisBackend +with IbisBackend(settings_params) as backend: + backend.create_table("users", df).upsert("users", new_df, conflict_columns=["id"]) + tables = backend.list_tables() + tbl = backend.ibis_connection().table("users") +``` From 2309da6194fb0edbffe1c495fcc2afaa4a9a7bc6 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 17:52:42 +1000 Subject: [PATCH 05/12] chore(specs): address Codex review findings in backend-as-single-handle spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Keep core/connection.py (BaseDBConnection) — Iceberg depends on it - Document Redshift URL ambiguity and resolution - Add explicit compatibility section (internal package, no deprecation needed) - Add out-of-scope section for Iceberg migration and future dialect hooks Co-Authored-By: Claude Opus 4.6 (1M context) --- ...6-04-27-backend-as-single-handle-design.md | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md b/docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md index 52bd406..a2f14a8 100644 --- a/docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md +++ b/docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md @@ -42,7 +42,7 @@ Both raise `RuntimeError` if not connected. | `OperationsFactory` | `core/factories/operations_factory.py` | Replaced by operations on `IbisBackend` | | `SettingsFactory` | `core/factories/settings_factory.py` | URL detection now in `_SCHEME_TO_DIALECT`; settings creation via `IbisBackend(settings_params)` | | `DatabaseUtils` | `core/utils.py` | Entire class — `IbisBackend` is the API | -| `BaseDBConnection` | `core/connection.py` | Abstract base no longer needed | +| ~~`BaseDBConnection`~~ | ~~`core/connection.py`~~ | **KEPT** — `IcebergConnectionBase` subclasses it; migrate Iceberg separately | | `BaseIbisConnection` + 12 subclasses | `backends/ibis/connection.py` | Replaced by `IbisConnection` (internal) + `DialectSpec` registry | | `BaseIbisOperations` + concrete subclasses | `backends/ibis/operations.py` | Replaced by operations on `IbisBackend` + `DialectSpec` hooks | | `_DuckDBFamilyOperationsMixin` | `backends/ibis/operations.py` | Implementations become `DialectSpec` hooks | @@ -57,6 +57,7 @@ Both raise `RuntimeError` if not connected. | `DialectSpec` registry | `backends/ibis/dialects/_registry.py` | Expanded with operation hooks | | Module-level operation functions | `backends/ibis/operations.py` | Wired as `DialectSpec` hooks | | `IcebergBackend` | `backends/iceberg/backend.py` | Unchanged (separate backend) | +| `BaseDBConnection` | `core/connection.py` | Kept — Iceberg depends on it; migrate separately | | `Backend` protocol | `core/protocol.py` | Updated — `connect()` returns `Self` | | Inspection model | `core/inspection.py` | Unchanged | | Settings classes | `core/settings/` | Unchanged | @@ -321,7 +322,7 @@ to use `IbisBackend` directly. `test_backend.py` is expanded. |------|--------| | `core/factories/` (entire directory) | All factories replaced by `IbisBackend` | | `core/utils.py` | `DatabaseUtils` replaced by `IbisBackend` | -| `core/connection.py` | `BaseDBConnection` no longer needed | +| ~~`core/connection.py`~~ | **KEPT** — Iceberg depends on `BaseDBConnection`; migrate separately | | `backends/ibis/connection.py` | `BaseIbisConnection` + 12 subclasses replaced | | `tests/test_unit/factories/` | All factory tests | | `tests/test_unit/test_database_utils.py` | `DatabaseUtils` tests | @@ -358,3 +359,39 @@ with IbisBackend(settings_params) as backend: tables = backend.list_tables() tbl = backend.ibis_connection().table("users") ``` + +## Redshift URL Ambiguity + +Redshift is registered with `connection_string_scheme="postgres://"` because +it uses the postgres wire protocol. This means `_SCHEME_TO_DIALECT` maps +`postgres://` → `"postgres"` (first writer wins), and Redshift is not +reachable by URL alone. + +**Resolution:** Redshift connections must use either: +- `IbisBackend(dialect="redshift", ...)` — explicit dialect keyword +- `IbisBackend(settings_params)` — settings path with Redshift settings class + +This is an inherent limitation: `postgres://` URLs are ambiguous between +postgres and Redshift. The old `SettingsFactory` had the same problem — it +mapped `postgres://` to postgres, not Redshift. No regression. + +If a dedicated `redshift://` scheme is needed in future, add a `url_schemes` +list field on `DialectSpec` separate from `connection_string_scheme`. + +## Compatibility + +This package is internal to the mountainash-io organisation — there are zero +external consumers. All imports of the removed names (`ConnectionFactory`, +`OperationsFactory`, `SettingsFactory`, `DatabaseUtils`, `Connection`) are +in this repo's own test files, which are rewritten in the same change. + +No deprecation window or major-version bump is needed. The removed exports +are cleaned up atomically: deletion + test migration in the same branch. + +## Out of Scope + +| Item | Reason | +|------|--------| +| Iceberg migration off `BaseDBConnection` | Separate spec — `core/connection.py` kept for now | +| `redshift://` URL scheme | No current need — use settings or `dialect=` | +| Operations for non-DuckDB-family dialects | Add hooks when needed (Snowflake, BigQuery, etc.) | From 0b1be77cf512df0587aec2da233def4be6204ede Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 21:00:12 +1000 Subject: [PATCH 06/12] chore(plans): add backend-as-single-handle implementation plan 7 tasks: protocol+lifecycle, DialectSpec hooks, operations on backend, IcebergBackend update, delete legacy code, rewrite tests, validation. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../2026-04-27-backend-as-single-handle.md | 1636 +++++++++++++++++ 1 file changed, 1636 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-27-backend-as-single-handle.md diff --git a/docs/superpowers/plans/2026-04-27-backend-as-single-handle.md b/docs/superpowers/plans/2026-04-27-backend-as-single-handle.md new file mode 100644 index 0000000..d8ea677 --- /dev/null +++ b/docs/superpowers/plans/2026-04-27-backend-as-single-handle.md @@ -0,0 +1,1636 @@ +# Backend as Single Handle — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `IbisBackend` the single public handle for all ibis interaction — lifecycle, inspection, and operations — then delete all legacy factories, utils, and class hierarchies. + +**Architecture:** `IbisBackend` composes an internal `IbisConnection` for inspection, delegates thin wrapper operations to the raw ibis connection, and dispatches per-dialect operations (upsert, indexes) via callable hooks on `DialectSpec`. Fluent methods return `self`; terminal methods return data. + +**Tech Stack:** Python 3.12, ibis-framework 10.4.0, pytest 8.3.5, hatch + +**Spec:** `docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md` +**Branch:** `feature/settings-aware-ibis-backend` (continuing from Phase 1) +**Test command:** `hatch run test:test-quick` + +--- + +## File Structure + +### Modified + +| File | Responsibility | +|------|----------------| +| `src/mountainash_data/core/protocol.py` | Updated `Backend` protocol (connect→Self, context manager, inspection). Remove `Connection` protocol. | +| `src/mountainash_data/backends/ibis/backend.py` | Lifecycle (`connect`/`close`/context manager), accessor methods, thin wrappers, hook-dispatched operations | +| `src/mountainash_data/backends/ibis/dialects/_registry.py` | New hook fields on `DialectSpec`, wiring to standalone functions | +| `src/mountainash_data/backends/ibis/operations.py` | Extract mixin methods to standalone hook functions. Delete class hierarchy (keep functions only). | +| `src/mountainash_data/backends/iceberg/backend.py` | `connect()` returns `Self`, add `__enter__`/`__exit__`/`close()` | +| `src/mountainash_data/__init__.py` | Remove factory/utils/Connection exports | +| `tests/test_unit/backends/ibis/test_backend.py` | Expand with lifecycle + operations tests | +| `tests/test_unit/test_mountainash_data.py` | Update import assertions for new public API | +| `tests/test_unit/databases/settings/test_settings_parametrized.py` | Replace factory/utils calls with `IbisBackend` | +| `tests/test_integration/test_end_to_end_workflows.py` | Rewrite to use `IbisBackend` | + +### Deleted + +| File/Directory | Reason | +|----------------|--------| +| `src/mountainash_data/core/factories/` | Entire directory — all factories replaced | +| `src/mountainash_data/core/utils.py` | `DatabaseUtils` replaced by `IbisBackend` | +| `src/mountainash_data/backends/ibis/connection.py` | `BaseIbisConnection` + 12 subclasses replaced | +| `tests/test_unit/factories/` | All factory tests | +| `tests/test_unit/test_database_utils.py` | `DatabaseUtils` tests | +| `tests/test_unit/databases/test_database_connections.py` | Legacy connection tests | +| `tests/test_unit/databases/connections/` | Legacy connection lifecycle tests | +| `tests/test_unit/databases/test_ibis_backends.py` | Legacy ibis backend tests | +| `tests/test_unit/databases/operations/` | Legacy operations tests (rewritten in test_backend.py) | + +### Kept (no changes) + +| File | Reason | +|------|--------| +| `src/mountainash_data/core/connection.py` | Iceberg depends on `BaseDBConnection` | +| `src/mountainash_data/core/inspection.py` | Unchanged | +| `src/mountainash_data/core/settings/` | Unchanged | +| `src/mountainash_data/backends/ibis/inspect.py` | Unchanged | + +--- + +### Task 1: Update Backend protocol and add lifecycle to IbisBackend + +**Files:** +- Modify: `src/mountainash_data/core/protocol.py` +- Modify: `src/mountainash_data/backends/ibis/backend.py` +- Test: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing lifecycle tests** + +Add these tests to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + +def test_connect_returns_self(): + """connect() must return the backend instance itself.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + result = backend.connect() + assert result is backend + + +def test_close_returns_self(): + """close() must return the backend instance itself.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + result = backend.close() + assert result is backend + + +def test_context_manager(): + """with IbisBackend(...) as backend: must connect and close.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + assert backend.list_tables() == [] + # After exit, should be closed + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_double_close_is_idempotent(): + """Calling close() twice must not raise.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + backend.close() + backend.close() # Must not raise + + +def test_use_before_connect_raises(): + """Calling methods before connect() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_use_after_close_raises(): + """Calling methods after close() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + backend.close() + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_ibis_connection_accessor(): + """ibis_connection() returns the raw ibis backend object.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + raw = backend.ibis_connection() + assert hasattr(raw, "list_tables") + + +def test_ibis_connection_before_connect_raises(): + """ibis_connection() before connect() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + with pytest.raises(RuntimeError, match="not connected"): + backend.ibis_connection() + + +def test_get_connection_accessor(): + """get_connection() returns our IbisConnection wrapper.""" + from mountainash_data.backends.ibis.backend import IbisConnection + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + conn = backend.get_connection() + assert isinstance(conn, IbisConnection) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_connect_returns_self -v` +Expected: FAIL (connect() returns IbisConnection, not self) + +- [ ] **Step 3: Update Backend protocol** + +Replace the entire contents of `src/mountainash_data/core/protocol.py` with: + +```python +"""Backend protocol. + +This is the structural contract every backend implementation must +satisfy. Implementations are plain classes — there is no inheritance. +""" + +from __future__ import annotations + +import typing as t + +from mountainash_data.core.inspection import ( + CatalogInfo, + NamespaceInfo, + TableInfo, +) + + +@t.runtime_checkable +class Backend(t.Protocol): + """The single handle for interacting with a backend service. + + Backends are constructed with config, connected via connect(), + used for inspection and operations, then closed. + """ + + name: str + + def connect(self) -> t.Self: ... + def close(self) -> t.Self: ... + def __enter__(self) -> t.Self: ... + def __exit__(self, *args: t.Any) -> None: ... + + def list_tables(self, namespace: str | None = None) -> list[str]: ... + def list_namespaces(self) -> list[str]: ... + + def inspect_table( + self, name: str, namespace: str | None = None + ) -> TableInfo: ... + + def inspect_namespace(self, name: str) -> NamespaceInfo: ... + def inspect_catalog(self) -> CatalogInfo: ... +``` + +- [ ] **Step 4: Add lifecycle and accessor methods to IbisBackend** + +In `src/mountainash_data/backends/ibis/backend.py`, make these changes: + +4a. Add `_conn: IbisConnection | None = None` initialisation to each `_init_from_*` method (set `self._conn = None`). + +4b. Replace the existing `connect()` method and add `close()`, `__enter__`, `__exit__`, accessor methods, and a `_require_connected` helper. Replace everything from `def connect(self)` to end of file with: + +```python + def _require_connected(self) -> IbisConnection: + if self._conn is None: + raise RuntimeError( + "IbisBackend is not connected. Call connect() first." + ) + return self._conn + + def connect(self) -> IbisBackend: + """Build a live ibis connection. Returns self for fluent chaining.""" + if self._conn is not None: + return self + if self._spec.connection_builder is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} has no connection_builder configured" + ) + if self._url is not None: + import ibis + ibis_conn = ibis.connect(self._url, **self._config) + else: + cleaned_config = { + k: v for k, v in self._config.items() + if not (isinstance(v, (list, tuple)) and len(v) == 0) + } + ibis_conn = self._spec.connection_builder(**cleaned_config) + self._conn = IbisConnection(ibis_conn, self._spec) + return self + + def close(self) -> IbisBackend: + """Release the connection. Idempotent. Returns self.""" + if self._conn is not None: + self._conn.close() + self._conn = None + return self + + def __enter__(self) -> IbisBackend: + self.connect() + return self + + def __exit__(self, *args: t.Any) -> None: + self.close() + + def ibis_connection(self) -> t.Any: + """Return the raw ibis backend object.""" + return self._require_connected()._ibis_conn + + def get_connection(self) -> IbisConnection: + """Return the internal IbisConnection wrapper.""" + return self._require_connected() + + # --- Inspection (terminal — delegates to IbisConnection) --- + + def list_tables(self, namespace: str | None = None) -> list[str]: + return self._require_connected().list_tables(namespace=namespace) + + def list_namespaces(self) -> list[str]: + return self._require_connected().list_namespaces() + + def inspect_table( + self, name: str, namespace: str | None = None + ) -> TableInfo: + return self._require_connected().inspect_table(name, namespace=namespace) + + def inspect_namespace(self, name: str) -> NamespaceInfo: + return self._require_connected().inspect_namespace(name) + + def inspect_catalog(self) -> CatalogInfo: + return self._require_connected().inspect_catalog() +``` + +4c. In each `_init_from_*` method, add `self._conn = None` after setting `self._config`: +- `_init_from_dialect`: after `self._config = config`, add `self._conn = None` +- `_init_from_url`: after `self._config = config`, add `self._conn = None` +- `_init_from_settings`: after `self._config = driver_kwargs`, add `self._conn = None` + +- [ ] **Step 5: Update existing tests that use the old connect() return** + +In `tests/test_unit/backends/ibis/test_backend.py`, update `test_in_memory_sqlite_connect_and_inspect` — it currently does `conn = backend.connect()` expecting an `IbisConnection`. Change it to use the backend directly: + +```python +def test_in_memory_sqlite_connect_and_inspect(): + """End-to-end test with the only dialect that needs no external service.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + try: + assert backend.list_tables() == [] + finally: + backend.close() +``` + +Update the settings path tests similarly: + +```python +def test_settings_path_sqlite(): + """Construct IbisBackend from SQLite SettingsParameters and connect.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import SQLiteAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=SQLiteAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "sqlite" + backend.connect() + tables = backend.list_tables() + assert isinstance(tables, list) + backend.close() + + +def test_settings_path_duckdb_empty_extensions(): + """DuckDB settings with default EXTENSIONS=[] must not crash ibis.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "duckdb" + backend.connect() # Must not raise — empty-list filter active + backend.close() +``` + +Update URL path tests: + +```python +def test_url_path_sqlite(): + """Construct IbisBackend from sqlite:// URL and connect.""" + backend = IbisBackend("sqlite://") + assert backend.dialect == "sqlite" + backend.connect() + backend.close() + + +def test_url_path_duckdb(): + """Construct IbisBackend from duckdb:// URL and connect.""" + backend = IbisBackend("duckdb://") + assert backend.dialect == "duckdb" + backend.connect() + backend.close() + + +def test_url_path_preserves_database(tmp_path): + """URL database component must reach the driver, not be discarded.""" + db_file = tmp_path / "test.db" + backend = IbisBackend(f"sqlite:///{db_file}") + assert backend.dialect == "sqlite" + backend.connect() + backend.close() + assert db_file.exists() +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py -v` +Expected: ALL PASS + +- [ ] **Step 7: Commit** + +```bash +git add src/mountainash_data/core/protocol.py src/mountainash_data/backends/ibis/backend.py tests/test_unit/backends/ibis/test_backend.py +git commit -m "feat(backend): add lifecycle, context manager, and accessor methods to IbisBackend + +connect() and close() return self for fluent chaining. Context manager +support via __enter__/__exit__. Inspection methods delegate to internal +IbisConnection. Backend protocol updated: connect() returns Self, +Connection protocol removed." +``` + +--- + +### Task 2: Expand DialectSpec with operation hooks and extract standalone functions + +**Files:** +- Modify: `src/mountainash_data/backends/ibis/dialects/_registry.py` +- Modify: `src/mountainash_data/backends/ibis/operations.py` +- Test: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing test for hook presence on DialectSpec** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +# --------------------------------------------------------------------------- +# DialectSpec hooks +# --------------------------------------------------------------------------- + +def test_duckdb_dialect_has_upsert_hook(): + """DuckDB DialectSpec must have upsert_hook wired.""" + spec = DIALECTS["duckdb"] + assert spec.upsert_hook is not None + + +def test_sqlite_dialect_has_create_index_hook(): + """SQLite DialectSpec must have create_index_hook wired.""" + spec = DIALECTS["sqlite"] + assert spec.create_index_hook is not None + + +def test_postgres_dialect_has_no_upsert_hook(): + """Postgres DialectSpec has no upsert_hook (not DuckDB family).""" + spec = DIALECTS["postgres"] + assert spec.upsert_hook is None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_duckdb_dialect_has_upsert_hook -v` +Expected: FAIL (DialectSpec has no `upsert_hook` attribute) + +- [ ] **Step 3: Add hook fields to DialectSpec** + +In `src/mountainash_data/backends/ibis/dialects/_registry.py`, add new type aliases and fields. + +After the existing type aliases (line 27), add: + +```python +UpsertHook = t.Callable[..., None] +CreateIndexHook = t.Callable[..., None] +DropIndexHook = t.Callable[..., None] +RenameTableHook = t.Callable[..., None] +``` + +Add new fields to the `DialectSpec` dataclass, after `get_list_indexes_sql` and before `extras`: + +```python + upsert_hook: t.Optional[UpsertHook] = None + create_index_hook: t.Optional[CreateIndexHook] = None + drop_index_hook: t.Optional[DropIndexHook] = None + rename_table_hook: t.Optional[RenameTableHook] = None +``` + +- [ ] **Step 4: Extract standalone hook functions from operations.py** + +In `src/mountainash_data/backends/ibis/operations.py`, convert the `_DuckDBFamilyOperationsMixin` class methods into standalone functions. Add these after the existing per-dialect SQL functions (after `motherduck_list_tables`, around line 213) and before the `_DuckDBFamilyOperationsMixin` class: + +```python +# =========================================================================== +# STANDALONE HOOK FUNCTIONS +# Extracted from _DuckDBFamilyOperationsMixin for DialectSpec wiring. +# =========================================================================== + +def duckdb_family_create_index( + ibis_conn: t.Any, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + unique: bool = False, + index_type: str | None = None, + where_condition: str | None = None, + database: str | None = None, + if_not_exists: bool = True, +) -> None: + """Create an index using DuckDB/SQLite syntax.""" + columns_list = _normalize_columns(columns) + + if index_name is None: + index_name = _generate_index_name(table_name, columns_list, unique=unique) + + qualified_table = _format_qualified_table(table_name, database=database) + columns_sql = ", ".join(columns_list) + + unique_sql = "UNIQUE " if unique else "" + if_not_exists_sql = "IF NOT EXISTS " if if_not_exists else "" + where_sql = f" WHERE {where_condition}" if where_condition else "" + + if index_type and index_type != CONST_INDEX_TYPE.BTREE: + warnings.warn( + f"Index type {index_type} not supported, using default BTREE" + ) + + create_sql = ( + f"CREATE {unique_sql}INDEX {if_not_exists_sql}{index_name} " + f"ON {qualified_table} ({columns_sql}){where_sql}" + ) + + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(create_sql) + + +def duckdb_family_drop_index( + ibis_conn: t.Any, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + if_exists: bool = True, +) -> None: + """Drop an index using DuckDB/SQLite syntax.""" + if_exists_sql = "IF EXISTS " if if_exists else "" + drop_sql = f"DROP INDEX {if_exists_sql}{index_name}" + + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(drop_sql) + + +def duckdb_family_upsert( + ibis_conn: t.Any, + table_name: str, + df: t.Any, + *, + conflict_columns: list[str] | str, + update_columns: list[str] | str | None = None, + conflict_action: str = CONST_CONFLICT_ACTION.UPDATE, + update_condition: str | None = None, + database: str | None = None, + schema: str | None = None, +) -> None: + """Perform upsert using INSERT ... ON CONFLICT syntax (DuckDB/SQLite).""" + conflict_cols = _normalize_columns(conflict_columns) + all_columns = ma.relation(df).columns + + if update_columns is None: + update_cols = [col for col in all_columns if col not in conflict_cols] + else: + update_cols = _normalize_columns(update_columns) + + tables = ibis_conn.list_tables() + if table_name not in tables: + raise ValueError(f"Target table '{table_name}' does not exist") + + if conflict_action not in [CONST_CONFLICT_ACTION.UPDATE, CONST_CONFLICT_ACTION.NOTHING]: + raise ValueError( + f"conflict_action must be '{CONST_CONFLICT_ACTION.UPDATE}' or " + f"'{CONST_CONFLICT_ACTION.NOTHING}', got '{conflict_action}'" + ) + + if conflict_action == CONST_CONFLICT_ACTION.NOTHING: + if update_cols or update_condition: + warnings.warn( + "update_columns and update_condition are ignored when " + "conflict_action='NOTHING'" + ) + + staging_table = f"temp_upsert_{uuid.uuid4().hex[:8]}" + qualified_table = _format_qualified_table(table_name, database=database, schema=schema) + + all_cols_sql = ", ".join(all_columns) + conflict_cols_sql = ", ".join(conflict_cols) + + if conflict_action == CONST_CONFLICT_ACTION.UPDATE: + if not update_cols: + raise ValueError( + "No columns to update. Either provide update_columns or ensure " + "dataframe has columns beyond conflict_columns" + ) + update_set_sql = ", ".join([f"{col} = EXCLUDED.{col}" for col in update_cols]) + where_sql = f" WHERE {update_condition}" if update_condition else "" + on_conflict_sql = ( + f"ON CONFLICT ({conflict_cols_sql}) DO UPDATE SET {update_set_sql}{where_sql}" + ) + else: + on_conflict_sql = f"ON CONFLICT ({conflict_cols_sql}) DO NOTHING" + + upsert_sql = f""" + INSERT INTO {qualified_table} ({all_cols_sql}) + SELECT {all_cols_sql} FROM {staging_table} + WHERE true + {on_conflict_sql} + """ + + if hasattr(ibis_conn.con, 'register'): + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute("BEGIN TRANSACTION") + cur.register(staging_table, df) + cur.execute(upsert_sql) + cur.unregister(staging_table) + cur.execute("COMMIT") + else: + ibis_conn.create_table(staging_table, df, temp=True, overwrite=True) + try: + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(upsert_sql) + ibis_conn.con.commit() + finally: + try: + ibis_conn.drop_table(staging_table, force=True) + except Exception: + pass +``` + +- [ ] **Step 5: Wire hooks in DIALECTS registry** + +In `src/mountainash_data/backends/ibis/dialects/_registry.py`, add imports for the new hook functions alongside the existing SQL function imports: + +```python +from mountainash_data.backends.ibis.operations import ( # noqa: E402 + duckdb_get_index_exists_sql, + duckdb_get_list_indexes_sql, + sqlite_get_index_exists_sql, + sqlite_get_list_indexes_sql, + motherduck_get_index_exists_sql, + motherduck_get_list_indexes_sql, + duckdb_family_upsert, + duckdb_family_create_index, + duckdb_family_drop_index, +) +``` + +Then add hook fields to the `sqlite`, `duckdb`, and `motherduck` entries in `DIALECTS`: + +For `"sqlite"`: +```python + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, +``` + +For `"duckdb"`: +```python + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, +``` + +For `"motherduck"`: +```python + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, +``` + +- [ ] **Step 6: Run tests** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py -v` +Expected: ALL PASS + +- [ ] **Step 7: Commit** + +```bash +git add src/mountainash_data/backends/ibis/dialects/_registry.py src/mountainash_data/backends/ibis/operations.py tests/test_unit/backends/ibis/test_backend.py +git commit -m "feat(registry): expand DialectSpec with operation hooks + +Add upsert_hook, create_index_hook, drop_index_hook, rename_table_hook +to DialectSpec. Extract standalone hook functions from +_DuckDBFamilyOperationsMixin. Wire DuckDB/SQLite/MotherDuck entries." +``` + +--- + +### Task 3: Add thin wrapper and hook-dispatched operations to IbisBackend + +**Files:** +- Modify: `src/mountainash_data/backends/ibis/backend.py` +- Test: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing tests for thin wrapper operations** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +import polars as pl + +# --------------------------------------------------------------------------- +# Thin wrapper operations (fluent) +# --------------------------------------------------------------------------- + +def test_create_table_returns_self(): + """create_table() must return self for fluent chaining.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + result = backend.create_table("t", {"id": [1, 2]}) + assert result is backend + assert "t" in backend.list_tables() + + +def test_drop_table_returns_self(): + """drop_table() must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.drop_table("t") + assert result is backend + assert "t" not in backend.list_tables() + + +def test_insert_returns_self(): + """insert() must return self.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.insert("t", {"id": [2]}) + assert result is backend + + +def test_truncate_returns_self(): + """truncate() must return self.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.truncate("t") + assert result is backend + + +def test_table_returns_ibis_table(): + """table() must return an ibis table expression.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2]}) + tbl = backend.table("t") + assert tbl is not None + + +def test_run_sql_returns_result(): + """run_sql() must return an ibis table expression.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2, 3]}) + result = backend.run_sql("SELECT COUNT(*) as cnt FROM t") + assert result is not None + + +def test_table_exists_returns_bool(): + """table_exists() must return True/False.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + assert backend.table_exists("nope") is False + backend.create_table("t", {"id": [1]}) + assert backend.table_exists("t") is True + + +def test_fluent_chaining(): + """Multiple fluent calls can be chained.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("a", {"id": [1]}).create_table("b", {"id": [2]}) + assert sorted(backend.list_tables()) == ["a", "b"] +``` + +- [ ] **Step 2: Write failing tests for hook-dispatched operations** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +# --------------------------------------------------------------------------- +# Hook-dispatched operations +# --------------------------------------------------------------------------- + +def test_create_index_returns_self(): + """create_index() via hook must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + result = backend.create_index("t", ["name"]) + assert result is backend + + +def test_create_unique_index_returns_self(): + """create_unique_index() must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + result = backend.create_unique_index("t", ["name"]) + assert result is backend + + +def test_drop_index_returns_self(): + """drop_index() via hook must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + backend.create_index("t", ["name"], index_name="idx_name") + result = backend.drop_index("idx_name") + assert result is backend + + +def test_index_exists(): + """index_exists() must detect created indexes.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + backend.create_index("t", ["id"], index_name="idx_id") + assert backend.index_exists("idx_id") is True + assert backend.index_exists("no_such_idx") is False + + +def test_list_indexes(): + """list_indexes() must return index info.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + backend.create_index("t", ["id"], index_name="idx_id") + indexes = backend.list_indexes("t") + assert isinstance(indexes, list) + assert len(indexes) >= 1 + + +def test_upsert_duckdb(): + """upsert() must work on DuckDB via hook.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "val": [10, 20]}) + backend.create_table("t", initial) + backend.create_unique_index("t", ["id"]) + + update = pl.DataFrame({"id": [2, 3], "val": [25, 30]}) + result = backend.upsert("t", update, conflict_columns=["id"]) + assert result is backend + + count_result = backend.run_sql("SELECT COUNT(*) as cnt FROM t") + count = count_result.to_polars()["cnt"][0] + assert count == 3 + + +def test_upsert_unsupported_dialect_raises(): + """upsert() on a dialect without upsert_hook must raise NotImplementedError.""" + backend = IbisBackend(dialect="postgres") + # Don't connect (can't connect to postgres anyway) — just check the method + backend._conn = type("FakeConn", (), {"_ibis_conn": None, "_dialect_spec": DIALECTS["postgres"], "_closed": False})() + with pytest.raises(NotImplementedError, match="does not support upsert"): + backend.upsert("t", {}, conflict_columns=["id"]) +``` + +- [ ] **Step 3: Run tests to verify they fail** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_create_table_returns_self -v` +Expected: FAIL (IbisBackend has no `create_table` method) + +- [ ] **Step 4: Add thin wrapper operations to IbisBackend** + +In `src/mountainash_data/backends/ibis/backend.py`, add these methods to the `IbisBackend` class after the inspection methods: + +```python + # --- Thin wrapper operations (fluent — return self) --- + + def create_table( + self, + name: str, + obj: t.Any, + *, + schema: t.Any | None = None, + database: str | None = None, + temp: bool = False, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.create_table( + name, obj=obj, schema=schema, database=database, + temp=temp, overwrite=overwrite, + ) + return self + + def drop_table( + self, + name: str, + *, + database: str | None = None, + force: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.drop_table(name, database=database, force=force) + return self + + def create_view( + self, + name: str, + obj: t.Any, + *, + database: str | None = None, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.create_view(name, obj=obj, database=database, overwrite=overwrite) + return self + + def drop_view( + self, + name: str, + *, + database: str | None = None, + force: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.drop_view(name, database=database, force=force) + return self + + def insert( + self, + name: str, + obj: t.Any, + *, + database: str | None = None, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.insert(name, obj=obj, database=database, overwrite=overwrite) + return self + + def truncate( + self, + name: str, + *, + database: str | None = None, + schema: str | None = None, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.truncate_table(name, schema=schema, database=database) + return self + + def rename_table(self, old_name: str, new_name: str) -> IbisBackend: + if self._spec.rename_table_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support rename_table" + ) + conn = self._require_connected() + self._spec.rename_table_hook(conn._ibis_conn, old_name, new_name) + return self + + # --- Terminal operations (return data) --- + + def table(self, name: str, *, database: str | None = None) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.table(name, database=database) + + def table_exists( + self, name: str, database: str | None = None + ) -> bool: + tables = self.list_tables() + return name in tables + + def run_sql( + self, + query: str, + *, + schema: t.Any | None = None, + dialect: str | None = None, + ) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.sql(query, schema=schema, dialect=dialect) + + def run_expr( + self, + expr: t.Any, + *, + params: dict | None = None, + limit: str | None = "default", + **kwargs: t.Any, + ) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.execute(expr, params=params, limit=limit, **kwargs) + + def to_sql( + self, + expr: t.Any, + *, + params: t.Any = None, + limit: str | None = None, + pretty: bool = False, + **kwargs: t.Any, + ) -> str | None: + conn = self._require_connected() + return conn._ibis_conn.compile(expr, params=params, limit=limit, pretty=pretty, **kwargs) +``` + +- [ ] **Step 5: Add hook-dispatched operations to IbisBackend** + +Continue adding methods in `src/mountainash_data/backends/ibis/backend.py`: + +```python + # --- Hook-dispatched operations (fluent — return self) --- + + def upsert( + self, + name: str, + obj: t.Any, + *, + conflict_columns: list[str] | str, + update_columns: list[str] | str | None = None, + conflict_action: str = "UPDATE", + update_condition: str | None = None, + database: str | None = None, + schema: str | None = None, + ) -> IbisBackend: + if self._spec.upsert_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support upsert" + ) + conn = self._require_connected() + self._spec.upsert_hook( + conn._ibis_conn, name, obj, + conflict_columns=conflict_columns, + update_columns=update_columns, + conflict_action=conflict_action, + update_condition=update_condition, + database=database, + schema=schema, + ) + return self + + def create_index( + self, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + unique: bool = False, + index_type: str | None = None, + where_condition: str | None = None, + database: str | None = None, + if_not_exists: bool = True, + ) -> IbisBackend: + if self._spec.create_index_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support create_index" + ) + conn = self._require_connected() + self._spec.create_index_hook( + conn._ibis_conn, table_name, columns, + index_name=index_name, unique=unique, index_type=index_type, + where_condition=where_condition, database=database, + if_not_exists=if_not_exists, + ) + return self + + def create_unique_index( + self, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + where_condition: str | None = None, + database: str | None = None, + ) -> IbisBackend: + return self.create_index( + table_name, columns, + index_name=index_name, unique=True, + where_condition=where_condition, database=database, + ) + + def drop_index( + self, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + if_exists: bool = True, + ) -> IbisBackend: + if self._spec.drop_index_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support drop_index" + ) + conn = self._require_connected() + self._spec.drop_index_hook( + conn._ibis_conn, index_name, + table_name=table_name, database=database, if_exists=if_exists, + ) + return self + + def index_exists( + self, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + ) -> bool: + if self._spec.get_index_exists_sql is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support index_exists" + ) + conn = self._require_connected() + check_sql = self._spec.get_index_exists_sql(index_name, table_name, database) + result = conn._ibis_conn.sql(check_sql) + if result is None: + return False + import mountainash as ma + count = ma.relation(result).to_dict()["count"][0] + return count > 0 + + def list_indexes( + self, + table_name: str, + *, + database: str | None = None, + ) -> list[dict]: + if self._spec.get_list_indexes_sql is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support list_indexes" + ) + conn = self._require_connected() + list_sql = self._spec.get_list_indexes_sql(table_name, database) + result = conn._ibis_conn.sql(list_sql) + if result is None: + return [] + import mountainash as ma + return ma.relation(result).to_dicts() +``` + +- [ ] **Step 6: Run all backend tests** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py -v` +Expected: ALL PASS + +- [ ] **Step 7: Commit** + +```bash +git add src/mountainash_data/backends/ibis/backend.py tests/test_unit/backends/ibis/test_backend.py +git commit -m "feat(backend): add thin wrapper and hook-dispatched operations + +Fluent methods (create_table, insert, upsert, create_index, etc.) return +self. Terminal methods (table, run_sql, list_indexes, etc.) return data. +Hook dispatch: upsert/index operations via DialectSpec callables." +``` + +--- + +### Task 4: Update IcebergBackend for protocol compliance + +**Files:** +- Modify: `src/mountainash_data/backends/iceberg/backend.py` +- Test: Run existing iceberg tests + +- [ ] **Step 1: Update IcebergBackend** + +Replace the contents of `src/mountainash_data/backends/iceberg/backend.py` with: + +```python +"""IcebergBackend — implements core.protocol.Backend for iceberg catalogs.""" + +from __future__ import annotations + +import typing as t + +from mountainash_data.backends.iceberg.catalogs.rest import IcebergRestConnection +from mountainash_data.backends.iceberg.connection import IcebergConnectionBase + + +_CATALOG_REGISTRY: dict[str, type[IcebergConnectionBase]] = { + "rest": IcebergRestConnection, +} + + +class IcebergBackend: + """Iceberg backend — single entry point for iceberg catalog interaction. + + Construction takes a catalog type (e.g. ``'rest'``) and config kwargs. + ``connect()`` returns ``self``. Use as a context manager. + """ + + name = "iceberg" + + def __init__(self, catalog: str, **config: t.Any) -> None: + if catalog not in _CATALOG_REGISTRY: + raise KeyError( + f"Unknown iceberg catalog type {catalog!r}. " + f"Available: {sorted(_CATALOG_REGISTRY)}" + ) + self._catalog_cls = _CATALOG_REGISTRY[catalog] + self._config = config + self._conn: IcebergConnectionBase | None = None + + def connect(self) -> IcebergBackend: + """Open a connection. Returns self for fluent chaining.""" + if self._conn is None: + self._conn = self._catalog_cls(**self._config) + return self + + def close(self) -> IcebergBackend: + """Release the connection. Idempotent. Returns self.""" + if self._conn is not None: + if hasattr(self._conn, "close"): + self._conn.close() + self._conn = None + return self + + def __enter__(self) -> IcebergBackend: + self.connect() + return self + + def __exit__(self, *args: t.Any) -> None: + self.close() + + def _require_connected(self) -> IcebergConnectionBase: + if self._conn is None: + raise RuntimeError( + "IcebergBackend is not connected. Call connect() first." + ) + return self._conn + + def list_tables(self, namespace: str | None = None) -> list[str]: + return self._require_connected().list_tables(namespace=namespace) + + def list_namespaces(self) -> list[str]: + return self._require_connected().list_namespaces() + + def inspect_table( + self, name: str, namespace: str | None = None + ) -> t.Any: + return self._require_connected().inspect_table(name, namespace=namespace) + + def inspect_namespace(self, name: str) -> t.Any: + return self._require_connected().inspect_namespace(name) + + def inspect_catalog(self) -> t.Any: + return self._require_connected().inspect_catalog() +``` + +- [ ] **Step 2: Run iceberg tests (if any pass without external services)** + +Run: `hatch run test:test-target tests/test_unit/backends/iceberg/ -v` +Expected: PASS (or skip if pyiceberg not installed) + +- [ ] **Step 3: Commit** + +```bash +git add src/mountainash_data/backends/iceberg/backend.py +git commit -m "feat(iceberg): update IcebergBackend for new Backend protocol + +connect() returns self, context manager support, inspection methods +delegate to internal connection." +``` + +--- + +### Task 5: Delete legacy code and update public API + +**Files:** +- Delete: `src/mountainash_data/core/factories/` (entire directory) +- Delete: `src/mountainash_data/core/utils.py` +- Delete: `src/mountainash_data/backends/ibis/connection.py` +- Modify: `src/mountainash_data/__init__.py` +- Modify: `src/mountainash_data/backends/ibis/operations.py` (delete class hierarchy, keep functions) + +- [ ] **Step 1: Delete factory directory, utils, and legacy connection module** + +```bash +rm -rf src/mountainash_data/core/factories/ +rm src/mountainash_data/core/utils.py +rm src/mountainash_data/backends/ibis/connection.py +``` + +- [ ] **Step 2: Clean up operations.py — delete class hierarchy, keep hook functions** + +In `src/mountainash_data/backends/ibis/operations.py`, delete everything from the `_DuckDBFamilyOperationsMixin` class definition onwards (from line ~221 to end of file). This removes: +- `_DuckDBFamilyOperationsMixin` class +- `_BaseIbisMixin` compatibility shim +- `BaseIbisOperations` abstract class +- All concrete operations subclasses (`DuckDB_IbisOperations`, `SQLite_IbisOperations`, etc.) + +Keep: +- All imports at the top +- All module-level helper functions (`_generate_index_name`, `_format_qualified_table`, `_normalize_columns`) +- All per-dialect SQL functions (`duckdb_get_index_exists_sql`, `sqlite_get_index_exists_sql`, etc.) +- All standalone hook functions (`duckdb_family_create_index`, `duckdb_family_drop_index`, `duckdb_family_upsert`) +- The `motherduck_list_tables` function + +Remove the imports that are only used by the deleted classes. After cleanup, the remaining imports should be: + +```python +import typing as t +import contextlib +import warnings +import uuid + +import mountainash as ma + +from mountainash_data.core.constants import ( + CONST_CONFLICT_ACTION, + CONST_INDEX_TYPE, +) +``` + +Remove: `from abc import abstractmethod, ABC`, `import ibis`, `import ibis.expr.types.relations as ir`, `from ibis.expr.schema import SchemaLike`, `from ibis.backends.sql import SQLBackend`, `from mountainash_settings import SettingsParameters`, `from mountainash_data.core.constants import CONST_DB_BACKEND`. + +- [ ] **Step 3: Update `__init__.py`** + +Replace `src/mountainash_data/__init__.py` with: + +```python +"""mountainash-data: physical access to backend data services. + +Public API: + Backend — protocol (core.protocol) + IbisBackend — ibis-style relational backends (backends.ibis.backend) + IcebergBackend — iceberg-style table-format catalogs (backends.iceberg.backend) + CatalogInfo, NamespaceInfo, TableInfo, ColumnInfo — inspection model +""" + +from mountainash_data.__version__ import __version__ +from mountainash_data.core.protocol import Backend +from mountainash_data.core.inspection import ( + CatalogInfo, + ColumnInfo, + NamespaceInfo, + TableInfo, +) +from mountainash_data.backends.ibis.backend import IbisBackend + +try: + from mountainash_data.backends.iceberg.backend import IcebergBackend +except ImportError: + IcebergBackend = None # type: ignore[assignment,misc] + +__all__ = [ + "__version__", + "Backend", + "CatalogInfo", + "ColumnInfo", + "NamespaceInfo", + "TableInfo", + "IbisBackend", + "IcebergBackend", +] +``` + +- [ ] **Step 4: Verify import works** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py -v` +Expected: ALL PASS (the backend tests don't import deleted modules) + +- [ ] **Step 5: Commit** + +```bash +git add -A +git commit -m "refactor: delete legacy factories, DatabaseUtils, and connection hierarchies + +Remove ConnectionFactory, OperationsFactory, SettingsFactory, DatabaseUtils, +BaseIbisConnection + 12 subclasses, BaseIbisOperations + concrete subclasses. +Keep BaseDBConnection (Iceberg depends on it). +Update __init__.py to export only Backend, IbisBackend, IcebergBackend, +and inspection model." +``` + +--- + +### Task 6: Delete legacy test files + +**Files:** +- Delete: `tests/test_unit/factories/` +- Delete: `tests/test_unit/test_database_utils.py` +- Delete: `tests/test_unit/databases/test_database_connections.py` +- Delete: `tests/test_unit/databases/connections/` +- Delete: `tests/test_unit/databases/test_ibis_backends.py` +- Delete: `tests/test_unit/databases/operations/` + +- [ ] **Step 1: Delete legacy test files and directories** + +```bash +rm -rf tests/test_unit/factories/ +rm tests/test_unit/test_database_utils.py +rm tests/test_unit/databases/test_database_connections.py +rm -rf tests/test_unit/databases/connections/ +rm tests/test_unit/databases/test_ibis_backends.py +rm -rf tests/test_unit/databases/operations/ +``` + +- [ ] **Step 2: Update test_mountainash_data.py** + +Replace `tests/test_unit/test_mountainash_data.py` with: + +```python +"""Tests for main mountainash_data package.""" + +import pytest +import mountainash_data + + +class TestPackageImports: + """Test package-level imports and structure.""" + + def test_version_import(self): + assert hasattr(mountainash_data, '__version__') + assert isinstance(mountainash_data.__version__, str) + assert len(mountainash_data.__version__) > 0 + + def test_version_format(self): + version_parts = mountainash_data.__version__.split('.') + assert len(version_parts) >= 2 + assert version_parts[0].isdigit() + assert version_parts[1].isdigit() + + def test_core_imports_available(self): + from mountainash_data.core.connection import BaseDBConnection + assert BaseDBConnection is not None + + +class TestPackageStructure: + """Test package structure and organization.""" + + def test_package_has_init(self): + assert hasattr(mountainash_data, '__file__') + + def test_new_submodules_exist(self): + import mountainash_data.core + import mountainash_data.backends + assert hasattr(mountainash_data, 'core') + assert hasattr(mountainash_data, 'backends') + + def test_public_api_ibis_backend(self): + from mountainash_data import IbisBackend + assert IbisBackend is not None + + def test_public_api_backend_protocol(self): + from mountainash_data import Backend + assert Backend is not None + + def test_public_api_inspection_model(self): + from mountainash_data import CatalogInfo, ColumnInfo, NamespaceInfo, TableInfo + assert CatalogInfo is not None + assert ColumnInfo is not None + assert NamespaceInfo is not None + assert TableInfo is not None + + def test_removed_exports_not_available(self): + assert not hasattr(mountainash_data, 'ConnectionFactory') + assert not hasattr(mountainash_data, 'OperationsFactory') + assert not hasattr(mountainash_data, 'SettingsFactory') + assert not hasattr(mountainash_data, 'DatabaseUtils') +``` + +- [ ] **Step 3: Update test_settings_parametrized.py** + +In `tests/test_unit/databases/settings/test_settings_parametrized.py`, find the tests that use `ConnectionFactory` and `DatabaseUtils` (around lines 148-175) and replace them with `IbisBackend` equivalents: + +Replace the `ConnectionFactory` test block (around line 148) with: + +```python + def test_settings_work_with_ibis_backend(self, settings_params): + """Test that settings work with IbisBackend.""" + from mountainash_data.backends.ibis.backend import IbisBackend + backend = IbisBackend(settings_params) + assert backend.dialect is not None +``` + +Replace the `DatabaseUtils` test block (around line 163) with: + +```python + def test_settings_work_with_ibis_backend_connect(self, settings_params): + """Test that settings can create a connected backend via IbisBackend.""" + from mountainash_data.backends.ibis.backend import IbisBackend + backend = IbisBackend(settings_params) + backend.connect() + tables = backend.list_tables() + assert isinstance(tables, list) + backend.close() +``` + +- [ ] **Step 4: Rewrite integration tests** + +Replace `tests/test_integration/test_end_to_end_workflows.py` with: + +```python +"""End-to-end integration tests using IbisBackend.""" + +import pytest +import polars as pl +from mountainash_data.backends.ibis.backend import IbisBackend +from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings +from mountainash_settings import SettingsParameters + + +@pytest.mark.integration +class TestIbisBackendWorkflow: + """Test complete workflows through IbisBackend.""" + + def test_sqlite_dialect_workflow(self): + """Full workflow with dialect= keyword.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("users", {"id": [1, 2], "name": ["a", "b"]}) + assert "users" in backend.list_tables() + tbl = backend.table("users") + assert tbl is not None + + def test_sqlite_url_workflow(self, tmp_path): + """Full workflow from URL.""" + db_file = tmp_path / "test.db" + with IbisBackend(f"sqlite:///{db_file}") as backend: + backend.create_table("t", {"id": [1, 2, 3]}) + assert "t" in backend.list_tables() + assert db_file.exists() + + def test_duckdb_settings_workflow(self): + """Full workflow from SettingsParameters.""" + from mountainash_data.core.settings import NoAuth + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + with IbisBackend(params) as backend: + backend.create_table("t", {"id": [1, 2]}) + assert "t" in backend.list_tables() + + def test_fluent_chaining_workflow(self): + """Fluent API chaining across multiple operations.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + ( + backend + .create_table("users", {"id": [1], "email": ["a@b.com"]}) + .create_index("users", ["email"], unique=True) + .create_table("orders", {"id": [1], "user_id": [1]}) + ) + assert sorted(backend.list_tables()) == ["orders", "users"] + + def test_inspect_workflow(self): + """Inspection methods work through the backend.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + info = backend.inspect_table("t") + assert info.name == "t" + assert len(info.columns) == 2 + + def test_ibis_connection_seam(self): + """ibis_connection() provides the seam to mountainash-expressions.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2]}) + raw = backend.ibis_connection() + tbl = raw.table("t") + assert tbl is not None + + +@pytest.mark.integration +class TestDuckDBOperationsWorkflow: + """Test DuckDB-specific operations (upsert, indexes) through IbisBackend.""" + + def test_upsert_insert_new_rows(self): + """Upsert inserts new rows when no conflicts exist.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"]}) + backend.create_table("users", initial) + backend.create_unique_index("users", ["id"]) + + new_data = pl.DataFrame({"id": [3, 4], "name": ["Charlie", "Diana"]}) + backend.upsert("users", new_data, conflict_columns=["id"]) + + count = backend.run_sql("SELECT COUNT(*) as cnt FROM users").to_polars()["cnt"][0] + assert count == 4 + + def test_upsert_update_existing_rows(self): + """Upsert updates existing rows on conflict.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"], "score": [100, 200]}) + backend.create_table("users", initial) + backend.create_unique_index("users", ["id"]) + + update = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"], "score": [150, 250]}) + backend.upsert("users", update, conflict_columns=["id"]) + + result = backend.run_sql("SELECT score FROM users ORDER BY id").to_polars() + assert list(result["score"]) == [150, 250] + + def test_index_lifecycle(self): + """Create, check, list, drop index.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2], "name": ["a", "b"]}) + + backend.create_index("t", ["name"], index_name="idx_name") + assert backend.index_exists("idx_name") is True + + indexes = backend.list_indexes("t") + assert any(idx.get("name") == "idx_name" for idx in indexes) + + backend.drop_index("idx_name") + assert backend.index_exists("idx_name") is False +``` + +- [ ] **Step 5: Run full test suite** + +Run: `hatch run test:test-quick` +Expected: ALL PASS (some tests will have been removed, count should be lower but zero failures) + +- [ ] **Step 6: Commit** + +```bash +git add -A +git commit -m "test: rewrite all tests to use IbisBackend, delete legacy test files + +Delete factory tests, DatabaseUtils tests, legacy connection tests, +and legacy operations tests. Rewrite integration tests and settings +parametrized tests. Update package structure tests." +``` + +--- + +### Task 7: Full test suite validation and cleanup + +**Files:** +- Verify: All files + +- [ ] **Step 1: Run full test suite** + +Run: `hatch run test:test-quick` +Expected: ALL PASS + +- [ ] **Step 2: Check for any remaining imports of deleted modules** + +```bash +grep -rn "from mountainash_data.core.factories" src/ tests/ --include="*.py" +grep -rn "from mountainash_data.core.utils" src/ tests/ --include="*.py" +grep -rn "from mountainash_data.backends.ibis.connection import" src/ tests/ --include="*.py" +grep -rn "ConnectionFactory\|OperationsFactory\|SettingsFactory\|DatabaseUtils" src/ tests/ --include="*.py" +grep -rn "BaseIbisOperations\|BaseIbisConnection" src/ tests/ --include="*.py" +``` + +Expected: No matches in src/ or tests/ (only in docs/ specs/plans which is fine). + +- [ ] **Step 3: Fix any remaining references found in step 2** + +If any stale imports are found, update them. Common places to check: +- `conftest.py` files +- `__init__.py` files in test directories +- Fixture files in `tests/fixtures/` + +- [ ] **Step 4: Run full test suite one final time** + +Run: `hatch run test:test-quick` +Expected: ALL PASS + +- [ ] **Step 5: Commit any cleanup** + +```bash +git add -A +git commit -m "chore: clean up stale imports and references to deleted modules" +``` + +(Skip this commit if no changes were needed.) + +--- + +## Self-Review + +**Spec coverage check:** +- ✅ IbisBackend lifecycle (connect/close/context manager) — Task 1 +- ✅ Fluent API (return self) — Task 3 +- ✅ Terminal methods (return data) — Task 3 +- ✅ DialectSpec hooks — Task 2 +- ✅ Hook-dispatched operations (upsert, indexes) — Task 3 +- ✅ Thin wrapper operations — Task 3 +- ✅ Accessor methods (ibis_connection, get_connection) — Task 1 +- ✅ Protocol update (Backend, remove Connection) — Task 1 +- ✅ IcebergBackend update — Task 4 +- ✅ Delete factories/utils — Task 5 +- ✅ Delete legacy hierarchies — Task 5 +- ✅ Public API cleanup — Task 5 +- ✅ Test rewrite — Task 6 +- ✅ BaseDBConnection kept — Task 5 (only deletes ibis connection.py, not core/connection.py) +- ✅ Error handling (no silent bool/None, NotImplementedError for unsupported) — Task 3 + +**Placeholder scan:** No TBD/TODO/placeholders found. + +**Type consistency:** `IbisBackend` return type used consistently across all tasks. Hook function signatures match between Task 2 (extraction) and Task 3 (dispatch). From b85ece20271054ca83bd9e527c5cbe4d40f16d8a Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 21:48:50 +1000 Subject: [PATCH 07/12] feat(backend): add lifecycle, context manager, and accessor methods to IbisBackend connect() and close() return self for fluent chaining. Context manager support via __enter__/__exit__. Inspection methods delegate to internal IbisConnection. Backend protocol updated: connect() returns Self, Connection protocol removed. Co-Authored-By: Claude Sonnet 4.6 --- src/mountainash_data/__init__.py | 5 +- src/mountainash_data/backends/ibis/backend.py | 65 ++++++++-- .../backends/iceberg/backend.py | 3 +- src/mountainash_data/core/protocol.py | 59 +++------ tests/test_unit/backends/ibis/test_backend.py | 116 +++++++++++++----- 5 files changed, 164 insertions(+), 84 deletions(-) diff --git a/src/mountainash_data/__init__.py b/src/mountainash_data/__init__.py index 2d91947..c1a0dca 100644 --- a/src/mountainash_data/__init__.py +++ b/src/mountainash_data/__init__.py @@ -1,7 +1,7 @@ """mountainash-data: physical access to backend data services. Public API: - Backend, Connection — protocols (core.protocol) + Backend — protocol (core.protocol) IbisBackend — ibis-style relational backends (backends.ibis.backend) IcebergBackend — iceberg-style table-format catalogs (backends.iceberg.backend) CatalogInfo, NamespaceInfo, TableInfo, ColumnInfo — inspection model @@ -11,7 +11,7 @@ """ from mountainash_data.__version__ import __version__ -from mountainash_data.core.protocol import Backend, Connection +from mountainash_data.core.protocol import Backend from mountainash_data.core.inspection import ( CatalogInfo, ColumnInfo, @@ -37,7 +37,6 @@ __all__ = [ "__version__", "Backend", - "Connection", "CatalogInfo", "ColumnInfo", "NamespaceInfo", diff --git a/src/mountainash_data/backends/ibis/backend.py b/src/mountainash_data/backends/ibis/backend.py index e1bf456..a738797 100644 --- a/src/mountainash_data/backends/ibis/backend.py +++ b/src/mountainash_data/backends/ibis/backend.py @@ -195,6 +195,7 @@ def _init_from_dialect( self._spec: DialectSpec = DIALECTS[dialect_name] self._url: str | None = None self._config = config + self._conn: IbisConnection | None = None def _init_from_url( self, url: str, config: dict[str, t.Any] @@ -218,6 +219,7 @@ def _init_from_url( self._spec = DIALECTS[resolved_dialect] self._url = url self._config = config + self._conn: IbisConnection | None = None def _init_from_settings( self, settings_params: t.Any, config: dict[str, t.Any] @@ -244,25 +246,72 @@ def _init_from_settings( self._spec = DIALECTS[resolved_dialect] self._url = None self._config = driver_kwargs + self._conn: IbisConnection | None = None - def connect(self) -> IbisConnection: - """Build and return a live ibis connection.""" + def _require_connected(self) -> IbisConnection: + if self._conn is None: + raise RuntimeError( + "IbisBackend is not connected. Call connect() first." + ) + return self._conn + + def connect(self) -> IbisBackend: + """Build a live ibis connection. Returns self for fluent chaining.""" + if self._conn is not None: + return self if self._spec.connection_builder is None: raise NotImplementedError( f"Dialect {self.dialect!r} has no connection_builder configured" ) if self._url is not None: - # URL path: delegate directly to ibis.connect() which - # natively handles all URL forms and preserves all URL - # components (host, port, credentials, database, query params). import ibis ibis_conn = ibis.connect(self._url, **self._config) else: - # Settings/dialect path: go through the dialect builder - # with empty-list normalization (e.g. DuckDB extensions=[]). cleaned_config = { k: v for k, v in self._config.items() if not (isinstance(v, (list, tuple)) and len(v) == 0) } ibis_conn = self._spec.connection_builder(**cleaned_config) - return IbisConnection(ibis_conn, self._spec) + self._conn = IbisConnection(ibis_conn, self._spec) + return self + + def close(self) -> IbisBackend: + """Release the connection. Idempotent. Returns self.""" + if self._conn is not None: + self._conn.close() + self._conn = None + return self + + def __enter__(self) -> IbisBackend: + self.connect() + return self + + def __exit__(self, *args: t.Any) -> None: + self.close() + + def ibis_connection(self) -> t.Any: + """Return the raw ibis backend object.""" + return self._require_connected()._ibis_conn + + def get_connection(self) -> IbisConnection: + """Return the internal IbisConnection wrapper.""" + return self._require_connected() + + # --- Inspection (terminal — delegates to IbisConnection) --- + + def list_tables(self, namespace: str | None = None) -> list[str]: + return self._require_connected().list_tables(namespace=namespace) + + def list_namespaces(self) -> list[str]: + return self._require_connected().list_namespaces() + + def inspect_table( + self, name: str, namespace: str | None = None + ) -> TableInfo: + return self._require_connected().inspect_table(name, namespace=namespace) + + def inspect_namespace(self, name: str) -> NamespaceInfo: + return self._require_connected().inspect_namespace(name) + + def inspect_catalog(self) -> CatalogInfo: + return self._require_connected().inspect_catalog() diff --git a/src/mountainash_data/backends/iceberg/backend.py b/src/mountainash_data/backends/iceberg/backend.py index e5555d7..00a5e9a 100644 --- a/src/mountainash_data/backends/iceberg/backend.py +++ b/src/mountainash_data/backends/iceberg/backend.py @@ -11,7 +11,6 @@ from mountainash_data.backends.iceberg.catalogs.rest import IcebergRestConnection from mountainash_data.backends.iceberg.connection import IcebergConnectionBase -from mountainash_data.core.protocol import Connection _CATALOG_REGISTRY: dict[str, type[IcebergConnectionBase]] = { @@ -55,7 +54,7 @@ def __init__(self, catalog: str, **config: t.Any) -> None: self._catalog_cls = _CATALOG_REGISTRY[catalog] self._config = config - def connect(self) -> Connection: + def connect(self) -> IcebergConnectionBase: """Open a connection. Caller is responsible for closing it. Note: The legacy IcebergConnectionBase requires a diff --git a/src/mountainash_data/core/protocol.py b/src/mountainash_data/core/protocol.py index b528bc9..26c60fb 100644 --- a/src/mountainash_data/core/protocol.py +++ b/src/mountainash_data/core/protocol.py @@ -1,4 +1,4 @@ -"""Backend and Connection protocols. +"""Backend protocol. This is the structural contract every backend implementation must satisfy. Implementations are plain classes — there is no inheritance. @@ -16,53 +16,26 @@ @t.runtime_checkable -class Connection(t.Protocol): - """A live, owned connection to a backend. +class Backend(t.Protocol): + """The single handle for interacting with a backend service. - Connections are obtained by calling Backend.connect(). They expose - physical introspection and lifecycle methods. Logical query - construction is the job of mountainash-expressions, reached via - to_relation() on backends that support it. + Backends are constructed with config, connected via connect(), + used for inspection and operations, then closed. """ - def list_namespaces(self) -> list[str]: - """Return the names of all namespaces (schemas) visible to this connection.""" - ... + name: str + + def connect(self) -> t.Self: ... + def close(self) -> t.Self: ... + def __enter__(self) -> t.Self: ... + def __exit__(self, *args: t.Any) -> None: ... - def list_tables(self, namespace: str | None = None) -> list[str]: - """Return the names of tables in the given namespace.""" - ... + def list_tables(self, namespace: str | None = None) -> list[str]: ... + def list_namespaces(self) -> list[str]: ... def inspect_table( self, name: str, namespace: str | None = None - ) -> TableInfo: - """Return shared-model metadata for one table.""" - ... - - def inspect_namespace(self, name: str) -> NamespaceInfo: - """Return shared-model metadata for one namespace.""" - ... - - def inspect_catalog(self) -> CatalogInfo: - """Return shared-model metadata for the connection's catalog.""" - ... - - def close(self) -> None: - """Release the connection. Idempotent.""" - ... - - -@t.runtime_checkable -class Backend(t.Protocol): - """A factory for Connections to a particular backend service. - - Backends are constructed with config and are stateless from the - consumer's perspective. State lives on the Connection returned by - connect(). - """ - - name: str + ) -> TableInfo: ... - def connect(self) -> Connection: - """Open a connection. Caller is responsible for closing it.""" - ... + def inspect_namespace(self, name: str) -> NamespaceInfo: ... + def inspect_catalog(self) -> CatalogInfo: ... diff --git a/tests/test_unit/backends/ibis/test_backend.py b/tests/test_unit/backends/ibis/test_backend.py index f07db8b..2e065cb 100644 --- a/tests/test_unit/backends/ibis/test_backend.py +++ b/tests/test_unit/backends/ibis/test_backend.py @@ -27,12 +27,11 @@ def test_all_registered_dialects_construct(): def test_in_memory_sqlite_connect_and_inspect(): """End-to-end test with the only dialect that needs no external service.""" backend = IbisBackend(dialect="sqlite", database=":memory:") - conn = backend.connect() + backend.connect() try: - # Should expose protocol methods even if no tables exist - assert conn.list_tables() == [] + assert backend.list_tables() == [] finally: - conn.close() + backend.close() def test_neither_positional_nor_dialect_raises(): @@ -61,7 +60,6 @@ def test_settings_path_sqlite(): """Construct IbisBackend from SQLite SettingsParameters and connect.""" from mountainash_settings import SettingsParameters from mountainash_data.core.settings import SQLiteAuthSettings, NoAuth - from mountainash_data.backends.ibis.backend import IbisConnection params = SettingsParameters.create( settings_class=SQLiteAuthSettings, @@ -70,18 +68,16 @@ def test_settings_path_sqlite(): ) backend = IbisBackend(params) assert backend.dialect == "sqlite" - conn = backend.connect() - assert isinstance(conn, IbisConnection) - tables = conn.list_tables() + backend.connect() + tables = backend.list_tables() assert isinstance(tables, list) - conn.close() + backend.close() def test_settings_path_duckdb_empty_extensions(): """DuckDB settings with default EXTENSIONS=[] must not crash ibis.""" from mountainash_settings import SettingsParameters from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth - from mountainash_data.backends.ibis.backend import IbisConnection params = SettingsParameters.create( settings_class=DuckDBAuthSettings, @@ -90,9 +86,8 @@ def test_settings_path_duckdb_empty_extensions(): ) backend = IbisBackend(params) assert backend.dialect == "duckdb" - conn = backend.connect() # Must not raise — empty-list filter active - assert isinstance(conn, IbisConnection) - conn.close() + backend.connect() + backend.close() # --------------------------------------------------------------------------- @@ -101,34 +96,99 @@ def test_settings_path_duckdb_empty_extensions(): def test_url_path_sqlite(): """Construct IbisBackend from sqlite:// URL and connect.""" - from mountainash_data.backends.ibis.backend import IbisConnection - backend = IbisBackend("sqlite://") assert backend.dialect == "sqlite" - conn = backend.connect() - assert isinstance(conn, IbisConnection) - conn.close() + backend.connect() + backend.close() def test_url_path_duckdb(): """Construct IbisBackend from duckdb:// URL and connect.""" - from mountainash_data.backends.ibis.backend import IbisConnection - backend = IbisBackend("duckdb://") assert backend.dialect == "duckdb" - conn = backend.connect() - assert isinstance(conn, IbisConnection) - conn.close() + backend.connect() + backend.close() def test_url_path_preserves_database(tmp_path): """URL database component must reach the driver, not be discarded.""" - from mountainash_data.backends.ibis.backend import IbisConnection - db_file = tmp_path / "test.db" backend = IbisBackend(f"sqlite:///{db_file}") assert backend.dialect == "sqlite" - conn = backend.connect() - assert isinstance(conn, IbisConnection) - conn.close() + backend.connect() + backend.close() assert db_file.exists() + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + +def test_connect_returns_self(): + """connect() must return the backend instance itself.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + result = backend.connect() + assert result is backend + + +def test_close_returns_self(): + """close() must return the backend instance itself.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + result = backend.close() + assert result is backend + + +def test_context_manager(): + """with IbisBackend(...) as backend: must connect and close.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + assert backend.list_tables() == [] + # After exit, should be closed + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_double_close_is_idempotent(): + """Calling close() twice must not raise.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + backend.close() + backend.close() # Must not raise + + +def test_use_before_connect_raises(): + """Calling methods before connect() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_use_after_close_raises(): + """Calling methods after close() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + backend.close() + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_ibis_connection_accessor(): + """ibis_connection() returns the raw ibis backend object.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + raw = backend.ibis_connection() + assert hasattr(raw, "list_tables") + + +def test_ibis_connection_before_connect_raises(): + """ibis_connection() before connect() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + with pytest.raises(RuntimeError, match="not connected"): + backend.ibis_connection() + + +def test_get_connection_accessor(): + """get_connection() returns our IbisConnection wrapper.""" + from mountainash_data.backends.ibis.backend import IbisConnection + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + conn = backend.get_connection() + assert isinstance(conn, IbisConnection) From 836438017fadfa1bc91d2d0a2ab6b4f5f6269248 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 21:52:20 +1000 Subject: [PATCH 08/12] feat(registry): expand DialectSpec with operation hooks Add upsert_hook, create_index_hook, drop_index_hook, rename_table_hook to DialectSpec. Extract standalone hook functions from _DuckDBFamilyOperationsMixin. Wire DuckDB/SQLite/MotherDuck entries. Co-Authored-By: Claude Sonnet 4.6 --- .../backends/ibis/dialects/_registry.py | 20 +++ .../backends/ibis/operations.py | 145 ++++++++++++++++++ tests/test_unit/backends/ibis/test_backend.py | 22 +++ 3 files changed, 187 insertions(+) diff --git a/src/mountainash_data/backends/ibis/dialects/_registry.py b/src/mountainash_data/backends/ibis/dialects/_registry.py index 0a997c6..085d43a 100644 --- a/src/mountainash_data/backends/ibis/dialects/_registry.py +++ b/src/mountainash_data/backends/ibis/dialects/_registry.py @@ -25,6 +25,10 @@ GetIndexExistsSql = t.Callable[[str, str, t.Optional[str]], str] # (index_name, table_name, database) -> SQL GetListIndexesSql = t.Callable[[str, t.Optional[str]], str] # (table_name, database) -> SQL ConnectionBuilder = t.Callable[..., t.Any] # (**config) -> ibis backend connection +UpsertHook = t.Callable[..., None] +CreateIndexHook = t.Callable[..., None] +DropIndexHook = t.Callable[..., None] +RenameTableHook = t.Callable[..., None] @dataclass(frozen=True) @@ -37,6 +41,10 @@ class DialectSpec: connection_builder: t.Optional[ConnectionBuilder] = None get_index_exists_sql: t.Optional[GetIndexExistsSql] = None get_list_indexes_sql: t.Optional[GetListIndexesSql] = None + upsert_hook: t.Optional[UpsertHook] = None + create_index_hook: t.Optional[CreateIndexHook] = None + drop_index_hook: t.Optional[DropIndexHook] = None + rename_table_hook: t.Optional[RenameTableHook] = None extras: t.Mapping[str, t.Any] = field(default_factory=dict) @@ -391,6 +399,9 @@ def _build_pyspark_connection(**config: t.Any) -> t.Any: sqlite_get_list_indexes_sql, motherduck_get_index_exists_sql, motherduck_get_list_indexes_sql, + duckdb_family_upsert, + duckdb_family_create_index, + duckdb_family_drop_index, ) @@ -402,6 +413,9 @@ def _build_pyspark_connection(**config: t.Any) -> t.Any: connection_builder=_build_sqlite_connection, get_index_exists_sql=sqlite_get_index_exists_sql, get_list_indexes_sql=sqlite_get_list_indexes_sql, + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, ), "duckdb": DialectSpec( ibis_backend_name="duckdb", @@ -410,6 +424,9 @@ def _build_pyspark_connection(**config: t.Any) -> t.Any: connection_builder=_build_duckdb_connection, get_index_exists_sql=duckdb_get_index_exists_sql, get_list_indexes_sql=duckdb_get_list_indexes_sql, + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, ), "motherduck": DialectSpec( ibis_backend_name="duckdb", @@ -418,6 +435,9 @@ def _build_pyspark_connection(**config: t.Any) -> t.Any: connection_builder=_build_motherduck_connection, get_index_exists_sql=motherduck_get_index_exists_sql, get_list_indexes_sql=motherduck_get_list_indexes_sql, + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, ), "postgres": DialectSpec( ibis_backend_name="postgres", diff --git a/src/mountainash_data/backends/ibis/operations.py b/src/mountainash_data/backends/ibis/operations.py index baf120f..c3a3843 100644 --- a/src/mountainash_data/backends/ibis/operations.py +++ b/src/mountainash_data/backends/ibis/operations.py @@ -212,6 +212,151 @@ def motherduck_list_tables( return ibis_backend.list_tables(like=like, database=database) if ibis_backend is not None else [] +# =========================================================================== +# STANDALONE HOOK FUNCTIONS +# Extracted from _DuckDBFamilyOperationsMixin for DialectSpec wiring. +# =========================================================================== + +def duckdb_family_create_index( + ibis_conn: t.Any, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + unique: bool = False, + index_type: str | None = None, + where_condition: str | None = None, + database: str | None = None, + if_not_exists: bool = True, +) -> None: + """Create an index using DuckDB/SQLite syntax.""" + columns_list = _normalize_columns(columns) + + if index_name is None: + index_name = _generate_index_name(table_name, columns_list, unique=unique) + + qualified_table = _format_qualified_table(table_name, database=database) + columns_sql = ", ".join(columns_list) + + unique_sql = "UNIQUE " if unique else "" + if_not_exists_sql = "IF NOT EXISTS " if if_not_exists else "" + where_sql = f" WHERE {where_condition}" if where_condition else "" + + if index_type and index_type != CONST_INDEX_TYPE.BTREE: + warnings.warn( + f"Index type {index_type} not supported, using default BTREE" + ) + + create_sql = ( + f"CREATE {unique_sql}INDEX {if_not_exists_sql}{index_name} " + f"ON {qualified_table} ({columns_sql}){where_sql}" + ) + + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(create_sql) + + +def duckdb_family_drop_index( + ibis_conn: t.Any, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + if_exists: bool = True, +) -> None: + """Drop an index using DuckDB/SQLite syntax.""" + if_exists_sql = "IF EXISTS " if if_exists else "" + drop_sql = f"DROP INDEX {if_exists_sql}{index_name}" + + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(drop_sql) + + +def duckdb_family_upsert( + ibis_conn: t.Any, + table_name: str, + df: t.Any, + *, + conflict_columns: list[str] | str, + update_columns: list[str] | str | None = None, + conflict_action: str = CONST_CONFLICT_ACTION.UPDATE, + update_condition: str | None = None, + database: str | None = None, + schema: str | None = None, +) -> None: + """Perform upsert using INSERT ... ON CONFLICT syntax (DuckDB/SQLite).""" + conflict_cols = _normalize_columns(conflict_columns) + all_columns = ma.relation(df).columns + + if update_columns is None: + update_cols = [col for col in all_columns if col not in conflict_cols] + else: + update_cols = _normalize_columns(update_columns) + + tables = ibis_conn.list_tables() + if table_name not in tables: + raise ValueError(f"Target table '{table_name}' does not exist") + + if conflict_action not in [CONST_CONFLICT_ACTION.UPDATE, CONST_CONFLICT_ACTION.NOTHING]: + raise ValueError( + f"conflict_action must be '{CONST_CONFLICT_ACTION.UPDATE}' or " + f"'{CONST_CONFLICT_ACTION.NOTHING}', got '{conflict_action}'" + ) + + if conflict_action == CONST_CONFLICT_ACTION.NOTHING: + if update_cols or update_condition: + warnings.warn( + "update_columns and update_condition are ignored when " + "conflict_action='NOTHING'" + ) + + staging_table = f"temp_upsert_{uuid.uuid4().hex[:8]}" + qualified_table = _format_qualified_table(table_name, database=database, schema=schema) + + all_cols_sql = ", ".join(all_columns) + conflict_cols_sql = ", ".join(conflict_cols) + + if conflict_action == CONST_CONFLICT_ACTION.UPDATE: + if not update_cols: + raise ValueError( + "No columns to update. Either provide update_columns or ensure " + "dataframe has columns beyond conflict_columns" + ) + update_set_sql = ", ".join([f"{col} = EXCLUDED.{col}" for col in update_cols]) + where_sql = f" WHERE {update_condition}" if update_condition else "" + on_conflict_sql = ( + f"ON CONFLICT ({conflict_cols_sql}) DO UPDATE SET {update_set_sql}{where_sql}" + ) + else: + on_conflict_sql = f"ON CONFLICT ({conflict_cols_sql}) DO NOTHING" + + upsert_sql = f""" + INSERT INTO {qualified_table} ({all_cols_sql}) + SELECT {all_cols_sql} FROM {staging_table} + WHERE true + {on_conflict_sql} + """ + + if hasattr(ibis_conn.con, 'register'): + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute("BEGIN TRANSACTION") + cur.register(staging_table, df) + cur.execute(upsert_sql) + cur.unregister(staging_table) + cur.execute("COMMIT") + else: + ibis_conn.create_table(staging_table, df, temp=True, overwrite=True) + try: + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(upsert_sql) + ibis_conn.con.commit() + finally: + try: + ibis_conn.drop_table(staging_table, force=True) + except Exception: + pass + + # =========================================================================== # _DuckDBFamilyOperationsMixin # Salvaged from _duckdb_family_mixin.py — retained as class for backward diff --git a/tests/test_unit/backends/ibis/test_backend.py b/tests/test_unit/backends/ibis/test_backend.py index 2e065cb..eabaebd 100644 --- a/tests/test_unit/backends/ibis/test_backend.py +++ b/tests/test_unit/backends/ibis/test_backend.py @@ -192,3 +192,25 @@ def test_get_connection_accessor(): with IbisBackend(dialect="sqlite", database=":memory:") as backend: conn = backend.get_connection() assert isinstance(conn, IbisConnection) + + +# --------------------------------------------------------------------------- +# DialectSpec hooks +# --------------------------------------------------------------------------- + +def test_duckdb_dialect_has_upsert_hook(): + """DuckDB DialectSpec must have upsert_hook wired.""" + spec = DIALECTS["duckdb"] + assert spec.upsert_hook is not None + + +def test_sqlite_dialect_has_create_index_hook(): + """SQLite DialectSpec must have create_index_hook wired.""" + spec = DIALECTS["sqlite"] + assert spec.create_index_hook is not None + + +def test_postgres_dialect_has_no_upsert_hook(): + """Postgres DialectSpec has no upsert_hook (not DuckDB family).""" + spec = DIALECTS["postgres"] + assert spec.upsert_hook is None From d74f94e54bdb18f1048ab0bdec93289e8ba3621b Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 21:55:54 +1000 Subject: [PATCH 09/12] feat(backend): add thin wrapper and hook-dispatched operations Fluent methods (create_table, insert, upsert, create_index, etc.) return self. Terminal methods (table, run_sql, list_indexes, etc.) return data. Hook dispatch: upsert/index operations via DialectSpec callables. One deviation from spec: truncate() omits the schema kwarg when calling ibis truncate_table() because SQLBackend does not accept it at the base level; schema is preserved in the method signature for future dialects. Co-Authored-By: Claude Sonnet 4.6 --- src/mountainash_data/backends/ibis/backend.py | 262 ++++++++++++++++++ tests/test_unit/backends/ibis/test_backend.py | 143 ++++++++++ 2 files changed, 405 insertions(+) diff --git a/src/mountainash_data/backends/ibis/backend.py b/src/mountainash_data/backends/ibis/backend.py index a738797..79267b1 100644 --- a/src/mountainash_data/backends/ibis/backend.py +++ b/src/mountainash_data/backends/ibis/backend.py @@ -315,3 +315,265 @@ def inspect_namespace(self, name: str) -> NamespaceInfo: def inspect_catalog(self) -> CatalogInfo: return self._require_connected().inspect_catalog() + + # --- Thin wrapper operations (fluent — return self) --- + + def create_table( + self, + name: str, + obj: t.Any, + *, + schema: t.Any | None = None, + database: str | None = None, + temp: bool = False, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.create_table( + name, obj=obj, schema=schema, database=database, + temp=temp, overwrite=overwrite, + ) + return self + + def drop_table( + self, + name: str, + *, + database: str | None = None, + force: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.drop_table(name, database=database, force=force) + return self + + def create_view( + self, + name: str, + obj: t.Any, + *, + database: str | None = None, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.create_view(name, obj=obj, database=database, overwrite=overwrite) + return self + + def drop_view( + self, + name: str, + *, + database: str | None = None, + force: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.drop_view(name, database=database, force=force) + return self + + def insert( + self, + name: str, + obj: t.Any, + *, + database: str | None = None, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.insert(name, obj=obj, database=database, overwrite=overwrite) + return self + + def truncate( + self, + name: str, + *, + database: str | None = None, + schema: str | None = None, + ) -> IbisBackend: + conn = self._require_connected() + # ibis SQLBackend.truncate_table() accepts only table_name + database; + # schema is not a standard kwarg at the SQLBackend level. + kwargs: dict[str, t.Any] = {} + if database is not None: + kwargs["database"] = database + conn._ibis_conn.truncate_table(name, **kwargs) + return self + + def rename_table(self, old_name: str, new_name: str) -> IbisBackend: + if self._spec.rename_table_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support rename_table" + ) + conn = self._require_connected() + self._spec.rename_table_hook(conn._ibis_conn, old_name, new_name) + return self + + # --- Terminal operations (return data) --- + + def table(self, name: str, *, database: str | None = None) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.table(name, database=database) + + def table_exists( + self, name: str, database: str | None = None + ) -> bool: + tables = self.list_tables() + return name in tables + + def run_sql( + self, + query: str, + *, + schema: t.Any | None = None, + dialect: str | None = None, + ) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.sql(query, schema=schema, dialect=dialect) + + def run_expr( + self, + expr: t.Any, + *, + params: dict | None = None, + limit: str | None = "default", + **kwargs: t.Any, + ) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.execute(expr, params=params, limit=limit, **kwargs) + + def to_sql( + self, + expr: t.Any, + *, + params: t.Any = None, + limit: str | None = None, + pretty: bool = False, + **kwargs: t.Any, + ) -> str | None: + conn = self._require_connected() + return conn._ibis_conn.compile(expr, params=params, limit=limit, pretty=pretty, **kwargs) + + # --- Hook-dispatched operations (fluent — return self) --- + + def upsert( + self, + name: str, + obj: t.Any, + *, + conflict_columns: list[str] | str, + update_columns: list[str] | str | None = None, + conflict_action: str = "UPDATE", + update_condition: str | None = None, + database: str | None = None, + schema: str | None = None, + ) -> IbisBackend: + if self._spec.upsert_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support upsert" + ) + conn = self._require_connected() + self._spec.upsert_hook( + conn._ibis_conn, name, obj, + conflict_columns=conflict_columns, + update_columns=update_columns, + conflict_action=conflict_action, + update_condition=update_condition, + database=database, + schema=schema, + ) + return self + + def create_index( + self, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + unique: bool = False, + index_type: str | None = None, + where_condition: str | None = None, + database: str | None = None, + if_not_exists: bool = True, + ) -> IbisBackend: + if self._spec.create_index_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support create_index" + ) + conn = self._require_connected() + self._spec.create_index_hook( + conn._ibis_conn, table_name, columns, + index_name=index_name, unique=unique, index_type=index_type, + where_condition=where_condition, database=database, + if_not_exists=if_not_exists, + ) + return self + + def create_unique_index( + self, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + where_condition: str | None = None, + database: str | None = None, + ) -> IbisBackend: + return self.create_index( + table_name, columns, + index_name=index_name, unique=True, + where_condition=where_condition, database=database, + ) + + def drop_index( + self, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + if_exists: bool = True, + ) -> IbisBackend: + if self._spec.drop_index_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support drop_index" + ) + conn = self._require_connected() + self._spec.drop_index_hook( + conn._ibis_conn, index_name, + table_name=table_name, database=database, if_exists=if_exists, + ) + return self + + def index_exists( + self, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + ) -> bool: + if self._spec.get_index_exists_sql is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support index_exists" + ) + conn = self._require_connected() + check_sql = self._spec.get_index_exists_sql(index_name, table_name, database) + result = conn._ibis_conn.sql(check_sql) + if result is None: + return False + import mountainash as ma + count = ma.relation(result).to_dict()["count"][0] + return count > 0 + + def list_indexes( + self, + table_name: str, + *, + database: str | None = None, + ) -> list[dict]: + if self._spec.get_list_indexes_sql is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support list_indexes" + ) + conn = self._require_connected() + list_sql = self._spec.get_list_indexes_sql(table_name, database) + result = conn._ibis_conn.sql(list_sql) + if result is None: + return [] + import mountainash as ma + return ma.relation(result).to_dicts() diff --git a/tests/test_unit/backends/ibis/test_backend.py b/tests/test_unit/backends/ibis/test_backend.py index eabaebd..458c9ba 100644 --- a/tests/test_unit/backends/ibis/test_backend.py +++ b/tests/test_unit/backends/ibis/test_backend.py @@ -1,6 +1,7 @@ """Tests for IbisBackend factory.""" import pytest +import polars as pl from mountainash_data.backends.ibis.backend import IbisBackend from mountainash_data.backends.ibis.dialects._registry import DIALECTS @@ -214,3 +215,145 @@ def test_postgres_dialect_has_no_upsert_hook(): """Postgres DialectSpec has no upsert_hook (not DuckDB family).""" spec = DIALECTS["postgres"] assert spec.upsert_hook is None + + +# --------------------------------------------------------------------------- +# Thin wrapper operations (fluent) +# --------------------------------------------------------------------------- + +def test_create_table_returns_self(): + """create_table() must return self for fluent chaining.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + result = backend.create_table("t", {"id": [1, 2]}) + assert result is backend + assert "t" in backend.list_tables() + + +def test_drop_table_returns_self(): + """drop_table() must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.drop_table("t") + assert result is backend + assert "t" not in backend.list_tables() + + +def test_insert_returns_self(): + """insert() must return self.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.insert("t", {"id": [2]}) + assert result is backend + + +def test_truncate_returns_self(): + """truncate() must return self.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.truncate("t") + assert result is backend + + +def test_table_returns_ibis_table(): + """table() must return an ibis table expression.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2]}) + tbl = backend.table("t") + assert tbl is not None + + +def test_run_sql_returns_result(): + """run_sql() must return an ibis table expression.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2, 3]}) + result = backend.run_sql("SELECT COUNT(*) as cnt FROM t") + assert result is not None + + +def test_table_exists_returns_bool(): + """table_exists() must return True/False.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + assert backend.table_exists("nope") is False + backend.create_table("t", {"id": [1]}) + assert backend.table_exists("t") is True + + +def test_fluent_chaining(): + """Multiple fluent calls can be chained.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("a", {"id": [1]}).create_table("b", {"id": [2]}) + assert sorted(backend.list_tables()) == ["a", "b"] + + +# --------------------------------------------------------------------------- +# Hook-dispatched operations +# --------------------------------------------------------------------------- + +def test_create_index_returns_self(): + """create_index() via hook must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + result = backend.create_index("t", ["name"]) + assert result is backend + + +def test_create_unique_index_returns_self(): + """create_unique_index() must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + result = backend.create_unique_index("t", ["name"]) + assert result is backend + + +def test_drop_index_returns_self(): + """drop_index() via hook must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + backend.create_index("t", ["name"], index_name="idx_name") + result = backend.drop_index("idx_name") + assert result is backend + + +def test_index_exists(): + """index_exists() must detect created indexes.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + backend.create_index("t", ["id"], index_name="idx_id") + assert backend.index_exists("idx_id") is True + assert backend.index_exists("no_such_idx") is False + + +def test_list_indexes(): + """list_indexes() must return index info.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + backend.create_index("t", ["id"], index_name="idx_id") + indexes = backend.list_indexes("t") + assert isinstance(indexes, list) + assert len(indexes) >= 1 + + +def test_upsert_duckdb(): + """upsert() must work on DuckDB via hook.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "val": [10, 20]}) + backend.create_table("t", initial) + backend.create_unique_index("t", ["id"]) + + update = pl.DataFrame({"id": [2, 3], "val": [25, 30]}) + result = backend.upsert("t", update, conflict_columns=["id"]) + assert result is backend + + count_result = backend.run_sql("SELECT COUNT(*) as cnt FROM t") + count = count_result.to_polars()["cnt"][0] + assert count == 3 + + +def test_upsert_unsupported_dialect_raises(): + """upsert() on a dialect without upsert_hook must raise NotImplementedError.""" + backend = IbisBackend(dialect="postgres") + # Can't actually connect to postgres, so mock the connection state + from mountainash_data.backends.ibis.backend import IbisConnection + backend._conn = IbisConnection(None, DIALECTS["postgres"]) + with pytest.raises(NotImplementedError, match="does not support upsert"): + backend.upsert("t", {}, conflict_columns=["id"]) From e634c893bace29dee07c6c0861fc4f3a65441770 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 21:56:57 +1000 Subject: [PATCH 10/12] feat(iceberg): update IcebergBackend for new Backend protocol connect() returns self, context manager support, inspection methods delegate to internal connection. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../backends/iceberg/backend.py | 81 +++++++++++-------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/src/mountainash_data/backends/iceberg/backend.py b/src/mountainash_data/backends/iceberg/backend.py index 00a5e9a..f353437 100644 --- a/src/mountainash_data/backends/iceberg/backend.py +++ b/src/mountainash_data/backends/iceberg/backend.py @@ -1,9 +1,4 @@ -"""IcebergBackend — implements core.protocol.Backend for iceberg catalogs. - -NOTE: ``to_relation()`` is intentionally NOT implemented on this backend. -It requires mountainash-expressions to gain an Iceberg adapter, which is -a separate work item. The gap is documented in the Phase 3 spec. -""" +"""IcebergBackend — implements core.protocol.Backend for iceberg catalogs.""" from __future__ import annotations @@ -19,28 +14,10 @@ class IcebergBackend: - """Iceberg backend factory. + """Iceberg backend — single entry point for iceberg catalog interaction. Construction takes a catalog type (e.g. ``'rest'``) and config kwargs. - ``connect()`` returns a live ``IcebergConnectionBase`` instance that - satisfies ``core.protocol.Connection``. - - Args: - catalog: One of the keys in ``_CATALOG_REGISTRY`` (currently - only ``'rest'``). - **config: Keyword arguments forwarded verbatim to the connection - class constructor (minus the ``db_auth_settings_parameters`` - which must be provided separately when calling ``connect()``). - - Raises: - KeyError: If ``catalog`` is not a known catalog type. - - Example:: - - backend = IcebergBackend(catalog="rest", uri="http://localhost:8181") - conn = backend.connect() - tables = conn.list_tables() - conn.close() + ``connect()`` returns ``self``. Use as a context manager. """ name = "iceberg" @@ -53,13 +30,49 @@ def __init__(self, catalog: str, **config: t.Any) -> None: ) self._catalog_cls = _CATALOG_REGISTRY[catalog] self._config = config + self._conn: IcebergConnectionBase | None = None + + def connect(self) -> IcebergBackend: + """Open a connection. Returns self for fluent chaining.""" + if self._conn is None: + self._conn = self._catalog_cls(**self._config) + return self + + def close(self) -> IcebergBackend: + """Release the connection. Idempotent. Returns self.""" + if self._conn is not None: + if hasattr(self._conn, "close"): + self._conn.close() + self._conn = None + return self + + def __enter__(self) -> IcebergBackend: + self.connect() + return self + + def __exit__(self, *args: t.Any) -> None: + self.close() + + def _require_connected(self) -> IcebergConnectionBase: + if self._conn is None: + raise RuntimeError( + "IcebergBackend is not connected. Call connect() first." + ) + return self._conn + + def list_tables(self, namespace: str | None = None) -> list[str]: + return self._require_connected().list_tables(namespace=namespace) + + def list_namespaces(self) -> list[str]: + return self._require_connected().list_namespaces() + + def inspect_table( + self, name: str, namespace: str | None = None + ) -> t.Any: + return self._require_connected().inspect_table(name, namespace=namespace) - def connect(self) -> IcebergConnectionBase: - """Open a connection. Caller is responsible for closing it. + def inspect_namespace(self, name: str) -> t.Any: + return self._require_connected().inspect_namespace(name) - Note: The legacy IcebergConnectionBase requires a - ``db_auth_settings_parameters`` argument. When ``_config`` does not - include one, this will raise at the base class constructor level. - This mirrors the legacy behaviour. - """ - return self._catalog_cls(**self._config) + def inspect_catalog(self) -> t.Any: + return self._require_connected().inspect_catalog() From 18ec2469fb2a1b0a792eac2f7b94c0ced396be7d Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 22:02:02 +1000 Subject: [PATCH 11/12] refactor: delete legacy factories, DatabaseUtils, and connection hierarchies Remove ConnectionFactory, OperationsFactory, SettingsFactory, DatabaseUtils, BaseIbisConnection + 12 subclasses, BaseIbisOperations + concrete subclasses. Keep BaseDBConnection (Iceberg depends on it). Update __init__.py to export only Backend, IbisBackend, IcebergBackend, and inspection model. Co-Authored-By: Claude Sonnet 4.6 --- src/mountainash_data/__init__.py | 16 - .../backends/ibis/connection.py | 720 ---------------- .../backends/ibis/operations.py | 780 +----------------- .../core/factories/__init__.py | 13 - .../core/factories/base_strategy_factory.py | 173 ---- .../core/factories/connection_factory.py | 108 --- .../core/factories/operations_factory.py | 110 --- .../core/factories/settings_factory.py | 206 ----- .../factories/settings_type_factory_mixin.py | 184 ----- src/mountainash_data/core/utils.py | 224 ----- 10 files changed, 6 insertions(+), 2528 deletions(-) delete mode 100644 src/mountainash_data/backends/ibis/connection.py delete mode 100644 src/mountainash_data/core/factories/__init__.py delete mode 100644 src/mountainash_data/core/factories/base_strategy_factory.py delete mode 100644 src/mountainash_data/core/factories/connection_factory.py delete mode 100644 src/mountainash_data/core/factories/operations_factory.py delete mode 100644 src/mountainash_data/core/factories/settings_factory.py delete mode 100644 src/mountainash_data/core/factories/settings_type_factory_mixin.py delete mode 100644 src/mountainash_data/core/utils.py diff --git a/src/mountainash_data/__init__.py b/src/mountainash_data/__init__.py index c1a0dca..a690c51 100644 --- a/src/mountainash_data/__init__.py +++ b/src/mountainash_data/__init__.py @@ -5,9 +5,6 @@ IbisBackend — ibis-style relational backends (backends.ibis.backend) IcebergBackend — iceberg-style table-format catalogs (backends.iceberg.backend) CatalogInfo, NamespaceInfo, TableInfo, ColumnInfo — inspection model - DatabaseUtils — high-level facade - ConnectionFactory, OperationsFactory, SettingsFactory — factories - *Settings classes — see mountainash_data.core.settings """ from mountainash_data.__version__ import __version__ @@ -18,17 +15,8 @@ NamespaceInfo, TableInfo, ) -from mountainash_data.core.utils import DatabaseUtils -from mountainash_data.core.factories import ( - ConnectionFactory, - OperationsFactory, - SettingsFactory, -) from mountainash_data.backends.ibis.backend import IbisBackend -# IcebergBackend requires the optional pyiceberg dependency. -# It is imported lazily so that consumers without pyiceberg installed -# still get the rest of the package. try: from mountainash_data.backends.iceberg.backend import IcebergBackend except ImportError: @@ -41,10 +29,6 @@ "ColumnInfo", "NamespaceInfo", "TableInfo", - "DatabaseUtils", - "ConnectionFactory", - "OperationsFactory", - "SettingsFactory", "IbisBackend", "IcebergBackend", ] diff --git a/src/mountainash_data/backends/ibis/connection.py b/src/mountainash_data/backends/ibis/connection.py deleted file mode 100644 index 8a37904..0000000 --- a/src/mountainash_data/backends/ibis/connection.py +++ /dev/null @@ -1,720 +0,0 @@ -import typing as t # import t.Any, t.Dict, t.Optional - -import ibis -from ibis.backends.sql import SQLBackend -from abc import abstractmethod -from pydantic_settings import BaseSettings as _BaseSettings - -from mountainash_settings import SettingsParameters - -from mountainash_data.core.connection import BaseDBConnection -from mountainash_data.core.constants import ( - IBIS_DB_CONNECTION_MODE, - CONST_DB_ABSTRACTION_LAYER, - CONST_DB_PROVIDER_TYPE, - CONST_DB_BACKEND as _CONST_DB_BACKEND, -) -# from mountainash_dataframes.utils.dataframe_utils import DataFrameUtils - - -class BaseIbisConnection(BaseDBConnection): - - def __init__(self, - db_auth_settings_parameters: SettingsParameters, - ): - - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters, - ) - - - @property - def db_abstraction_layer(self) -> CONST_DB_ABSTRACTION_LAYER: - return CONST_DB_ABSTRACTION_LAYER.IBIS - - @property - @abstractmethod - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - """Database provider identifier.""" - ... - - - @property - @abstractmethod - def ibis_backend(self) -> t.Optional[SQLBackend|t.Any]: - """Concrete Ibis backend connection object.""" - pass - - @property - @abstractmethod - def ibis_connection_mode(self) -> str: - """Connect via a connection string, kwargs or both.""" - pass - - @property - @abstractmethod - def connection_string_scheme(self) -> str: - """Template string for database connection.""" - pass - - # @property - # @abstractmethod - # def supports_upsert(self) -> str: - # """Whether upserts are supported""" - # pass - - - - - ########################### - # Core Functions - - - def connect(self, - *, - connection_string: t.Optional[str] = None, - connection_kwargs: t.Optional[t.Dict[str, t.Any]] = None, - **kwargs) -> SQLBackend: - - """Connect with explicitly provided connection parameters""" - - - if self.ibis_backend is None: - - if connection_string is not None: - if connection_kwargs is not None: - self._connect(connection_string=connection_string, connection_kwargs=connection_kwargs, **kwargs) - else: - self._connect(connection_string=connection_string, **kwargs) - - else: - if connection_kwargs is not None: - self._connect(connection_string = self.connection_string_scheme, connection_kwargs=connection_kwargs, **kwargs) - else: - self.connect_default(**kwargs) - - if self.ibis_backend is None: - raise Exception(f"Unable to establish connection to {self.db_backend_name}") - - - return self.ibis_backend - - - def connect_default(self, **kwargs) -> SQLBackend: - """Connect using default configuration""" - - if self.ibis_backend is None: - - settings_class = self.db_auth_settings_parameters.settings_class - if settings_class is not None: - obj_settings = settings_class.get_settings( - settings_parameters=self.db_auth_settings_parameters - ) - driver_kwargs = obj_settings.to_driver_kwargs() - # Filter out empty lists/sequences that some drivers reject. - # (e.g. ibis.duckdb.connect() does not accept extensions=[]) - driver_kwargs = {k: v for k, v in driver_kwargs.items() - if not (isinstance(v, (list, tuple)) and len(v) == 0)} - driver_kwargs.update(kwargs) - # Use the ibis dialect string to call ibis..connect(**driver_kwargs) - descriptor = getattr(obj_settings, "__descriptor__", None) - ibis_dialect = descriptor.ibis_dialect if descriptor else None - if ibis_dialect: - dialect_backend = getattr(ibis, ibis_dialect, None) - if dialect_backend is not None: - self._ibis_backend = dialect_backend.connect(**driver_kwargs) - if self.ibis_backend is None: - raise Exception(f"Unable to establish default connection to {self.db_backend_name}") - return self.ibis_backend - # Fallback: build KWARGS-mode connection - self._connect( - connection_string=self.connection_string_scheme, - connection_kwargs=driver_kwargs if driver_kwargs else None, - ) - if self.ibis_backend is None: - raise Exception(f"Unable to establish default connection to {self.db_backend_name}") - return self.ibis_backend - - return self.ibis_backend - - - - def _connect(self, - connection_string: t.Optional[str] = None, - connection_kwargs: t.Optional[t.Dict[str, t.Any]] = None, - **kwargs - ) -> SQLBackend: - """ - Default Implementation to connect to the database using the provided connection string. - By default this relies on the connection_string_scheme to determine the backend dynamically. - Over-ride in subclasses if a different implementation is necessary, such as using the custom backend directly. - """ - - if connection_kwargs is None: - connection_kwargs = {} - - #combine connection_kwargs and kwargs - connection_kwargs = {**connection_kwargs, **kwargs} - - if connection_string is None: - raise ValueError(f"{self.db_backend_name}: Connection string is required to establish connection") - - self._ibis_backend : t.Any = ibis.connect(connection_string, **connection_kwargs) - - if self.ibis_backend is None: - raise Exception(f"Unable to establish connection to {self.db_backend_name}") - - - return self.ibis_backend - - - - def close(self): - """Close the connection to the database.""" - - self.disconnect() - - def disconnect(self): - """Close the connection to the database.""" - - if self.ibis_backend is not None: - self.ibis_backend.disconnect() - self._ibis_backend = None - - def is_connected(self) -> bool: - """ Is the connection open?""" - if self.ibis_backend is None: - return False - else: - return True - - -# =========================================================================== -# Concrete per-dialect connection classes -# These preserve the per-backend concrete subclasses needed by the legacy -# ConnectionFactory and existing tests. Each implements the abstract properties -# that BaseIbisConnection requires from BaseDBConnection. -# =========================================================================== - -class SQLite_IbisConnection(BaseIbisConnection): - """SQLite ibis connection — concrete subclass for factory compatibility.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.SQLITE - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.SQLITE - - @property - def connection_string_scheme(self) -> str: - return "sqlite://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import SQLiteAuthSettings - return SQLiteAuthSettings - - -class DuckDB_IbisConnection(BaseIbisConnection): - """DuckDB ibis connection — concrete subclass for factory compatibility.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None, - read_only: t.Optional[bool] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - self._read_only = read_only - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.DUCKDB - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.DUCKDB - - @property - def connection_string_scheme(self) -> str: - return "duckdb://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import DuckDBAuthSettings - return DuckDBAuthSettings - - def _connect(self, connection_string: t.Optional[str] = None, - connection_kwargs: t.Optional[t.Dict[str, t.Any]] = None, - **kwargs) -> t.Any: - from mountainash_data.core.settings import DuckDBAuthSettings - - if connection_kwargs is None: - connection_kwargs = {} - if kwargs is None: - kwargs = {} - - settings: DuckDBAuthSettings = DuckDBAuthSettings.get_settings(self.db_auth_settings_parameters) - - if settings.DATABASE is None: - kwargs["read_only"] = False - else: - if connection_kwargs.get("read_only") is not None and self._read_only is not None: - kwargs["read_only"] = connection_kwargs["read_only"] or self._read_only - elif self._read_only is not None: - kwargs["read_only"] = self._read_only - elif connection_kwargs.get("read_only") is None: - kwargs["read_only"] = False - - connection_kwargs = {**connection_kwargs, **kwargs} - return super()._connect(connection_string, connection_kwargs) - - def disconnect(self): - """Close the DuckDB connection, ensuring the underlying connection is properly closed.""" - if self.ibis_backend is not None: - try: - if hasattr(self.ibis_backend, 'con'): - try: - if hasattr(self.ibis_backend.con, '_cursors'): - for cursor in list(self.ibis_backend.con._cursors): - try: - cursor.close() - except Exception: - pass - self.ibis_backend.con.close() - except Exception as e: - print(f"Warning: Error closing DuckDB connection: {str(e)}") - except Exception as e: - print(f"Warning: Error during DuckDB disconnect: {str(e)}") - finally: - try: - super().disconnect() - except Exception as e: - print(f"Warning: Error during Ibis backend disconnect: {str(e)}") - self._ibis_backend = None - - -class MotherDuck_IbisConnection(BaseIbisConnection): - """MotherDuck ibis connection — concrete subclass for factory compatibility.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - self.supports_upsert = True - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.MOTHERDUCK - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.MOTHERDUCK - - @property - def connection_string_scheme(self) -> str: - return "duckdb://md:" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import MotherDuckAuthSettings - return MotherDuckAuthSettings - - -class Postgres_IbisConnection(BaseIbisConnection): - """PostgreSQL ibis connection — concrete subclass for factory compatibility.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.POSTGRESQL - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.POSTGRES - - @property - def connection_string_scheme(self) -> str: - return "postgres://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import PostgreSQLAuthSettings - return PostgreSQLAuthSettings - - -class MySQL_IbisConnection(BaseIbisConnection): - """MySQL ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.MYSQL - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.MYSQL - - @property - def connection_string_scheme(self) -> str: - return "mysql://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import MySQLAuthSettings - return MySQLAuthSettings - - -class MSSQL_IbisConnection(BaseIbisConnection): - """MSSQL ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.MSSQL - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.MSSQL - - @property - def connection_string_scheme(self) -> str: - return "mssql://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import MSSQLAuthSettings - return MSSQLAuthSettings - - -class Oracle_IbisConnection(BaseIbisConnection): - """Oracle ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.ORACLE - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.ORACLE - - @property - def connection_string_scheme(self) -> str: - return "oracle://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - return None # Oracle settings not yet implemented - - -class Snowflake_IbisConnection(BaseIbisConnection): - """Snowflake ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.HYBRID - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.SNOWFLAKE - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.SNOWFLAKE - - @property - def connection_string_scheme(self) -> str: - return "snowflake://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import SnowflakeAuthSettings - return SnowflakeAuthSettings - - -class BigQuery_IbisConnection(BaseIbisConnection): - """BigQuery ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.KWARGS - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.BIGQUERY - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.BIGQUERY - - @property - def connection_string_scheme(self) -> str: - return "bigquery://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import BigQueryAuthSettings - return BigQueryAuthSettings - - def _connect(self, connection_string: t.Optional[str] = None, - connection_kwargs: t.Optional[t.Dict[str, t.Any]] = None, - **kwargs) -> t.Any: - import ibis.backends.bigquery as ir_backend - - credentials_info = connection_kwargs.get('credentials_info', None) if connection_kwargs else None - dataset_id = connection_kwargs.get('dataset_id', "") if connection_kwargs else "" - project_id = connection_kwargs.get('project_id', None) if connection_kwargs else None - - if credentials_info: - from google.oauth2 import service_account - credentials = service_account.Credentials.from_service_account_info(credentials_info) - self._ibis_backend = ir_backend.connect(dataset_id=dataset_id, credentials=credentials) - else: - self._ibis_backend = ir_backend.connect(project_id=project_id, dataset_id=dataset_id) - - return self.ibis_backend - - -class Redshift_IbisConnection(BaseIbisConnection): - """Redshift ibis connection (uses postgres protocol).""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.REDSHIFT - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.REDSHIFT - - @property - def connection_string_scheme(self) -> str: - return "postgres://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import RedshiftAuthSettings - return RedshiftAuthSettings - - -class Trino_IbisConnection(BaseIbisConnection): - """Trino ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.HYBRID - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.TRINO - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.TRINO - - @property - def connection_string_scheme(self) -> str: - return "trino://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import TrinoAuthSettings - return TrinoAuthSettings - - -class PySpark_IbisConnection(BaseIbisConnection): - """PySpark ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.PYSPARK - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.PYSPARK - - @property - def connection_string_scheme(self) -> str: - return "pyspark://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import PySparkAuthSettings - return PySparkAuthSettings diff --git a/src/mountainash_data/backends/ibis/operations.py b/src/mountainash_data/backends/ibis/operations.py index c3a3843..d8d5119 100644 --- a/src/mountainash_data/backends/ibis/operations.py +++ b/src/mountainash_data/backends/ibis/operations.py @@ -1,32 +1,20 @@ -"""Merged ibis operations module. +"""Ibis operations module — module-level hook functions only. -Consolidates: -- base_ibis_operations.py (676 LOC) — actual operations implementation -- _base_ibis_mixin.py (98 LOC) — helper methods as module-level functions -- _duckdb_family_mixin.py (314 LOC) — DuckDB-family shared SQL as class + module functions -- Per-dialect index SQL functions (duckdb, sqlite, motherduck) - -This module is the single source of operations truth for all ibis backends. -The original files have been replaced with shims that re-export from here. +Contains: +- Module-level helper functions: _generate_index_name, _format_qualified_table, _normalize_columns +- Per-dialect SQL functions: duckdb, sqlite, motherduck index SQL generators +- Standalone hook functions: duckdb_family_create_index, duckdb_family_drop_index, + duckdb_family_upsert """ import typing as t - -from abc import abstractmethod, ABC import contextlib import warnings import uuid -import ibis -import ibis.expr.types.relations as ir -from ibis.expr.schema import SchemaLike -from ibis.backends.sql import SQLBackend - -from mountainash_settings import SettingsParameters import mountainash as ma from mountainash_data.core.constants import ( - CONST_DB_BACKEND, CONST_CONFLICT_ACTION, CONST_INDEX_TYPE, ) @@ -34,7 +22,6 @@ # =========================================================================== # MODULE-LEVEL HELPER FUNCTIONS -# Salvaged from _base_ibis_mixin.py — converted from mixin methods to functions # =========================================================================== def _generate_index_name( @@ -83,8 +70,6 @@ def _normalize_columns( # =========================================================================== # PER-DIALECT CAPABILITY HOOK FUNCTIONS -# Salvaged from duckdb_ibis_operations.py, sqlite_ibis_operations.py, -# motherduck_ibis_operations.py # =========================================================================== # --- DuckDB --- @@ -214,7 +199,6 @@ def motherduck_list_tables( # =========================================================================== # STANDALONE HOOK FUNCTIONS -# Extracted from _DuckDBFamilyOperationsMixin for DialectSpec wiring. # =========================================================================== def duckdb_family_create_index( @@ -355,755 +339,3 @@ def duckdb_family_upsert( ibis_conn.drop_table(staging_table, force=True) except Exception: pass - - -# =========================================================================== -# _DuckDBFamilyOperationsMixin -# Salvaged from _duckdb_family_mixin.py — retained as class for backward -# compatibility. Internal helpers now delegate to module-level functions. -# =========================================================================== - -class _DuckDBFamilyOperationsMixin: - """ - Shared operations for DuckDB-family databases. - - This class provides common implementation for databases that share - DuckDB's SQL syntax: DuckDB, MotherDuck, and SQLite. - - Internal helpers delegate to module-level functions (see top of file). - """ - - # Delegate helper methods to module-level functions so existing code - # that calls cls._generate_index_name() etc. still works. - _generate_index_name = staticmethod(_generate_index_name) - _format_qualified_table = staticmethod(_format_qualified_table) - _normalize_columns = staticmethod(_normalize_columns) - - @classmethod - def create_index( - cls, - ibis_backend: SQLBackend, - table_name: str, - columns: list[str] | str, - *, - index_name: str | None = None, - unique: bool = False, - index_type: str | None = None, - where_condition: str | None = None, - database: str | None = None, - if_not_exists: bool = True - ) -> bool: - """Create an index using DuckDB/SQLite syntax.""" - columns_list = _normalize_columns(columns) - - if index_name is None: - index_name = _generate_index_name(table_name, columns_list, unique=unique) - - qualified_table = _format_qualified_table(table_name, database=database) - columns_sql = ", ".join(columns_list) - - unique_sql = "UNIQUE " if unique else "" - if_not_exists_sql = "IF NOT EXISTS " if if_not_exists else "" - where_sql = f" WHERE {where_condition}" if where_condition else "" - - if index_type and index_type != CONST_INDEX_TYPE.BTREE: - warnings.warn( - f"Index type {index_type} not supported, using default BTREE" - ) - - create_sql = ( - f"CREATE {unique_sql}INDEX {if_not_exists_sql}{index_name} " - f"ON {qualified_table} ({columns_sql}){where_sql}" - ) - - try: - with contextlib.closing(ibis_backend.con.cursor()) as cur: - cur.execute(create_sql) - return True - except Exception as e: - print(f"Error creating index {index_name}: {e}") - return False - - @classmethod - def drop_index( - cls, - ibis_backend: SQLBackend, - index_name: str, - *, - table_name: str | None = None, - database: str | None = None, - if_exists: bool = True - ) -> bool: - """Drop an index.""" - if_exists_sql = "IF EXISTS " if if_exists else "" - drop_sql = f"DROP INDEX {if_exists_sql}{index_name}" - - try: - with contextlib.closing(ibis_backend.con.cursor()) as cur: - cur.execute(drop_sql) - return True - except Exception as e: - print(f"Error dropping index {index_name}: {e}") - return False - - @classmethod - def index_exists( - cls, - ibis_backend: SQLBackend, - index_name: str, - *, - table_name: str | None = None, - database: str | None = None - ) -> bool: - """Check if an index exists.""" - check_sql = cls._get_index_exists_sql(index_name, table_name, database) - result = cls.run_sql(ibis_backend, check_sql) - - if result is None: - return False - - count = ma.relation(result).to_dict()["count"][0] - return count > 0 - - @classmethod - def list_indexes( - cls, - ibis_backend: SQLBackend, - table_name: str, - *, - database: str | None = None - ) -> list[dict]: - """List all indexes for a table.""" - list_sql = cls._get_list_indexes_sql(table_name, database) - result = cls.run_sql(ibis_backend, list_sql) - - if result is None: - return [] - - return ma.relation(result).to_dicts() - - @classmethod - def _get_index_exists_sql( - cls, - index_name: str, - table_name: str | None, - database: str | None - ) -> str: - """Provider-specific SQL to check index existence. Must be overridden.""" - raise NotImplementedError( - f"{cls.__name__}: _get_index_exists_sql must be implemented" - ) - - @classmethod - def _get_list_indexes_sql( - cls, - table_name: str, - database: str | None - ) -> str: - """Provider-specific SQL to list all indexes for a table. Must be overridden.""" - raise NotImplementedError( - f"{cls.__name__}: _get_list_indexes_sql must be implemented" - ) - - @classmethod - def _upsert( - cls, - ibis_backend: SQLBackend, - table_name: str, - df: ir.Table | t.Any, - *, - conflict_columns: list[str] | str, - update_columns: list[str] | str | None = None, - conflict_action: str = CONST_CONFLICT_ACTION.UPDATE, - update_condition: str | None = None, - database: str | None = None, - schema: str | None = None, - ) -> None: - """Perform upsert using INSERT ... ON CONFLICT syntax.""" - conflict_cols = _normalize_columns(conflict_columns) - all_columns = ma.relation(df).columns - - if update_columns is None: - update_cols = [col for col in all_columns if col not in conflict_cols] - else: - update_cols = _normalize_columns(update_columns) - - if not cls.table_exists(ibis_backend, table_name, database=database): - raise ValueError(f"Target table '{table_name}' does not exist") - - if conflict_action not in [CONST_CONFLICT_ACTION.UPDATE, CONST_CONFLICT_ACTION.NOTHING]: - raise ValueError( - f"conflict_action must be '{CONST_CONFLICT_ACTION.UPDATE}' or " - f"'{CONST_CONFLICT_ACTION.NOTHING}', got '{conflict_action}'" - ) - - if conflict_action == CONST_CONFLICT_ACTION.NOTHING: - if update_cols or update_condition: - warnings.warn( - "update_columns and update_condition are ignored when " - "conflict_action='NOTHING'" - ) - - staging_table = f"temp_upsert_{uuid.uuid4().hex[:8]}" - qualified_table = _format_qualified_table(table_name, database=database, schema=schema) - - all_cols_sql = ", ".join(all_columns) - conflict_cols_sql = ", ".join(conflict_cols) - - if conflict_action == CONST_CONFLICT_ACTION.UPDATE: - if not update_cols: - raise ValueError( - "No columns to update. Either provide update_columns or ensure " - "dataframe has columns beyond conflict_columns" - ) - update_set_sql = ", ".join([f"{col} = EXCLUDED.{col}" for col in update_cols]) - where_sql = f" WHERE {update_condition}" if update_condition else "" - on_conflict_sql = ( - f"ON CONFLICT ({conflict_cols_sql}) DO UPDATE SET {update_set_sql}{where_sql}" - ) - else: # NOTHING - on_conflict_sql = f"ON CONFLICT ({conflict_cols_sql}) DO NOTHING" - - upsert_sql = f""" - INSERT INTO {qualified_table} ({all_cols_sql}) - SELECT {all_cols_sql} FROM {staging_table} - WHERE true - {on_conflict_sql} - """ - - print(f"DEBUG: Upsert SQL:\n{upsert_sql}") - print(f"DEBUG: Qualified table: {qualified_table}") - print(f"DEBUG: Staging table: {staging_table}") - print(f"DEBUG: Conflict columns: {conflict_cols}") - print(f"DEBUG: Update columns: {update_cols}") - - try: - if hasattr(ibis_backend.con, 'register'): - with contextlib.closing(ibis_backend.con.cursor()) as cur: - cur.execute("BEGIN TRANSACTION") - cur.register(staging_table, df) - cur.execute(upsert_sql) - cur.unregister(staging_table) - cur.execute("COMMIT") - else: - ibis_backend.create_table(staging_table, df, temp=True, overwrite=True) - - try: - with contextlib.closing(ibis_backend.con.cursor()) as cur: - cur.execute(upsert_sql) - ibis_backend.con.commit() - finally: - try: - ibis_backend.drop_table(staging_table, force=True) - except Exception: - pass - except Exception as e: - print(f"Error during upsert to {table_name}: {e}") - raise - - -# Compatibility shim: _BaseIbisMixin was the original mixin class -class _BaseIbisMixin: - """DEPRECATED compatibility shim — methods are now module-level functions - in mountainash_data.backends.ibis.operations.""" - - _generate_index_name = staticmethod(_generate_index_name) - _format_qualified_table = staticmethod(_format_qualified_table) - _normalize_columns = staticmethod(_normalize_columns) - - -# =========================================================================== -# BaseIbisOperations -# Salvaged verbatim from base_ibis_operations.py. -# =========================================================================== - -class BaseIbisOperations(ABC): - - def __init__(self, - db_auth_settings_parameters: SettingsParameters, - ): - ... - - ## SQL Queries - - @property - @abstractmethod - def db_backend_name(self) -> str: - return CONST_DB_BACKEND.DUCKDB - - @classmethod - def run_sql(cls, - ibis_backend: SQLBackend, - query: str, - /, - *, - schema: SchemaLike | None = None, - dialect: str | None = None, - ) -> t.Optional[ir.Table]: - - try: - return ibis_backend.sql(query, - schema=schema, - dialect=dialect - ) - except Exception as e: - print(f"Error executing SQL: {e}") - return None - - @classmethod - def run_expr( - cls, - ibis_backend: ibis.BaseBackend, - ibis_expr: ir.Expr, - /, - params: t.Dict | None = None, - limit: str | None = "default", - **kwargs: t.Any, - ) -> t.Any: - - try: - return ibis_backend.execute(ibis_expr, - params=params, - limit=limit, - **kwargs - ) if ibis_backend is not None else None - except Exception as e: - print(f"Error executing expression: {e}") - return None - - @classmethod - def to_sql( - cls, - ibis_backend: ibis.BaseBackend, - expr: ir.Expr, - /, - params=None, - limit: str | None = None, - pretty: bool = False, - **kwargs: t.Any, - ) -> t.Optional[str]: - - try: - return ibis_backend.compile(expr, - params=params, - limit=limit, - pretty=pretty, - **kwargs - ) if ibis_backend is not None else None - except Exception as e: - print(f"Error compiling expression to SQL: {e}") - return None - - ## Tables - @classmethod - def table( - cls, - ibis_backend: ibis.BaseBackend, - object_name: str, - /, - schema: str | None = None, - database: tuple[str, str] | str | None = None - ) -> t.Optional[ir.Table]: - - try: - return ibis_backend.table(object_name, - database=database - ) if ibis_backend is not None else None - except Exception as e: - print(f"Error getting table {object_name}: {e}") - return None - - @classmethod - def create_table(cls, - ibis_backend: ibis.BaseBackend, - table_name: str, - df: ir.Table|t.Any, - /, - schema: t.Optional[ibis.Schema] = None, - database: str | None = None, - temp: bool = False, - overwrite: bool = False, - ) -> None: - try: - ibis_backend.create_table(table_name, - obj=df, - schema=schema, - database=database, - temp=temp, - overwrite=overwrite) if ibis_backend is not None else None - except Exception as e: - print(f"Error creating table {table_name}: {e}") - return None - - @classmethod - def drop_table( - cls, - ibis_backend: ibis.BaseBackend, - table_name: str, - /, - database: str | None = None, - force: bool = False, - ) -> bool: - - try: - ibis_backend.drop_table(table_name, - database=database, - force=force) if ibis_backend is not None else None - return True - except Exception: - return False - - ## Views - @classmethod - def create_view( - cls, - ibis_backend: ibis.BaseBackend, - view_name: str, - ibis_table_expr: ir.Table, - /, - database: str | None = None, - schema: str | None = None, - overwrite: bool = False, - ) -> t.Optional[ir.Table]: - - try: - return ibis_backend.create_view(view_name, - obj=ibis_table_expr, - database=database, - overwrite=overwrite) if ibis_backend is not None else None - except Exception as e: - print(f"Error creating view {view_name}: {e}") - return None - - @classmethod - def drop_view( - cls, - ibis_backend: ibis.BaseBackend, - view_name: str, - /, - database: str | None = None, - schema: str | None = None, - force: bool = False, - ) -> bool: - - try: - ibis_backend.drop_view(view_name, - database=database, - force=force) if ibis_backend is not None else None - return True - except Exception: - print(f"Error dropping view {view_name}") - return False - - # Backend Data Manipulation - - @classmethod - def insert( - cls, - ibis_backend: SQLBackend, - table_name: str, - /, - df: ir.Table|t.Any, - database: str | None = None, - schema: str | None = None, - overwrite: bool = False, - ) -> bool: - - try: - ibis_backend.insert(table_name, - obj=df, - database=database, - overwrite=overwrite) if ibis_backend is not None else None - return True - except Exception as e: - print(f"Error inserting into table {table_name}: {e}") - return False - - @classmethod - def truncate( - cls, - ibis_backend: SQLBackend, - table_name: str, - /, - database: str | None = None, - schema: str | None = None - ) -> None: - - try: - ibis_backend.truncate_table( - table_name, - schema=schema, - database=database) if ibis_backend is not None else None - except Exception as e: - print(f"Error truncating table {table_name}: {e}") - return None - - @classmethod - def upsert( - cls, - ibis_backend: SQLBackend, - table_name: str, - df: ir.Table|t.Any, - /, - conflict_columns: list[str] | str, - update_columns: list[str] | str | None = None, - conflict_action: str = CONST_CONFLICT_ACTION.UPDATE, - update_condition: str | None = None, - database: str | None = None, - schema: str | None = None, - ) -> None: - - try: - cls._upsert( - ibis_backend, - table_name, - df, - conflict_columns=conflict_columns, - update_columns=update_columns, - conflict_action=conflict_action, - update_condition=update_condition, - database=database, - schema=schema) if ibis_backend is not None else None - except Exception as e: - print(f"Error upserting into table {table_name}: {e}") - raise - - @classmethod - @abstractmethod - def _upsert( - cls, - ibis_backend: SQLBackend, - table_name: str, - df: ir.Table|t.Any, - *, - conflict_columns: list[str] | str, - update_columns: list[str] | str | None = None, - conflict_action: str = CONST_CONFLICT_ACTION.UPDATE, - update_condition: str | None = None, - database: str | None = None, - schema: str | None = None, - ) -> None: - raise NotImplementedError("Upsert is not implemented for this backend") - - # =========================== - # INDEX MANAGEMENT - # =========================== - - @classmethod - @abstractmethod - def create_index( - cls, - ibis_backend: SQLBackend, - table_name: str, - columns: list[str] | str, - *, - index_name: str | None = None, - unique: bool = False, - index_type: str | None = None, - where_condition: str | None = None, - database: str | None = None, - if_not_exists: bool = True - ) -> bool: - raise NotImplementedError("create_index is not implemented for this backend") - - @classmethod - def create_unique_index( - cls, - ibis_backend: SQLBackend, - table_name: str, - columns: list[str] | str, - *, - index_name: str | None = None, - where_condition: str | None = None, - database: str | None = None - ) -> bool: - return cls.create_index( - ibis_backend, table_name, columns, - index_name=index_name, unique=True, - where_condition=where_condition, database=database - ) - - @classmethod - @abstractmethod - def drop_index( - cls, - ibis_backend: SQLBackend, - index_name: str, - *, - table_name: str | None = None, - database: str | None = None, - if_exists: bool = True - ) -> bool: - raise NotImplementedError("drop_index is not implemented for this backend") - - @classmethod - @abstractmethod - def index_exists( - cls, - ibis_backend: SQLBackend, - index_name: str, - *, - table_name: str | None = None, - database: str | None = None - ) -> bool: - raise NotImplementedError("index_exists is not implemented for this backend") - - @classmethod - @abstractmethod - def list_indexes( - cls, - ibis_backend: SQLBackend, - table_name: str, - *, - database: str | None = None - ) -> list[dict]: - raise NotImplementedError("list_indexes is not implemented for this backend") - - # =========================== - # t.Optionally Implemented Functions - # =========================== - - @classmethod - def list_tables(cls, - ibis_backend: ibis.BaseBackend, - table_name: str | None = None, - database: tuple[str, str] | str | None = None - ) -> t.List[str]: - - try: - return ibis_backend.list_tables(like=table_name, database=database) if ibis_backend is not None else [] - except Exception as e: - print(f"Error listing tables: {e}") - return [] - - @classmethod - def rename_table(cls, - ibis_backend: ibis.BaseBackend, - old_name: str, - new_name: str, - ) -> None: - - try: - return cls._rename_table(old_name=old_name, new_name=new_name) if ibis_backend is not None else None - except Exception as e: - print(f"Error renaming table {old_name} to {new_name}: {e}") - return None - - @classmethod - @abstractmethod - def _rename_table(cls, - old_name: str, - new_name: str, - ) -> None: - raise NotImplementedError - - @classmethod - def table_exists(cls, - ibis_backend: ibis.BaseBackend, - table_name: str | None = None, - database: tuple[str, str] | str | None = None - ) -> bool: - - tables = cls.list_tables(ibis_backend, table_name=table_name, database=database) - return True if table_name in tables else False - - -# =========================================================================== -# Concrete per-dialect operations classes -# Preserved as concrete classes so the OperationsFactory can instantiate them. -# Each class combines _DuckDBFamilyOperationsMixin (concrete upsert/index -# implementations) with BaseIbisOperations (abstract template) and provides -# the dialect-specific index catalog SQL via overriding _get_index_exists_sql -# and _get_list_indexes_sql. -# =========================================================================== - -class DuckDB_IbisOperations(_DuckDBFamilyOperationsMixin, BaseIbisOperations): - """DuckDB operations — concrete implementation.""" - - @property - def db_backend_name(self) -> str: - return CONST_DB_BACKEND.DUCKDB - - @classmethod - def _rename_table(cls, old_name: str, new_name: str) -> None: - raise NotImplementedError(f"{CONST_DB_BACKEND.DUCKDB}: rename_table not yet implemented") - - @classmethod - def _get_index_exists_sql(cls, index_name: str, table_name: str | None, database: str | None) -> str: - return duckdb_get_index_exists_sql(index_name, table_name, database) - - @classmethod - def _get_list_indexes_sql(cls, table_name: str, database: str | None) -> str: - return duckdb_get_list_indexes_sql(table_name, database) - - -class SQLite_IbisOperations(_DuckDBFamilyOperationsMixin, BaseIbisOperations): - """SQLite operations — concrete implementation.""" - - @property - def db_backend_name(self) -> str: - return CONST_DB_BACKEND.SQLITE - - @classmethod - def _rename_table(cls, old_name: str, new_name: str) -> None: - raise NotImplementedError(f"{CONST_DB_BACKEND.SQLITE}: rename_table not yet implemented") - - @classmethod - def _get_index_exists_sql(cls, index_name: str, table_name: str | None, database: str | None) -> str: - return sqlite_get_index_exists_sql(index_name, table_name, database) - - @classmethod - def _get_list_indexes_sql(cls, table_name: str, database: str | None) -> str: - return sqlite_get_list_indexes_sql(table_name, database) - - -class MotherDuck_IbisOperations(_DuckDBFamilyOperationsMixin, BaseIbisOperations): - """MotherDuck operations — concrete implementation.""" - - @property - def db_backend_name(self) -> str: - return CONST_DB_BACKEND.MOTHERDUCK - - @classmethod - def _rename_table(cls, old_name: str, new_name: str) -> None: - raise NotImplementedError(f"{CONST_DB_BACKEND.MOTHERDUCK}: rename_table not yet implemented") - - @classmethod - def _get_index_exists_sql(cls, index_name: str, table_name: str | None, database: str | None) -> str: - return motherduck_get_index_exists_sql(index_name, table_name, database) - - @classmethod - def _get_list_indexes_sql(cls, table_name: str, database: str | None) -> str: - return motherduck_get_list_indexes_sql(table_name, database) - - -class Trino_IbisOperations(BaseIbisOperations): - """Trino operations — connection mode is handled via DialectSpec registry.""" - - @property - def db_backend_name(self) -> str: - return CONST_DB_BACKEND.TRINO - - @classmethod - def _rename_table(cls, old_name: str, new_name: str) -> None: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: rename_table not yet implemented") - - @classmethod - def _upsert(cls, *args, **kwargs) -> None: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: upsert not supported") - - @classmethod - def create_index(cls, *args, **kwargs) -> bool: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: create_index not supported") - - @classmethod - def drop_index(cls, *args, **kwargs) -> bool: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: drop_index not supported") - - @classmethod - def index_exists(cls, *args, **kwargs) -> bool: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: index_exists not supported") - - @classmethod - def list_indexes(cls, *args, **kwargs) -> list[dict]: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: list_indexes not supported") diff --git a/src/mountainash_data/core/factories/__init__.py b/src/mountainash_data/core/factories/__init__.py deleted file mode 100644 index 3bc812f..0000000 --- a/src/mountainash_data/core/factories/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from .base_strategy_factory import BaseStrategyFactory -from .settings_type_factory_mixin import SettingsTypeFactoryMixin -from .connection_factory import ConnectionFactory -from .operations_factory import OperationsFactory -from .settings_factory import SettingsFactory - -__all__ = [ - "BaseStrategyFactory", - "SettingsTypeFactoryMixin", - "ConnectionFactory", - "OperationsFactory", - "SettingsFactory", -] diff --git a/src/mountainash_data/core/factories/base_strategy_factory.py b/src/mountainash_data/core/factories/base_strategy_factory.py deleted file mode 100644 index 8d1489c..0000000 --- a/src/mountainash_data/core/factories/base_strategy_factory.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -Base Strategy Factory with dual-generic pattern for database connections and operations. - -This module implements the factory pattern using lazy loading and runtime strategy selection, -adapted from mountainash-dataframes architecture for database connection/operation strategies. -""" - -import importlib -import logging -from abc import ABC, abstractmethod -from enum import Enum -from typing import ClassVar, Dict, Generic, Optional, Type, TypeVar - -# Type variables for dual-generic pattern -InputT = TypeVar("InputT") # Input type (SettingsParameters) -StrategyT = TypeVar("StrategyT") # Strategy type (connection or operation class) - -logger = logging.getLogger(__name__) - - -class BaseStrategyFactory(ABC, Generic[InputT, StrategyT]): - """ - Universal base factory with dual-generic design for database strategies. - - Generic Type Parameters: - InputT: Type constraint for input (e.g., SettingsParameters) - StrategyT: Strategy class type returned by factory (e.g., BaseIbisConnection, BaseIbisOperations) - - Key Features: - - Lazy loading: Strategies imported only when first used - - Runtime detection: Auto-detect backend from input - - Strategy caching: Once loaded, strategies cached for reuse - - Zero import cost: String-based configuration, no imports until needed - """ - - # Class-level caches for lazy loading - _strategy_cache: ClassVar[Dict[Enum, Type[StrategyT]]] = {} - _strategy_modules: ClassVar[Dict[Enum, str]] = {} - _strategy_classes: ClassVar[Dict[Enum, str]] = {} - - def __init__(self): - """Initialize factory and configure strategy mappings.""" - if not self._strategy_modules: - self._configure_strategy_mapping() - - @classmethod - @abstractmethod - def _configure_strategy_mapping(cls) -> None: - """ - Configure strategy mappings using ONLY strings (no imports). - - Subclasses MUST implement this to define: - cls._strategy_modules: {backend_type: "module.path"} - cls._strategy_classes: {backend_type: "ClassName"} - - Example: - cls._strategy_modules = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "mountainash_data.databases.connections.ibis", - } - cls._strategy_classes = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "Postgres_IbisConnection", - } - """ - pass - - @classmethod - @abstractmethod - def _get_strategy_key(cls, input_data: InputT, **kwargs) -> Optional[Enum]: - """ - Determine strategy key from input data. - - This is implemented by mixin classes (e.g., SettingsTypeFactoryMixin) - to detect backend type from SettingsParameters. - - Args: - input_data: Input data to analyze (e.g., SettingsParameters) - **kwargs: Additional context for detection - - Returns: - Strategy key (enum) or None if detection fails - """ - pass - - @classmethod - def _lazy_load_strategy_class(cls, strategy_key: Enum) -> Type[StrategyT]: - """ - Lazy load strategy class using runtime import. - - Args: - strategy_key: Backend type enum - - Returns: - Strategy class (not instance) - - Raises: - KeyError: If strategy not configured - ImportError: If module import fails - """ - # Check cache first - if strategy_key in cls._strategy_cache: - return cls._strategy_cache[strategy_key] - - # Ensure mappings are configured - if not cls._strategy_modules: - cls._configure_strategy_mapping() - - # Get module and class name - if strategy_key not in cls._strategy_modules: - raise KeyError( - f"No strategy configured for {strategy_key}. " - f"Available: {list(cls._strategy_modules.keys())}" - ) - - module_path = cls._strategy_modules[strategy_key] - class_name = cls._strategy_classes[strategy_key] - - # Runtime import - try: - module = importlib.import_module(module_path) - strategy_class = getattr(module, class_name) - - # Cache for future use - cls._strategy_cache[strategy_key] = strategy_class - - logger.debug( - f"Loaded strategy {class_name} from {module_path} for {strategy_key}" - ) - - return strategy_class - - except ImportError as e: - raise ImportError( - f"Failed to import {class_name} from {module_path} for {strategy_key}: {e}" - ) from e - - @classmethod - def get_strategy(cls, input_data: InputT, **kwargs) -> StrategyT: - """ - Get appropriate strategy instance for input data. - - This is the main entry point for factory usage. - - Args: - input_data: Input to analyze (e.g., SettingsParameters) - **kwargs: Additional arguments passed to strategy constructor - - Returns: - Strategy instance ready to use - - Raises: - ValueError: If backend cannot be detected - ImportError: If strategy import fails - """ - # Detect backend type - strategy_key = cls._get_strategy_key(input_data, **kwargs) - - if strategy_key is None: - raise ValueError( - f"Could not determine strategy for input: {input_data}" - ) - - # Load strategy class - strategy_class = cls._lazy_load_strategy_class(strategy_key) - - # Return instance (strategies are typically instantiated with settings) - # Note: Subclasses may override this to customize instantiation - return strategy_class # type: ignore - - @classmethod - def clear_cache(cls) -> None: - """Clear strategy cache (mainly for testing).""" - cls._strategy_cache.clear() - logger.debug(f"Cleared strategy cache for {cls.__name__}") diff --git a/src/mountainash_data/core/factories/connection_factory.py b/src/mountainash_data/core/factories/connection_factory.py deleted file mode 100644 index 50a7ac8..0000000 --- a/src/mountainash_data/core/factories/connection_factory.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -Connection Factory for settings-driven database connection creation. - -Uses SettingsParameters to auto-detect backend and create appropriate connection. -""" - -import logging -from typing import Type - -from mountainash_settings import SettingsParameters - -from ..connection import BaseDBConnection -from ..constants import CONST_DB_PROVIDER_TYPE -from .base_strategy_factory import BaseStrategyFactory -from .settings_type_factory_mixin import SettingsTypeFactoryMixin - -logger = logging.getLogger(__name__) - - -class ConnectionFactory( - SettingsTypeFactoryMixin, - BaseStrategyFactory[SettingsParameters, Type[BaseDBConnection]], -): - """ - Settings-driven factory for database connections. - - Detects backend type from SettingsParameters.settings_class and returns - appropriate connection class with lazy loading. - - Example: - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - - factory = ConnectionFactory() - connection_class = factory.get_strategy(settings_params) - connection = connection_class(db_auth_settings_parameters=settings_params) - backend = connection.connect() - """ - - # Each subclass needs its own dictionaries to avoid sharing with other factories - _strategy_cache = {} - _strategy_modules = {} - _strategy_classes = {} - - @classmethod - def _configure_strategy_mapping(cls) -> None: - """ - Configure strategy mappings using ONLY strings (no imports). - - Maps backend types to connection module paths and class names. - """ - # Ibis-based connections — point directly at backends.ibis.connection - # (bypasses the databases.connections.ibis shim chain from Phase 4) - cls._strategy_modules = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.SQLITE: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.DUCKDB: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.MOTHERDUCK: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.SNOWFLAKE: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.BIGQUERY: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.MSSQL: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.MYSQL: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.TRINO: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.PYSPARK: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.REDSHIFT: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.ORACLE: "mountainash_data.backends.ibis.connection", - # Iceberg connections — point directly at backends.iceberg - CONST_DB_PROVIDER_TYPE.PYICEBERG_REST: "mountainash_data.backends.iceberg.catalogs.rest", - } - - cls._strategy_classes = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "Postgres_IbisConnection", - CONST_DB_PROVIDER_TYPE.SQLITE: "SQLite_IbisConnection", - CONST_DB_PROVIDER_TYPE.DUCKDB: "DuckDB_IbisConnection", - CONST_DB_PROVIDER_TYPE.MOTHERDUCK: "MotherDuck_IbisConnection", - CONST_DB_PROVIDER_TYPE.SNOWFLAKE: "Snowflake_IbisConnection", - CONST_DB_PROVIDER_TYPE.BIGQUERY: "BigQuery_IbisConnection", - CONST_DB_PROVIDER_TYPE.MSSQL: "MSSQL_IbisConnection", - CONST_DB_PROVIDER_TYPE.MYSQL: "MySQL_IbisConnection", - CONST_DB_PROVIDER_TYPE.TRINO: "Trino_IbisConnection", - CONST_DB_PROVIDER_TYPE.PYSPARK: "PySpark_IbisConnection", - CONST_DB_PROVIDER_TYPE.REDSHIFT: "Redshift_IbisConnection", - CONST_DB_PROVIDER_TYPE.ORACLE: "Oracle_IbisConnection", - CONST_DB_PROVIDER_TYPE.PYICEBERG_REST: "IcebergRestConnection", - } - - @classmethod - def get_connection( - cls, settings_parameters: SettingsParameters - ) -> BaseDBConnection: - """ - Convenience method to get connection instance directly. - - Args: - settings_parameters: SettingsParameters with settings_class - - Returns: - Connection instance ready to use - - Example: - factory = ConnectionFactory() - connection = factory.get_connection(settings_params) - backend = connection.connect() - """ - connection_class = cls.get_strategy(settings_parameters) - return connection_class(db_auth_settings_parameters=settings_parameters) diff --git a/src/mountainash_data/core/factories/operations_factory.py b/src/mountainash_data/core/factories/operations_factory.py deleted file mode 100644 index 5bf1306..0000000 --- a/src/mountainash_data/core/factories/operations_factory.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -Operations Factory for settings-driven database operations creation. - -Uses SettingsParameters to auto-detect backend and create appropriate operations instance. -""" - -import logging -from typing import Type - -from mountainash_settings import SettingsParameters - -from mountainash_data.backends.ibis.operations import BaseIbisOperations -from ..constants import CONST_DB_PROVIDER_TYPE -from .base_strategy_factory import BaseStrategyFactory -from .settings_type_factory_mixin import SettingsTypeFactoryMixin - -logger = logging.getLogger(__name__) - - -class OperationsFactory( - SettingsTypeFactoryMixin, - BaseStrategyFactory[SettingsParameters, Type[BaseIbisOperations]], -): - """ - Settings-driven factory for database operations. - - Detects backend type from SettingsParameters.settings_class and returns - appropriate operations class with lazy loading. - - Example: - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - - factory = OperationsFactory() - operations_class = factory.get_strategy(settings_params) - operations = operations_class(db_auth_settings_parameters=settings_params) - operations.create_table(backend, "my_table", dataframe) - """ - - # Each subclass needs its own dictionaries to avoid sharing with other factories - _strategy_cache = {} - _strategy_modules = {} - _strategy_classes = {} - - @classmethod - def _configure_strategy_mapping(cls) -> None: - """ - Configure strategy mappings using ONLY strings (no imports). - - Maps backend types to operations module paths and class names. - """ - # All ibis backends point directly at backends.ibis.operations - # (bypasses the databases.operations.ibis shim chain from Phase 4). - # Per-backend ops classes (SQLite_IbisOperations, etc.) are in the - # same module; POSTGRESQL/SNOWFLAKE/etc. fall back to BaseIbisOperations. - cls._strategy_modules = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.SQLITE: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.DUCKDB: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.MOTHERDUCK: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.SNOWFLAKE: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.BIGQUERY: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.MSSQL: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.MYSQL: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.TRINO: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.PYSPARK: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.REDSHIFT: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.ORACLE: "mountainash_data.backends.ibis.operations", - # Iceberg REST — operations are merged into IcebergRestConnection - CONST_DB_PROVIDER_TYPE.PYICEBERG_REST: "mountainash_data.backends.iceberg.catalogs.rest", - } - - cls._strategy_classes = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.SQLITE: "SQLite_IbisOperations", - CONST_DB_PROVIDER_TYPE.DUCKDB: "DuckDB_IbisOperations", - CONST_DB_PROVIDER_TYPE.MOTHERDUCK: "MotherDuck_IbisOperations", - CONST_DB_PROVIDER_TYPE.SNOWFLAKE: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.BIGQUERY: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.MSSQL: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.MYSQL: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.TRINO: "Trino_IbisOperations", - CONST_DB_PROVIDER_TYPE.PYSPARK: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.REDSHIFT: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.ORACLE: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.PYICEBERG_REST: "IcebergRestConnection", - } - - @classmethod - def get_operations( - cls, settings_parameters: SettingsParameters - ) -> BaseIbisOperations: - """ - Convenience method to get operations instance directly. - - Args: - settings_parameters: SettingsParameters with settings_class - - Returns: - Operations instance ready to use - - Example: - factory = OperationsFactory() - operations = factory.get_operations(settings_params) - operations.create_table(backend, "my_table", dataframe) - """ - operations_class = cls.get_strategy(settings_parameters) - return operations_class(db_auth_settings_parameters=settings_parameters) diff --git a/src/mountainash_data/core/factories/settings_factory.py b/src/mountainash_data/core/factories/settings_factory.py deleted file mode 100644 index 9408e89..0000000 --- a/src/mountainash_data/core/factories/settings_factory.py +++ /dev/null @@ -1,206 +0,0 @@ -""" -Settings Factory for auto-detecting and creating appropriate settings classes. - -Provides utilities to auto-detect database backend from connection strings -and create the corresponding settings class. -""" - -import logging -import re -from typing import Dict, Type -from urllib.parse import urlparse - -from mountainash_settings import MountainAshBaseSettings - -from ..constants import CONST_DB_PROVIDER_TYPE - -logger = logging.getLogger(__name__) - - -class SettingsFactory: - """ - Factory for auto-detecting and loading appropriate settings classes. - - Provides intelligent backend detection from connection strings and - automatic settings class selection. - """ - - # Mapping of URL schemes to backend types - SCHEME_MAP: Dict[str, CONST_DB_PROVIDER_TYPE] = { - "postgresql": CONST_DB_PROVIDER_TYPE.POSTGRESQL, - "postgres": CONST_DB_PROVIDER_TYPE.POSTGRESQL, - "sqlite": CONST_DB_PROVIDER_TYPE.SQLITE, - "duckdb": CONST_DB_PROVIDER_TYPE.DUCKDB, - "motherduck": CONST_DB_PROVIDER_TYPE.MOTHERDUCK, - "md": CONST_DB_PROVIDER_TYPE.MOTHERDUCK, - "snowflake": CONST_DB_PROVIDER_TYPE.SNOWFLAKE, - "bigquery": CONST_DB_PROVIDER_TYPE.BIGQUERY, - "mssql": CONST_DB_PROVIDER_TYPE.MSSQL, - "mysql": CONST_DB_PROVIDER_TYPE.MYSQL, - "trino": CONST_DB_PROVIDER_TYPE.TRINO, - "pyspark": CONST_DB_PROVIDER_TYPE.PYSPARK, - "redshift": CONST_DB_PROVIDER_TYPE.REDSHIFT, - "oracle": CONST_DB_PROVIDER_TYPE.ORACLE, - } - - # Mapping of backend types to settings classes (lazy loaded) - SETTINGS_CLASS_MAP: Dict[CONST_DB_PROVIDER_TYPE, Type[MountainAshBaseSettings]] = {} - - @classmethod - def _ensure_settings_classes_loaded(cls) -> None: - """ - Lazy load settings class mappings. - - This avoids circular imports and reduces initial load time. - """ - if cls.SETTINGS_CLASS_MAP: - return # Already loaded - - # Lazy import settings classes only when needed - from ..settings import ( - PostgreSQLAuthSettings, - SQLiteAuthSettings, - DuckDBAuthSettings, - MotherDuckAuthSettings, - SnowflakeAuthSettings, - BigQueryAuthSettings, - MSSQLAuthSettings, - MySQLAuthSettings, - TrinoAuthSettings, - PySparkAuthSettings, - RedshiftAuthSettings, - PyIcebergRestAuthSettings, - ) - - cls.SETTINGS_CLASS_MAP = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: PostgreSQLAuthSettings, - CONST_DB_PROVIDER_TYPE.SQLITE: SQLiteAuthSettings, - CONST_DB_PROVIDER_TYPE.DUCKDB: DuckDBAuthSettings, - CONST_DB_PROVIDER_TYPE.MOTHERDUCK: MotherDuckAuthSettings, - CONST_DB_PROVIDER_TYPE.SNOWFLAKE: SnowflakeAuthSettings, - CONST_DB_PROVIDER_TYPE.BIGQUERY: BigQueryAuthSettings, - CONST_DB_PROVIDER_TYPE.MSSQL: MSSQLAuthSettings, - CONST_DB_PROVIDER_TYPE.MYSQL: MySQLAuthSettings, - CONST_DB_PROVIDER_TYPE.TRINO: TrinoAuthSettings, - CONST_DB_PROVIDER_TYPE.PYSPARK: PySparkAuthSettings, - CONST_DB_PROVIDER_TYPE.REDSHIFT: RedshiftAuthSettings, - CONST_DB_PROVIDER_TYPE.PYICEBERG_REST: PyIcebergRestAuthSettings, - } - - logger.debug("Settings class mappings loaded") - - @classmethod - def from_backend_type( - cls, backend_type: CONST_DB_PROVIDER_TYPE, **kwargs - ) -> MountainAshBaseSettings: - """ - Create settings instance from backend type. - - Args: - backend_type: Database backend type enum - **kwargs: Arguments passed to settings constructor - - Returns: - Settings instance for the backend - - Raises: - KeyError: If backend type not supported - """ - cls._ensure_settings_classes_loaded() - - if backend_type not in cls.SETTINGS_CLASS_MAP: - raise KeyError( - f"No settings class for {backend_type}. " - f"Available: {list(cls.SETTINGS_CLASS_MAP.keys())}" - ) - - settings_class = cls.SETTINGS_CLASS_MAP[backend_type] - - # Auto-inject auth=NoAuth() for ConnectionProfile subclasses that only - # support NoAuth and haven't been passed an auth kwarg by the caller. - if "auth" not in kwargs: - try: - from ..settings.profile import ConnectionProfile - from ..settings.auth import NoAuth - if ( - isinstance(settings_class, type) - and issubclass(settings_class, ConnectionProfile) - ): - descriptor = getattr(settings_class, "__descriptor__", None) - if descriptor is not None and descriptor.auth_modes == [NoAuth]: - kwargs["auth"] = NoAuth() - except (ImportError, AttributeError): - pass - - return settings_class(**kwargs) - - @classmethod - def from_connection_string( - cls, connection_url: str, **kwargs - ) -> MountainAshBaseSettings: - """ - Auto-detect backend from connection URL and create settings. - - Args: - connection_url: Database connection URL - **kwargs: Arguments passed to settings constructor - - Returns: - Settings instance for detected backend - - Raises: - ValueError: If backend cannot be detected from URL - - Example: - settings = SettingsFactory.from_connection_string( - "postgresql://user:pass@localhost:5432/db", - config_files=["postgres.env"] - ) - """ - backend_type = cls.detect_backend_from_url(connection_url) - return cls.from_backend_type(backend_type, **kwargs) - - @classmethod - def detect_backend_from_url(cls, connection_url: str) -> CONST_DB_PROVIDER_TYPE: - """ - Detect backend type from connection URL. - - Args: - connection_url: Database connection URL - - Returns: - Detected backend type - - Raises: - ValueError: If backend cannot be detected - - Example: - backend = SettingsFactory.detect_backend_from_url( - "postgresql://localhost/db" - ) - # Returns: CONST_DB_PROVIDER_TYPE.POSTGRESQL - """ - parsed = urlparse(connection_url) - scheme = parsed.scheme.lower() - - # Check exact scheme match - if scheme in cls.SCHEME_MAP: - return cls.SCHEME_MAP[scheme] - - # Check for special patterns (e.g., duckdb://md: for MotherDuck) - if scheme == "duckdb" and connection_url.startswith("duckdb://md:"): - return CONST_DB_PROVIDER_TYPE.MOTHERDUCK - - # Pattern matching for complex URLs - for pattern, backend_type in cls.SCHEME_MAP.items(): - if re.match(pattern, scheme): - logger.debug( - f"Pattern matched: {scheme} → {backend_type} (pattern: {pattern})" - ) - return backend_type - - raise ValueError( - f"Cannot detect backend from URL scheme: {scheme}. " - f"URL: {connection_url}. " - f"Supported schemes: {list(cls.SCHEME_MAP.keys())}" - ) diff --git a/src/mountainash_data/core/factories/settings_type_factory_mixin.py b/src/mountainash_data/core/factories/settings_type_factory_mixin.py deleted file mode 100644 index ae26d10..0000000 --- a/src/mountainash_data/core/factories/settings_type_factory_mixin.py +++ /dev/null @@ -1,184 +0,0 @@ -""" -Settings Type Detection Mixin for Factory Pattern. - -Detects database backend type from SettingsParameters.settings_class, -similar to DataFrame type detection in mountainash-dataframes. -""" - -import logging -import re -from typing import Optional, Type - -from mountainash_settings import SettingsParameters, MountainAshBaseSettings - -from ..constants import CONST_DB_PROVIDER_TYPE - -logger = logging.getLogger(__name__) - - -class SettingsTypeFactoryMixin: - """ - Mixin for detecting database backend type from settings class. - - Uses three-tier detection system: - 1. Exact Match (fast path): Direct mapping of known settings classes - 2. Pattern Matching (flexible): Regex-based class name matching - 3. Logging: Track unmapped types for future registration - """ - - # Layer 1: Exact Match - Direct settings class mapping (O(1) lookup) - TYPE_MAP = { - # Map settings class → backend type - # These will be populated when settings classes are imported - } - - # Layer 2: Pattern Matching - Class name patterns for flexible detection - PATTERN_MAP = { - # Regex pattern → backend type - r"PostgreSQL.*Settings": CONST_DB_PROVIDER_TYPE.POSTGRESQL, - r"SQLite.*Settings": CONST_DB_PROVIDER_TYPE.SQLITE, - r"DuckDB.*Settings": CONST_DB_PROVIDER_TYPE.DUCKDB, - r"MotherDuck.*Settings": CONST_DB_PROVIDER_TYPE.MOTHERDUCK, - r"Snowflake.*Settings": CONST_DB_PROVIDER_TYPE.SNOWFLAKE, - r"BigQuery.*Settings": CONST_DB_PROVIDER_TYPE.BIGQUERY, - r"MSSQL.*Settings": CONST_DB_PROVIDER_TYPE.MSSQL, - r"MySQL.*Settings": CONST_DB_PROVIDER_TYPE.MYSQL, - r"Trino.*Settings": CONST_DB_PROVIDER_TYPE.TRINO, - r"PySpark.*Settings": CONST_DB_PROVIDER_TYPE.PYSPARK, - r"Redshift.*Settings": CONST_DB_PROVIDER_TYPE.REDSHIFT, - r"Oracle.*Settings": CONST_DB_PROVIDER_TYPE.ORACLE, - r"PyIceberg.*Settings": CONST_DB_PROVIDER_TYPE.PYICEBERG_REST, - } - - @classmethod - def _get_strategy_key( - cls, settings_parameters: SettingsParameters, **kwargs - ) -> Optional[CONST_DB_PROVIDER_TYPE]: - """ - Detect backend type from SettingsParameters. - - Args: - settings_parameters: SettingsParameters with settings_class - **kwargs: Additional context (unused) - - Returns: - Backend type enum or None if detection fails - """ - if settings_parameters is None: - logger.warning("SettingsParameters is None, cannot detect backend") - return None - - settings_class = settings_parameters.settings_class - - if settings_class is None: - logger.warning("settings_class is None in SettingsParameters") - return None - - # Layer 1: Exact Match (Fast Path) - backend_type = cls._detect_from_exact_match(settings_class) - if backend_type: - return backend_type - - # Layer 2: Pattern Matching (Flexible Fallback) - backend_type = cls._detect_from_pattern_match(settings_class) - if backend_type: - # Auto-register for future fast-path lookup - cls._register_settings_class(settings_class, backend_type) - return backend_type - - # Layer 3: Logging (Detection Failed) - cls._log_unmapped_settings_class(settings_class) - return None - - @classmethod - def _detect_from_exact_match( - cls, settings_class: Type[MountainAshBaseSettings] - ) -> Optional[CONST_DB_PROVIDER_TYPE]: - """ - Fast path: Direct lookup in TYPE_MAP. - - Args: - settings_class: Settings class to detect - - Returns: - Backend type or None - """ - return cls.TYPE_MAP.get(settings_class) - - @classmethod - def _detect_from_pattern_match( - cls, settings_class: Type[MountainAshBaseSettings] - ) -> Optional[CONST_DB_PROVIDER_TYPE]: - """ - Flexible fallback: Regex pattern matching on class name. - - Args: - settings_class: Settings class to detect - - Returns: - Backend type or None - """ - class_name = settings_class.__name__ - - for pattern, backend_type in cls.PATTERN_MAP.items(): - if re.match(pattern, class_name): - logger.debug( - f"Pattern matched: {class_name} → {backend_type} (pattern: {pattern})" - ) - return backend_type - - return None - - @classmethod - def _register_settings_class( - cls, - settings_class: Type[MountainAshBaseSettings], - backend_type: CONST_DB_PROVIDER_TYPE, - ) -> None: - """ - Register settings class in TYPE_MAP for future fast-path lookup. - - Args: - settings_class: Settings class to register - backend_type: Detected backend type - """ - cls.TYPE_MAP[settings_class] = backend_type - logger.debug( - f"Auto-registered {settings_class.__name__} → {backend_type} for fast lookup" - ) - - @classmethod - def _log_unmapped_settings_class( - cls, settings_class: Type[MountainAshBaseSettings] - ) -> None: - """ - Log unmapped settings class for debugging. - - Args: - settings_class: Unmapped settings class - """ - class_name = settings_class.__name__ - module_name = settings_class.__module__ - - logger.warning( - f"Unmapped settings class: {class_name} (module: {module_name}). " - f"Consider adding to TYPE_MAP or PATTERN_MAP." - ) - - @classmethod - def register_settings_class_mapping( - cls, - settings_class: Type[MountainAshBaseSettings], - backend_type: CONST_DB_PROVIDER_TYPE, - ) -> None: - """ - Manually register a settings class mapping. - - Useful for custom settings classes or explicit registration. - - Args: - settings_class: Settings class to register - backend_type: Backend type to map to - """ - cls.TYPE_MAP[settings_class] = backend_type - logger.info(f"Manually registered {settings_class.__name__} → {backend_type}") diff --git a/src/mountainash_data/core/utils.py b/src/mountainash_data/core/utils.py deleted file mode 100644 index de16714..0000000 --- a/src/mountainash_data/core/utils.py +++ /dev/null @@ -1,224 +0,0 @@ -""" -High-level database utilities for settings-driven connection and operations management. - -Provides a unified API for creating database connections and operations -with automatic backend detection and lazy loading. -""" - -import logging -from typing import Any, Optional - -from mountainash_settings import SettingsParameters, MountainAshBaseSettings - -from .connection import BaseDBConnection -from mountainash_data.backends.ibis.operations import BaseIbisOperations -from .constants import CONST_DB_PROVIDER_TYPE -from .factories import ConnectionFactory, OperationsFactory, SettingsFactory - -logger = logging.getLogger(__name__) - - -class DatabaseUtils: - """ - High-level API for settings-driven database operations. - - All operations driven by SettingsParameters, with automatic backend - detection and lazy loading of backend-specific implementations. - - Example: - # Create settings parameters - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - - # Auto-detect and create connection - connection = DatabaseUtils.create_connection(settings_params) - backend = connection.connect() - - # Auto-detect and create operations - operations = DatabaseUtils.create_operations(settings_params) - operations.create_table(backend, "my_table", dataframe) - """ - - @classmethod - def create_connection( - cls, settings_parameters: SettingsParameters - ) -> BaseDBConnection: - """ - Create database connection from settings parameters. - - Auto-detects backend from settings_class and returns appropriate connection. - - Args: - settings_parameters: SettingsParameters with settings_class - - Returns: - Connection instance ready to use - - Example: - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - connection = DatabaseUtils.create_connection(settings_params) - backend = connection.connect() - """ - factory = ConnectionFactory() - return factory.get_connection(settings_parameters) - - @classmethod - def create_operations( - cls, settings_parameters: SettingsParameters - ) -> BaseIbisOperations: - """ - Create database operations from settings parameters. - - Auto-detects backend from settings_class and returns appropriate operations. - - Args: - settings_parameters: SettingsParameters with settings_class - - Returns: - Operations instance ready to use - - Example: - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - operations = DatabaseUtils.create_operations(settings_params) - operations.upsert(backend, "my_table", df, natural_key_columns=["id"]) - """ - factory = OperationsFactory() - return factory.get_operations(settings_parameters) - - @classmethod - def create_backend( - cls, settings_parameters: SettingsParameters, **connect_kwargs - ) -> Any: - """ - Create connection and connect to backend in one step. - - Convenience method combining connection creation and connection. - - Args: - settings_parameters: SettingsParameters with settings_class - **connect_kwargs: Additional arguments passed to connect() - - Returns: - Connected backend (Ibis backend or PyIceberg catalog) - - Example: - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - backend = DatabaseUtils.create_backend(settings_params) - tables = backend.list_tables() - """ - connection = cls.create_connection(settings_parameters) - return connection.connect(**connect_kwargs) - - @classmethod - def create_settings_from_url( - cls, connection_url: str, **kwargs - ) -> MountainAshBaseSettings: - """ - Auto-detect backend from URL and create appropriate settings. - - Args: - connection_url: Database connection URL - **kwargs: Arguments passed to settings constructor - - Returns: - Settings instance for detected backend - - Example: - settings = DatabaseUtils.create_settings_from_url( - "postgresql://user:pass@localhost:5432/db", - config_files=["postgres.env"] - ) - """ - return SettingsFactory.from_connection_string(connection_url, **kwargs) - - @classmethod - def create_settings_from_backend_type( - cls, backend_type: CONST_DB_PROVIDER_TYPE, **kwargs - ) -> MountainAshBaseSettings: - """ - Create settings for specific backend type. - - Args: - backend_type: Database backend type enum - **kwargs: Arguments passed to settings constructor - - Returns: - Settings instance for the backend - - Example: - settings = DatabaseUtils.create_settings_from_backend_type( - CONST_DB_PROVIDER_TYPE.POSTGRESQL, - config_files=["postgres.env"] - ) - """ - return SettingsFactory.from_backend_type(backend_type, **kwargs) - - @classmethod - def detect_backend_from_url(cls, connection_url: str) -> CONST_DB_PROVIDER_TYPE: - """ - Detect backend type from connection URL. - - Args: - connection_url: Database connection URL - - Returns: - Detected backend type - - Example: - backend_type = DatabaseUtils.detect_backend_from_url( - "postgresql://localhost/db" - ) - """ - return SettingsFactory.detect_backend_from_url(connection_url) - - @classmethod - def create_from_url( - cls, - connection_url: str, - config_files: Optional[list] = None, - **settings_kwargs, - ) -> tuple[BaseDBConnection, Any]: - """ - Complete workflow: URL → settings → connection → backend. - - Convenience method for quick setup from connection URL. - - Args: - connection_url: Database connection URL - config_files: Optional configuration files - **settings_kwargs: Additional settings constructor arguments - - Returns: - Tuple of (connection, connected_backend) - - Example: - connection, backend = DatabaseUtils.create_from_url( - "postgresql://user:pass@localhost:5432/db", - config_files=["postgres.env"] - ) - tables = backend.list_tables() - """ - # Create settings from URL - settings = cls.create_settings_from_url( - connection_url, config_files=config_files, **settings_kwargs - ) - - # Create settings parameters - settings_params = settings.extract_settings_parameters() - - # Create connection and connect - connection = cls.create_connection(settings_params) - backend = connection.connect() - - return connection, backend From 57ec49fcd70602752d083ec5f42b010561b57baf Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 27 Apr 2026 22:02:07 +1000 Subject: [PATCH 12/12] test: rewrite all tests to use IbisBackend, delete legacy test files Delete factory tests, DatabaseUtils tests, legacy connection tests, and legacy operations tests. Rewrite integration tests and settings parametrized tests to use IbisBackend directly. Update package structure tests to verify removed exports are gone. Co-Authored-By: Claude Sonnet 4.6 --- .../test_end_to_end_workflows.py | 354 +++------- tests/test_unit/core/test_protocol.py | 4 +- .../ibis/test_connection_lifecycle.py | 236 ------- .../connections/test_base_db_connection.py | 31 - .../ibis/test_base_ibis_operations.py | 333 ---------- .../operations/test_upsert_and_indexes.py | 626 ------------------ .../settings/test_settings_parametrized.py | 34 +- .../databases/test_database_connections.py | 277 -------- .../test_unit/databases/test_ibis_backends.py | 137 ---- .../factories/test_connection_factory.py | 246 ------- .../factories/test_operations_factory.py | 239 ------- .../factories/test_settings_factory.py | 257 ------- tests/test_unit/test_database_utils.py | 269 -------- tests/test_unit/test_mountainash_data.py | 66 +- 14 files changed, 122 insertions(+), 2987 deletions(-) delete mode 100644 tests/test_unit/databases/connections/ibis/test_connection_lifecycle.py delete mode 100644 tests/test_unit/databases/connections/test_base_db_connection.py delete mode 100644 tests/test_unit/databases/operations/ibis/test_base_ibis_operations.py delete mode 100644 tests/test_unit/databases/operations/test_upsert_and_indexes.py delete mode 100644 tests/test_unit/databases/test_database_connections.py delete mode 100644 tests/test_unit/databases/test_ibis_backends.py delete mode 100644 tests/test_unit/factories/test_connection_factory.py delete mode 100644 tests/test_unit/factories/test_operations_factory.py delete mode 100644 tests/test_unit/factories/test_settings_factory.py delete mode 100644 tests/test_unit/test_database_utils.py diff --git a/tests/test_integration/test_end_to_end_workflows.py b/tests/test_integration/test_end_to_end_workflows.py index 04d0107..3f7c718 100644 --- a/tests/test_integration/test_end_to_end_workflows.py +++ b/tests/test_integration/test_end_to_end_workflows.py @@ -1,288 +1,112 @@ -"""End-to-end integration tests for complete workflows.""" +"""End-to-end integration tests using IbisBackend.""" import pytest -from pathlib import Path -from mountainash_data.core.utils import DatabaseUtils -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings -from mountainash_data.core.constants import CONST_DB_PROVIDER_TYPE +import polars as pl +from mountainash_data.backends.ibis.backend import IbisBackend +from mountainash_data.core.settings import DuckDBAuthSettings from mountainash_settings import SettingsParameters @pytest.mark.integration -class TestCompleteWorkflowFromURL: - """Test complete workflow starting from connection URL.""" - - def test_sqlite_url_to_backend_workflow(self, temp_sqlite_db): - """Test complete SQLite workflow from URL to backend operations.""" - url = f"sqlite:///{temp_sqlite_db}" - - # Complete workflow: URL → settings → connection → backend - connection, backend = DatabaseUtils.create_from_url(url) - - # Verify backend is connected and functional - assert backend is not None - tables = backend.list_tables() - assert isinstance(tables, list) - # Note: test_table from fixture may not be visible in new connection - # Instead, test that we can create and query tables - - # Can create new table - backend.create_table('new_table', {'id': [1, 2, 3]}, overwrite=True) - assert 'new_table' in backend.list_tables() - - # Cleanup - connection.disconnect() - - def test_duckdb_url_to_backend_workflow(self): - """Test complete DuckDB workflow from URL to backend operations.""" - url = "duckdb:///:memory:" - - connection, backend = DatabaseUtils.create_from_url(url) - - # Verify backend is connected and functional - assert backend is not None - - # Can create and query tables - backend.create_table('test', {'id': [1, 2, 3], 'value': [10, 20, 30]}, overwrite=True) - tables = backend.list_tables() - assert 'test' in tables - - # Can query table - table = backend.table('test') - assert table is not None - - # Cleanup - connection.disconnect() - - -@pytest.mark.integration -class TestFactoryDrivenWorkflow: - """Test workflows driven by factory pattern.""" - - def test_settings_factory_to_connection_workflow(self): - """Test Settings Factory → Connection Factory workflow.""" - from mountainash_data.core.factories import SettingsFactory, ConnectionFactory - - # Create settings from backend type - settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:" - ) - - # Extract parameters - settings_params = settings.extract_settings_parameters() - - # Create connection from settings - connection = ConnectionFactory.get_connection(settings_params) - - # Connect and verify - backend = connection.connect() - assert backend is not None - - # Can perform operations - backend.create_table('factory_test', {'id': [1, 2]}, overwrite=True) - assert 'factory_test' in backend.list_tables() - - # Cleanup - connection.disconnect() - - def test_url_detection_to_connection_workflow(self): - """Test URL detection → Settings → Connection workflow.""" - from mountainash_data.core.factories import SettingsFactory, ConnectionFactory - - url = "sqlite:///:memory:" - - # Detect backend type - backend_type = SettingsFactory.detect_backend_from_url(url) - assert backend_type == CONST_DB_PROVIDER_TYPE.SQLITE - - # Create settings from URL - settings = SettingsFactory.from_connection_string(url) - - # Create connection - settings_params = settings.extract_settings_parameters() - connection = ConnectionFactory.get_connection(settings_params) - - # Verify functionality - backend = connection.connect() - assert backend is not None - - # Cleanup - connection.disconnect() - - -@pytest.mark.integration -class TestDatabaseUtilsEndToEnd: - """Test DatabaseUtils high-level API end-to-end.""" - - def test_create_connection_to_operations_workflow(self): - """Test creating connection and performing operations.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - # Create connection - connection = DatabaseUtils.create_connection(settings_params) - - # Connect - backend = connection.connect() - - # Perform operations - backend.create_table('workflow_test', {'id': [1, 2, 3], 'value': [10, 20, 30]}, overwrite=True) - - # Verify - tables = backend.list_tables() - assert 'workflow_test' in tables - - table = backend.table('workflow_test') - assert table is not None - - # Cleanup - connection.disconnect() - - def test_create_backend_shortcut_workflow(self): - """Test create_backend convenience method.""" - settings_params = SettingsParameters.create( +class TestIbisBackendWorkflow: + """Test complete workflows through IbisBackend.""" + + def test_sqlite_dialect_workflow(self): + """Full workflow with dialect= keyword.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("users", {"id": [1, 2], "name": ["a", "b"]}) + assert "users" in backend.list_tables() + tbl = backend.table("users") + assert tbl is not None + + def test_sqlite_url_workflow(self, tmp_path): + """Full workflow from URL.""" + db_file = tmp_path / "test.db" + with IbisBackend(f"sqlite:///{db_file}") as backend: + backend.create_table("t", {"id": [1, 2, 3]}) + assert "t" in backend.list_tables() + assert db_file.exists() + + def test_duckdb_settings_workflow(self): + """Full workflow from SettingsParameters.""" + from mountainash_data.core.settings import NoAuth + params = SettingsParameters.create( settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - # Single call to get connected backend - backend = DatabaseUtils.create_backend(settings_params) - - # Should be ready to use - assert backend is not None - - # Can immediately perform operations - backend.create_table('shortcut_test', {'id': [1, 2]}, overwrite=True) - tables = backend.list_tables() - assert 'shortcut_test' in tables - - -@pytest.mark.integration -@pytest.mark.parametrize("url,expected_backend_type", [ - ("sqlite:///:memory:", CONST_DB_PROVIDER_TYPE.SQLITE), - ("duckdb:///:memory:", CONST_DB_PROVIDER_TYPE.DUCKDB), -]) -class TestCrossBackendWorkflows: - """Test workflows work across different backends.""" - - def test_url_to_backend_works_for_all_backends(self, url, expected_backend_type): - """Test that URL workflow works for different backends.""" - connection, backend = DatabaseUtils.create_from_url(url) - - # Verify correct backend type detected - detected = DatabaseUtils.detect_backend_from_url(url) - assert detected == expected_backend_type - - # Verify backend works - assert backend is not None - backend.create_table('cross_backend_test', {'id': [1, 2, 3]}, overwrite=True) - tables = backend.list_tables() - assert 'cross_backend_test' in tables - - # Cleanup - connection.disconnect() - - def test_backend_type_to_connection_works_for_all_backends(self, url, expected_backend_type): - """Test that backend type workflow works for different backends.""" - settings = DatabaseUtils.create_settings_from_backend_type( - expected_backend_type, - DATABASE=":memory:" + DATABASE=":memory:", + auth=NoAuth(), ) - - settings_params = settings.extract_settings_parameters() - connection = DatabaseUtils.create_connection(settings_params) - backend = connection.connect() - - # Verify backend works - assert backend is not None - backend.create_table('backend_type_test', {'id': [1]}, overwrite=True) - assert 'backend_type_test' in backend.list_tables() - - # Cleanup - connection.disconnect() - - -@pytest.mark.integration -class TestDataTransferWorkflows: - """Test workflows involving data transfer between backends.""" - - def test_create_table_and_query_workflow(self): - """Test creating table and querying it.""" - backend = DatabaseUtils.create_backend( - SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} + with IbisBackend(params) as backend: + backend.create_table("t", {"id": [1, 2]}) + assert "t" in backend.list_tables() + + def test_fluent_chaining_workflow(self): + """Fluent API chaining across multiple operations.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + ( + backend + .create_table("users", {"id": [1], "email": ["a@b.com"]}) + .create_index("users", ["email"], unique=True) + .create_table("orders", {"id": [1], "user_id": [1]}) ) - ) + assert sorted(backend.list_tables()) == ["orders", "users"] - # Create table with data - data = { - 'id': [1, 2, 3, 4, 5], - 'name': ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'], - 'value': [100.5, 200.7, 300.9, 400.2, 500.8] - } + def test_inspect_workflow(self): + """Inspection methods work through the backend.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + info = backend.inspect_table("t") + assert info.name == "t" + assert len(info.columns) == 2 - backend.create_table('data_transfer_test', data, overwrite=True) + def test_ibis_connection_seam(self): + """ibis_connection() provides the seam to mountainash-expressions.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2]}) + raw = backend.ibis_connection() + tbl = raw.table("t") + assert tbl is not None - # Query table - table = backend.table('data_transfer_test') - assert table is not None - # Verify data - result = table.execute() - assert len(result) == 5 +@pytest.mark.integration +class TestDuckDBOperationsWorkflow: + """Test DuckDB-specific operations (upsert, indexes) through IbisBackend.""" - def test_insert_and_retrieve_workflow(self): - """Test inserting and retrieving data.""" - backend = DatabaseUtils.create_backend( - SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - ) + def test_upsert_insert_new_rows(self): + """Upsert inserts new rows when no conflicts exist.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"]}) + backend.create_table("users", initial) + backend.create_unique_index("users", ["id"]) - # Create empty table structure - backend.create_table('insert_test', {'id': [1], 'value': [10]}, overwrite=True) + new_data = pl.DataFrame({"id": [3, 4], "name": ["Charlie", "Diana"]}) + backend.upsert("users", new_data, conflict_columns=["id"]) - # Insert more data - backend.insert('insert_test', {'id': [2, 3], 'value': [20, 30]}) + count = backend.run_sql("SELECT COUNT(*) as cnt FROM users").to_polars()["cnt"][0] + assert count == 4 - # Retrieve and verify - table = backend.table('insert_test') - assert table is not None + def test_upsert_update_existing_rows(self): + """Upsert updates existing rows on conflict.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"], "score": [100, 200]}) + backend.create_table("users", initial) + backend.create_unique_index("users", ["id"]) + update = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"], "score": [150, 250]}) + backend.upsert("users", update, conflict_columns=["id"]) -@pytest.mark.integration -class TestErrorHandlingWorkflows: - """Test error handling in complete workflows.""" + result = backend.run_sql("SELECT score FROM users ORDER BY id").to_polars() + assert list(result["score"]) == [150, 250] - def test_invalid_url_workflow(self): - """Test that invalid URL is handled appropriately.""" - with pytest.raises((ValueError, KeyError, AttributeError, NotImplementedError)): - DatabaseUtils.create_from_url("invalid://url") + def test_index_lifecycle(self): + """Create, check, list, drop index.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2], "name": ["a", "b"]}) - @pytest.mark.skip(reason="Settings classes have optional/default values") - def test_missing_required_config_workflow(self): - """Test that missing config is handled.""" - # NOTE: Settings have default values, so this doesn't raise - with pytest.raises((ValueError, KeyError, TypeError)): - SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={} # Missing DATABASE - ) + backend.create_index("t", ["name"], index_name="idx_name") + assert backend.index_exists("idx_name") is True - def test_nonexistent_table_query_workflow(self): - """Test querying non-existent table is handled.""" - backend = DatabaseUtils.create_backend( - SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - ) + indexes = backend.list_indexes("t") + assert any(idx.get("name") == "idx_name" for idx in indexes) - # Try to query non-existent table - with pytest.raises((Exception, AttributeError)): - backend.table('definitely_does_not_exist').execute() + backend.drop_index("idx_name") + assert backend.index_exists("idx_name") is False diff --git a/tests/test_unit/core/test_protocol.py b/tests/test_unit/core/test_protocol.py index 33f91ca..5e6826f 100644 --- a/tests/test_unit/core/test_protocol.py +++ b/tests/test_unit/core/test_protocol.py @@ -14,7 +14,7 @@ NamespaceInfo, TableInfo, ) -from mountainash_data.core.protocol import Backend, Connection +from mountainash_data.core.protocol import Backend class _FakeConnection: @@ -57,7 +57,7 @@ def test_fake_backend_satisfies_protocol(): def test_fake_connection_satisfies_protocol(): - conn: Connection = _FakeConnection() + conn = _FakeConnection() assert conn.list_namespaces() == ["public"] diff --git a/tests/test_unit/databases/connections/ibis/test_connection_lifecycle.py b/tests/test_unit/databases/connections/ibis/test_connection_lifecycle.py deleted file mode 100644 index 83f5344..0000000 --- a/tests/test_unit/databases/connections/ibis/test_connection_lifecycle.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Parametrized tests for Ibis connection lifecycle across backends.""" - -import pytest -from mountainash_data.backends.ibis.connection import ( - SQLite_IbisConnection, - DuckDB_IbisConnection, -) -from mountainash_data.core.connection import BaseDBConnection -from mountainash_data.core.settings import ( - SQLiteAuthSettings, - DuckDBAuthSettings, - NoAuth, -) -from mountainash_settings import SettingsParameters - - -@pytest.mark.unit -@pytest.mark.parametrize("connection_class,settings_class,db_config", [ - (SQLite_IbisConnection, SQLiteAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), - (DuckDB_IbisConnection, DuckDBAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), -]) -class TestConnectionLifecycle: - """Test connection lifecycle for all Ibis backends.""" - - def test_connection_instantiation(self, connection_class, settings_class, db_config): - """Test that connection can be instantiated.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - - assert connection is not None - assert isinstance(connection, BaseDBConnection) - assert isinstance(connection, connection_class) - - def test_connection_not_connected_initially(self, connection_class, settings_class, db_config): - """Test that connection is not connected initially.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - - # Initially should not be connected - assert not connection.is_connected() - - def test_connect_method(self, connection_class, settings_class, db_config): - """Test that connect method works.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - backend = connection.connect() - - assert backend is not None - assert connection.is_connected() - - # Cleanup - connection.disconnect() - - def test_disconnect_method(self, connection_class, settings_class, db_config): - """Test that disconnect method works.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - connection.connect() - connection.disconnect() - - assert not connection.is_connected() - - def test_reconnect_after_disconnect(self, connection_class, settings_class, db_config): - """Test that connection can reconnect after disconnect.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - - # First connection - backend1 = connection.connect() - assert connection.is_connected() - - # Disconnect - connection.disconnect() - assert not connection.is_connected() - - # Reconnect - backend2 = connection.connect() - assert connection.is_connected() - assert backend2 is not None - - # Cleanup - connection.disconnect() - - def test_multiple_connect_calls(self, connection_class, settings_class, db_config): - """Test that multiple connect calls don't cause errors.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - - # Connect multiple times - backend1 = connection.connect() - backend2 = connection.connect() - - assert backend1 is not None - assert backend2 is not None - assert connection.is_connected() - - # Cleanup - connection.disconnect() - - -@pytest.mark.integration -@pytest.mark.parametrize("connection_class,settings_class,db_config", [ - (SQLite_IbisConnection, SQLiteAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), - (DuckDB_IbisConnection, DuckDBAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), -]) -class TestConnectionFunctionality: - """Test actual backend functionality for all Ibis backends.""" - - def test_backend_can_create_table(self, connection_class, settings_class, db_config): - """Test that connected backend can create tables.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - backend = connection.connect() - - # Create a simple table - backend.create_table("test_table", {"id": [1, 2, 3], "value": [10, 20, 30]}, overwrite=True) - - # Verify table exists - tables = backend.list_tables() - assert "test_table" in tables - - # Cleanup - connection.disconnect() - - def test_backend_can_list_tables(self, connection_class, settings_class, db_config): - """Test that connected backend can list tables.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - backend = connection.connect() - - # Create some tables - backend.create_table("table1", {"id": [1]}, overwrite=True) - backend.create_table("table2", {"id": [2]}, overwrite=True) - - # List tables - tables = backend.list_tables() - - assert isinstance(tables, list) - assert "table1" in tables - assert "table2" in tables - - # Cleanup - connection.disconnect() - - def test_backend_can_query_table(self, connection_class, settings_class, db_config): - """Test that connected backend can query tables.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - backend = connection.connect() - - # Create table with data - backend.create_table("query_test", {"id": [1, 2, 3], "value": [10, 20, 30]}, overwrite=True) - - # Query table - table = backend.table("query_test") - assert table is not None - - # Cleanup - connection.disconnect() - - -@pytest.mark.unit -@pytest.mark.parametrize("settings_class,db_config", [ - (SQLiteAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), - (DuckDBAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), -]) -class TestConnectionWithFactory: - """Test connections work with factory pattern.""" - - def test_factory_creates_correct_connection(self, settings_class, db_config): - """Test that factory creates correct connection type.""" - from mountainash_data.core.factories import ConnectionFactory - - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = ConnectionFactory.get_connection(settings_params) - - assert connection is not None - assert isinstance(connection, BaseDBConnection) - - def test_factory_connection_can_connect(self, settings_class, db_config): - """Test that factory-created connection can connect.""" - from mountainash_data.core.factories import ConnectionFactory - - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = ConnectionFactory.get_connection(settings_params) - backend = connection.connect() - - assert backend is not None - assert connection.is_connected() - - # Cleanup - connection.disconnect() diff --git a/tests/test_unit/databases/connections/test_base_db_connection.py b/tests/test_unit/databases/connections/test_base_db_connection.py deleted file mode 100644 index 8b9b7af..0000000 --- a/tests/test_unit/databases/connections/test_base_db_connection.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Tests for BaseDBConnection abstract base class.""" - -import pytest -from mountainash_data.core.connection import BaseDBConnection - - -@pytest.mark.unit -class TestBaseDBConnection: - """Tests for BaseDBConnection abstract base class.""" - - def test_base_db_connection_is_abstract(self): - """Test that BaseDBConnection cannot be instantiated directly.""" - with pytest.raises(TypeError, match="Can't instantiate abstract class"): - BaseDBConnection() - - def test_base_db_connection_has_required_abstract_methods(self): - """Test that BaseDBConnection defines required abstract methods.""" - assert hasattr(BaseDBConnection, 'connect') - assert hasattr(BaseDBConnection, 'disconnect') - assert hasattr(BaseDBConnection, 'is_connected') - - def test_base_db_connection_has_db_backend_property(self): - """Test that BaseDBConnection has db_backend_name property.""" - assert hasattr(BaseDBConnection, 'db_backend_name') - - def test_base_db_connection_has_settings_parameter(self): - """Test that BaseDBConnection expects db_auth_settings_parameters.""" - # This is verified by the abstract __init__ signature - import inspect - sig = inspect.signature(BaseDBConnection.__init__) - assert 'db_auth_settings_parameters' in sig.parameters diff --git a/tests/test_unit/databases/operations/ibis/test_base_ibis_operations.py b/tests/test_unit/databases/operations/ibis/test_base_ibis_operations.py deleted file mode 100644 index 18abc61..0000000 --- a/tests/test_unit/databases/operations/ibis/test_base_ibis_operations.py +++ /dev/null @@ -1,333 +0,0 @@ -"""Tests for BaseIbisOperations.""" - -import pytest -import ibis -from mountainash_data.backends.ibis.operations import BaseIbisOperations -from mountainash_data.core.utils import DatabaseUtils -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings -from mountainash_settings import SettingsParameters - - -@pytest.mark.unit -class TestBaseIbisOperationsInstantiation: - """Tests for BaseIbisOperations instantiation.""" - - def test_base_ibis_operations_is_abstract(self): - """Test that BaseIbisOperations cannot be instantiated directly.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - with pytest.raises(TypeError): - BaseIbisOperations(db_auth_settings_parameters=settings_params) - - -@pytest.mark.integration -class TestIbisOperationsRunSQL: - """Tests for run_sql method.""" - - def test_run_sql_simple_query(self, sqlite_settings_params, sample_table_data): - """Test running a simple SQL query.""" - # Create backend and operations - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create a test table - backend.create_table("test_sql", sample_table_data["simple"], overwrite=True) - - # Run SQL query - result = operations.run_sql(backend, "SELECT * FROM test_sql") - - assert result is not None - - def test_run_sql_with_where_clause(self, sqlite_settings_params, sample_table_data): - """Test SQL query with WHERE clause.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - backend.create_table("test_where", sample_table_data["simple"], overwrite=True) - - result = operations.run_sql(backend, "SELECT * FROM test_where WHERE id > 1") - - assert result is not None - - -@pytest.mark.integration -class TestIbisOperationsTable: - """Tests for table method.""" - - def test_table_retrieval(self, sqlite_settings_params, sample_table_data): - """Test retrieving a table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create test table - backend.create_table("test_retrieve", sample_table_data["simple"], overwrite=True) - - # Retrieve table - table = operations.table(backend, "test_retrieve") - - assert table is not None - - def test_table_nonexistent(self, sqlite_settings_params): - """Test retrieving non-existent table returns None.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Try to retrieve non-existent table - table = operations.table(backend, "nonexistent_table") - - assert table is None - - -@pytest.mark.integration -class TestIbisOperationsCreateTable: - """Tests for create_table method.""" - - def test_create_table_from_dict(self, sqlite_settings_params): - """Test creating table from dictionary data.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - data = {"id": [1, 2, 3], "value": [10, 20, 30]} - - operations.create_table(backend, "test_create", data, overwrite=True) - - # Verify table exists - assert operations.table_exists(backend, "test_create") - - def test_create_table_overwrite(self, sqlite_settings_params): - """Test creating table with overwrite=True.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - data1 = {"id": [1, 2], "value": [10, 20]} - data2 = {"id": [3, 4, 5], "value": [30, 40, 50]} - - # Create first table - operations.create_table(backend, "test_overwrite", data1, overwrite=True) - - # Overwrite with new data - operations.create_table(backend, "test_overwrite", data2, overwrite=True) - - # Verify table exists - assert operations.table_exists(backend, "test_overwrite") - - def test_create_table_temp(self, sqlite_settings_params): - """Test creating temporary table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - data = {"id": [1, 2, 3]} - - operations.create_table(backend, "test_temp", data, temp=True, overwrite=True) - - # Verify table exists (temp tables should be queryable in same session) - assert operations.table_exists(backend, "test_temp") or True # Some backends may not list temp tables - - -@pytest.mark.integration -class TestIbisOperationsDropTable: - """Tests for drop_table method.""" - - def test_drop_existing_table(self, sqlite_settings_params): - """Test dropping an existing table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create table - backend.create_table("test_drop", {"id": [1, 2, 3]}, overwrite=True) - - # Drop table - result = operations.drop_table(backend, "test_drop") - - assert result is True - assert not operations.table_exists(backend, "test_drop") - - def test_drop_nonexistent_table(self, sqlite_settings_params): - """Test dropping non-existent table returns False.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - result = operations.drop_table(backend, "nonexistent_drop") - - assert result is False - - -@pytest.mark.integration -class TestIbisOperationsListTables: - """Tests for list_tables method.""" - - def test_list_tables_empty(self, sqlite_memory_settings_params): - """Test listing tables in empty database.""" - backend = DatabaseUtils.create_backend(sqlite_memory_settings_params) - operations = DatabaseUtils.create_operations(sqlite_memory_settings_params) - - tables = operations.list_tables(backend) - - assert isinstance(tables, list) - - def test_list_tables_with_tables(self, sqlite_memory_settings_params): - """Test listing tables when tables exist.""" - backend = DatabaseUtils.create_backend(sqlite_memory_settings_params) - operations = DatabaseUtils.create_operations(sqlite_memory_settings_params) - - # Create some tables - backend.create_table("table1", {"id": [1, 2]}, overwrite=True) - backend.create_table("table2", {"id": [3, 4]}, overwrite=True) - - tables = operations.list_tables(backend) - - assert "table1" in tables - assert "table2" in tables - - -@pytest.mark.integration -class TestIbisOperationsTableExists: - """Tests for table_exists method.""" - - def test_table_exists_true(self, sqlite_settings_params): - """Test table_exists returns True for existing table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - backend.create_table("test_exists", {"id": [1]}, overwrite=True) - - assert operations.table_exists(backend, "test_exists") is True - - def test_table_exists_false(self, sqlite_settings_params): - """Test table_exists returns False for non-existent table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - assert operations.table_exists(backend, "definitely_not_exists") is False - - -@pytest.mark.integration -class TestIbisOperationsInsert: - """Tests for insert method.""" - - def test_insert_data(self, sqlite_settings_params): - """Test inserting data into table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create table - backend.create_table("test_insert", {"id": [1], "value": [10]}, overwrite=True) - - # Insert more data - new_data = {"id": [2, 3], "value": [20, 30]} - result = operations.insert(backend, "test_insert", new_data) - - assert result is True - - def test_insert_overwrite(self, sqlite_settings_params): - """Test insert with overwrite=True.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create table with initial data - backend.create_table("test_insert_overwrite", {"id": [1, 2]}, overwrite=True) - - # Insert with overwrite - new_data = {"id": [3, 4, 5]} - result = operations.insert(backend, "test_insert_overwrite", new_data, overwrite=True) - - assert result is True - - -@pytest.mark.integration -class TestIbisOperationsTruncate: - """Tests for truncate method.""" - - def test_truncate_table(self, sqlite_settings_params): - """Test truncating a table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create table with data - backend.create_table("test_truncate", {"id": [1, 2, 3], "value": [10, 20, 30]}, overwrite=True) - - # Truncate (may not be supported by all backends) - try: - operations.truncate(backend, "test_truncate") - # If successful, table should exist but be empty - assert operations.table_exists(backend, "test_truncate") - except (NotImplementedError, Exception): - # Some backends may not support truncate - pytest.skip("Truncate not supported by this backend") - - -@pytest.mark.integration -class TestIbisOperationsViews: - """Tests for view operations.""" - - def test_create_view(self, sqlite_settings_params, sample_table_data): - """Test creating a view.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create base table - backend.create_table("base_for_view", sample_table_data["simple"], overwrite=True) - - # Get table expression - table_expr = operations.table(backend, "base_for_view") - - # Create view - view = operations.create_view(backend, "test_view", table_expr, overwrite=True) - - # View should be created (result may vary by backend) - assert view is not None or True - - def test_drop_view(self, sqlite_settings_params, sample_table_data): - """Test dropping a view.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create base table and view - backend.create_table("base_for_drop_view", sample_table_data["simple"], overwrite=True) - table_expr = operations.table(backend, "base_for_drop_view") - operations.create_view(backend, "view_to_drop", table_expr, overwrite=True) - - # Drop view - result = operations.drop_view(backend, "view_to_drop") - - # Result depends on backend support - assert isinstance(result, bool) - - -@pytest.mark.integration -@pytest.mark.parametrize("backend_settings", [ - "sqlite_memory_settings_params", - "duckdb_settings_params", -]) -class TestIbisOperationsMultiBackend: - """Test operations work across different backends.""" - - def test_create_and_list_tables_multi_backend(self, backend_settings, request): - """Test basic operations work on multiple backends.""" - settings = request.getfixturevalue(backend_settings) - - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - backend.create_table("multi_test", {"id": [1, 2, 3]}, overwrite=True) - - # List tables - tables = operations.list_tables(backend) - - assert "multi_test" in tables - - def test_table_exists_multi_backend(self, backend_settings, request): - """Test table_exists works on multiple backends.""" - settings = request.getfixturevalue(backend_settings) - - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - backend.create_table("exists_test", {"id": [1]}, overwrite=True) - - assert operations.table_exists(backend, "exists_test") is True - assert operations.table_exists(backend, "not_exists") is False diff --git a/tests/test_unit/databases/operations/test_upsert_and_indexes.py b/tests/test_unit/databases/operations/test_upsert_and_indexes.py deleted file mode 100644 index 6660755..0000000 --- a/tests/test_unit/databases/operations/test_upsert_and_indexes.py +++ /dev/null @@ -1,626 +0,0 @@ -"""Comprehensive tests for upsert and index management operations. - -Tests cover: -- Upsert operations (various scenarios) -- Index creation, deletion, and querying -- Cross-database compatibility (DuckDB, SQLite) -""" - -import pytest -import polars as pl -from mountainash_data.core.utils import DatabaseUtils -from mountainash_data.core.constants import CONST_CONFLICT_ACTION - - -@pytest.mark.integration -@pytest.mark.parametrize("backend_fixture", [ - "sqlite_memory_settings_params", - "duckdb_settings_params", -]) -class TestUpsertOperations: - """Tests for upsert operations across different scenarios.""" - - def test_simple_upsert_insert_new_rows(self, backend_fixture, request): - """Test upsert inserts new rows when no conflicts exist.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table - initial_data = pl.DataFrame({ - "id": [1, 2], - "email": ["alice@example.com", "bob@example.com"], - "name": ["Alice", "Bob"] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - - # Create unique index on email - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert with new data (no conflicts) - new_data = pl.DataFrame({ - "id": [3, 4], - "email": ["charlie@example.com", "diana@example.com"], - "name": ["Charlie", "Diana"] - }) - operations.upsert( - backend, "users", new_data, - conflict_columns=["email"] - ) - - # Verify all 4 rows exist - result = operations.run_sql(backend, "SELECT COUNT(*) as count FROM users") - count = result.to_polars()["count"][0] - assert count == 4 - - def test_simple_upsert_update_existing_rows(self, backend_fixture, request): - """Test upsert updates existing rows on conflict.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table with index - initial_data = pl.DataFrame({ - "id": [1, 2], - "email": ["alice@example.com", "bob@example.com"], - "name": ["Alice", "Bob"], - "score": [100, 200] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert with conflicting emails but different data - update_data = pl.DataFrame({ - "id": [1, 2], - "email": ["alice@example.com", "bob@example.com"], - "name": ["Alice Updated", "Bob Updated"], - "score": [150, 250] - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"] - ) - - # Verify rows were updated - result = operations.run_sql(backend, "SELECT * FROM users ORDER BY id") - df = result.to_polars() - assert len(df) == 2 - # Names should be updated - names = df["name"].to_list() - assert "Alice Updated" in names - assert "Bob Updated" in names - - def test_upsert_mixed_insert_and_update(self, backend_fixture, request): - """Test upsert handles both inserts and updates in same operation.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table - initial_data = pl.DataFrame({ - "email": ["alice@example.com", "bob@example.com"], - "name": ["Alice", "Bob"] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert with mix of existing and new emails - mixed_data = pl.DataFrame({ - "email": ["alice@example.com", "charlie@example.com"], - "name": ["Alice Updated", "Charlie"] - }) - operations.upsert( - backend, "users", mixed_data, - conflict_columns=["email"] - ) - - # Verify: 3 rows total (alice updated, bob unchanged, charlie added) - result = operations.run_sql(backend, "SELECT COUNT(*) as count FROM users") - count = result.to_polars()["count"][0] - assert count == 3 - - # Verify Alice was updated - result = operations.run_sql(backend, "SELECT name FROM users WHERE email = 'alice@example.com'") - alice_name = result.to_polars()["name"][0] - assert alice_name == "Alice Updated" - - def test_upsert_with_conflict_action_nothing(self, backend_fixture, request): - """Test upsert with DO NOTHING ignores conflicts.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table - initial_data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["Alice"], - "score": [100] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert with DO NOTHING - should skip conflicting row - conflict_data = pl.DataFrame({ - "email": ["alice@example.com", "bob@example.com"], - "name": ["Alice Updated", "Bob"], - "score": [999, 200] - }) - operations.upsert( - backend, "users", conflict_data, - conflict_columns=["email"], - conflict_action=CONST_CONFLICT_ACTION.NOTHING - ) - - # Verify: Alice unchanged (still 100), Bob added - result = operations.run_sql(backend, "SELECT email, score FROM users ORDER BY email") - df = result.to_polars() - assert len(df) == 2 - # Alice should still have original score - alice_score = df.filter(pl.col("email") == "alice@example.com")["score"][0] - assert alice_score == 100 - - def test_upsert_with_specific_update_columns(self, backend_fixture, request): - """Test upsert updates only specified columns.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table - initial_data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["Alice"], - "score": [100], - "status": ["active"] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert updating only score (not name or status) - update_data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["SHOULD NOT UPDATE"], - "score": [200], - "status": ["SHOULD NOT UPDATE"] - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"], - update_columns=["score"] # Only update score - ) - - # Verify: score updated, name and status unchanged - result = operations.run_sql(backend, "SELECT name, score, status FROM users WHERE email = 'alice@example.com'") - df = result.to_polars() - assert df["name"][0] == "Alice" # name unchanged - assert df["score"][0] == 200 # score updated - assert df["status"][0] == "active" # status unchanged - - def test_upsert_with_conditional_update(self, backend_fixture, request): - """Test upsert with conditional WHERE clause on update.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table - initial_data = pl.DataFrame({ - "email": ["alice@example.com", "bob@example.com"], - "last_login": [100, 50] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert with condition: only update if new last_login is greater - update_data = pl.DataFrame({ - "email": ["alice@example.com", "bob@example.com"], - "last_login": [90, 75] # Alice: 90 < 100 (no update), Bob: 75 > 50 (update) - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"], - update_columns=["last_login"], - update_condition="users.last_login < EXCLUDED.last_login" - ) - - # Verify: Alice unchanged (100), Bob updated (75) - result = operations.run_sql(backend, "SELECT email, last_login FROM users ORDER BY email") - df = result.to_polars() - alice_login = df.filter(pl.col("email") == "alice@example.com")["last_login"][0] - bob_login = df.filter(pl.col("email") == "bob@example.com")["last_login"][0] - assert alice_login == 100 # Not updated (condition failed) - assert bob_login == 75 # Updated (condition passed) - - def test_upsert_composite_conflict_columns(self, backend_fixture, request): - """Test upsert with multiple conflict columns (composite key).""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table with composite key - initial_data = pl.DataFrame({ - "user_id": [1, 1, 2], - "product_id": [100, 101, 100], - "quantity": [5, 3, 7] - }) - operations.create_table(backend, "cart", initial_data, overwrite=True) - operations.create_unique_index(backend, "cart", ["user_id", "product_id"]) - - # Upsert with composite conflict - update_data = pl.DataFrame({ - "user_id": [1, 2, 2], - "product_id": [100, 100, 102], - "quantity": [10, 15, 2] - }) - operations.upsert( - backend, "cart", update_data, - conflict_columns=["user_id", "product_id"] - ) - - # Verify: user 1/product 100 updated, user 2/product 100 updated, user 2/product 102 added - result = operations.run_sql(backend, "SELECT user_id, product_id, quantity FROM cart ORDER BY user_id, product_id") - df = result.to_polars() - assert len(df) == 4 - - def test_upsert_auto_column_detection(self, backend_fixture, request): - """Test upsert auto-detects update columns when not specified.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - initial_data = pl.DataFrame({ - "id": [1], - "email": ["alice@example.com"], - "name": ["Alice"], - "score": [100] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert without specifying update_columns (should auto-detect) - update_data = pl.DataFrame({ - "id": [1], - "email": ["alice@example.com"], - "name": ["Alice Updated"], - "score": [200] - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"] - # update_columns not specified - should update id, name, score - ) - - # Verify all non-conflict columns were updated - result = operations.run_sql(backend, "SELECT name, score FROM users WHERE email = 'alice@example.com'") - df = result.to_polars() - assert df["name"][0] == "Alice Updated" - assert df["score"][0] == 200 - - def test_upsert_validation_table_not_exists(self, backend_fixture, request): - """Test upsert raises error when target table doesn't exist.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Try to upsert to non-existent table - data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["Alice"] - }) - - with pytest.raises(ValueError, match="does not exist"): - operations.upsert( - backend, "nonexistent_table", data, - conflict_columns=["email"] - ) - - def test_upsert_validation_empty_conflict_columns(self, backend_fixture, request): - """Test upsert raises error when conflict_columns is empty.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - initial_data = pl.DataFrame({"email": ["alice@example.com"]}) - operations.create_table(backend, "users", initial_data, overwrite=True) - - # Try upsert with empty conflict columns - with pytest.raises(ValueError, match="At least one column"): - operations.upsert( - backend, "users", initial_data, - conflict_columns=[] - ) - - -@pytest.mark.integration -@pytest.mark.parametrize("backend_fixture", [ - "sqlite_memory_settings_params", - "duckdb_settings_params", -]) -class TestIndexManagement: - """Tests for index creation, deletion, and querying.""" - - def test_create_simple_index(self, backend_fixture, request): - """Test creating a simple index.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({ - "id": [1, 2, 3], - "email": ["a@example.com", "b@example.com", "c@example.com"] - }) - operations.create_table(backend, "users", data, overwrite=True) - - # Create index - result = operations.create_index(backend, "users", ["email"]) - assert result is True - - # Verify index exists - assert operations.index_exists(backend, "idx_users_email") is True - - def test_create_unique_index(self, backend_fixture, request): - """Test creating a unique index.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({ - "id": [1, 2, 3], - "email": ["a@example.com", "b@example.com", "c@example.com"] - }) - operations.create_table(backend, "users", data, overwrite=True) - - # Create unique index - result = operations.create_unique_index(backend, "users", ["email"]) - assert result is True - - # Verify index exists - assert operations.index_exists(backend, "uidx_users_email") is True - - def test_create_composite_index(self, backend_fixture, request): - """Test creating a composite (multi-column) index.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({ - "user_id": [1, 2, 3], - "product_id": [100, 200, 300], - "quantity": [5, 10, 15] - }) - operations.create_table(backend, "orders", data, overwrite=True) - - # Create composite index - result = operations.create_index( - backend, "orders", ["user_id", "product_id"] - ) - assert result is True - - # Verify index exists - assert operations.index_exists(backend, "idx_orders_product_id_user_id") is True - - def test_create_index_with_custom_name(self, backend_fixture, request): - """Test creating index with custom name.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({"email": ["a@example.com"]}) - operations.create_table(backend, "users", data, overwrite=True) - - # Create index with custom name - result = operations.create_index( - backend, "users", ["email"], - index_name="my_custom_index" - ) - assert result is True - - # Verify custom name - assert operations.index_exists(backend, "my_custom_index") is True - - def test_create_partial_index(self, backend_fixture, request): - """Test creating partial index with WHERE clause.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({ - "email": ["a@example.com", "b@example.com"], - "status": ["active", "inactive"] - }) - operations.create_table(backend, "users", data, overwrite=True) - - # Create partial index (only for active users) - # Note: DuckDB does not support partial indexes as of v1.3.2 - result = operations.create_index( - backend, "users", ["email"], - where_condition="status = 'active'" - ) - - # DuckDB doesn't support partial indexes, SQLite does - if "duckdb" in backend_fixture: - assert result is False # Expected: DuckDB doesn't support partial indexes - else: - assert result is True - # Verify index exists (only for SQLite) - assert operations.index_exists(backend, "idx_users_email") is True - - def test_create_index_if_not_exists(self, backend_fixture, request): - """Test creating index with IF NOT EXISTS doesn't fail on duplicate.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table and index - data = pl.DataFrame({"email": ["a@example.com"]}) - operations.create_table(backend, "users", data, overwrite=True) - operations.create_index(backend, "users", ["email"]) - - # Create same index again with if_not_exists=True (should not fail) - result = operations.create_index( - backend, "users", ["email"], - if_not_exists=True - ) - # Result may vary, but should not raise error - assert result in [True, False] - - def test_drop_index(self, backend_fixture, request): - """Test dropping an index.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table and index - data = pl.DataFrame({"email": ["a@example.com"]}) - operations.create_table(backend, "users", data, overwrite=True) - operations.create_index(backend, "users", ["email"]) - - # Verify index exists - assert operations.index_exists(backend, "idx_users_email") is True - - # Drop index - result = operations.drop_index(backend, "idx_users_email") - assert result is True - - # Verify index no longer exists - assert operations.index_exists(backend, "idx_users_email") is False - - def test_drop_index_if_exists(self, backend_fixture, request): - """Test dropping non-existent index with IF EXISTS doesn't fail.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Try to drop non-existent index with if_exists=True - result = operations.drop_index( - backend, "nonexistent_index", - if_exists=True - ) - # Should not raise error - assert result in [True, False] - - def test_index_exists_returns_false_for_nonexistent(self, backend_fixture, request): - """Test index_exists returns False for non-existent index.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Check non-existent index - assert operations.index_exists(backend, "nonexistent_index") is False - - def test_list_indexes(self, backend_fixture, request): - """Test listing all indexes for a table.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table with multiple indexes - data = pl.DataFrame({ - "id": [1, 2], - "email": ["a@example.com", "b@example.com"], - "name": ["Alice", "Bob"] - }) - operations.create_table(backend, "users", data, overwrite=True) - operations.create_index(backend, "users", ["email"]) - operations.create_unique_index(backend, "users", ["id"]) - - # List indexes - indexes = operations.list_indexes(backend, "users") - assert isinstance(indexes, list) - assert len(indexes) >= 2 # At least the two we created - - # Verify index names are in the list - index_names = [idx.get("name") for idx in indexes] - assert "idx_users_email" in index_names - assert "uidx_users_id" in index_names - - def test_create_unique_index_convenience_method(self, backend_fixture, request): - """Test create_unique_index convenience method.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({"email": ["a@example.com"]}) - operations.create_table(backend, "users", data, overwrite=True) - - # Use convenience method - result = operations.create_unique_index(backend, "users", "email") - assert result is True - - # Verify unique index created - assert operations.index_exists(backend, "uidx_users_email") is True - - -@pytest.mark.integration -class TestUpsertIndexIntegration: - """Integration tests combining upsert and index operations.""" - - def test_upsert_requires_unique_index_on_conflict_columns(self, duckdb_settings_params): - """Test that upsert works correctly with unique index on conflict columns.""" - backend = DatabaseUtils.create_backend(duckdb_settings_params) - operations = DatabaseUtils.create_operations(duckdb_settings_params) - - # Create table - initial_data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["Alice"] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - - # Create unique index on conflict column - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert should work - update_data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["Alice Updated"] - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"] - ) - - # Verify update worked - result = operations.run_sql(backend, "SELECT name FROM users WHERE email = 'alice@example.com'") - df = result.to_polars() - assert df["name"][0] == "Alice Updated" - - def test_multiple_indexes_and_upserts(self, sqlite_memory_settings_params): - """Test multiple indexes don't interfere with upserts.""" - backend = DatabaseUtils.create_backend(sqlite_memory_settings_params) - operations = DatabaseUtils.create_operations(sqlite_memory_settings_params) - - # Create table with multiple indexes - data = pl.DataFrame({ - "id": [1, 2], - "email": ["a@example.com", "b@example.com"], - "username": ["alice", "bob"] - }) - operations.create_table(backend, "users", data, overwrite=True) - - # Create multiple indexes - operations.create_unique_index(backend, "users", ["email"]) - operations.create_unique_index(backend, "users", ["username"]) - operations.create_index(backend, "users", ["id"]) - - # Upsert on email should work - update_data = pl.DataFrame({ - "id": [1], - "email": ["a@example.com"], - "username": ["alice_updated"] - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"] - ) - - # Verify - result = operations.run_sql(backend, "SELECT username FROM users WHERE email = 'a@example.com'") - df = result.to_polars() - assert df["username"][0] == "alice_updated" diff --git a/tests/test_unit/databases/settings/test_settings_parametrized.py b/tests/test_unit/databases/settings/test_settings_parametrized.py index e86a73e..cb9c20a 100644 --- a/tests/test_unit/databases/settings/test_settings_parametrized.py +++ b/tests/test_unit/databases/settings/test_settings_parametrized.py @@ -144,38 +144,28 @@ def test_extracted_settings_have_parameters_method(self, settings_class): class TestSettingsWithConnections: """Test that settings work with actual connections.""" - def test_settings_work_with_connection_factory(self, settings_class, db_config): - """Test that settings work with ConnectionFactory.""" - from mountainash_data.core.factories import ConnectionFactory - + def test_settings_work_with_ibis_backend(self, settings_class, db_config): + """Test that settings work with IbisBackend.""" + from mountainash_data.backends.ibis.backend import IbisBackend settings_params = SettingsParameters.create( settings_class=settings_class, kwargs=db_config ) + backend = IbisBackend(settings_params) + assert backend.dialect is not None - # Should be able to get connection - connection = ConnectionFactory.get_connection(settings_params) - - assert connection is not None - - def test_settings_enable_backend_connection(self, settings_class, db_config): - """Test that settings enable actual backend connection.""" - from mountainash_data.core.utils import DatabaseUtils - + def test_settings_work_with_ibis_backend_connect(self, settings_class, db_config): + """Test that settings can create a connected backend via IbisBackend.""" + from mountainash_data.backends.ibis.backend import IbisBackend settings_params = SettingsParameters.create( settings_class=settings_class, kwargs=db_config ) - - # Should be able to create backend - backend = DatabaseUtils.create_backend(settings_params) - - assert backend is not None - - # Backend should be functional - backend.create_table("test", {"id": [1, 2, 3]}, overwrite=True) + backend = IbisBackend(settings_params) + backend.connect() tables = backend.list_tables() - assert "test" in tables + assert isinstance(tables, list) + backend.close() @pytest.mark.unit diff --git a/tests/test_unit/databases/test_database_connections.py b/tests/test_unit/databases/test_database_connections.py deleted file mode 100644 index c5ade05..0000000 --- a/tests/test_unit/databases/test_database_connections.py +++ /dev/null @@ -1,277 +0,0 @@ -"""Tests for database connection classes.""" - -import pytest -from pathlib import Path - -from mountainash_data.core.connection import BaseDBConnection -from mountainash_data.backends.ibis.connection import BaseIbisConnection -from mountainash_data.backends.ibis.connection import SQLite_IbisConnection -from mountainash_data.backends.ibis.connection import DuckDB_IbisConnection -from mountainash_settings import SettingsParameters -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings, NoAuth - - -class TestBaseDBConnection: - """Tests for base database connection class.""" - - def test_base_db_connection_is_abstract(self): - """Test that BaseDBConnection cannot be instantiated directly.""" - with pytest.raises(TypeError): - BaseDBConnection() - - def test_base_db_connection_has_required_methods(self): - """Test that BaseDBConnection defines required abstract methods.""" - # Check that abstract methods are defined - assert hasattr(BaseDBConnection, 'connect') - assert hasattr(BaseDBConnection, 'disconnect') - assert hasattr(BaseDBConnection, 'is_connected') - - -class TestBaseIbisConnection: - """Tests for base Ibis connection class.""" - - def test_base_ibis_connection_is_abstract(self): - """Test that BaseIbisConnection cannot be instantiated directly.""" - with pytest.raises(TypeError): - BaseIbisConnection() - - def test_inherits_from_base_db_connection(self): - """Test that BaseIbisConnection inherits from BaseDBConnection.""" - assert issubclass(BaseIbisConnection, BaseDBConnection) - - -class TestSQLiteIbisConnection: - """Tests for SQLite Ibis connection.""" - - def test_sqlite_connection_initialization(self, temp_sqlite_db): - """Test SQLite connection can be initialized.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - conn = SQLite_IbisConnection(db_auth_settings_parameters=settings_params) - assert conn is not None - assert hasattr(conn, 'db_auth_settings_parameters') - assert conn.db_auth_settings_parameters == settings_params - - def test_sqlite_connect_method(self, temp_sqlite_db): - """Test SQLite connect method.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - conn = SQLite_IbisConnection(db_auth_settings_parameters=settings_params) - backend = conn.connect() - - # Test actual functionality - assert backend is not None - # Verify we can list tables (should include test_table from fixture) - tables = backend.list_tables() - assert 'test_table' in tables - # Test we can actually query the table - table = backend.table('test_table') - assert table is not None - count = table.count().execute() - assert count == 3 # From fixture data - - def test_sqlite_connect_with_memory_db(self): - """Test SQLite connection with in-memory database.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - conn = SQLite_IbisConnection(db_auth_settings_parameters=settings_params) - backend = conn.connect() - - # Test actual functionality with in-memory database - assert backend is not None - # Should be able to create and query tables - backend.create_table('test_mem', {'id': [1, 2, 3], 'value': [10, 20, 30]}) - tables = backend.list_tables() - assert 'test_mem' in tables - - # Verify data operations work - table = backend.table('test_mem') - assert table is not None - # Note: .count().execute() has pyarrow compatibility issues in ibis 10.4.0 - # Verify table is queryable by converting to pandas (works around the issue) - try: - df = table.to_pandas() - assert len(df) == 3 - except AttributeError: - # Fallback if to_pandas also fails - just verify table exists - pass - - # def test_sqlite_connection_error_handling(self): - # """Test SQLite connection error handling.""" - # # Use invalid path to trigger real error - # settings_params = SettingsParameters.create( - # settings_class=SQLiteAuthSettings, - # kwargs={"DATABASE": "/invalid/nonexistent/path/test.db"} - # ) - - # conn = SQLite_IbisConnection(db_auth_settings_parameters=settings_params) - - # # This should raise a real connection error - # with pytest.raises(Exception): # Real error from ibis/sqlite - # conn.connect() - - -class TestDuckDBIbisConnection: - """Tests for DuckDB Ibis connection.""" - - def test_duckdb_connection_initialization(self): - """Test DuckDB connection can be initialized.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - conn = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params) - assert conn is not None - assert hasattr(conn, 'db_auth_settings_parameters') - assert conn.db_auth_settings_parameters == settings_params - - def test_duckdb_connect_method(self): - """Test DuckDB connect method.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - conn = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params) - backend = conn.connect() - - # Test actual functionality - assert backend is not None - # Create test data and verify operations - backend.create_table('test_duckdb', {'id': [1, 2, 3], 'name': ['A', 'B', 'C']}) - tables = backend.list_tables() - assert 'test_duckdb' in tables - - # Test actual query operations - table = backend.table('test_duckdb') - assert table is not None - # Note: .count().execute() has pyarrow compatibility issues in ibis 10.4.0 - # Verify table is queryable by converting to pandas (works around the issue) - try: - df = table.to_pandas() - assert len(df) == 3 - except AttributeError: - # Fallback if to_pandas also fails - just verify table exists - pass - - def test_duckdb_connect_with_memory_db(self): - """Test DuckDB connection with in-memory database.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": None, "auth": NoAuth()} # None triggers memory mode for DuckDB - ) - - conn = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params) - backend = conn.connect() - - # Test actual functionality with in-memory database - assert backend is not None - # Should be able to create and query tables - backend.create_table('test_mem_duck', {'x': [1, 2, 3], 'y': [4, 5, 6]}) - tables = backend.list_tables() - assert 'test_mem_duck' in tables - - # Verify data operations work - table = backend.table('test_mem_duck') - assert table is not None - # Note: .count().execute() has pyarrow compatibility issues in ibis 10.4.0 - # Verify table is queryable by converting to pandas (works around the issue) - try: - df = table.to_pandas() - assert len(df) == 3 - except AttributeError: - # Fallback if to_pandas also fails - just verify table exists - pass - - # def test_duckdb_connection_error_handling(self): - # """Test DuckDB connection error handling.""" - # # Use invalid path to trigger real error - # settings_params = SettingsParameters.create( - # settings_class=DuckDBAuthSettings, - # kwargs={"DATABASE": "/invalid/nonexistent/directory/test.duckdb"} - # ) - - # conn = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params) - - # # This should raise a real connection error - # with pytest.raises(Exception): # Real error from ibis/duckdb - # conn.connect() - - -class TestConnectionFactory: - """Tests for connection factory patterns.""" - - def test_can_create_multiple_sqlite_connections(self, temp_sqlite_db): - """Test that multiple SQLite connections can be created.""" - settings_params1 = SettingsParameters.create( - namespace="settings_params1", - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - settings_params2 = SettingsParameters.create( - namespace="settings_params2", - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - conn1 = SQLite_IbisConnection(db_auth_settings_parameters=settings_params1) - conn2 = SQLite_IbisConnection(db_auth_settings_parameters=settings_params2) - - assert conn1 != conn2 - assert conn1.db_auth_settings_parameters.kwargs != conn2.db_auth_settings_parameters.kwargs - - # Test they can both connect and work independently - backend1 = conn1.connect() - backend2 = conn2.connect() - - assert backend1 != backend2 - # Backend1 should have test_table from fixture, backend2 should not - assert 'test_table' in backend1.list_tables() - assert 'test_table' not in backend2.list_tables() - - def test_can_create_multiple_duckdb_connections(self): - """Test that multiple DuckDB connections can be created.""" - settings_params1 = SettingsParameters.create( - namespace="settings_params1", - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - settings_params2 = SettingsParameters.create( - namespace="settings_params2", - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": None, "auth": NoAuth()} - ) - - conn1 = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params1) - conn2 = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params2) - - assert conn1 != conn2 - assert conn1.db_auth_settings_parameters.kwargs != conn2.db_auth_settings_parameters.kwargs - - # Test they can both connect and work independently - backend1 = conn1.connect() - backend2 = conn2.connect() - - # Verify backends are different instances (object identity) - assert backend1 is not backend2 - - # Create different tables in each to verify independence - backend1.create_table('table1', {'a': [1, 2]}) - backend2.create_table('table2', {'b': [3, 4]}) - - assert 'table1' in backend1.list_tables() - assert 'table1' not in backend2.list_tables() - assert 'table2' not in backend1.list_tables() - assert 'table2' in backend2.list_tables() diff --git a/tests/test_unit/databases/test_ibis_backends.py b/tests/test_unit/databases/test_ibis_backends.py deleted file mode 100644 index 8d165ef..0000000 --- a/tests/test_unit/databases/test_ibis_backends.py +++ /dev/null @@ -1,137 +0,0 @@ -import pytest -import ibis -import os -from unittest import mock - -from mountainash_data.backends.ibis.connection import SQLite_IbisConnection, DuckDB_IbisConnection -from mountainash_settings import SettingsParameters, MountainAshBaseSettings -from mountainash_data.core.settings import DuckDBAuthSettings, SQLiteAuthSettings, NoAuth - -@pytest.fixture -def mock_settings_parameters_1(): - return SettingsParameters.create(settings_class = MountainAshBaseSettings, namespace="mock") - -@pytest.fixture -def mock_settings_parameters_2(): - kwargs = {"USERNAME": "ngods", "PASSWORD": "ngods", "HOST": "host", "PORT": "5432"} - return SettingsParameters.create(settings_class = MountainAshBaseSettings, namespace="mock_pg", kwargs=kwargs) - -@pytest.fixture -def mock_settings_parameters_3(): - kwargs = {"USERNAME": "ngods", "PASSWORD": "ngods", "HOST": "host", "PORT": "5432", "DATABASE_NAME": "database_name"} - return SettingsParameters.create(settings_class = MountainAshBaseSettings, namespace="mock_mssql", kwargs=kwargs) - -################ -# Connections - -# @pytest.fixture -# def mock_sqlite_connection(mock_settings_parameters_1): -# with mock.patch('mountainash_data.databases.connections.ibis.SQLite_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield SQLite_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_1) - -# @pytest.fixture -# def mock_duckdb_connection(mock_settings_parameters_1): -# with mock.patch('mountainash_data.databases.connections.ibis.DuckDB_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield DuckDB_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_1) - -@pytest.fixture -def sqlite_connection(): - - settings_parameters = SettingsParameters.create(settings_class = SQLiteAuthSettings, namespace="SQLiteAuthSettings", kwargs={"auth": NoAuth()}) - - return SQLite_IbisConnection(db_auth_settings_parameters=settings_parameters) - -@pytest.fixture -def duckdb_connection(): - - settings_parameters = SettingsParameters.create(settings_class = DuckDBAuthSettings, namespace="DuckDBAuthSettings", kwargs={"auth": NoAuth()}) - - return DuckDB_IbisConnection(db_auth_settings_parameters=settings_parameters) - - - -# @pytest.fixture -# def mock_postgres_connection(mock_settings_parameters_2): -# with mock.patch('mountainash_data.databases.connections.ibis.Postgres_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield Postgres_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_2) - -# @pytest.fixture -# def mock_mssql_connection(mock_settings_parameters_3): -# with mock.patch('mountainash_data.databases.connections.ibis.MSSQL_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield MSSQL_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_3) - -# @pytest.fixture -# def mock_snowflake_connection(mock_settings_parameters_3): -# with mock.patch('mountainash_data.databases.connections.ibis.Snowflake_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield Snowflake_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_3) - - -# @pytest.fixture -# def mock_mysql_connection(mock_settings_parameters_3): -# with mock.patch('mountainash_data.databases.connections.ibis.MySQL_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield MySQL_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_3) - - - - -################# -# # Tests - -# def test_sqlite_connection(mock_sqlite_connection): -# mock_sqlite_connection.connect() -# assert mock_sqlite_connection.ibis_backend is not None - -# def test_duckdb_connection(mock_duckdb_connection): -# mock_duckdb_connection.connect() -# assert mock_duckdb_connection.ibis_backend is not None - -def test_sqlite_connection(sqlite_connection): - sqlite_connection.connect() - assert sqlite_connection.ibis_backend is not None - -def test_duckdb_connection(duckdb_connection): - duckdb_connection.connect() - assert duckdb_connection.ibis_backend is not None - - -# def test_postgres_connection(mock_postgres_connection): -# mock_postgres_connection.connect() -# assert mock_postgres_connection.ibis_backend is not None - -# def test_mysql_connection(mock_mysql_connection): -# mock_mysql_connection.connect() -# assert mock_mysql_connection.ibis_backend is not None - -# def test_mssql_connection(mock_mssql_connection): -# mock_mssql_connection.connect() -# assert mock_mssql_connection.ibis_backend is not None - - -# def test_snowflake_connection(mock_snowflake_connection): -# mock_snowflake_connection.connect() -# assert mock_snowflake_connection.ibis_backend is not None - - - - -# @pytest.fixture(scope="module") -# def docker_compose_file(pytestconfig): -# return os.path.join(str(pytestconfig.rootdir), "docker-compose.yml") - -# @pytest.fixture(scope="module") -# def docker_services(docker_ip, docker_services): -# docker_services.start("postgres") -# docker_services.wait_for_service("postgres", 5432) -# yield -# docker_services.stop("postgres") - -# def test_postgres_docker_connection(docker_services): -# connection = Postgres_IbisConnection(db_auth_settings_parameters=mock.MagicMock()) -# connection.connect() -# assert connection.ibis_backend is not None diff --git a/tests/test_unit/factories/test_connection_factory.py b/tests/test_unit/factories/test_connection_factory.py deleted file mode 100644 index 50cc710..0000000 --- a/tests/test_unit/factories/test_connection_factory.py +++ /dev/null @@ -1,246 +0,0 @@ -"""Tests for ConnectionFactory.""" - -import pytest -from mountainash_data.core.factories.connection_factory import ConnectionFactory -from mountainash_data.core.connection import BaseDBConnection -from mountainash_data.backends.ibis.connection import SQLite_IbisConnection, DuckDB_IbisConnection -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings, NoAuth -from mountainash_data.core.constants import CONST_DB_PROVIDER_TYPE -from mountainash_settings import SettingsParameters - - -@pytest.mark.unit -class TestConnectionFactoryGetStrategy: - """Tests for ConnectionFactory.get_strategy method.""" - - def test_get_strategy_for_sqlite(self): - """Test getting SQLite connection class from factory.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - connection_class = ConnectionFactory.get_strategy(settings_params) - - assert connection_class is not None - assert connection_class == SQLite_IbisConnection - - def test_get_strategy_for_duckdb(self): - """Test getting DuckDB connection class from factory.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - connection_class = ConnectionFactory.get_strategy(settings_params) - - assert connection_class is not None - assert connection_class == DuckDB_IbisConnection - - @pytest.mark.parametrize("settings_class,expected_connection", [ - (SQLiteAuthSettings, SQLite_IbisConnection), - (DuckDBAuthSettings, DuckDB_IbisConnection), - ]) - def test_get_strategy_parametrized(self, settings_class, expected_connection): - """Test strategy selection for various backend types.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs={"DATABASE": ":memory:"} - ) - - connection_class = ConnectionFactory.get_strategy(settings_params) - - assert connection_class == expected_connection - - -@pytest.mark.unit -class TestConnectionFactoryGetConnection: - """Tests for ConnectionFactory.get_connection convenience method.""" - - def test_get_connection_returns_instance(self): - """Test that get_connection returns a connection instance.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - connection = ConnectionFactory.get_connection(settings_params) - - assert connection is not None - assert isinstance(connection, BaseDBConnection) - assert isinstance(connection, SQLite_IbisConnection) - - def test_get_connection_instance_can_connect(self, temp_sqlite_db): - """Test that returned connection instance can actually connect.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - connection = ConnectionFactory.get_connection(settings_params) - backend = connection.connect() - - assert backend is not None - assert connection.is_connected() - - # Cleanup - connection.disconnect() - - def test_get_connection_with_different_backends(self): - """Test getting connections for different backends.""" - # SQLite - sqlite_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - sqlite_conn = ConnectionFactory.get_connection(sqlite_params) - assert isinstance(sqlite_conn, SQLite_IbisConnection) - - # DuckDB - duckdb_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - duckdb_conn = ConnectionFactory.get_connection(duckdb_params) - assert isinstance(duckdb_conn, DuckDB_IbisConnection) - - -@pytest.mark.unit -class TestConnectionFactoryConfiguration: - """Tests for ConnectionFactory configuration and mapping.""" - - def test_factory_has_strategy_configuration(self): - """Test that factory configures strategies.""" - # Trigger configuration by accessing class methods - factory = ConnectionFactory() - - # Should have strategy mappings configured - assert hasattr(ConnectionFactory, '_strategy_modules') - assert hasattr(ConnectionFactory, '_strategy_classes') - - def test_strategy_modules_configured(self): - """Test that strategy modules are properly configured.""" - # Access the configuration - factory = ConnectionFactory() - - # Should have mappings for key backends - assert ConnectionFactory._strategy_modules is not None - assert CONST_DB_PROVIDER_TYPE.SQLITE in ConnectionFactory._strategy_modules - assert CONST_DB_PROVIDER_TYPE.DUCKDB in ConnectionFactory._strategy_modules - - def test_strategy_classes_configured(self): - """Test that strategy classes are properly configured.""" - factory = ConnectionFactory() - - assert ConnectionFactory._strategy_classes is not None - assert CONST_DB_PROVIDER_TYPE.SQLITE in ConnectionFactory._strategy_classes - assert CONST_DB_PROVIDER_TYPE.DUCKDB in ConnectionFactory._strategy_classes - - def test_sqlite_strategy_mapping(self): - """Test SQLite strategy configuration. - - Updated in Phase 5 Task 5.2: module path now points directly at - backends.ibis.connection (bypassing the databases.connections.ibis shim chain). - """ - factory = ConnectionFactory() - - module_path = ConnectionFactory._strategy_modules.get(CONST_DB_PROVIDER_TYPE.SQLITE) - class_name = ConnectionFactory._strategy_classes.get(CONST_DB_PROVIDER_TYPE.SQLITE) - - assert module_path == "mountainash_data.backends.ibis.connection" - assert class_name == "SQLite_IbisConnection" - - def test_duckdb_strategy_mapping(self): - """Test DuckDB strategy configuration. - - Updated in Phase 5 Task 5.2: module path now points directly at - backends.ibis.connection (bypassing the databases.connections.ibis shim chain). - """ - factory = ConnectionFactory() - - module_path = ConnectionFactory._strategy_modules.get(CONST_DB_PROVIDER_TYPE.DUCKDB) - class_name = ConnectionFactory._strategy_classes.get(CONST_DB_PROVIDER_TYPE.DUCKDB) - - assert module_path == "mountainash_data.backends.ibis.connection" - assert class_name == "DuckDB_IbisConnection" - - -@pytest.mark.unit -class TestConnectionFactoryLazyLoading: - """Tests for lazy loading behavior of ConnectionFactory.""" - - def test_factory_lazy_loads_connection_classes(self): - """Test that connection classes are loaded lazily.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - # Getting strategy should work without pre-importing - connection_class = ConnectionFactory.get_strategy(settings_params) - - assert connection_class is not None - assert callable(connection_class) - - def test_multiple_calls_return_same_class(self): - """Test that multiple calls for same backend return same class.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - class1 = ConnectionFactory.get_strategy(settings_params) - class2 = ConnectionFactory.get_strategy(settings_params) - - assert class1 is class2 # Same class object - - -@pytest.mark.integration -class TestConnectionFactoryIntegration: - """Integration tests for ConnectionFactory.""" - - def test_factory_to_connection_to_backend_workflow(self, temp_sqlite_db): - """Test complete workflow: factory → connection → backend.""" - # Create settings - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - # Get connection from factory - connection = ConnectionFactory.get_connection(settings_params) - - # Connect to backend - backend = connection.connect() - - # Verify backend is functional - assert backend is not None - tables = backend.list_tables() - assert isinstance(tables, list) - assert 'test_table' in tables - - # Cleanup - connection.disconnect() - - def test_factory_works_with_multiple_concurrent_connections(self): - """Test factory can create multiple connections concurrently.""" - sqlite_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - duckdb_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - conn1 = ConnectionFactory.get_connection(sqlite_params) - conn2 = ConnectionFactory.get_connection(duckdb_params) - conn3 = ConnectionFactory.get_connection(sqlite_params) - - # All should be valid instances - assert isinstance(conn1, SQLite_IbisConnection) - assert isinstance(conn2, DuckDB_IbisConnection) - assert isinstance(conn3, SQLite_IbisConnection) - - # But conn1 and conn3 should be different instances - assert conn1 is not conn3 diff --git a/tests/test_unit/factories/test_operations_factory.py b/tests/test_unit/factories/test_operations_factory.py deleted file mode 100644 index 4b2abb3..0000000 --- a/tests/test_unit/factories/test_operations_factory.py +++ /dev/null @@ -1,239 +0,0 @@ -"""Tests for OperationsFactory.""" - -import pytest -from mountainash_data.core.factories.operations_factory import OperationsFactory -from mountainash_data.backends.ibis.operations import BaseIbisOperations -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings, NoAuth -from mountainash_data.core.constants import CONST_DB_PROVIDER_TYPE -from mountainash_settings import SettingsParameters - - -@pytest.mark.unit -class TestOperationsFactoryGetStrategy: - """Tests for OperationsFactory.get_strategy method.""" - - def test_get_strategy_for_sqlite(self): - """Test getting SQLite operations class from factory.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations_class = OperationsFactory.get_strategy(settings_params) - - assert operations_class is not None - # Should be a subclass of BaseIbisOperations - assert issubclass(operations_class, BaseIbisOperations) - - def test_get_strategy_for_duckdb(self): - """Test getting DuckDB operations class from factory.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations_class = OperationsFactory.get_strategy(settings_params) - - assert operations_class is not None - assert issubclass(operations_class, BaseIbisOperations) - - @pytest.mark.parametrize("settings_class", [ - SQLiteAuthSettings, - DuckDBAuthSettings, - ]) - def test_get_strategy_parametrized(self, settings_class): - """Test strategy selection for various backend types.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs={"DATABASE": ":memory:"} - ) - - operations_class = OperationsFactory.get_strategy(settings_params) - - assert operations_class is not None - assert issubclass(operations_class, BaseIbisOperations) - - -@pytest.mark.unit -class TestOperationsFactoryGetOperations: - """Tests for OperationsFactory.get_operations convenience method.""" - - def test_get_operations_returns_instance(self): - """Test that get_operations returns an operations instance.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations = OperationsFactory.get_operations(settings_params) - - assert operations is not None - assert isinstance(operations, BaseIbisOperations) - - def test_get_operations_for_different_backends(self): - """Test getting operations for different backends.""" - # SQLite - sqlite_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - sqlite_ops = OperationsFactory.get_operations(sqlite_params) - assert isinstance(sqlite_ops, BaseIbisOperations) - - # DuckDB - duckdb_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - duckdb_ops = OperationsFactory.get_operations(duckdb_params) - assert isinstance(duckdb_ops, BaseIbisOperations) - - -@pytest.mark.unit -class TestOperationsFactoryConfiguration: - """Tests for OperationsFactory configuration and mapping.""" - - def test_factory_has_strategy_configuration(self): - """Test that factory configures strategies.""" - factory = OperationsFactory() - - assert hasattr(OperationsFactory, '_strategy_modules') - assert hasattr(OperationsFactory, '_strategy_classes') - - def test_strategy_modules_configured(self): - """Test that strategy modules are properly configured.""" - factory = OperationsFactory() - - assert OperationsFactory._strategy_modules is not None - assert CONST_DB_PROVIDER_TYPE.SQLITE in OperationsFactory._strategy_modules - assert CONST_DB_PROVIDER_TYPE.DUCKDB in OperationsFactory._strategy_modules - - def test_strategy_classes_configured(self): - """Test that strategy classes are properly configured.""" - factory = OperationsFactory() - - assert OperationsFactory._strategy_classes is not None - assert CONST_DB_PROVIDER_TYPE.SQLITE in OperationsFactory._strategy_classes - assert CONST_DB_PROVIDER_TYPE.DUCKDB in OperationsFactory._strategy_classes - - def test_sqlite_strategy_mapping(self): - """Test SQLite operations strategy configuration. - - Updated in Phase 5 Task 5.2: module path now points directly at - backends.ibis.operations (bypassing the databases.operations.ibis shim chain). - """ - factory = OperationsFactory() - - module_path = OperationsFactory._strategy_modules.get(CONST_DB_PROVIDER_TYPE.SQLITE) - class_name = OperationsFactory._strategy_classes.get(CONST_DB_PROVIDER_TYPE.SQLITE) - - assert "mountainash_data.backends.ibis.operations" in module_path - assert "Operations" in class_name - - def test_duckdb_strategy_mapping(self): - """Test DuckDB operations strategy configuration. - - Updated in Phase 5 Task 5.2: module path now points directly at - backends.ibis.operations (bypassing the databases.operations.ibis shim chain). - """ - factory = OperationsFactory() - - module_path = OperationsFactory._strategy_modules.get(CONST_DB_PROVIDER_TYPE.DUCKDB) - class_name = OperationsFactory._strategy_classes.get(CONST_DB_PROVIDER_TYPE.DUCKDB) - - assert "mountainash_data.backends.ibis.operations" in module_path - assert "Operations" in class_name - - -@pytest.mark.unit -class TestOperationsFactoryLazyLoading: - """Tests for lazy loading behavior of OperationsFactory.""" - - def test_factory_lazy_loads_operations_classes(self): - """Test that operations classes are loaded lazily.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations_class = OperationsFactory.get_strategy(settings_params) - - assert operations_class is not None - assert callable(operations_class) - - def test_multiple_calls_return_same_class(self): - """Test that multiple calls for same backend return same class.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - class1 = OperationsFactory.get_strategy(settings_params) - class2 = OperationsFactory.get_strategy(settings_params) - - assert class1 is class2 - - -@pytest.mark.unit -class TestOperationsFactoryMatchesConnectionFactory: - """Tests to ensure operations factory aligns with connection factory.""" - - def test_operations_backend_matches_connection_backend(self): - """Test that operations backend type matches connection backend type.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations = OperationsFactory.get_operations(settings_params) - - # Operations should have correct backend type - assert hasattr(operations, 'db_backend_name') - - -@pytest.mark.integration -class TestOperationsFactoryIntegration: - """Integration tests for OperationsFactory.""" - - def test_operations_work_with_actual_backend(self, temp_sqlite_db): - """Test that factory-created operations work with real backend.""" - from mountainash_data.core.utils import DatabaseUtils - - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - # Get operations from factory - operations = OperationsFactory.get_operations(settings_params) - - # Get connected backend - backend = DatabaseUtils.create_backend(settings_params) - - # Test operations with backend - tables = operations.list_tables(backend) - assert isinstance(tables, list) - assert 'test_table' in tables - - def test_factory_creates_multiple_operations_instances(self): - """Test factory can create multiple operations instances.""" - sqlite_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - duckdb_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - ops1 = OperationsFactory.get_operations(sqlite_params) - ops2 = OperationsFactory.get_operations(duckdb_params) - ops3 = OperationsFactory.get_operations(sqlite_params) - - # All should be valid instances - assert isinstance(ops1, BaseIbisOperations) - assert isinstance(ops2, BaseIbisOperations) - assert isinstance(ops3, BaseIbisOperations) - - # ops1 and ops3 should be different instances - assert ops1 is not ops3 diff --git a/tests/test_unit/factories/test_settings_factory.py b/tests/test_unit/factories/test_settings_factory.py deleted file mode 100644 index 1c07ea5..0000000 --- a/tests/test_unit/factories/test_settings_factory.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Tests for SettingsFactory.""" - -import pytest -from mountainash_data.core.factories.settings_factory import SettingsFactory -from mountainash_data.core.settings import ( - SQLiteAuthSettings, - DuckDBAuthSettings, - ConnectionProfile, -) -from mountainash_data.core.constants import CONST_DB_PROVIDER_TYPE - - -@pytest.mark.unit -class TestSettingsFactoryFromConnectionString: - """Tests for SettingsFactory.from_connection_string method.""" - - def test_from_sqlite_connection_string(self): - """Test creating settings from SQLite connection string.""" - url = "sqlite:///test.db" - - settings = SettingsFactory.from_connection_string(url) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - def test_from_duckdb_connection_string(self): - """Test creating settings from DuckDB connection string.""" - url = "duckdb:///test.duckdb" - - settings = SettingsFactory.from_connection_string(url) - - assert settings is not None - assert isinstance(settings, DuckDBAuthSettings) - - @pytest.mark.parametrize("url,expected_type", [ - ("sqlite:///:memory:", SQLiteAuthSettings), - ("sqlite:///test.db", SQLiteAuthSettings), - ("duckdb:///:memory:", DuckDBAuthSettings), - ("duckdb:///test.duckdb", DuckDBAuthSettings), - ]) - def test_from_connection_string_parametrized(self, url, expected_type): - """Test connection string parsing for various database types.""" - settings = SettingsFactory.from_connection_string(url) - - assert settings is not None - assert isinstance(settings, expected_type) - - def test_from_connection_string_with_kwargs(self): - """Test from_connection_string with additional kwargs.""" - url = "sqlite:///:memory:" - - settings = SettingsFactory.from_connection_string( - url, - timeout=30 - ) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - -@pytest.mark.unit -class TestSettingsFactoryFromBackendType: - """Tests for SettingsFactory.from_backend_type method.""" - - def test_from_sqlite_backend_type(self): - """Test creating settings from SQLite backend type.""" - settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - def test_from_duckdb_backend_type(self): - """Test creating settings from DuckDB backend type.""" - settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.DUCKDB, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, DuckDBAuthSettings) - - @pytest.mark.parametrize("backend_type,expected_settings", [ - (CONST_DB_PROVIDER_TYPE.SQLITE, SQLiteAuthSettings), - (CONST_DB_PROVIDER_TYPE.DUCKDB, DuckDBAuthSettings), - ]) - def test_from_backend_type_parametrized(self, backend_type, expected_settings): - """Test settings creation for various backend types.""" - settings = SettingsFactory.from_backend_type( - backend_type, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, expected_settings) - - def test_from_backend_type_with_multiple_kwargs(self): - """Test from_backend_type with multiple configuration parameters.""" - settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:", - timeout=30, - read_only=False - ) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - -@pytest.mark.unit -class TestSettingsFactoryDetectBackend: - """Tests for SettingsFactory.detect_backend_from_url method.""" - - @pytest.mark.parametrize("url,expected_backend", [ - ("sqlite:///test.db", CONST_DB_PROVIDER_TYPE.SQLITE), - ("sqlite:///:memory:", CONST_DB_PROVIDER_TYPE.SQLITE), - ("duckdb:///test.duckdb", CONST_DB_PROVIDER_TYPE.DUCKDB), - ("duckdb:///:memory:", CONST_DB_PROVIDER_TYPE.DUCKDB), - ("postgresql://localhost/db", CONST_DB_PROVIDER_TYPE.POSTGRESQL), - ("postgres://localhost/db", CONST_DB_PROVIDER_TYPE.POSTGRESQL), - ]) - def test_detect_backend_from_url(self, url, expected_backend): - """Test backend detection from various URL formats.""" - detected_backend = SettingsFactory.detect_backend_from_url(url) - - assert detected_backend == expected_backend - - def test_detect_backend_sqlite_variations(self): - """Test SQLite URL variations.""" - sqlite_urls = [ - "sqlite:///absolute/path/to/db.sqlite", - "sqlite:///:memory:", - "sqlite:///relative/path.db", - ] - - for url in sqlite_urls: - backend = SettingsFactory.detect_backend_from_url(url) - assert backend == CONST_DB_PROVIDER_TYPE.SQLITE, f"Failed for URL: {url}" - - def test_detect_backend_duckdb_variations(self): - """Test DuckDB URL variations.""" - duckdb_urls = [ - "duckdb:///path/to/db.duckdb", - "duckdb:///:memory:", - "duckdb:///test.db", - ] - - for url in duckdb_urls: - backend = SettingsFactory.detect_backend_from_url(url) - assert backend == CONST_DB_PROVIDER_TYPE.DUCKDB, f"Failed for URL: {url}" - - -@pytest.mark.unit -class TestSettingsFactoryConfiguration: - """Tests for SettingsFactory configuration.""" - - def test_factory_has_backend_mappings(self): - """Test that factory has backend to settings mappings.""" - # This tests that the factory properly maps backend types to settings classes - sqlite_settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:" - ) - duckdb_settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.DUCKDB, - DATABASE=":memory:" - ) - - assert type(sqlite_settings) != type(duckdb_settings) - assert isinstance(sqlite_settings, ConnectionProfile) - assert isinstance(duckdb_settings, ConnectionProfile) - - def test_factory_lazy_loads_settings_classes(self): - """Test that settings classes are loaded lazily.""" - # Creating settings should work without pre-importing - settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:" - ) - - assert settings is not None - - -@pytest.mark.unit -class TestSettingsFactoryErrorHandling: - """Tests for SettingsFactory error handling.""" - - def test_invalid_url_format(self): - """Test handling of invalid URL formats.""" - invalid_url = "not-a-valid-url" - - with pytest.raises((ValueError, KeyError, AttributeError)): - SettingsFactory.from_connection_string(invalid_url) - - def test_unsupported_backend_type(self): - """Test handling of URL with unsupported backend.""" - # This should either raise an error or handle gracefully - unsupported_url = "unknown-backend://localhost/db" - - with pytest.raises((ValueError, KeyError, AttributeError, NotImplementedError)): - SettingsFactory.from_connection_string(unsupported_url) - - -@pytest.mark.integration -class TestSettingsFactoryIntegration: - """Integration tests for SettingsFactory.""" - - def test_settings_factory_to_connection_workflow(self, temp_sqlite_db): - """Test workflow: factory creates settings → settings create connection.""" - from mountainash_data.core.utils import DatabaseUtils - - # Factory creates settings - url = f"sqlite:///{temp_sqlite_db}" - settings = SettingsFactory.from_connection_string(url) - - # Settings can be used to create connection - settings_params = settings.extract_settings_parameters() - connection = DatabaseUtils.create_connection(settings_params) - - assert connection is not None - - # Connection can connect - backend = connection.connect() - assert backend is not None - - # Cleanup - connection.disconnect() - - def test_settings_factory_round_trip(self): - """Test that settings created from URL can generate correct connection strings.""" - original_url = "sqlite:///:memory:" - - # Create settings from URL - settings = SettingsFactory.from_connection_string(original_url) - - # Settings should have connection info - assert settings is not None - assert hasattr(settings, 'DATABASE') - - def test_factory_creates_independent_settings_instances(self): - """Test that factory creates independent settings instances.""" - settings1 = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE="db1.sqlite" - ) - settings2 = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE="db2.sqlite" - ) - - # Different instances - assert settings1 is not settings2 - - # Different databases - assert settings1.DATABASE != settings2.DATABASE diff --git a/tests/test_unit/test_database_utils.py b/tests/test_unit/test_database_utils.py deleted file mode 100644 index a40f45d..0000000 --- a/tests/test_unit/test_database_utils.py +++ /dev/null @@ -1,269 +0,0 @@ -"""Tests for DatabaseUtils high-level API.""" - -import pytest -from pathlib import Path -from mountainash_data.core.utils import DatabaseUtils -from mountainash_data.core.connection import BaseDBConnection -from mountainash_data.backends.ibis.connection import SQLite_IbisConnection, DuckDB_IbisConnection -from mountainash_data.backends.ibis.operations import BaseIbisOperations -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings, NoAuth -from mountainash_data.core.constants import CONST_DB_PROVIDER_TYPE -from mountainash_settings import SettingsParameters - - -@pytest.mark.unit -class TestDatabaseUtilsCreateConnection: - """Tests for DatabaseUtils.create_connection method.""" - - def test_create_connection_with_sqlite_settings(self, temp_sqlite_db): - """Test creating SQLite connection from settings parameters.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db)} - ) - - connection = DatabaseUtils.create_connection(settings_params) - - assert connection is not None - assert isinstance(connection, BaseDBConnection) - assert isinstance(connection, SQLite_IbisConnection) - - def test_create_connection_with_duckdb_settings(self): - """Test creating DuckDB connection from settings parameters.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - connection = DatabaseUtils.create_connection(settings_params) - - assert connection is not None - assert isinstance(connection, BaseDBConnection) - assert isinstance(connection, DuckDB_IbisConnection) - - def test_created_connection_can_connect(self, temp_sqlite_db): - """Test that created connection can actually connect.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - connection = DatabaseUtils.create_connection(settings_params) - backend = connection.connect() - - assert backend is not None - assert connection.is_connected() - - # Cleanup - connection.disconnect() - - -@pytest.mark.unit -class TestDatabaseUtilsCreateOperations: - """Tests for DatabaseUtils.create_operations method.""" - - def test_create_operations_with_sqlite_settings(self, temp_sqlite_db): - """Test creating operations from SQLite settings parameters.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db)} - ) - - operations = DatabaseUtils.create_operations(settings_params) - - assert operations is not None - assert isinstance(operations, BaseIbisOperations) - - def test_create_operations_with_duckdb_settings(self): - """Test creating operations from DuckDB settings parameters.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations = DatabaseUtils.create_operations(settings_params) - - assert operations is not None - assert isinstance(operations, BaseIbisOperations) - - -@pytest.mark.unit -class TestDatabaseUtilsCreateBackend: - """Tests for DatabaseUtils.create_backend convenience method.""" - - def test_create_backend_connects_immediately(self, temp_sqlite_db): - """Test that create_backend returns connected backend.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - backend = DatabaseUtils.create_backend(settings_params) - - assert backend is not None - # Should be able to list tables - tables = backend.list_tables() - assert isinstance(tables, list) - assert 'test_table' in tables - - def test_create_backend_with_duckdb(self): - """Test create_backend with DuckDB in-memory.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - backend = DatabaseUtils.create_backend(settings_params) - - assert backend is not None - # Should be able to create and list tables - backend.create_table('test', {'id': [1, 2, 3]}) - tables = backend.list_tables() - assert 'test' in tables - - -@pytest.mark.unit -class TestDatabaseUtilsSettingsFromURL: - """Tests for DatabaseUtils.create_settings_from_url method.""" - - def test_create_settings_from_sqlite_url(self): - """Test creating settings from SQLite URL.""" - url = "sqlite:///test.db" - - settings = DatabaseUtils.create_settings_from_url(url) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - def test_create_settings_from_duckdb_url(self): - """Test creating settings from DuckDB URL.""" - url = "duckdb:///test.duckdb" - - settings = DatabaseUtils.create_settings_from_url(url) - - assert settings is not None - assert isinstance(settings, DuckDBAuthSettings) - - @pytest.mark.parametrize("url,expected_type", [ - ("sqlite:///:memory:", SQLiteAuthSettings), - ("duckdb:///:memory:", DuckDBAuthSettings), - ]) - def test_create_settings_from_url_parametrized(self, url, expected_type): - """Test URL detection for various database types.""" - settings = DatabaseUtils.create_settings_from_url(url) - - assert settings is not None - assert isinstance(settings, expected_type) - - -@pytest.mark.unit -class TestDatabaseUtilsSettingsFromBackendType: - """Tests for DatabaseUtils.create_settings_from_backend_type method.""" - - def test_create_settings_from_sqlite_backend_type(self): - """Test creating settings from backend type enum.""" - settings = DatabaseUtils.create_settings_from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - def test_create_settings_from_duckdb_backend_type(self): - """Test creating settings from DuckDB backend type.""" - settings = DatabaseUtils.create_settings_from_backend_type( - CONST_DB_PROVIDER_TYPE.DUCKDB, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, DuckDBAuthSettings) - - @pytest.mark.parametrize("backend_type,expected_settings", [ - (CONST_DB_PROVIDER_TYPE.SQLITE, SQLiteAuthSettings), - (CONST_DB_PROVIDER_TYPE.DUCKDB, DuckDBAuthSettings), - ]) - def test_create_settings_from_backend_type_parametrized(self, backend_type, expected_settings): - """Test settings creation for various backend types.""" - settings = DatabaseUtils.create_settings_from_backend_type( - backend_type, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, expected_settings) - - -@pytest.mark.unit -class TestDatabaseUtilsDetectBackend: - """Tests for DatabaseUtils.detect_backend_from_url method.""" - - @pytest.mark.parametrize("url,expected_backend", [ - ("sqlite:///test.db", CONST_DB_PROVIDER_TYPE.SQLITE), - ("sqlite:///:memory:", CONST_DB_PROVIDER_TYPE.SQLITE), - ("duckdb:///test.duckdb", CONST_DB_PROVIDER_TYPE.DUCKDB), - ("duckdb:///:memory:", CONST_DB_PROVIDER_TYPE.DUCKDB), - ]) - def test_detect_backend_from_url(self, url, expected_backend): - """Test backend detection from various URL formats.""" - detected_backend = DatabaseUtils.detect_backend_from_url(url) - - assert detected_backend == expected_backend - - -@pytest.mark.integration -class TestDatabaseUtilsCreateFromURL: - """Tests for DatabaseUtils.create_from_url end-to-end method.""" - - def test_create_from_sqlite_url_end_to_end(self, temp_sqlite_db): - """Test complete workflow from URL to connected backend.""" - url = f"sqlite:///{temp_sqlite_db}" - - connection, backend = DatabaseUtils.create_from_url(url) - - assert connection is not None - assert isinstance(connection, SQLite_IbisConnection) - assert backend is not None - # Verify backend is actually connected and functional - tables = backend.list_tables() - assert isinstance(tables, list) - - # Create a table to verify functionality - backend.create_table('workflow_test', {'id': [1, 2, 3]}, overwrite=True) - assert 'workflow_test' in backend.list_tables() - - # Cleanup - connection.disconnect() - - def test_create_from_duckdb_url_end_to_end(self): - """Test complete workflow with DuckDB in-memory.""" - url = "duckdb:///:memory:" - - connection, backend = DatabaseUtils.create_from_url(url) - - assert connection is not None - assert isinstance(connection, DuckDB_IbisConnection) - assert backend is not None - # Verify we can perform operations - backend.create_table('test_table', {'id': [1, 2, 3], 'value': [10, 20, 30]}) - tables = backend.list_tables() - assert 'test_table' in tables - - # Cleanup - connection.disconnect() - - def test_create_from_url_with_config_files(self, temp_sqlite_db, tmp_path): - """Test create_from_url with additional config files.""" - url = f"sqlite:///{temp_sqlite_db}" - - connection, backend = DatabaseUtils.create_from_url( - url, - config_files=[] # Empty list, just testing parameter works - ) - - assert connection is not None - assert backend is not None - - # Cleanup - connection.disconnect() diff --git a/tests/test_unit/test_mountainash_data.py b/tests/test_unit/test_mountainash_data.py index 62ca7bb..48b0495 100644 --- a/tests/test_unit/test_mountainash_data.py +++ b/tests/test_unit/test_mountainash_data.py @@ -1,7 +1,6 @@ """Tests for main mountainash_data package.""" import pytest -from mountainash_data import __version__ import mountainash_data @@ -9,77 +8,50 @@ class TestPackageImports: """Test package-level imports and structure.""" def test_version_import(self): - """Test that version can be imported.""" assert hasattr(mountainash_data, '__version__') assert isinstance(mountainash_data.__version__, str) assert len(mountainash_data.__version__) > 0 def test_version_format(self): - """Test version follows expected format.""" version_parts = mountainash_data.__version__.split('.') - assert len(version_parts) >= 2, "Version should have at least major.minor" - - # Test that major and minor are numeric - assert version_parts[0].isdigit(), "Major version should be numeric" - assert version_parts[1].isdigit(), "Minor version should be numeric" + assert len(version_parts) >= 2 + assert version_parts[0].isdigit() + assert version_parts[1].isdigit() def test_core_imports_available(self): - """Test that core classes can be imported.""" from mountainash_data.core.connection import BaseDBConnection - - # Check classes are properly defined assert BaseDBConnection is not None - def test_database_connections_available(self): - """Test that database connection classes can be imported.""" - from mountainash_data.backends.ibis.connection import SQLite_IbisConnection - from mountainash_data.backends.ibis.connection import DuckDB_IbisConnection - - assert SQLite_IbisConnection is not None - assert DuckDB_IbisConnection is not None - class TestPackageStructure: """Test package structure and organization.""" def test_package_has_init(self): - """Test that package has proper __init__.py.""" - import mountainash_data assert hasattr(mountainash_data, '__file__') def test_new_submodules_exist(self): - """Test that the new core and backends submodules exist.""" import mountainash_data.core import mountainash_data.backends - assert hasattr(mountainash_data, 'core') assert hasattr(mountainash_data, 'backends') - @pytest.mark.parametrize("module_name", [ - "core.connection", - "backends.ibis.connection", - ]) - def test_module_importable(self, module_name: str): - """Test that core modules can be imported.""" - from importlib import import_module - full_module = f"mountainash_data.{module_name}" - - module = import_module(full_module) - assert module is not None - def test_public_api_ibis_backend(self): - """Test that IbisBackend is available from the top-level package.""" from mountainash_data import IbisBackend assert IbisBackend is not None - def test_public_api_factories(self): - """Test that factories are available from the top-level package.""" - from mountainash_data import ConnectionFactory, OperationsFactory, SettingsFactory - assert ConnectionFactory is not None - assert OperationsFactory is not None - assert SettingsFactory is not None - - def test_public_api_database_utils(self): - """Test that DatabaseUtils is available from the top-level package.""" - from mountainash_data import DatabaseUtils - assert DatabaseUtils is not None + def test_public_api_backend_protocol(self): + from mountainash_data import Backend + assert Backend is not None + + def test_public_api_inspection_model(self): + from mountainash_data import CatalogInfo, ColumnInfo, NamespaceInfo, TableInfo + assert CatalogInfo is not None + assert ColumnInfo is not None + assert NamespaceInfo is not None + assert TableInfo is not None + + def test_removed_exports_not_available(self): + assert not hasattr(mountainash_data, 'ConnectionFactory') + assert not hasattr(mountainash_data, 'OperationsFactory') + assert not hasattr(mountainash_data, 'SettingsFactory') + assert not hasattr(mountainash_data, 'DatabaseUtils')