diff --git a/docs/superpowers/plans/2026-04-27-backend-as-single-handle.md b/docs/superpowers/plans/2026-04-27-backend-as-single-handle.md new file mode 100644 index 0000000..d8ea677 --- /dev/null +++ b/docs/superpowers/plans/2026-04-27-backend-as-single-handle.md @@ -0,0 +1,1636 @@ +# Backend as Single Handle — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `IbisBackend` the single public handle for all ibis interaction — lifecycle, inspection, and operations — then delete all legacy factories, utils, and class hierarchies. + +**Architecture:** `IbisBackend` composes an internal `IbisConnection` for inspection, delegates thin wrapper operations to the raw ibis connection, and dispatches per-dialect operations (upsert, indexes) via callable hooks on `DialectSpec`. Fluent methods return `self`; terminal methods return data. + +**Tech Stack:** Python 3.12, ibis-framework 10.4.0, pytest 8.3.5, hatch + +**Spec:** `docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md` +**Branch:** `feature/settings-aware-ibis-backend` (continuing from Phase 1) +**Test command:** `hatch run test:test-quick` + +--- + +## File Structure + +### Modified + +| File | Responsibility | +|------|----------------| +| `src/mountainash_data/core/protocol.py` | Updated `Backend` protocol (connect→Self, context manager, inspection). Remove `Connection` protocol. | +| `src/mountainash_data/backends/ibis/backend.py` | Lifecycle (`connect`/`close`/context manager), accessor methods, thin wrappers, hook-dispatched operations | +| `src/mountainash_data/backends/ibis/dialects/_registry.py` | New hook fields on `DialectSpec`, wiring to standalone functions | +| `src/mountainash_data/backends/ibis/operations.py` | Extract mixin methods to standalone hook functions. Delete class hierarchy (keep functions only). | +| `src/mountainash_data/backends/iceberg/backend.py` | `connect()` returns `Self`, add `__enter__`/`__exit__`/`close()` | +| `src/mountainash_data/__init__.py` | Remove factory/utils/Connection exports | +| `tests/test_unit/backends/ibis/test_backend.py` | Expand with lifecycle + operations tests | +| `tests/test_unit/test_mountainash_data.py` | Update import assertions for new public API | +| `tests/test_unit/databases/settings/test_settings_parametrized.py` | Replace factory/utils calls with `IbisBackend` | +| `tests/test_integration/test_end_to_end_workflows.py` | Rewrite to use `IbisBackend` | + +### Deleted + +| File/Directory | Reason | +|----------------|--------| +| `src/mountainash_data/core/factories/` | Entire directory — all factories replaced | +| `src/mountainash_data/core/utils.py` | `DatabaseUtils` replaced by `IbisBackend` | +| `src/mountainash_data/backends/ibis/connection.py` | `BaseIbisConnection` + 12 subclasses replaced | +| `tests/test_unit/factories/` | All factory tests | +| `tests/test_unit/test_database_utils.py` | `DatabaseUtils` tests | +| `tests/test_unit/databases/test_database_connections.py` | Legacy connection tests | +| `tests/test_unit/databases/connections/` | Legacy connection lifecycle tests | +| `tests/test_unit/databases/test_ibis_backends.py` | Legacy ibis backend tests | +| `tests/test_unit/databases/operations/` | Legacy operations tests (rewritten in test_backend.py) | + +### Kept (no changes) + +| File | Reason | +|------|--------| +| `src/mountainash_data/core/connection.py` | Iceberg depends on `BaseDBConnection` | +| `src/mountainash_data/core/inspection.py` | Unchanged | +| `src/mountainash_data/core/settings/` | Unchanged | +| `src/mountainash_data/backends/ibis/inspect.py` | Unchanged | + +--- + +### Task 1: Update Backend protocol and add lifecycle to IbisBackend + +**Files:** +- Modify: `src/mountainash_data/core/protocol.py` +- Modify: `src/mountainash_data/backends/ibis/backend.py` +- Test: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing lifecycle tests** + +Add these tests to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + +def test_connect_returns_self(): + """connect() must return the backend instance itself.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + result = backend.connect() + assert result is backend + + +def test_close_returns_self(): + """close() must return the backend instance itself.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + result = backend.close() + assert result is backend + + +def test_context_manager(): + """with IbisBackend(...) as backend: must connect and close.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + assert backend.list_tables() == [] + # After exit, should be closed + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_double_close_is_idempotent(): + """Calling close() twice must not raise.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + backend.close() + backend.close() # Must not raise + + +def test_use_before_connect_raises(): + """Calling methods before connect() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_use_after_close_raises(): + """Calling methods after close() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + backend.close() + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_ibis_connection_accessor(): + """ibis_connection() returns the raw ibis backend object.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + raw = backend.ibis_connection() + assert hasattr(raw, "list_tables") + + +def test_ibis_connection_before_connect_raises(): + """ibis_connection() before connect() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + with pytest.raises(RuntimeError, match="not connected"): + backend.ibis_connection() + + +def test_get_connection_accessor(): + """get_connection() returns our IbisConnection wrapper.""" + from mountainash_data.backends.ibis.backend import IbisConnection + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + conn = backend.get_connection() + assert isinstance(conn, IbisConnection) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_connect_returns_self -v` +Expected: FAIL (connect() returns IbisConnection, not self) + +- [ ] **Step 3: Update Backend protocol** + +Replace the entire contents of `src/mountainash_data/core/protocol.py` with: + +```python +"""Backend protocol. + +This is the structural contract every backend implementation must +satisfy. Implementations are plain classes — there is no inheritance. +""" + +from __future__ import annotations + +import typing as t + +from mountainash_data.core.inspection import ( + CatalogInfo, + NamespaceInfo, + TableInfo, +) + + +@t.runtime_checkable +class Backend(t.Protocol): + """The single handle for interacting with a backend service. + + Backends are constructed with config, connected via connect(), + used for inspection and operations, then closed. + """ + + name: str + + def connect(self) -> t.Self: ... + def close(self) -> t.Self: ... + def __enter__(self) -> t.Self: ... + def __exit__(self, *args: t.Any) -> None: ... + + def list_tables(self, namespace: str | None = None) -> list[str]: ... + def list_namespaces(self) -> list[str]: ... + + def inspect_table( + self, name: str, namespace: str | None = None + ) -> TableInfo: ... + + def inspect_namespace(self, name: str) -> NamespaceInfo: ... + def inspect_catalog(self) -> CatalogInfo: ... +``` + +- [ ] **Step 4: Add lifecycle and accessor methods to IbisBackend** + +In `src/mountainash_data/backends/ibis/backend.py`, make these changes: + +4a. Add `_conn: IbisConnection | None = None` initialisation to each `_init_from_*` method (set `self._conn = None`). + +4b. Replace the existing `connect()` method and add `close()`, `__enter__`, `__exit__`, accessor methods, and a `_require_connected` helper. Replace everything from `def connect(self)` to end of file with: + +```python + def _require_connected(self) -> IbisConnection: + if self._conn is None: + raise RuntimeError( + "IbisBackend is not connected. Call connect() first." + ) + return self._conn + + def connect(self) -> IbisBackend: + """Build a live ibis connection. Returns self for fluent chaining.""" + if self._conn is not None: + return self + if self._spec.connection_builder is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} has no connection_builder configured" + ) + if self._url is not None: + import ibis + ibis_conn = ibis.connect(self._url, **self._config) + else: + cleaned_config = { + k: v for k, v in self._config.items() + if not (isinstance(v, (list, tuple)) and len(v) == 0) + } + ibis_conn = self._spec.connection_builder(**cleaned_config) + self._conn = IbisConnection(ibis_conn, self._spec) + return self + + def close(self) -> IbisBackend: + """Release the connection. Idempotent. Returns self.""" + if self._conn is not None: + self._conn.close() + self._conn = None + return self + + def __enter__(self) -> IbisBackend: + self.connect() + return self + + def __exit__(self, *args: t.Any) -> None: + self.close() + + def ibis_connection(self) -> t.Any: + """Return the raw ibis backend object.""" + return self._require_connected()._ibis_conn + + def get_connection(self) -> IbisConnection: + """Return the internal IbisConnection wrapper.""" + return self._require_connected() + + # --- Inspection (terminal — delegates to IbisConnection) --- + + def list_tables(self, namespace: str | None = None) -> list[str]: + return self._require_connected().list_tables(namespace=namespace) + + def list_namespaces(self) -> list[str]: + return self._require_connected().list_namespaces() + + def inspect_table( + self, name: str, namespace: str | None = None + ) -> TableInfo: + return self._require_connected().inspect_table(name, namespace=namespace) + + def inspect_namespace(self, name: str) -> NamespaceInfo: + return self._require_connected().inspect_namespace(name) + + def inspect_catalog(self) -> CatalogInfo: + return self._require_connected().inspect_catalog() +``` + +4c. In each `_init_from_*` method, add `self._conn = None` after setting `self._config`: +- `_init_from_dialect`: after `self._config = config`, add `self._conn = None` +- `_init_from_url`: after `self._config = config`, add `self._conn = None` +- `_init_from_settings`: after `self._config = driver_kwargs`, add `self._conn = None` + +- [ ] **Step 5: Update existing tests that use the old connect() return** + +In `tests/test_unit/backends/ibis/test_backend.py`, update `test_in_memory_sqlite_connect_and_inspect` — it currently does `conn = backend.connect()` expecting an `IbisConnection`. Change it to use the backend directly: + +```python +def test_in_memory_sqlite_connect_and_inspect(): + """End-to-end test with the only dialect that needs no external service.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + try: + assert backend.list_tables() == [] + finally: + backend.close() +``` + +Update the settings path tests similarly: + +```python +def test_settings_path_sqlite(): + """Construct IbisBackend from SQLite SettingsParameters and connect.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import SQLiteAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=SQLiteAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "sqlite" + backend.connect() + tables = backend.list_tables() + assert isinstance(tables, list) + backend.close() + + +def test_settings_path_duckdb_empty_extensions(): + """DuckDB settings with default EXTENSIONS=[] must not crash ibis.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "duckdb" + backend.connect() # Must not raise — empty-list filter active + backend.close() +``` + +Update URL path tests: + +```python +def test_url_path_sqlite(): + """Construct IbisBackend from sqlite:// URL and connect.""" + backend = IbisBackend("sqlite://") + assert backend.dialect == "sqlite" + backend.connect() + backend.close() + + +def test_url_path_duckdb(): + """Construct IbisBackend from duckdb:// URL and connect.""" + backend = IbisBackend("duckdb://") + assert backend.dialect == "duckdb" + backend.connect() + backend.close() + + +def test_url_path_preserves_database(tmp_path): + """URL database component must reach the driver, not be discarded.""" + db_file = tmp_path / "test.db" + backend = IbisBackend(f"sqlite:///{db_file}") + assert backend.dialect == "sqlite" + backend.connect() + backend.close() + assert db_file.exists() +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py -v` +Expected: ALL PASS + +- [ ] **Step 7: Commit** + +```bash +git add src/mountainash_data/core/protocol.py src/mountainash_data/backends/ibis/backend.py tests/test_unit/backends/ibis/test_backend.py +git commit -m "feat(backend): add lifecycle, context manager, and accessor methods to IbisBackend + +connect() and close() return self for fluent chaining. Context manager +support via __enter__/__exit__. Inspection methods delegate to internal +IbisConnection. Backend protocol updated: connect() returns Self, +Connection protocol removed." +``` + +--- + +### Task 2: Expand DialectSpec with operation hooks and extract standalone functions + +**Files:** +- Modify: `src/mountainash_data/backends/ibis/dialects/_registry.py` +- Modify: `src/mountainash_data/backends/ibis/operations.py` +- Test: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing test for hook presence on DialectSpec** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +# --------------------------------------------------------------------------- +# DialectSpec hooks +# --------------------------------------------------------------------------- + +def test_duckdb_dialect_has_upsert_hook(): + """DuckDB DialectSpec must have upsert_hook wired.""" + spec = DIALECTS["duckdb"] + assert spec.upsert_hook is not None + + +def test_sqlite_dialect_has_create_index_hook(): + """SQLite DialectSpec must have create_index_hook wired.""" + spec = DIALECTS["sqlite"] + assert spec.create_index_hook is not None + + +def test_postgres_dialect_has_no_upsert_hook(): + """Postgres DialectSpec has no upsert_hook (not DuckDB family).""" + spec = DIALECTS["postgres"] + assert spec.upsert_hook is None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_duckdb_dialect_has_upsert_hook -v` +Expected: FAIL (DialectSpec has no `upsert_hook` attribute) + +- [ ] **Step 3: Add hook fields to DialectSpec** + +In `src/mountainash_data/backends/ibis/dialects/_registry.py`, add new type aliases and fields. + +After the existing type aliases (line 27), add: + +```python +UpsertHook = t.Callable[..., None] +CreateIndexHook = t.Callable[..., None] +DropIndexHook = t.Callable[..., None] +RenameTableHook = t.Callable[..., None] +``` + +Add new fields to the `DialectSpec` dataclass, after `get_list_indexes_sql` and before `extras`: + +```python + upsert_hook: t.Optional[UpsertHook] = None + create_index_hook: t.Optional[CreateIndexHook] = None + drop_index_hook: t.Optional[DropIndexHook] = None + rename_table_hook: t.Optional[RenameTableHook] = None +``` + +- [ ] **Step 4: Extract standalone hook functions from operations.py** + +In `src/mountainash_data/backends/ibis/operations.py`, convert the `_DuckDBFamilyOperationsMixin` class methods into standalone functions. Add these after the existing per-dialect SQL functions (after `motherduck_list_tables`, around line 213) and before the `_DuckDBFamilyOperationsMixin` class: + +```python +# =========================================================================== +# STANDALONE HOOK FUNCTIONS +# Extracted from _DuckDBFamilyOperationsMixin for DialectSpec wiring. +# =========================================================================== + +def duckdb_family_create_index( + ibis_conn: t.Any, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + unique: bool = False, + index_type: str | None = None, + where_condition: str | None = None, + database: str | None = None, + if_not_exists: bool = True, +) -> None: + """Create an index using DuckDB/SQLite syntax.""" + columns_list = _normalize_columns(columns) + + if index_name is None: + index_name = _generate_index_name(table_name, columns_list, unique=unique) + + qualified_table = _format_qualified_table(table_name, database=database) + columns_sql = ", ".join(columns_list) + + unique_sql = "UNIQUE " if unique else "" + if_not_exists_sql = "IF NOT EXISTS " if if_not_exists else "" + where_sql = f" WHERE {where_condition}" if where_condition else "" + + if index_type and index_type != CONST_INDEX_TYPE.BTREE: + warnings.warn( + f"Index type {index_type} not supported, using default BTREE" + ) + + create_sql = ( + f"CREATE {unique_sql}INDEX {if_not_exists_sql}{index_name} " + f"ON {qualified_table} ({columns_sql}){where_sql}" + ) + + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(create_sql) + + +def duckdb_family_drop_index( + ibis_conn: t.Any, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + if_exists: bool = True, +) -> None: + """Drop an index using DuckDB/SQLite syntax.""" + if_exists_sql = "IF EXISTS " if if_exists else "" + drop_sql = f"DROP INDEX {if_exists_sql}{index_name}" + + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(drop_sql) + + +def duckdb_family_upsert( + ibis_conn: t.Any, + table_name: str, + df: t.Any, + *, + conflict_columns: list[str] | str, + update_columns: list[str] | str | None = None, + conflict_action: str = CONST_CONFLICT_ACTION.UPDATE, + update_condition: str | None = None, + database: str | None = None, + schema: str | None = None, +) -> None: + """Perform upsert using INSERT ... ON CONFLICT syntax (DuckDB/SQLite).""" + conflict_cols = _normalize_columns(conflict_columns) + all_columns = ma.relation(df).columns + + if update_columns is None: + update_cols = [col for col in all_columns if col not in conflict_cols] + else: + update_cols = _normalize_columns(update_columns) + + tables = ibis_conn.list_tables() + if table_name not in tables: + raise ValueError(f"Target table '{table_name}' does not exist") + + if conflict_action not in [CONST_CONFLICT_ACTION.UPDATE, CONST_CONFLICT_ACTION.NOTHING]: + raise ValueError( + f"conflict_action must be '{CONST_CONFLICT_ACTION.UPDATE}' or " + f"'{CONST_CONFLICT_ACTION.NOTHING}', got '{conflict_action}'" + ) + + if conflict_action == CONST_CONFLICT_ACTION.NOTHING: + if update_cols or update_condition: + warnings.warn( + "update_columns and update_condition are ignored when " + "conflict_action='NOTHING'" + ) + + staging_table = f"temp_upsert_{uuid.uuid4().hex[:8]}" + qualified_table = _format_qualified_table(table_name, database=database, schema=schema) + + all_cols_sql = ", ".join(all_columns) + conflict_cols_sql = ", ".join(conflict_cols) + + if conflict_action == CONST_CONFLICT_ACTION.UPDATE: + if not update_cols: + raise ValueError( + "No columns to update. Either provide update_columns or ensure " + "dataframe has columns beyond conflict_columns" + ) + update_set_sql = ", ".join([f"{col} = EXCLUDED.{col}" for col in update_cols]) + where_sql = f" WHERE {update_condition}" if update_condition else "" + on_conflict_sql = ( + f"ON CONFLICT ({conflict_cols_sql}) DO UPDATE SET {update_set_sql}{where_sql}" + ) + else: + on_conflict_sql = f"ON CONFLICT ({conflict_cols_sql}) DO NOTHING" + + upsert_sql = f""" + INSERT INTO {qualified_table} ({all_cols_sql}) + SELECT {all_cols_sql} FROM {staging_table} + WHERE true + {on_conflict_sql} + """ + + if hasattr(ibis_conn.con, 'register'): + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute("BEGIN TRANSACTION") + cur.register(staging_table, df) + cur.execute(upsert_sql) + cur.unregister(staging_table) + cur.execute("COMMIT") + else: + ibis_conn.create_table(staging_table, df, temp=True, overwrite=True) + try: + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(upsert_sql) + ibis_conn.con.commit() + finally: + try: + ibis_conn.drop_table(staging_table, force=True) + except Exception: + pass +``` + +- [ ] **Step 5: Wire hooks in DIALECTS registry** + +In `src/mountainash_data/backends/ibis/dialects/_registry.py`, add imports for the new hook functions alongside the existing SQL function imports: + +```python +from mountainash_data.backends.ibis.operations import ( # noqa: E402 + duckdb_get_index_exists_sql, + duckdb_get_list_indexes_sql, + sqlite_get_index_exists_sql, + sqlite_get_list_indexes_sql, + motherduck_get_index_exists_sql, + motherduck_get_list_indexes_sql, + duckdb_family_upsert, + duckdb_family_create_index, + duckdb_family_drop_index, +) +``` + +Then add hook fields to the `sqlite`, `duckdb`, and `motherduck` entries in `DIALECTS`: + +For `"sqlite"`: +```python + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, +``` + +For `"duckdb"`: +```python + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, +``` + +For `"motherduck"`: +```python + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, +``` + +- [ ] **Step 6: Run tests** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py -v` +Expected: ALL PASS + +- [ ] **Step 7: Commit** + +```bash +git add src/mountainash_data/backends/ibis/dialects/_registry.py src/mountainash_data/backends/ibis/operations.py tests/test_unit/backends/ibis/test_backend.py +git commit -m "feat(registry): expand DialectSpec with operation hooks + +Add upsert_hook, create_index_hook, drop_index_hook, rename_table_hook +to DialectSpec. Extract standalone hook functions from +_DuckDBFamilyOperationsMixin. Wire DuckDB/SQLite/MotherDuck entries." +``` + +--- + +### Task 3: Add thin wrapper and hook-dispatched operations to IbisBackend + +**Files:** +- Modify: `src/mountainash_data/backends/ibis/backend.py` +- Test: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing tests for thin wrapper operations** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +import polars as pl + +# --------------------------------------------------------------------------- +# Thin wrapper operations (fluent) +# --------------------------------------------------------------------------- + +def test_create_table_returns_self(): + """create_table() must return self for fluent chaining.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + result = backend.create_table("t", {"id": [1, 2]}) + assert result is backend + assert "t" in backend.list_tables() + + +def test_drop_table_returns_self(): + """drop_table() must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.drop_table("t") + assert result is backend + assert "t" not in backend.list_tables() + + +def test_insert_returns_self(): + """insert() must return self.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.insert("t", {"id": [2]}) + assert result is backend + + +def test_truncate_returns_self(): + """truncate() must return self.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.truncate("t") + assert result is backend + + +def test_table_returns_ibis_table(): + """table() must return an ibis table expression.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2]}) + tbl = backend.table("t") + assert tbl is not None + + +def test_run_sql_returns_result(): + """run_sql() must return an ibis table expression.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2, 3]}) + result = backend.run_sql("SELECT COUNT(*) as cnt FROM t") + assert result is not None + + +def test_table_exists_returns_bool(): + """table_exists() must return True/False.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + assert backend.table_exists("nope") is False + backend.create_table("t", {"id": [1]}) + assert backend.table_exists("t") is True + + +def test_fluent_chaining(): + """Multiple fluent calls can be chained.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("a", {"id": [1]}).create_table("b", {"id": [2]}) + assert sorted(backend.list_tables()) == ["a", "b"] +``` + +- [ ] **Step 2: Write failing tests for hook-dispatched operations** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +# --------------------------------------------------------------------------- +# Hook-dispatched operations +# --------------------------------------------------------------------------- + +def test_create_index_returns_self(): + """create_index() via hook must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + result = backend.create_index("t", ["name"]) + assert result is backend + + +def test_create_unique_index_returns_self(): + """create_unique_index() must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + result = backend.create_unique_index("t", ["name"]) + assert result is backend + + +def test_drop_index_returns_self(): + """drop_index() via hook must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + backend.create_index("t", ["name"], index_name="idx_name") + result = backend.drop_index("idx_name") + assert result is backend + + +def test_index_exists(): + """index_exists() must detect created indexes.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + backend.create_index("t", ["id"], index_name="idx_id") + assert backend.index_exists("idx_id") is True + assert backend.index_exists("no_such_idx") is False + + +def test_list_indexes(): + """list_indexes() must return index info.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + backend.create_index("t", ["id"], index_name="idx_id") + indexes = backend.list_indexes("t") + assert isinstance(indexes, list) + assert len(indexes) >= 1 + + +def test_upsert_duckdb(): + """upsert() must work on DuckDB via hook.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "val": [10, 20]}) + backend.create_table("t", initial) + backend.create_unique_index("t", ["id"]) + + update = pl.DataFrame({"id": [2, 3], "val": [25, 30]}) + result = backend.upsert("t", update, conflict_columns=["id"]) + assert result is backend + + count_result = backend.run_sql("SELECT COUNT(*) as cnt FROM t") + count = count_result.to_polars()["cnt"][0] + assert count == 3 + + +def test_upsert_unsupported_dialect_raises(): + """upsert() on a dialect without upsert_hook must raise NotImplementedError.""" + backend = IbisBackend(dialect="postgres") + # Don't connect (can't connect to postgres anyway) — just check the method + backend._conn = type("FakeConn", (), {"_ibis_conn": None, "_dialect_spec": DIALECTS["postgres"], "_closed": False})() + with pytest.raises(NotImplementedError, match="does not support upsert"): + backend.upsert("t", {}, conflict_columns=["id"]) +``` + +- [ ] **Step 3: Run tests to verify they fail** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_create_table_returns_self -v` +Expected: FAIL (IbisBackend has no `create_table` method) + +- [ ] **Step 4: Add thin wrapper operations to IbisBackend** + +In `src/mountainash_data/backends/ibis/backend.py`, add these methods to the `IbisBackend` class after the inspection methods: + +```python + # --- Thin wrapper operations (fluent — return self) --- + + def create_table( + self, + name: str, + obj: t.Any, + *, + schema: t.Any | None = None, + database: str | None = None, + temp: bool = False, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.create_table( + name, obj=obj, schema=schema, database=database, + temp=temp, overwrite=overwrite, + ) + return self + + def drop_table( + self, + name: str, + *, + database: str | None = None, + force: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.drop_table(name, database=database, force=force) + return self + + def create_view( + self, + name: str, + obj: t.Any, + *, + database: str | None = None, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.create_view(name, obj=obj, database=database, overwrite=overwrite) + return self + + def drop_view( + self, + name: str, + *, + database: str | None = None, + force: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.drop_view(name, database=database, force=force) + return self + + def insert( + self, + name: str, + obj: t.Any, + *, + database: str | None = None, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.insert(name, obj=obj, database=database, overwrite=overwrite) + return self + + def truncate( + self, + name: str, + *, + database: str | None = None, + schema: str | None = None, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.truncate_table(name, schema=schema, database=database) + return self + + def rename_table(self, old_name: str, new_name: str) -> IbisBackend: + if self._spec.rename_table_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support rename_table" + ) + conn = self._require_connected() + self._spec.rename_table_hook(conn._ibis_conn, old_name, new_name) + return self + + # --- Terminal operations (return data) --- + + def table(self, name: str, *, database: str | None = None) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.table(name, database=database) + + def table_exists( + self, name: str, database: str | None = None + ) -> bool: + tables = self.list_tables() + return name in tables + + def run_sql( + self, + query: str, + *, + schema: t.Any | None = None, + dialect: str | None = None, + ) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.sql(query, schema=schema, dialect=dialect) + + def run_expr( + self, + expr: t.Any, + *, + params: dict | None = None, + limit: str | None = "default", + **kwargs: t.Any, + ) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.execute(expr, params=params, limit=limit, **kwargs) + + def to_sql( + self, + expr: t.Any, + *, + params: t.Any = None, + limit: str | None = None, + pretty: bool = False, + **kwargs: t.Any, + ) -> str | None: + conn = self._require_connected() + return conn._ibis_conn.compile(expr, params=params, limit=limit, pretty=pretty, **kwargs) +``` + +- [ ] **Step 5: Add hook-dispatched operations to IbisBackend** + +Continue adding methods in `src/mountainash_data/backends/ibis/backend.py`: + +```python + # --- Hook-dispatched operations (fluent — return self) --- + + def upsert( + self, + name: str, + obj: t.Any, + *, + conflict_columns: list[str] | str, + update_columns: list[str] | str | None = None, + conflict_action: str = "UPDATE", + update_condition: str | None = None, + database: str | None = None, + schema: str | None = None, + ) -> IbisBackend: + if self._spec.upsert_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support upsert" + ) + conn = self._require_connected() + self._spec.upsert_hook( + conn._ibis_conn, name, obj, + conflict_columns=conflict_columns, + update_columns=update_columns, + conflict_action=conflict_action, + update_condition=update_condition, + database=database, + schema=schema, + ) + return self + + def create_index( + self, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + unique: bool = False, + index_type: str | None = None, + where_condition: str | None = None, + database: str | None = None, + if_not_exists: bool = True, + ) -> IbisBackend: + if self._spec.create_index_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support create_index" + ) + conn = self._require_connected() + self._spec.create_index_hook( + conn._ibis_conn, table_name, columns, + index_name=index_name, unique=unique, index_type=index_type, + where_condition=where_condition, database=database, + if_not_exists=if_not_exists, + ) + return self + + def create_unique_index( + self, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + where_condition: str | None = None, + database: str | None = None, + ) -> IbisBackend: + return self.create_index( + table_name, columns, + index_name=index_name, unique=True, + where_condition=where_condition, database=database, + ) + + def drop_index( + self, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + if_exists: bool = True, + ) -> IbisBackend: + if self._spec.drop_index_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support drop_index" + ) + conn = self._require_connected() + self._spec.drop_index_hook( + conn._ibis_conn, index_name, + table_name=table_name, database=database, if_exists=if_exists, + ) + return self + + def index_exists( + self, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + ) -> bool: + if self._spec.get_index_exists_sql is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support index_exists" + ) + conn = self._require_connected() + check_sql = self._spec.get_index_exists_sql(index_name, table_name, database) + result = conn._ibis_conn.sql(check_sql) + if result is None: + return False + import mountainash as ma + count = ma.relation(result).to_dict()["count"][0] + return count > 0 + + def list_indexes( + self, + table_name: str, + *, + database: str | None = None, + ) -> list[dict]: + if self._spec.get_list_indexes_sql is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support list_indexes" + ) + conn = self._require_connected() + list_sql = self._spec.get_list_indexes_sql(table_name, database) + result = conn._ibis_conn.sql(list_sql) + if result is None: + return [] + import mountainash as ma + return ma.relation(result).to_dicts() +``` + +- [ ] **Step 6: Run all backend tests** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py -v` +Expected: ALL PASS + +- [ ] **Step 7: Commit** + +```bash +git add src/mountainash_data/backends/ibis/backend.py tests/test_unit/backends/ibis/test_backend.py +git commit -m "feat(backend): add thin wrapper and hook-dispatched operations + +Fluent methods (create_table, insert, upsert, create_index, etc.) return +self. Terminal methods (table, run_sql, list_indexes, etc.) return data. +Hook dispatch: upsert/index operations via DialectSpec callables." +``` + +--- + +### Task 4: Update IcebergBackend for protocol compliance + +**Files:** +- Modify: `src/mountainash_data/backends/iceberg/backend.py` +- Test: Run existing iceberg tests + +- [ ] **Step 1: Update IcebergBackend** + +Replace the contents of `src/mountainash_data/backends/iceberg/backend.py` with: + +```python +"""IcebergBackend — implements core.protocol.Backend for iceberg catalogs.""" + +from __future__ import annotations + +import typing as t + +from mountainash_data.backends.iceberg.catalogs.rest import IcebergRestConnection +from mountainash_data.backends.iceberg.connection import IcebergConnectionBase + + +_CATALOG_REGISTRY: dict[str, type[IcebergConnectionBase]] = { + "rest": IcebergRestConnection, +} + + +class IcebergBackend: + """Iceberg backend — single entry point for iceberg catalog interaction. + + Construction takes a catalog type (e.g. ``'rest'``) and config kwargs. + ``connect()`` returns ``self``. Use as a context manager. + """ + + name = "iceberg" + + def __init__(self, catalog: str, **config: t.Any) -> None: + if catalog not in _CATALOG_REGISTRY: + raise KeyError( + f"Unknown iceberg catalog type {catalog!r}. " + f"Available: {sorted(_CATALOG_REGISTRY)}" + ) + self._catalog_cls = _CATALOG_REGISTRY[catalog] + self._config = config + self._conn: IcebergConnectionBase | None = None + + def connect(self) -> IcebergBackend: + """Open a connection. Returns self for fluent chaining.""" + if self._conn is None: + self._conn = self._catalog_cls(**self._config) + return self + + def close(self) -> IcebergBackend: + """Release the connection. Idempotent. Returns self.""" + if self._conn is not None: + if hasattr(self._conn, "close"): + self._conn.close() + self._conn = None + return self + + def __enter__(self) -> IcebergBackend: + self.connect() + return self + + def __exit__(self, *args: t.Any) -> None: + self.close() + + def _require_connected(self) -> IcebergConnectionBase: + if self._conn is None: + raise RuntimeError( + "IcebergBackend is not connected. Call connect() first." + ) + return self._conn + + def list_tables(self, namespace: str | None = None) -> list[str]: + return self._require_connected().list_tables(namespace=namespace) + + def list_namespaces(self) -> list[str]: + return self._require_connected().list_namespaces() + + def inspect_table( + self, name: str, namespace: str | None = None + ) -> t.Any: + return self._require_connected().inspect_table(name, namespace=namespace) + + def inspect_namespace(self, name: str) -> t.Any: + return self._require_connected().inspect_namespace(name) + + def inspect_catalog(self) -> t.Any: + return self._require_connected().inspect_catalog() +``` + +- [ ] **Step 2: Run iceberg tests (if any pass without external services)** + +Run: `hatch run test:test-target tests/test_unit/backends/iceberg/ -v` +Expected: PASS (or skip if pyiceberg not installed) + +- [ ] **Step 3: Commit** + +```bash +git add src/mountainash_data/backends/iceberg/backend.py +git commit -m "feat(iceberg): update IcebergBackend for new Backend protocol + +connect() returns self, context manager support, inspection methods +delegate to internal connection." +``` + +--- + +### Task 5: Delete legacy code and update public API + +**Files:** +- Delete: `src/mountainash_data/core/factories/` (entire directory) +- Delete: `src/mountainash_data/core/utils.py` +- Delete: `src/mountainash_data/backends/ibis/connection.py` +- Modify: `src/mountainash_data/__init__.py` +- Modify: `src/mountainash_data/backends/ibis/operations.py` (delete class hierarchy, keep functions) + +- [ ] **Step 1: Delete factory directory, utils, and legacy connection module** + +```bash +rm -rf src/mountainash_data/core/factories/ +rm src/mountainash_data/core/utils.py +rm src/mountainash_data/backends/ibis/connection.py +``` + +- [ ] **Step 2: Clean up operations.py — delete class hierarchy, keep hook functions** + +In `src/mountainash_data/backends/ibis/operations.py`, delete everything from the `_DuckDBFamilyOperationsMixin` class definition onwards (from line ~221 to end of file). This removes: +- `_DuckDBFamilyOperationsMixin` class +- `_BaseIbisMixin` compatibility shim +- `BaseIbisOperations` abstract class +- All concrete operations subclasses (`DuckDB_IbisOperations`, `SQLite_IbisOperations`, etc.) + +Keep: +- All imports at the top +- All module-level helper functions (`_generate_index_name`, `_format_qualified_table`, `_normalize_columns`) +- All per-dialect SQL functions (`duckdb_get_index_exists_sql`, `sqlite_get_index_exists_sql`, etc.) +- All standalone hook functions (`duckdb_family_create_index`, `duckdb_family_drop_index`, `duckdb_family_upsert`) +- The `motherduck_list_tables` function + +Remove the imports that are only used by the deleted classes. After cleanup, the remaining imports should be: + +```python +import typing as t +import contextlib +import warnings +import uuid + +import mountainash as ma + +from mountainash_data.core.constants import ( + CONST_CONFLICT_ACTION, + CONST_INDEX_TYPE, +) +``` + +Remove: `from abc import abstractmethod, ABC`, `import ibis`, `import ibis.expr.types.relations as ir`, `from ibis.expr.schema import SchemaLike`, `from ibis.backends.sql import SQLBackend`, `from mountainash_settings import SettingsParameters`, `from mountainash_data.core.constants import CONST_DB_BACKEND`. + +- [ ] **Step 3: Update `__init__.py`** + +Replace `src/mountainash_data/__init__.py` with: + +```python +"""mountainash-data: physical access to backend data services. + +Public API: + Backend — protocol (core.protocol) + IbisBackend — ibis-style relational backends (backends.ibis.backend) + IcebergBackend — iceberg-style table-format catalogs (backends.iceberg.backend) + CatalogInfo, NamespaceInfo, TableInfo, ColumnInfo — inspection model +""" + +from mountainash_data.__version__ import __version__ +from mountainash_data.core.protocol import Backend +from mountainash_data.core.inspection import ( + CatalogInfo, + ColumnInfo, + NamespaceInfo, + TableInfo, +) +from mountainash_data.backends.ibis.backend import IbisBackend + +try: + from mountainash_data.backends.iceberg.backend import IcebergBackend +except ImportError: + IcebergBackend = None # type: ignore[assignment,misc] + +__all__ = [ + "__version__", + "Backend", + "CatalogInfo", + "ColumnInfo", + "NamespaceInfo", + "TableInfo", + "IbisBackend", + "IcebergBackend", +] +``` + +- [ ] **Step 4: Verify import works** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py -v` +Expected: ALL PASS (the backend tests don't import deleted modules) + +- [ ] **Step 5: Commit** + +```bash +git add -A +git commit -m "refactor: delete legacy factories, DatabaseUtils, and connection hierarchies + +Remove ConnectionFactory, OperationsFactory, SettingsFactory, DatabaseUtils, +BaseIbisConnection + 12 subclasses, BaseIbisOperations + concrete subclasses. +Keep BaseDBConnection (Iceberg depends on it). +Update __init__.py to export only Backend, IbisBackend, IcebergBackend, +and inspection model." +``` + +--- + +### Task 6: Delete legacy test files + +**Files:** +- Delete: `tests/test_unit/factories/` +- Delete: `tests/test_unit/test_database_utils.py` +- Delete: `tests/test_unit/databases/test_database_connections.py` +- Delete: `tests/test_unit/databases/connections/` +- Delete: `tests/test_unit/databases/test_ibis_backends.py` +- Delete: `tests/test_unit/databases/operations/` + +- [ ] **Step 1: Delete legacy test files and directories** + +```bash +rm -rf tests/test_unit/factories/ +rm tests/test_unit/test_database_utils.py +rm tests/test_unit/databases/test_database_connections.py +rm -rf tests/test_unit/databases/connections/ +rm tests/test_unit/databases/test_ibis_backends.py +rm -rf tests/test_unit/databases/operations/ +``` + +- [ ] **Step 2: Update test_mountainash_data.py** + +Replace `tests/test_unit/test_mountainash_data.py` with: + +```python +"""Tests for main mountainash_data package.""" + +import pytest +import mountainash_data + + +class TestPackageImports: + """Test package-level imports and structure.""" + + def test_version_import(self): + assert hasattr(mountainash_data, '__version__') + assert isinstance(mountainash_data.__version__, str) + assert len(mountainash_data.__version__) > 0 + + def test_version_format(self): + version_parts = mountainash_data.__version__.split('.') + assert len(version_parts) >= 2 + assert version_parts[0].isdigit() + assert version_parts[1].isdigit() + + def test_core_imports_available(self): + from mountainash_data.core.connection import BaseDBConnection + assert BaseDBConnection is not None + + +class TestPackageStructure: + """Test package structure and organization.""" + + def test_package_has_init(self): + assert hasattr(mountainash_data, '__file__') + + def test_new_submodules_exist(self): + import mountainash_data.core + import mountainash_data.backends + assert hasattr(mountainash_data, 'core') + assert hasattr(mountainash_data, 'backends') + + def test_public_api_ibis_backend(self): + from mountainash_data import IbisBackend + assert IbisBackend is not None + + def test_public_api_backend_protocol(self): + from mountainash_data import Backend + assert Backend is not None + + def test_public_api_inspection_model(self): + from mountainash_data import CatalogInfo, ColumnInfo, NamespaceInfo, TableInfo + assert CatalogInfo is not None + assert ColumnInfo is not None + assert NamespaceInfo is not None + assert TableInfo is not None + + def test_removed_exports_not_available(self): + assert not hasattr(mountainash_data, 'ConnectionFactory') + assert not hasattr(mountainash_data, 'OperationsFactory') + assert not hasattr(mountainash_data, 'SettingsFactory') + assert not hasattr(mountainash_data, 'DatabaseUtils') +``` + +- [ ] **Step 3: Update test_settings_parametrized.py** + +In `tests/test_unit/databases/settings/test_settings_parametrized.py`, find the tests that use `ConnectionFactory` and `DatabaseUtils` (around lines 148-175) and replace them with `IbisBackend` equivalents: + +Replace the `ConnectionFactory` test block (around line 148) with: + +```python + def test_settings_work_with_ibis_backend(self, settings_params): + """Test that settings work with IbisBackend.""" + from mountainash_data.backends.ibis.backend import IbisBackend + backend = IbisBackend(settings_params) + assert backend.dialect is not None +``` + +Replace the `DatabaseUtils` test block (around line 163) with: + +```python + def test_settings_work_with_ibis_backend_connect(self, settings_params): + """Test that settings can create a connected backend via IbisBackend.""" + from mountainash_data.backends.ibis.backend import IbisBackend + backend = IbisBackend(settings_params) + backend.connect() + tables = backend.list_tables() + assert isinstance(tables, list) + backend.close() +``` + +- [ ] **Step 4: Rewrite integration tests** + +Replace `tests/test_integration/test_end_to_end_workflows.py` with: + +```python +"""End-to-end integration tests using IbisBackend.""" + +import pytest +import polars as pl +from mountainash_data.backends.ibis.backend import IbisBackend +from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings +from mountainash_settings import SettingsParameters + + +@pytest.mark.integration +class TestIbisBackendWorkflow: + """Test complete workflows through IbisBackend.""" + + def test_sqlite_dialect_workflow(self): + """Full workflow with dialect= keyword.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("users", {"id": [1, 2], "name": ["a", "b"]}) + assert "users" in backend.list_tables() + tbl = backend.table("users") + assert tbl is not None + + def test_sqlite_url_workflow(self, tmp_path): + """Full workflow from URL.""" + db_file = tmp_path / "test.db" + with IbisBackend(f"sqlite:///{db_file}") as backend: + backend.create_table("t", {"id": [1, 2, 3]}) + assert "t" in backend.list_tables() + assert db_file.exists() + + def test_duckdb_settings_workflow(self): + """Full workflow from SettingsParameters.""" + from mountainash_data.core.settings import NoAuth + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + with IbisBackend(params) as backend: + backend.create_table("t", {"id": [1, 2]}) + assert "t" in backend.list_tables() + + def test_fluent_chaining_workflow(self): + """Fluent API chaining across multiple operations.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + ( + backend + .create_table("users", {"id": [1], "email": ["a@b.com"]}) + .create_index("users", ["email"], unique=True) + .create_table("orders", {"id": [1], "user_id": [1]}) + ) + assert sorted(backend.list_tables()) == ["orders", "users"] + + def test_inspect_workflow(self): + """Inspection methods work through the backend.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + info = backend.inspect_table("t") + assert info.name == "t" + assert len(info.columns) == 2 + + def test_ibis_connection_seam(self): + """ibis_connection() provides the seam to mountainash-expressions.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2]}) + raw = backend.ibis_connection() + tbl = raw.table("t") + assert tbl is not None + + +@pytest.mark.integration +class TestDuckDBOperationsWorkflow: + """Test DuckDB-specific operations (upsert, indexes) through IbisBackend.""" + + def test_upsert_insert_new_rows(self): + """Upsert inserts new rows when no conflicts exist.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"]}) + backend.create_table("users", initial) + backend.create_unique_index("users", ["id"]) + + new_data = pl.DataFrame({"id": [3, 4], "name": ["Charlie", "Diana"]}) + backend.upsert("users", new_data, conflict_columns=["id"]) + + count = backend.run_sql("SELECT COUNT(*) as cnt FROM users").to_polars()["cnt"][0] + assert count == 4 + + def test_upsert_update_existing_rows(self): + """Upsert updates existing rows on conflict.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"], "score": [100, 200]}) + backend.create_table("users", initial) + backend.create_unique_index("users", ["id"]) + + update = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"], "score": [150, 250]}) + backend.upsert("users", update, conflict_columns=["id"]) + + result = backend.run_sql("SELECT score FROM users ORDER BY id").to_polars() + assert list(result["score"]) == [150, 250] + + def test_index_lifecycle(self): + """Create, check, list, drop index.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2], "name": ["a", "b"]}) + + backend.create_index("t", ["name"], index_name="idx_name") + assert backend.index_exists("idx_name") is True + + indexes = backend.list_indexes("t") + assert any(idx.get("name") == "idx_name" for idx in indexes) + + backend.drop_index("idx_name") + assert backend.index_exists("idx_name") is False +``` + +- [ ] **Step 5: Run full test suite** + +Run: `hatch run test:test-quick` +Expected: ALL PASS (some tests will have been removed, count should be lower but zero failures) + +- [ ] **Step 6: Commit** + +```bash +git add -A +git commit -m "test: rewrite all tests to use IbisBackend, delete legacy test files + +Delete factory tests, DatabaseUtils tests, legacy connection tests, +and legacy operations tests. Rewrite integration tests and settings +parametrized tests. Update package structure tests." +``` + +--- + +### Task 7: Full test suite validation and cleanup + +**Files:** +- Verify: All files + +- [ ] **Step 1: Run full test suite** + +Run: `hatch run test:test-quick` +Expected: ALL PASS + +- [ ] **Step 2: Check for any remaining imports of deleted modules** + +```bash +grep -rn "from mountainash_data.core.factories" src/ tests/ --include="*.py" +grep -rn "from mountainash_data.core.utils" src/ tests/ --include="*.py" +grep -rn "from mountainash_data.backends.ibis.connection import" src/ tests/ --include="*.py" +grep -rn "ConnectionFactory\|OperationsFactory\|SettingsFactory\|DatabaseUtils" src/ tests/ --include="*.py" +grep -rn "BaseIbisOperations\|BaseIbisConnection" src/ tests/ --include="*.py" +``` + +Expected: No matches in src/ or tests/ (only in docs/ specs/plans which is fine). + +- [ ] **Step 3: Fix any remaining references found in step 2** + +If any stale imports are found, update them. Common places to check: +- `conftest.py` files +- `__init__.py` files in test directories +- Fixture files in `tests/fixtures/` + +- [ ] **Step 4: Run full test suite one final time** + +Run: `hatch run test:test-quick` +Expected: ALL PASS + +- [ ] **Step 5: Commit any cleanup** + +```bash +git add -A +git commit -m "chore: clean up stale imports and references to deleted modules" +``` + +(Skip this commit if no changes were needed.) + +--- + +## Self-Review + +**Spec coverage check:** +- ✅ IbisBackend lifecycle (connect/close/context manager) — Task 1 +- ✅ Fluent API (return self) — Task 3 +- ✅ Terminal methods (return data) — Task 3 +- ✅ DialectSpec hooks — Task 2 +- ✅ Hook-dispatched operations (upsert, indexes) — Task 3 +- ✅ Thin wrapper operations — Task 3 +- ✅ Accessor methods (ibis_connection, get_connection) — Task 1 +- ✅ Protocol update (Backend, remove Connection) — Task 1 +- ✅ IcebergBackend update — Task 4 +- ✅ Delete factories/utils — Task 5 +- ✅ Delete legacy hierarchies — Task 5 +- ✅ Public API cleanup — Task 5 +- ✅ Test rewrite — Task 6 +- ✅ BaseDBConnection kept — Task 5 (only deletes ibis connection.py, not core/connection.py) +- ✅ Error handling (no silent bool/None, NotImplementedError for unsupported) — Task 3 + +**Placeholder scan:** No TBD/TODO/placeholders found. + +**Type consistency:** `IbisBackend` return type used consistently across all tasks. Hook function signatures match between Task 2 (extraction) and Task 3 (dispatch). diff --git a/docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md b/docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md new file mode 100644 index 0000000..b22f9c8 --- /dev/null +++ b/docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md @@ -0,0 +1,432 @@ +# Settings-Aware Backends + to_relation() Implementation Plan + +> **Status:** ABANDONED -- superseded by `docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md`. to_relation() descoped; constructor design revised. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `IbisBackend` accept `SettingsParameters` as an alternative constructor, add `to_relation()` to the Connection protocol and both Ibis connection paths, and declare `mountainash` as an optional dependency. + +**Architecture:** `IbisBackend.__init__` detects whether it received a `str` (dialect) or `SettingsParameters` and resolves config accordingly. `to_relation()` is added to the `Connection` protocol and implemented on `IbisConnection` (new-style) and `BaseIbisConnection` (factory path), both using an import-guarded call to `mountainash.relations.relation()`. Iceberg gets a `NotImplementedError` stub. + +**Tech Stack:** Python, ibis-framework, mountainash-settings, mountainash (optional) + +**Spec:** `docs/superpowers/specs/2026-04-26-to-relation-design.md` + +--- + +## File Map + +| File | Action | What changes | +|------|--------|--------------| +| `src/mountainash_data/backends/ibis/backend.py` | Modify | Settings-aware `__init__` + `to_relation()` on `IbisConnection` | +| `src/mountainash_data/core/protocol.py` | Modify | Add `to_relation()` to `Connection` protocol | +| `src/mountainash_data/backends/ibis/connection.py` | Modify | Add `to_relation()` to `BaseIbisConnection` | +| `src/mountainash_data/backends/iceberg/connection.py` | Modify | Add `to_relation()` stub to `IcebergConnectionBase` | +| `pyproject.toml` | Modify | Add `relations` optional extra | +| `tests/test_unit/core/test_protocol.py` | Modify | Add `to_relation` to `_FakeConnection` | +| `tests/test_unit/backends/ibis/test_backend_settings.py` | Create | Tests for settings-aware `IbisBackend` | +| `tests/test_unit/backends/ibis/test_to_relation.py` | Create | Tests for `to_relation()` on both Ibis paths | + +--- + +### Task 1: Settings-aware IbisBackend + +**Files:** +- Modify: `src/mountainash_data/backends/ibis/backend.py:106-142` +- Create: `tests/test_unit/backends/ibis/test_backend_settings.py` + +- [ ] **Step 1: Create test directory and write failing tests** + +Create `tests/test_unit/backends/__init__.py` and `tests/test_unit/backends/ibis/__init__.py` if they don't exist, then create the test file: + +```python +"""Tests for settings-aware IbisBackend constructor.""" + +import pytest +from mountainash_data.backends.ibis.backend import IbisBackend, IbisConnection +from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings, NoAuth +from mountainash_settings import SettingsParameters + + +class TestIbisBackendFromSettings: + """Test IbisBackend constructed from SettingsParameters.""" + + def test_sqlite_settings_creates_backend(self): + params = SettingsParameters.create( + settings_class=SQLiteAuthSettings, + kwargs={"DATABASE": ":memory:", "auth": NoAuth()}, + ) + backend = IbisBackend(params) + assert backend.dialect == "sqlite" + + def test_duckdb_settings_creates_backend(self): + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + kwargs={"DATABASE": ":memory:", "auth": NoAuth()}, + ) + backend = IbisBackend(params) + assert backend.dialect == "duckdb" + + def test_settings_backend_connects(self): + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + kwargs={"DATABASE": ":memory:", "auth": NoAuth()}, + ) + backend = IbisBackend(params) + conn = backend.connect() + try: + assert isinstance(conn, IbisConnection) + tables = conn.list_tables() + assert isinstance(tables, list) + finally: + conn.close() + + def test_direct_dialect_still_works(self): + backend = IbisBackend(dialect="duckdb", database=":memory:") + conn = backend.connect() + try: + assert isinstance(conn, IbisConnection) + finally: + conn.close() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend_settings.py -v` +Expected: FAIL — `IbisBackend` does not accept `SettingsParameters` + +- [ ] **Step 3: Implement settings-aware constructor** + +In `src/mountainash_data/backends/ibis/backend.py`, replace the `IbisBackend` class (lines 106-142) with: + +```python +class IbisBackend: + """Ibis backend factory. + + Construction takes either a dialect name with config kwargs, or a + SettingsParameters object that resolves both automatically. + + Usage: + # Direct config + backend = IbisBackend(dialect="sqlite", database=":memory:") + + # Settings-driven + backend = IbisBackend(settings_params) + + conn = backend.connect() + try: + tables = conn.list_tables() + finally: + conn.close() + """ + + name = "ibis" + + def __init__(self, dialect: str | t.Any = "", **config: t.Any): + from mountainash_settings import SettingsParameters + + if isinstance(dialect, SettingsParameters): + settings_params = dialect + settings = settings_params.settings_class.get_settings(settings_params) + descriptor = settings.__descriptor__ + ibis_dialect = descriptor.ibis_dialect + if ibis_dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {ibis_dialect!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = ibis_dialect + self._spec: DialectSpec = DIALECTS[ibis_dialect] + driver_kwargs = settings.to_driver_kwargs() + self._config = { + k: v for k, v in driver_kwargs.items() + if not (isinstance(v, (list, tuple)) and len(v) == 0) + } + else: + if dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {dialect!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = dialect + self._spec = DIALECTS[dialect] + self._config = config + + def connect(self) -> IbisConnection: + """Build and return a live ibis connection.""" + if self._spec.connection_builder is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} has no connection_builder configured" + ) + ibis_conn = self._spec.connection_builder(**self._config) + return IbisConnection(ibis_conn, self._spec) +``` + +Note: The parameter is named `dialect` (not `dialect_or_settings`) to +preserve backward compatibility with `IbisBackend(dialect="sqlite")`. +Type detection via `isinstance` distinguishes `SettingsParameters` from a +string. The `SettingsParameters` import is inside `__init__` to avoid a +top-level dependency for the direct-config path. + +Empty list/tuple values are filtered from settings-derived kwargs (e.g. +`extensions=[]` from DuckDB) because some ibis drivers reject them. This +matches the normalization in `BaseIbisConnection.connect_default()`. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend_settings.py -v` +Expected: PASS — all 4 tests + +- [ ] **Step 5: Run full test suite** + +Run: `hatch run test:test-quick` +Expected: All existing tests still pass + +- [ ] **Step 6: Commit** + +```bash +git add src/mountainash_data/backends/ibis/backend.py tests/test_unit/backends/ +git commit -m "feat(backend): make IbisBackend settings-aware" +``` + +--- + +### Task 2: Add to_relation() to protocol and all implementations + +**Files:** +- Modify: `src/mountainash_data/core/protocol.py:19-52` +- Modify: `src/mountainash_data/backends/ibis/backend.py:21-103` (IbisConnection) +- Modify: `src/mountainash_data/backends/ibis/connection.py:20-99` (BaseIbisConnection) +- Modify: `src/mountainash_data/backends/iceberg/connection.py:50-114` (IcebergConnectionBase) +- Modify: `tests/test_unit/core/test_protocol.py` (_FakeConnection) +- Create: `tests/test_unit/backends/ibis/test_to_relation.py` + +- [ ] **Step 1: Add to_relation() to the Connection protocol** + +In `src/mountainash_data/core/protocol.py`, add after `close()` (after line 52): + +```python + def to_relation( + self, name: str, namespace: str | None = None + ) -> t.Any: + """Return a mountainash Relation for the named table. + + Requires the mountainash package. Raises ImportError if not installed, + or NotImplementedError if the backend does not support it. + """ + ... +``` + +- [ ] **Step 2: Update _FakeConnection in test_protocol.py** + +In `tests/test_unit/core/test_protocol.py`, add to `_FakeConnection` (after `close`): + +```python + def to_relation(self, name: str, namespace: str | None = None) -> t.Any: + return f"relation:{name}" +``` + +- [ ] **Step 3: Run protocol tests to verify they still pass** + +Run: `hatch run test:test-target tests/test_unit/core/test_protocol.py -v` +Expected: PASS + +- [ ] **Step 4: Add to_relation() to IbisConnection** + +In `src/mountainash_data/backends/ibis/backend.py`, add to `IbisConnection` +(after `inspect_catalog`, before `close`): + +```python + def to_relation( + self, name: str, namespace: str | None = None + ) -> t.Any: + """Return a mountainash Relation wrapping the named ibis table.""" + try: + from mountainash.relations import relation + except ImportError: + raise ImportError( + "mountainash package is required for to_relation(). " + "Install it with: pip install mountainash" + ) + ibis_table = self._ibis_conn.table(name, database=namespace) + return relation(ibis_table) +``` + +- [ ] **Step 5: Add to_relation() to BaseIbisConnection** + +In `src/mountainash_data/backends/ibis/connection.py`, add to +`BaseIbisConnection` (after `connect_default`, before `_connect` — after +line 137): + +```python + def to_relation( + self, name: str, namespace: str | None = None + ) -> t.Any: + """Return a mountainash Relation wrapping the named ibis table.""" + try: + from mountainash.relations import relation + except ImportError: + raise ImportError( + "mountainash package is required for to_relation(). " + "Install it with: pip install mountainash" + ) + self.connect() + ibis_table = self.ibis_backend.table(name, database=namespace) + return relation(ibis_table) +``` + +- [ ] **Step 6: Add to_relation() stub to IcebergConnectionBase** + +In `src/mountainash_data/backends/iceberg/connection.py`, add to +`IcebergConnectionBase` (after `is_connected`, before the schema cache +section — after line 138): + +```python + def to_relation( + self, name: str, namespace: str | None = None + ) -> t.Any: + """Not yet supported for Iceberg connections.""" + raise NotImplementedError( + "to_relation() is not yet supported for Iceberg connections. " + "Use table() to get the native pyiceberg Table object." + ) +``` + +- [ ] **Step 7: Write tests for to_relation()** + +Create `tests/test_unit/backends/ibis/test_to_relation.py`: + +```python +"""Tests for to_relation() on Ibis connection paths.""" + +import pytest +import typing as t +from unittest.mock import patch + +from mountainash_data.backends.ibis.backend import IbisBackend, IbisConnection + + +class TestIbisConnectionToRelation: + """Test to_relation() on the new-style IbisConnection path.""" + + @pytest.fixture + def conn(self): + backend = IbisBackend(dialect="duckdb", database=":memory:") + conn = backend.connect() + yield conn + conn.close() + + def test_to_relation_returns_relation(self, conn): + try: + from mountainash.relations import Relation + except ImportError: + pytest.skip("mountainash package not installed") + + conn._ibis_conn.raw_sql("CREATE TABLE test_tbl (id INTEGER, name VARCHAR)") + result = conn.to_relation("test_tbl") + assert isinstance(result, Relation) + + def test_to_relation_import_error(self, conn): + conn._ibis_conn.raw_sql("CREATE TABLE test_tbl2 (id INTEGER)") + with patch.dict("sys.modules", {"mountainash": None, "mountainash.relations": None}): + with pytest.raises(ImportError, match="mountainash package is required"): + conn.to_relation("test_tbl2") + + +class TestBaseIbisConnectionToRelation: + """Test to_relation() on the settings/factory path.""" + + @pytest.fixture + def conn(self): + from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + from mountainash_settings import SettingsParameters + from mountainash_data.core.factories import ConnectionFactory + + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + kwargs={"DATABASE": ":memory:", "auth": NoAuth()}, + ) + connection = ConnectionFactory.get_connection(params) + connection.connect() + yield connection + connection.disconnect() + + def test_to_relation_returns_relation(self, conn): + try: + from mountainash.relations import Relation + except ImportError: + pytest.skip("mountainash package not installed") + + conn.ibis_backend.raw_sql("CREATE TABLE test_tbl3 (id INTEGER, name VARCHAR)") + result = conn.to_relation("test_tbl3") + assert isinstance(result, Relation) + + +class TestIcebergToRelationStub: + """Test that Iceberg raises NotImplementedError.""" + + def test_raises_not_implemented(self): + from mountainash_data.backends.iceberg.connection import IcebergConnectionBase + with pytest.raises(NotImplementedError, match="not yet supported"): + IcebergConnectionBase.to_relation(None, "some_table") +``` + +- [ ] **Step 8: Run tests** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_to_relation.py -v` +Expected: Tests pass (or skip if mountainash not installed) + +Run: `hatch run test:test-quick` +Expected: Full suite passes + +- [ ] **Step 9: Commit** + +```bash +git add src/mountainash_data/core/protocol.py \ + src/mountainash_data/backends/ibis/backend.py \ + src/mountainash_data/backends/ibis/connection.py \ + src/mountainash_data/backends/iceberg/connection.py \ + tests/test_unit/core/test_protocol.py \ + tests/test_unit/backends/ibis/test_to_relation.py +git commit -m "feat(protocol): add to_relation() to Connection protocol and Ibis implementations" +``` + +--- + +### Task 3: Add mountainash as optional dependency + +**Files:** +- Modify: `pyproject.toml:51-68` + +- [ ] **Step 1: Add relations extra to pyproject.toml** + +In `pyproject.toml`, after the `trino` optional dependency line (line 68), add: + +```toml +relations = ["mountainash"] +``` + +- [ ] **Step 2: Run tests** + +Run: `hatch run test:test-quick` +Expected: All tests pass + +- [ ] **Step 3: Commit** + +```bash +git add pyproject.toml +git commit -m "feat(deps): add mountainash as optional relations extra" +``` + +--- + +## Follow-up (not in this plan) + +- **Phase 2:** Migrate `DatabaseUtils` consumers to `IbisBackend(settings_params)` +- **Phase 3:** Deprecate `ConnectionFactory`, `OperationsFactory`, 12 concrete subclasses +- **Dead code candidate:** After the legacy branch removal in PR #78, + `BaseIbisConnection.ibis_connection_mode` and `connection_string_scheme` + abstract properties are unused by `connect_default()` but still declared + on all 12 subclasses. Track separately. + +Tracked in: `mountainash-central/01.principles/mountainash-data/f.backlog/settings-aware-backends.md` diff --git a/docs/superpowers/plans/2026-04-27-settings-aware-ibis-backend.md b/docs/superpowers/plans/2026-04-27-settings-aware-ibis-backend.md new file mode 100644 index 0000000..2bc0a8f --- /dev/null +++ b/docs/superpowers/plans/2026-04-27-settings-aware-ibis-backend.md @@ -0,0 +1,452 @@ +# Settings-Aware IbisBackend Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Extend `IbisBackend` to accept `SettingsParameters` and connection URLs alongside the existing dialect keyword, producing `IbisConnection` from all three input forms. + +**Architecture:** The `IbisBackend.__init__` constructor gains a positional-only first parameter that accepts `SettingsParameters` or a URL string. A `dialect=` keyword preserves the existing path. All forms resolve to `(self.dialect, self._spec, self._config)` so `connect()` stays simple. A module-level `_SCHEME_TO_DIALECT` map (built from the `DIALECTS` registry) handles URL scheme detection. + +**Tech Stack:** ibis-framework, mountainash-settings (`SettingsParameters`), mountainash-data settings (`ConnectionProfile`, `BackendDescriptor`) + +**Spec:** `docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md` + +--- + +## File Map + +| File | Responsibility | Change | +|------|---------------|--------| +| `src/mountainash_data/backends/ibis/backend.py` | `IbisBackend` + `IbisConnection` | New constructor, `_SCHEME_TO_DIALECT` map, empty-list filter in `connect()` | +| `tests/test_unit/backends/ibis/test_backend.py` | Unit tests for `IbisBackend` | New tests for settings, URL, error paths | + +No new files. `IbisConnection`, `DialectSpec` registry, and builders are untouched. + +--- + +### Task 1: Error-case tests and `_SCHEME_TO_DIALECT` map + +Establishes the constructor signature, dispatch validation, and the scheme lookup — all tested before the happy paths. + +**Files:** +- Modify: `src/mountainash_data/backends/ibis/backend.py:106-142` +- Modify: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing tests for constructor validation** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +import pytest +from mountainash_data.backends.ibis.backend import IbisBackend + + +def test_neither_positional_nor_dialect_raises(): + """Constructor with no arguments must raise ValueError.""" + with pytest.raises(ValueError, match="Either.*or.*dialect"): + IbisBackend() + + +def test_both_positional_and_dialect_raises(): + """Cannot supply both a positional arg and dialect= keyword.""" + with pytest.raises(ValueError, match="Cannot specify both"): + IbisBackend("sqlite://", dialect="sqlite") + + +def test_unknown_url_scheme_raises(): + """URL with unrecognised scheme must raise ValueError.""" + with pytest.raises(ValueError, match="Cannot detect ibis dialect"): + IbisBackend("nosuch://localhost/db") +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_neither_positional_nor_dialect_raises tests/test_unit/backends/ibis/test_backend.py::test_both_positional_and_dialect_raises tests/test_unit/backends/ibis/test_backend.py::test_unknown_url_scheme_raises -v` + +Expected: FAIL — the current constructor signature `__init__(self, dialect: str, **config)` doesn't match these call forms. + +- [ ] **Step 3: Implement `_SCHEME_TO_DIALECT` map and new constructor** + +Replace the `IbisBackend` class in `src/mountainash_data/backends/ibis/backend.py` (lines 106–142) with: + +```python +# --------------------------------------------------------------------------- +# Scheme → dialect reverse lookup (built once from the DIALECTS registry) +# --------------------------------------------------------------------------- +def _build_scheme_to_dialect() -> dict[str, str]: + """Build a map from URL scheme (e.g. 'sqlite', 'postgres') to dialect name.""" + result: dict[str, str] = {} + for dialect_name, spec in DIALECTS.items(): + # connection_string_scheme is e.g. "postgres://", "duckdb://md:" + scheme = spec.connection_string_scheme.split("://")[0].lower() + # First dialect wins — e.g. "postgres" maps to "postgres", not "redshift" + if scheme not in result: + result[scheme] = dialect_name + # Common aliases + result.setdefault("postgresql", result.get("postgres", "postgres")) + return result + + +_SCHEME_TO_DIALECT: dict[str, str] = _build_scheme_to_dialect() + + +class IbisBackend: + """Ibis backend — single entry point for all Ibis connections. + + Three input forms, all producing IbisConnection via connect(): + + # Settings object (deployment, env-driven config) + backend = IbisBackend(settings_params) + + # Connection URL (universal connection strings) + backend = IbisBackend("postgresql://user:pass@host:5432/db") + + # Dialect keyword + kwargs (tests, scripts) + backend = IbisBackend(dialect="sqlite", database=":memory:") + """ + + name = "ibis" + + def __init__( + self, + settings_or_connection_string: str | t.Any | None = None, + /, + *, + dialect: str | None = None, + **config: t.Any, + ): + if settings_or_connection_string is not None and dialect is not None: + raise ValueError( + "Cannot specify both a positional settings/URL argument " + "and dialect= keyword" + ) + + if settings_or_connection_string is not None: + self._init_from_positional(settings_or_connection_string, config) + elif dialect is not None: + self._init_from_dialect(dialect, config) + else: + raise ValueError( + "Either a SettingsParameters/URL positional argument " + "or a dialect= keyword is required" + ) + + def _init_from_positional( + self, value: str | t.Any, config: dict[str, t.Any] + ) -> None: + # Lazy import — only pay for it on the settings/URL paths + from mountainash_settings import SettingsParameters + + if isinstance(value, SettingsParameters): + self._init_from_settings(value, config) + elif isinstance(value, str): + if "://" in value: + self._init_from_url(value, config) + else: + # Plain string — treat as dialect name + self._init_from_dialect(value, config) + else: + raise TypeError( + f"Expected SettingsParameters or str, got {type(value).__name__}" + ) + + def _init_from_dialect( + self, dialect_name: str, config: dict[str, t.Any] + ) -> None: + if dialect_name not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {dialect_name!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = dialect_name + self._spec: DialectSpec = DIALECTS[dialect_name] + self._config = config + + def _init_from_dialect( + self, dialect_name: str, config: dict[str, t.Any] + ) -> None: + if dialect_name not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {dialect_name!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = dialect_name + self._spec: DialectSpec = DIALECTS[dialect_name] + self._url: str | None = None + self._config = config + + def _init_from_url( + self, url: str, config: dict[str, t.Any] + ) -> None: + from urllib.parse import urlparse + + scheme = urlparse(url).scheme.lower() + + # Special case: MotherDuck URLs are "duckdb://md:..." + if scheme == "duckdb" and url.startswith("duckdb://md:"): + resolved_dialect = "motherduck" + else: + resolved_dialect = _SCHEME_TO_DIALECT.get(scheme) + + if resolved_dialect is None: + raise ValueError( + f"Cannot detect ibis dialect from URL scheme: {scheme!r}" + ) + + self.dialect = resolved_dialect + self._spec = DIALECTS[resolved_dialect] + self._url = url + self._config = config + + def _init_from_settings( + self, settings_params: t.Any, config: dict[str, t.Any] + ) -> None: + obj_settings = settings_params.settings_class.get_settings( + settings_parameters=settings_params + ) + descriptor = getattr(obj_settings, "__descriptor__", None) + if descriptor is None or getattr(descriptor, "ibis_dialect", None) is None: + raise ValueError( + f"Settings class {type(obj_settings).__name__} has no " + f"ibis_dialect on its descriptor" + ) + resolved_dialect = descriptor.ibis_dialect + if resolved_dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {resolved_dialect!r} from descriptor. " + f"Available: {sorted(DIALECTS)}" + ) + driver_kwargs = obj_settings.to_driver_kwargs() + driver_kwargs.update(config) + + self.dialect = resolved_dialect + self._spec = DIALECTS[resolved_dialect] + self._url = None + self._config = driver_kwargs + + def connect(self) -> IbisConnection: + """Build and return a live ibis connection.""" + if self._spec.connection_builder is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} has no connection_builder configured" + ) + if self._url is not None: + # URL path: delegate directly to ibis.connect() which + # natively handles all URL forms and preserves all URL + # components (host, port, credentials, database, query params). + import ibis + ibis_conn = ibis.connect(self._url, **self._config) + else: + # Settings/dialect path: go through the dialect builder + # with empty-list normalization (e.g. DuckDB extensions=[]). + cleaned_config = { + k: v for k, v in self._config.items() + if not (isinstance(v, (list, tuple)) and len(v) == 0) + } + ibis_conn = self._spec.connection_builder(**cleaned_config) + return IbisConnection(ibis_conn, self._spec) +``` + +Key details: +- The type annotation for `settings_or_connection_string` uses `str | t.Any | None` rather than `str | SettingsParameters | None` to avoid a top-level import of `mountainash_settings`. The `SettingsParameters` isinstance check happens inside `_init_from_positional` with a lazy import. +- `_init_from_url` stores the raw URL on `self._url`. In `connect()`, when `_url` is set, the URL is passed directly to `ibis.connect(url)` — this natively handles all URL forms for all backends and preserves all URL components. The dialect builders are NOT used for the URL path (they don't all handle `connection_string` kwargs). +- `_init_from_settings` resolves the settings object, extracts `ibis_dialect` from the descriptor, and calls `to_driver_kwargs()`. Sets `_url = None`. +- `_init_from_dialect` is the existing path, unchanged except for initialising `_url = None`. +- `connect()` branches on `self._url`: URL path uses `ibis.connect(url)`, settings/dialect path uses the builder with empty-list filtering. The empty-list filter is essential because `DuckDBAuthSettings.to_driver_kwargs()` returns `extensions: []` by default, which `ibis.duckdb.connect()` rejects. + +- [ ] **Step 4: Run the three error tests** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_neither_positional_nor_dialect_raises tests/test_unit/backends/ibis/test_backend.py::test_both_positional_and_dialect_raises tests/test_unit/backends/ibis/test_backend.py::test_unknown_url_scheme_raises -v` + +Expected: PASS — all three error cases now handled by the new constructor. + +- [ ] **Step 5: Run existing tests to verify no regressions** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py -v` + +Expected: All 4 existing tests + 3 new tests PASS. The existing tests use `IbisBackend(dialect="sqlite", ...)` which hits `_init_from_dialect` — the unchanged path. + +- [ ] **Step 6: Commit** + +```bash +git add src/mountainash_data/backends/ibis/backend.py tests/test_unit/backends/ibis/test_backend.py +git commit -m "feat(backend): new IbisBackend constructor with dispatch and error validation + +Add _SCHEME_TO_DIALECT map, three-way dispatch (settings, URL, dialect), +empty-list normalization in connect(), and URL-direct via ibis.connect(). +Error cases tested: no args, both args, unknown scheme." +``` + +--- + +### Task 2: Settings path — SQLite and DuckDB + +Wire and test the `SettingsParameters` input form. + +**Files:** +- Modify: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing tests for settings path** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +from mountainash_data.backends.ibis.backend import IbisBackend, IbisConnection + + +def test_settings_path_sqlite(): + """Construct IbisBackend from SQLite SettingsParameters and connect.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import SQLiteAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=SQLiteAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "sqlite" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + tables = conn.list_tables() + assert isinstance(tables, list) + conn.close() + + +def test_settings_path_duckdb_empty_extensions(): + """DuckDB settings with default EXTENSIONS=[] must not crash ibis.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "duckdb" + conn = backend.connect() # Must not raise — empty-list filter active + assert isinstance(conn, IbisConnection) + conn.close() +``` + +- [ ] **Step 2: Run tests to verify they pass** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_settings_path_sqlite tests/test_unit/backends/ibis/test_backend.py::test_settings_path_duckdb_empty_extensions -v` + +Expected: PASS — the constructor's `_init_from_settings` path was implemented in Task 1. + +If the DuckDB test fails with an error about `extensions=[]`, verify that `connect()` properly filters empty lists before calling the builder. + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_unit/backends/ibis/test_backend.py +git commit -m "test(backend): settings path — SQLite and DuckDB SettingsParameters + +Verifies constructor resolves SettingsParameters via descriptor.ibis_dialect +and to_driver_kwargs(). DuckDB test confirms empty-list normalization +filters extensions=[] before reaching ibis." +``` + +--- + +### Task 3: URL path — SQLite and DuckDB + +Wire and test the connection URL input form. + +**Files:** +- Modify: `tests/test_unit/backends/ibis/test_backend.py` + +- [ ] **Step 1: Write failing tests for URL path** + +Add to `tests/test_unit/backends/ibis/test_backend.py`: + +```python +def test_url_path_sqlite(): + """Construct IbisBackend from sqlite:// URL and connect.""" + backend = IbisBackend("sqlite://") + assert backend.dialect == "sqlite" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + + +def test_url_path_duckdb(): + """Construct IbisBackend from duckdb:// URL and connect.""" + backend = IbisBackend("duckdb://") + assert backend.dialect == "duckdb" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + + +def test_url_path_preserves_database(tmp_path): + """URL database component must reach the driver, not be discarded.""" + db_file = tmp_path / "test.db" + backend = IbisBackend(f"sqlite:///{db_file}") + assert backend.dialect == "sqlite" + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + assert db_file.exists() +``` + +- [ ] **Step 2: Run tests to verify they pass** + +Run: `hatch run test:test-target tests/test_unit/backends/ibis/test_backend.py::test_url_path_sqlite tests/test_unit/backends/ibis/test_backend.py::test_url_path_duckdb tests/test_unit/backends/ibis/test_backend.py::test_url_path_preserves_database -v` + +Expected: PASS — the URL path calls `ibis.connect(url)` directly, which handles all URL forms natively. + +The `test_url_path_preserves_database` test is the critical regression test identified by Codex review: it proves the URL's database path actually reaches the driver (the file must exist on disk after connect). + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_unit/backends/ibis/test_backend.py +git commit -m "test(backend): URL path — sqlite://, duckdb://, file path preservation + +Verifies URL dispatch via _SCHEME_TO_DIALECT, ibis.connect(url) delegation. +Critical regression: sqlite:///path creates the file on disk, proving +URL components are not discarded." +``` + +--- + +### Task 4: Full test suite and mark old docs abandoned + +Run the entire test suite, then commit the abandoned old spec/plan. + +**Files:** +- Modify: `docs/superpowers/specs/2026-04-26-to-relation-design.md` (already marked) +- Modify: `docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md` (already marked) + +- [ ] **Step 1: Run full test suite** + +Run: `hatch run test:test-quick` + +Expected: All tests PASS (441+ existing + 8 new). No regressions. + +If any existing tests fail, investigate — the existing `test_ibis_backend_satisfies_protocol`, `test_unknown_dialect_raises`, `test_all_registered_dialects_construct`, and `test_in_memory_sqlite_connect_and_inspect` should all still pass because they use `dialect=` keyword. + +- [ ] **Step 2: Verify old spec and plan are marked abandoned** + +Check that these files already have ABANDONED status (done earlier in the brainstorming session): +- `docs/superpowers/specs/2026-04-26-to-relation-design.md` — line 4 should say `ABANDONED` +- `docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md` — line 3 should say `ABANDONED` + +- [ ] **Step 3: Commit abandoned docs** + +```bash +git add docs/superpowers/specs/2026-04-26-to-relation-design.md docs/superpowers/plans/2026-04-27-settings-aware-backends-to-relation.md docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md docs/superpowers/plans/2026-04-27-settings-aware-ibis-backend.md +git commit -m "chore(specs): abandon old to-relation spec/plan, add new settings-aware spec/plan + +Old spec bundled to_relation() which has been descoped per revised +principle. New spec focuses solely on settings-aware IbisBackend +constructor (Phase 1 of connection consolidation)." +``` + +- [ ] **Step 4: Run full test suite one final time** + +Run: `hatch run test:test-quick` + +Expected: All tests PASS. Clean working tree (no uncommitted changes to source). diff --git a/docs/superpowers/specs/2026-04-26-to-relation-design.md b/docs/superpowers/specs/2026-04-26-to-relation-design.md new file mode 100644 index 0000000..c11e8c8 --- /dev/null +++ b/docs/superpowers/specs/2026-04-26-to-relation-design.md @@ -0,0 +1,210 @@ +# Settings-Aware Backends + to_relation() + +> **Date:** 2026-04-26 (updated 2026-04-27) +> **Status:** ABANDONED -- superseded by `2026-04-27-settings-aware-ibis-backend-design.md`. to_relation() descoped per revised principle. +> **Backlog refs:** +> - `mountainash-central/01.principles/mountainash-data/f.backlog/to-relation-gap.md` +> - `mountainash-central/01.principles/mountainash-data/f.backlog/settings-aware-backends.md` (Phase 1) + +## Context + +mountainash-data has two parallel connection paths that produce the same +result with different indirection: + +1. **New-style:** `IbisBackend(dialect="sqlite", **config).connect()` → + `IbisConnection` (protocol-compliant) +2. **Settings-driven:** `ConnectionFactory.get_connection(settings_params)` → + `BaseIbisConnection` subclass (old hierarchy) + +The gap: `IbisBackend` only accepts direct config. Consumers with a +`SettingsParameters` must use the old factory path or manually resolve +settings. This keeps the factory alive despite adding no value post-refactor. + +Separately, `to_relation()` is defined on the `Connection` protocol concept +but not yet implemented. The `mountainash` package (formerly +mountainash-expressions) provides `relation()` which accepts ibis tables +and wraps them in a `Relation` AST node. + +This spec covers: +- Making `IbisBackend` settings-aware (Phase 1 of settings-aware-backends backlog) +- Wiring `to_relation()` on both connection paths + +## Design + +### 1. Settings-aware IbisBackend + +`IbisBackend.__init__` accepts either a dialect string + kwargs (existing) +or a `SettingsParameters` object (new): + +```python +class IbisBackend: + name = "ibis" + + def __init__(self, dialect_or_settings: str | SettingsParameters, **config: Any): + if isinstance(dialect_or_settings, SettingsParameters): + settings_params = dialect_or_settings + settings = settings_params.settings_class.get_settings(settings_params) + descriptor = settings.__descriptor__ + ibis_dialect = descriptor.ibis_dialect + if ibis_dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {ibis_dialect!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = ibis_dialect + self._spec = DIALECTS[ibis_dialect] + self._config = settings.to_driver_kwargs() + else: + dialect = dialect_or_settings + if dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {dialect!r}. " + f"Available: {sorted(DIALECTS)}" + ) + self.dialect = dialect + self._spec = DIALECTS[dialect] + self._config = config +``` + +Usage: + +```python +from mountainash_data import IbisBackend + +# Direct +backend = IbisBackend(dialect="sqlite", database=":memory:") + +# Settings-driven +backend = IbisBackend(settings_params) + +# Both produce the same IbisConnection +conn = backend.connect() +``` + +### 2. Protocol — to_relation() + +Add `to_relation()` to the `Connection` protocol in `core/protocol.py`: + +```python +def to_relation(self, name: str, namespace: str | None = None) -> "Relation": + """Return a mountainash Relation for the named table. + + Requires the mountainash package. Raises ImportError if not installed. + """ + ... +``` + +Signature matches `inspect_table(name, namespace)` for consistency. Return +type uses a string forward reference to avoid importing mountainash at module +level. + +### 3. Ibis implementation — new-style path (IbisConnection) + +In `backends/ibis/backend.py`, add to `IbisConnection`: + +```python +def to_relation(self, name: str, namespace: str | None = None) -> "Relation": + try: + from mountainash.relations import relation + except ImportError: + raise ImportError( + "mountainash package is required for to_relation(). " + "Install it with: pip install mountainash" + ) + ibis_table = self._ibis_conn.table(name, database=namespace) + return relation(ibis_table) +``` + +### 4. Ibis implementation — settings/factory path (BaseIbisConnection) + +In `backends/ibis/connection.py`, add to `BaseIbisConnection`: + +```python +def to_relation(self, name: str, namespace: str | None = None) -> "Relation": + try: + from mountainash.relations import relation + except ImportError: + raise ImportError( + "mountainash package is required for to_relation(). " + "Install it with: pip install mountainash" + ) + self.connect() + ibis_table = self.ibis_backend.table(name, database=namespace) + return relation(ibis_table) +``` + +This covers connections obtained via `ConnectionFactory` and `DatabaseUtils`. +The factory path gets `to_relation()` now; it will be deprecated in Phase 3 +of the settings-aware-backends backlog. + +### 5. Iceberg stub + +In `backends/iceberg/connection.py`, add to `IcebergConnectionBase`: + +```python +def to_relation(self, name: str, namespace: str | None = None) -> "Relation": + raise NotImplementedError( + "to_relation() is not yet supported for Iceberg connections. " + "Use table() to get the native pyiceberg Table object." + ) +``` + +### 6. Dependency + +Add `mountainash` as an optional extra in `pyproject.toml`: + +```toml +[project.optional-dependencies] +relations = ["mountainash"] +``` + +The core package does NOT depend on mountainash — the import is guarded at +call time in `to_relation()`. + +### 7. Extensibility pattern + +Future backends (DataFusion, etc.) follow the same two-step pattern: + +1. Make the backend class settings-aware (`__init__` accepts `SettingsParameters`) +2. Add `to_relation()` — get native table handle, pass to `relation()` + +The only prerequisite is that `identify_backend()` in mountainash recognises +the native table type and a corresponding relation system backend exists. + +## Testing + +### Settings-aware IbisBackend +- Construct `IbisBackend(settings_params)` with SQLite/DuckDB settings, + verify `.connect()` returns working `IbisConnection` +- Verify `IbisBackend(settings_params).dialect` matches expected dialect +- Verify invalid settings raise `KeyError` +- Verify existing direct path still works unchanged + +### to_relation() +- `IbisConnection.to_relation()` (new-style) with in-memory DuckDB: create + table, call `to_relation()`, verify returns `Relation` instance +- `BaseIbisConnection.to_relation()` (factory path) via + `ConnectionFactory.get_connection()`: same verification +- Round-trip test: `to_relation()` → `.collect()` returns expected data +- `ImportError` path: mock the import to verify clear error message +- `IcebergConnectionBase.to_relation()` raises `NotImplementedError` + +## Commit strategy + +Three commits, single branch, single PR targeting `develop`: + +1. `feat(backend): make IbisBackend settings-aware` +2. `feat(protocol): add to_relation() to Connection protocol and Ibis implementations` +3. `feat(deps): add mountainash as optional relations extra` + +## Follow-up (not in this PR) + +- **Phase 2:** Migrate `DatabaseUtils` consumers to use `IbisBackend(settings_params)` directly +- **Phase 3:** Deprecate `ConnectionFactory`, `OperationsFactory`, `DatabaseUtils.create_connection()`/`create_operations()`, and the 12 concrete `BaseIbisConnection` subclasses + +Tracked in: `mountainash-central/01.principles/mountainash-data/f.backlog/settings-aware-backends.md` + +## Backlog updates after merge + +- `to-relation-gap.md` → **RESOLVED** (Ibis wired; Iceberg stub) +- `settings-aware-backends.md` → Phase 1 **RESOLVED** diff --git a/docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md b/docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md new file mode 100644 index 0000000..a2f14a8 --- /dev/null +++ b/docs/superpowers/specs/2026-04-27-backend-as-single-handle-design.md @@ -0,0 +1,397 @@ +# Backend as Single Handle — Design Spec + +> **Status:** APPROVED +> **Date:** 2026-04-27 +> **Branch:** `feature/settings-aware-ibis-backend` (continues Phase 1 work) +> **Supersedes:** Connection consolidation Phases 2+3 (collapsed into one) + +## Goal + +Make `IbisBackend` the single public handle for all ibis interaction — +construction, connection lifecycle, inspection, and operations. Delete +`ConnectionFactory`, `OperationsFactory`, `DatabaseUtils`, the 12 concrete +`BaseIbisConnection` subclasses, and the separate `BaseIbisOperations` class +hierarchy. + +## Architecture + +`IbisBackend` composes three internal concerns: + +1. **Config resolution** (existing, unchanged) — three-way constructor + dispatch: `SettingsParameters`, URL, or `dialect=` keyword. +2. **Connection lifecycle** — `connect()` returns `self`, `close()` returns + `self`, context manager support. +3. **Operations dispatch** — instance methods on `IbisBackend` that delegate + to the internal `IbisConnection` (inspection), the raw ibis connection + (thin wrappers), or `DialectSpec` callable hooks (per-dialect operations). + +`IbisConnection` stays as an **internal** class. The consumer never imports +or interacts with it directly. `IbisBackend` exposes two accessor methods: + +- `ibis_connection()` → raw ibis backend object (for the seam with + mountainash-expressions: `backend.ibis_connection().table("users")`) +- `get_connection()` → our `IbisConnection` wrapper (for internal use) + +Both raise `RuntimeError` if not connected. + +### What Gets Deleted + +| Target | Location | Reason | +|--------|----------|--------| +| `ConnectionFactory` | `core/factories/connection_factory.py` | Replaced by `IbisBackend` constructor | +| `OperationsFactory` | `core/factories/operations_factory.py` | Replaced by operations on `IbisBackend` | +| `SettingsFactory` | `core/factories/settings_factory.py` | URL detection now in `_SCHEME_TO_DIALECT`; settings creation via `IbisBackend(settings_params)` | +| `DatabaseUtils` | `core/utils.py` | Entire class — `IbisBackend` is the API | +| ~~`BaseDBConnection`~~ | ~~`core/connection.py`~~ | **KEPT** — `IcebergConnectionBase` subclasses it; migrate Iceberg separately | +| `BaseIbisConnection` + 12 subclasses | `backends/ibis/connection.py` | Replaced by `IbisConnection` (internal) + `DialectSpec` registry | +| `BaseIbisOperations` + concrete subclasses | `backends/ibis/operations.py` | Replaced by operations on `IbisBackend` + `DialectSpec` hooks | +| `_DuckDBFamilyOperationsMixin` | `backends/ibis/operations.py` | Implementations become `DialectSpec` hooks | +| `_BaseIbisMixin` | `backends/ibis/operations.py` | Already a deprecated shim | + +### What Stays + +| Component | Location | Role | +|-----------|----------|------| +| `IbisBackend` | `backends/ibis/backend.py` | Single public handle (expanded) | +| `IbisConnection` | `backends/ibis/backend.py` | Internal wrapper — inspection delegation | +| `DialectSpec` registry | `backends/ibis/dialects/_registry.py` | Expanded with operation hooks | +| Module-level operation functions | `backends/ibis/operations.py` | Wired as `DialectSpec` hooks | +| `IcebergBackend` | `backends/iceberg/backend.py` | Unchanged (separate backend) | +| `BaseDBConnection` | `core/connection.py` | Kept — Iceberg depends on it; migrate separately | +| `Backend` protocol | `core/protocol.py` | Updated — `connect()` returns `Self` | +| Inspection model | `core/inspection.py` | Unchanged | +| Settings classes | `core/settings/` | Unchanged | + +## Protocol Changes + +### `Backend` protocol (updated) + +```python +@t.runtime_checkable +class Backend(t.Protocol): + name: str + + def connect(self) -> Self: ... + def close(self) -> Self: ... + def __enter__(self) -> Self: ... + def __exit__(self, *args) -> None: ... + + # Inspection (terminal — return data) + def list_tables(self, namespace: str | None = None) -> list[str]: ... + def list_namespaces(self) -> list[str]: ... + def inspect_table(self, name: str, namespace: str | None = None) -> TableInfo: ... + def inspect_namespace(self, name: str) -> NamespaceInfo: ... + def inspect_catalog(self) -> CatalogInfo: ... +``` + +### `Connection` protocol — REMOVED + +No longer part of the public API. `IbisConnection` is internal. + +## Connection Lifecycle + +```python +# Explicit connect/close +backend = IbisBackend(dialect="sqlite", database=":memory:") +backend.connect() +tables = backend.list_tables() +backend.close() + +# Context manager (recommended) +with IbisBackend(dialect="sqlite", database=":memory:") as backend: + tables = backend.list_tables() + +# Fluent chaining +with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("users", df).create_index("users", ["email"], unique=True) + rows = backend.list_tables() +``` + +`__enter__` calls `connect()` and returns `self`. +`__exit__` calls `close()`. + +Calling any method before `connect()` or after `close()` raises `RuntimeError`. + +## Fluent API + +Methods are categorised into two groups: + +### Fluent (return `self` — chainable mutations) + +| Method | Signature | +|--------|-----------| +| `connect()` | `() -> Self` | +| `close()` | `() -> Self` | +| `create_table()` | `(name, obj, *, schema?, database?, temp?, overwrite?) -> Self` | +| `drop_table()` | `(name, *, database?, force?) -> Self` | +| `create_view()` | `(name, obj, *, database?, overwrite?) -> Self` | +| `drop_view()` | `(name, *, database?, force?) -> Self` | +| `insert()` | `(name, obj, *, database?, overwrite?) -> Self` | +| `upsert()` | `(name, obj, *, conflict_columns, update_columns?, conflict_action?, ...) -> Self` | +| `truncate()` | `(name, *, database?, schema?) -> Self` | +| `rename_table()` | `(old_name, new_name) -> Self` | +| `create_index()` | `(table, columns, *, index_name?, unique?, ...) -> Self` | +| `create_unique_index()` | `(table, columns, *, index_name?, ...) -> Self` | +| `drop_index()` | `(index_name, *, table_name?, database?, if_exists?) -> Self` | + +### Terminal (return data — end of chain) + +| Method | Returns | +|--------|---------| +| `list_tables(namespace?)` | `list[str]` | +| `list_namespaces()` | `list[str]` | +| `table_exists(name, database?)` | `bool` | +| `inspect_table(name, namespace?)` | `TableInfo` | +| `inspect_namespace(name)` | `NamespaceInfo` | +| `inspect_catalog()` | `CatalogInfo` | +| `table(name, *, database?)` | `ir.Table` | +| `run_sql(query, *, schema?, dialect?)` | `ir.Table | None` | +| `run_expr(expr, *, params?, limit?)` | `Any` | +| `to_sql(expr, *, params?, limit?, pretty?)` | `str | None` | +| `index_exists(index_name, *, table_name?, database?)` | `bool` | +| `list_indexes(table_name, *, database?)` | `list[dict]` | +| `ibis_connection()` | raw ibis backend object | +| `get_connection()` | `IbisConnection` (internal wrapper) | + +## DialectSpec Expansion + +`DialectSpec` gains operation hook fields: + +```python +@dataclass(frozen=True) +class DialectSpec: + # Existing + ibis_backend_name: str + connection_mode: str + connection_string_scheme: str + connection_builder: Callable | None = None + get_index_exists_sql: Callable | None = None # existing + get_list_indexes_sql: Callable | None = None # existing + + # New operation hooks + upsert_hook: Callable | None = None + create_index_hook: Callable | None = None + drop_index_hook: Callable | None = None + rename_table_hook: Callable | None = None + + extras: Mapping[str, Any] = field(default_factory=dict) +``` + +### Hook wiring + +The existing `_DuckDBFamilyOperationsMixin` methods become standalone +functions and are wired as hooks: + +| Hook | DuckDB/MotherDuck | SQLite | Others | +|------|-------------------|--------|--------| +| `upsert_hook` | `duckdb_family_upsert` | `duckdb_family_upsert` | `None` | +| `create_index_hook` | `duckdb_family_create_index` | `duckdb_family_create_index` | `None` | +| `drop_index_hook` | `duckdb_family_drop_index` | `duckdb_family_drop_index` | `None` | +| `rename_table_hook` | `None` (not implemented) | `None` (not implemented) | `None` | +| `get_index_exists_sql` | `duckdb_get_index_exists_sql` | `sqlite_get_index_exists_sql` | `None` | +| `get_list_indexes_sql` | `duckdb_get_list_indexes_sql` | `sqlite_get_list_indexes_sql` | `None` | + +`IbisBackend.upsert()` checks `self._spec.upsert_hook`; if `None`, raises +`NotImplementedError(f"Dialect {self.dialect!r} does not support upsert")`. + +### Hook function signatures + +Hooks receive the raw ibis connection as their first argument (not the +`IbisBackend` instance), plus the method's arguments: + +```python +# upsert_hook signature +def duckdb_family_upsert( + ibis_conn: Any, + table_name: str, + df: Any, + *, + conflict_columns: list[str] | str, + update_columns: list[str] | str | None = None, + conflict_action: str = "UPDATE", + update_condition: str | None = None, + database: str | None = None, + schema: str | None = None, +) -> None: ... + +# create_index_hook signature +def duckdb_family_create_index( + ibis_conn: Any, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + unique: bool = False, + index_type: str | None = None, + where_condition: str | None = None, + database: str | None = None, + if_not_exists: bool = True, +) -> None: ... +``` + +Note: hooks return `None`, not `bool`. The `IbisBackend` wrapper converts +the call to fluent `return self`. Errors raise exceptions (no silent +`False` returns). + +### Thin wrapper methods + +Methods that are simple ibis delegations don't need hooks — they work the +same across all dialects: + +- `create_table` → `ibis_conn.create_table(...)` +- `drop_table` → `ibis_conn.drop_table(...)` +- `create_view` → `ibis_conn.create_view(...)` +- `drop_view` → `ibis_conn.drop_view(...)` +- `insert` → `ibis_conn.insert(...)` +- `truncate` → `ibis_conn.truncate_table(...)` +- `table` → `ibis_conn.table(...)` +- `run_sql` → `ibis_conn.sql(...)` +- `run_expr` → `ibis_conn.execute(...)` +- `to_sql` → `ibis_conn.compile(...)` +- `list_tables` → delegates to `IbisConnection.list_tables()` +- `list_namespaces` → delegates to `IbisConnection.list_namespaces()` +- `inspect_*` → delegates to `IbisConnection.inspect_*()` + +## Public API Changes (`__init__.py`) + +### Removed exports + +- `Connection` (protocol removed) +- `ConnectionFactory` +- `OperationsFactory` +- `SettingsFactory` +- `DatabaseUtils` + +### Retained exports + +- `Backend` (protocol, updated) +- `IbisBackend` (expanded) +- `IcebergBackend` +- `CatalogInfo`, `ColumnInfo`, `NamespaceInfo`, `TableInfo` + +## IcebergBackend Impact + +`IcebergBackend` must also satisfy the updated `Backend` protocol +(`connect()` returns `Self`, context manager). This is a minor change — +same pattern as `IbisBackend`. Operations that don't apply (upsert, indexes) +are simply not on the protocol. + +## Test Strategy + +### New tests for `IbisBackend` operations + +Test with SQLite and DuckDB (in-memory, no external services): + +- **Lifecycle**: `connect()` → use → `close()`, context manager, double-close idempotent, use-before-connect raises +- **Fluent API**: chain `create_table().create_index()`, verify returns `self` +- **Thin wrappers**: `create_table`, `drop_table`, `insert`, `list_tables`, `table`, `run_sql` +- **Hook-based operations**: `upsert` (DuckDB), `create_index`/`drop_index`/`index_exists`/`list_indexes` (SQLite + DuckDB) +- **Unsupported operations**: `upsert` on Trino dialect raises `NotImplementedError` +- **Accessor methods**: `ibis_connection()` returns raw ibis, `get_connection()` returns `IbisConnection` + +### Existing tests — rewrite + +Factory test files (`test_connection_factory.py`, `test_operations_factory.py`) +are deleted. `test_database_utils.py` is deleted. Operations tests +(`test_base_ibis_operations.py`, `test_upsert_and_indexes.py`) are rewritten +to use `IbisBackend` directly. `test_backend.py` is expanded. + +### Existing tests — verify unchanged + +- Inspection tests remain valid +- Settings tests remain valid +- Iceberg tests need minor update for protocol change + +## Files Modified or Deleted + +### Modified + +| File | Change | +|------|--------| +| `backends/ibis/backend.py` | Add lifecycle, operations methods, accessor methods | +| `backends/ibis/dialects/_registry.py` | Add operation hook fields to `DialectSpec`, wire hooks | +| `backends/ibis/operations.py` | Extract mixin methods into standalone hook functions; delete class hierarchy | +| `backends/iceberg/backend.py` | Update to return `Self` from `connect()`, add context manager | +| `core/protocol.py` | Update `Backend` protocol, remove `Connection` protocol | +| `__init__.py` | Remove factory/utils exports, remove `Connection` | +| `tests/test_unit/backends/ibis/test_backend.py` | Expand with operations + lifecycle tests | + +### Deleted + +| File | Reason | +|------|--------| +| `core/factories/` (entire directory) | All factories replaced by `IbisBackend` | +| `core/utils.py` | `DatabaseUtils` replaced by `IbisBackend` | +| ~~`core/connection.py`~~ | **KEPT** — Iceberg depends on `BaseDBConnection`; migrate separately | +| `backends/ibis/connection.py` | `BaseIbisConnection` + 12 subclasses replaced | +| `tests/test_unit/factories/` | All factory tests | +| `tests/test_unit/test_database_utils.py` | `DatabaseUtils` tests | +| `tests/test_unit/databases/test_database_connections.py` | Legacy connection tests | +| `tests/test_unit/databases/connections/` | Legacy connection lifecycle tests | + +## Error Handling + +- Methods that previously returned `bool` or `None` now raise on failure + (no silent swallowing). The `print(f"Error: ...")` pattern throughout + `BaseIbisOperations` is replaced with proper exception propagation. +- Unsupported operations (hook is `None`) raise `NotImplementedError` + with dialect name in the message. +- Use-before-connect and use-after-close raise `RuntimeError`. + +## Consumer Migration + +Before: +```python +from mountainash_data import DatabaseUtils, ConnectionFactory +conn = DatabaseUtils.create_connection(settings_params) +backend = conn.connect() +ops = DatabaseUtils.create_operations(settings_params) +ops.create_table(backend, "users", df) +ops.upsert(backend, "users", new_df, conflict_columns=["id"]) +tables = ops.list_tables(backend) +``` + +After: +```python +from mountainash_data import IbisBackend +with IbisBackend(settings_params) as backend: + backend.create_table("users", df).upsert("users", new_df, conflict_columns=["id"]) + tables = backend.list_tables() + tbl = backend.ibis_connection().table("users") +``` + +## Redshift URL Ambiguity + +Redshift is registered with `connection_string_scheme="postgres://"` because +it uses the postgres wire protocol. This means `_SCHEME_TO_DIALECT` maps +`postgres://` → `"postgres"` (first writer wins), and Redshift is not +reachable by URL alone. + +**Resolution:** Redshift connections must use either: +- `IbisBackend(dialect="redshift", ...)` — explicit dialect keyword +- `IbisBackend(settings_params)` — settings path with Redshift settings class + +This is an inherent limitation: `postgres://` URLs are ambiguous between +postgres and Redshift. The old `SettingsFactory` had the same problem — it +mapped `postgres://` to postgres, not Redshift. No regression. + +If a dedicated `redshift://` scheme is needed in future, add a `url_schemes` +list field on `DialectSpec` separate from `connection_string_scheme`. + +## Compatibility + +This package is internal to the mountainash-io organisation — there are zero +external consumers. All imports of the removed names (`ConnectionFactory`, +`OperationsFactory`, `SettingsFactory`, `DatabaseUtils`, `Connection`) are +in this repo's own test files, which are rewritten in the same change. + +No deprecation window or major-version bump is needed. The removed exports +are cleaned up atomically: deletion + test migration in the same branch. + +## Out of Scope + +| Item | Reason | +|------|--------| +| Iceberg migration off `BaseDBConnection` | Separate spec — `core/connection.py` kept for now | +| `redshift://` URL scheme | No current need — use settings or `dialect=` | +| Operations for non-DuckDB-family dialects | Add hooks when needed (Snowflake, BigQuery, etc.) | diff --git a/docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md b/docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md new file mode 100644 index 0000000..e914601 --- /dev/null +++ b/docs/superpowers/specs/2026-04-27-settings-aware-ibis-backend-design.md @@ -0,0 +1,262 @@ +# Settings-Aware IbisBackend + +> **Date:** 2026-04-27 +> **Status:** Approved +> **Principle:** `mountainash-central/01.principles/mountainash-data/b.connection-management/connection-consolidation.md` (Phase 1) +> **Backlog ref:** `mountainash-central/01.principles/mountainash-data/f.backlog/settings-aware-backends.md` +> **Supersedes:** `docs/superpowers/specs/2026-04-26-to-relation-design.md` (ABANDONED) + +## Goal + +Extend `IbisBackend` to accept `SettingsParameters` and connection URLs +alongside the existing dialect keyword. All three input forms resolve to +the same internal state and produce `IbisConnection` via `connect()`. +This is Phase 1 of connection consolidation. + +## Constructor Signature + +```python +def __init__( + self, + settings_or_connection_string: str | SettingsParameters | None = None, + /, + *, + dialect: str | None = None, + **config: t.Any, +): +``` + +### Input Forms + +```python +from mountainash_data import IbisBackend +from mountainash_settings import SettingsParameters +from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + +# Form 1: Settings object (deployment, env-driven config) +settings_params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), +) +backend = IbisBackend(settings_params) + +# Form 2: Connection URL (universal connection strings) +backend = IbisBackend("postgresql://user:pass@host:5432/db") + +# Form 3: Dialect keyword + kwargs (tests, scripts) +backend = IbisBackend(dialect="sqlite", database=":memory:") + +# All produce the same IbisConnection +conn = backend.connect() +``` + +## Dispatch Logic + +The constructor resolves all input forms to `(self.dialect, self._spec, +self._config)` so that `connect()` requires no changes beyond empty-list +normalization. + +1. **Both positional and `dialect=` provided** -> `ValueError`. +2. **Positional is `SettingsParameters`** -> settings path: + - `settings_class.get_settings(settings_parameters)` to resolve. + - `descriptor.ibis_dialect` to get dialect name. + - `to_driver_kwargs()` for config dict. + - `**config` kwargs merged on top (caller overrides). +3. **Positional is `str` containing `://`** -> URL path: + - Detect dialect from URL scheme using a reverse lookup on + `DialectSpec.connection_string_scheme`. + - Store the raw URL as `connection_string` in config — the dialect + builder passes it straight to `ibis.connect(url)`. + - `**config` kwargs merged on top (caller overrides). +4. **Positional is a plain `str`** -> treat as dialect name (same as + `dialect=` keyword). +5. **`dialect` keyword provided** -> existing dialect path (unchanged). +6. **Neither provided** -> `ValueError`. + +### Settings Resolution Detail + +```python +# Inside __init__, settings path: +from mountainash_settings import SettingsParameters + +obj_settings = settings_or_connection_string.settings_class.get_settings( + settings_parameters=settings_or_connection_string +) +descriptor = getattr(obj_settings, "__descriptor__", None) +if descriptor is None or descriptor.ibis_dialect is None: + raise ValueError( + f"Settings class {type(obj_settings).__name__} has no ibis_dialect on its descriptor" + ) +resolved_dialect = descriptor.ibis_dialect +driver_kwargs = obj_settings.to_driver_kwargs() +driver_kwargs.update(config) # caller overrides +``` + +### URL Resolution Detail + +The URL path bypasses settings entirely. The raw URL is passed to the +dialect builder as `connection_string`, which forwards it to +`ibis.connect(url)`. This preserves all URL components (host, port, +credentials, database, query params) without lossy round-tripping +through settings fields. + +Dialect detection uses a reverse lookup built from the `DIALECTS` +registry — each `DialectSpec` already carries `connection_string_scheme`. + +```python +# Inside __init__, URL path: +from urllib.parse import urlparse + +# Build reverse scheme -> dialect map from registry +# e.g. {"sqlite": "sqlite", "duckdb": "duckdb", "postgres": "postgres", ...} +scheme = urlparse(settings_or_connection_string).scheme.lower() + +# Special cases: "postgresql" -> "postgres", "md" -> "motherduck" +# MotherDuck also detected by "duckdb://md:" prefix +resolved_dialect = _SCHEME_TO_DIALECT.get(scheme) +if resolved_dialect is None: + raise ValueError( + f"Cannot detect ibis dialect from URL scheme: {scheme!r}" + ) + +# Store raw URL as connection_string — builders pass it to ibis.connect() +driver_kwargs = {"connection_string": settings_or_connection_string} +driver_kwargs.update(config) # caller overrides +``` + +The `_SCHEME_TO_DIALECT` map is built once at module level from the +`DIALECTS` registry, with additional aliases for common scheme variants +(e.g. `postgresql` -> `postgres`). + +## Empty-List Normalization + +Happens in `connect()`, not `__init__`. Some ibis drivers reject empty +sequences (e.g. `ibis.duckdb.connect(extensions=[])` fails). + +```python +def connect(self) -> IbisConnection: + cleaned_config = { + k: v for k, v in self._config.items() + if not (isinstance(v, (list, tuple)) and len(v) == 0) + } + ibis_conn = self._spec.connection_builder(**cleaned_config) + return IbisConnection(ibis_conn, self._spec) +``` + +## Files Changed + +| File | Change | +|------|--------| +| `src/mountainash_data/backends/ibis/backend.py` | New constructor signature, dispatch logic, empty-list normalization in `connect()`, `_SCHEME_TO_DIALECT` map | +| `tests/test_unit/backends/ibis/test_backend.py` | New tests for settings, URL, and error paths | + +## Files NOT Changed + +- `IbisConnection` -- untouched +- `DialectSpec` registry / builders -- untouched (URL path uses existing `connection_string` kwarg) +- `ConnectionFactory`, `DatabaseUtils`, `BaseIbisConnection` -- no deprecation yet (Phase 2) +- `core/protocol.py` -- untouched +- No new files + +## Testing + +All tests use SQLite and DuckDB (in-memory, no external deps). + +### 1. Dialect Path (existing, unchanged) + +```python +def test_dialect_path(): + backend = IbisBackend(dialect="sqlite", database=":memory:") + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() +``` + +### 2. Settings Path + +```python +def test_settings_path_sqlite(): + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import SQLiteAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=SQLiteAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + conn = backend.connect() + assert isinstance(conn, IbisConnection) + tables = conn.list_tables() + assert isinstance(tables, list) + conn.close() + +def test_settings_path_duckdb_empty_extensions(): + """DuckDB settings with default EXTENSIONS=[] must not reach ibis.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + conn = backend.connect() # Must not raise + assert isinstance(conn, IbisConnection) + conn.close() +``` + +### 3. URL Path + +```python +def test_url_path_sqlite(): + backend = IbisBackend("sqlite://") + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + +def test_url_path_duckdb(): + backend = IbisBackend("duckdb://") + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + +def test_url_path_preserves_database(tmp_path): + """URL database component must reach the driver, not be discarded.""" + db_file = tmp_path / "test.db" + backend = IbisBackend(f"sqlite:///{db_file}") + conn = backend.connect() + assert isinstance(conn, IbisConnection) + conn.close() + assert db_file.exists() + +def test_url_path_unknown_scheme_raises(): + with pytest.raises(ValueError, match="Cannot detect ibis dialect"): + IbisBackend("nosuch://localhost/db") +``` + +### 4. Error Cases + +```python +def test_both_positional_and_dialect_raises(): + with pytest.raises(ValueError): + IbisBackend("sqlite://", dialect="sqlite") + +def test_neither_provided_raises(): + with pytest.raises(ValueError): + IbisBackend() + +def test_unknown_dialect_raises(): + with pytest.raises(KeyError): + IbisBackend(dialect="nosuch") +``` + +## Commit Strategy + +Single branch (`feature/settings-aware-ibis-backend`) targeting `develop`. +Two commits: + +1. `feat(backend): make IbisBackend settings-aware` -- constructor + tests +2. `chore(specs): abandon old to-relation spec and plan` -- mark old docs diff --git a/src/mountainash_data/__init__.py b/src/mountainash_data/__init__.py index 2d91947..a690c51 100644 --- a/src/mountainash_data/__init__.py +++ b/src/mountainash_data/__init__.py @@ -1,34 +1,22 @@ """mountainash-data: physical access to backend data services. Public API: - Backend, Connection — protocols (core.protocol) + Backend — protocol (core.protocol) IbisBackend — ibis-style relational backends (backends.ibis.backend) IcebergBackend — iceberg-style table-format catalogs (backends.iceberg.backend) CatalogInfo, NamespaceInfo, TableInfo, ColumnInfo — inspection model - DatabaseUtils — high-level facade - ConnectionFactory, OperationsFactory, SettingsFactory — factories - *Settings classes — see mountainash_data.core.settings """ from mountainash_data.__version__ import __version__ -from mountainash_data.core.protocol import Backend, Connection +from mountainash_data.core.protocol import Backend from mountainash_data.core.inspection import ( CatalogInfo, ColumnInfo, NamespaceInfo, TableInfo, ) -from mountainash_data.core.utils import DatabaseUtils -from mountainash_data.core.factories import ( - ConnectionFactory, - OperationsFactory, - SettingsFactory, -) from mountainash_data.backends.ibis.backend import IbisBackend -# IcebergBackend requires the optional pyiceberg dependency. -# It is imported lazily so that consumers without pyiceberg installed -# still get the rest of the package. try: from mountainash_data.backends.iceberg.backend import IcebergBackend except ImportError: @@ -37,15 +25,10 @@ __all__ = [ "__version__", "Backend", - "Connection", "CatalogInfo", "ColumnInfo", "NamespaceInfo", "TableInfo", - "DatabaseUtils", - "ConnectionFactory", - "OperationsFactory", - "SettingsFactory", "IbisBackend", "IcebergBackend", ] diff --git a/src/mountainash_data/backends/ibis/backend.py b/src/mountainash_data/backends/ibis/backend.py index 8c6ddd9..79267b1 100644 --- a/src/mountainash_data/backends/ibis/backend.py +++ b/src/mountainash_data/backends/ibis/backend.py @@ -103,39 +103,477 @@ def __exit__(self, *args): self.close() +# --------------------------------------------------------------------------- +# Scheme → dialect reverse lookup (built once from the DIALECTS registry) +# --------------------------------------------------------------------------- +def _build_scheme_to_dialect() -> dict[str, str]: + """Build a map from URL scheme (e.g. 'sqlite', 'postgres') to dialect name.""" + result: dict[str, str] = {} + for dialect_name, spec in DIALECTS.items(): + # connection_string_scheme is e.g. "postgres://", "duckdb://md:" + scheme = spec.connection_string_scheme.split("://")[0].lower() + # First dialect wins — e.g. "postgres" maps to "postgres", not "redshift" + if scheme not in result: + result[scheme] = dialect_name + # Common aliases + result.setdefault("postgresql", result.get("postgres", "postgres")) + return result + + +_SCHEME_TO_DIALECT: dict[str, str] = _build_scheme_to_dialect() + + class IbisBackend: - """Ibis backend factory. + """Ibis backend — single entry point for all Ibis connections. - Construction takes a dialect name (e.g. 'postgres') and config. - connect() returns a live connection that satisfies - core.protocol.Connection. + Three input forms, all producing IbisConnection via connect(): - Usage: + # Settings object (deployment, env-driven config) + backend = IbisBackend(settings_params) + + # Connection URL (universal connection strings) + backend = IbisBackend("postgresql://user:pass@host:5432/db") + + # Dialect keyword + kwargs (tests, scripts) backend = IbisBackend(dialect="sqlite", database=":memory:") - conn = backend.connect() - try: - tables = conn.list_tables() - finally: - conn.close() """ name = "ibis" - def __init__(self, dialect: str, **config: t.Any): - if dialect not in DIALECTS: + def __init__( + self, + settings_or_connection_string: str | t.Any | None = None, + /, + *, + dialect: str | None = None, + **config: t.Any, + ): + if settings_or_connection_string is not None and dialect is not None: + raise ValueError( + "Cannot specify both a positional settings/URL argument " + "and dialect= keyword" + ) + + if settings_or_connection_string is not None: + self._init_from_positional(settings_or_connection_string, config) + elif dialect is not None: + self._init_from_dialect(dialect, config) + else: + raise ValueError( + "Either a SettingsParameters/URL positional argument " + "or a dialect= keyword is required" + ) + + def _init_from_positional( + self, value: str | t.Any, config: dict[str, t.Any] + ) -> None: + # Lazy import — only pay for it on the settings/URL paths + from mountainash_settings import SettingsParameters + + if isinstance(value, SettingsParameters): + self._init_from_settings(value, config) + elif isinstance(value, str): + if "://" in value: + self._init_from_url(value, config) + else: + # Plain string — treat as dialect name + self._init_from_dialect(value, config) + else: + raise TypeError( + f"Expected SettingsParameters or str, got {type(value).__name__}" + ) + + def _init_from_dialect( + self, dialect_name: str, config: dict[str, t.Any] + ) -> None: + if dialect_name not in DIALECTS: raise KeyError( - f"Unknown ibis dialect {dialect!r}. " + f"Unknown ibis dialect {dialect_name!r}. " f"Available: {sorted(DIALECTS)}" ) - self.dialect = dialect - self._spec: DialectSpec = DIALECTS[dialect] + self.dialect = dialect_name + self._spec: DialectSpec = DIALECTS[dialect_name] + self._url: str | None = None self._config = config + self._conn: IbisConnection | None = None + + def _init_from_url( + self, url: str, config: dict[str, t.Any] + ) -> None: + from urllib.parse import urlparse + + scheme = urlparse(url).scheme.lower() + + # Special case: MotherDuck URLs are "duckdb://md:..." + if scheme == "duckdb" and url.startswith("duckdb://md:"): + resolved_dialect = "motherduck" + else: + resolved_dialect = _SCHEME_TO_DIALECT.get(scheme) - def connect(self) -> IbisConnection: - """Build and return a live ibis connection.""" + if resolved_dialect is None: + raise ValueError( + f"Cannot detect ibis dialect from URL scheme: {scheme!r}" + ) + + self.dialect = resolved_dialect + self._spec = DIALECTS[resolved_dialect] + self._url = url + self._config = config + self._conn: IbisConnection | None = None + + def _init_from_settings( + self, settings_params: t.Any, config: dict[str, t.Any] + ) -> None: + obj_settings = settings_params.settings_class.get_settings( + settings_parameters=settings_params + ) + descriptor = getattr(obj_settings, "__descriptor__", None) + if descriptor is None or getattr(descriptor, "ibis_dialect", None) is None: + raise ValueError( + f"Settings class {type(obj_settings).__name__} has no " + f"ibis_dialect on its descriptor" + ) + resolved_dialect = descriptor.ibis_dialect + if resolved_dialect not in DIALECTS: + raise KeyError( + f"Unknown ibis dialect {resolved_dialect!r} from descriptor. " + f"Available: {sorted(DIALECTS)}" + ) + driver_kwargs = obj_settings.to_driver_kwargs() + driver_kwargs.update(config) + + self.dialect = resolved_dialect + self._spec = DIALECTS[resolved_dialect] + self._url = None + self._config = driver_kwargs + self._conn: IbisConnection | None = None + + def _require_connected(self) -> IbisConnection: + if self._conn is None: + raise RuntimeError( + "IbisBackend is not connected. Call connect() first." + ) + return self._conn + + def connect(self) -> IbisBackend: + """Build a live ibis connection. Returns self for fluent chaining.""" + if self._conn is not None: + return self if self._spec.connection_builder is None: raise NotImplementedError( f"Dialect {self.dialect!r} has no connection_builder configured" ) - ibis_conn = self._spec.connection_builder(**self._config) - return IbisConnection(ibis_conn, self._spec) + if self._url is not None: + import ibis + ibis_conn = ibis.connect(self._url, **self._config) + else: + cleaned_config = { + k: v for k, v in self._config.items() + if not (isinstance(v, (list, tuple)) and len(v) == 0) + } + ibis_conn = self._spec.connection_builder(**cleaned_config) + self._conn = IbisConnection(ibis_conn, self._spec) + return self + + def close(self) -> IbisBackend: + """Release the connection. Idempotent. Returns self.""" + if self._conn is not None: + self._conn.close() + self._conn = None + return self + + def __enter__(self) -> IbisBackend: + self.connect() + return self + + def __exit__(self, *args: t.Any) -> None: + self.close() + + def ibis_connection(self) -> t.Any: + """Return the raw ibis backend object.""" + return self._require_connected()._ibis_conn + + def get_connection(self) -> IbisConnection: + """Return the internal IbisConnection wrapper.""" + return self._require_connected() + + # --- Inspection (terminal — delegates to IbisConnection) --- + + def list_tables(self, namespace: str | None = None) -> list[str]: + return self._require_connected().list_tables(namespace=namespace) + + def list_namespaces(self) -> list[str]: + return self._require_connected().list_namespaces() + + def inspect_table( + self, name: str, namespace: str | None = None + ) -> TableInfo: + return self._require_connected().inspect_table(name, namespace=namespace) + + def inspect_namespace(self, name: str) -> NamespaceInfo: + return self._require_connected().inspect_namespace(name) + + def inspect_catalog(self) -> CatalogInfo: + return self._require_connected().inspect_catalog() + + # --- Thin wrapper operations (fluent — return self) --- + + def create_table( + self, + name: str, + obj: t.Any, + *, + schema: t.Any | None = None, + database: str | None = None, + temp: bool = False, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.create_table( + name, obj=obj, schema=schema, database=database, + temp=temp, overwrite=overwrite, + ) + return self + + def drop_table( + self, + name: str, + *, + database: str | None = None, + force: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.drop_table(name, database=database, force=force) + return self + + def create_view( + self, + name: str, + obj: t.Any, + *, + database: str | None = None, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.create_view(name, obj=obj, database=database, overwrite=overwrite) + return self + + def drop_view( + self, + name: str, + *, + database: str | None = None, + force: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.drop_view(name, database=database, force=force) + return self + + def insert( + self, + name: str, + obj: t.Any, + *, + database: str | None = None, + overwrite: bool = False, + ) -> IbisBackend: + conn = self._require_connected() + conn._ibis_conn.insert(name, obj=obj, database=database, overwrite=overwrite) + return self + + def truncate( + self, + name: str, + *, + database: str | None = None, + schema: str | None = None, + ) -> IbisBackend: + conn = self._require_connected() + # ibis SQLBackend.truncate_table() accepts only table_name + database; + # schema is not a standard kwarg at the SQLBackend level. + kwargs: dict[str, t.Any] = {} + if database is not None: + kwargs["database"] = database + conn._ibis_conn.truncate_table(name, **kwargs) + return self + + def rename_table(self, old_name: str, new_name: str) -> IbisBackend: + if self._spec.rename_table_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support rename_table" + ) + conn = self._require_connected() + self._spec.rename_table_hook(conn._ibis_conn, old_name, new_name) + return self + + # --- Terminal operations (return data) --- + + def table(self, name: str, *, database: str | None = None) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.table(name, database=database) + + def table_exists( + self, name: str, database: str | None = None + ) -> bool: + tables = self.list_tables() + return name in tables + + def run_sql( + self, + query: str, + *, + schema: t.Any | None = None, + dialect: str | None = None, + ) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.sql(query, schema=schema, dialect=dialect) + + def run_expr( + self, + expr: t.Any, + *, + params: dict | None = None, + limit: str | None = "default", + **kwargs: t.Any, + ) -> t.Any: + conn = self._require_connected() + return conn._ibis_conn.execute(expr, params=params, limit=limit, **kwargs) + + def to_sql( + self, + expr: t.Any, + *, + params: t.Any = None, + limit: str | None = None, + pretty: bool = False, + **kwargs: t.Any, + ) -> str | None: + conn = self._require_connected() + return conn._ibis_conn.compile(expr, params=params, limit=limit, pretty=pretty, **kwargs) + + # --- Hook-dispatched operations (fluent — return self) --- + + def upsert( + self, + name: str, + obj: t.Any, + *, + conflict_columns: list[str] | str, + update_columns: list[str] | str | None = None, + conflict_action: str = "UPDATE", + update_condition: str | None = None, + database: str | None = None, + schema: str | None = None, + ) -> IbisBackend: + if self._spec.upsert_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support upsert" + ) + conn = self._require_connected() + self._spec.upsert_hook( + conn._ibis_conn, name, obj, + conflict_columns=conflict_columns, + update_columns=update_columns, + conflict_action=conflict_action, + update_condition=update_condition, + database=database, + schema=schema, + ) + return self + + def create_index( + self, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + unique: bool = False, + index_type: str | None = None, + where_condition: str | None = None, + database: str | None = None, + if_not_exists: bool = True, + ) -> IbisBackend: + if self._spec.create_index_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support create_index" + ) + conn = self._require_connected() + self._spec.create_index_hook( + conn._ibis_conn, table_name, columns, + index_name=index_name, unique=unique, index_type=index_type, + where_condition=where_condition, database=database, + if_not_exists=if_not_exists, + ) + return self + + def create_unique_index( + self, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + where_condition: str | None = None, + database: str | None = None, + ) -> IbisBackend: + return self.create_index( + table_name, columns, + index_name=index_name, unique=True, + where_condition=where_condition, database=database, + ) + + def drop_index( + self, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + if_exists: bool = True, + ) -> IbisBackend: + if self._spec.drop_index_hook is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support drop_index" + ) + conn = self._require_connected() + self._spec.drop_index_hook( + conn._ibis_conn, index_name, + table_name=table_name, database=database, if_exists=if_exists, + ) + return self + + def index_exists( + self, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + ) -> bool: + if self._spec.get_index_exists_sql is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support index_exists" + ) + conn = self._require_connected() + check_sql = self._spec.get_index_exists_sql(index_name, table_name, database) + result = conn._ibis_conn.sql(check_sql) + if result is None: + return False + import mountainash as ma + count = ma.relation(result).to_dict()["count"][0] + return count > 0 + + def list_indexes( + self, + table_name: str, + *, + database: str | None = None, + ) -> list[dict]: + if self._spec.get_list_indexes_sql is None: + raise NotImplementedError( + f"Dialect {self.dialect!r} does not support list_indexes" + ) + conn = self._require_connected() + list_sql = self._spec.get_list_indexes_sql(table_name, database) + result = conn._ibis_conn.sql(list_sql) + if result is None: + return [] + import mountainash as ma + return ma.relation(result).to_dicts() diff --git a/src/mountainash_data/backends/ibis/connection.py b/src/mountainash_data/backends/ibis/connection.py deleted file mode 100644 index 8a37904..0000000 --- a/src/mountainash_data/backends/ibis/connection.py +++ /dev/null @@ -1,720 +0,0 @@ -import typing as t # import t.Any, t.Dict, t.Optional - -import ibis -from ibis.backends.sql import SQLBackend -from abc import abstractmethod -from pydantic_settings import BaseSettings as _BaseSettings - -from mountainash_settings import SettingsParameters - -from mountainash_data.core.connection import BaseDBConnection -from mountainash_data.core.constants import ( - IBIS_DB_CONNECTION_MODE, - CONST_DB_ABSTRACTION_LAYER, - CONST_DB_PROVIDER_TYPE, - CONST_DB_BACKEND as _CONST_DB_BACKEND, -) -# from mountainash_dataframes.utils.dataframe_utils import DataFrameUtils - - -class BaseIbisConnection(BaseDBConnection): - - def __init__(self, - db_auth_settings_parameters: SettingsParameters, - ): - - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters, - ) - - - @property - def db_abstraction_layer(self) -> CONST_DB_ABSTRACTION_LAYER: - return CONST_DB_ABSTRACTION_LAYER.IBIS - - @property - @abstractmethod - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - """Database provider identifier.""" - ... - - - @property - @abstractmethod - def ibis_backend(self) -> t.Optional[SQLBackend|t.Any]: - """Concrete Ibis backend connection object.""" - pass - - @property - @abstractmethod - def ibis_connection_mode(self) -> str: - """Connect via a connection string, kwargs or both.""" - pass - - @property - @abstractmethod - def connection_string_scheme(self) -> str: - """Template string for database connection.""" - pass - - # @property - # @abstractmethod - # def supports_upsert(self) -> str: - # """Whether upserts are supported""" - # pass - - - - - ########################### - # Core Functions - - - def connect(self, - *, - connection_string: t.Optional[str] = None, - connection_kwargs: t.Optional[t.Dict[str, t.Any]] = None, - **kwargs) -> SQLBackend: - - """Connect with explicitly provided connection parameters""" - - - if self.ibis_backend is None: - - if connection_string is not None: - if connection_kwargs is not None: - self._connect(connection_string=connection_string, connection_kwargs=connection_kwargs, **kwargs) - else: - self._connect(connection_string=connection_string, **kwargs) - - else: - if connection_kwargs is not None: - self._connect(connection_string = self.connection_string_scheme, connection_kwargs=connection_kwargs, **kwargs) - else: - self.connect_default(**kwargs) - - if self.ibis_backend is None: - raise Exception(f"Unable to establish connection to {self.db_backend_name}") - - - return self.ibis_backend - - - def connect_default(self, **kwargs) -> SQLBackend: - """Connect using default configuration""" - - if self.ibis_backend is None: - - settings_class = self.db_auth_settings_parameters.settings_class - if settings_class is not None: - obj_settings = settings_class.get_settings( - settings_parameters=self.db_auth_settings_parameters - ) - driver_kwargs = obj_settings.to_driver_kwargs() - # Filter out empty lists/sequences that some drivers reject. - # (e.g. ibis.duckdb.connect() does not accept extensions=[]) - driver_kwargs = {k: v for k, v in driver_kwargs.items() - if not (isinstance(v, (list, tuple)) and len(v) == 0)} - driver_kwargs.update(kwargs) - # Use the ibis dialect string to call ibis..connect(**driver_kwargs) - descriptor = getattr(obj_settings, "__descriptor__", None) - ibis_dialect = descriptor.ibis_dialect if descriptor else None - if ibis_dialect: - dialect_backend = getattr(ibis, ibis_dialect, None) - if dialect_backend is not None: - self._ibis_backend = dialect_backend.connect(**driver_kwargs) - if self.ibis_backend is None: - raise Exception(f"Unable to establish default connection to {self.db_backend_name}") - return self.ibis_backend - # Fallback: build KWARGS-mode connection - self._connect( - connection_string=self.connection_string_scheme, - connection_kwargs=driver_kwargs if driver_kwargs else None, - ) - if self.ibis_backend is None: - raise Exception(f"Unable to establish default connection to {self.db_backend_name}") - return self.ibis_backend - - return self.ibis_backend - - - - def _connect(self, - connection_string: t.Optional[str] = None, - connection_kwargs: t.Optional[t.Dict[str, t.Any]] = None, - **kwargs - ) -> SQLBackend: - """ - Default Implementation to connect to the database using the provided connection string. - By default this relies on the connection_string_scheme to determine the backend dynamically. - Over-ride in subclasses if a different implementation is necessary, such as using the custom backend directly. - """ - - if connection_kwargs is None: - connection_kwargs = {} - - #combine connection_kwargs and kwargs - connection_kwargs = {**connection_kwargs, **kwargs} - - if connection_string is None: - raise ValueError(f"{self.db_backend_name}: Connection string is required to establish connection") - - self._ibis_backend : t.Any = ibis.connect(connection_string, **connection_kwargs) - - if self.ibis_backend is None: - raise Exception(f"Unable to establish connection to {self.db_backend_name}") - - - return self.ibis_backend - - - - def close(self): - """Close the connection to the database.""" - - self.disconnect() - - def disconnect(self): - """Close the connection to the database.""" - - if self.ibis_backend is not None: - self.ibis_backend.disconnect() - self._ibis_backend = None - - def is_connected(self) -> bool: - """ Is the connection open?""" - if self.ibis_backend is None: - return False - else: - return True - - -# =========================================================================== -# Concrete per-dialect connection classes -# These preserve the per-backend concrete subclasses needed by the legacy -# ConnectionFactory and existing tests. Each implements the abstract properties -# that BaseIbisConnection requires from BaseDBConnection. -# =========================================================================== - -class SQLite_IbisConnection(BaseIbisConnection): - """SQLite ibis connection — concrete subclass for factory compatibility.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.SQLITE - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.SQLITE - - @property - def connection_string_scheme(self) -> str: - return "sqlite://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import SQLiteAuthSettings - return SQLiteAuthSettings - - -class DuckDB_IbisConnection(BaseIbisConnection): - """DuckDB ibis connection — concrete subclass for factory compatibility.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None, - read_only: t.Optional[bool] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - self._read_only = read_only - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.DUCKDB - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.DUCKDB - - @property - def connection_string_scheme(self) -> str: - return "duckdb://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import DuckDBAuthSettings - return DuckDBAuthSettings - - def _connect(self, connection_string: t.Optional[str] = None, - connection_kwargs: t.Optional[t.Dict[str, t.Any]] = None, - **kwargs) -> t.Any: - from mountainash_data.core.settings import DuckDBAuthSettings - - if connection_kwargs is None: - connection_kwargs = {} - if kwargs is None: - kwargs = {} - - settings: DuckDBAuthSettings = DuckDBAuthSettings.get_settings(self.db_auth_settings_parameters) - - if settings.DATABASE is None: - kwargs["read_only"] = False - else: - if connection_kwargs.get("read_only") is not None and self._read_only is not None: - kwargs["read_only"] = connection_kwargs["read_only"] or self._read_only - elif self._read_only is not None: - kwargs["read_only"] = self._read_only - elif connection_kwargs.get("read_only") is None: - kwargs["read_only"] = False - - connection_kwargs = {**connection_kwargs, **kwargs} - return super()._connect(connection_string, connection_kwargs) - - def disconnect(self): - """Close the DuckDB connection, ensuring the underlying connection is properly closed.""" - if self.ibis_backend is not None: - try: - if hasattr(self.ibis_backend, 'con'): - try: - if hasattr(self.ibis_backend.con, '_cursors'): - for cursor in list(self.ibis_backend.con._cursors): - try: - cursor.close() - except Exception: - pass - self.ibis_backend.con.close() - except Exception as e: - print(f"Warning: Error closing DuckDB connection: {str(e)}") - except Exception as e: - print(f"Warning: Error during DuckDB disconnect: {str(e)}") - finally: - try: - super().disconnect() - except Exception as e: - print(f"Warning: Error during Ibis backend disconnect: {str(e)}") - self._ibis_backend = None - - -class MotherDuck_IbisConnection(BaseIbisConnection): - """MotherDuck ibis connection — concrete subclass for factory compatibility.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - self.supports_upsert = True - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.MOTHERDUCK - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.MOTHERDUCK - - @property - def connection_string_scheme(self) -> str: - return "duckdb://md:" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import MotherDuckAuthSettings - return MotherDuckAuthSettings - - -class Postgres_IbisConnection(BaseIbisConnection): - """PostgreSQL ibis connection — concrete subclass for factory compatibility.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.POSTGRESQL - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.POSTGRES - - @property - def connection_string_scheme(self) -> str: - return "postgres://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import PostgreSQLAuthSettings - return PostgreSQLAuthSettings - - -class MySQL_IbisConnection(BaseIbisConnection): - """MySQL ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.MYSQL - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.MYSQL - - @property - def connection_string_scheme(self) -> str: - return "mysql://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import MySQLAuthSettings - return MySQLAuthSettings - - -class MSSQL_IbisConnection(BaseIbisConnection): - """MSSQL ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.MSSQL - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.MSSQL - - @property - def connection_string_scheme(self) -> str: - return "mssql://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import MSSQLAuthSettings - return MSSQLAuthSettings - - -class Oracle_IbisConnection(BaseIbisConnection): - """Oracle ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.ORACLE - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.ORACLE - - @property - def connection_string_scheme(self) -> str: - return "oracle://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - return None # Oracle settings not yet implemented - - -class Snowflake_IbisConnection(BaseIbisConnection): - """Snowflake ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.HYBRID - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.SNOWFLAKE - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.SNOWFLAKE - - @property - def connection_string_scheme(self) -> str: - return "snowflake://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import SnowflakeAuthSettings - return SnowflakeAuthSettings - - -class BigQuery_IbisConnection(BaseIbisConnection): - """BigQuery ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.KWARGS - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.BIGQUERY - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.BIGQUERY - - @property - def connection_string_scheme(self) -> str: - return "bigquery://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import BigQueryAuthSettings - return BigQueryAuthSettings - - def _connect(self, connection_string: t.Optional[str] = None, - connection_kwargs: t.Optional[t.Dict[str, t.Any]] = None, - **kwargs) -> t.Any: - import ibis.backends.bigquery as ir_backend - - credentials_info = connection_kwargs.get('credentials_info', None) if connection_kwargs else None - dataset_id = connection_kwargs.get('dataset_id', "") if connection_kwargs else "" - project_id = connection_kwargs.get('project_id', None) if connection_kwargs else None - - if credentials_info: - from google.oauth2 import service_account - credentials = service_account.Credentials.from_service_account_info(credentials_info) - self._ibis_backend = ir_backend.connect(dataset_id=dataset_id, credentials=credentials) - else: - self._ibis_backend = ir_backend.connect(project_id=project_id, dataset_id=dataset_id) - - return self.ibis_backend - - -class Redshift_IbisConnection(BaseIbisConnection): - """Redshift ibis connection (uses postgres protocol).""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.REDSHIFT - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.REDSHIFT - - @property - def connection_string_scheme(self) -> str: - return "postgres://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import RedshiftAuthSettings - return RedshiftAuthSettings - - -class Trino_IbisConnection(BaseIbisConnection): - """Trino ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.HYBRID - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.TRINO - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.TRINO - - @property - def connection_string_scheme(self) -> str: - return "trino://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import TrinoAuthSettings - return TrinoAuthSettings - - -class PySpark_IbisConnection(BaseIbisConnection): - """PySpark ibis connection.""" - - def __init__(self, db_auth_settings_parameters: SettingsParameters, - connection_mode: t.Optional[str] = None): - self._ibis_backend: t.Optional[t.Any] = None - self._ibis_connection_mode: str = ( - connection_mode if connection_mode is not None - else IBIS_DB_CONNECTION_MODE.CONNECTION_STRING - ) - super().__init__(db_auth_settings_parameters=db_auth_settings_parameters) - - @property - def provider_type(self) -> CONST_DB_PROVIDER_TYPE: - return CONST_DB_PROVIDER_TYPE.PYSPARK - - @property - def ibis_backend(self) -> t.Optional[t.Any]: - return self._ibis_backend - - @property - def ibis_connection_mode(self) -> str: - return self._ibis_connection_mode - - @property - def db_backend_name(self) -> str: - return _CONST_DB_BACKEND.PYSPARK - - @property - def connection_string_scheme(self) -> str: - return "pyspark://" - - @property - def settings_class(self) -> t.Type[_BaseSettings]: - from mountainash_data.core.settings import PySparkAuthSettings - return PySparkAuthSettings diff --git a/src/mountainash_data/backends/ibis/dialects/_registry.py b/src/mountainash_data/backends/ibis/dialects/_registry.py index 0a997c6..085d43a 100644 --- a/src/mountainash_data/backends/ibis/dialects/_registry.py +++ b/src/mountainash_data/backends/ibis/dialects/_registry.py @@ -25,6 +25,10 @@ GetIndexExistsSql = t.Callable[[str, str, t.Optional[str]], str] # (index_name, table_name, database) -> SQL GetListIndexesSql = t.Callable[[str, t.Optional[str]], str] # (table_name, database) -> SQL ConnectionBuilder = t.Callable[..., t.Any] # (**config) -> ibis backend connection +UpsertHook = t.Callable[..., None] +CreateIndexHook = t.Callable[..., None] +DropIndexHook = t.Callable[..., None] +RenameTableHook = t.Callable[..., None] @dataclass(frozen=True) @@ -37,6 +41,10 @@ class DialectSpec: connection_builder: t.Optional[ConnectionBuilder] = None get_index_exists_sql: t.Optional[GetIndexExistsSql] = None get_list_indexes_sql: t.Optional[GetListIndexesSql] = None + upsert_hook: t.Optional[UpsertHook] = None + create_index_hook: t.Optional[CreateIndexHook] = None + drop_index_hook: t.Optional[DropIndexHook] = None + rename_table_hook: t.Optional[RenameTableHook] = None extras: t.Mapping[str, t.Any] = field(default_factory=dict) @@ -391,6 +399,9 @@ def _build_pyspark_connection(**config: t.Any) -> t.Any: sqlite_get_list_indexes_sql, motherduck_get_index_exists_sql, motherduck_get_list_indexes_sql, + duckdb_family_upsert, + duckdb_family_create_index, + duckdb_family_drop_index, ) @@ -402,6 +413,9 @@ def _build_pyspark_connection(**config: t.Any) -> t.Any: connection_builder=_build_sqlite_connection, get_index_exists_sql=sqlite_get_index_exists_sql, get_list_indexes_sql=sqlite_get_list_indexes_sql, + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, ), "duckdb": DialectSpec( ibis_backend_name="duckdb", @@ -410,6 +424,9 @@ def _build_pyspark_connection(**config: t.Any) -> t.Any: connection_builder=_build_duckdb_connection, get_index_exists_sql=duckdb_get_index_exists_sql, get_list_indexes_sql=duckdb_get_list_indexes_sql, + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, ), "motherduck": DialectSpec( ibis_backend_name="duckdb", @@ -418,6 +435,9 @@ def _build_pyspark_connection(**config: t.Any) -> t.Any: connection_builder=_build_motherduck_connection, get_index_exists_sql=motherduck_get_index_exists_sql, get_list_indexes_sql=motherduck_get_list_indexes_sql, + upsert_hook=duckdb_family_upsert, + create_index_hook=duckdb_family_create_index, + drop_index_hook=duckdb_family_drop_index, ), "postgres": DialectSpec( ibis_backend_name="postgres", diff --git a/src/mountainash_data/backends/ibis/operations.py b/src/mountainash_data/backends/ibis/operations.py index baf120f..d8d5119 100644 --- a/src/mountainash_data/backends/ibis/operations.py +++ b/src/mountainash_data/backends/ibis/operations.py @@ -1,32 +1,20 @@ -"""Merged ibis operations module. +"""Ibis operations module — module-level hook functions only. -Consolidates: -- base_ibis_operations.py (676 LOC) — actual operations implementation -- _base_ibis_mixin.py (98 LOC) — helper methods as module-level functions -- _duckdb_family_mixin.py (314 LOC) — DuckDB-family shared SQL as class + module functions -- Per-dialect index SQL functions (duckdb, sqlite, motherduck) - -This module is the single source of operations truth for all ibis backends. -The original files have been replaced with shims that re-export from here. +Contains: +- Module-level helper functions: _generate_index_name, _format_qualified_table, _normalize_columns +- Per-dialect SQL functions: duckdb, sqlite, motherduck index SQL generators +- Standalone hook functions: duckdb_family_create_index, duckdb_family_drop_index, + duckdb_family_upsert """ import typing as t - -from abc import abstractmethod, ABC import contextlib import warnings import uuid -import ibis -import ibis.expr.types.relations as ir -from ibis.expr.schema import SchemaLike -from ibis.backends.sql import SQLBackend - -from mountainash_settings import SettingsParameters import mountainash as ma from mountainash_data.core.constants import ( - CONST_DB_BACKEND, CONST_CONFLICT_ACTION, CONST_INDEX_TYPE, ) @@ -34,7 +22,6 @@ # =========================================================================== # MODULE-LEVEL HELPER FUNCTIONS -# Salvaged from _base_ibis_mixin.py — converted from mixin methods to functions # =========================================================================== def _generate_index_name( @@ -83,8 +70,6 @@ def _normalize_columns( # =========================================================================== # PER-DIALECT CAPABILITY HOOK FUNCTIONS -# Salvaged from duckdb_ibis_operations.py, sqlite_ibis_operations.py, -# motherduck_ibis_operations.py # =========================================================================== # --- DuckDB --- @@ -213,752 +198,144 @@ def motherduck_list_tables( # =========================================================================== -# _DuckDBFamilyOperationsMixin -# Salvaged from _duckdb_family_mixin.py — retained as class for backward -# compatibility. Internal helpers now delegate to module-level functions. +# STANDALONE HOOK FUNCTIONS # =========================================================================== -class _DuckDBFamilyOperationsMixin: - """ - Shared operations for DuckDB-family databases. - - This class provides common implementation for databases that share - DuckDB's SQL syntax: DuckDB, MotherDuck, and SQLite. - - Internal helpers delegate to module-level functions (see top of file). - """ - - # Delegate helper methods to module-level functions so existing code - # that calls cls._generate_index_name() etc. still works. - _generate_index_name = staticmethod(_generate_index_name) - _format_qualified_table = staticmethod(_format_qualified_table) - _normalize_columns = staticmethod(_normalize_columns) - - @classmethod - def create_index( - cls, - ibis_backend: SQLBackend, - table_name: str, - columns: list[str] | str, - *, - index_name: str | None = None, - unique: bool = False, - index_type: str | None = None, - where_condition: str | None = None, - database: str | None = None, - if_not_exists: bool = True - ) -> bool: - """Create an index using DuckDB/SQLite syntax.""" - columns_list = _normalize_columns(columns) - - if index_name is None: - index_name = _generate_index_name(table_name, columns_list, unique=unique) - - qualified_table = _format_qualified_table(table_name, database=database) - columns_sql = ", ".join(columns_list) - - unique_sql = "UNIQUE " if unique else "" - if_not_exists_sql = "IF NOT EXISTS " if if_not_exists else "" - where_sql = f" WHERE {where_condition}" if where_condition else "" - - if index_type and index_type != CONST_INDEX_TYPE.BTREE: - warnings.warn( - f"Index type {index_type} not supported, using default BTREE" - ) +def duckdb_family_create_index( + ibis_conn: t.Any, + table_name: str, + columns: list[str] | str, + *, + index_name: str | None = None, + unique: bool = False, + index_type: str | None = None, + where_condition: str | None = None, + database: str | None = None, + if_not_exists: bool = True, +) -> None: + """Create an index using DuckDB/SQLite syntax.""" + columns_list = _normalize_columns(columns) - create_sql = ( - f"CREATE {unique_sql}INDEX {if_not_exists_sql}{index_name} " - f"ON {qualified_table} ({columns_sql}){where_sql}" - ) + if index_name is None: + index_name = _generate_index_name(table_name, columns_list, unique=unique) - try: - with contextlib.closing(ibis_backend.con.cursor()) as cur: - cur.execute(create_sql) - return True - except Exception as e: - print(f"Error creating index {index_name}: {e}") - return False - - @classmethod - def drop_index( - cls, - ibis_backend: SQLBackend, - index_name: str, - *, - table_name: str | None = None, - database: str | None = None, - if_exists: bool = True - ) -> bool: - """Drop an index.""" - if_exists_sql = "IF EXISTS " if if_exists else "" - drop_sql = f"DROP INDEX {if_exists_sql}{index_name}" + qualified_table = _format_qualified_table(table_name, database=database) + columns_sql = ", ".join(columns_list) - try: - with contextlib.closing(ibis_backend.con.cursor()) as cur: - cur.execute(drop_sql) - return True - except Exception as e: - print(f"Error dropping index {index_name}: {e}") - return False - - @classmethod - def index_exists( - cls, - ibis_backend: SQLBackend, - index_name: str, - *, - table_name: str | None = None, - database: str | None = None - ) -> bool: - """Check if an index exists.""" - check_sql = cls._get_index_exists_sql(index_name, table_name, database) - result = cls.run_sql(ibis_backend, check_sql) - - if result is None: - return False - - count = ma.relation(result).to_dict()["count"][0] - return count > 0 - - @classmethod - def list_indexes( - cls, - ibis_backend: SQLBackend, - table_name: str, - *, - database: str | None = None - ) -> list[dict]: - """List all indexes for a table.""" - list_sql = cls._get_list_indexes_sql(table_name, database) - result = cls.run_sql(ibis_backend, list_sql) - - if result is None: - return [] - - return ma.relation(result).to_dicts() - - @classmethod - def _get_index_exists_sql( - cls, - index_name: str, - table_name: str | None, - database: str | None - ) -> str: - """Provider-specific SQL to check index existence. Must be overridden.""" - raise NotImplementedError( - f"{cls.__name__}: _get_index_exists_sql must be implemented" - ) + unique_sql = "UNIQUE " if unique else "" + if_not_exists_sql = "IF NOT EXISTS " if if_not_exists else "" + where_sql = f" WHERE {where_condition}" if where_condition else "" - @classmethod - def _get_list_indexes_sql( - cls, - table_name: str, - database: str | None - ) -> str: - """Provider-specific SQL to list all indexes for a table. Must be overridden.""" - raise NotImplementedError( - f"{cls.__name__}: _get_list_indexes_sql must be implemented" + if index_type and index_type != CONST_INDEX_TYPE.BTREE: + warnings.warn( + f"Index type {index_type} not supported, using default BTREE" ) - @classmethod - def _upsert( - cls, - ibis_backend: SQLBackend, - table_name: str, - df: ir.Table | t.Any, - *, - conflict_columns: list[str] | str, - update_columns: list[str] | str | None = None, - conflict_action: str = CONST_CONFLICT_ACTION.UPDATE, - update_condition: str | None = None, - database: str | None = None, - schema: str | None = None, - ) -> None: - """Perform upsert using INSERT ... ON CONFLICT syntax.""" - conflict_cols = _normalize_columns(conflict_columns) - all_columns = ma.relation(df).columns - - if update_columns is None: - update_cols = [col for col in all_columns if col not in conflict_cols] - else: - update_cols = _normalize_columns(update_columns) - - if not cls.table_exists(ibis_backend, table_name, database=database): - raise ValueError(f"Target table '{table_name}' does not exist") - - if conflict_action not in [CONST_CONFLICT_ACTION.UPDATE, CONST_CONFLICT_ACTION.NOTHING]: - raise ValueError( - f"conflict_action must be '{CONST_CONFLICT_ACTION.UPDATE}' or " - f"'{CONST_CONFLICT_ACTION.NOTHING}', got '{conflict_action}'" - ) + create_sql = ( + f"CREATE {unique_sql}INDEX {if_not_exists_sql}{index_name} " + f"ON {qualified_table} ({columns_sql}){where_sql}" + ) - if conflict_action == CONST_CONFLICT_ACTION.NOTHING: - if update_cols or update_condition: - warnings.warn( - "update_columns and update_condition are ignored when " - "conflict_action='NOTHING'" - ) - - staging_table = f"temp_upsert_{uuid.uuid4().hex[:8]}" - qualified_table = _format_qualified_table(table_name, database=database, schema=schema) - - all_cols_sql = ", ".join(all_columns) - conflict_cols_sql = ", ".join(conflict_cols) - - if conflict_action == CONST_CONFLICT_ACTION.UPDATE: - if not update_cols: - raise ValueError( - "No columns to update. Either provide update_columns or ensure " - "dataframe has columns beyond conflict_columns" - ) - update_set_sql = ", ".join([f"{col} = EXCLUDED.{col}" for col in update_cols]) - where_sql = f" WHERE {update_condition}" if update_condition else "" - on_conflict_sql = ( - f"ON CONFLICT ({conflict_cols_sql}) DO UPDATE SET {update_set_sql}{where_sql}" - ) - else: # NOTHING - on_conflict_sql = f"ON CONFLICT ({conflict_cols_sql}) DO NOTHING" - - upsert_sql = f""" - INSERT INTO {qualified_table} ({all_cols_sql}) - SELECT {all_cols_sql} FROM {staging_table} - WHERE true - {on_conflict_sql} - """ - - print(f"DEBUG: Upsert SQL:\n{upsert_sql}") - print(f"DEBUG: Qualified table: {qualified_table}") - print(f"DEBUG: Staging table: {staging_table}") - print(f"DEBUG: Conflict columns: {conflict_cols}") - print(f"DEBUG: Update columns: {update_cols}") - - try: - if hasattr(ibis_backend.con, 'register'): - with contextlib.closing(ibis_backend.con.cursor()) as cur: - cur.execute("BEGIN TRANSACTION") - cur.register(staging_table, df) - cur.execute(upsert_sql) - cur.unregister(staging_table) - cur.execute("COMMIT") - else: - ibis_backend.create_table(staging_table, df, temp=True, overwrite=True) - - try: - with contextlib.closing(ibis_backend.con.cursor()) as cur: - cur.execute(upsert_sql) - ibis_backend.con.commit() - finally: - try: - ibis_backend.drop_table(staging_table, force=True) - except Exception: - pass - except Exception as e: - print(f"Error during upsert to {table_name}: {e}") - raise - - -# Compatibility shim: _BaseIbisMixin was the original mixin class -class _BaseIbisMixin: - """DEPRECATED compatibility shim — methods are now module-level functions - in mountainash_data.backends.ibis.operations.""" - - _generate_index_name = staticmethod(_generate_index_name) - _format_qualified_table = staticmethod(_format_qualified_table) - _normalize_columns = staticmethod(_normalize_columns) + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(create_sql) -# =========================================================================== -# BaseIbisOperations -# Salvaged verbatim from base_ibis_operations.py. -# =========================================================================== - -class BaseIbisOperations(ABC): - - def __init__(self, - db_auth_settings_parameters: SettingsParameters, - ): - ... - - ## SQL Queries - - @property - @abstractmethod - def db_backend_name(self) -> str: - return CONST_DB_BACKEND.DUCKDB - - @classmethod - def run_sql(cls, - ibis_backend: SQLBackend, - query: str, - /, - *, - schema: SchemaLike | None = None, - dialect: str | None = None, - ) -> t.Optional[ir.Table]: - - try: - return ibis_backend.sql(query, - schema=schema, - dialect=dialect - ) - except Exception as e: - print(f"Error executing SQL: {e}") - return None - - @classmethod - def run_expr( - cls, - ibis_backend: ibis.BaseBackend, - ibis_expr: ir.Expr, - /, - params: t.Dict | None = None, - limit: str | None = "default", - **kwargs: t.Any, - ) -> t.Any: - - try: - return ibis_backend.execute(ibis_expr, - params=params, - limit=limit, - **kwargs - ) if ibis_backend is not None else None - except Exception as e: - print(f"Error executing expression: {e}") - return None - - @classmethod - def to_sql( - cls, - ibis_backend: ibis.BaseBackend, - expr: ir.Expr, - /, - params=None, - limit: str | None = None, - pretty: bool = False, - **kwargs: t.Any, - ) -> t.Optional[str]: +def duckdb_family_drop_index( + ibis_conn: t.Any, + index_name: str, + *, + table_name: str | None = None, + database: str | None = None, + if_exists: bool = True, +) -> None: + """Drop an index using DuckDB/SQLite syntax.""" + if_exists_sql = "IF EXISTS " if if_exists else "" + drop_sql = f"DROP INDEX {if_exists_sql}{index_name}" - try: - return ibis_backend.compile(expr, - params=params, - limit=limit, - pretty=pretty, - **kwargs - ) if ibis_backend is not None else None - except Exception as e: - print(f"Error compiling expression to SQL: {e}") - return None - - ## Tables - @classmethod - def table( - cls, - ibis_backend: ibis.BaseBackend, - object_name: str, - /, - schema: str | None = None, - database: tuple[str, str] | str | None = None - ) -> t.Optional[ir.Table]: + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(drop_sql) - try: - return ibis_backend.table(object_name, - database=database - ) if ibis_backend is not None else None - except Exception as e: - print(f"Error getting table {object_name}: {e}") - return None - - @classmethod - def create_table(cls, - ibis_backend: ibis.BaseBackend, - table_name: str, - df: ir.Table|t.Any, - /, - schema: t.Optional[ibis.Schema] = None, - database: str | None = None, - temp: bool = False, - overwrite: bool = False, - ) -> None: - try: - ibis_backend.create_table(table_name, - obj=df, - schema=schema, - database=database, - temp=temp, - overwrite=overwrite) if ibis_backend is not None else None - except Exception as e: - print(f"Error creating table {table_name}: {e}") - return None - - @classmethod - def drop_table( - cls, - ibis_backend: ibis.BaseBackend, - table_name: str, - /, - database: str | None = None, - force: bool = False, - ) -> bool: - try: - ibis_backend.drop_table(table_name, - database=database, - force=force) if ibis_backend is not None else None - return True - except Exception: - return False - - ## Views - @classmethod - def create_view( - cls, - ibis_backend: ibis.BaseBackend, - view_name: str, - ibis_table_expr: ir.Table, - /, - database: str | None = None, - schema: str | None = None, - overwrite: bool = False, - ) -> t.Optional[ir.Table]: - - try: - return ibis_backend.create_view(view_name, - obj=ibis_table_expr, - database=database, - overwrite=overwrite) if ibis_backend is not None else None - except Exception as e: - print(f"Error creating view {view_name}: {e}") - return None - - @classmethod - def drop_view( - cls, - ibis_backend: ibis.BaseBackend, - view_name: str, - /, - database: str | None = None, - schema: str | None = None, - force: bool = False, - ) -> bool: +def duckdb_family_upsert( + ibis_conn: t.Any, + table_name: str, + df: t.Any, + *, + conflict_columns: list[str] | str, + update_columns: list[str] | str | None = None, + conflict_action: str = CONST_CONFLICT_ACTION.UPDATE, + update_condition: str | None = None, + database: str | None = None, + schema: str | None = None, +) -> None: + """Perform upsert using INSERT ... ON CONFLICT syntax (DuckDB/SQLite).""" + conflict_cols = _normalize_columns(conflict_columns) + all_columns = ma.relation(df).columns + + if update_columns is None: + update_cols = [col for col in all_columns if col not in conflict_cols] + else: + update_cols = _normalize_columns(update_columns) + + tables = ibis_conn.list_tables() + if table_name not in tables: + raise ValueError(f"Target table '{table_name}' does not exist") + + if conflict_action not in [CONST_CONFLICT_ACTION.UPDATE, CONST_CONFLICT_ACTION.NOTHING]: + raise ValueError( + f"conflict_action must be '{CONST_CONFLICT_ACTION.UPDATE}' or " + f"'{CONST_CONFLICT_ACTION.NOTHING}', got '{conflict_action}'" + ) - try: - ibis_backend.drop_view(view_name, - database=database, - force=force) if ibis_backend is not None else None - return True - except Exception: - print(f"Error dropping view {view_name}") - return False - - # Backend Data Manipulation - - @classmethod - def insert( - cls, - ibis_backend: SQLBackend, - table_name: str, - /, - df: ir.Table|t.Any, - database: str | None = None, - schema: str | None = None, - overwrite: bool = False, - ) -> bool: + if conflict_action == CONST_CONFLICT_ACTION.NOTHING: + if update_cols or update_condition: + warnings.warn( + "update_columns and update_condition are ignored when " + "conflict_action='NOTHING'" + ) - try: - ibis_backend.insert(table_name, - obj=df, - database=database, - overwrite=overwrite) if ibis_backend is not None else None - return True - except Exception as e: - print(f"Error inserting into table {table_name}: {e}") - return False - - @classmethod - def truncate( - cls, - ibis_backend: SQLBackend, - table_name: str, - /, - database: str | None = None, - schema: str | None = None - ) -> None: + staging_table = f"temp_upsert_{uuid.uuid4().hex[:8]}" + qualified_table = _format_qualified_table(table_name, database=database, schema=schema) - try: - ibis_backend.truncate_table( - table_name, - schema=schema, - database=database) if ibis_backend is not None else None - except Exception as e: - print(f"Error truncating table {table_name}: {e}") - return None - - @classmethod - def upsert( - cls, - ibis_backend: SQLBackend, - table_name: str, - df: ir.Table|t.Any, - /, - conflict_columns: list[str] | str, - update_columns: list[str] | str | None = None, - conflict_action: str = CONST_CONFLICT_ACTION.UPDATE, - update_condition: str | None = None, - database: str | None = None, - schema: str | None = None, - ) -> None: + all_cols_sql = ", ".join(all_columns) + conflict_cols_sql = ", ".join(conflict_cols) - try: - cls._upsert( - ibis_backend, - table_name, - df, - conflict_columns=conflict_columns, - update_columns=update_columns, - conflict_action=conflict_action, - update_condition=update_condition, - database=database, - schema=schema) if ibis_backend is not None else None - except Exception as e: - print(f"Error upserting into table {table_name}: {e}") - raise - - @classmethod - @abstractmethod - def _upsert( - cls, - ibis_backend: SQLBackend, - table_name: str, - df: ir.Table|t.Any, - *, - conflict_columns: list[str] | str, - update_columns: list[str] | str | None = None, - conflict_action: str = CONST_CONFLICT_ACTION.UPDATE, - update_condition: str | None = None, - database: str | None = None, - schema: str | None = None, - ) -> None: - raise NotImplementedError("Upsert is not implemented for this backend") - - # =========================== - # INDEX MANAGEMENT - # =========================== - - @classmethod - @abstractmethod - def create_index( - cls, - ibis_backend: SQLBackend, - table_name: str, - columns: list[str] | str, - *, - index_name: str | None = None, - unique: bool = False, - index_type: str | None = None, - where_condition: str | None = None, - database: str | None = None, - if_not_exists: bool = True - ) -> bool: - raise NotImplementedError("create_index is not implemented for this backend") - - @classmethod - def create_unique_index( - cls, - ibis_backend: SQLBackend, - table_name: str, - columns: list[str] | str, - *, - index_name: str | None = None, - where_condition: str | None = None, - database: str | None = None - ) -> bool: - return cls.create_index( - ibis_backend, table_name, columns, - index_name=index_name, unique=True, - where_condition=where_condition, database=database + if conflict_action == CONST_CONFLICT_ACTION.UPDATE: + if not update_cols: + raise ValueError( + "No columns to update. Either provide update_columns or ensure " + "dataframe has columns beyond conflict_columns" + ) + update_set_sql = ", ".join([f"{col} = EXCLUDED.{col}" for col in update_cols]) + where_sql = f" WHERE {update_condition}" if update_condition else "" + on_conflict_sql = ( + f"ON CONFLICT ({conflict_cols_sql}) DO UPDATE SET {update_set_sql}{where_sql}" ) + else: + on_conflict_sql = f"ON CONFLICT ({conflict_cols_sql}) DO NOTHING" + + upsert_sql = f""" + INSERT INTO {qualified_table} ({all_cols_sql}) + SELECT {all_cols_sql} FROM {staging_table} + WHERE true + {on_conflict_sql} + """ - @classmethod - @abstractmethod - def drop_index( - cls, - ibis_backend: SQLBackend, - index_name: str, - *, - table_name: str | None = None, - database: str | None = None, - if_exists: bool = True - ) -> bool: - raise NotImplementedError("drop_index is not implemented for this backend") - - @classmethod - @abstractmethod - def index_exists( - cls, - ibis_backend: SQLBackend, - index_name: str, - *, - table_name: str | None = None, - database: str | None = None - ) -> bool: - raise NotImplementedError("index_exists is not implemented for this backend") - - @classmethod - @abstractmethod - def list_indexes( - cls, - ibis_backend: SQLBackend, - table_name: str, - *, - database: str | None = None - ) -> list[dict]: - raise NotImplementedError("list_indexes is not implemented for this backend") - - # =========================== - # t.Optionally Implemented Functions - # =========================== - - @classmethod - def list_tables(cls, - ibis_backend: ibis.BaseBackend, - table_name: str | None = None, - database: tuple[str, str] | str | None = None - ) -> t.List[str]: - - try: - return ibis_backend.list_tables(like=table_name, database=database) if ibis_backend is not None else [] - except Exception as e: - print(f"Error listing tables: {e}") - return [] - - @classmethod - def rename_table(cls, - ibis_backend: ibis.BaseBackend, - old_name: str, - new_name: str, - ) -> None: - + if hasattr(ibis_conn.con, 'register'): + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute("BEGIN TRANSACTION") + cur.register(staging_table, df) + cur.execute(upsert_sql) + cur.unregister(staging_table) + cur.execute("COMMIT") + else: + ibis_conn.create_table(staging_table, df, temp=True, overwrite=True) try: - return cls._rename_table(old_name=old_name, new_name=new_name) if ibis_backend is not None else None - except Exception as e: - print(f"Error renaming table {old_name} to {new_name}: {e}") - return None - - @classmethod - @abstractmethod - def _rename_table(cls, - old_name: str, - new_name: str, - ) -> None: - raise NotImplementedError - - @classmethod - def table_exists(cls, - ibis_backend: ibis.BaseBackend, - table_name: str | None = None, - database: tuple[str, str] | str | None = None - ) -> bool: - - tables = cls.list_tables(ibis_backend, table_name=table_name, database=database) - return True if table_name in tables else False - - -# =========================================================================== -# Concrete per-dialect operations classes -# Preserved as concrete classes so the OperationsFactory can instantiate them. -# Each class combines _DuckDBFamilyOperationsMixin (concrete upsert/index -# implementations) with BaseIbisOperations (abstract template) and provides -# the dialect-specific index catalog SQL via overriding _get_index_exists_sql -# and _get_list_indexes_sql. -# =========================================================================== - -class DuckDB_IbisOperations(_DuckDBFamilyOperationsMixin, BaseIbisOperations): - """DuckDB operations — concrete implementation.""" - - @property - def db_backend_name(self) -> str: - return CONST_DB_BACKEND.DUCKDB - - @classmethod - def _rename_table(cls, old_name: str, new_name: str) -> None: - raise NotImplementedError(f"{CONST_DB_BACKEND.DUCKDB}: rename_table not yet implemented") - - @classmethod - def _get_index_exists_sql(cls, index_name: str, table_name: str | None, database: str | None) -> str: - return duckdb_get_index_exists_sql(index_name, table_name, database) - - @classmethod - def _get_list_indexes_sql(cls, table_name: str, database: str | None) -> str: - return duckdb_get_list_indexes_sql(table_name, database) - - -class SQLite_IbisOperations(_DuckDBFamilyOperationsMixin, BaseIbisOperations): - """SQLite operations — concrete implementation.""" - - @property - def db_backend_name(self) -> str: - return CONST_DB_BACKEND.SQLITE - - @classmethod - def _rename_table(cls, old_name: str, new_name: str) -> None: - raise NotImplementedError(f"{CONST_DB_BACKEND.SQLITE}: rename_table not yet implemented") - - @classmethod - def _get_index_exists_sql(cls, index_name: str, table_name: str | None, database: str | None) -> str: - return sqlite_get_index_exists_sql(index_name, table_name, database) - - @classmethod - def _get_list_indexes_sql(cls, table_name: str, database: str | None) -> str: - return sqlite_get_list_indexes_sql(table_name, database) - - -class MotherDuck_IbisOperations(_DuckDBFamilyOperationsMixin, BaseIbisOperations): - """MotherDuck operations — concrete implementation.""" - - @property - def db_backend_name(self) -> str: - return CONST_DB_BACKEND.MOTHERDUCK - - @classmethod - def _rename_table(cls, old_name: str, new_name: str) -> None: - raise NotImplementedError(f"{CONST_DB_BACKEND.MOTHERDUCK}: rename_table not yet implemented") - - @classmethod - def _get_index_exists_sql(cls, index_name: str, table_name: str | None, database: str | None) -> str: - return motherduck_get_index_exists_sql(index_name, table_name, database) - - @classmethod - def _get_list_indexes_sql(cls, table_name: str, database: str | None) -> str: - return motherduck_get_list_indexes_sql(table_name, database) - - -class Trino_IbisOperations(BaseIbisOperations): - """Trino operations — connection mode is handled via DialectSpec registry.""" - - @property - def db_backend_name(self) -> str: - return CONST_DB_BACKEND.TRINO - - @classmethod - def _rename_table(cls, old_name: str, new_name: str) -> None: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: rename_table not yet implemented") - - @classmethod - def _upsert(cls, *args, **kwargs) -> None: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: upsert not supported") - - @classmethod - def create_index(cls, *args, **kwargs) -> bool: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: create_index not supported") - - @classmethod - def drop_index(cls, *args, **kwargs) -> bool: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: drop_index not supported") - - @classmethod - def index_exists(cls, *args, **kwargs) -> bool: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: index_exists not supported") - - @classmethod - def list_indexes(cls, *args, **kwargs) -> list[dict]: - raise NotImplementedError(f"{CONST_DB_BACKEND.TRINO}: list_indexes not supported") + with contextlib.closing(ibis_conn.con.cursor()) as cur: + cur.execute(upsert_sql) + ibis_conn.con.commit() + finally: + try: + ibis_conn.drop_table(staging_table, force=True) + except Exception: + pass diff --git a/src/mountainash_data/backends/iceberg/backend.py b/src/mountainash_data/backends/iceberg/backend.py index e5555d7..f353437 100644 --- a/src/mountainash_data/backends/iceberg/backend.py +++ b/src/mountainash_data/backends/iceberg/backend.py @@ -1,9 +1,4 @@ -"""IcebergBackend — implements core.protocol.Backend for iceberg catalogs. - -NOTE: ``to_relation()`` is intentionally NOT implemented on this backend. -It requires mountainash-expressions to gain an Iceberg adapter, which is -a separate work item. The gap is documented in the Phase 3 spec. -""" +"""IcebergBackend — implements core.protocol.Backend for iceberg catalogs.""" from __future__ import annotations @@ -11,7 +6,6 @@ from mountainash_data.backends.iceberg.catalogs.rest import IcebergRestConnection from mountainash_data.backends.iceberg.connection import IcebergConnectionBase -from mountainash_data.core.protocol import Connection _CATALOG_REGISTRY: dict[str, type[IcebergConnectionBase]] = { @@ -20,28 +14,10 @@ class IcebergBackend: - """Iceberg backend factory. + """Iceberg backend — single entry point for iceberg catalog interaction. Construction takes a catalog type (e.g. ``'rest'``) and config kwargs. - ``connect()`` returns a live ``IcebergConnectionBase`` instance that - satisfies ``core.protocol.Connection``. - - Args: - catalog: One of the keys in ``_CATALOG_REGISTRY`` (currently - only ``'rest'``). - **config: Keyword arguments forwarded verbatim to the connection - class constructor (minus the ``db_auth_settings_parameters`` - which must be provided separately when calling ``connect()``). - - Raises: - KeyError: If ``catalog`` is not a known catalog type. - - Example:: - - backend = IcebergBackend(catalog="rest", uri="http://localhost:8181") - conn = backend.connect() - tables = conn.list_tables() - conn.close() + ``connect()`` returns ``self``. Use as a context manager. """ name = "iceberg" @@ -54,13 +30,49 @@ def __init__(self, catalog: str, **config: t.Any) -> None: ) self._catalog_cls = _CATALOG_REGISTRY[catalog] self._config = config + self._conn: IcebergConnectionBase | None = None + + def connect(self) -> IcebergBackend: + """Open a connection. Returns self for fluent chaining.""" + if self._conn is None: + self._conn = self._catalog_cls(**self._config) + return self + + def close(self) -> IcebergBackend: + """Release the connection. Idempotent. Returns self.""" + if self._conn is not None: + if hasattr(self._conn, "close"): + self._conn.close() + self._conn = None + return self + + def __enter__(self) -> IcebergBackend: + self.connect() + return self + + def __exit__(self, *args: t.Any) -> None: + self.close() + + def _require_connected(self) -> IcebergConnectionBase: + if self._conn is None: + raise RuntimeError( + "IcebergBackend is not connected. Call connect() first." + ) + return self._conn + + def list_tables(self, namespace: str | None = None) -> list[str]: + return self._require_connected().list_tables(namespace=namespace) + + def list_namespaces(self) -> list[str]: + return self._require_connected().list_namespaces() + + def inspect_table( + self, name: str, namespace: str | None = None + ) -> t.Any: + return self._require_connected().inspect_table(name, namespace=namespace) - def connect(self) -> Connection: - """Open a connection. Caller is responsible for closing it. + def inspect_namespace(self, name: str) -> t.Any: + return self._require_connected().inspect_namespace(name) - Note: The legacy IcebergConnectionBase requires a - ``db_auth_settings_parameters`` argument. When ``_config`` does not - include one, this will raise at the base class constructor level. - This mirrors the legacy behaviour. - """ - return self._catalog_cls(**self._config) + def inspect_catalog(self) -> t.Any: + return self._require_connected().inspect_catalog() diff --git a/src/mountainash_data/core/factories/__init__.py b/src/mountainash_data/core/factories/__init__.py deleted file mode 100644 index 3bc812f..0000000 --- a/src/mountainash_data/core/factories/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from .base_strategy_factory import BaseStrategyFactory -from .settings_type_factory_mixin import SettingsTypeFactoryMixin -from .connection_factory import ConnectionFactory -from .operations_factory import OperationsFactory -from .settings_factory import SettingsFactory - -__all__ = [ - "BaseStrategyFactory", - "SettingsTypeFactoryMixin", - "ConnectionFactory", - "OperationsFactory", - "SettingsFactory", -] diff --git a/src/mountainash_data/core/factories/base_strategy_factory.py b/src/mountainash_data/core/factories/base_strategy_factory.py deleted file mode 100644 index 8d1489c..0000000 --- a/src/mountainash_data/core/factories/base_strategy_factory.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -Base Strategy Factory with dual-generic pattern for database connections and operations. - -This module implements the factory pattern using lazy loading and runtime strategy selection, -adapted from mountainash-dataframes architecture for database connection/operation strategies. -""" - -import importlib -import logging -from abc import ABC, abstractmethod -from enum import Enum -from typing import ClassVar, Dict, Generic, Optional, Type, TypeVar - -# Type variables for dual-generic pattern -InputT = TypeVar("InputT") # Input type (SettingsParameters) -StrategyT = TypeVar("StrategyT") # Strategy type (connection or operation class) - -logger = logging.getLogger(__name__) - - -class BaseStrategyFactory(ABC, Generic[InputT, StrategyT]): - """ - Universal base factory with dual-generic design for database strategies. - - Generic Type Parameters: - InputT: Type constraint for input (e.g., SettingsParameters) - StrategyT: Strategy class type returned by factory (e.g., BaseIbisConnection, BaseIbisOperations) - - Key Features: - - Lazy loading: Strategies imported only when first used - - Runtime detection: Auto-detect backend from input - - Strategy caching: Once loaded, strategies cached for reuse - - Zero import cost: String-based configuration, no imports until needed - """ - - # Class-level caches for lazy loading - _strategy_cache: ClassVar[Dict[Enum, Type[StrategyT]]] = {} - _strategy_modules: ClassVar[Dict[Enum, str]] = {} - _strategy_classes: ClassVar[Dict[Enum, str]] = {} - - def __init__(self): - """Initialize factory and configure strategy mappings.""" - if not self._strategy_modules: - self._configure_strategy_mapping() - - @classmethod - @abstractmethod - def _configure_strategy_mapping(cls) -> None: - """ - Configure strategy mappings using ONLY strings (no imports). - - Subclasses MUST implement this to define: - cls._strategy_modules: {backend_type: "module.path"} - cls._strategy_classes: {backend_type: "ClassName"} - - Example: - cls._strategy_modules = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "mountainash_data.databases.connections.ibis", - } - cls._strategy_classes = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "Postgres_IbisConnection", - } - """ - pass - - @classmethod - @abstractmethod - def _get_strategy_key(cls, input_data: InputT, **kwargs) -> Optional[Enum]: - """ - Determine strategy key from input data. - - This is implemented by mixin classes (e.g., SettingsTypeFactoryMixin) - to detect backend type from SettingsParameters. - - Args: - input_data: Input data to analyze (e.g., SettingsParameters) - **kwargs: Additional context for detection - - Returns: - Strategy key (enum) or None if detection fails - """ - pass - - @classmethod - def _lazy_load_strategy_class(cls, strategy_key: Enum) -> Type[StrategyT]: - """ - Lazy load strategy class using runtime import. - - Args: - strategy_key: Backend type enum - - Returns: - Strategy class (not instance) - - Raises: - KeyError: If strategy not configured - ImportError: If module import fails - """ - # Check cache first - if strategy_key in cls._strategy_cache: - return cls._strategy_cache[strategy_key] - - # Ensure mappings are configured - if not cls._strategy_modules: - cls._configure_strategy_mapping() - - # Get module and class name - if strategy_key not in cls._strategy_modules: - raise KeyError( - f"No strategy configured for {strategy_key}. " - f"Available: {list(cls._strategy_modules.keys())}" - ) - - module_path = cls._strategy_modules[strategy_key] - class_name = cls._strategy_classes[strategy_key] - - # Runtime import - try: - module = importlib.import_module(module_path) - strategy_class = getattr(module, class_name) - - # Cache for future use - cls._strategy_cache[strategy_key] = strategy_class - - logger.debug( - f"Loaded strategy {class_name} from {module_path} for {strategy_key}" - ) - - return strategy_class - - except ImportError as e: - raise ImportError( - f"Failed to import {class_name} from {module_path} for {strategy_key}: {e}" - ) from e - - @classmethod - def get_strategy(cls, input_data: InputT, **kwargs) -> StrategyT: - """ - Get appropriate strategy instance for input data. - - This is the main entry point for factory usage. - - Args: - input_data: Input to analyze (e.g., SettingsParameters) - **kwargs: Additional arguments passed to strategy constructor - - Returns: - Strategy instance ready to use - - Raises: - ValueError: If backend cannot be detected - ImportError: If strategy import fails - """ - # Detect backend type - strategy_key = cls._get_strategy_key(input_data, **kwargs) - - if strategy_key is None: - raise ValueError( - f"Could not determine strategy for input: {input_data}" - ) - - # Load strategy class - strategy_class = cls._lazy_load_strategy_class(strategy_key) - - # Return instance (strategies are typically instantiated with settings) - # Note: Subclasses may override this to customize instantiation - return strategy_class # type: ignore - - @classmethod - def clear_cache(cls) -> None: - """Clear strategy cache (mainly for testing).""" - cls._strategy_cache.clear() - logger.debug(f"Cleared strategy cache for {cls.__name__}") diff --git a/src/mountainash_data/core/factories/connection_factory.py b/src/mountainash_data/core/factories/connection_factory.py deleted file mode 100644 index 50a7ac8..0000000 --- a/src/mountainash_data/core/factories/connection_factory.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -Connection Factory for settings-driven database connection creation. - -Uses SettingsParameters to auto-detect backend and create appropriate connection. -""" - -import logging -from typing import Type - -from mountainash_settings import SettingsParameters - -from ..connection import BaseDBConnection -from ..constants import CONST_DB_PROVIDER_TYPE -from .base_strategy_factory import BaseStrategyFactory -from .settings_type_factory_mixin import SettingsTypeFactoryMixin - -logger = logging.getLogger(__name__) - - -class ConnectionFactory( - SettingsTypeFactoryMixin, - BaseStrategyFactory[SettingsParameters, Type[BaseDBConnection]], -): - """ - Settings-driven factory for database connections. - - Detects backend type from SettingsParameters.settings_class and returns - appropriate connection class with lazy loading. - - Example: - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - - factory = ConnectionFactory() - connection_class = factory.get_strategy(settings_params) - connection = connection_class(db_auth_settings_parameters=settings_params) - backend = connection.connect() - """ - - # Each subclass needs its own dictionaries to avoid sharing with other factories - _strategy_cache = {} - _strategy_modules = {} - _strategy_classes = {} - - @classmethod - def _configure_strategy_mapping(cls) -> None: - """ - Configure strategy mappings using ONLY strings (no imports). - - Maps backend types to connection module paths and class names. - """ - # Ibis-based connections — point directly at backends.ibis.connection - # (bypasses the databases.connections.ibis shim chain from Phase 4) - cls._strategy_modules = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.SQLITE: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.DUCKDB: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.MOTHERDUCK: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.SNOWFLAKE: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.BIGQUERY: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.MSSQL: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.MYSQL: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.TRINO: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.PYSPARK: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.REDSHIFT: "mountainash_data.backends.ibis.connection", - CONST_DB_PROVIDER_TYPE.ORACLE: "mountainash_data.backends.ibis.connection", - # Iceberg connections — point directly at backends.iceberg - CONST_DB_PROVIDER_TYPE.PYICEBERG_REST: "mountainash_data.backends.iceberg.catalogs.rest", - } - - cls._strategy_classes = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "Postgres_IbisConnection", - CONST_DB_PROVIDER_TYPE.SQLITE: "SQLite_IbisConnection", - CONST_DB_PROVIDER_TYPE.DUCKDB: "DuckDB_IbisConnection", - CONST_DB_PROVIDER_TYPE.MOTHERDUCK: "MotherDuck_IbisConnection", - CONST_DB_PROVIDER_TYPE.SNOWFLAKE: "Snowflake_IbisConnection", - CONST_DB_PROVIDER_TYPE.BIGQUERY: "BigQuery_IbisConnection", - CONST_DB_PROVIDER_TYPE.MSSQL: "MSSQL_IbisConnection", - CONST_DB_PROVIDER_TYPE.MYSQL: "MySQL_IbisConnection", - CONST_DB_PROVIDER_TYPE.TRINO: "Trino_IbisConnection", - CONST_DB_PROVIDER_TYPE.PYSPARK: "PySpark_IbisConnection", - CONST_DB_PROVIDER_TYPE.REDSHIFT: "Redshift_IbisConnection", - CONST_DB_PROVIDER_TYPE.ORACLE: "Oracle_IbisConnection", - CONST_DB_PROVIDER_TYPE.PYICEBERG_REST: "IcebergRestConnection", - } - - @classmethod - def get_connection( - cls, settings_parameters: SettingsParameters - ) -> BaseDBConnection: - """ - Convenience method to get connection instance directly. - - Args: - settings_parameters: SettingsParameters with settings_class - - Returns: - Connection instance ready to use - - Example: - factory = ConnectionFactory() - connection = factory.get_connection(settings_params) - backend = connection.connect() - """ - connection_class = cls.get_strategy(settings_parameters) - return connection_class(db_auth_settings_parameters=settings_parameters) diff --git a/src/mountainash_data/core/factories/operations_factory.py b/src/mountainash_data/core/factories/operations_factory.py deleted file mode 100644 index 5bf1306..0000000 --- a/src/mountainash_data/core/factories/operations_factory.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -Operations Factory for settings-driven database operations creation. - -Uses SettingsParameters to auto-detect backend and create appropriate operations instance. -""" - -import logging -from typing import Type - -from mountainash_settings import SettingsParameters - -from mountainash_data.backends.ibis.operations import BaseIbisOperations -from ..constants import CONST_DB_PROVIDER_TYPE -from .base_strategy_factory import BaseStrategyFactory -from .settings_type_factory_mixin import SettingsTypeFactoryMixin - -logger = logging.getLogger(__name__) - - -class OperationsFactory( - SettingsTypeFactoryMixin, - BaseStrategyFactory[SettingsParameters, Type[BaseIbisOperations]], -): - """ - Settings-driven factory for database operations. - - Detects backend type from SettingsParameters.settings_class and returns - appropriate operations class with lazy loading. - - Example: - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - - factory = OperationsFactory() - operations_class = factory.get_strategy(settings_params) - operations = operations_class(db_auth_settings_parameters=settings_params) - operations.create_table(backend, "my_table", dataframe) - """ - - # Each subclass needs its own dictionaries to avoid sharing with other factories - _strategy_cache = {} - _strategy_modules = {} - _strategy_classes = {} - - @classmethod - def _configure_strategy_mapping(cls) -> None: - """ - Configure strategy mappings using ONLY strings (no imports). - - Maps backend types to operations module paths and class names. - """ - # All ibis backends point directly at backends.ibis.operations - # (bypasses the databases.operations.ibis shim chain from Phase 4). - # Per-backend ops classes (SQLite_IbisOperations, etc.) are in the - # same module; POSTGRESQL/SNOWFLAKE/etc. fall back to BaseIbisOperations. - cls._strategy_modules = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.SQLITE: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.DUCKDB: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.MOTHERDUCK: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.SNOWFLAKE: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.BIGQUERY: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.MSSQL: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.MYSQL: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.TRINO: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.PYSPARK: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.REDSHIFT: "mountainash_data.backends.ibis.operations", - CONST_DB_PROVIDER_TYPE.ORACLE: "mountainash_data.backends.ibis.operations", - # Iceberg REST — operations are merged into IcebergRestConnection - CONST_DB_PROVIDER_TYPE.PYICEBERG_REST: "mountainash_data.backends.iceberg.catalogs.rest", - } - - cls._strategy_classes = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.SQLITE: "SQLite_IbisOperations", - CONST_DB_PROVIDER_TYPE.DUCKDB: "DuckDB_IbisOperations", - CONST_DB_PROVIDER_TYPE.MOTHERDUCK: "MotherDuck_IbisOperations", - CONST_DB_PROVIDER_TYPE.SNOWFLAKE: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.BIGQUERY: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.MSSQL: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.MYSQL: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.TRINO: "Trino_IbisOperations", - CONST_DB_PROVIDER_TYPE.PYSPARK: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.REDSHIFT: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.ORACLE: "BaseIbisOperations", - CONST_DB_PROVIDER_TYPE.PYICEBERG_REST: "IcebergRestConnection", - } - - @classmethod - def get_operations( - cls, settings_parameters: SettingsParameters - ) -> BaseIbisOperations: - """ - Convenience method to get operations instance directly. - - Args: - settings_parameters: SettingsParameters with settings_class - - Returns: - Operations instance ready to use - - Example: - factory = OperationsFactory() - operations = factory.get_operations(settings_params) - operations.create_table(backend, "my_table", dataframe) - """ - operations_class = cls.get_strategy(settings_parameters) - return operations_class(db_auth_settings_parameters=settings_parameters) diff --git a/src/mountainash_data/core/factories/settings_factory.py b/src/mountainash_data/core/factories/settings_factory.py deleted file mode 100644 index 9408e89..0000000 --- a/src/mountainash_data/core/factories/settings_factory.py +++ /dev/null @@ -1,206 +0,0 @@ -""" -Settings Factory for auto-detecting and creating appropriate settings classes. - -Provides utilities to auto-detect database backend from connection strings -and create the corresponding settings class. -""" - -import logging -import re -from typing import Dict, Type -from urllib.parse import urlparse - -from mountainash_settings import MountainAshBaseSettings - -from ..constants import CONST_DB_PROVIDER_TYPE - -logger = logging.getLogger(__name__) - - -class SettingsFactory: - """ - Factory for auto-detecting and loading appropriate settings classes. - - Provides intelligent backend detection from connection strings and - automatic settings class selection. - """ - - # Mapping of URL schemes to backend types - SCHEME_MAP: Dict[str, CONST_DB_PROVIDER_TYPE] = { - "postgresql": CONST_DB_PROVIDER_TYPE.POSTGRESQL, - "postgres": CONST_DB_PROVIDER_TYPE.POSTGRESQL, - "sqlite": CONST_DB_PROVIDER_TYPE.SQLITE, - "duckdb": CONST_DB_PROVIDER_TYPE.DUCKDB, - "motherduck": CONST_DB_PROVIDER_TYPE.MOTHERDUCK, - "md": CONST_DB_PROVIDER_TYPE.MOTHERDUCK, - "snowflake": CONST_DB_PROVIDER_TYPE.SNOWFLAKE, - "bigquery": CONST_DB_PROVIDER_TYPE.BIGQUERY, - "mssql": CONST_DB_PROVIDER_TYPE.MSSQL, - "mysql": CONST_DB_PROVIDER_TYPE.MYSQL, - "trino": CONST_DB_PROVIDER_TYPE.TRINO, - "pyspark": CONST_DB_PROVIDER_TYPE.PYSPARK, - "redshift": CONST_DB_PROVIDER_TYPE.REDSHIFT, - "oracle": CONST_DB_PROVIDER_TYPE.ORACLE, - } - - # Mapping of backend types to settings classes (lazy loaded) - SETTINGS_CLASS_MAP: Dict[CONST_DB_PROVIDER_TYPE, Type[MountainAshBaseSettings]] = {} - - @classmethod - def _ensure_settings_classes_loaded(cls) -> None: - """ - Lazy load settings class mappings. - - This avoids circular imports and reduces initial load time. - """ - if cls.SETTINGS_CLASS_MAP: - return # Already loaded - - # Lazy import settings classes only when needed - from ..settings import ( - PostgreSQLAuthSettings, - SQLiteAuthSettings, - DuckDBAuthSettings, - MotherDuckAuthSettings, - SnowflakeAuthSettings, - BigQueryAuthSettings, - MSSQLAuthSettings, - MySQLAuthSettings, - TrinoAuthSettings, - PySparkAuthSettings, - RedshiftAuthSettings, - PyIcebergRestAuthSettings, - ) - - cls.SETTINGS_CLASS_MAP = { - CONST_DB_PROVIDER_TYPE.POSTGRESQL: PostgreSQLAuthSettings, - CONST_DB_PROVIDER_TYPE.SQLITE: SQLiteAuthSettings, - CONST_DB_PROVIDER_TYPE.DUCKDB: DuckDBAuthSettings, - CONST_DB_PROVIDER_TYPE.MOTHERDUCK: MotherDuckAuthSettings, - CONST_DB_PROVIDER_TYPE.SNOWFLAKE: SnowflakeAuthSettings, - CONST_DB_PROVIDER_TYPE.BIGQUERY: BigQueryAuthSettings, - CONST_DB_PROVIDER_TYPE.MSSQL: MSSQLAuthSettings, - CONST_DB_PROVIDER_TYPE.MYSQL: MySQLAuthSettings, - CONST_DB_PROVIDER_TYPE.TRINO: TrinoAuthSettings, - CONST_DB_PROVIDER_TYPE.PYSPARK: PySparkAuthSettings, - CONST_DB_PROVIDER_TYPE.REDSHIFT: RedshiftAuthSettings, - CONST_DB_PROVIDER_TYPE.PYICEBERG_REST: PyIcebergRestAuthSettings, - } - - logger.debug("Settings class mappings loaded") - - @classmethod - def from_backend_type( - cls, backend_type: CONST_DB_PROVIDER_TYPE, **kwargs - ) -> MountainAshBaseSettings: - """ - Create settings instance from backend type. - - Args: - backend_type: Database backend type enum - **kwargs: Arguments passed to settings constructor - - Returns: - Settings instance for the backend - - Raises: - KeyError: If backend type not supported - """ - cls._ensure_settings_classes_loaded() - - if backend_type not in cls.SETTINGS_CLASS_MAP: - raise KeyError( - f"No settings class for {backend_type}. " - f"Available: {list(cls.SETTINGS_CLASS_MAP.keys())}" - ) - - settings_class = cls.SETTINGS_CLASS_MAP[backend_type] - - # Auto-inject auth=NoAuth() for ConnectionProfile subclasses that only - # support NoAuth and haven't been passed an auth kwarg by the caller. - if "auth" not in kwargs: - try: - from ..settings.profile import ConnectionProfile - from ..settings.auth import NoAuth - if ( - isinstance(settings_class, type) - and issubclass(settings_class, ConnectionProfile) - ): - descriptor = getattr(settings_class, "__descriptor__", None) - if descriptor is not None and descriptor.auth_modes == [NoAuth]: - kwargs["auth"] = NoAuth() - except (ImportError, AttributeError): - pass - - return settings_class(**kwargs) - - @classmethod - def from_connection_string( - cls, connection_url: str, **kwargs - ) -> MountainAshBaseSettings: - """ - Auto-detect backend from connection URL and create settings. - - Args: - connection_url: Database connection URL - **kwargs: Arguments passed to settings constructor - - Returns: - Settings instance for detected backend - - Raises: - ValueError: If backend cannot be detected from URL - - Example: - settings = SettingsFactory.from_connection_string( - "postgresql://user:pass@localhost:5432/db", - config_files=["postgres.env"] - ) - """ - backend_type = cls.detect_backend_from_url(connection_url) - return cls.from_backend_type(backend_type, **kwargs) - - @classmethod - def detect_backend_from_url(cls, connection_url: str) -> CONST_DB_PROVIDER_TYPE: - """ - Detect backend type from connection URL. - - Args: - connection_url: Database connection URL - - Returns: - Detected backend type - - Raises: - ValueError: If backend cannot be detected - - Example: - backend = SettingsFactory.detect_backend_from_url( - "postgresql://localhost/db" - ) - # Returns: CONST_DB_PROVIDER_TYPE.POSTGRESQL - """ - parsed = urlparse(connection_url) - scheme = parsed.scheme.lower() - - # Check exact scheme match - if scheme in cls.SCHEME_MAP: - return cls.SCHEME_MAP[scheme] - - # Check for special patterns (e.g., duckdb://md: for MotherDuck) - if scheme == "duckdb" and connection_url.startswith("duckdb://md:"): - return CONST_DB_PROVIDER_TYPE.MOTHERDUCK - - # Pattern matching for complex URLs - for pattern, backend_type in cls.SCHEME_MAP.items(): - if re.match(pattern, scheme): - logger.debug( - f"Pattern matched: {scheme} → {backend_type} (pattern: {pattern})" - ) - return backend_type - - raise ValueError( - f"Cannot detect backend from URL scheme: {scheme}. " - f"URL: {connection_url}. " - f"Supported schemes: {list(cls.SCHEME_MAP.keys())}" - ) diff --git a/src/mountainash_data/core/factories/settings_type_factory_mixin.py b/src/mountainash_data/core/factories/settings_type_factory_mixin.py deleted file mode 100644 index ae26d10..0000000 --- a/src/mountainash_data/core/factories/settings_type_factory_mixin.py +++ /dev/null @@ -1,184 +0,0 @@ -""" -Settings Type Detection Mixin for Factory Pattern. - -Detects database backend type from SettingsParameters.settings_class, -similar to DataFrame type detection in mountainash-dataframes. -""" - -import logging -import re -from typing import Optional, Type - -from mountainash_settings import SettingsParameters, MountainAshBaseSettings - -from ..constants import CONST_DB_PROVIDER_TYPE - -logger = logging.getLogger(__name__) - - -class SettingsTypeFactoryMixin: - """ - Mixin for detecting database backend type from settings class. - - Uses three-tier detection system: - 1. Exact Match (fast path): Direct mapping of known settings classes - 2. Pattern Matching (flexible): Regex-based class name matching - 3. Logging: Track unmapped types for future registration - """ - - # Layer 1: Exact Match - Direct settings class mapping (O(1) lookup) - TYPE_MAP = { - # Map settings class → backend type - # These will be populated when settings classes are imported - } - - # Layer 2: Pattern Matching - Class name patterns for flexible detection - PATTERN_MAP = { - # Regex pattern → backend type - r"PostgreSQL.*Settings": CONST_DB_PROVIDER_TYPE.POSTGRESQL, - r"SQLite.*Settings": CONST_DB_PROVIDER_TYPE.SQLITE, - r"DuckDB.*Settings": CONST_DB_PROVIDER_TYPE.DUCKDB, - r"MotherDuck.*Settings": CONST_DB_PROVIDER_TYPE.MOTHERDUCK, - r"Snowflake.*Settings": CONST_DB_PROVIDER_TYPE.SNOWFLAKE, - r"BigQuery.*Settings": CONST_DB_PROVIDER_TYPE.BIGQUERY, - r"MSSQL.*Settings": CONST_DB_PROVIDER_TYPE.MSSQL, - r"MySQL.*Settings": CONST_DB_PROVIDER_TYPE.MYSQL, - r"Trino.*Settings": CONST_DB_PROVIDER_TYPE.TRINO, - r"PySpark.*Settings": CONST_DB_PROVIDER_TYPE.PYSPARK, - r"Redshift.*Settings": CONST_DB_PROVIDER_TYPE.REDSHIFT, - r"Oracle.*Settings": CONST_DB_PROVIDER_TYPE.ORACLE, - r"PyIceberg.*Settings": CONST_DB_PROVIDER_TYPE.PYICEBERG_REST, - } - - @classmethod - def _get_strategy_key( - cls, settings_parameters: SettingsParameters, **kwargs - ) -> Optional[CONST_DB_PROVIDER_TYPE]: - """ - Detect backend type from SettingsParameters. - - Args: - settings_parameters: SettingsParameters with settings_class - **kwargs: Additional context (unused) - - Returns: - Backend type enum or None if detection fails - """ - if settings_parameters is None: - logger.warning("SettingsParameters is None, cannot detect backend") - return None - - settings_class = settings_parameters.settings_class - - if settings_class is None: - logger.warning("settings_class is None in SettingsParameters") - return None - - # Layer 1: Exact Match (Fast Path) - backend_type = cls._detect_from_exact_match(settings_class) - if backend_type: - return backend_type - - # Layer 2: Pattern Matching (Flexible Fallback) - backend_type = cls._detect_from_pattern_match(settings_class) - if backend_type: - # Auto-register for future fast-path lookup - cls._register_settings_class(settings_class, backend_type) - return backend_type - - # Layer 3: Logging (Detection Failed) - cls._log_unmapped_settings_class(settings_class) - return None - - @classmethod - def _detect_from_exact_match( - cls, settings_class: Type[MountainAshBaseSettings] - ) -> Optional[CONST_DB_PROVIDER_TYPE]: - """ - Fast path: Direct lookup in TYPE_MAP. - - Args: - settings_class: Settings class to detect - - Returns: - Backend type or None - """ - return cls.TYPE_MAP.get(settings_class) - - @classmethod - def _detect_from_pattern_match( - cls, settings_class: Type[MountainAshBaseSettings] - ) -> Optional[CONST_DB_PROVIDER_TYPE]: - """ - Flexible fallback: Regex pattern matching on class name. - - Args: - settings_class: Settings class to detect - - Returns: - Backend type or None - """ - class_name = settings_class.__name__ - - for pattern, backend_type in cls.PATTERN_MAP.items(): - if re.match(pattern, class_name): - logger.debug( - f"Pattern matched: {class_name} → {backend_type} (pattern: {pattern})" - ) - return backend_type - - return None - - @classmethod - def _register_settings_class( - cls, - settings_class: Type[MountainAshBaseSettings], - backend_type: CONST_DB_PROVIDER_TYPE, - ) -> None: - """ - Register settings class in TYPE_MAP for future fast-path lookup. - - Args: - settings_class: Settings class to register - backend_type: Detected backend type - """ - cls.TYPE_MAP[settings_class] = backend_type - logger.debug( - f"Auto-registered {settings_class.__name__} → {backend_type} for fast lookup" - ) - - @classmethod - def _log_unmapped_settings_class( - cls, settings_class: Type[MountainAshBaseSettings] - ) -> None: - """ - Log unmapped settings class for debugging. - - Args: - settings_class: Unmapped settings class - """ - class_name = settings_class.__name__ - module_name = settings_class.__module__ - - logger.warning( - f"Unmapped settings class: {class_name} (module: {module_name}). " - f"Consider adding to TYPE_MAP or PATTERN_MAP." - ) - - @classmethod - def register_settings_class_mapping( - cls, - settings_class: Type[MountainAshBaseSettings], - backend_type: CONST_DB_PROVIDER_TYPE, - ) -> None: - """ - Manually register a settings class mapping. - - Useful for custom settings classes or explicit registration. - - Args: - settings_class: Settings class to register - backend_type: Backend type to map to - """ - cls.TYPE_MAP[settings_class] = backend_type - logger.info(f"Manually registered {settings_class.__name__} → {backend_type}") diff --git a/src/mountainash_data/core/protocol.py b/src/mountainash_data/core/protocol.py index b528bc9..26c60fb 100644 --- a/src/mountainash_data/core/protocol.py +++ b/src/mountainash_data/core/protocol.py @@ -1,4 +1,4 @@ -"""Backend and Connection protocols. +"""Backend protocol. This is the structural contract every backend implementation must satisfy. Implementations are plain classes — there is no inheritance. @@ -16,53 +16,26 @@ @t.runtime_checkable -class Connection(t.Protocol): - """A live, owned connection to a backend. +class Backend(t.Protocol): + """The single handle for interacting with a backend service. - Connections are obtained by calling Backend.connect(). They expose - physical introspection and lifecycle methods. Logical query - construction is the job of mountainash-expressions, reached via - to_relation() on backends that support it. + Backends are constructed with config, connected via connect(), + used for inspection and operations, then closed. """ - def list_namespaces(self) -> list[str]: - """Return the names of all namespaces (schemas) visible to this connection.""" - ... + name: str + + def connect(self) -> t.Self: ... + def close(self) -> t.Self: ... + def __enter__(self) -> t.Self: ... + def __exit__(self, *args: t.Any) -> None: ... - def list_tables(self, namespace: str | None = None) -> list[str]: - """Return the names of tables in the given namespace.""" - ... + def list_tables(self, namespace: str | None = None) -> list[str]: ... + def list_namespaces(self) -> list[str]: ... def inspect_table( self, name: str, namespace: str | None = None - ) -> TableInfo: - """Return shared-model metadata for one table.""" - ... - - def inspect_namespace(self, name: str) -> NamespaceInfo: - """Return shared-model metadata for one namespace.""" - ... - - def inspect_catalog(self) -> CatalogInfo: - """Return shared-model metadata for the connection's catalog.""" - ... - - def close(self) -> None: - """Release the connection. Idempotent.""" - ... - - -@t.runtime_checkable -class Backend(t.Protocol): - """A factory for Connections to a particular backend service. - - Backends are constructed with config and are stateless from the - consumer's perspective. State lives on the Connection returned by - connect(). - """ - - name: str + ) -> TableInfo: ... - def connect(self) -> Connection: - """Open a connection. Caller is responsible for closing it.""" - ... + def inspect_namespace(self, name: str) -> NamespaceInfo: ... + def inspect_catalog(self) -> CatalogInfo: ... diff --git a/src/mountainash_data/core/utils.py b/src/mountainash_data/core/utils.py deleted file mode 100644 index de16714..0000000 --- a/src/mountainash_data/core/utils.py +++ /dev/null @@ -1,224 +0,0 @@ -""" -High-level database utilities for settings-driven connection and operations management. - -Provides a unified API for creating database connections and operations -with automatic backend detection and lazy loading. -""" - -import logging -from typing import Any, Optional - -from mountainash_settings import SettingsParameters, MountainAshBaseSettings - -from .connection import BaseDBConnection -from mountainash_data.backends.ibis.operations import BaseIbisOperations -from .constants import CONST_DB_PROVIDER_TYPE -from .factories import ConnectionFactory, OperationsFactory, SettingsFactory - -logger = logging.getLogger(__name__) - - -class DatabaseUtils: - """ - High-level API for settings-driven database operations. - - All operations driven by SettingsParameters, with automatic backend - detection and lazy loading of backend-specific implementations. - - Example: - # Create settings parameters - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - - # Auto-detect and create connection - connection = DatabaseUtils.create_connection(settings_params) - backend = connection.connect() - - # Auto-detect and create operations - operations = DatabaseUtils.create_operations(settings_params) - operations.create_table(backend, "my_table", dataframe) - """ - - @classmethod - def create_connection( - cls, settings_parameters: SettingsParameters - ) -> BaseDBConnection: - """ - Create database connection from settings parameters. - - Auto-detects backend from settings_class and returns appropriate connection. - - Args: - settings_parameters: SettingsParameters with settings_class - - Returns: - Connection instance ready to use - - Example: - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - connection = DatabaseUtils.create_connection(settings_params) - backend = connection.connect() - """ - factory = ConnectionFactory() - return factory.get_connection(settings_parameters) - - @classmethod - def create_operations( - cls, settings_parameters: SettingsParameters - ) -> BaseIbisOperations: - """ - Create database operations from settings parameters. - - Auto-detects backend from settings_class and returns appropriate operations. - - Args: - settings_parameters: SettingsParameters with settings_class - - Returns: - Operations instance ready to use - - Example: - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - operations = DatabaseUtils.create_operations(settings_params) - operations.upsert(backend, "my_table", df, natural_key_columns=["id"]) - """ - factory = OperationsFactory() - return factory.get_operations(settings_parameters) - - @classmethod - def create_backend( - cls, settings_parameters: SettingsParameters, **connect_kwargs - ) -> Any: - """ - Create connection and connect to backend in one step. - - Convenience method combining connection creation and connection. - - Args: - settings_parameters: SettingsParameters with settings_class - **connect_kwargs: Additional arguments passed to connect() - - Returns: - Connected backend (Ibis backend or PyIceberg catalog) - - Example: - settings_params = SettingsParameters.create( - settings_class=PostgreSQLAuthSettings, - config_files=["postgres.env"] - ) - backend = DatabaseUtils.create_backend(settings_params) - tables = backend.list_tables() - """ - connection = cls.create_connection(settings_parameters) - return connection.connect(**connect_kwargs) - - @classmethod - def create_settings_from_url( - cls, connection_url: str, **kwargs - ) -> MountainAshBaseSettings: - """ - Auto-detect backend from URL and create appropriate settings. - - Args: - connection_url: Database connection URL - **kwargs: Arguments passed to settings constructor - - Returns: - Settings instance for detected backend - - Example: - settings = DatabaseUtils.create_settings_from_url( - "postgresql://user:pass@localhost:5432/db", - config_files=["postgres.env"] - ) - """ - return SettingsFactory.from_connection_string(connection_url, **kwargs) - - @classmethod - def create_settings_from_backend_type( - cls, backend_type: CONST_DB_PROVIDER_TYPE, **kwargs - ) -> MountainAshBaseSettings: - """ - Create settings for specific backend type. - - Args: - backend_type: Database backend type enum - **kwargs: Arguments passed to settings constructor - - Returns: - Settings instance for the backend - - Example: - settings = DatabaseUtils.create_settings_from_backend_type( - CONST_DB_PROVIDER_TYPE.POSTGRESQL, - config_files=["postgres.env"] - ) - """ - return SettingsFactory.from_backend_type(backend_type, **kwargs) - - @classmethod - def detect_backend_from_url(cls, connection_url: str) -> CONST_DB_PROVIDER_TYPE: - """ - Detect backend type from connection URL. - - Args: - connection_url: Database connection URL - - Returns: - Detected backend type - - Example: - backend_type = DatabaseUtils.detect_backend_from_url( - "postgresql://localhost/db" - ) - """ - return SettingsFactory.detect_backend_from_url(connection_url) - - @classmethod - def create_from_url( - cls, - connection_url: str, - config_files: Optional[list] = None, - **settings_kwargs, - ) -> tuple[BaseDBConnection, Any]: - """ - Complete workflow: URL → settings → connection → backend. - - Convenience method for quick setup from connection URL. - - Args: - connection_url: Database connection URL - config_files: Optional configuration files - **settings_kwargs: Additional settings constructor arguments - - Returns: - Tuple of (connection, connected_backend) - - Example: - connection, backend = DatabaseUtils.create_from_url( - "postgresql://user:pass@localhost:5432/db", - config_files=["postgres.env"] - ) - tables = backend.list_tables() - """ - # Create settings from URL - settings = cls.create_settings_from_url( - connection_url, config_files=config_files, **settings_kwargs - ) - - # Create settings parameters - settings_params = settings.extract_settings_parameters() - - # Create connection and connect - connection = cls.create_connection(settings_params) - backend = connection.connect() - - return connection, backend diff --git a/tests/test_integration/test_end_to_end_workflows.py b/tests/test_integration/test_end_to_end_workflows.py index 04d0107..3f7c718 100644 --- a/tests/test_integration/test_end_to_end_workflows.py +++ b/tests/test_integration/test_end_to_end_workflows.py @@ -1,288 +1,112 @@ -"""End-to-end integration tests for complete workflows.""" +"""End-to-end integration tests using IbisBackend.""" import pytest -from pathlib import Path -from mountainash_data.core.utils import DatabaseUtils -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings -from mountainash_data.core.constants import CONST_DB_PROVIDER_TYPE +import polars as pl +from mountainash_data.backends.ibis.backend import IbisBackend +from mountainash_data.core.settings import DuckDBAuthSettings from mountainash_settings import SettingsParameters @pytest.mark.integration -class TestCompleteWorkflowFromURL: - """Test complete workflow starting from connection URL.""" - - def test_sqlite_url_to_backend_workflow(self, temp_sqlite_db): - """Test complete SQLite workflow from URL to backend operations.""" - url = f"sqlite:///{temp_sqlite_db}" - - # Complete workflow: URL → settings → connection → backend - connection, backend = DatabaseUtils.create_from_url(url) - - # Verify backend is connected and functional - assert backend is not None - tables = backend.list_tables() - assert isinstance(tables, list) - # Note: test_table from fixture may not be visible in new connection - # Instead, test that we can create and query tables - - # Can create new table - backend.create_table('new_table', {'id': [1, 2, 3]}, overwrite=True) - assert 'new_table' in backend.list_tables() - - # Cleanup - connection.disconnect() - - def test_duckdb_url_to_backend_workflow(self): - """Test complete DuckDB workflow from URL to backend operations.""" - url = "duckdb:///:memory:" - - connection, backend = DatabaseUtils.create_from_url(url) - - # Verify backend is connected and functional - assert backend is not None - - # Can create and query tables - backend.create_table('test', {'id': [1, 2, 3], 'value': [10, 20, 30]}, overwrite=True) - tables = backend.list_tables() - assert 'test' in tables - - # Can query table - table = backend.table('test') - assert table is not None - - # Cleanup - connection.disconnect() - - -@pytest.mark.integration -class TestFactoryDrivenWorkflow: - """Test workflows driven by factory pattern.""" - - def test_settings_factory_to_connection_workflow(self): - """Test Settings Factory → Connection Factory workflow.""" - from mountainash_data.core.factories import SettingsFactory, ConnectionFactory - - # Create settings from backend type - settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:" - ) - - # Extract parameters - settings_params = settings.extract_settings_parameters() - - # Create connection from settings - connection = ConnectionFactory.get_connection(settings_params) - - # Connect and verify - backend = connection.connect() - assert backend is not None - - # Can perform operations - backend.create_table('factory_test', {'id': [1, 2]}, overwrite=True) - assert 'factory_test' in backend.list_tables() - - # Cleanup - connection.disconnect() - - def test_url_detection_to_connection_workflow(self): - """Test URL detection → Settings → Connection workflow.""" - from mountainash_data.core.factories import SettingsFactory, ConnectionFactory - - url = "sqlite:///:memory:" - - # Detect backend type - backend_type = SettingsFactory.detect_backend_from_url(url) - assert backend_type == CONST_DB_PROVIDER_TYPE.SQLITE - - # Create settings from URL - settings = SettingsFactory.from_connection_string(url) - - # Create connection - settings_params = settings.extract_settings_parameters() - connection = ConnectionFactory.get_connection(settings_params) - - # Verify functionality - backend = connection.connect() - assert backend is not None - - # Cleanup - connection.disconnect() - - -@pytest.mark.integration -class TestDatabaseUtilsEndToEnd: - """Test DatabaseUtils high-level API end-to-end.""" - - def test_create_connection_to_operations_workflow(self): - """Test creating connection and performing operations.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - # Create connection - connection = DatabaseUtils.create_connection(settings_params) - - # Connect - backend = connection.connect() - - # Perform operations - backend.create_table('workflow_test', {'id': [1, 2, 3], 'value': [10, 20, 30]}, overwrite=True) - - # Verify - tables = backend.list_tables() - assert 'workflow_test' in tables - - table = backend.table('workflow_test') - assert table is not None - - # Cleanup - connection.disconnect() - - def test_create_backend_shortcut_workflow(self): - """Test create_backend convenience method.""" - settings_params = SettingsParameters.create( +class TestIbisBackendWorkflow: + """Test complete workflows through IbisBackend.""" + + def test_sqlite_dialect_workflow(self): + """Full workflow with dialect= keyword.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("users", {"id": [1, 2], "name": ["a", "b"]}) + assert "users" in backend.list_tables() + tbl = backend.table("users") + assert tbl is not None + + def test_sqlite_url_workflow(self, tmp_path): + """Full workflow from URL.""" + db_file = tmp_path / "test.db" + with IbisBackend(f"sqlite:///{db_file}") as backend: + backend.create_table("t", {"id": [1, 2, 3]}) + assert "t" in backend.list_tables() + assert db_file.exists() + + def test_duckdb_settings_workflow(self): + """Full workflow from SettingsParameters.""" + from mountainash_data.core.settings import NoAuth + params = SettingsParameters.create( settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - # Single call to get connected backend - backend = DatabaseUtils.create_backend(settings_params) - - # Should be ready to use - assert backend is not None - - # Can immediately perform operations - backend.create_table('shortcut_test', {'id': [1, 2]}, overwrite=True) - tables = backend.list_tables() - assert 'shortcut_test' in tables - - -@pytest.mark.integration -@pytest.mark.parametrize("url,expected_backend_type", [ - ("sqlite:///:memory:", CONST_DB_PROVIDER_TYPE.SQLITE), - ("duckdb:///:memory:", CONST_DB_PROVIDER_TYPE.DUCKDB), -]) -class TestCrossBackendWorkflows: - """Test workflows work across different backends.""" - - def test_url_to_backend_works_for_all_backends(self, url, expected_backend_type): - """Test that URL workflow works for different backends.""" - connection, backend = DatabaseUtils.create_from_url(url) - - # Verify correct backend type detected - detected = DatabaseUtils.detect_backend_from_url(url) - assert detected == expected_backend_type - - # Verify backend works - assert backend is not None - backend.create_table('cross_backend_test', {'id': [1, 2, 3]}, overwrite=True) - tables = backend.list_tables() - assert 'cross_backend_test' in tables - - # Cleanup - connection.disconnect() - - def test_backend_type_to_connection_works_for_all_backends(self, url, expected_backend_type): - """Test that backend type workflow works for different backends.""" - settings = DatabaseUtils.create_settings_from_backend_type( - expected_backend_type, - DATABASE=":memory:" + DATABASE=":memory:", + auth=NoAuth(), ) - - settings_params = settings.extract_settings_parameters() - connection = DatabaseUtils.create_connection(settings_params) - backend = connection.connect() - - # Verify backend works - assert backend is not None - backend.create_table('backend_type_test', {'id': [1]}, overwrite=True) - assert 'backend_type_test' in backend.list_tables() - - # Cleanup - connection.disconnect() - - -@pytest.mark.integration -class TestDataTransferWorkflows: - """Test workflows involving data transfer between backends.""" - - def test_create_table_and_query_workflow(self): - """Test creating table and querying it.""" - backend = DatabaseUtils.create_backend( - SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} + with IbisBackend(params) as backend: + backend.create_table("t", {"id": [1, 2]}) + assert "t" in backend.list_tables() + + def test_fluent_chaining_workflow(self): + """Fluent API chaining across multiple operations.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + ( + backend + .create_table("users", {"id": [1], "email": ["a@b.com"]}) + .create_index("users", ["email"], unique=True) + .create_table("orders", {"id": [1], "user_id": [1]}) ) - ) + assert sorted(backend.list_tables()) == ["orders", "users"] - # Create table with data - data = { - 'id': [1, 2, 3, 4, 5], - 'name': ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'], - 'value': [100.5, 200.7, 300.9, 400.2, 500.8] - } + def test_inspect_workflow(self): + """Inspection methods work through the backend.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + info = backend.inspect_table("t") + assert info.name == "t" + assert len(info.columns) == 2 - backend.create_table('data_transfer_test', data, overwrite=True) + def test_ibis_connection_seam(self): + """ibis_connection() provides the seam to mountainash-expressions.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2]}) + raw = backend.ibis_connection() + tbl = raw.table("t") + assert tbl is not None - # Query table - table = backend.table('data_transfer_test') - assert table is not None - # Verify data - result = table.execute() - assert len(result) == 5 +@pytest.mark.integration +class TestDuckDBOperationsWorkflow: + """Test DuckDB-specific operations (upsert, indexes) through IbisBackend.""" - def test_insert_and_retrieve_workflow(self): - """Test inserting and retrieving data.""" - backend = DatabaseUtils.create_backend( - SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - ) + def test_upsert_insert_new_rows(self): + """Upsert inserts new rows when no conflicts exist.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"]}) + backend.create_table("users", initial) + backend.create_unique_index("users", ["id"]) - # Create empty table structure - backend.create_table('insert_test', {'id': [1], 'value': [10]}, overwrite=True) + new_data = pl.DataFrame({"id": [3, 4], "name": ["Charlie", "Diana"]}) + backend.upsert("users", new_data, conflict_columns=["id"]) - # Insert more data - backend.insert('insert_test', {'id': [2, 3], 'value': [20, 30]}) + count = backend.run_sql("SELECT COUNT(*) as cnt FROM users").to_polars()["cnt"][0] + assert count == 4 - # Retrieve and verify - table = backend.table('insert_test') - assert table is not None + def test_upsert_update_existing_rows(self): + """Upsert updates existing rows on conflict.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"], "score": [100, 200]}) + backend.create_table("users", initial) + backend.create_unique_index("users", ["id"]) + update = pl.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"], "score": [150, 250]}) + backend.upsert("users", update, conflict_columns=["id"]) -@pytest.mark.integration -class TestErrorHandlingWorkflows: - """Test error handling in complete workflows.""" + result = backend.run_sql("SELECT score FROM users ORDER BY id").to_polars() + assert list(result["score"]) == [150, 250] - def test_invalid_url_workflow(self): - """Test that invalid URL is handled appropriately.""" - with pytest.raises((ValueError, KeyError, AttributeError, NotImplementedError)): - DatabaseUtils.create_from_url("invalid://url") + def test_index_lifecycle(self): + """Create, check, list, drop index.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2], "name": ["a", "b"]}) - @pytest.mark.skip(reason="Settings classes have optional/default values") - def test_missing_required_config_workflow(self): - """Test that missing config is handled.""" - # NOTE: Settings have default values, so this doesn't raise - with pytest.raises((ValueError, KeyError, TypeError)): - SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={} # Missing DATABASE - ) + backend.create_index("t", ["name"], index_name="idx_name") + assert backend.index_exists("idx_name") is True - def test_nonexistent_table_query_workflow(self): - """Test querying non-existent table is handled.""" - backend = DatabaseUtils.create_backend( - SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - ) + indexes = backend.list_indexes("t") + assert any(idx.get("name") == "idx_name" for idx in indexes) - # Try to query non-existent table - with pytest.raises((Exception, AttributeError)): - backend.table('definitely_does_not_exist').execute() + backend.drop_index("idx_name") + assert backend.index_exists("idx_name") is False diff --git a/tests/test_unit/backends/ibis/test_backend.py b/tests/test_unit/backends/ibis/test_backend.py index 9dee07d..458c9ba 100644 --- a/tests/test_unit/backends/ibis/test_backend.py +++ b/tests/test_unit/backends/ibis/test_backend.py @@ -1,6 +1,7 @@ """Tests for IbisBackend factory.""" import pytest +import polars as pl from mountainash_data.backends.ibis.backend import IbisBackend from mountainash_data.backends.ibis.dialects._registry import DIALECTS @@ -27,9 +28,332 @@ def test_all_registered_dialects_construct(): def test_in_memory_sqlite_connect_and_inspect(): """End-to-end test with the only dialect that needs no external service.""" backend = IbisBackend(dialect="sqlite", database=":memory:") - conn = backend.connect() + backend.connect() try: - # Should expose protocol methods even if no tables exist - assert conn.list_tables() == [] + assert backend.list_tables() == [] finally: - conn.close() + backend.close() + + +def test_neither_positional_nor_dialect_raises(): + """Constructor with no arguments must raise ValueError.""" + with pytest.raises(ValueError, match="Either.*or.*dialect"): + IbisBackend() + + +def test_both_positional_and_dialect_raises(): + """Cannot supply both a positional arg and dialect= keyword.""" + with pytest.raises(ValueError, match="Cannot specify both"): + IbisBackend("sqlite://", dialect="sqlite") + + +def test_unknown_url_scheme_raises(): + """URL with unrecognised scheme must raise ValueError.""" + with pytest.raises(ValueError, match="Cannot detect ibis dialect"): + IbisBackend("nosuch://localhost/db") + + +# --------------------------------------------------------------------------- +# Settings path +# --------------------------------------------------------------------------- + +def test_settings_path_sqlite(): + """Construct IbisBackend from SQLite SettingsParameters and connect.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import SQLiteAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=SQLiteAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "sqlite" + backend.connect() + tables = backend.list_tables() + assert isinstance(tables, list) + backend.close() + + +def test_settings_path_duckdb_empty_extensions(): + """DuckDB settings with default EXTENSIONS=[] must not crash ibis.""" + from mountainash_settings import SettingsParameters + from mountainash_data.core.settings import DuckDBAuthSettings, NoAuth + + params = SettingsParameters.create( + settings_class=DuckDBAuthSettings, + DATABASE=":memory:", + auth=NoAuth(), + ) + backend = IbisBackend(params) + assert backend.dialect == "duckdb" + backend.connect() + backend.close() + + +# --------------------------------------------------------------------------- +# URL path +# --------------------------------------------------------------------------- + +def test_url_path_sqlite(): + """Construct IbisBackend from sqlite:// URL and connect.""" + backend = IbisBackend("sqlite://") + assert backend.dialect == "sqlite" + backend.connect() + backend.close() + + +def test_url_path_duckdb(): + """Construct IbisBackend from duckdb:// URL and connect.""" + backend = IbisBackend("duckdb://") + assert backend.dialect == "duckdb" + backend.connect() + backend.close() + + +def test_url_path_preserves_database(tmp_path): + """URL database component must reach the driver, not be discarded.""" + db_file = tmp_path / "test.db" + backend = IbisBackend(f"sqlite:///{db_file}") + assert backend.dialect == "sqlite" + backend.connect() + backend.close() + assert db_file.exists() + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + +def test_connect_returns_self(): + """connect() must return the backend instance itself.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + result = backend.connect() + assert result is backend + + +def test_close_returns_self(): + """close() must return the backend instance itself.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + result = backend.close() + assert result is backend + + +def test_context_manager(): + """with IbisBackend(...) as backend: must connect and close.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + assert backend.list_tables() == [] + # After exit, should be closed + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_double_close_is_idempotent(): + """Calling close() twice must not raise.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + backend.close() + backend.close() # Must not raise + + +def test_use_before_connect_raises(): + """Calling methods before connect() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_use_after_close_raises(): + """Calling methods after close() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + backend.connect() + backend.close() + with pytest.raises(RuntimeError, match="not connected"): + backend.list_tables() + + +def test_ibis_connection_accessor(): + """ibis_connection() returns the raw ibis backend object.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + raw = backend.ibis_connection() + assert hasattr(raw, "list_tables") + + +def test_ibis_connection_before_connect_raises(): + """ibis_connection() before connect() must raise RuntimeError.""" + backend = IbisBackend(dialect="sqlite", database=":memory:") + with pytest.raises(RuntimeError, match="not connected"): + backend.ibis_connection() + + +def test_get_connection_accessor(): + """get_connection() returns our IbisConnection wrapper.""" + from mountainash_data.backends.ibis.backend import IbisConnection + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + conn = backend.get_connection() + assert isinstance(conn, IbisConnection) + + +# --------------------------------------------------------------------------- +# DialectSpec hooks +# --------------------------------------------------------------------------- + +def test_duckdb_dialect_has_upsert_hook(): + """DuckDB DialectSpec must have upsert_hook wired.""" + spec = DIALECTS["duckdb"] + assert spec.upsert_hook is not None + + +def test_sqlite_dialect_has_create_index_hook(): + """SQLite DialectSpec must have create_index_hook wired.""" + spec = DIALECTS["sqlite"] + assert spec.create_index_hook is not None + + +def test_postgres_dialect_has_no_upsert_hook(): + """Postgres DialectSpec has no upsert_hook (not DuckDB family).""" + spec = DIALECTS["postgres"] + assert spec.upsert_hook is None + + +# --------------------------------------------------------------------------- +# Thin wrapper operations (fluent) +# --------------------------------------------------------------------------- + +def test_create_table_returns_self(): + """create_table() must return self for fluent chaining.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + result = backend.create_table("t", {"id": [1, 2]}) + assert result is backend + assert "t" in backend.list_tables() + + +def test_drop_table_returns_self(): + """drop_table() must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.drop_table("t") + assert result is backend + assert "t" not in backend.list_tables() + + +def test_insert_returns_self(): + """insert() must return self.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.insert("t", {"id": [2]}) + assert result is backend + + +def test_truncate_returns_self(): + """truncate() must return self.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + result = backend.truncate("t") + assert result is backend + + +def test_table_returns_ibis_table(): + """table() must return an ibis table expression.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2]}) + tbl = backend.table("t") + assert tbl is not None + + +def test_run_sql_returns_result(): + """run_sql() must return an ibis table expression.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1, 2, 3]}) + result = backend.run_sql("SELECT COUNT(*) as cnt FROM t") + assert result is not None + + +def test_table_exists_returns_bool(): + """table_exists() must return True/False.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + assert backend.table_exists("nope") is False + backend.create_table("t", {"id": [1]}) + assert backend.table_exists("t") is True + + +def test_fluent_chaining(): + """Multiple fluent calls can be chained.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("a", {"id": [1]}).create_table("b", {"id": [2]}) + assert sorted(backend.list_tables()) == ["a", "b"] + + +# --------------------------------------------------------------------------- +# Hook-dispatched operations +# --------------------------------------------------------------------------- + +def test_create_index_returns_self(): + """create_index() via hook must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + result = backend.create_index("t", ["name"]) + assert result is backend + + +def test_create_unique_index_returns_self(): + """create_unique_index() must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + result = backend.create_unique_index("t", ["name"]) + assert result is backend + + +def test_drop_index_returns_self(): + """drop_index() via hook must return self.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + backend.create_index("t", ["name"], index_name="idx_name") + result = backend.drop_index("idx_name") + assert result is backend + + +def test_index_exists(): + """index_exists() must detect created indexes.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1]}) + backend.create_index("t", ["id"], index_name="idx_id") + assert backend.index_exists("idx_id") is True + assert backend.index_exists("no_such_idx") is False + + +def test_list_indexes(): + """list_indexes() must return index info.""" + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", {"id": [1], "name": ["a"]}) + backend.create_index("t", ["id"], index_name="idx_id") + indexes = backend.list_indexes("t") + assert isinstance(indexes, list) + assert len(indexes) >= 1 + + +def test_upsert_duckdb(): + """upsert() must work on DuckDB via hook.""" + with IbisBackend(dialect="duckdb", database=":memory:") as backend: + initial = pl.DataFrame({"id": [1, 2], "val": [10, 20]}) + backend.create_table("t", initial) + backend.create_unique_index("t", ["id"]) + + update = pl.DataFrame({"id": [2, 3], "val": [25, 30]}) + result = backend.upsert("t", update, conflict_columns=["id"]) + assert result is backend + + count_result = backend.run_sql("SELECT COUNT(*) as cnt FROM t") + count = count_result.to_polars()["cnt"][0] + assert count == 3 + + +def test_upsert_unsupported_dialect_raises(): + """upsert() on a dialect without upsert_hook must raise NotImplementedError.""" + backend = IbisBackend(dialect="postgres") + # Can't actually connect to postgres, so mock the connection state + from mountainash_data.backends.ibis.backend import IbisConnection + backend._conn = IbisConnection(None, DIALECTS["postgres"]) + with pytest.raises(NotImplementedError, match="does not support upsert"): + backend.upsert("t", {}, conflict_columns=["id"]) diff --git a/tests/test_unit/core/test_protocol.py b/tests/test_unit/core/test_protocol.py index 33f91ca..5e6826f 100644 --- a/tests/test_unit/core/test_protocol.py +++ b/tests/test_unit/core/test_protocol.py @@ -14,7 +14,7 @@ NamespaceInfo, TableInfo, ) -from mountainash_data.core.protocol import Backend, Connection +from mountainash_data.core.protocol import Backend class _FakeConnection: @@ -57,7 +57,7 @@ def test_fake_backend_satisfies_protocol(): def test_fake_connection_satisfies_protocol(): - conn: Connection = _FakeConnection() + conn = _FakeConnection() assert conn.list_namespaces() == ["public"] diff --git a/tests/test_unit/databases/connections/ibis/test_connection_lifecycle.py b/tests/test_unit/databases/connections/ibis/test_connection_lifecycle.py deleted file mode 100644 index 83f5344..0000000 --- a/tests/test_unit/databases/connections/ibis/test_connection_lifecycle.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Parametrized tests for Ibis connection lifecycle across backends.""" - -import pytest -from mountainash_data.backends.ibis.connection import ( - SQLite_IbisConnection, - DuckDB_IbisConnection, -) -from mountainash_data.core.connection import BaseDBConnection -from mountainash_data.core.settings import ( - SQLiteAuthSettings, - DuckDBAuthSettings, - NoAuth, -) -from mountainash_settings import SettingsParameters - - -@pytest.mark.unit -@pytest.mark.parametrize("connection_class,settings_class,db_config", [ - (SQLite_IbisConnection, SQLiteAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), - (DuckDB_IbisConnection, DuckDBAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), -]) -class TestConnectionLifecycle: - """Test connection lifecycle for all Ibis backends.""" - - def test_connection_instantiation(self, connection_class, settings_class, db_config): - """Test that connection can be instantiated.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - - assert connection is not None - assert isinstance(connection, BaseDBConnection) - assert isinstance(connection, connection_class) - - def test_connection_not_connected_initially(self, connection_class, settings_class, db_config): - """Test that connection is not connected initially.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - - # Initially should not be connected - assert not connection.is_connected() - - def test_connect_method(self, connection_class, settings_class, db_config): - """Test that connect method works.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - backend = connection.connect() - - assert backend is not None - assert connection.is_connected() - - # Cleanup - connection.disconnect() - - def test_disconnect_method(self, connection_class, settings_class, db_config): - """Test that disconnect method works.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - connection.connect() - connection.disconnect() - - assert not connection.is_connected() - - def test_reconnect_after_disconnect(self, connection_class, settings_class, db_config): - """Test that connection can reconnect after disconnect.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - - # First connection - backend1 = connection.connect() - assert connection.is_connected() - - # Disconnect - connection.disconnect() - assert not connection.is_connected() - - # Reconnect - backend2 = connection.connect() - assert connection.is_connected() - assert backend2 is not None - - # Cleanup - connection.disconnect() - - def test_multiple_connect_calls(self, connection_class, settings_class, db_config): - """Test that multiple connect calls don't cause errors.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - - # Connect multiple times - backend1 = connection.connect() - backend2 = connection.connect() - - assert backend1 is not None - assert backend2 is not None - assert connection.is_connected() - - # Cleanup - connection.disconnect() - - -@pytest.mark.integration -@pytest.mark.parametrize("connection_class,settings_class,db_config", [ - (SQLite_IbisConnection, SQLiteAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), - (DuckDB_IbisConnection, DuckDBAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), -]) -class TestConnectionFunctionality: - """Test actual backend functionality for all Ibis backends.""" - - def test_backend_can_create_table(self, connection_class, settings_class, db_config): - """Test that connected backend can create tables.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - backend = connection.connect() - - # Create a simple table - backend.create_table("test_table", {"id": [1, 2, 3], "value": [10, 20, 30]}, overwrite=True) - - # Verify table exists - tables = backend.list_tables() - assert "test_table" in tables - - # Cleanup - connection.disconnect() - - def test_backend_can_list_tables(self, connection_class, settings_class, db_config): - """Test that connected backend can list tables.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - backend = connection.connect() - - # Create some tables - backend.create_table("table1", {"id": [1]}, overwrite=True) - backend.create_table("table2", {"id": [2]}, overwrite=True) - - # List tables - tables = backend.list_tables() - - assert isinstance(tables, list) - assert "table1" in tables - assert "table2" in tables - - # Cleanup - connection.disconnect() - - def test_backend_can_query_table(self, connection_class, settings_class, db_config): - """Test that connected backend can query tables.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = connection_class(db_auth_settings_parameters=settings_params) - backend = connection.connect() - - # Create table with data - backend.create_table("query_test", {"id": [1, 2, 3], "value": [10, 20, 30]}, overwrite=True) - - # Query table - table = backend.table("query_test") - assert table is not None - - # Cleanup - connection.disconnect() - - -@pytest.mark.unit -@pytest.mark.parametrize("settings_class,db_config", [ - (SQLiteAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), - (DuckDBAuthSettings, {"DATABASE": ":memory:", "auth": NoAuth()}), -]) -class TestConnectionWithFactory: - """Test connections work with factory pattern.""" - - def test_factory_creates_correct_connection(self, settings_class, db_config): - """Test that factory creates correct connection type.""" - from mountainash_data.core.factories import ConnectionFactory - - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = ConnectionFactory.get_connection(settings_params) - - assert connection is not None - assert isinstance(connection, BaseDBConnection) - - def test_factory_connection_can_connect(self, settings_class, db_config): - """Test that factory-created connection can connect.""" - from mountainash_data.core.factories import ConnectionFactory - - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs=db_config - ) - - connection = ConnectionFactory.get_connection(settings_params) - backend = connection.connect() - - assert backend is not None - assert connection.is_connected() - - # Cleanup - connection.disconnect() diff --git a/tests/test_unit/databases/connections/test_base_db_connection.py b/tests/test_unit/databases/connections/test_base_db_connection.py deleted file mode 100644 index 8b9b7af..0000000 --- a/tests/test_unit/databases/connections/test_base_db_connection.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Tests for BaseDBConnection abstract base class.""" - -import pytest -from mountainash_data.core.connection import BaseDBConnection - - -@pytest.mark.unit -class TestBaseDBConnection: - """Tests for BaseDBConnection abstract base class.""" - - def test_base_db_connection_is_abstract(self): - """Test that BaseDBConnection cannot be instantiated directly.""" - with pytest.raises(TypeError, match="Can't instantiate abstract class"): - BaseDBConnection() - - def test_base_db_connection_has_required_abstract_methods(self): - """Test that BaseDBConnection defines required abstract methods.""" - assert hasattr(BaseDBConnection, 'connect') - assert hasattr(BaseDBConnection, 'disconnect') - assert hasattr(BaseDBConnection, 'is_connected') - - def test_base_db_connection_has_db_backend_property(self): - """Test that BaseDBConnection has db_backend_name property.""" - assert hasattr(BaseDBConnection, 'db_backend_name') - - def test_base_db_connection_has_settings_parameter(self): - """Test that BaseDBConnection expects db_auth_settings_parameters.""" - # This is verified by the abstract __init__ signature - import inspect - sig = inspect.signature(BaseDBConnection.__init__) - assert 'db_auth_settings_parameters' in sig.parameters diff --git a/tests/test_unit/databases/operations/ibis/test_base_ibis_operations.py b/tests/test_unit/databases/operations/ibis/test_base_ibis_operations.py deleted file mode 100644 index 18abc61..0000000 --- a/tests/test_unit/databases/operations/ibis/test_base_ibis_operations.py +++ /dev/null @@ -1,333 +0,0 @@ -"""Tests for BaseIbisOperations.""" - -import pytest -import ibis -from mountainash_data.backends.ibis.operations import BaseIbisOperations -from mountainash_data.core.utils import DatabaseUtils -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings -from mountainash_settings import SettingsParameters - - -@pytest.mark.unit -class TestBaseIbisOperationsInstantiation: - """Tests for BaseIbisOperations instantiation.""" - - def test_base_ibis_operations_is_abstract(self): - """Test that BaseIbisOperations cannot be instantiated directly.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - with pytest.raises(TypeError): - BaseIbisOperations(db_auth_settings_parameters=settings_params) - - -@pytest.mark.integration -class TestIbisOperationsRunSQL: - """Tests for run_sql method.""" - - def test_run_sql_simple_query(self, sqlite_settings_params, sample_table_data): - """Test running a simple SQL query.""" - # Create backend and operations - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create a test table - backend.create_table("test_sql", sample_table_data["simple"], overwrite=True) - - # Run SQL query - result = operations.run_sql(backend, "SELECT * FROM test_sql") - - assert result is not None - - def test_run_sql_with_where_clause(self, sqlite_settings_params, sample_table_data): - """Test SQL query with WHERE clause.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - backend.create_table("test_where", sample_table_data["simple"], overwrite=True) - - result = operations.run_sql(backend, "SELECT * FROM test_where WHERE id > 1") - - assert result is not None - - -@pytest.mark.integration -class TestIbisOperationsTable: - """Tests for table method.""" - - def test_table_retrieval(self, sqlite_settings_params, sample_table_data): - """Test retrieving a table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create test table - backend.create_table("test_retrieve", sample_table_data["simple"], overwrite=True) - - # Retrieve table - table = operations.table(backend, "test_retrieve") - - assert table is not None - - def test_table_nonexistent(self, sqlite_settings_params): - """Test retrieving non-existent table returns None.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Try to retrieve non-existent table - table = operations.table(backend, "nonexistent_table") - - assert table is None - - -@pytest.mark.integration -class TestIbisOperationsCreateTable: - """Tests for create_table method.""" - - def test_create_table_from_dict(self, sqlite_settings_params): - """Test creating table from dictionary data.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - data = {"id": [1, 2, 3], "value": [10, 20, 30]} - - operations.create_table(backend, "test_create", data, overwrite=True) - - # Verify table exists - assert operations.table_exists(backend, "test_create") - - def test_create_table_overwrite(self, sqlite_settings_params): - """Test creating table with overwrite=True.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - data1 = {"id": [1, 2], "value": [10, 20]} - data2 = {"id": [3, 4, 5], "value": [30, 40, 50]} - - # Create first table - operations.create_table(backend, "test_overwrite", data1, overwrite=True) - - # Overwrite with new data - operations.create_table(backend, "test_overwrite", data2, overwrite=True) - - # Verify table exists - assert operations.table_exists(backend, "test_overwrite") - - def test_create_table_temp(self, sqlite_settings_params): - """Test creating temporary table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - data = {"id": [1, 2, 3]} - - operations.create_table(backend, "test_temp", data, temp=True, overwrite=True) - - # Verify table exists (temp tables should be queryable in same session) - assert operations.table_exists(backend, "test_temp") or True # Some backends may not list temp tables - - -@pytest.mark.integration -class TestIbisOperationsDropTable: - """Tests for drop_table method.""" - - def test_drop_existing_table(self, sqlite_settings_params): - """Test dropping an existing table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create table - backend.create_table("test_drop", {"id": [1, 2, 3]}, overwrite=True) - - # Drop table - result = operations.drop_table(backend, "test_drop") - - assert result is True - assert not operations.table_exists(backend, "test_drop") - - def test_drop_nonexistent_table(self, sqlite_settings_params): - """Test dropping non-existent table returns False.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - result = operations.drop_table(backend, "nonexistent_drop") - - assert result is False - - -@pytest.mark.integration -class TestIbisOperationsListTables: - """Tests for list_tables method.""" - - def test_list_tables_empty(self, sqlite_memory_settings_params): - """Test listing tables in empty database.""" - backend = DatabaseUtils.create_backend(sqlite_memory_settings_params) - operations = DatabaseUtils.create_operations(sqlite_memory_settings_params) - - tables = operations.list_tables(backend) - - assert isinstance(tables, list) - - def test_list_tables_with_tables(self, sqlite_memory_settings_params): - """Test listing tables when tables exist.""" - backend = DatabaseUtils.create_backend(sqlite_memory_settings_params) - operations = DatabaseUtils.create_operations(sqlite_memory_settings_params) - - # Create some tables - backend.create_table("table1", {"id": [1, 2]}, overwrite=True) - backend.create_table("table2", {"id": [3, 4]}, overwrite=True) - - tables = operations.list_tables(backend) - - assert "table1" in tables - assert "table2" in tables - - -@pytest.mark.integration -class TestIbisOperationsTableExists: - """Tests for table_exists method.""" - - def test_table_exists_true(self, sqlite_settings_params): - """Test table_exists returns True for existing table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - backend.create_table("test_exists", {"id": [1]}, overwrite=True) - - assert operations.table_exists(backend, "test_exists") is True - - def test_table_exists_false(self, sqlite_settings_params): - """Test table_exists returns False for non-existent table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - assert operations.table_exists(backend, "definitely_not_exists") is False - - -@pytest.mark.integration -class TestIbisOperationsInsert: - """Tests for insert method.""" - - def test_insert_data(self, sqlite_settings_params): - """Test inserting data into table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create table - backend.create_table("test_insert", {"id": [1], "value": [10]}, overwrite=True) - - # Insert more data - new_data = {"id": [2, 3], "value": [20, 30]} - result = operations.insert(backend, "test_insert", new_data) - - assert result is True - - def test_insert_overwrite(self, sqlite_settings_params): - """Test insert with overwrite=True.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create table with initial data - backend.create_table("test_insert_overwrite", {"id": [1, 2]}, overwrite=True) - - # Insert with overwrite - new_data = {"id": [3, 4, 5]} - result = operations.insert(backend, "test_insert_overwrite", new_data, overwrite=True) - - assert result is True - - -@pytest.mark.integration -class TestIbisOperationsTruncate: - """Tests for truncate method.""" - - def test_truncate_table(self, sqlite_settings_params): - """Test truncating a table.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create table with data - backend.create_table("test_truncate", {"id": [1, 2, 3], "value": [10, 20, 30]}, overwrite=True) - - # Truncate (may not be supported by all backends) - try: - operations.truncate(backend, "test_truncate") - # If successful, table should exist but be empty - assert operations.table_exists(backend, "test_truncate") - except (NotImplementedError, Exception): - # Some backends may not support truncate - pytest.skip("Truncate not supported by this backend") - - -@pytest.mark.integration -class TestIbisOperationsViews: - """Tests for view operations.""" - - def test_create_view(self, sqlite_settings_params, sample_table_data): - """Test creating a view.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create base table - backend.create_table("base_for_view", sample_table_data["simple"], overwrite=True) - - # Get table expression - table_expr = operations.table(backend, "base_for_view") - - # Create view - view = operations.create_view(backend, "test_view", table_expr, overwrite=True) - - # View should be created (result may vary by backend) - assert view is not None or True - - def test_drop_view(self, sqlite_settings_params, sample_table_data): - """Test dropping a view.""" - backend = DatabaseUtils.create_backend(sqlite_settings_params) - operations = DatabaseUtils.create_operations(sqlite_settings_params) - - # Create base table and view - backend.create_table("base_for_drop_view", sample_table_data["simple"], overwrite=True) - table_expr = operations.table(backend, "base_for_drop_view") - operations.create_view(backend, "view_to_drop", table_expr, overwrite=True) - - # Drop view - result = operations.drop_view(backend, "view_to_drop") - - # Result depends on backend support - assert isinstance(result, bool) - - -@pytest.mark.integration -@pytest.mark.parametrize("backend_settings", [ - "sqlite_memory_settings_params", - "duckdb_settings_params", -]) -class TestIbisOperationsMultiBackend: - """Test operations work across different backends.""" - - def test_create_and_list_tables_multi_backend(self, backend_settings, request): - """Test basic operations work on multiple backends.""" - settings = request.getfixturevalue(backend_settings) - - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - backend.create_table("multi_test", {"id": [1, 2, 3]}, overwrite=True) - - # List tables - tables = operations.list_tables(backend) - - assert "multi_test" in tables - - def test_table_exists_multi_backend(self, backend_settings, request): - """Test table_exists works on multiple backends.""" - settings = request.getfixturevalue(backend_settings) - - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - backend.create_table("exists_test", {"id": [1]}, overwrite=True) - - assert operations.table_exists(backend, "exists_test") is True - assert operations.table_exists(backend, "not_exists") is False diff --git a/tests/test_unit/databases/operations/test_upsert_and_indexes.py b/tests/test_unit/databases/operations/test_upsert_and_indexes.py deleted file mode 100644 index 6660755..0000000 --- a/tests/test_unit/databases/operations/test_upsert_and_indexes.py +++ /dev/null @@ -1,626 +0,0 @@ -"""Comprehensive tests for upsert and index management operations. - -Tests cover: -- Upsert operations (various scenarios) -- Index creation, deletion, and querying -- Cross-database compatibility (DuckDB, SQLite) -""" - -import pytest -import polars as pl -from mountainash_data.core.utils import DatabaseUtils -from mountainash_data.core.constants import CONST_CONFLICT_ACTION - - -@pytest.mark.integration -@pytest.mark.parametrize("backend_fixture", [ - "sqlite_memory_settings_params", - "duckdb_settings_params", -]) -class TestUpsertOperations: - """Tests for upsert operations across different scenarios.""" - - def test_simple_upsert_insert_new_rows(self, backend_fixture, request): - """Test upsert inserts new rows when no conflicts exist.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table - initial_data = pl.DataFrame({ - "id": [1, 2], - "email": ["alice@example.com", "bob@example.com"], - "name": ["Alice", "Bob"] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - - # Create unique index on email - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert with new data (no conflicts) - new_data = pl.DataFrame({ - "id": [3, 4], - "email": ["charlie@example.com", "diana@example.com"], - "name": ["Charlie", "Diana"] - }) - operations.upsert( - backend, "users", new_data, - conflict_columns=["email"] - ) - - # Verify all 4 rows exist - result = operations.run_sql(backend, "SELECT COUNT(*) as count FROM users") - count = result.to_polars()["count"][0] - assert count == 4 - - def test_simple_upsert_update_existing_rows(self, backend_fixture, request): - """Test upsert updates existing rows on conflict.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table with index - initial_data = pl.DataFrame({ - "id": [1, 2], - "email": ["alice@example.com", "bob@example.com"], - "name": ["Alice", "Bob"], - "score": [100, 200] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert with conflicting emails but different data - update_data = pl.DataFrame({ - "id": [1, 2], - "email": ["alice@example.com", "bob@example.com"], - "name": ["Alice Updated", "Bob Updated"], - "score": [150, 250] - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"] - ) - - # Verify rows were updated - result = operations.run_sql(backend, "SELECT * FROM users ORDER BY id") - df = result.to_polars() - assert len(df) == 2 - # Names should be updated - names = df["name"].to_list() - assert "Alice Updated" in names - assert "Bob Updated" in names - - def test_upsert_mixed_insert_and_update(self, backend_fixture, request): - """Test upsert handles both inserts and updates in same operation.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table - initial_data = pl.DataFrame({ - "email": ["alice@example.com", "bob@example.com"], - "name": ["Alice", "Bob"] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert with mix of existing and new emails - mixed_data = pl.DataFrame({ - "email": ["alice@example.com", "charlie@example.com"], - "name": ["Alice Updated", "Charlie"] - }) - operations.upsert( - backend, "users", mixed_data, - conflict_columns=["email"] - ) - - # Verify: 3 rows total (alice updated, bob unchanged, charlie added) - result = operations.run_sql(backend, "SELECT COUNT(*) as count FROM users") - count = result.to_polars()["count"][0] - assert count == 3 - - # Verify Alice was updated - result = operations.run_sql(backend, "SELECT name FROM users WHERE email = 'alice@example.com'") - alice_name = result.to_polars()["name"][0] - assert alice_name == "Alice Updated" - - def test_upsert_with_conflict_action_nothing(self, backend_fixture, request): - """Test upsert with DO NOTHING ignores conflicts.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table - initial_data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["Alice"], - "score": [100] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert with DO NOTHING - should skip conflicting row - conflict_data = pl.DataFrame({ - "email": ["alice@example.com", "bob@example.com"], - "name": ["Alice Updated", "Bob"], - "score": [999, 200] - }) - operations.upsert( - backend, "users", conflict_data, - conflict_columns=["email"], - conflict_action=CONST_CONFLICT_ACTION.NOTHING - ) - - # Verify: Alice unchanged (still 100), Bob added - result = operations.run_sql(backend, "SELECT email, score FROM users ORDER BY email") - df = result.to_polars() - assert len(df) == 2 - # Alice should still have original score - alice_score = df.filter(pl.col("email") == "alice@example.com")["score"][0] - assert alice_score == 100 - - def test_upsert_with_specific_update_columns(self, backend_fixture, request): - """Test upsert updates only specified columns.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table - initial_data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["Alice"], - "score": [100], - "status": ["active"] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert updating only score (not name or status) - update_data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["SHOULD NOT UPDATE"], - "score": [200], - "status": ["SHOULD NOT UPDATE"] - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"], - update_columns=["score"] # Only update score - ) - - # Verify: score updated, name and status unchanged - result = operations.run_sql(backend, "SELECT name, score, status FROM users WHERE email = 'alice@example.com'") - df = result.to_polars() - assert df["name"][0] == "Alice" # name unchanged - assert df["score"][0] == 200 # score updated - assert df["status"][0] == "active" # status unchanged - - def test_upsert_with_conditional_update(self, backend_fixture, request): - """Test upsert with conditional WHERE clause on update.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create initial table - initial_data = pl.DataFrame({ - "email": ["alice@example.com", "bob@example.com"], - "last_login": [100, 50] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert with condition: only update if new last_login is greater - update_data = pl.DataFrame({ - "email": ["alice@example.com", "bob@example.com"], - "last_login": [90, 75] # Alice: 90 < 100 (no update), Bob: 75 > 50 (update) - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"], - update_columns=["last_login"], - update_condition="users.last_login < EXCLUDED.last_login" - ) - - # Verify: Alice unchanged (100), Bob updated (75) - result = operations.run_sql(backend, "SELECT email, last_login FROM users ORDER BY email") - df = result.to_polars() - alice_login = df.filter(pl.col("email") == "alice@example.com")["last_login"][0] - bob_login = df.filter(pl.col("email") == "bob@example.com")["last_login"][0] - assert alice_login == 100 # Not updated (condition failed) - assert bob_login == 75 # Updated (condition passed) - - def test_upsert_composite_conflict_columns(self, backend_fixture, request): - """Test upsert with multiple conflict columns (composite key).""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table with composite key - initial_data = pl.DataFrame({ - "user_id": [1, 1, 2], - "product_id": [100, 101, 100], - "quantity": [5, 3, 7] - }) - operations.create_table(backend, "cart", initial_data, overwrite=True) - operations.create_unique_index(backend, "cart", ["user_id", "product_id"]) - - # Upsert with composite conflict - update_data = pl.DataFrame({ - "user_id": [1, 2, 2], - "product_id": [100, 100, 102], - "quantity": [10, 15, 2] - }) - operations.upsert( - backend, "cart", update_data, - conflict_columns=["user_id", "product_id"] - ) - - # Verify: user 1/product 100 updated, user 2/product 100 updated, user 2/product 102 added - result = operations.run_sql(backend, "SELECT user_id, product_id, quantity FROM cart ORDER BY user_id, product_id") - df = result.to_polars() - assert len(df) == 4 - - def test_upsert_auto_column_detection(self, backend_fixture, request): - """Test upsert auto-detects update columns when not specified.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - initial_data = pl.DataFrame({ - "id": [1], - "email": ["alice@example.com"], - "name": ["Alice"], - "score": [100] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert without specifying update_columns (should auto-detect) - update_data = pl.DataFrame({ - "id": [1], - "email": ["alice@example.com"], - "name": ["Alice Updated"], - "score": [200] - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"] - # update_columns not specified - should update id, name, score - ) - - # Verify all non-conflict columns were updated - result = operations.run_sql(backend, "SELECT name, score FROM users WHERE email = 'alice@example.com'") - df = result.to_polars() - assert df["name"][0] == "Alice Updated" - assert df["score"][0] == 200 - - def test_upsert_validation_table_not_exists(self, backend_fixture, request): - """Test upsert raises error when target table doesn't exist.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Try to upsert to non-existent table - data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["Alice"] - }) - - with pytest.raises(ValueError, match="does not exist"): - operations.upsert( - backend, "nonexistent_table", data, - conflict_columns=["email"] - ) - - def test_upsert_validation_empty_conflict_columns(self, backend_fixture, request): - """Test upsert raises error when conflict_columns is empty.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - initial_data = pl.DataFrame({"email": ["alice@example.com"]}) - operations.create_table(backend, "users", initial_data, overwrite=True) - - # Try upsert with empty conflict columns - with pytest.raises(ValueError, match="At least one column"): - operations.upsert( - backend, "users", initial_data, - conflict_columns=[] - ) - - -@pytest.mark.integration -@pytest.mark.parametrize("backend_fixture", [ - "sqlite_memory_settings_params", - "duckdb_settings_params", -]) -class TestIndexManagement: - """Tests for index creation, deletion, and querying.""" - - def test_create_simple_index(self, backend_fixture, request): - """Test creating a simple index.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({ - "id": [1, 2, 3], - "email": ["a@example.com", "b@example.com", "c@example.com"] - }) - operations.create_table(backend, "users", data, overwrite=True) - - # Create index - result = operations.create_index(backend, "users", ["email"]) - assert result is True - - # Verify index exists - assert operations.index_exists(backend, "idx_users_email") is True - - def test_create_unique_index(self, backend_fixture, request): - """Test creating a unique index.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({ - "id": [1, 2, 3], - "email": ["a@example.com", "b@example.com", "c@example.com"] - }) - operations.create_table(backend, "users", data, overwrite=True) - - # Create unique index - result = operations.create_unique_index(backend, "users", ["email"]) - assert result is True - - # Verify index exists - assert operations.index_exists(backend, "uidx_users_email") is True - - def test_create_composite_index(self, backend_fixture, request): - """Test creating a composite (multi-column) index.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({ - "user_id": [1, 2, 3], - "product_id": [100, 200, 300], - "quantity": [5, 10, 15] - }) - operations.create_table(backend, "orders", data, overwrite=True) - - # Create composite index - result = operations.create_index( - backend, "orders", ["user_id", "product_id"] - ) - assert result is True - - # Verify index exists - assert operations.index_exists(backend, "idx_orders_product_id_user_id") is True - - def test_create_index_with_custom_name(self, backend_fixture, request): - """Test creating index with custom name.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({"email": ["a@example.com"]}) - operations.create_table(backend, "users", data, overwrite=True) - - # Create index with custom name - result = operations.create_index( - backend, "users", ["email"], - index_name="my_custom_index" - ) - assert result is True - - # Verify custom name - assert operations.index_exists(backend, "my_custom_index") is True - - def test_create_partial_index(self, backend_fixture, request): - """Test creating partial index with WHERE clause.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({ - "email": ["a@example.com", "b@example.com"], - "status": ["active", "inactive"] - }) - operations.create_table(backend, "users", data, overwrite=True) - - # Create partial index (only for active users) - # Note: DuckDB does not support partial indexes as of v1.3.2 - result = operations.create_index( - backend, "users", ["email"], - where_condition="status = 'active'" - ) - - # DuckDB doesn't support partial indexes, SQLite does - if "duckdb" in backend_fixture: - assert result is False # Expected: DuckDB doesn't support partial indexes - else: - assert result is True - # Verify index exists (only for SQLite) - assert operations.index_exists(backend, "idx_users_email") is True - - def test_create_index_if_not_exists(self, backend_fixture, request): - """Test creating index with IF NOT EXISTS doesn't fail on duplicate.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table and index - data = pl.DataFrame({"email": ["a@example.com"]}) - operations.create_table(backend, "users", data, overwrite=True) - operations.create_index(backend, "users", ["email"]) - - # Create same index again with if_not_exists=True (should not fail) - result = operations.create_index( - backend, "users", ["email"], - if_not_exists=True - ) - # Result may vary, but should not raise error - assert result in [True, False] - - def test_drop_index(self, backend_fixture, request): - """Test dropping an index.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table and index - data = pl.DataFrame({"email": ["a@example.com"]}) - operations.create_table(backend, "users", data, overwrite=True) - operations.create_index(backend, "users", ["email"]) - - # Verify index exists - assert operations.index_exists(backend, "idx_users_email") is True - - # Drop index - result = operations.drop_index(backend, "idx_users_email") - assert result is True - - # Verify index no longer exists - assert operations.index_exists(backend, "idx_users_email") is False - - def test_drop_index_if_exists(self, backend_fixture, request): - """Test dropping non-existent index with IF EXISTS doesn't fail.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Try to drop non-existent index with if_exists=True - result = operations.drop_index( - backend, "nonexistent_index", - if_exists=True - ) - # Should not raise error - assert result in [True, False] - - def test_index_exists_returns_false_for_nonexistent(self, backend_fixture, request): - """Test index_exists returns False for non-existent index.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Check non-existent index - assert operations.index_exists(backend, "nonexistent_index") is False - - def test_list_indexes(self, backend_fixture, request): - """Test listing all indexes for a table.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table with multiple indexes - data = pl.DataFrame({ - "id": [1, 2], - "email": ["a@example.com", "b@example.com"], - "name": ["Alice", "Bob"] - }) - operations.create_table(backend, "users", data, overwrite=True) - operations.create_index(backend, "users", ["email"]) - operations.create_unique_index(backend, "users", ["id"]) - - # List indexes - indexes = operations.list_indexes(backend, "users") - assert isinstance(indexes, list) - assert len(indexes) >= 2 # At least the two we created - - # Verify index names are in the list - index_names = [idx.get("name") for idx in indexes] - assert "idx_users_email" in index_names - assert "uidx_users_id" in index_names - - def test_create_unique_index_convenience_method(self, backend_fixture, request): - """Test create_unique_index convenience method.""" - settings = request.getfixturevalue(backend_fixture) - backend = DatabaseUtils.create_backend(settings) - operations = DatabaseUtils.create_operations(settings) - - # Create table - data = pl.DataFrame({"email": ["a@example.com"]}) - operations.create_table(backend, "users", data, overwrite=True) - - # Use convenience method - result = operations.create_unique_index(backend, "users", "email") - assert result is True - - # Verify unique index created - assert operations.index_exists(backend, "uidx_users_email") is True - - -@pytest.mark.integration -class TestUpsertIndexIntegration: - """Integration tests combining upsert and index operations.""" - - def test_upsert_requires_unique_index_on_conflict_columns(self, duckdb_settings_params): - """Test that upsert works correctly with unique index on conflict columns.""" - backend = DatabaseUtils.create_backend(duckdb_settings_params) - operations = DatabaseUtils.create_operations(duckdb_settings_params) - - # Create table - initial_data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["Alice"] - }) - operations.create_table(backend, "users", initial_data, overwrite=True) - - # Create unique index on conflict column - operations.create_unique_index(backend, "users", ["email"]) - - # Upsert should work - update_data = pl.DataFrame({ - "email": ["alice@example.com"], - "name": ["Alice Updated"] - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"] - ) - - # Verify update worked - result = operations.run_sql(backend, "SELECT name FROM users WHERE email = 'alice@example.com'") - df = result.to_polars() - assert df["name"][0] == "Alice Updated" - - def test_multiple_indexes_and_upserts(self, sqlite_memory_settings_params): - """Test multiple indexes don't interfere with upserts.""" - backend = DatabaseUtils.create_backend(sqlite_memory_settings_params) - operations = DatabaseUtils.create_operations(sqlite_memory_settings_params) - - # Create table with multiple indexes - data = pl.DataFrame({ - "id": [1, 2], - "email": ["a@example.com", "b@example.com"], - "username": ["alice", "bob"] - }) - operations.create_table(backend, "users", data, overwrite=True) - - # Create multiple indexes - operations.create_unique_index(backend, "users", ["email"]) - operations.create_unique_index(backend, "users", ["username"]) - operations.create_index(backend, "users", ["id"]) - - # Upsert on email should work - update_data = pl.DataFrame({ - "id": [1], - "email": ["a@example.com"], - "username": ["alice_updated"] - }) - operations.upsert( - backend, "users", update_data, - conflict_columns=["email"] - ) - - # Verify - result = operations.run_sql(backend, "SELECT username FROM users WHERE email = 'a@example.com'") - df = result.to_polars() - assert df["username"][0] == "alice_updated" diff --git a/tests/test_unit/databases/settings/test_settings_parametrized.py b/tests/test_unit/databases/settings/test_settings_parametrized.py index e86a73e..cb9c20a 100644 --- a/tests/test_unit/databases/settings/test_settings_parametrized.py +++ b/tests/test_unit/databases/settings/test_settings_parametrized.py @@ -144,38 +144,28 @@ def test_extracted_settings_have_parameters_method(self, settings_class): class TestSettingsWithConnections: """Test that settings work with actual connections.""" - def test_settings_work_with_connection_factory(self, settings_class, db_config): - """Test that settings work with ConnectionFactory.""" - from mountainash_data.core.factories import ConnectionFactory - + def test_settings_work_with_ibis_backend(self, settings_class, db_config): + """Test that settings work with IbisBackend.""" + from mountainash_data.backends.ibis.backend import IbisBackend settings_params = SettingsParameters.create( settings_class=settings_class, kwargs=db_config ) + backend = IbisBackend(settings_params) + assert backend.dialect is not None - # Should be able to get connection - connection = ConnectionFactory.get_connection(settings_params) - - assert connection is not None - - def test_settings_enable_backend_connection(self, settings_class, db_config): - """Test that settings enable actual backend connection.""" - from mountainash_data.core.utils import DatabaseUtils - + def test_settings_work_with_ibis_backend_connect(self, settings_class, db_config): + """Test that settings can create a connected backend via IbisBackend.""" + from mountainash_data.backends.ibis.backend import IbisBackend settings_params = SettingsParameters.create( settings_class=settings_class, kwargs=db_config ) - - # Should be able to create backend - backend = DatabaseUtils.create_backend(settings_params) - - assert backend is not None - - # Backend should be functional - backend.create_table("test", {"id": [1, 2, 3]}, overwrite=True) + backend = IbisBackend(settings_params) + backend.connect() tables = backend.list_tables() - assert "test" in tables + assert isinstance(tables, list) + backend.close() @pytest.mark.unit diff --git a/tests/test_unit/databases/test_database_connections.py b/tests/test_unit/databases/test_database_connections.py deleted file mode 100644 index c5ade05..0000000 --- a/tests/test_unit/databases/test_database_connections.py +++ /dev/null @@ -1,277 +0,0 @@ -"""Tests for database connection classes.""" - -import pytest -from pathlib import Path - -from mountainash_data.core.connection import BaseDBConnection -from mountainash_data.backends.ibis.connection import BaseIbisConnection -from mountainash_data.backends.ibis.connection import SQLite_IbisConnection -from mountainash_data.backends.ibis.connection import DuckDB_IbisConnection -from mountainash_settings import SettingsParameters -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings, NoAuth - - -class TestBaseDBConnection: - """Tests for base database connection class.""" - - def test_base_db_connection_is_abstract(self): - """Test that BaseDBConnection cannot be instantiated directly.""" - with pytest.raises(TypeError): - BaseDBConnection() - - def test_base_db_connection_has_required_methods(self): - """Test that BaseDBConnection defines required abstract methods.""" - # Check that abstract methods are defined - assert hasattr(BaseDBConnection, 'connect') - assert hasattr(BaseDBConnection, 'disconnect') - assert hasattr(BaseDBConnection, 'is_connected') - - -class TestBaseIbisConnection: - """Tests for base Ibis connection class.""" - - def test_base_ibis_connection_is_abstract(self): - """Test that BaseIbisConnection cannot be instantiated directly.""" - with pytest.raises(TypeError): - BaseIbisConnection() - - def test_inherits_from_base_db_connection(self): - """Test that BaseIbisConnection inherits from BaseDBConnection.""" - assert issubclass(BaseIbisConnection, BaseDBConnection) - - -class TestSQLiteIbisConnection: - """Tests for SQLite Ibis connection.""" - - def test_sqlite_connection_initialization(self, temp_sqlite_db): - """Test SQLite connection can be initialized.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - conn = SQLite_IbisConnection(db_auth_settings_parameters=settings_params) - assert conn is not None - assert hasattr(conn, 'db_auth_settings_parameters') - assert conn.db_auth_settings_parameters == settings_params - - def test_sqlite_connect_method(self, temp_sqlite_db): - """Test SQLite connect method.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - conn = SQLite_IbisConnection(db_auth_settings_parameters=settings_params) - backend = conn.connect() - - # Test actual functionality - assert backend is not None - # Verify we can list tables (should include test_table from fixture) - tables = backend.list_tables() - assert 'test_table' in tables - # Test we can actually query the table - table = backend.table('test_table') - assert table is not None - count = table.count().execute() - assert count == 3 # From fixture data - - def test_sqlite_connect_with_memory_db(self): - """Test SQLite connection with in-memory database.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - conn = SQLite_IbisConnection(db_auth_settings_parameters=settings_params) - backend = conn.connect() - - # Test actual functionality with in-memory database - assert backend is not None - # Should be able to create and query tables - backend.create_table('test_mem', {'id': [1, 2, 3], 'value': [10, 20, 30]}) - tables = backend.list_tables() - assert 'test_mem' in tables - - # Verify data operations work - table = backend.table('test_mem') - assert table is not None - # Note: .count().execute() has pyarrow compatibility issues in ibis 10.4.0 - # Verify table is queryable by converting to pandas (works around the issue) - try: - df = table.to_pandas() - assert len(df) == 3 - except AttributeError: - # Fallback if to_pandas also fails - just verify table exists - pass - - # def test_sqlite_connection_error_handling(self): - # """Test SQLite connection error handling.""" - # # Use invalid path to trigger real error - # settings_params = SettingsParameters.create( - # settings_class=SQLiteAuthSettings, - # kwargs={"DATABASE": "/invalid/nonexistent/path/test.db"} - # ) - - # conn = SQLite_IbisConnection(db_auth_settings_parameters=settings_params) - - # # This should raise a real connection error - # with pytest.raises(Exception): # Real error from ibis/sqlite - # conn.connect() - - -class TestDuckDBIbisConnection: - """Tests for DuckDB Ibis connection.""" - - def test_duckdb_connection_initialization(self): - """Test DuckDB connection can be initialized.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - conn = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params) - assert conn is not None - assert hasattr(conn, 'db_auth_settings_parameters') - assert conn.db_auth_settings_parameters == settings_params - - def test_duckdb_connect_method(self): - """Test DuckDB connect method.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - conn = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params) - backend = conn.connect() - - # Test actual functionality - assert backend is not None - # Create test data and verify operations - backend.create_table('test_duckdb', {'id': [1, 2, 3], 'name': ['A', 'B', 'C']}) - tables = backend.list_tables() - assert 'test_duckdb' in tables - - # Test actual query operations - table = backend.table('test_duckdb') - assert table is not None - # Note: .count().execute() has pyarrow compatibility issues in ibis 10.4.0 - # Verify table is queryable by converting to pandas (works around the issue) - try: - df = table.to_pandas() - assert len(df) == 3 - except AttributeError: - # Fallback if to_pandas also fails - just verify table exists - pass - - def test_duckdb_connect_with_memory_db(self): - """Test DuckDB connection with in-memory database.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": None, "auth": NoAuth()} # None triggers memory mode for DuckDB - ) - - conn = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params) - backend = conn.connect() - - # Test actual functionality with in-memory database - assert backend is not None - # Should be able to create and query tables - backend.create_table('test_mem_duck', {'x': [1, 2, 3], 'y': [4, 5, 6]}) - tables = backend.list_tables() - assert 'test_mem_duck' in tables - - # Verify data operations work - table = backend.table('test_mem_duck') - assert table is not None - # Note: .count().execute() has pyarrow compatibility issues in ibis 10.4.0 - # Verify table is queryable by converting to pandas (works around the issue) - try: - df = table.to_pandas() - assert len(df) == 3 - except AttributeError: - # Fallback if to_pandas also fails - just verify table exists - pass - - # def test_duckdb_connection_error_handling(self): - # """Test DuckDB connection error handling.""" - # # Use invalid path to trigger real error - # settings_params = SettingsParameters.create( - # settings_class=DuckDBAuthSettings, - # kwargs={"DATABASE": "/invalid/nonexistent/directory/test.duckdb"} - # ) - - # conn = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params) - - # # This should raise a real connection error - # with pytest.raises(Exception): # Real error from ibis/duckdb - # conn.connect() - - -class TestConnectionFactory: - """Tests for connection factory patterns.""" - - def test_can_create_multiple_sqlite_connections(self, temp_sqlite_db): - """Test that multiple SQLite connections can be created.""" - settings_params1 = SettingsParameters.create( - namespace="settings_params1", - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - settings_params2 = SettingsParameters.create( - namespace="settings_params2", - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - conn1 = SQLite_IbisConnection(db_auth_settings_parameters=settings_params1) - conn2 = SQLite_IbisConnection(db_auth_settings_parameters=settings_params2) - - assert conn1 != conn2 - assert conn1.db_auth_settings_parameters.kwargs != conn2.db_auth_settings_parameters.kwargs - - # Test they can both connect and work independently - backend1 = conn1.connect() - backend2 = conn2.connect() - - assert backend1 != backend2 - # Backend1 should have test_table from fixture, backend2 should not - assert 'test_table' in backend1.list_tables() - assert 'test_table' not in backend2.list_tables() - - def test_can_create_multiple_duckdb_connections(self): - """Test that multiple DuckDB connections can be created.""" - settings_params1 = SettingsParameters.create( - namespace="settings_params1", - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - settings_params2 = SettingsParameters.create( - namespace="settings_params2", - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": None, "auth": NoAuth()} - ) - - conn1 = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params1) - conn2 = DuckDB_IbisConnection(db_auth_settings_parameters=settings_params2) - - assert conn1 != conn2 - assert conn1.db_auth_settings_parameters.kwargs != conn2.db_auth_settings_parameters.kwargs - - # Test they can both connect and work independently - backend1 = conn1.connect() - backend2 = conn2.connect() - - # Verify backends are different instances (object identity) - assert backend1 is not backend2 - - # Create different tables in each to verify independence - backend1.create_table('table1', {'a': [1, 2]}) - backend2.create_table('table2', {'b': [3, 4]}) - - assert 'table1' in backend1.list_tables() - assert 'table1' not in backend2.list_tables() - assert 'table2' not in backend1.list_tables() - assert 'table2' in backend2.list_tables() diff --git a/tests/test_unit/databases/test_ibis_backends.py b/tests/test_unit/databases/test_ibis_backends.py deleted file mode 100644 index 8d165ef..0000000 --- a/tests/test_unit/databases/test_ibis_backends.py +++ /dev/null @@ -1,137 +0,0 @@ -import pytest -import ibis -import os -from unittest import mock - -from mountainash_data.backends.ibis.connection import SQLite_IbisConnection, DuckDB_IbisConnection -from mountainash_settings import SettingsParameters, MountainAshBaseSettings -from mountainash_data.core.settings import DuckDBAuthSettings, SQLiteAuthSettings, NoAuth - -@pytest.fixture -def mock_settings_parameters_1(): - return SettingsParameters.create(settings_class = MountainAshBaseSettings, namespace="mock") - -@pytest.fixture -def mock_settings_parameters_2(): - kwargs = {"USERNAME": "ngods", "PASSWORD": "ngods", "HOST": "host", "PORT": "5432"} - return SettingsParameters.create(settings_class = MountainAshBaseSettings, namespace="mock_pg", kwargs=kwargs) - -@pytest.fixture -def mock_settings_parameters_3(): - kwargs = {"USERNAME": "ngods", "PASSWORD": "ngods", "HOST": "host", "PORT": "5432", "DATABASE_NAME": "database_name"} - return SettingsParameters.create(settings_class = MountainAshBaseSettings, namespace="mock_mssql", kwargs=kwargs) - -################ -# Connections - -# @pytest.fixture -# def mock_sqlite_connection(mock_settings_parameters_1): -# with mock.patch('mountainash_data.databases.connections.ibis.SQLite_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield SQLite_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_1) - -# @pytest.fixture -# def mock_duckdb_connection(mock_settings_parameters_1): -# with mock.patch('mountainash_data.databases.connections.ibis.DuckDB_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield DuckDB_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_1) - -@pytest.fixture -def sqlite_connection(): - - settings_parameters = SettingsParameters.create(settings_class = SQLiteAuthSettings, namespace="SQLiteAuthSettings", kwargs={"auth": NoAuth()}) - - return SQLite_IbisConnection(db_auth_settings_parameters=settings_parameters) - -@pytest.fixture -def duckdb_connection(): - - settings_parameters = SettingsParameters.create(settings_class = DuckDBAuthSettings, namespace="DuckDBAuthSettings", kwargs={"auth": NoAuth()}) - - return DuckDB_IbisConnection(db_auth_settings_parameters=settings_parameters) - - - -# @pytest.fixture -# def mock_postgres_connection(mock_settings_parameters_2): -# with mock.patch('mountainash_data.databases.connections.ibis.Postgres_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield Postgres_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_2) - -# @pytest.fixture -# def mock_mssql_connection(mock_settings_parameters_3): -# with mock.patch('mountainash_data.databases.connections.ibis.MSSQL_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield MSSQL_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_3) - -# @pytest.fixture -# def mock_snowflake_connection(mock_settings_parameters_3): -# with mock.patch('mountainash_data.databases.connections.ibis.Snowflake_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield Snowflake_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_3) - - -# @pytest.fixture -# def mock_mysql_connection(mock_settings_parameters_3): -# with mock.patch('mountainash_data.databases.connections.ibis.MySQL_IbisConnection.connect') as mock_connect: -# mock_connect.return_value = mock.MagicMock() -# yield MySQL_IbisConnection(db_auth_settings_parameters=mock_settings_parameters_3) - - - - -################# -# # Tests - -# def test_sqlite_connection(mock_sqlite_connection): -# mock_sqlite_connection.connect() -# assert mock_sqlite_connection.ibis_backend is not None - -# def test_duckdb_connection(mock_duckdb_connection): -# mock_duckdb_connection.connect() -# assert mock_duckdb_connection.ibis_backend is not None - -def test_sqlite_connection(sqlite_connection): - sqlite_connection.connect() - assert sqlite_connection.ibis_backend is not None - -def test_duckdb_connection(duckdb_connection): - duckdb_connection.connect() - assert duckdb_connection.ibis_backend is not None - - -# def test_postgres_connection(mock_postgres_connection): -# mock_postgres_connection.connect() -# assert mock_postgres_connection.ibis_backend is not None - -# def test_mysql_connection(mock_mysql_connection): -# mock_mysql_connection.connect() -# assert mock_mysql_connection.ibis_backend is not None - -# def test_mssql_connection(mock_mssql_connection): -# mock_mssql_connection.connect() -# assert mock_mssql_connection.ibis_backend is not None - - -# def test_snowflake_connection(mock_snowflake_connection): -# mock_snowflake_connection.connect() -# assert mock_snowflake_connection.ibis_backend is not None - - - - -# @pytest.fixture(scope="module") -# def docker_compose_file(pytestconfig): -# return os.path.join(str(pytestconfig.rootdir), "docker-compose.yml") - -# @pytest.fixture(scope="module") -# def docker_services(docker_ip, docker_services): -# docker_services.start("postgres") -# docker_services.wait_for_service("postgres", 5432) -# yield -# docker_services.stop("postgres") - -# def test_postgres_docker_connection(docker_services): -# connection = Postgres_IbisConnection(db_auth_settings_parameters=mock.MagicMock()) -# connection.connect() -# assert connection.ibis_backend is not None diff --git a/tests/test_unit/factories/test_connection_factory.py b/tests/test_unit/factories/test_connection_factory.py deleted file mode 100644 index 50cc710..0000000 --- a/tests/test_unit/factories/test_connection_factory.py +++ /dev/null @@ -1,246 +0,0 @@ -"""Tests for ConnectionFactory.""" - -import pytest -from mountainash_data.core.factories.connection_factory import ConnectionFactory -from mountainash_data.core.connection import BaseDBConnection -from mountainash_data.backends.ibis.connection import SQLite_IbisConnection, DuckDB_IbisConnection -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings, NoAuth -from mountainash_data.core.constants import CONST_DB_PROVIDER_TYPE -from mountainash_settings import SettingsParameters - - -@pytest.mark.unit -class TestConnectionFactoryGetStrategy: - """Tests for ConnectionFactory.get_strategy method.""" - - def test_get_strategy_for_sqlite(self): - """Test getting SQLite connection class from factory.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - connection_class = ConnectionFactory.get_strategy(settings_params) - - assert connection_class is not None - assert connection_class == SQLite_IbisConnection - - def test_get_strategy_for_duckdb(self): - """Test getting DuckDB connection class from factory.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - connection_class = ConnectionFactory.get_strategy(settings_params) - - assert connection_class is not None - assert connection_class == DuckDB_IbisConnection - - @pytest.mark.parametrize("settings_class,expected_connection", [ - (SQLiteAuthSettings, SQLite_IbisConnection), - (DuckDBAuthSettings, DuckDB_IbisConnection), - ]) - def test_get_strategy_parametrized(self, settings_class, expected_connection): - """Test strategy selection for various backend types.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs={"DATABASE": ":memory:"} - ) - - connection_class = ConnectionFactory.get_strategy(settings_params) - - assert connection_class == expected_connection - - -@pytest.mark.unit -class TestConnectionFactoryGetConnection: - """Tests for ConnectionFactory.get_connection convenience method.""" - - def test_get_connection_returns_instance(self): - """Test that get_connection returns a connection instance.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - connection = ConnectionFactory.get_connection(settings_params) - - assert connection is not None - assert isinstance(connection, BaseDBConnection) - assert isinstance(connection, SQLite_IbisConnection) - - def test_get_connection_instance_can_connect(self, temp_sqlite_db): - """Test that returned connection instance can actually connect.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - connection = ConnectionFactory.get_connection(settings_params) - backend = connection.connect() - - assert backend is not None - assert connection.is_connected() - - # Cleanup - connection.disconnect() - - def test_get_connection_with_different_backends(self): - """Test getting connections for different backends.""" - # SQLite - sqlite_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - sqlite_conn = ConnectionFactory.get_connection(sqlite_params) - assert isinstance(sqlite_conn, SQLite_IbisConnection) - - # DuckDB - duckdb_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - duckdb_conn = ConnectionFactory.get_connection(duckdb_params) - assert isinstance(duckdb_conn, DuckDB_IbisConnection) - - -@pytest.mark.unit -class TestConnectionFactoryConfiguration: - """Tests for ConnectionFactory configuration and mapping.""" - - def test_factory_has_strategy_configuration(self): - """Test that factory configures strategies.""" - # Trigger configuration by accessing class methods - factory = ConnectionFactory() - - # Should have strategy mappings configured - assert hasattr(ConnectionFactory, '_strategy_modules') - assert hasattr(ConnectionFactory, '_strategy_classes') - - def test_strategy_modules_configured(self): - """Test that strategy modules are properly configured.""" - # Access the configuration - factory = ConnectionFactory() - - # Should have mappings for key backends - assert ConnectionFactory._strategy_modules is not None - assert CONST_DB_PROVIDER_TYPE.SQLITE in ConnectionFactory._strategy_modules - assert CONST_DB_PROVIDER_TYPE.DUCKDB in ConnectionFactory._strategy_modules - - def test_strategy_classes_configured(self): - """Test that strategy classes are properly configured.""" - factory = ConnectionFactory() - - assert ConnectionFactory._strategy_classes is not None - assert CONST_DB_PROVIDER_TYPE.SQLITE in ConnectionFactory._strategy_classes - assert CONST_DB_PROVIDER_TYPE.DUCKDB in ConnectionFactory._strategy_classes - - def test_sqlite_strategy_mapping(self): - """Test SQLite strategy configuration. - - Updated in Phase 5 Task 5.2: module path now points directly at - backends.ibis.connection (bypassing the databases.connections.ibis shim chain). - """ - factory = ConnectionFactory() - - module_path = ConnectionFactory._strategy_modules.get(CONST_DB_PROVIDER_TYPE.SQLITE) - class_name = ConnectionFactory._strategy_classes.get(CONST_DB_PROVIDER_TYPE.SQLITE) - - assert module_path == "mountainash_data.backends.ibis.connection" - assert class_name == "SQLite_IbisConnection" - - def test_duckdb_strategy_mapping(self): - """Test DuckDB strategy configuration. - - Updated in Phase 5 Task 5.2: module path now points directly at - backends.ibis.connection (bypassing the databases.connections.ibis shim chain). - """ - factory = ConnectionFactory() - - module_path = ConnectionFactory._strategy_modules.get(CONST_DB_PROVIDER_TYPE.DUCKDB) - class_name = ConnectionFactory._strategy_classes.get(CONST_DB_PROVIDER_TYPE.DUCKDB) - - assert module_path == "mountainash_data.backends.ibis.connection" - assert class_name == "DuckDB_IbisConnection" - - -@pytest.mark.unit -class TestConnectionFactoryLazyLoading: - """Tests for lazy loading behavior of ConnectionFactory.""" - - def test_factory_lazy_loads_connection_classes(self): - """Test that connection classes are loaded lazily.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - # Getting strategy should work without pre-importing - connection_class = ConnectionFactory.get_strategy(settings_params) - - assert connection_class is not None - assert callable(connection_class) - - def test_multiple_calls_return_same_class(self): - """Test that multiple calls for same backend return same class.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - class1 = ConnectionFactory.get_strategy(settings_params) - class2 = ConnectionFactory.get_strategy(settings_params) - - assert class1 is class2 # Same class object - - -@pytest.mark.integration -class TestConnectionFactoryIntegration: - """Integration tests for ConnectionFactory.""" - - def test_factory_to_connection_to_backend_workflow(self, temp_sqlite_db): - """Test complete workflow: factory → connection → backend.""" - # Create settings - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - # Get connection from factory - connection = ConnectionFactory.get_connection(settings_params) - - # Connect to backend - backend = connection.connect() - - # Verify backend is functional - assert backend is not None - tables = backend.list_tables() - assert isinstance(tables, list) - assert 'test_table' in tables - - # Cleanup - connection.disconnect() - - def test_factory_works_with_multiple_concurrent_connections(self): - """Test factory can create multiple connections concurrently.""" - sqlite_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - duckdb_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - conn1 = ConnectionFactory.get_connection(sqlite_params) - conn2 = ConnectionFactory.get_connection(duckdb_params) - conn3 = ConnectionFactory.get_connection(sqlite_params) - - # All should be valid instances - assert isinstance(conn1, SQLite_IbisConnection) - assert isinstance(conn2, DuckDB_IbisConnection) - assert isinstance(conn3, SQLite_IbisConnection) - - # But conn1 and conn3 should be different instances - assert conn1 is not conn3 diff --git a/tests/test_unit/factories/test_operations_factory.py b/tests/test_unit/factories/test_operations_factory.py deleted file mode 100644 index 4b2abb3..0000000 --- a/tests/test_unit/factories/test_operations_factory.py +++ /dev/null @@ -1,239 +0,0 @@ -"""Tests for OperationsFactory.""" - -import pytest -from mountainash_data.core.factories.operations_factory import OperationsFactory -from mountainash_data.backends.ibis.operations import BaseIbisOperations -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings, NoAuth -from mountainash_data.core.constants import CONST_DB_PROVIDER_TYPE -from mountainash_settings import SettingsParameters - - -@pytest.mark.unit -class TestOperationsFactoryGetStrategy: - """Tests for OperationsFactory.get_strategy method.""" - - def test_get_strategy_for_sqlite(self): - """Test getting SQLite operations class from factory.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations_class = OperationsFactory.get_strategy(settings_params) - - assert operations_class is not None - # Should be a subclass of BaseIbisOperations - assert issubclass(operations_class, BaseIbisOperations) - - def test_get_strategy_for_duckdb(self): - """Test getting DuckDB operations class from factory.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations_class = OperationsFactory.get_strategy(settings_params) - - assert operations_class is not None - assert issubclass(operations_class, BaseIbisOperations) - - @pytest.mark.parametrize("settings_class", [ - SQLiteAuthSettings, - DuckDBAuthSettings, - ]) - def test_get_strategy_parametrized(self, settings_class): - """Test strategy selection for various backend types.""" - settings_params = SettingsParameters.create( - settings_class=settings_class, - kwargs={"DATABASE": ":memory:"} - ) - - operations_class = OperationsFactory.get_strategy(settings_params) - - assert operations_class is not None - assert issubclass(operations_class, BaseIbisOperations) - - -@pytest.mark.unit -class TestOperationsFactoryGetOperations: - """Tests for OperationsFactory.get_operations convenience method.""" - - def test_get_operations_returns_instance(self): - """Test that get_operations returns an operations instance.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations = OperationsFactory.get_operations(settings_params) - - assert operations is not None - assert isinstance(operations, BaseIbisOperations) - - def test_get_operations_for_different_backends(self): - """Test getting operations for different backends.""" - # SQLite - sqlite_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - sqlite_ops = OperationsFactory.get_operations(sqlite_params) - assert isinstance(sqlite_ops, BaseIbisOperations) - - # DuckDB - duckdb_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - duckdb_ops = OperationsFactory.get_operations(duckdb_params) - assert isinstance(duckdb_ops, BaseIbisOperations) - - -@pytest.mark.unit -class TestOperationsFactoryConfiguration: - """Tests for OperationsFactory configuration and mapping.""" - - def test_factory_has_strategy_configuration(self): - """Test that factory configures strategies.""" - factory = OperationsFactory() - - assert hasattr(OperationsFactory, '_strategy_modules') - assert hasattr(OperationsFactory, '_strategy_classes') - - def test_strategy_modules_configured(self): - """Test that strategy modules are properly configured.""" - factory = OperationsFactory() - - assert OperationsFactory._strategy_modules is not None - assert CONST_DB_PROVIDER_TYPE.SQLITE in OperationsFactory._strategy_modules - assert CONST_DB_PROVIDER_TYPE.DUCKDB in OperationsFactory._strategy_modules - - def test_strategy_classes_configured(self): - """Test that strategy classes are properly configured.""" - factory = OperationsFactory() - - assert OperationsFactory._strategy_classes is not None - assert CONST_DB_PROVIDER_TYPE.SQLITE in OperationsFactory._strategy_classes - assert CONST_DB_PROVIDER_TYPE.DUCKDB in OperationsFactory._strategy_classes - - def test_sqlite_strategy_mapping(self): - """Test SQLite operations strategy configuration. - - Updated in Phase 5 Task 5.2: module path now points directly at - backends.ibis.operations (bypassing the databases.operations.ibis shim chain). - """ - factory = OperationsFactory() - - module_path = OperationsFactory._strategy_modules.get(CONST_DB_PROVIDER_TYPE.SQLITE) - class_name = OperationsFactory._strategy_classes.get(CONST_DB_PROVIDER_TYPE.SQLITE) - - assert "mountainash_data.backends.ibis.operations" in module_path - assert "Operations" in class_name - - def test_duckdb_strategy_mapping(self): - """Test DuckDB operations strategy configuration. - - Updated in Phase 5 Task 5.2: module path now points directly at - backends.ibis.operations (bypassing the databases.operations.ibis shim chain). - """ - factory = OperationsFactory() - - module_path = OperationsFactory._strategy_modules.get(CONST_DB_PROVIDER_TYPE.DUCKDB) - class_name = OperationsFactory._strategy_classes.get(CONST_DB_PROVIDER_TYPE.DUCKDB) - - assert "mountainash_data.backends.ibis.operations" in module_path - assert "Operations" in class_name - - -@pytest.mark.unit -class TestOperationsFactoryLazyLoading: - """Tests for lazy loading behavior of OperationsFactory.""" - - def test_factory_lazy_loads_operations_classes(self): - """Test that operations classes are loaded lazily.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations_class = OperationsFactory.get_strategy(settings_params) - - assert operations_class is not None - assert callable(operations_class) - - def test_multiple_calls_return_same_class(self): - """Test that multiple calls for same backend return same class.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - class1 = OperationsFactory.get_strategy(settings_params) - class2 = OperationsFactory.get_strategy(settings_params) - - assert class1 is class2 - - -@pytest.mark.unit -class TestOperationsFactoryMatchesConnectionFactory: - """Tests to ensure operations factory aligns with connection factory.""" - - def test_operations_backend_matches_connection_backend(self): - """Test that operations backend type matches connection backend type.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations = OperationsFactory.get_operations(settings_params) - - # Operations should have correct backend type - assert hasattr(operations, 'db_backend_name') - - -@pytest.mark.integration -class TestOperationsFactoryIntegration: - """Integration tests for OperationsFactory.""" - - def test_operations_work_with_actual_backend(self, temp_sqlite_db): - """Test that factory-created operations work with real backend.""" - from mountainash_data.core.utils import DatabaseUtils - - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - # Get operations from factory - operations = OperationsFactory.get_operations(settings_params) - - # Get connected backend - backend = DatabaseUtils.create_backend(settings_params) - - # Test operations with backend - tables = operations.list_tables(backend) - assert isinstance(tables, list) - assert 'test_table' in tables - - def test_factory_creates_multiple_operations_instances(self): - """Test factory can create multiple operations instances.""" - sqlite_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - duckdb_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - ops1 = OperationsFactory.get_operations(sqlite_params) - ops2 = OperationsFactory.get_operations(duckdb_params) - ops3 = OperationsFactory.get_operations(sqlite_params) - - # All should be valid instances - assert isinstance(ops1, BaseIbisOperations) - assert isinstance(ops2, BaseIbisOperations) - assert isinstance(ops3, BaseIbisOperations) - - # ops1 and ops3 should be different instances - assert ops1 is not ops3 diff --git a/tests/test_unit/factories/test_settings_factory.py b/tests/test_unit/factories/test_settings_factory.py deleted file mode 100644 index 1c07ea5..0000000 --- a/tests/test_unit/factories/test_settings_factory.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Tests for SettingsFactory.""" - -import pytest -from mountainash_data.core.factories.settings_factory import SettingsFactory -from mountainash_data.core.settings import ( - SQLiteAuthSettings, - DuckDBAuthSettings, - ConnectionProfile, -) -from mountainash_data.core.constants import CONST_DB_PROVIDER_TYPE - - -@pytest.mark.unit -class TestSettingsFactoryFromConnectionString: - """Tests for SettingsFactory.from_connection_string method.""" - - def test_from_sqlite_connection_string(self): - """Test creating settings from SQLite connection string.""" - url = "sqlite:///test.db" - - settings = SettingsFactory.from_connection_string(url) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - def test_from_duckdb_connection_string(self): - """Test creating settings from DuckDB connection string.""" - url = "duckdb:///test.duckdb" - - settings = SettingsFactory.from_connection_string(url) - - assert settings is not None - assert isinstance(settings, DuckDBAuthSettings) - - @pytest.mark.parametrize("url,expected_type", [ - ("sqlite:///:memory:", SQLiteAuthSettings), - ("sqlite:///test.db", SQLiteAuthSettings), - ("duckdb:///:memory:", DuckDBAuthSettings), - ("duckdb:///test.duckdb", DuckDBAuthSettings), - ]) - def test_from_connection_string_parametrized(self, url, expected_type): - """Test connection string parsing for various database types.""" - settings = SettingsFactory.from_connection_string(url) - - assert settings is not None - assert isinstance(settings, expected_type) - - def test_from_connection_string_with_kwargs(self): - """Test from_connection_string with additional kwargs.""" - url = "sqlite:///:memory:" - - settings = SettingsFactory.from_connection_string( - url, - timeout=30 - ) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - -@pytest.mark.unit -class TestSettingsFactoryFromBackendType: - """Tests for SettingsFactory.from_backend_type method.""" - - def test_from_sqlite_backend_type(self): - """Test creating settings from SQLite backend type.""" - settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - def test_from_duckdb_backend_type(self): - """Test creating settings from DuckDB backend type.""" - settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.DUCKDB, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, DuckDBAuthSettings) - - @pytest.mark.parametrize("backend_type,expected_settings", [ - (CONST_DB_PROVIDER_TYPE.SQLITE, SQLiteAuthSettings), - (CONST_DB_PROVIDER_TYPE.DUCKDB, DuckDBAuthSettings), - ]) - def test_from_backend_type_parametrized(self, backend_type, expected_settings): - """Test settings creation for various backend types.""" - settings = SettingsFactory.from_backend_type( - backend_type, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, expected_settings) - - def test_from_backend_type_with_multiple_kwargs(self): - """Test from_backend_type with multiple configuration parameters.""" - settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:", - timeout=30, - read_only=False - ) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - -@pytest.mark.unit -class TestSettingsFactoryDetectBackend: - """Tests for SettingsFactory.detect_backend_from_url method.""" - - @pytest.mark.parametrize("url,expected_backend", [ - ("sqlite:///test.db", CONST_DB_PROVIDER_TYPE.SQLITE), - ("sqlite:///:memory:", CONST_DB_PROVIDER_TYPE.SQLITE), - ("duckdb:///test.duckdb", CONST_DB_PROVIDER_TYPE.DUCKDB), - ("duckdb:///:memory:", CONST_DB_PROVIDER_TYPE.DUCKDB), - ("postgresql://localhost/db", CONST_DB_PROVIDER_TYPE.POSTGRESQL), - ("postgres://localhost/db", CONST_DB_PROVIDER_TYPE.POSTGRESQL), - ]) - def test_detect_backend_from_url(self, url, expected_backend): - """Test backend detection from various URL formats.""" - detected_backend = SettingsFactory.detect_backend_from_url(url) - - assert detected_backend == expected_backend - - def test_detect_backend_sqlite_variations(self): - """Test SQLite URL variations.""" - sqlite_urls = [ - "sqlite:///absolute/path/to/db.sqlite", - "sqlite:///:memory:", - "sqlite:///relative/path.db", - ] - - for url in sqlite_urls: - backend = SettingsFactory.detect_backend_from_url(url) - assert backend == CONST_DB_PROVIDER_TYPE.SQLITE, f"Failed for URL: {url}" - - def test_detect_backend_duckdb_variations(self): - """Test DuckDB URL variations.""" - duckdb_urls = [ - "duckdb:///path/to/db.duckdb", - "duckdb:///:memory:", - "duckdb:///test.db", - ] - - for url in duckdb_urls: - backend = SettingsFactory.detect_backend_from_url(url) - assert backend == CONST_DB_PROVIDER_TYPE.DUCKDB, f"Failed for URL: {url}" - - -@pytest.mark.unit -class TestSettingsFactoryConfiguration: - """Tests for SettingsFactory configuration.""" - - def test_factory_has_backend_mappings(self): - """Test that factory has backend to settings mappings.""" - # This tests that the factory properly maps backend types to settings classes - sqlite_settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:" - ) - duckdb_settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.DUCKDB, - DATABASE=":memory:" - ) - - assert type(sqlite_settings) != type(duckdb_settings) - assert isinstance(sqlite_settings, ConnectionProfile) - assert isinstance(duckdb_settings, ConnectionProfile) - - def test_factory_lazy_loads_settings_classes(self): - """Test that settings classes are loaded lazily.""" - # Creating settings should work without pre-importing - settings = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:" - ) - - assert settings is not None - - -@pytest.mark.unit -class TestSettingsFactoryErrorHandling: - """Tests for SettingsFactory error handling.""" - - def test_invalid_url_format(self): - """Test handling of invalid URL formats.""" - invalid_url = "not-a-valid-url" - - with pytest.raises((ValueError, KeyError, AttributeError)): - SettingsFactory.from_connection_string(invalid_url) - - def test_unsupported_backend_type(self): - """Test handling of URL with unsupported backend.""" - # This should either raise an error or handle gracefully - unsupported_url = "unknown-backend://localhost/db" - - with pytest.raises((ValueError, KeyError, AttributeError, NotImplementedError)): - SettingsFactory.from_connection_string(unsupported_url) - - -@pytest.mark.integration -class TestSettingsFactoryIntegration: - """Integration tests for SettingsFactory.""" - - def test_settings_factory_to_connection_workflow(self, temp_sqlite_db): - """Test workflow: factory creates settings → settings create connection.""" - from mountainash_data.core.utils import DatabaseUtils - - # Factory creates settings - url = f"sqlite:///{temp_sqlite_db}" - settings = SettingsFactory.from_connection_string(url) - - # Settings can be used to create connection - settings_params = settings.extract_settings_parameters() - connection = DatabaseUtils.create_connection(settings_params) - - assert connection is not None - - # Connection can connect - backend = connection.connect() - assert backend is not None - - # Cleanup - connection.disconnect() - - def test_settings_factory_round_trip(self): - """Test that settings created from URL can generate correct connection strings.""" - original_url = "sqlite:///:memory:" - - # Create settings from URL - settings = SettingsFactory.from_connection_string(original_url) - - # Settings should have connection info - assert settings is not None - assert hasattr(settings, 'DATABASE') - - def test_factory_creates_independent_settings_instances(self): - """Test that factory creates independent settings instances.""" - settings1 = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE="db1.sqlite" - ) - settings2 = SettingsFactory.from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE="db2.sqlite" - ) - - # Different instances - assert settings1 is not settings2 - - # Different databases - assert settings1.DATABASE != settings2.DATABASE diff --git a/tests/test_unit/test_database_utils.py b/tests/test_unit/test_database_utils.py deleted file mode 100644 index a40f45d..0000000 --- a/tests/test_unit/test_database_utils.py +++ /dev/null @@ -1,269 +0,0 @@ -"""Tests for DatabaseUtils high-level API.""" - -import pytest -from pathlib import Path -from mountainash_data.core.utils import DatabaseUtils -from mountainash_data.core.connection import BaseDBConnection -from mountainash_data.backends.ibis.connection import SQLite_IbisConnection, DuckDB_IbisConnection -from mountainash_data.backends.ibis.operations import BaseIbisOperations -from mountainash_data.core.settings import SQLiteAuthSettings, DuckDBAuthSettings, NoAuth -from mountainash_data.core.constants import CONST_DB_PROVIDER_TYPE -from mountainash_settings import SettingsParameters - - -@pytest.mark.unit -class TestDatabaseUtilsCreateConnection: - """Tests for DatabaseUtils.create_connection method.""" - - def test_create_connection_with_sqlite_settings(self, temp_sqlite_db): - """Test creating SQLite connection from settings parameters.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db)} - ) - - connection = DatabaseUtils.create_connection(settings_params) - - assert connection is not None - assert isinstance(connection, BaseDBConnection) - assert isinstance(connection, SQLite_IbisConnection) - - def test_create_connection_with_duckdb_settings(self): - """Test creating DuckDB connection from settings parameters.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - connection = DatabaseUtils.create_connection(settings_params) - - assert connection is not None - assert isinstance(connection, BaseDBConnection) - assert isinstance(connection, DuckDB_IbisConnection) - - def test_created_connection_can_connect(self, temp_sqlite_db): - """Test that created connection can actually connect.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - connection = DatabaseUtils.create_connection(settings_params) - backend = connection.connect() - - assert backend is not None - assert connection.is_connected() - - # Cleanup - connection.disconnect() - - -@pytest.mark.unit -class TestDatabaseUtilsCreateOperations: - """Tests for DatabaseUtils.create_operations method.""" - - def test_create_operations_with_sqlite_settings(self, temp_sqlite_db): - """Test creating operations from SQLite settings parameters.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db)} - ) - - operations = DatabaseUtils.create_operations(settings_params) - - assert operations is not None - assert isinstance(operations, BaseIbisOperations) - - def test_create_operations_with_duckdb_settings(self): - """Test creating operations from DuckDB settings parameters.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:"} - ) - - operations = DatabaseUtils.create_operations(settings_params) - - assert operations is not None - assert isinstance(operations, BaseIbisOperations) - - -@pytest.mark.unit -class TestDatabaseUtilsCreateBackend: - """Tests for DatabaseUtils.create_backend convenience method.""" - - def test_create_backend_connects_immediately(self, temp_sqlite_db): - """Test that create_backend returns connected backend.""" - settings_params = SettingsParameters.create( - settings_class=SQLiteAuthSettings, - kwargs={"DATABASE": str(temp_sqlite_db), "auth": NoAuth()} - ) - - backend = DatabaseUtils.create_backend(settings_params) - - assert backend is not None - # Should be able to list tables - tables = backend.list_tables() - assert isinstance(tables, list) - assert 'test_table' in tables - - def test_create_backend_with_duckdb(self): - """Test create_backend with DuckDB in-memory.""" - settings_params = SettingsParameters.create( - settings_class=DuckDBAuthSettings, - kwargs={"DATABASE": ":memory:", "auth": NoAuth()} - ) - - backend = DatabaseUtils.create_backend(settings_params) - - assert backend is not None - # Should be able to create and list tables - backend.create_table('test', {'id': [1, 2, 3]}) - tables = backend.list_tables() - assert 'test' in tables - - -@pytest.mark.unit -class TestDatabaseUtilsSettingsFromURL: - """Tests for DatabaseUtils.create_settings_from_url method.""" - - def test_create_settings_from_sqlite_url(self): - """Test creating settings from SQLite URL.""" - url = "sqlite:///test.db" - - settings = DatabaseUtils.create_settings_from_url(url) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - def test_create_settings_from_duckdb_url(self): - """Test creating settings from DuckDB URL.""" - url = "duckdb:///test.duckdb" - - settings = DatabaseUtils.create_settings_from_url(url) - - assert settings is not None - assert isinstance(settings, DuckDBAuthSettings) - - @pytest.mark.parametrize("url,expected_type", [ - ("sqlite:///:memory:", SQLiteAuthSettings), - ("duckdb:///:memory:", DuckDBAuthSettings), - ]) - def test_create_settings_from_url_parametrized(self, url, expected_type): - """Test URL detection for various database types.""" - settings = DatabaseUtils.create_settings_from_url(url) - - assert settings is not None - assert isinstance(settings, expected_type) - - -@pytest.mark.unit -class TestDatabaseUtilsSettingsFromBackendType: - """Tests for DatabaseUtils.create_settings_from_backend_type method.""" - - def test_create_settings_from_sqlite_backend_type(self): - """Test creating settings from backend type enum.""" - settings = DatabaseUtils.create_settings_from_backend_type( - CONST_DB_PROVIDER_TYPE.SQLITE, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, SQLiteAuthSettings) - - def test_create_settings_from_duckdb_backend_type(self): - """Test creating settings from DuckDB backend type.""" - settings = DatabaseUtils.create_settings_from_backend_type( - CONST_DB_PROVIDER_TYPE.DUCKDB, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, DuckDBAuthSettings) - - @pytest.mark.parametrize("backend_type,expected_settings", [ - (CONST_DB_PROVIDER_TYPE.SQLITE, SQLiteAuthSettings), - (CONST_DB_PROVIDER_TYPE.DUCKDB, DuckDBAuthSettings), - ]) - def test_create_settings_from_backend_type_parametrized(self, backend_type, expected_settings): - """Test settings creation for various backend types.""" - settings = DatabaseUtils.create_settings_from_backend_type( - backend_type, - DATABASE=":memory:" - ) - - assert settings is not None - assert isinstance(settings, expected_settings) - - -@pytest.mark.unit -class TestDatabaseUtilsDetectBackend: - """Tests for DatabaseUtils.detect_backend_from_url method.""" - - @pytest.mark.parametrize("url,expected_backend", [ - ("sqlite:///test.db", CONST_DB_PROVIDER_TYPE.SQLITE), - ("sqlite:///:memory:", CONST_DB_PROVIDER_TYPE.SQLITE), - ("duckdb:///test.duckdb", CONST_DB_PROVIDER_TYPE.DUCKDB), - ("duckdb:///:memory:", CONST_DB_PROVIDER_TYPE.DUCKDB), - ]) - def test_detect_backend_from_url(self, url, expected_backend): - """Test backend detection from various URL formats.""" - detected_backend = DatabaseUtils.detect_backend_from_url(url) - - assert detected_backend == expected_backend - - -@pytest.mark.integration -class TestDatabaseUtilsCreateFromURL: - """Tests for DatabaseUtils.create_from_url end-to-end method.""" - - def test_create_from_sqlite_url_end_to_end(self, temp_sqlite_db): - """Test complete workflow from URL to connected backend.""" - url = f"sqlite:///{temp_sqlite_db}" - - connection, backend = DatabaseUtils.create_from_url(url) - - assert connection is not None - assert isinstance(connection, SQLite_IbisConnection) - assert backend is not None - # Verify backend is actually connected and functional - tables = backend.list_tables() - assert isinstance(tables, list) - - # Create a table to verify functionality - backend.create_table('workflow_test', {'id': [1, 2, 3]}, overwrite=True) - assert 'workflow_test' in backend.list_tables() - - # Cleanup - connection.disconnect() - - def test_create_from_duckdb_url_end_to_end(self): - """Test complete workflow with DuckDB in-memory.""" - url = "duckdb:///:memory:" - - connection, backend = DatabaseUtils.create_from_url(url) - - assert connection is not None - assert isinstance(connection, DuckDB_IbisConnection) - assert backend is not None - # Verify we can perform operations - backend.create_table('test_table', {'id': [1, 2, 3], 'value': [10, 20, 30]}) - tables = backend.list_tables() - assert 'test_table' in tables - - # Cleanup - connection.disconnect() - - def test_create_from_url_with_config_files(self, temp_sqlite_db, tmp_path): - """Test create_from_url with additional config files.""" - url = f"sqlite:///{temp_sqlite_db}" - - connection, backend = DatabaseUtils.create_from_url( - url, - config_files=[] # Empty list, just testing parameter works - ) - - assert connection is not None - assert backend is not None - - # Cleanup - connection.disconnect() diff --git a/tests/test_unit/test_mountainash_data.py b/tests/test_unit/test_mountainash_data.py index 62ca7bb..48b0495 100644 --- a/tests/test_unit/test_mountainash_data.py +++ b/tests/test_unit/test_mountainash_data.py @@ -1,7 +1,6 @@ """Tests for main mountainash_data package.""" import pytest -from mountainash_data import __version__ import mountainash_data @@ -9,77 +8,50 @@ class TestPackageImports: """Test package-level imports and structure.""" def test_version_import(self): - """Test that version can be imported.""" assert hasattr(mountainash_data, '__version__') assert isinstance(mountainash_data.__version__, str) assert len(mountainash_data.__version__) > 0 def test_version_format(self): - """Test version follows expected format.""" version_parts = mountainash_data.__version__.split('.') - assert len(version_parts) >= 2, "Version should have at least major.minor" - - # Test that major and minor are numeric - assert version_parts[0].isdigit(), "Major version should be numeric" - assert version_parts[1].isdigit(), "Minor version should be numeric" + assert len(version_parts) >= 2 + assert version_parts[0].isdigit() + assert version_parts[1].isdigit() def test_core_imports_available(self): - """Test that core classes can be imported.""" from mountainash_data.core.connection import BaseDBConnection - - # Check classes are properly defined assert BaseDBConnection is not None - def test_database_connections_available(self): - """Test that database connection classes can be imported.""" - from mountainash_data.backends.ibis.connection import SQLite_IbisConnection - from mountainash_data.backends.ibis.connection import DuckDB_IbisConnection - - assert SQLite_IbisConnection is not None - assert DuckDB_IbisConnection is not None - class TestPackageStructure: """Test package structure and organization.""" def test_package_has_init(self): - """Test that package has proper __init__.py.""" - import mountainash_data assert hasattr(mountainash_data, '__file__') def test_new_submodules_exist(self): - """Test that the new core and backends submodules exist.""" import mountainash_data.core import mountainash_data.backends - assert hasattr(mountainash_data, 'core') assert hasattr(mountainash_data, 'backends') - @pytest.mark.parametrize("module_name", [ - "core.connection", - "backends.ibis.connection", - ]) - def test_module_importable(self, module_name: str): - """Test that core modules can be imported.""" - from importlib import import_module - full_module = f"mountainash_data.{module_name}" - - module = import_module(full_module) - assert module is not None - def test_public_api_ibis_backend(self): - """Test that IbisBackend is available from the top-level package.""" from mountainash_data import IbisBackend assert IbisBackend is not None - def test_public_api_factories(self): - """Test that factories are available from the top-level package.""" - from mountainash_data import ConnectionFactory, OperationsFactory, SettingsFactory - assert ConnectionFactory is not None - assert OperationsFactory is not None - assert SettingsFactory is not None - - def test_public_api_database_utils(self): - """Test that DatabaseUtils is available from the top-level package.""" - from mountainash_data import DatabaseUtils - assert DatabaseUtils is not None + def test_public_api_backend_protocol(self): + from mountainash_data import Backend + assert Backend is not None + + def test_public_api_inspection_model(self): + from mountainash_data import CatalogInfo, ColumnInfo, NamespaceInfo, TableInfo + assert CatalogInfo is not None + assert ColumnInfo is not None + assert NamespaceInfo is not None + assert TableInfo is not None + + def test_removed_exports_not_available(self): + assert not hasattr(mountainash_data, 'ConnectionFactory') + assert not hasattr(mountainash_data, 'OperationsFactory') + assert not hasattr(mountainash_data, 'SettingsFactory') + assert not hasattr(mountainash_data, 'DatabaseUtils')