diff --git a/src/mountainash_data/backends/ibis/_render.py b/src/mountainash_data/backends/ibis/_render.py index 9dc194a..6738746 100644 --- a/src/mountainash_data/backends/ibis/_render.py +++ b/src/mountainash_data/backends/ibis/_render.py @@ -13,6 +13,7 @@ import ibis.expr.operations as ops import ibis.expr.types as ir from sqlglot import exp +from mountainash_data.backends.ibis._sqlite_compat import ensure_sqlite_nat_adapter def dialect_of(ibis_conn: t.Any) -> t.Any: @@ -143,6 +144,7 @@ def compiled_source( raise ValueError(f"source columns absent from target: {sorted(extra)}") cols = [c for c in target_schema.names if c in src_cols] projected = src.select([src[c].cast(target_schema[c]).name(c) for c in cols]) + ensure_sqlite_nat_adapter() ibis_conn._register_in_memory_tables(projected) # REQUIRED: stage memtables return ibis_conn.compile(projected), cols diff --git a/src/mountainash_data/backends/ibis/_sqlite_compat.py b/src/mountainash_data/backends/ibis/_sqlite_compat.py new file mode 100644 index 0000000..cbd6e3f --- /dev/null +++ b/src/mountainash_data/backends/ibis/_sqlite_compat.py @@ -0,0 +1,66 @@ +"""Compatibility shim for Ibis's SQLite backend. + +Ibis's SQLite backend (``ibis.backends.sqlite.Backend._register_in_memory_table``) +always stages an in-memory table via a pandas roundtrip (``op.data.to_frame()``) +before binding rows through the stdlib ``sqlite3`` module -- regardless of +whether the table was built from a dict, a PyArrow table, a Polars/pandas +DataFrame, or an ``ibis.memtable(..., schema=...)`` call with an explicit +temporal schema. A null ``date``/``timestamp`` value becomes pandas ``NaT`` +during that roundtrip, and ``sqlite3`` has no adapter for ``NaTType`` -- +``cur.executemany()`` raises ``sqlite3.ProgrammingError("Error binding +parameter N: type 'NaTType' is not supported")`` before any of +``create_table``/``insert``/``upsert`` reach the database -- for the entire +portable write surface (DEBT-13). + +Verified empirically against ibis 12.0.0 (2026-08-19): this reproduces +identically whether the source frame is Polars, pandas, or an explicit +pyarrow/ibis schema. Tracked upstream as ``IB-DT-19`` in the +``mountainash`` repo's ``registry/upstream-issues.yaml`` (status: +``needs_filing`` as of 2026-08-18 -- no upstream ibis issue exists yet). +Same root cause, same fix shape as the sibling ``mountainash`` package's +``relations/backends/relation_systems/ibis/_sqlite_compat.py`` (item 112, +PR #303) -- kept as a separate, self-contained module here rather than a +hard dependency on ``mountainash``, since ``mountainash_data`` treats +``mountainash`` as optional (see ``operations.py::_coerce_dtype``). + +This module registers a single process-global ``sqlite3`` adapter that binds +``NaT`` as ``NULL``, matching how every other backend already treats a +missing temporal value. It does not touch Ibis's own type inference or +compiled SQL, so schema fidelity (including ``Boolean`` columns, which a +naive ``pandas.DataFrame.to_sql()`` bypass would silently degrade to +``Int64``) is unaffected. +""" +from __future__ import annotations + +_NAT_ADAPTER_INSTALLED = False + + +def ensure_sqlite_nat_adapter() -> None: + """Register a ``sqlite3`` adapter so pandas ``NaT`` binds as ``NULL``. + + Idempotent and process-global: ``sqlite3.register_adapter`` just + overwrites one dict entry, so repeated calls are a cheap no-op. Called + unconditionally (no dialect check needed -- registering the adapter is + a no-op for every other dialect's connection) at the top of every + ``IbisBackend`` write path that can reach + ``ibis.Backend._register_in_memory_table``: ``create_table``, ``insert``, + and ``compiled_source`` (the shared staging step behind every + ``upsert`` renderer). + + Silently returns if pandas is not importable: Ibis's own SQLite roundtrip + requires pandas too (``ibis-framework[sqlite]`` depends on it), so if + pandas is missing here, the crash this guards against cannot occur either. + """ + global _NAT_ADAPTER_INSTALLED + if _NAT_ADAPTER_INSTALLED: + return + + import sqlite3 + + try: + import pandas as pd + except ImportError: + return + + sqlite3.register_adapter(type(pd.NaT), lambda _: None) + _NAT_ADAPTER_INSTALLED = True diff --git a/src/mountainash_data/backends/ibis/backend.py b/src/mountainash_data/backends/ibis/backend.py index f936966..13b286c 100644 --- a/src/mountainash_data/backends/ibis/backend.py +++ b/src/mountainash_data/backends/ibis/backend.py @@ -12,6 +12,7 @@ from mountainash_data.backends.ibis.dialects._registry import DIALECTS, DialectSpec, TransactionSupport from mountainash_data.backends.ibis._transaction import run_transaction, is_active +from mountainash_data.backends.ibis._sqlite_compat import ensure_sqlite_nat_adapter from mountainash_data.backends.ibis._adoption import ( apply_options, snapshot_options, restore_options, ) @@ -625,6 +626,7 @@ def create_table( ) -> IbisBackend: conn = self._require_connected() rendered = _render_ibis_database(Namespace.coerce(namespace)) + ensure_sqlite_nat_adapter() conn._ibis_conn.create_table( name, obj=obj, schema=schema, database=rendered, temp=temp, overwrite=overwrite, @@ -678,6 +680,7 @@ def insert( ) -> IbisBackend: conn = self._require_connected() rendered = _render_ibis_database(Namespace.coerce(namespace)) + ensure_sqlite_nat_adapter() conn._ibis_conn.insert(name, obj=obj, database=rendered, overwrite=overwrite) return self diff --git a/tests/test_unit/backends/ibis/test_sqlite_nat_binding.py b/tests/test_unit/backends/ibis/test_sqlite_nat_binding.py new file mode 100644 index 0000000..3470dd3 --- /dev/null +++ b/tests/test_unit/backends/ibis/test_sqlite_nat_binding.py @@ -0,0 +1,137 @@ +"""Regression tests for DEBT-13 — sqlite write ops crash on null temporal +values (NaT binding). + +ibis's SQLite backend stages every in-memory table via a pandas roundtrip +(``op.data.to_frame()``) before binding rows through stdlib ``sqlite3``. A +null ``date``/``timestamp`` value becomes pandas ``NaT`` during that +roundtrip, and ``sqlite3`` has no adapter for ``NaTType`` — this crashes +``create_table``, ``insert``, and ``upsert`` for any frame containing a null +temporal value. Tracked upstream as ``IB-DT-19`` in +``mountainash/registry/upstream-issues.yaml`` (status: ``needs_filing``). + +See ``mountainash-central`` backlog: +``04.planning/mountainash-data/a.backlog/2026-08-18-sqlite-null-temporal-binding.md``. +""" + +import datetime +import subprocess +import sys +import textwrap + +import polars as pl +import pytest + +from mountainash_data.backends.ibis.backend import IbisBackend + + +class TestCreateTableNullTemporal: + def test_null_date_column(self): + df = pl.DataFrame( + {"id": [1, 2], "d": [datetime.date(2024, 1, 1), None]}, + schema={"id": pl.Int64, "d": pl.Date}, + ) + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", df) + result = backend.table("t").to_pandas() + assert result["d"].isna().sum() == 1 + + def test_null_datetime_column(self): + df = pl.DataFrame( + { + "id": [1, 2], + "ts": [datetime.datetime(2024, 1, 1, 12, 0, 0), None], + }, + schema={"id": pl.Int64, "ts": pl.Datetime}, + ) + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table("t", df) + result = backend.table("t").to_pandas() + assert result["ts"].isna().sum() == 1 + + +class TestInsertNullTemporal: + def test_null_datetime_column(self): + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table( + "t", None, schema={"id": "int64", "ts": "timestamp"} + ) + df = pl.DataFrame( + { + "id": [1, 2], + "ts": [datetime.datetime(2024, 1, 1, 12, 0, 0), None], + }, + schema={"id": pl.Int64, "ts": pl.Datetime}, + ) + backend.insert("t", df) + result = backend.table("t").to_pandas() + assert result["ts"].isna().sum() == 1 + + +class TestUpsertNullTemporal: + def test_null_datetime_column_update_style(self): + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table( + "t", None, schema={"id": "int64", "ts": "timestamp"} + ) + backend.create_index("t", ["id"], unique=True) + df = pl.DataFrame( + { + "id": [1, 2], + "ts": [datetime.datetime(2024, 1, 1, 12, 0, 0), None], + }, + schema={"id": pl.Int64, "ts": pl.Datetime}, + ) + backend.upsert("t", df, conflict_columns=["id"]) + result = backend.table("t").to_pandas() + assert result["ts"].isna().sum() == 1 + + def test_null_datetime_column_nothing_style(self): + with IbisBackend(dialect="sqlite", database=":memory:") as backend: + backend.create_table( + "t", None, schema={"id": "int64", "ts": "timestamp"} + ) + backend.create_index("t", ["id"], unique=True) + df = pl.DataFrame( + { + "id": [1, 2], + "ts": [datetime.datetime(2024, 1, 1, 12, 0, 0), None], + }, + schema={"id": pl.Int64, "ts": pl.Datetime}, + ) + backend.upsert( + "t", df, conflict_columns=["id"], conflict_action="NOTHING" + ) + result = backend.table("t").to_pandas() + assert result["ts"].isna().sum() == 1 + + +def test_raw_ibis_sqlite_null_temporal_upstream_bug_ib_dt_19(): + """Upstream-fix monitor (IB-DT-19), isolated from mountainash-data's own + process-global sqlite3 adapter patch via a subprocess. + + Reproduces the raw ibis-sqlite crash directly (no mountainash-data + involved) so this flips to a failure the moment ibis fixes the bug + upstream — a signal to remove mountainash-data's own workaround and + close DEBT-13/IB-DT-19 for good. + """ + script = textwrap.dedent( + """ + import ibis, polars as pl, datetime + con = ibis.sqlite.connect() + df = pl.DataFrame( + {"id": [1, 2], "ts": [datetime.datetime(2024, 1, 1), None]}, + schema={"id": pl.Int64, "ts": pl.Datetime}, + ) + con.create_table("t", df) + """ + ) + proc = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True + ) + if proc.returncode == 0: + pytest.fail( + "ibis-sqlite no longer crashes on null datetime binding — " + "IB-DT-19 appears fixed upstream. Remove mountainash-data's " + "_sqlite_compat workaround and close DEBT-13." + ) + assert "NaTType" in proc.stderr and "not supported" in proc.stderr