From 642c762aa4443d0e2249515cc367ac8e06f5e83f Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Sun, 16 Aug 2026 17:50:12 +0000 Subject: [PATCH 1/7] feat: native Spark add_months scalar function DuckDB lacks add_months, which appears in Spark SQL translated through lpts. Register add_months(DATE, INTEGER) -> DATE implementing full Spark semantics: day-of-month preserved, clamped to target month length, with the end-of-month rule (last day of source month maps to last day of target month). Reuses the DuckDB Date API for leap-year-correct month lengths. Registered in LoadInternal so any consumer loading lpts (or compiling its sources) gets the function. Adds test/sql/spark_add_months.test covering scalar correctness across month-end clamping, leap years, negative offsets, year rollover, and NULL propagation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CMakeLists.txt | 3 +- src/include/spark_scalar_functions.hpp | 11 +++++ src/lpts_extension.cpp | 5 ++ src/spark_scalar_functions.cpp | 52 +++++++++++++++++++++ test/sql/spark_add_months.test | 65 ++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 src/include/spark_scalar_functions.hpp create mode 100644 src/spark_scalar_functions.cpp create mode 100644 test/sql/spark_add_months.test diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ff1cfd..ba3e400 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,8 @@ set(EXTENSION_SOURCES src/lpts_ast_renderer.cpp src/lpts_ast_builder.cpp src/lpts_ast_flattener.cpp - src/dialect_function_map.cpp) + src/dialect_function_map.cpp + src/spark_scalar_functions.cpp) build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) diff --git a/src/include/spark_scalar_functions.hpp b/src/include/spark_scalar_functions.hpp new file mode 100644 index 0000000..4b1bdab --- /dev/null +++ b/src/include/spark_scalar_functions.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include "duckdb/main/extension/extension_loader.hpp" + +namespace duckdb { + +// Registers Spark-compatible scalar functions that DuckDB lacks natively but that +// appear in Spark SQL translated through lpts (compile/binding coverage). +void RegisterSparkScalarFunctions(ExtensionLoader &loader); + +} // namespace duckdb diff --git a/src/lpts_extension.cpp b/src/lpts_extension.cpp index 2f773a6..1e62079 100644 --- a/src/lpts_extension.cpp +++ b/src/lpts_extension.cpp @@ -8,6 +8,7 @@ #include "lpts_helpers.hpp" #include "lpts_debug.hpp" #include "lpts_parser.hpp" +#include "spark_scalar_functions.hpp" #include "duckdb.hpp" #include "duckdb/common/exception.hpp" @@ -1310,6 +1311,10 @@ static void LptsCheckOptimize(OptimizerExtensionInput &input, unique_ptr(year) * 12 + (month - 1) + num_months; + int32_t new_year = NumericCast(zero_based_months / 12); + int32_t new_month = static_cast(zero_based_months % 12); + if (new_month < 0) { + new_month += 12; + new_year -= 1; + } + new_month += 1; + + int32_t target_month_days = Date::MonthDays(new_year, new_month); + int32_t new_day = input_is_month_end ? target_month_days : MinValue(day, target_month_days); + return Date::FromDate(new_year, new_month, new_day); +} + +static void AddMonthsFunction(DataChunk &args, ExpressionState &state, Vector &result) { + BinaryExecutor::Execute( + args.data[0], args.data[1], result, args.size(), + [](date_t input, int32_t num_months) { return AddMonthsSparkImpl(input, num_months); }); +} + +void RegisterSparkScalarFunctions(ExtensionLoader &loader) { + ScalarFunction add_months("add_months", {LogicalType::DATE, LogicalType::INTEGER}, LogicalType::DATE, + AddMonthsFunction); + loader.RegisterFunction(add_months); +} + +} // namespace duckdb diff --git a/test/sql/spark_add_months.test b/test/sql/spark_add_months.test new file mode 100644 index 0000000..10fcc66 --- /dev/null +++ b/test/sql/spark_add_months.test @@ -0,0 +1,65 @@ +# name: test/sql/spark_add_months.test +# description: Spark-compatible add_months scalar function (scalar correctness) +# group: [sql] + +require lpts + +# Month-end input -> target month-end (with clamp) +query I +SELECT add_months(DATE '2015-01-31', 1) = DATE '2015-02-28'; +---- +true + +# Non month-end input, day clamps to target month length +query I +SELECT add_months(DATE '2015-01-30', 1) = DATE '2015-02-28'; +---- +true + +# Non month-end input, day fits +query I +SELECT add_months(DATE '2015-01-28', 1) = DATE '2015-02-28'; +---- +true + +# Leap-year target month-end +query I +SELECT add_months(DATE '2016-01-31', 1) = DATE '2016-02-29'; +---- +true + +# Feb month-end (non-leap) -> March month-end +query I +SELECT add_months(DATE '2015-02-28', 1) = DATE '2015-03-31'; +---- +true + +# Leap Feb month-end -> non-leap Feb month-end across a year +query I +SELECT add_months(DATE '2016-02-29', 12) = DATE '2017-02-28'; +---- +true + +# Negative months, month-end preserved +query I +SELECT add_months(DATE '2015-03-31', -1) = DATE '2015-02-28'; +---- +true + +# Year rollover, day preserved +query I +SELECT add_months(DATE '2015-01-15', 13) = DATE '2016-02-15'; +---- +true + +# December month-end -> January month-end (year rollover) +query I +SELECT add_months(DATE '2015-12-31', 1) = DATE '2016-01-31'; +---- +true + +# NULL propagation +query I +SELECT add_months(NULL::DATE, 1) IS NULL; +---- +true From 592d469f305fd56cff625e05b1860c86f18ffeae Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Sun, 16 Aug 2026 23:04:49 +0000 Subject: [PATCH 2/7] test: require icu for SPARK dialect timestamptz assertions CURRENT_TIMESTAMP folds to a TIMESTAMPTZ constant during lpts_query binding, which autoloads icu. Offline linux_amd64 CI cannot fetch icu and failed the whole file; `require icu` makes those runners skip instead (arm64 still runs it fully). Fixes the pre-existing dialect_spark.test amd64 failure surfaced on this add_months PR via the merge with main. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- test/sql/dialect_spark.test | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/sql/dialect_spark.test b/test/sql/dialect_spark.test index 1011b69..e6af677 100644 --- a/test/sql/dialect_spark.test +++ b/test/sql/dialect_spark.test @@ -9,6 +9,12 @@ require lpts +# Some SPARK-dialect assertions translate queries whose binding folds +# CURRENT_TIMESTAMP to a TIMESTAMPTZ constant, which pulls in icu. CI runners +# that cannot autoload icu (e.g. offline linux_amd64) would otherwise fail the +# whole file; require it so those runners skip instead of erroring. +require icu + statement ok CREATE TABLE users (id INTEGER, name VARCHAR, "order" INTEGER); From 77ed5e585f26e2702a7b2af3cc9e8623769256c8 Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Sat, 22 Aug 2026 20:06:14 +0000 Subject: [PATCH 3/7] Fix set-op column binding remaps Restore the lhs column_map before traversing each set-op sibling so later UNION/EXCEPT/INTERSECT branches cannot leak bindings into the parent. Also resolve projection refs above set-ops against the set-op output binding when a rewrite leaves a stale child binding in place. Add focused regressions for downstream joins over N-ary UNION ALL and for duplicated UNION ALL key projections. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/lpts_ast_builder.cpp | 86 ++++++++++++++++++++++-------- test/sql/fail_reduction.test | 100 +++++++++++++++++++++++++++++++++++ test/sql/union.test | 32 +++++++++++ 3 files changed, 195 insertions(+), 23 deletions(-) diff --git a/src/lpts_ast_builder.cpp b/src/lpts_ast_builder.cpp index 43f0e5a..0d4618b 100644 --- a/src/lpts_ast_builder.cpp +++ b/src/lpts_ast_builder.cpp @@ -99,6 +99,8 @@ class AstBuilder { } }; + using ColumnMap = std::map>; + // A set of emitted CTE column names with case-insensitive membership. DuckDB resolves identifiers // case-insensitively, so two generated names differing only in case (t1_hello vs t1_HeLlO) would // collide when referenced. Deduping against this set treats them as equal, so the second gets a suffix. @@ -142,6 +144,20 @@ class AstBuilder { return false; } + static bool IsSetOperationType(LogicalOperatorType type) { + return type == LogicalOperatorType::LOGICAL_UNION || type == LogicalOperatorType::LOGICAL_EXCEPT || + type == LogicalOperatorType::LOGICAL_INTERSECT; + } + + static ColumnMap CloneColumnMap(const ColumnMap &source) { + ColumnMap cloned; + for (const auto &entry : source) { + cloned[entry.first] = + make_uniq(entry.second->table_index, entry.second->column_name, entry.second->alias); + } + return cloned; + } + static void AddUniqueBinding(vector &bindings, const ColumnBinding &binding) { if (!HasBinding(bindings, binding)) { bindings.push_back(binding); @@ -462,7 +478,7 @@ class AstBuilder { /// Global map: ColumnBinding → ColStruct. /// Populated bottom-up; each operator registers its output columns here. - std::map> column_map; + ColumnMap column_map; /// Maps DELIM_GET table_index → source column names (from the outer/left CTE). /// Populated by PreregisterDelimGetColumns before the right subtree is traversed. @@ -491,6 +507,23 @@ class AstBuilder { context, (unsigned long long)binding.table_index, (unsigned long long)binding.column_index); } + bool TryResolveProjectionBinding(const LogicalProjection &proj, const ColumnBinding &binding, + ColumnBinding &resolved) const { + if (proj.children.empty()) { + return false; + } + const auto child_bindings = proj.children[0]->GetColumnBindings(); + resolved = binding; + if ((!HasBinding(child_bindings, resolved) && IsSetOperationType(proj.children[0]->type)) || + column_map.find(MappableColumnBinding(resolved)) == column_map.end()) { + if (resolved.column_index >= child_bindings.size()) { + return false; + } + resolved = child_bindings[resolved.column_index]; + } + return true; + } + void RegisterChildBindingFallbacks(Expression &expr, const vector &child_bindings) { if (expr.type == ExpressionType::BOUND_COLUMN_REF) { auto &bcr = expr.Cast(); @@ -508,6 +541,25 @@ class AstBuilder { }); } + void RegisterChildBindingFallbacks(Expression &expr, const LogicalProjection &proj) { + if (expr.type == ExpressionType::BOUND_COLUMN_REF) { + auto &bcr = expr.Cast(); + ColumnBinding resolved; + if (TryResolveProjectionBinding(proj, bcr.binding, resolved) && + (!(resolved == bcr.binding) || + column_map.find(MappableColumnBinding(bcr.binding)) == column_map.end())) { + auto &src = FindColumnBinding(resolved, "projection fallback"); + column_map[MappableColumnBinding(bcr.binding)] = + make_uniq(src->table_index, src->column_name, src->alias); + } + } + ExpressionIterator::EnumerateChildren(expr, [&](unique_ptr &child) { + if (child) { + RegisterChildBindingFallbacks(*child, proj); + } + }); + } + bool EnsureBindingAvailableFrom(LogicalOperator *op, const ColumnBinding &binding) { if (!op) { return false; @@ -835,12 +887,8 @@ class AstBuilder { } const auto child_bindings = proj.children[0]->GetColumnBindings(); auto &bcr = expr->Cast(); - resolved = bcr.binding; - if (column_map.find(MappableColumnBinding(resolved)) == column_map.end()) { - if (resolved.column_index >= child_bindings.size()) { - return false; - } - resolved = child_bindings[resolved.column_index]; + if (!TryResolveProjectionBinding(proj, bcr.binding, resolved)) { + return false; } if (ordinal >= child_bindings.size()) { return false; @@ -1555,12 +1603,8 @@ class AstBuilder { if (expr->type == ExpressionType::BOUND_COLUMN_REF) { BoundColumnRefExpression &bcr = expr->Cast(); ColumnBinding lookup_binding = bcr.binding; - if (column_map.find(MappableColumnBinding(lookup_binding)) == column_map.end() && - !proj.children.empty()) { - auto child_bindings = proj.children[0]->GetColumnBindings(); - if (lookup_binding.column_index < child_bindings.size()) { - lookup_binding = child_bindings[lookup_binding.column_index]; - } + if (TryResolveProjectionBinding(proj, bcr.binding, lookup_binding)) { + // lookup_binding already resolved against the set-op child when needed. } const unique_ptr &desc = FindColumnBinding(lookup_binding, "projection"); const string src_name = desc->ToUniqueColumnName(); @@ -1585,7 +1629,7 @@ class AstBuilder { column_map[MappableColumnBinding(new_cb)] = std::move(new_col); } else { if (!proj.children.empty()) { - RegisterChildBindingFallbacks(*expr, proj.children[0]->GetColumnBindings()); + RegisterChildBindingFallbacks(*expr, proj); } string expr_str = ExpressionToAliasedString(expr); expressions.emplace_back(expr_str); @@ -2617,19 +2661,15 @@ class AstBuilder { unique_ptr RecursiveTraversal(unique_ptr &op, bool is_root = false) { // 1. Recurse into children first (post-order). vector> child_nodes; - if ((op->type == LogicalOperatorType::LOGICAL_UNION || op->type == LogicalOperatorType::LOGICAL_EXCEPT || - op->type == LogicalOperatorType::LOGICAL_INTERSECT) && - op->children.size() >= 2) { + if (IsSetOperationType(op->type) && op->children.size() >= 2) { // Set operations: scope column_map to prevent sibling children from overwriting // each other's entries when subtrees share table indices. child_nodes.push_back(RecursiveTraversal(op->children[0])); - // Save column_map after first child; restore before each subsequent child - std::map> saved_map; - for (auto &entry : column_map) { - saved_map[entry.first] = - make_uniq(entry.second->table_index, entry.second->column_name, entry.second->alias); - } + // Save the lhs-visible bindings, then replay each sibling under that same scope so one branch + // cannot leak its table-index mappings into another or into the parent set-op output. + ColumnMap saved_map = CloneColumnMap(column_map); for (size_t ci = 1; ci < op->children.size(); ci++) { + column_map = CloneColumnMap(saved_map); child_nodes.push_back(RecursiveTraversal(op->children[ci])); } column_map = std::move(saved_map); diff --git a/test/sql/fail_reduction.test b/test/sql/fail_reduction.test index d6eae5c..aa7546f 100644 --- a/test/sql/fail_reduction.test +++ b/test/sql/fail_reduction.test @@ -254,6 +254,106 @@ query I 2 3 +# --- A projection that duplicates a UNION ALL key must render both aliases from the set-op output, +# not a stale child binding from one branch. Closest SQL shape to the OpenIVM hidden-left-key rewrite. --- +statement ok +CREATE TABLE isf_bugf(instance_arm_collection_key INT, event_time INT); + +statement ok +INSERT INTO isf_bugf VALUES (1, 10), (2, 20); + +statement ok +CREATE TABLE acd_bugf(arm_collection_key INT, subscription_id INT, arm_id VARCHAR, rn INT); + +statement ok +INSERT INTO acd_bugf VALUES (1, 100, 'arm-1', 1), (2, 200, 'arm-2', 1); + +statement ok +CREATE TABLE imlmv_bugf(instance_arm_id VARCHAR, container_resource_id VARCHAR); + +statement ok +INSERT INTO imlmv_bugf VALUES ('arm-1', 'machine-1'); + +statement ok +CREATE TABLE cores_bugf(instance_arm_id VARCHAR, instance_effective_cores DOUBLE); + +statement ok +INSERT INTO cores_bugf VALUES ('arm-1', 4.0), ('arm-2', 8.0); + +query TT rowsort +WITH base_filtered_fact AS ( + SELECT isf.event_time, acd.subscription_id, acd.arm_id AS instance_arm_id, + lower(coalesce(imlmv.container_resource_id, 'NA')) AS container_resource_id + FROM isf_bugf isf + INNER JOIN (SELECT arm_collection_key, subscription_id, arm_id FROM acd_bugf WHERE rn = 1) AS acd + ON isf.instance_arm_collection_key = acd.arm_collection_key + LEFT JOIN imlmv_bugf imlmv ON acd.arm_id = imlmv.instance_arm_id +), +keyed AS ( + SELECT bff.event_time, bff.subscription_id, bff.instance_arm_id, bff.container_resource_id, + coalesce(cores.instance_effective_cores, 0.0) AS instance_effective_cores + FROM base_filtered_fact AS bff + LEFT JOIN cores_bugf cores ON bff.instance_arm_id = cores.instance_arm_id + WHERE lower(bff.container_resource_id) <> lower('NA') OR bff.container_resource_id IS NULL +), +no_machine AS ( + SELECT bff.event_time, bff.subscription_id, bff.instance_arm_id, bff.container_resource_id, + coalesce(cores.instance_effective_cores, 0.0) AS instance_effective_cores + FROM base_filtered_fact AS bff + LEFT JOIN cores_bugf cores ON bff.instance_arm_id = cores.instance_arm_id + WHERE lower(bff.container_resource_id) = lower('NA') +) +SELECT combined.instance_arm_id AS instance_arm_id, combined.instance_arm_id AS openivm_left_key +FROM ( + SELECT keyed.subscription_id, keyed.instance_arm_id, keyed.instance_effective_cores FROM keyed + UNION ALL + SELECT no_machine.subscription_id, no_machine.instance_arm_id, no_machine.instance_effective_cores FROM no_machine +) combined +ORDER BY 1, 2; +---- +arm-1 arm-1 +arm-2 arm-2 + +query I +SELECT regexp_extract(sql, + '(?s).*SELECT\\s+([^ ,]+) AS instance_arm_id,\\s+([^ ,]+) AS openivm_left_key.*', + 1) = + regexp_extract(sql, + '(?s).*SELECT\\s+([^ ,]+) AS instance_arm_id,\\s+([^ ,]+) AS openivm_left_key.*', + 2) +FROM lpts_query($$ +WITH base_filtered_fact AS ( + SELECT isf.event_time, acd.subscription_id, acd.arm_id AS instance_arm_id, + lower(coalesce(imlmv.container_resource_id, 'NA')) AS container_resource_id + FROM isf_bugf isf + INNER JOIN (SELECT arm_collection_key, subscription_id, arm_id FROM acd_bugf WHERE rn = 1) AS acd + ON isf.instance_arm_collection_key = acd.arm_collection_key + LEFT JOIN imlmv_bugf imlmv ON acd.arm_id = imlmv.instance_arm_id +), +keyed AS ( + SELECT bff.event_time, bff.subscription_id, bff.instance_arm_id, bff.container_resource_id, + coalesce(cores.instance_effective_cores, 0.0) AS instance_effective_cores + FROM base_filtered_fact AS bff + LEFT JOIN cores_bugf cores ON bff.instance_arm_id = cores.instance_arm_id + WHERE lower(bff.container_resource_id) <> lower('NA') OR bff.container_resource_id IS NULL +), +no_machine AS ( + SELECT bff.event_time, bff.subscription_id, bff.instance_arm_id, bff.container_resource_id, + coalesce(cores.instance_effective_cores, 0.0) AS instance_effective_cores + FROM base_filtered_fact AS bff + LEFT JOIN cores_bugf cores ON bff.instance_arm_id = cores.instance_arm_id + WHERE lower(bff.container_resource_id) = lower('NA') +) +SELECT combined.instance_arm_id AS instance_arm_id, combined.instance_arm_id AS openivm_left_key +FROM ( + SELECT keyed.subscription_id, keyed.instance_arm_id, keyed.instance_effective_cores FROM keyed + UNION ALL + SELECT no_machine.subscription_id, no_machine.instance_arm_id, no_machine.instance_effective_cores FROM no_machine +) combined +$$); +---- +true + # --- A pushed-down filter on a column that is NOT projected (filter-only) must still resolve through # the table-function alias. --- query I diff --git a/test/sql/union.test b/test/sql/union.test index a4c7518..e8c54eb 100644 --- a/test/sql/union.test +++ b/test/sql/union.test @@ -146,6 +146,38 @@ INSERT INTO extra VALUES (99); statement ok SELECT id FROM users UNION ALL SELECT id FROM staff UNION ALL SELECT user_id FROM orders UNION ALL SELECT val FROM extra; +# --- N-ary UNION ALL with a downstream join/projection keeps each sibling under the lhs scope. --- +statement ok +CREATE TABLE union_labels(id INTEGER, tag VARCHAR); + +statement ok +INSERT INTO union_labels VALUES (1, 'one'), (2, 'two'), (99, 'ninety-nine'); + +query IIT rowsort +SELECT u.id, l.tag, u.label +FROM ( + SELECT u.id, u.name AS label + FROM users u + INNER JOIN orders o ON u.id = o.user_id + UNION ALL + SELECT s.id, s.dept AS label + FROM staff s + INNER JOIN orders o ON s.id = o.user_id + UNION ALL + SELECT val AS id, 'extra' AS label + FROM extra +) u +LEFT JOIN union_labels l ON u.id = l.id +ORDER BY 1, 2, 3; +---- +1 one Alice +1 one Alice +1 one eng +1 one eng +2 two Bob +2 two sales +99 ninety-nine extra + # --- N-ary UNION with aggregate on top (regression) --- # The AstFlattener splits N-ary UNION into left-deep binary UNIONs and appends # intermediate CTEs to cte_nodes. Parent operators must reference each child From 754c797781c9b56429e05c1781cea5ca99c628e8 Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Sun, 23 Aug 2026 06:02:41 +0000 Subject: [PATCH 4/7] Fix bounded Spark HUGEINT and UNION aliases Map only provably bounded HUGEINT values to Spark DECIMAL(38,0), and retain trailing rewritten UNION bindings as aliases of their physical multiplicity output. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/lpts_ast_builder.cpp | 21 ++++++++++++- src/lpts_expression_renderer.cpp | 46 ++++++++++++++++++++++++++- test/sql/dialect_spark.test | 35 +++++++++++++++++++++ test/sql/union.test | 54 ++++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 2 deletions(-) diff --git a/src/lpts_ast_builder.cpp b/src/lpts_ast_builder.cpp index 0d4618b..376bbca 100644 --- a/src/lpts_ast_builder.cpp +++ b/src/lpts_ast_builder.cpp @@ -2128,17 +2128,36 @@ class AstBuilder { vector cte_column_names; const auto &lhs_bindings = op->children[0]->GetColumnBindings(); const auto &union_bindings = op->GetColumnBindings(); + const idx_t physical_column_count = + op->types.empty() ? std::min(lhs_bindings.size(), union_bindings.size()) : op->types.size(); + if (lhs_bindings.size() < physical_column_count || union_bindings.size() < physical_column_count) { + throw InternalException("LPTS UNION: physical output arity exceeds available column bindings"); + } // Two union output columns can derive from source columns with the same name (e.g. // SELECT t1.a, t2.a ... UNION ...), which would emit a header like (t7_a, t7_a) — DuckDB // resolves later references to the first, silently dropping the second column. Dedup so each // output column gets a distinct generated name. CaseInsensitiveNameSet seen_names; - for (size_t i = 0; i < lhs_bindings.size(); ++i) { + for (idx_t i = 0; i < physical_column_count; ++i) { const unique_ptr &lhs_col = FindColumnBinding(lhs_bindings[i], "union lhs"); auto new_col = MakeDedupedColumn(table_index, lhs_col->column_name, lhs_col->alias, seen_names, 1); cte_column_names.push_back(new_col->ToUniqueColumnName()); column_map[MappableColumnBinding(union_bindings[i])] = std::move(new_col); } + if (union_bindings.size() > physical_column_count) { + // Some post-optimizer rewrites expose one trailing alias binding for the + // physical multiplicity column without widening the UNION types or children. + // Preserve that binding by pointing it at the existing last positional output. + if (physical_column_count == 0 || union_bindings.size() != physical_column_count + 1) { + throw NotImplementedException( + "LPTS_UNSUPPORTED_COLUMN_REF: UNION exposes %llu bindings for %llu physical columns", + (unsigned long long)union_bindings.size(), (unsigned long long)physical_column_count); + } + const auto &last_output = + FindColumnBinding(union_bindings[physical_column_count - 1], "union trailing alias source"); + column_map[MappableColumnBinding(union_bindings[physical_column_count])] = + make_uniq(last_output->table_index, last_output->column_name, last_output->alias); + } return make_uniq(set_op.setop_all, std::move(cte_column_names)); } diff --git a/src/lpts_expression_renderer.cpp b/src/lpts_expression_renderer.cpp index 4efc035..f6ea14c 100644 --- a/src/lpts_expression_renderer.cpp +++ b/src/lpts_expression_renderer.cpp @@ -360,6 +360,26 @@ static string RenderCastTargetType(const LogicalType &type, SqlDialect dialect) } } +static bool SparkCanRepresentHugeintCastAsDecimal38(const LogicalType &source_type) { + switch (source_type.id()) { + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::UTINYINT: + case LogicalTypeId::USMALLINT: + case LogicalTypeId::UINTEGER: + case LogicalTypeId::UBIGINT: + case LogicalTypeId::DECIMAL: + case LogicalTypeId::SQLNULL: + // These source domains fit exactly in Spark DECIMAL(38,0). DuckDB + // HUGEINT itself does not: its signed 128-bit range needs 39 digits. + return true; + default: + return false; + } +} + static string RenderValueForDialect(const Value &value, SqlDialect dialect) { if (value.IsNull()) { // An untyped NULL literal (SQLNULL) has no target-dialect cast type; emit a @@ -370,6 +390,9 @@ static string RenderValueForDialect(const Value &value, SqlDialect dialect) { if (value.type().id() == LogicalTypeId::SQLNULL) { return "NULL"; } + if (dialect == SqlDialect::SPARK && value.type().id() == LogicalTypeId::HUGEINT) { + return "CAST(NULL AS DECIMAL(38,0))"; + } return "CAST(NULL AS " + RenderCastTargetType(value.type(), dialect) + ")"; } switch (value.type().id()) { @@ -390,6 +413,17 @@ static string RenderValueForDialect(const Value &value, SqlDialect dialect) { return "DATE '" + EscapeSingleQuotes(value.ToString()) + "'"; case LogicalTypeId::TIMESTAMP: return "TIMESTAMP '" + EscapeSingleQuotes(value.ToString()) + "'"; + case LogicalTypeId::HUGEINT: + if (dialect == SqlDialect::SPARK) { + const string integer = value.ToString(); + const idx_t digits = !integer.empty() && integer[0] == '-' ? integer.size() - 1 : integer.size(); + if (digits <= 38) { + return "CAST(" + integer + " AS DECIMAL(38,0))"; + } + ThrowLptsNotImplemented("LPTS_UNSUPPORTED_TYPE", dialect, "type", value.type().ToString(), "BOUND_CONSTANT", + "the HUGEINT constant needs 39 digits, beyond Spark DECIMAL(38,0)"); + } + return value.ToSQLString(); case LogicalTypeId::VARCHAR: return "'" + EscapeSingleQuotes(value.GetValue()) + "'"; case LogicalTypeId::INTERVAL: { @@ -1340,7 +1374,17 @@ string LptsExpressionRenderer::ExpressionToAliasedString(const unique_ptrreturn_type)) { + expr_str << " AS DECIMAL(38,0))"; + } else if (dialect == SqlDialect::SPARK && cast_expr.return_type.id() == LogicalTypeId::HUGEINT) { + ThrowLptsNotImplemented("LPTS_UNSUPPORTED_TYPE", dialect, "type", cast_expr.return_type.ToString(), + "BOUND_CAST", + "source type " + cast_expr.child->return_type.ToString() + + " may use DuckDB HUGEINT's 39-digit range, beyond Spark DECIMAL(38,0)"); + } else { + expr_str << " AS " + RenderCastTargetType(cast_expr.return_type, dialect) + ")"; + } break; } case ExpressionClass::BOUND_CONJUNCTION: { diff --git a/test/sql/dialect_spark.test b/test/sql/dialect_spark.test index e6af677..f88c646 100644 --- a/test/sql/dialect_spark.test +++ b/test/sql/dialect_spark.test @@ -212,6 +212,41 @@ FROM lpts_query('SELECT CAST(d AS VARCHAR), substring(name, 1, 2) FROM compiler_ ---- true true +# DuckDB/OpenIVM introduces HUGEINT casts while widening ordinary integer +# expressions. Spark has no 128-bit integer, so use DECIMAL(38,0) only when the +# source domain is provably no wider than 38 digits. +query II +SELECT sql LIKE '%CAST(id AS DECIMAL(38,0))%' AS bounded_hugeint_cast, + sql NOT LIKE '%HUGEINT%' AS no_spark_hugeint +FROM lpts_query('SELECT CAST(id AS HUGEINT) FROM users'); +---- +true true + +# Exact aggregate shape from the nine canary failures: SUM(BIGINT) widens to +# HUGEINT, so DuckDB binds the COALESCE fallback through a HUGEINT cast. +query I +SELECT sql LIKE '%COALESCE(%CAST(0 AS DECIMAL(38,0))%)%' AS bounded_sum_fallback +FROM lpts_query('SELECT CAST(COALESCE(SUM(CAST(id AS BIGINT)), 0) AS INTEGER) FROM users'); +---- +true + +query II +SELECT sql LIKE '%CAST(99999999999999999999999999999999999999 AS DECIMAL(38,0))%' AS bounded_hugeint_constant, + sql LIKE '%CAST(NULL AS DECIMAL(38,0))%' AS typed_hugeint_null +FROM lpts_query('SELECT 99999999999999999999999999999999999999::HUGEINT, NULL::HUGEINT'); +---- +true true + +statement error +SELECT * FROM lpts_query('SELECT CAST(name AS HUGEINT) FROM users'); +---- +LPTS_UNSUPPORTED_TYPE + +statement error +SELECT * FROM lpts_query('SELECT 170141183460469231731687303715884105727::HUGEINT'); +---- +LPTS_UNSUPPORTED_TYPE + query II SELECT sql LIKE '%INTERVAL ''30'' DAY%' AS spark_interval, sql NOT LIKE '%::INTERVAL%' AS no_postgres_interval_cast diff --git a/test/sql/union.test b/test/sql/union.test index e8c54eb..942c2d9 100644 --- a/test/sql/union.test +++ b/test/sql/union.test @@ -178,6 +178,60 @@ ORDER BY 1, 2, 3; 2 two sales 99 ninety-nine extra +# --- OpenIVM join-delta UNION terms carry eleven visible machine-status columns, +# followed by a duplicated hidden left key and multiplicity. A left-deep UNION +# must preserve and bind all thirteen positions, including the hidden key. --- +statement ok +CREATE TABLE machine_status_terms ( + event_time INTEGER, + machine_arm_id VARCHAR, + customer_key INTEGER, + subscription_offer_type_key INTEGER, + subscription_workload_type_key INTEGER, + infrastructure_key INTEGER, + operating_system_key INTEGER, + mssqldiscovered BOOLEAN, + mysqldiscovered BOOLEAN, + pgsqldiscovered BOOLEAN, + physical_cores INTEGER +); + +statement ok +INSERT INTO machine_status_terms VALUES + (10, 'arm-1', 1, 2, 3, 4, 5, true, false, false, 8), + (20, 'arm-2', 6, 7, 8, 9, 10, false, true, false, 16), + (30, 'arm-3', 11, 12, 13, 14, 15, false, false, true, 32); + +query ITIIIIIIIIITI rowsort +SELECT * +FROM ( + SELECT event_time, machine_arm_id, customer_key, subscription_offer_type_key, + subscription_workload_type_key, infrastructure_key, operating_system_key, + mssqldiscovered, mysqldiscovered, pgsqldiscovered, physical_cores, + machine_arm_id AS openivm_left_key, 1 AS openivm_multiplicity + FROM machine_status_terms + WHERE event_time = 10 + UNION ALL + SELECT event_time, machine_arm_id, customer_key, subscription_offer_type_key, + subscription_workload_type_key, infrastructure_key, operating_system_key, + mssqldiscovered, mysqldiscovered, pgsqldiscovered, physical_cores, + machine_arm_id AS openivm_left_key, -1 AS openivm_multiplicity + FROM machine_status_terms + WHERE event_time = 20 + UNION ALL + SELECT event_time, machine_arm_id, customer_key, subscription_offer_type_key, + subscription_workload_type_key, infrastructure_key, operating_system_key, + mssqldiscovered, mysqldiscovered, pgsqldiscovered, physical_cores, + machine_arm_id AS openivm_left_key, 1 AS openivm_multiplicity + FROM machine_status_terms + WHERE event_time = 30 +) AS delta_terms +ORDER BY event_time; +---- +10 arm-1 1 2 3 4 5 true false false 8 arm-1 1 +20 arm-2 6 7 8 9 10 false true false 16 arm-2 -1 +30 arm-3 11 12 13 14 15 false false true 32 arm-3 1 + # --- N-ary UNION with aggregate on top (regression) --- # The AstFlattener splits N-ary UNION into left-deep binary UNIONs and appends # intermediate CTEs to cte_nodes. Parent operators must reference each child From 66bf3ae66bd4a2072a3676866a440047d46043f2 Mon Sep 17 00:00:00 2001 From: Copilot CLI Date: Mon, 24 Aug 2026 20:39:03 +0000 Subject: [PATCH 5/7] Represent Spark/Delta time-travel pins instead of failing to parse them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Spark/Delta temporal clause (`FROM t VERSION AS OF 366`) reached DuckDB's parser verbatim, so any pinned relation died with `Parser Error: syntax error at or near "as"` before a plan was ever built. Downstream that surfaced as a silent correctness fallback: OpenIVM's `openivm_compile_with_facts` produced no result, the refresh span reported `compile_refresh_type=COMPILE_FAILED effective_refresh_type=FULL_REFRESH reason=compile_failed`, and the pinned snapshot the user asked for was lost. Input side (`lpts_input_dialect = 'spark'`): the temporal clause now normalizes to DuckDB's semantically equivalent `AT (...)` clause — `[FOR] VERSION|SYSTEM_VERSION AS OF ` becomes `AT (VERSION => )` and `[FOR] TIMESTAMP|SYSTEM_TIME AS OF ''` becomes `AT (TIMESTAMP => '')`. The pin is represented, never dropped: dropping it would silently promote every pinned scan to "read latest" and change the meaning of the query. A match requires the full ` AS OF ` sequence, so a column or alias merely named `version`/`timestamp` is untouched, as are string literals. A timestamp pin given a bare number is refused rather than mis-pinned. Output side: LPTS already carried a pin as a DuckDB `AT (...)` suffix on the table name (DuckLake time travel), but emitted that DuckDB spelling into every dialect. The suffix is now rendered per dialect — Spark gets `VERSION AS OF ` / `TIMESTAMP AS OF ''`, DuckDB keeps `AT (...)`, and a dialect with no verified time-travel syntax raises `LPTS_UNSUPPORTED_TIME_TRAVEL` instead of emitting SQL the target cannot parse. Unqualified renderings (Postgres/Redshift/Feldera, inline SQL) went through a raw `table_name` path that also leaked the suffix and then mistook its parentheses for a table-function argument list (`products AT (VERSION => 2) _tf("1")`); those paths now split the pin off explicitly. `test/sql/time_travel.test` pins the contract end to end: normalization fixtures for both temporal forms and the negative cases, and a DuckLake table whose version 2 holds 2 rows while the latest holds 3, so a dropped pin is observable in the result under `lpts_check`, not just in the generated SQL. Note for the pinned-scan compile path: a plain DuckDB catalog now answers `Binder Error: Catalog type does not support time travel` instead of a parser error. That is the honest outcome — LPTS represents the pin and the catalog decides — and it is the downstream signal OpenIVM must handle when it registers schema-only fact tables. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CLAUDE.md | 1 + src/cte_nodes.cpp | 8 ++ src/include/lpts_helpers.hpp | 6 ++ src/lpts_ast_flattener.cpp | 7 ++ src/lpts_helpers.cpp | 62 ++++++++++- src/lpts_parser.cpp | 89 ++++++++++++++++ test/sql/time_travel.test | 201 +++++++++++++++++++++++++++++++++++ 7 files changed, 370 insertions(+), 4 deletions(-) create mode 100644 test/sql/time_travel.test diff --git a/CLAUDE.md b/CLAUDE.md index f5246cf..87e201a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -251,6 +251,7 @@ CteBaseNode (base, ToQuery(SqlDialect)) | `test/sql/data_dependent_optimizers.test` | `lpts_enable_data_dependent_optimizers` | | `test/sql/tpch.test` | All 22 TPC-H queries under `lpts_check` | | `test/sql/ducklake.test` | DuckLake scans | +| `test/sql/time_travel.test` | Snapshot pinning: Spark/Delta `VERSION`/`TIMESTAMP AS OF` ↔ DuckDB `AT (...)` | | `test/sql/explain_format_sql.test` | `EXPLAIN (FORMAT SQL)` | | `test/sql/print_ast.test` | AST `ToString()` output | | `test/sql/check_mode.test` | `lpts_check` round-trip semantics (canonical example) | diff --git a/src/cte_nodes.cpp b/src/cte_nodes.cpp index f726332..1a3f097 100644 --- a/src/cte_nodes.cpp +++ b/src/cte_nodes.cpp @@ -384,9 +384,17 @@ string GetNode::ToQuery(SqlDialect dialect) { get_str << VecToSeparatedList(RenderGetSelectColumns(column_names, column_is_expression, dialect)); } get_str << " FROM "; + string base_table_name; + string snapshot_suffix; + const bool unqualified_snapshot = + catalog.empty() && TrySplitDialectSnapshotSuffix(table_name, dialect, base_table_name, snapshot_suffix); if (!catalog.empty()) { // Fully-qualified: catalog.schema.table (DuckDB / Spark dialect) get_str << DialectQualifiedTableName(catalog, schema, table_name, dialect); + } else if (unqualified_snapshot) { + // A pinned-snapshot scan rendered unqualified: the qualifier is dialect-specific and must not be + // mistaken for a table-function argument list by the `_tf` aliasing below. + get_str << base_table_name << snapshot_suffix; } else { // A TABLE-argument function: the child CTE is the function's argument, not a lateral input. const size_t table_arg_pos = table_name.find("%LPTS_TABLE_ARG%"); diff --git a/src/include/lpts_helpers.hpp b/src/include/lpts_helpers.hpp index f1cf3df..1b18534 100644 --- a/src/include/lpts_helpers.hpp +++ b/src/include/lpts_helpers.hpp @@ -25,6 +25,12 @@ string VecToQuotedIdentifierList(const vector &input_list, const string /// Quote a table name, preserving a DuckDB AT (...) snapshot suffix if present. string QuoteTableWithOptionalSuffix(const string &table_name); +/// Split a table name that carries a pinned snapshot (`name AT ( => )`, the DuckDB +/// spelling LPTS uses internally) into `base_name` and the `dialect`-rendered snapshot qualifier +/// (e.g. ` VERSION AS OF 366` for Spark). Returns false and leaves the outputs untouched when +/// `table_name` carries no snapshot. Throws when `dialect` has no verified time-travel syntax. +bool TrySplitDialectSnapshotSuffix(const string &table_name, SqlDialect dialect, string &base_name, string &suffix); + /// Build catalog.schema.table with each identifier quoted when needed. string QualifiedTableName(const string &catalog, const string &schema, const string &table_name); diff --git a/src/lpts_ast_flattener.cpp b/src/lpts_ast_flattener.cpp index 15d4098..cf86597 100644 --- a/src/lpts_ast_flattener.cpp +++ b/src/lpts_ast_flattener.cpp @@ -289,8 +289,15 @@ class AstFlattener { } } sql += " FROM "; + string inline_base_name; + string inline_snapshot_suffix; if (!get.catalog.empty()) { sql += DialectQualifiedTableName(get.catalog, get.schema, get.table_name, dialect); + } else if (TrySplitDialectSnapshotSuffix(get.table_name, dialect, inline_base_name, + inline_snapshot_suffix)) { + // Pinned-snapshot scan rendered unqualified: the dialect-specific qualifier must not be + // mistaken for a table-function argument list by the `_tf` aliasing below. + sql += inline_base_name + inline_snapshot_suffix; } else { // An in-out (lateral) table function has the delim/correlation source as its AST child: // inline it as the left comma-join input (mirrors GetNode's input_cte_name handling). diff --git a/src/lpts_helpers.cpp b/src/lpts_helpers.cpp index ad4da29..d8b335a 100644 --- a/src/lpts_helpers.cpp +++ b/src/lpts_helpers.cpp @@ -81,13 +81,67 @@ string DialectVecToQuotedIdentifierList(const vector &input_list, SqlDia return ret_str.str(); } -string DialectQuoteTableWithOptionalSuffix(const string &table_name, SqlDialect dialect) { - static const string at_suffix = " AT ("; - auto suffix_pos = table_name.find(at_suffix); +/// Split `name AT ( => )` — the DuckDB spelling LPTS uses internally to carry a pinned +/// snapshot on a table name — into its base name and `AT` parameter/value. +static bool TrySplitSnapshotSuffix(const string &table_name, string &base_name, string &at_parameter, + string &at_value) { + static const string AT_SUFFIX = " AT ("; + auto suffix_pos = table_name.find(AT_SUFFIX); if (suffix_pos == string::npos) { + return false; + } + string body = table_name.substr(suffix_pos + AT_SUFFIX.size()); + if (body.empty() || body.back() != ')') { + return false; + } + body.pop_back(); + auto arrow_pos = body.find("=>"); + if (arrow_pos == string::npos) { + return false; + } + base_name = table_name.substr(0, suffix_pos); + at_parameter = TrimCopy(body.substr(0, arrow_pos)); + at_value = TrimCopy(body.substr(arrow_pos + 2)); + return !at_parameter.empty() && !at_value.empty(); +} + +/// Render a pinned-snapshot qualifier in `dialect`. LPTS carries the pin in DuckDB's spelling +/// (`AT (VERSION => 366)`); Spark/Delta spells the same pin `VERSION AS OF 366`. A dialect with no +/// verified time-travel syntax refuses instead of emitting a qualifier the target cannot parse — +/// dropping it would silently turn a pinned scan into a read of the latest snapshot. +static string DialectSnapshotSuffix(const string &base_name, const string &at_parameter, const string &at_value, + SqlDialect dialect) { + if (dialect == SqlDialect::DUCKDB) { + return " AT (" + at_parameter + " => " + at_value + ")"; + } + string parameter = LowerCopy(at_parameter); + if (dialect == SqlDialect::SPARK && (parameter == "version" || parameter == "timestamp")) { + return (parameter == "version" ? string(" VERSION AS OF ") : string(" TIMESTAMP AS OF ")) + at_value; + } + ThrowLptsNotImplemented("LPTS_UNSUPPORTED_TIME_TRAVEL", dialect, "time_travel", at_parameter + " => " + at_value, + base_name, "no verified time-travel syntax for target dialect"); +} + +string DialectQuoteTableWithOptionalSuffix(const string &table_name, SqlDialect dialect) { + string base_name; + string at_parameter; + string at_value; + if (!TrySplitSnapshotSuffix(table_name, base_name, at_parameter, at_value)) { return DialectQuoteIdent(table_name, dialect); } - return DialectQuoteIdent(table_name.substr(0, suffix_pos), dialect) + table_name.substr(suffix_pos); + return DialectQuoteIdent(base_name, dialect) + DialectSnapshotSuffix(base_name, at_parameter, at_value, dialect); +} + +bool TrySplitDialectSnapshotSuffix(const string &table_name, SqlDialect dialect, string &base_name, string &suffix) { + string at_parameter; + string at_value; + string split_base; + if (!TrySplitSnapshotSuffix(table_name, split_base, at_parameter, at_value)) { + return false; + } + suffix = DialectSnapshotSuffix(split_base, at_parameter, at_value, dialect); + base_name = split_base; + return true; } string DialectQualifiedTableName(const string &catalog, const string &schema, const string &table_name, diff --git a/src/lpts_parser.cpp b/src/lpts_parser.cpp index 2279cfa..4647cfb 100644 --- a/src/lpts_parser.cpp +++ b/src/lpts_parser.cpp @@ -108,6 +108,94 @@ static string NormalizeBracketIdentifiers(const string &sql, SqlDialect dialect) return result; } +/// Match a Spark/Delta temporal-clause keyword at `pos` and map it to the DuckDB `AT (...)` named +/// parameter that carries the same snapshot. `VERSION`/`SYSTEM_VERSION` pin a commit version, +/// `TIMESTAMP`/`SYSTEM_TIME` pin a point in time. +static bool TryReadTimeTravelKeyword(const string &sql, idx_t pos, idx_t &end, string &at_parameter) { + struct TimeTravelKeyword { + const char *keyword; + const char *at_parameter; + }; + static const TimeTravelKeyword KEYWORDS[] = {{"system_version", "VERSION"}, + {"version", "VERSION"}, + {"system_time", "TIMESTAMP"}, + {"timestamp", "TIMESTAMP"}}; + for (const auto &candidate : KEYWORDS) { + if (MatchesKeywordAt(sql, pos, candidate.keyword)) { + end = pos + strlen(candidate.keyword); + at_parameter = candidate.at_parameter; + return true; + } + } + return false; +} + +/// Rewrite the Spark/Delta time-travel clause into DuckDB's `AT (...)` snapshot clause: +/// +/// FROM t [FOR] VERSION AS OF 366 -> FROM t AT (VERSION => 366) +/// FROM t [FOR] TIMESTAMP AS OF '2024-...' -> FROM t AT (TIMESTAMP => '2024-...') +/// +/// The pinned snapshot is *represented*, never dropped: DuckDB's `AT` clause is the semantically +/// equivalent spelling, so the plan keeps reading the pinned snapshot and the generated SQL can +/// render the pin back out in the target dialect. Dropping the clause here would silently promote +/// every pinned scan to "read latest", changing the meaning of the query. +/// +/// A match requires the full ` AS OF ` sequence, so a column or alias merely named +/// `version` / `timestamp` is left alone. +static string RewriteTimeTravelClauses(const string &sql, SqlDialect dialect) { + if (dialect != SqlDialect::SPARK) { + return sql; + } + + string result; + for (idx_t i = 0; i < sql.size(); i++) { + if (TryAppendSkippableSqlSpan(sql, i, result)) { + continue; + } + + idx_t keyword_start = i; + if (MatchesKeywordAt(sql, i, "for")) { + keyword_start = SkipWhitespace(sql, i + 3); + } + idx_t keyword_end = keyword_start; + string at_parameter; + if (!TryReadTimeTravelKeyword(sql, keyword_start, keyword_end, at_parameter)) { + result += sql[i]; + continue; + } + idx_t as_pos = SkipWhitespace(sql, keyword_end); + if (!MatchesKeywordAt(sql, as_pos, "as")) { + result += sql[i]; + continue; + } + idx_t of_pos = SkipWhitespace(sql, as_pos + 2); + if (!MatchesKeywordAt(sql, of_pos, "of")) { + result += sql[i]; + continue; + } + + idx_t value_start = SkipWhitespace(sql, of_pos + 2); + idx_t value_end = value_start; + string literal; + string value_sql; + if (TryReadSingleQuotedLiteral(sql, value_start, value_end, literal)) { + value_sql = SingleQuotedSqlString(literal); + } else if (TryReadNumericToken(sql, value_start, value_end, value_sql)) { + if (at_parameter == "TIMESTAMP") { + ThrowUnsupportedInputDialectFeature( + dialect, "time_travel", "timestamp time travel needs a string literal, got '" + value_sql + "'"); + } + } else { + result += sql[i]; + continue; + } + + result += "AT (" + at_parameter + " => " + value_sql + ")"; + i = value_end - 1; + } + return result; +} + enum class InputFunctionRewriteKind : uint8_t { RENAME, DATE_ADD_DAYS, @@ -652,6 +740,7 @@ string NormalizeInputSqlToDuckDB(const string &query, SqlDialect dialect) { string result = NormalizeBacktickIdentifiers(query, dialect); result = NormalizeBracketIdentifiers(result, dialect); + result = RewriteTimeTravelClauses(result, dialect); result = RewriteIntervals(result, dialect); result = RewriteCastTypes(result, dialect); RejectRiskyAliasReferences(result, dialect); diff --git a/test/sql/time_travel.test b/test/sql/time_travel.test new file mode 100644 index 0000000..82f3bbf --- /dev/null +++ b/test/sql/time_travel.test @@ -0,0 +1,201 @@ +# name: test/sql/time_travel.test +# description: Spark/Delta time travel (VERSION/TIMESTAMP AS OF) <-> DuckDB AT (...) snapshot pinning +# group: [sql] + +require lpts + +require parquet + +statement ok +SET lpts_check = true; + +# ============================================================ +# Input dialect: the Spark/Delta temporal clause normalizes to +# DuckDB's AT (...) clause. The pinned snapshot is represented, +# never dropped -- dropping it would silently turn a pinned scan +# into a read of the latest snapshot. +# ============================================================ + +statement ok +SET lpts_input_dialect = 'spark'; + +query T +SELECT sql FROM lpts_normalize_query('SELECT COUNT(*) AS row_count FROM arc_sql_db_bi.billing_meter_dim VERSION AS OF 366'); +---- +SELECT COUNT(*) AS row_count FROM arc_sql_db_bi.billing_meter_dim AT (VERSION => 366) + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t FOR VERSION AS OF 7'); +---- +SELECT * FROM t AT (VERSION => 7) + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t FOR SYSTEM_VERSION AS OF 7'); +---- +SELECT * FROM t AT (VERSION => 7) + +# Delta path relations keep their alias after the snapshot qualifier. +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM delta.`/mnt/mv` VERSION AS OF 42 v'); +---- +SELECT * FROM delta."/mnt/mv" AT (VERSION => 42) v + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t TIMESTAMP AS OF ''2024-01-15 08:09:10'''); +---- +SELECT * FROM t AT (TIMESTAMP => '2024-01-15 08:09:10') + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t FOR SYSTEM_TIME AS OF ''2024-01-15'''); +---- +SELECT * FROM t AT (TIMESTAMP => '2024-01-15') + +# A column/alias merely named `version` or `timestamp` is not a temporal clause. +query T +SELECT sql FROM lpts_normalize_query('SELECT version AS of FROM t'); +---- +SELECT version AS of FROM t + +query T +SELECT sql FROM lpts_normalize_query('SELECT timestamp AS of FROM t'); +---- +SELECT timestamp AS of FROM t + +# String literals are never rewritten. +query T +SELECT sql FROM lpts_normalize_query('SELECT ''x VERSION AS OF 3'' AS lit FROM t'); +---- +SELECT 'x VERSION AS OF 3' AS lit FROM t + +# A timestamp pin needs a timestamp literal, not a version number: refuse rather than +# silently pin to the wrong thing. +statement error +SELECT sql FROM lpts_normalize_query('SELECT * FROM t TIMESTAMP AS OF 366'); +---- +LPTS_UNSUPPORTED_INPUT_DIALECT_FEATURE + +# Non-Spark input dialects do not carry the Delta temporal clause. +statement ok +SET lpts_input_dialect = 'duckdb'; + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t VERSION AS OF 366'); +---- +SELECT * FROM t VERSION AS OF 366 + +statement ok +SET lpts_input_dialect = 'postgres'; + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t VERSION AS OF 366'); +---- +SELECT * FROM t VERSION AS OF 366 + +statement ok +SET lpts_input_dialect = 'duckdb'; + +# ============================================================ +# Round trip against a catalog that really supports time travel. +# products: version 2 has 2 rows, version 3 (latest) has 3 rows, +# so a dropped pin is observable in the result, not just the SQL. +# ============================================================ + +statement ok +INSTALL ducklake; + +statement ok +LOAD ducklake; + +statement ok +ATTACH 'ducklake:__TEST_DIR__/lpts_time_travel.ducklake' AS dl; + +statement ok +CREATE TABLE dl.products(product_id INTEGER, name VARCHAR); + +statement ok +INSERT INTO dl.products VALUES (1, 'Widget'), (2, 'Gadget'); + +statement ok +INSERT INTO dl.products VALUES (3, 'Gizmo'); + +query I +SELECT COUNT(*) FROM dl.products AT (VERSION => 2); +---- +2 + +query I +SELECT COUNT(*) FROM dl.products; +---- +3 + +# Spark input: the pinned scan plans against the pinned snapshot, and the regenerated +# DuckDB SQL still carries the pin (lpts_check re-runs the rewrite of the bare query above +# and compares result bags, so a dropped pin would surface as 3 rows instead of 2). +statement ok +SET lpts_input_dialect = 'spark'; + +query I +SELECT sql LIKE '%AT (VERSION => 2)%' AS spark_input_pin_survives_planning +FROM lpts_query('SELECT COUNT(*) AS row_count FROM dl.products VERSION AS OF 2'); +---- +true + +# ============================================================ +# Output dialect: the pin renders in the target dialect's own +# spelling. Spark gets `VERSION AS OF`, DuckDB keeps `AT (...)`, +# and a dialect with no verified time-travel syntax refuses +# instead of emitting SQL the target cannot parse. +# ============================================================ + +statement ok +SET lpts_dialect = 'spark'; + +query I +SELECT sql LIKE '%VERSION AS OF 2%' AND sql NOT LIKE '%AT (VERSION%' AS spark_snapshot_preserved +FROM lpts_query('SELECT COUNT(*) AS row_count FROM dl.products VERSION AS OF 2'); +---- +true + +query I +SELECT sql LIKE '%VERSION AS OF 2%' AND sql NOT LIKE '%AT (VERSION%' AS spark_snapshot_preserved_with_filter +FROM lpts_query('SELECT name FROM dl.products VERSION AS OF 2 WHERE product_id > 1'); +---- +true + +# An unpinned scan stays unpinned: no snapshot qualifier is invented. +query I +SELECT sql NOT LIKE '%VERSION AS OF%' AND sql NOT LIKE '%AT (VERSION%' AS unpinned_scan_unchanged +FROM lpts_query('SELECT COUNT(*) AS row_count FROM dl.products'); +---- +true + +statement ok +SET lpts_dialect = 'duckdb'; + +query I +SELECT sql LIKE '%AT (VERSION => 2)%' AS duckdb_snapshot_preserved +FROM lpts_query('SELECT COUNT(*) AS row_count FROM dl.products VERSION AS OF 2'); +---- +true + +statement ok +SET lpts_dialect = 'postgres'; + +statement error +SELECT sql FROM lpts_query('SELECT COUNT(*) AS row_count FROM dl.products VERSION AS OF 2'); +---- +LPTS_UNSUPPORTED_TIME_TRAVEL + +statement ok +SET lpts_dialect = 'hive'; + +statement error +SELECT sql FROM lpts_query('SELECT COUNT(*) AS row_count FROM dl.products VERSION AS OF 2'); +---- +LPTS_UNSUPPORTED_TIME_TRAVEL + +statement ok +SET lpts_dialect = 'duckdb'; + +statement ok +SET lpts_input_dialect = 'duckdb'; From dbac36de2e7e00d22fcb48250662fd69c0c106cd Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Tue, 25 Aug 2026 01:38:57 +0000 Subject: [PATCH 6/7] Move a pinned relation's alias ahead of the DuckDB AT (...) clause Spark spells a time-travel pin between the relation and its alias (`FROM t VERSION AS OF 2 p`), DuckDB spells it after the alias (`FROM t p AT (VERSION => 2)`). The Spark input normalizer emitted the qualifier in Spark's position, so every aliased pinned relation produced `FROM t AT (VERSION => 2) p`, which DuckDB refuses to parse -- the same class of failure the normalizer was added to remove. The normalizer now reads the optional `[AS] alias [(column, ...)]` that follows the temporal clause and emits it ahead of the rewritten qualifier, preserving the original quoting and case. A bare token is only taken as an alias when DuckDB's grammar allows it there (the `ColId` rule: reserved and type/function keywords are excluded), so `WHERE`, `GROUP BY`, `CROSS`/`NATURAL`/`ANTI JOIN`, a comma and a closing paren continue the query instead of being swallowed. test/sql/time_travel.test replaces the text-only expectation that asserted the unparseable form with executable DuckLake coverage: real pinned scans with a bare alias, an `AS` alias, a quoted alias, a column alias list, alias-qualified projections and filters, and two aliased relations pinned to two different non-latest versions -- all run under `lpts_check`, so the regenerated SQL is executed and its rows compared. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/lpts_parser.cpp | 67 +++++++++++- test/sql/time_travel.test | 213 +++++++++++++++++++++++++++++++++++--- 2 files changed, 266 insertions(+), 14 deletions(-) diff --git a/src/lpts_parser.cpp b/src/lpts_parser.cpp index 4647cfb..f2bed80 100644 --- a/src/lpts_parser.cpp +++ b/src/lpts_parser.cpp @@ -3,6 +3,8 @@ #include "lpts_date_format.hpp" #include "lpts_sql_scanner.hpp" +#include "duckdb/parser/parser.hpp" + #include #include #include @@ -130,10 +132,64 @@ static bool TryReadTimeTravelKeyword(const string &sql, idx_t pos, idx_t &end, s return false; } +/// True when `token` may name a relation on its own, without a leading `AS`. DuckDB's `alias_clause` +/// takes a `ColId`, which admits plain identifiers plus the unreserved and column-name keywords but +/// not the reserved or type/function ones -- so `WHERE`, `JOIN`, `NATURAL`, `TABLESAMPLE` and their +/// kin end the relation instead of naming it. +static bool CanBeBareRelationAlias(const string &token) { + const KeywordCategory category = Parser::IsKeyword(LowerCopy(token)); + return category != KeywordCategory::KEYWORD_RESERVED && category != KeywordCategory::KEYWORD_TYPE_FUNC; +} + +/// Read the optional `[AS] alias [(column, ...)]` that Spark allows *after* a temporal clause, +/// yielding the original text so quoting and case survive being moved. Returns false when the +/// relation is unaliased, i.e. when what follows continues the query (`WHERE`, `JOIN`, `,`, `)`). +static bool TryReadRelationAlias(const string &sql, idx_t pos, idx_t &end, string &alias_sql) { + const idx_t start = SkipWhitespace(sql, pos); + idx_t cursor = start; + const bool explicit_as = MatchesKeywordAt(sql, cursor, "as"); + if (explicit_as) { + cursor = SkipWhitespace(sql, cursor + 2); + } + + idx_t alias_end; + string token; + if (cursor < sql.size() && sql[cursor] == '"') { + if (!TryReadSkippableSqlSpan(sql, cursor, alias_end)) { + return false; + } + } else if (TryReadIdentifierToken(sql, cursor, alias_end, token)) { + // An explicit `AS` already committed to an alias; a bare token only aliases when the grammar + // lets it, otherwise it is the next clause. + if (!explicit_as && !CanBeBareRelationAlias(token)) { + return false; + } + } else { + return false; + } + + const idx_t columns_start = SkipWhitespace(sql, alias_end); + if (columns_start < sql.size() && sql[columns_start] == '(') { + const idx_t columns_end = FindMatchingParen(sql, columns_start); + if (columns_end == DConstants::INVALID_INDEX) { + return false; + } + alias_end = columns_end + 1; + } + + end = alias_end; + alias_sql = sql.substr(start, alias_end - start); + return true; +} + /// Rewrite the Spark/Delta time-travel clause into DuckDB's `AT (...)` snapshot clause: /// /// FROM t [FOR] VERSION AS OF 366 -> FROM t AT (VERSION => 366) /// FROM t [FOR] TIMESTAMP AS OF '2024-...' -> FROM t AT (TIMESTAMP => '2024-...') +/// FROM t VERSION AS OF 366 AS v -> FROM t AS v AT (VERSION => 366) +/// +/// Spark puts the clause between the relation and its alias; DuckDB puts it after the alias, so the +/// alias is carried across the rewrite rather than left stranded behind the qualifier. /// /// The pinned snapshot is *represented*, never dropped: DuckDB's `AT` clause is the semantically /// equivalent spelling, so the plan keeps reading the pinned snapshot and the generated SQL can @@ -190,8 +246,17 @@ static string RewriteTimeTravelClauses(const string &sql, SqlDialect dialect) { continue; } + // Spark: ` VERSION AS OF [AS] alias`. DuckDB: ` [AS] alias AT (...)`. + // Emit the alias first so the qualifier still attaches to the relation it pins. + idx_t clause_end = value_end; + idx_t alias_end; + string alias_sql; + if (TryReadRelationAlias(sql, value_end, alias_end, alias_sql)) { + result += alias_sql + " "; + clause_end = alias_end; + } result += "AT (" + at_parameter + " => " + value_sql + ")"; - i = value_end - 1; + i = clause_end - 1; } return result; } diff --git a/test/sql/time_travel.test b/test/sql/time_travel.test index 82f3bbf..207a52a 100644 --- a/test/sql/time_travel.test +++ b/test/sql/time_travel.test @@ -34,12 +34,6 @@ SELECT sql FROM lpts_normalize_query('SELECT * FROM t FOR SYSTEM_VERSION AS OF 7 ---- SELECT * FROM t AT (VERSION => 7) -# Delta path relations keep their alias after the snapshot qualifier. -query T -SELECT sql FROM lpts_normalize_query('SELECT * FROM delta.`/mnt/mv` VERSION AS OF 42 v'); ----- -SELECT * FROM delta."/mnt/mv" AT (VERSION => 42) v - query T SELECT sql FROM lpts_normalize_query('SELECT * FROM t TIMESTAMP AS OF ''2024-01-15 08:09:10'''); ---- @@ -74,6 +68,122 @@ SELECT sql FROM lpts_normalize_query('SELECT * FROM t TIMESTAMP AS OF 366'); ---- LPTS_UNSUPPORTED_INPUT_DIALECT_FEATURE +# ============================================================ +# Alias ordering. Spark writes the pin *between* the relation and +# its alias; DuckDB writes it *after* the alias. The alias has to +# travel with the rewrite -- `FROM t AT (VERSION => 2) p` and +# `FROM t AT (VERSION => 2) AS p` are both DuckDB parse errors. +# ============================================================ + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t VERSION AS OF 2 p'); +---- +SELECT * FROM t p AT (VERSION => 2) + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t VERSION AS OF 2 AS p'); +---- +SELECT * FROM t AS p AT (VERSION => 2) + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t TIMESTAMP AS OF ''2024-01-15'' AS p'); +---- +SELECT * FROM t AS p AT (TIMESTAMP => '2024-01-15') + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t FOR SYSTEM_VERSION AS OF 7 AS p'); +---- +SELECT * FROM t AS p AT (VERSION => 7) + +# Quoted aliases keep their quoting and their spaces. +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t VERSION AS OF 2 AS `odd alias`'); +---- +SELECT * FROM t AS "odd alias" AT (VERSION => 2) + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t VERSION AS OF 2 `odd alias`'); +---- +SELECT * FROM t "odd alias" AT (VERSION => 2) + +# An alias may carry a column list. +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t VERSION AS OF 2 p(a, b)'); +---- +SELECT * FROM t p(a, b) AT (VERSION => 2) + +# Delta path relations behave the same way. +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM delta.`/mnt/mv` VERSION AS OF 42 v'); +---- +SELECT * FROM delta."/mnt/mv" v AT (VERSION => 42) + +# Every relation in a join keeps its own alias and its own pin. +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM a VERSION AS OF 1 x JOIN b VERSION AS OF 2 y ON x.id = y.id'); +---- +SELECT * FROM a x AT (VERSION => 1) JOIN b y AT (VERSION => 2) ON x.id = y.id + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM a VERSION AS OF 1 AS x, b VERSION AS OF 2 AS y'); +---- +SELECT * FROM a AS x AT (VERSION => 1), b AS y AT (VERSION => 2) + +# ------------------------------------------------------------ +# What follows an unaliased pin must not be mistaken for its +# alias: a reserved or type/function keyword continues the query. +# ------------------------------------------------------------ + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t VERSION AS OF 2 WHERE x > 1'); +---- +SELECT * FROM t AT (VERSION => 2) WHERE x > 1 + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t VERSION AS OF 2 GROUP BY a'); +---- +SELECT * FROM t AT (VERSION => 2) GROUP BY a + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM t VERSION AS OF 2 ORDER BY a LIMIT 3'); +---- +SELECT * FROM t AT (VERSION => 2) ORDER BY a LIMIT 3 + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM a VERSION AS OF 1 CROSS JOIN b'); +---- +SELECT * FROM a AT (VERSION => 1) CROSS JOIN b + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM a VERSION AS OF 1 NATURAL JOIN b'); +---- +SELECT * FROM a AT (VERSION => 1) NATURAL JOIN b + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM a VERSION AS OF 1 LEFT JOIN b ON a.i = b.i'); +---- +SELECT * FROM a AT (VERSION => 1) LEFT JOIN b ON a.i = b.i + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM a VERSION AS OF 1 ANTI JOIN b ON a.i = b.i'); +---- +SELECT * FROM a AT (VERSION => 1) ANTI JOIN b ON a.i = b.i + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM a VERSION AS OF 1, b'); +---- +SELECT * FROM a AT (VERSION => 1), b + +query T +SELECT sql FROM lpts_normalize_query('SELECT * FROM (SELECT * FROM t VERSION AS OF 2) x'); +---- +SELECT * FROM (SELECT * FROM t AT (VERSION => 2)) x + +query T +SELECT sql FROM lpts_normalize_query('WITH c AS (SELECT * FROM t VERSION AS OF 2 v) SELECT * FROM c UNION ALL SELECT * FROM t VERSION AS OF 3'); +---- +WITH c AS (SELECT * FROM t v AT (VERSION => 2)) SELECT * FROM c UNION ALL SELECT * FROM t AT (VERSION => 3) + # Non-Spark input dialects do not carry the Delta temporal clause. statement ok SET lpts_input_dialect = 'duckdb'; @@ -96,8 +206,13 @@ SET lpts_input_dialect = 'duckdb'; # ============================================================ # Round trip against a catalog that really supports time travel. -# products: version 2 has 2 rows, version 3 (latest) has 3 rows, -# so a dropped pin is observable in the result, not just the SQL. +# products: version 2 has 2 rows, version 3 has 3 rows, version 4 +# (latest) has 4 rows -- so both pinned versions are non-latest and +# a dropped pin changes the rows, not just the SQL text. +# +# Every query below runs under `lpts_check = true`, which executes +# the LPTS-regenerated SQL alongside the original and compares the +# result bags: these are real scans, not string assertions. # ============================================================ statement ok @@ -118,28 +233,94 @@ INSERT INTO dl.products VALUES (1, 'Widget'), (2, 'Gadget'); statement ok INSERT INTO dl.products VALUES (3, 'Gizmo'); +statement ok +INSERT INTO dl.products VALUES (4, 'Doohickey'); + query I SELECT COUNT(*) FROM dl.products AT (VERSION => 2); ---- 2 query I -SELECT COUNT(*) FROM dl.products; +SELECT COUNT(*) FROM dl.products AT (VERSION => 3); ---- 3 -# Spark input: the pinned scan plans against the pinned snapshot, and the regenerated -# DuckDB SQL still carries the pin (lpts_check re-runs the rewrite of the bare query above -# and compares result bags, so a dropped pin would surface as 3 rows instead of 2). +query I +SELECT COUNT(*) FROM dl.products; +---- +4 + +# The exact shape the Spark normalizer emits, executed: alias before the pin, with the +# projection and the filter qualified by that alias. +query T +SELECT p.name FROM dl.products AS p AT (VERSION => 2) WHERE p.product_id > 1; +---- +Gadget + +query T +SELECT p.name FROM dl.products p AT (VERSION => 2) WHERE p.product_id > 1; +---- +Gadget + +query T +SELECT "odd alias".name FROM dl.products AS "odd alias" AT (VERSION => 2) WHERE "odd alias".product_id = 2; +---- +Gadget + +query I +SELECT pid FROM dl.products p(pid, pname) AT (VERSION => 2) WHERE pname = 'Gadget'; +---- +2 + +# Two aliased relations over the same table, pinned to two different non-latest versions. +query I +SELECT COUNT(*) FROM dl.products p AT (VERSION => 2) JOIN dl.products q AT (VERSION => 3) ON p.product_id = q.product_id; +---- +2 + +query I +SELECT COUNT(*) FROM dl.products AS p AT (VERSION => 2), dl.products AS q AT (VERSION => 3); +---- +6 + +# ------------------------------------------------------------ +# Spark input against the same catalog: normalization produces +# exactly the executable statements asserted above, and the +# pinned scan plans against the pinned snapshot. +# ------------------------------------------------------------ + statement ok SET lpts_input_dialect = 'spark'; +query T +SELECT sql FROM lpts_normalize_query('SELECT p.name FROM dl.products VERSION AS OF 2 AS p WHERE p.product_id > 1'); +---- +SELECT p.name FROM dl.products AS p AT (VERSION => 2) WHERE p.product_id > 1 + +query T +SELECT sql FROM lpts_normalize_query('SELECT COUNT(*) FROM dl.products VERSION AS OF 2 p JOIN dl.products VERSION AS OF 3 q ON p.product_id = q.product_id'); +---- +SELECT COUNT(*) FROM dl.products p AT (VERSION => 2) JOIN dl.products q AT (VERSION => 3) ON p.product_id = q.product_id + query I SELECT sql LIKE '%AT (VERSION => 2)%' AS spark_input_pin_survives_planning FROM lpts_query('SELECT COUNT(*) AS row_count FROM dl.products VERSION AS OF 2'); ---- true +query I +SELECT sql LIKE '%AT (VERSION => 2)%' AS aliased_spark_input_pin_survives_planning +FROM lpts_query('SELECT p.name AS n FROM dl.products VERSION AS OF 2 AS p WHERE p.product_id > 1'); +---- +true + +query I +SELECT sql LIKE '%AT (VERSION => 2)%' AND sql LIKE '%AT (VERSION => 3)%' AS both_pins_survive_planning +FROM lpts_query('SELECT COUNT(*) AS c FROM dl.products VERSION AS OF 2 p, dl.products VERSION AS OF 3 q WHERE p.product_id = q.product_id'); +---- +true + # ============================================================ # Output dialect: the pin renders in the target dialect's own # spelling. Spark gets `VERSION AS OF`, DuckDB keeps `AT (...)`, @@ -158,7 +339,13 @@ true query I SELECT sql LIKE '%VERSION AS OF 2%' AND sql NOT LIKE '%AT (VERSION%' AS spark_snapshot_preserved_with_filter -FROM lpts_query('SELECT name FROM dl.products VERSION AS OF 2 WHERE product_id > 1'); +FROM lpts_query('SELECT name FROM dl.products VERSION AS OF 2 AS p WHERE p.product_id > 1'); +---- +true + +query I +SELECT sql LIKE '%VERSION AS OF 2%' AND sql LIKE '%VERSION AS OF 3%' AS both_pins_render_in_spark +FROM lpts_query('SELECT COUNT(*) AS c FROM dl.products VERSION AS OF 2 p, dl.products VERSION AS OF 3 q WHERE p.product_id = q.product_id'); ---- true From 6980a13bedcef63e751087dcc25cac1a0db9a635 Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Sun, 30 Aug 2026 14:01:09 +0000 Subject: [PATCH 7/7] Fix Spark dialect sqllogictest separation Separate the HUGEINT error expectation from the following interval query so the ICU-enabled runner parses them as distinct records. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- test/sql/dialect_spark.test | 1 + 1 file changed, 1 insertion(+) diff --git a/test/sql/dialect_spark.test b/test/sql/dialect_spark.test index 9b7fc80..f0012fd 100644 --- a/test/sql/dialect_spark.test +++ b/test/sql/dialect_spark.test @@ -256,6 +256,7 @@ statement error SELECT * FROM lpts_query('SELECT 170141183460469231731687303715884105727::HUGEINT'); ---- LPTS_UNSUPPORTED_TYPE + query II SELECT sql LIKE '%INTERVAL ''30'' DAY%' AS spark_interval, sql NOT LIKE '%::INTERVAL%' AS no_postgres_interval_cast