From d2fcb11e786bb843d0c2021c072502f6949dda0b Mon Sep 17 00:00:00 2001 From: ila Date: Mon, 31 Aug 2026 12:50:20 +0200 Subject: [PATCH 1/2] Optimize filterless group contribution caps --- src/compiler/privacy_mechanisms.cpp | 15 ++++++++++----- test/sql/dp_filterless.test | 21 +++++++++++++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/compiler/privacy_mechanisms.cpp b/src/compiler/privacy_mechanisms.cpp index c91b40b..280b777 100644 --- a/src/compiler/privacy_mechanisms.cpp +++ b/src/compiler/privacy_mechanisms.cpp @@ -3226,8 +3226,9 @@ static FilterlessPreAggregationInput ApplyFilterlessMaxGroups(OptimizerExtension } // Flatten the lower aggregate's separate group/aggregate bindings into one projection. The - // lower aggregate already emits one row per logical (PU, SQL group), so dense rank directly - // caps the group set seen by both the answer and fixed-sample histogram channels. + // lower aggregate already emits exactly one row per logical (PU, SQL group), so ROW_NUMBER + // caps the group set without needing duplicate-aware DENSE_RANK. DuckDB can optimize this + // single hashed ordering into a bounded per-PU top-k aggregate. idx_t lower_aggregate_count = 2 * component_count + 1; idx_t projection_index = input.optimizer.binder.GenerateTableIndex(); vector> expressions; @@ -3247,9 +3248,12 @@ static FilterlessPreAggregationInput ApplyFilterlessMaxGroups(OptimizerExtension idx_t logical_pu_column = pre.num_original_groups; RankCapSpec spec; - spec.rank_type = ExpressionType::WINDOW_RANK_DENSE; + spec.rank_type = ExpressionType::WINDOW_ROW_NUMBER; spec.partition_cols = {logical_pu_column}; - spec.order_cols.reserve(pre.num_original_groups); + spec.order_cols.reserve(pre.num_original_groups + 1); + // Include the PU in the stable hash so each PU gets its own deterministic group + // ordering instead of every PU systematically favoring the same group. + spec.order_cols.push_back(logical_pu_column); for (idx_t i = 0; i < pre.num_original_groups; i++) { spec.order_cols.push_back(i); } @@ -3275,7 +3279,8 @@ static FilterlessPreAggregationInput ApplyFilterlessMaxGroups(OptimizerExtension for (idx_t i = 0; i < lower_aggregate_count; i++) { result.aggregate_types.push_back(pre.lower_agg->types[pre.num_original_groups + 1 + i]); } - PRIVACY_DEBUG_PRINT("[dp_filterless] capped distinct groups per logical PU at " + std::to_string(max_groups)); + PRIVACY_DEBUG_PRINT("[dp_filterless] capped unique groups per logical PU with bounded row selection at " + + std::to_string(max_groups)); return result; } diff --git a/test/sql/dp_filterless.test b/test/sql/dp_filterless.test index 1a86259..7fc049e 100644 --- a/test/sql/dp_filterless.test +++ b/test/sql/dp_filterless.test @@ -283,7 +283,8 @@ ORDER BY grp; a 100.000000 # The existing Google-style L0 rule is retained: the lower aggregate emits one -# row per logical PU/group, and at most C_u groups survive. +# row per logical PU/group, and hash-ranked row selection keeps at most C_u +# groups. Which bounded group survives is not part of the query contract. statement ok CREATE PU TABLE filterless_group_cap ( uid BIGINT, @@ -298,13 +299,25 @@ INSERT INTO filterless_group_cap VALUES (1, 'a', 50.0), (1, 'b', 50.0), (2, 'a', 50.0), (2, 'b', 50.0); -query IR -SELECT grp, SUM(amount) +query R +SELECT SUM(amount) FROM filterless_group_cap GROUP BY grp ORDER BY grp; ---- -a 100.000000 +100.000000 + +# Because the lower aggregate has unique PU/group rows, DuckDB can optimize the +# ROW_NUMBER <= C_u cap to a bounded per-PU arg_min instead of sorting a window. +statement ok +SET explain_output = 'physical_only'; + +query II +EXPLAIN SELECT grp, SUM(amount) +FROM filterless_group_cap +GROUP BY grp; +---- +physical_plan :[\s\S]*arg_min[\s\S]* statement ok SET dp_max_groups_contributed = 2; From a608eb95a666f3fa30ee20dd1fd52847d18f292a Mon Sep 17 00:00:00 2001 From: ila Date: Mon, 31 Aug 2026 12:54:06 +0200 Subject: [PATCH 2/2] Test filterless top-k caps above one --- test/sql/dp_filterless.test | 55 +++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/test/sql/dp_filterless.test b/test/sql/dp_filterless.test index 7fc049e..de06e1c 100644 --- a/test/sql/dp_filterless.test +++ b/test/sql/dp_filterless.test @@ -307,18 +307,6 @@ ORDER BY grp; ---- 100.000000 -# Because the lower aggregate has unique PU/group rows, DuckDB can optimize the -# ROW_NUMBER <= C_u cap to a bounded per-PU arg_min instead of sorting a window. -statement ok -SET explain_output = 'physical_only'; - -query II -EXPLAIN SELECT grp, SUM(amount) -FROM filterless_group_cap -GROUP BY grp; ----- -physical_plan :[\s\S]*arg_min[\s\S]* - statement ok SET dp_max_groups_contributed = 2; @@ -331,6 +319,49 @@ ORDER BY grp; a 100.000000 b 100.000000 +# Exercise a genuine top-k cap with more candidate groups than C_u. The private +# query must retain exactly three contributions per PU across its released rows. +statement ok +SET dp_max_groups_contributed = 3; + +statement ok +CREATE PU TABLE filterless_group_cap_three ( + uid BIGINT, + grp BIGINT, + amount DOUBLE, + PRIVACY_KEY (uid), + PROTECTED (amount) +); + +statement ok +INSERT INTO filterless_group_cap_three +SELECT uid, grp, 1.0 +FROM range(100) users(uid), range(10) groups(grp); + +statement ok +CREATE TEMP TABLE filterless_group_cap_three_result AS +SELECT grp, SUM(amount) AS total +FROM filterless_group_cap_three +GROUP BY grp; + +query IR +SELECT COUNT(*), SUM(total) +FROM filterless_group_cap_three_result; +---- +10 300.000000 + +# DuckDB should optimize ROW_NUMBER <= 3 into a bounded per-PU top-k aggregate +# plus UNNEST, instead of sorting a window. +statement ok +SET explain_output = 'physical_only'; + +query II +EXPLAIN SELECT grp, SUM(amount) +FROM filterless_group_cap_three +GROUP BY grp; +---- +physical_plan :[\s\S]*UNNEST[\s\S]*arg_min_nulls_last[\s\S]* + statement ok SET dp_max_groups_contributed = 1;