Skip to content

Optimize filterless group contribution caps #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ila merged 2 commits into from
Aug 31, 2026
Merged

Optimize filterless group contribution caps #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d2fcb11
Optimize filterless group contribution caps
ila Aug 31, 2026
a608eb9
Test filterless top-k caps above one
ila Aug 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions src/compiler/privacy_mechanisms.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3226,8 +3226,9 @@ static FilterlessPreAggregationInput ApplyFilterlessMaxGroups(OptimizerExtension
}

// Flatten the lower aggregate's separate group/aggregate bindings into one projection. The
// lower aggregate already emits one row per logical (PU, SQL group), so dense rank directly
// caps the group set seen by both the answer and fixed-sample histogram channels.
// lower aggregate already emits exactly one row per logical (PU, SQL group), so ROW_NUMBER
// caps the group set without needing duplicate-aware DENSE_RANK. DuckDB can optimize this
// single hashed ordering into a bounded per-PU top-k aggregate.
idx_t lower_aggregate_count = 2 * component_count + 1;
idx_t projection_index = input.optimizer.binder.GenerateTableIndex();
vector<unique_ptr<Expression>> expressions;
Expand All @@ -3247,9 +3248,12 @@ static FilterlessPreAggregationInput ApplyFilterlessMaxGroups(OptimizerExtension

idx_t logical_pu_column = pre.num_original_groups;
RankCapSpec spec;
spec.rank_type = ExpressionType::WINDOW_RANK_DENSE;
spec.rank_type = ExpressionType::WINDOW_ROW_NUMBER;
spec.partition_cols = {logical_pu_column};
spec.order_cols.reserve(pre.num_original_groups);
spec.order_cols.reserve(pre.num_original_groups + 1);
// Include the PU in the stable hash so each PU gets its own deterministic group
// ordering instead of every PU systematically favoring the same group.
spec.order_cols.push_back(logical_pu_column);
for (idx_t i = 0; i < pre.num_original_groups; i++) {
spec.order_cols.push_back(i);
}
Expand All @@ -3275,7 +3279,8 @@ static FilterlessPreAggregationInput ApplyFilterlessMaxGroups(OptimizerExtension
for (idx_t i = 0; i < lower_aggregate_count; i++) {
result.aggregate_types.push_back(pre.lower_agg->types[pre.num_original_groups + 1 + i]);
}
PRIVACY_DEBUG_PRINT("[dp_filterless] capped distinct groups per logical PU at " + std::to_string(max_groups));
PRIVACY_DEBUG_PRINT("[dp_filterless] capped unique groups per logical PU with bounded row selection at " +
std::to_string(max_groups));
return result;
}

Expand Down
52 changes: 48 additions & 4 deletions test/sql/dp_filterless.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,8 @@ ORDER BY grp;
a 100.000000

# The existing Google-style L0 rule is retained: the lower aggregate emits one
# row per logical PU/group, and at most C_u groups survive.
# row per logical PU/group, and hash-ranked row selection keeps at most C_u
# groups. Which bounded group survives is not part of the query contract.
statement ok
CREATE PU TABLE filterless_group_cap (
uid BIGINT,
Expand All @@ -298,13 +299,13 @@ INSERT INTO filterless_group_cap VALUES
(1, 'a', 50.0), (1, 'b', 50.0),
(2, 'a', 50.0), (2, 'b', 50.0);

query IR
SELECT grp, SUM(amount)
query R
SELECT SUM(amount)
FROM filterless_group_cap
GROUP BY grp
ORDER BY grp;
----
a 100.000000
100.000000

statement ok
SET dp_max_groups_contributed = 2;
Expand All @@ -318,6 +319,49 @@ ORDER BY grp;
a 100.000000
b 100.000000

# Exercise a genuine top-k cap with more candidate groups than C_u. The private
# query must retain exactly three contributions per PU across its released rows.
statement ok
SET dp_max_groups_contributed = 3;

statement ok
CREATE PU TABLE filterless_group_cap_three (
uid BIGINT,
grp BIGINT,
amount DOUBLE,
PRIVACY_KEY (uid),
PROTECTED (amount)
);

statement ok
INSERT INTO filterless_group_cap_three
SELECT uid, grp, 1.0
FROM range(100) users(uid), range(10) groups(grp);

statement ok
CREATE TEMP TABLE filterless_group_cap_three_result AS
SELECT grp, SUM(amount) AS total
FROM filterless_group_cap_three
GROUP BY grp;

query IR
SELECT COUNT(*), SUM(total)
FROM filterless_group_cap_three_result;
----
10 300.000000

# DuckDB should optimize ROW_NUMBER <= 3 into a bounded per-PU top-k aggregate
# plus UNNEST, instead of sorting a window.
statement ok
SET explain_output = 'physical_only';

query II
EXPLAIN SELECT grp, SUM(amount)
FROM filterless_group_cap_three
GROUP BY grp;
----
physical_plan <REGEX>:[\s\S]*UNNEST[\s\S]*arg_min_nulls_last[\s\S]*

statement ok
SET dp_max_groups_contributed = 1;

Expand Down
Loading