From b343f6e085446ddbaa587f141afdb96684717505 Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 00:41:14 -0400 Subject: [PATCH 01/13] fix(zql): normalize planner filters before costing --- .../src/planner/condition-normalizer.test.ts | 250 +++++++++ .../zql/src/planner/condition-normalizer.ts | 488 ++++++++++++++++++ packages/zql/src/planner/planner-builder.ts | 6 +- .../scenarios/duplicate-exists-or-dedupe.ts | 34 -- .../test/query-scenarios/scenarios/index.ts | 2 + .../scenarios/same-relationship-or-merge.ts | 43 +- .../shared-parent-filter-or-factor.ts | 47 +- .../scenarios/single-exists-child-or.ts | 2 +- .../scenarios/teacher-filter-or-in.ts | 52 ++ .../scenarios/two-student-membership-or.ts | 25 +- 10 files changed, 816 insertions(+), 133 deletions(-) create mode 100644 packages/zql/src/planner/condition-normalizer.test.ts create mode 100644 packages/zql/src/planner/condition-normalizer.ts create mode 100644 packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts diff --git a/packages/zql/src/planner/condition-normalizer.test.ts b/packages/zql/src/planner/condition-normalizer.test.ts new file mode 100644 index 0000000000..2f5c8ae8a2 --- /dev/null +++ b/packages/zql/src/planner/condition-normalizer.test.ts @@ -0,0 +1,250 @@ +import {expect, test} from 'vitest'; +import type { + AST, + Condition, + CorrelatedSubqueryCondition, + LiteralValue, +} from '../../../zero-protocol/src/ast.ts'; +import {normalizePlannerAST} from './condition-normalizer.ts'; + +const TRUE: Condition = {type: 'and', conditions: []}; +const FALSE: Condition = {type: 'or', conditions: []}; + +test('drops identity branches without losing real filters', () => { + const active = eq('active', true); + + expect(normalizeWhere({type: 'or', conditions: [active, FALSE]})).toEqual( + active, + ); + expect(normalizeWhere({type: 'and', conditions: [active, TRUE]})).toEqual( + active, + ); + expect(normalizeWhere({type: 'and', conditions: [active, FALSE]})).toEqual( + FALSE, + ); +}); + +test('rewrites same-column OR equalities to IN', () => { + expect( + normalizeWhere({ + type: 'or', + conditions: [eq('status', 'active'), eq('status', 'pending')], + }), + ).toEqual({ + type: 'simple', + left: {type: 'column', name: 'status'}, + op: 'IN', + right: {type: 'literal', value: ['active', 'pending']}, + }); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + eq('status', 'active'), + inCondition('status', 'IN', ['active']), + ], + }), + ).toEqual(eq('status', 'active')); +}); + +test('normalizes degenerate IN predicates', () => { + const notInEmpty = inCondition('status', 'NOT IN', []); + + expect(normalizeWhere(inCondition('status', 'IN', []))).toEqual(FALSE); + expect(normalizeWhere(inCondition('status', 'IN', ['active']))).toEqual( + eq('status', 'active'), + ); + expect(normalizeWhere(notInEmpty)).toEqual(notInEmpty); + expect(normalizeWhere(inCondition('status', 'NOT IN', ['active']))).toEqual({ + type: 'simple', + left: {type: 'column', name: 'status'}, + op: '!=', + right: {type: 'literal', value: 'active'}, + }); +}); + +test('merges OR exists branches over the same relationship', () => { + expect( + normalizeWhere({ + type: 'or', + conditions: [exists(eq('title', 'hello')), exists(eq('title', 'world'))], + }), + ).toEqual( + exists({ + type: 'simple', + left: {type: 'column', name: 'title'}, + op: 'IN', + right: {type: 'literal', value: ['hello', 'world']}, + }), + ); +}); + +test('factors common parent filters before merging child exists branches', () => { + const active = eq('active', true); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + { + type: 'and', + conditions: [active, exists(eq('title', 'hello'))], + }, + { + type: 'and', + conditions: [active, exists(eq('title', 'world'))], + }, + ], + }), + ).toEqual({ + type: 'and', + conditions: [ + active, + exists({ + type: 'simple', + left: {type: 'column', name: 'title'}, + op: 'IN', + right: {type: 'literal', value: ['hello', 'world']}, + }), + ], + }); +}); + +test('absorbs redundant branches after factoring common predicates', () => { + const active = eq('active', true); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + active, + { + type: 'and', + conditions: [active, eq('status', 'pending')], + }, + ], + }), + ).toEqual(active); +}); + +test('does not merge exists branches with different semantics', () => { + expect( + normalizeWhere({ + type: 'or', + conditions: [ + exists(eq('title', 'hello'), {op: 'NOT EXISTS'}), + exists(eq('title', 'world'), {op: 'NOT EXISTS'}), + ], + }), + ).toMatchObject({ + type: 'or', + conditions: [{op: 'NOT EXISTS'}, {op: 'NOT EXISTS'}], + }); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + exists(eq('title', 'hello'), {scalar: true}), + exists(eq('title', 'world'), {scalar: true}), + ], + }), + ).toMatchObject({type: 'or', conditions: [{scalar: true}, {scalar: true}]}); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + exists(eq('title', 'hello'), {subquery: {limit: 1}}), + exists(eq('title', 'world'), {subquery: {limit: 1}}), + ], + }), + ).toMatchObject({ + type: 'or', + conditions: [ + {related: {subquery: {limit: 1}}}, + {related: {subquery: {limit: 1}}}, + ], + }); +}); + +test('normalizes related subqueries recursively', () => { + expect( + normalizePlannerAST({ + table: 'users', + related: [ + { + correlation: { + parentField: ['id'], + childField: ['userId'], + }, + subquery: { + table: 'posts', + where: { + type: 'or', + conditions: [eq('title', 'hello'), eq('title', 'world')], + }, + }, + }, + ], + }).related?.[0].subquery.where, + ).toEqual({ + type: 'simple', + left: {type: 'column', name: 'title'}, + op: 'IN', + right: {type: 'literal', value: ['hello', 'world']}, + }); +}); + +function normalizeWhere(where: Condition): Condition | undefined { + return normalizePlannerAST({table: 'users', where}).where; +} + +function eq(name: string, value: LiteralValue): Condition { + return { + type: 'simple', + left: {type: 'column', name}, + op: '=', + right: {type: 'literal', value}, + }; +} + +function inCondition( + name: string, + op: 'IN' | 'NOT IN', + values: readonly (string | number | boolean)[], +): Condition { + return { + type: 'simple', + left: {type: 'column', name}, + op, + right: {type: 'literal', value: values}, + }; +} + +function exists( + where: Condition | undefined, + options: { + readonly op?: 'EXISTS' | 'NOT EXISTS' | undefined; + readonly scalar?: boolean | undefined; + readonly subquery?: Partial | undefined; + } = {}, +): CorrelatedSubqueryCondition { + return { + type: 'correlatedSubquery', + op: options.op ?? 'EXISTS', + scalar: options.scalar, + related: { + correlation: { + parentField: ['id'], + childField: ['userId'], + }, + subquery: { + table: 'posts', + where, + ...options.subquery, + }, + }, + }; +} diff --git a/packages/zql/src/planner/condition-normalizer.ts b/packages/zql/src/planner/condition-normalizer.ts new file mode 100644 index 0000000000..ae3a1f7bb0 --- /dev/null +++ b/packages/zql/src/planner/condition-normalizer.ts @@ -0,0 +1,488 @@ +import type { + AST, + Condition, + CorrelatedSubquery, + CorrelatedSubqueryCondition, + LiteralValue, + SimpleCondition, + ValuePosition, +} from '../../../zero-protocol/src/ast.ts'; +import {planIdSymbol} from '../../../zero-protocol/src/ast.ts'; + +const TRUE: Condition = {type: 'and', conditions: []}; +const FALSE: Condition = {type: 'or', conditions: []}; + +export function normalizePlannerAST(ast: AST): AST { + return { + ...ast, + where: ast.where ? normalizeCondition(ast.where) : undefined, + related: ast.related?.map(related => ({ + ...related, + subquery: normalizePlannerAST(related.subquery), + })), + }; +} + +function normalizeCondition(condition: Condition): Condition { + switch (condition.type) { + case 'simple': + return normalizeSimpleCondition(condition); + case 'correlatedSubquery': + return normalizeCorrelatedSubquery(condition); + case 'and': + return buildAnd(condition.conditions.map(normalizeCondition)); + case 'or': + return buildOr(condition.conditions.map(normalizeCondition)); + } +} + +function normalizeSimpleCondition(condition: SimpleCondition): Condition { + if (condition.right.type !== 'literal') { + return condition; + } + + const value = condition.right.value; + if (!Array.isArray(value)) { + return condition; + } + + if (condition.op === 'IN') { + const values = dedupeInLiteralValues(value); + switch (values.length) { + case 0: + return FALSE; + case 1: + return { + ...condition, + op: '=', + right: {type: 'literal', value: values[0]}, + }; + default: + return { + ...condition, + right: {type: 'literal', value: values}, + }; + } + } + + if (condition.op === 'NOT IN') { + const values = dedupeInLiteralValues(value); + switch (values.length) { + case 0: + return condition; + case 1: + return { + ...condition, + op: '!=', + right: {type: 'literal', value: values[0]}, + }; + default: + return { + ...condition, + right: {type: 'literal', value: values}, + }; + } + } + + return condition; +} + +function normalizeCorrelatedSubquery( + condition: CorrelatedSubqueryCondition, +): CorrelatedSubqueryCondition { + const {[planIdSymbol]: _planId, ...conditionWithoutPlanId} = condition; + return { + ...conditionWithoutPlanId, + related: { + ...condition.related, + subquery: normalizePlannerAST(condition.related.subquery), + }, + }; +} + +function buildAnd(conditions: readonly Condition[]): Condition { + const flattened = flatten('and', conditions); + if (flattened.some(isAlwaysFalse)) { + return FALSE; + } + + const deduped = dedupe( + flattened.filter(condition => !isAlwaysTrue(condition)), + ); + switch (deduped.length) { + case 0: + return TRUE; + case 1: + return deduped[0]; + default: + return {type: 'and', conditions: deduped}; + } +} + +function buildOr(conditions: readonly Condition[]): Condition { + const flattened = flatten('or', conditions); + if (flattened.some(isAlwaysTrue)) { + return TRUE; + } + + const deduped = dedupe( + flattened.filter(condition => !isAlwaysFalse(condition)), + ); + if (deduped.length === 0) { + return FALSE; + } + + const compacted = mergeEquivalentOrPredicates(deduped); + + const factored = factorCommonConjuncts(compacted); + if (factored) { + return normalizeCondition(factored); + } + + const merged = mergeSameRelationshipExists(compacted); + switch (merged.length) { + case 0: + return FALSE; + case 1: + return merged[0]; + default: + return {type: 'or', conditions: merged}; + } +} + +type InMergeCandidate = { + readonly left: ValuePosition; + readonly values: readonly InLiteralValue[]; + readonly isAlreadyIn: boolean; +}; + +type InLiteralValue = string | number | boolean; + +type InMergeGroup = { + readonly left: ValuePosition; + readonly firstIndex: number; + readonly values: InLiteralValue[]; + readonly valueKeys: Set; + changed: boolean; +}; + +function mergeEquivalentOrPredicates( + conditions: readonly Condition[], +): Condition[] { + const merged: Array = [...conditions]; + const groups = new Map(); + + for (const [index, condition] of conditions.entries()) { + const candidate = getInMergeCandidate(condition); + if (!candidate) { + continue; + } + + const key = stableStringify(candidate.left); + let group = groups.get(key); + if (!group) { + group = { + left: candidate.left, + firstIndex: index, + values: [], + valueKeys: new Set(), + changed: candidate.isAlreadyIn, + }; + groups.set(key, group); + } else { + merged[index] = undefined; + group.changed = true; + } + + for (const value of candidate.values) { + const valueKey = stableStringify(value); + if (group.valueKeys.has(valueKey)) { + group.changed = true; + continue; + } + group.valueKeys.add(valueKey); + group.values.push(value); + } + } + + for (const group of groups.values()) { + if (!group.changed) { + continue; + } + merged[group.firstIndex] = buildInCondition(group.left, group.values); + } + + return merged.filter((condition): condition is Condition => !!condition); +} + +function buildInCondition( + left: ValuePosition, + values: readonly InLiteralValue[], +): Condition { + switch (values.length) { + case 0: + return FALSE; + case 1: + return { + type: 'simple', + left, + op: '=', + right: {type: 'literal', value: values[0]}, + }; + default: + return { + type: 'simple', + left, + op: 'IN', + right: {type: 'literal', value: values}, + }; + } +} + +function getInMergeCandidate( + condition: Condition, +): InMergeCandidate | undefined { + if (condition.type !== 'simple') { + return undefined; + } + + if (condition.op === '=') { + const value = + condition.right.type === 'literal' ? condition.right.value : undefined; + if (isInLiteralValue(value)) { + return { + left: condition.left, + values: [value], + isAlreadyIn: false, + }; + } + return undefined; + } + + if (condition.op === 'IN' && condition.right.type === 'literal') { + const value = condition.right.value; + if (Array.isArray(value) && value.every(isInLiteralValue)) { + return { + left: condition.left, + values: value, + isAlreadyIn: true, + }; + } + } + + return undefined; +} + +function isInLiteralValue( + value: LiteralValue | undefined, +): value is InLiteralValue { + return value !== undefined && value !== null && !Array.isArray(value); +} + +function dedupeInLiteralValues( + values: readonly InLiteralValue[], +): InLiteralValue[] { + const seen = new Set(); + const deduped: InLiteralValue[] = []; + for (const value of values) { + const key = stableStringify(value); + if (seen.has(key)) { + continue; + } + seen.add(key); + deduped.push(value); + } + return deduped; +} + +function flatten( + type: 'and' | 'or', + conditions: readonly Condition[], +): Condition[] { + const flattened: Condition[] = []; + for (const condition of conditions) { + if (condition.type === type) { + flattened.push(...condition.conditions); + } else { + flattened.push(condition); + } + } + return flattened; +} + +function factorCommonConjuncts( + conditions: readonly Condition[], +): Condition | undefined { + if (conditions.length < 2) { + return undefined; + } + + const branches = conditions.map(condition => + condition.type === 'and' ? [...condition.conditions] : [condition], + ); + const commonKeys = new Set(branches[0].map(conditionKey)); + for (const branch of branches.slice(1)) { + const branchKeys = new Set(branch.map(conditionKey)); + for (const key of [...commonKeys]) { + if (!branchKeys.has(key)) { + commonKeys.delete(key); + } + } + } + + if (commonKeys.size === 0) { + return undefined; + } + + const common: Condition[] = []; + const firstBranch = branches[0]; + for (const condition of firstBranch) { + if (commonKeys.has(conditionKey(condition))) { + common.push(condition); + } + } + + const remainingBranches = branches.map(branch => + branch.filter(condition => !commonKeys.has(conditionKey(condition))), + ); + + return buildAnd([ + ...common, + buildOr(remainingBranches.map(branch => buildAnd(branch))), + ]); +} + +function mergeSameRelationshipExists( + conditions: readonly Condition[], +): Condition[] { + const merged: Array = [...conditions]; + const groups = new Map< + string, + { + readonly template: CorrelatedSubqueryCondition; + readonly firstIndex: number; + readonly filters: Condition[]; + } + >(); + + for (const [index, condition] of conditions.entries()) { + const key = mergeableExistsKey(condition); + if (!key || condition.type !== 'correlatedSubquery') { + continue; + } + + const group = groups.get(key); + const filter = condition.related.subquery.where ?? TRUE; + if (group) { + merged[index] = undefined; + group.filters.push(filter); + } else { + groups.set(key, { + template: condition, + firstIndex: index, + filters: [filter], + }); + } + } + + for (const group of groups.values()) { + merged[group.firstIndex] = mergeExistsGroup(group.template, group.filters); + } + + return dedupe( + merged.filter((condition): condition is Condition => !!condition), + ); +} + +function mergeExistsGroup( + template: CorrelatedSubqueryCondition, + filters: readonly Condition[], +): CorrelatedSubqueryCondition { + const where = buildOr(filters); + return { + ...template, + related: { + ...template.related, + subquery: { + ...template.related.subquery, + where: isAlwaysTrue(where) ? undefined : where, + }, + }, + }; +} + +function mergeableExistsKey(condition: Condition): string | undefined { + if (condition.type !== 'correlatedSubquery') { + return undefined; + } + if (condition.op !== 'EXISTS' || condition.scalar === true) { + return undefined; + } + + const {related} = condition; + const {subquery} = related; + if ( + subquery.related !== undefined || + subquery.start !== undefined || + subquery.limit !== undefined || + subquery.orderBy !== undefined + ) { + return undefined; + } + + const mergeShape: CorrelatedSubquery = { + ...related, + subquery: { + schema: subquery.schema, + table: subquery.table, + alias: subquery.alias, + }, + }; + return stableStringify({ + op: condition.op, + flip: condition.flip, + scalar: condition.scalar, + related: mergeShape, + }); +} + +function dedupe(conditions: readonly Condition[]): Condition[] { + const seen = new Set(); + const deduped: Condition[] = []; + for (const condition of conditions) { + const key = conditionKey(condition); + if (seen.has(key)) { + continue; + } + seen.add(key); + deduped.push(condition); + } + return deduped; +} + +function conditionKey(condition: Condition): string { + return stableStringify(condition); +} + +function stableStringify(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map(stableStringify).join(',')}]`; + } + if (value && typeof value === 'object') { + const entries = Object.entries(value) + .filter(([, entry]) => entry !== undefined) + .sort(([left], [right]) => left.localeCompare(right)); + return `{${entries + .map(([key, entry]) => `${JSON.stringify(key)}:${stableStringify(entry)}`) + .join(',')}}`; + } + return JSON.stringify(value) ?? 'undefined'; +} + +function isAlwaysTrue(condition: Condition): boolean { + return condition.type === 'and' && condition.conditions.length === 0; +} + +function isAlwaysFalse(condition: Condition): boolean { + return condition.type === 'or' && condition.conditions.length === 0; +} diff --git a/packages/zql/src/planner/planner-builder.ts b/packages/zql/src/planner/planner-builder.ts index 29eea8882a..16153cd061 100644 --- a/packages/zql/src/planner/planner-builder.ts +++ b/packages/zql/src/planner/planner-builder.ts @@ -9,6 +9,7 @@ import type { Disjunction, } from '../../../zero-protocol/src/ast.ts'; import {planIdSymbol} from '../../../zero-protocol/src/ast.ts'; +import {normalizePlannerAST} from './condition-normalizer.ts'; import type {ConnectionCostModel} from './planner-connection.ts'; import type {PlannerConstraint} from './planner-constraint.ts'; import type {PlanDebugger} from './planner-debug.ts'; @@ -314,9 +315,10 @@ export function planQuery( planDebugger?: PlanDebugger, lc?: LogContext, ): AST { - const plans = buildPlanGraph(ast, model, true); + const normalizedAST = normalizePlannerAST(ast); + const plans = buildPlanGraph(normalizedAST, model, true); planRecursively(plans, planDebugger, lc); - return applyPlansToAST(ast, plans); + return applyPlansToAST(normalizedAST, plans); } function applyToCondition( diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts index 8da1ae9d3d..2bf5aa80cf 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts @@ -18,40 +18,6 @@ const assignmentToStudentRelationship = relationshipName( export default { name: 'duplicate exists OR could dedupe before planning', - knownFailure: { - reason: - 'A OR A is exactly A, but duplicate exists branches still reach join enumeration as separate correlated subqueries.', - current: ` -OR - exists assignment_to_student where student_id = student-1 - exists assignment_to_student where student_id = student-1 - -Plan shape today: - -two equivalent exists branches enter the planner -`, - desired: ` -Deduped predicate: - -exists assignment_to_student where student_id = student-1 - -Desired plan shape: - -assignment_to_student student-1 => assignment -`, - currentSQL: [ - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', - }, - ], - engineIdea: - 'Add canonical condition keys during boolean normalization. Use them to collapse duplicate OR branches and duplicate AND branches before costing.', - }, schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/index.ts b/packages/zqlite/src/test/query-scenarios/scenarios/index.ts index b878303c36..780f18f5ad 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/index.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/index.ts @@ -12,12 +12,14 @@ import sharedParentFilterOrFactor from './shared-parent-filter-or-factor.ts'; import singleExistsChildOr from './single-exists-child-or.ts'; import studentMembershipMixedOr from './student-membership-mixed-or.ts'; import studentMembershipSimpleExists from './student-membership-simple-exists.ts'; +import teacherFilterOrIn from './teacher-filter-or-in.ts'; import teacherFilterWithMembership from './teacher-filter-with-membership.ts'; import twoStudentMembershipOr from './two-student-membership-or.ts'; export default [ studentMembershipMixedOr, studentMembershipSimpleExists, + teacherFilterOrIn, teacherFilterWithMembership, explicitFlipFalse, explicitFlipTrue, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts index d093dfdcf1..cb8e607593 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts @@ -18,42 +18,6 @@ const assignmentToStudentRelationship = relationshipName( export default { name: 'same relationship OR could merge into one flipped membership scan', - knownFailure: { - reason: - 'The planner currently flips each sibling exists independently. The desired plan would rewrite the OR into one exists over the same relationship, then flip that single child scan.', - current: ` -OR - exists assignment_to_student where student_id = student-1 - exists assignment_to_student where student_id = student-2 - -Plan shape today: - -assignment_to_student student-1 => assignment -assignment_to_student student-2 => assignment -`, - desired: ` -exists assignment_to_student where: - student_id = student-1 - OR - student_id = student-2 - -Desired plan shape: - -assignment_to_student student-1 OR student-2 => assignment -`, - currentSQL: [ - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', - }, - ], - engineIdea: - 'Add a boolean normalization pass that recognizes OR siblings with the same relationship, same correlation, and same exists options. Merge the child filters under one child OR before join enumeration.', - }, schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); @@ -97,11 +61,6 @@ assignment_to_student student-1 OR student-2 => assignment .orderBy(colName(assignment, 'created_at'), 'desc') .orderBy(colName(assignment, 'id'), 'asc'), expectations: { - // Current optimized AST keeps an OR with two flipped correlated subqueries. - // That means the optimized query shape is effectively two sibling child - // scans, each filtering one student_id, followed by parent lookups. The - // desired optimized AST is one flipped correlated subquery whose child - // filter is student-1 OR student-2, so one child scan feeds parent lookups. optimizedAST: { where: { type: 'correlatedSubquery', @@ -111,7 +70,7 @@ assignment_to_student student-1 OR student-2 => assignment sql: [ { table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE ("student_id" = ? OR "student_id" = ?) ORDER BY "assignment_id" asc, "student_id" asc', + sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" IN (SELECT value FROM json_each(?)) ORDER BY "assignment_id" asc, "student_id" asc', }, { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts index b46f02a742..27fe620c37 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts @@ -18,46 +18,6 @@ const assignmentToStudentRelationship = relationshipName( export default { name: 'shared parent filter across OR could factor before flipping child', - knownFailure: { - reason: - 'Both OR branches repeat the same parent predicate. Factoring that predicate out would leave one same relationship child OR that can flip as a single membership scan.', - current: ` -OR - AND - archived_at IS null - exists student-1 - AND - archived_at IS null - exists student-2 - -Plan shape today: - -student-1 => assignment with archived_at -student-2 => assignment with archived_at -`, - desired: ` -AND - archived_at IS null - exists child where: - student-1 OR student-2 - -Desired plan shape: - -assignment_to_student student-1 OR student-2 => assignment archived_at IS null -`, - currentSQL: [ - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND ("archived_at" IS ? OR "archived_at" IS ?) ORDER BY "created_at" desc, "id" asc', - }, - ], - engineIdea: - 'Run a small boolean algebra normalization before join enumeration. Factor common simple parent predicates out of OR branches, then run the same relationship OR merge pass on the remaining child predicates.', - }, schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); @@ -123,7 +83,10 @@ assignment_to_student student-1 OR student-2 => assignment archived_at IS null related: { subquery: { where: { - type: 'or', + type: 'simple', + op: 'IN', + left: {type: 'column', name: 'student_id'}, + right: {type: 'literal', value: ['student-1', 'student-2']}, }, }, }, @@ -134,7 +97,7 @@ assignment_to_student student-1 OR student-2 => assignment archived_at IS null sql: [ { table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE ("student_id" = ? OR "student_id" = ?) ORDER BY "assignment_id" asc, "student_id" asc', + sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" IN (SELECT value FROM json_each(?)) ORDER BY "assignment_id" asc, "student_id" asc', }, { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts index 2882a14dbc..b05d7c565f 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts @@ -61,7 +61,7 @@ export default { sql: [ { table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE ("student_id" = ? OR "student_id" = ?) ORDER BY "assignment_id" asc, "student_id" asc', + sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" IN (SELECT value FROM json_each(?)) ORDER BY "assignment_id" asc, "student_id" asc', }, { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts new file mode 100644 index 0000000000..9f33082feb --- /dev/null +++ b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts @@ -0,0 +1,52 @@ +import type {QueryScenario} from '../../query-scenario.ts'; +import { + colName, + createEducationAppTables, + educationAppSchema, + educationAppTables, + tableName, +} from '../education-app.ts'; + +const assignment = educationAppTables.assignment; + +export default { + name: 'same column parent OR compacts to an IN filter', + schema: educationAppSchema, + seed: db => { + const tables = createEducationAppTables(db); + const assignment = tables.assignment; + + const assignmentStmt = db.prepare( + `INSERT INTO ${tableName(assignment)} (${colName(assignment, 'id')}, ${colName(assignment, 'teacher_id')}, ${colName(assignment, 'archived_at')}, ${colName(assignment, 'created_at')}) VALUES (?, ?, ?, ?)`, + ); + for (let i = 1; i <= 2_000; i++) { + assignmentStmt.run(i, i % 25 === 0 ? 1 : 2, null, i); + } + }, + query: builder => + builder[assignment.name] + .where(({cmp, or}) => + or( + cmp(colName(assignment, 'teacher_id'), '=', 1), + cmp(colName(assignment, 'teacher_id'), '=', 2), + ), + ) + .orderBy(colName(assignment, 'created_at'), 'desc') + .orderBy(colName(assignment, 'id'), 'asc'), + expectations: { + optimizedAST: { + where: { + type: 'simple', + left: {type: 'column', name: 'teacher_id'}, + op: 'IN', + right: {type: 'literal', value: [1, 2]}, + }, + }, + sql: [ + { + table: 'assignment', + sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "teacher_id" IN (SELECT value FROM json_each(?)) ORDER BY "created_at" desc, "id" asc', + }, + ], + }, +} satisfies QueryScenario; diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts index 98068b47b8..793badde92 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts @@ -17,7 +17,7 @@ const assignmentToStudentRelationship = relationshipName( ); export default { - name: 'OR across two membership branches flips both branches', + name: 'OR across two membership branches merges and flips one child scan', schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); @@ -63,24 +63,25 @@ export default { expectations: { optimizedAST: { where: { - type: 'or', - conditions: [ - { - type: 'correlatedSubquery', - flip: true, + type: 'correlatedSubquery', + flip: true, + related: { + subquery: { + where: { + type: 'simple', + op: 'IN', + left: {type: 'column', name: 'student_id'}, + right: {type: 'literal', value: ['student-1', 'student-2']}, + }, }, - { - type: 'correlatedSubquery', - flip: true, - }, - ], + }, }, }, planDebug: ['flipped'], sql: [ { table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', + sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" IN (SELECT value FROM json_each(?)) ORDER BY "assignment_id" asc, "student_id" asc', }, { table: 'assignment', From 561ad13d0e7654d2bbe4128786c767f19d8506d1 Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 09:44:01 -0400 Subject: [PATCH 02/13] test(zql): harden planner normalization edge cases --- .../src/planner/condition-normalizer.test.ts | 199 ++++++++++++++++++ .../zql/src/planner/condition-normalizer.ts | 186 +++++++++++++++- .../scenarios/child-domain-intersection.ts | 85 ++++++++ .../scenarios/contradictory-parent-domain.ts | 50 +++++ .../test/query-scenarios/scenarios/index.ts | 4 + 5 files changed, 521 insertions(+), 3 deletions(-) create mode 100644 packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts create mode 100644 packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts diff --git a/packages/zql/src/planner/condition-normalizer.test.ts b/packages/zql/src/planner/condition-normalizer.test.ts index 2f5c8ae8a2..d51ed29e0a 100644 --- a/packages/zql/src/planner/condition-normalizer.test.ts +++ b/packages/zql/src/planner/condition-normalizer.test.ts @@ -1,10 +1,13 @@ +import fc from 'fast-check'; import {expect, test} from 'vitest'; import type { AST, Condition, CorrelatedSubqueryCondition, LiteralValue, + SimpleOperator, } from '../../../zero-protocol/src/ast.ts'; +import {createPredicate, type NoSubqueryCondition} from '../builder/filter.ts'; import {normalizePlannerAST} from './condition-normalizer.ts'; const TRUE: Condition = {type: 'and', conditions: []}; @@ -64,6 +67,65 @@ test('normalizes degenerate IN predicates', () => { }); }); +test('intersects same-column AND equalities and IN predicates', () => { + expect( + normalizeWhere({ + type: 'and', + conditions: [ + inCondition('status', 'IN', ['active', 'pending']), + eq('status', 'pending'), + ], + }), + ).toEqual(eq('status', 'pending')); + + expect( + normalizeWhere({ + type: 'and', + conditions: [ + inCondition('status', 'IN', ['active', 'pending']), + inCondition('status', 'IN', ['pending', 'draft']), + ], + }), + ).toEqual(eq('status', 'pending')); + + expect( + normalizeWhere({ + type: 'and', + conditions: [eq('status', 'active'), eq('status', 'pending')], + }), + ).toEqual(FALSE); +}); + +test('applies same-column AND exclusions', () => { + expect( + normalizeWhere({ + type: 'and', + conditions: [eq('status', 'active'), ne('status', 'active')], + }), + ).toEqual(FALSE); + + expect( + normalizeWhere({ + type: 'and', + conditions: [ + inCondition('status', 'IN', ['active', 'pending', 'draft']), + ne('status', 'pending'), + inCondition('status', 'NOT IN', ['draft']), + ], + }), + ).toEqual(eq('status', 'active')); + + expect( + normalizeWhere({ + type: 'and', + conditions: [ + inCondition('status', 'NOT IN', ['active']), + inCondition('status', 'NOT IN', ['pending']), + ], + }), + ).toEqual(inCondition('status', 'NOT IN', ['active', 'pending'])); +}); + test('merges OR exists branches over the same relationship', () => { expect( normalizeWhere({ @@ -80,6 +142,15 @@ test('merges OR exists branches over the same relationship', () => { ); }); +test('preserves unrestricted exists when merging narrower exists branches', () => { + expect( + normalizeWhere({ + type: 'or', + conditions: [exists(undefined), exists(eq('title', 'hello'))], + }), + ).toEqual(exists(undefined)); +}); + test('factors common parent filters before merging child exists branches', () => { const active = eq('active', true); @@ -197,6 +268,38 @@ test('normalizes related subqueries recursively', () => { }); }); +test('preserves simple filter semantics under generated rows', () => { + fc.assert( + fc.property( + filterConditionArbitrary(), + rowArbitrary(), + (condition, row) => { + const normalized = normalizePlannerAST({ + table: 'users', + where: condition, + }).where as NoSubqueryCondition | undefined; + const before = createPredicate(condition)(row); + const after = normalized ? createPredicate(normalized)(row) : true; + + expect(after).toBe(before); + }, + ), + {numRuns: 1_000}, + ); +}); + +test('is idempotent for generated simple filters', () => { + fc.assert( + fc.property(filterConditionArbitrary(), condition => { + const once = normalizePlannerAST({table: 'users', where: condition}); + const twice = normalizePlannerAST(once); + + expect(twice).toEqual(once); + }), + {numRuns: 1_000}, + ); +}); + function normalizeWhere(where: Condition): Condition | undefined { return normalizePlannerAST({table: 'users', where}).where; } @@ -210,6 +313,15 @@ function eq(name: string, value: LiteralValue): Condition { }; } +function ne(name: string, value: LiteralValue): Condition { + return { + type: 'simple', + left: {type: 'column', name}, + op: '!=', + right: {type: 'literal', value}, + }; +} + function inCondition( name: string, op: 'IN' | 'NOT IN', @@ -248,3 +360,90 @@ function exists( }, }; } + +function filterConditionArbitrary(): fc.Arbitrary { + return fc.letrec(tie => ({ + condition: fc.oneof( + simpleConditionArbitrary(), + fc.record({ + type: fc.constant('and' as const), + conditions: fc.array(tie('condition'), {maxLength: 4}), + }), + fc.record({ + type: fc.constant('or' as const), + conditions: fc.array(tie('condition'), {maxLength: 4}), + }), + ), + })).condition as fc.Arbitrary; +} + +function simpleConditionArbitrary(): fc.Arbitrary { + return fc.oneof( + fc.record({ + type: fc.constant('simple' as const), + left: columnReferenceArbitrary(), + op: fc.constantFrom('=', '!=', '<', '<=', '>', '>='), + right: fc.record({ + type: fc.constant('literal' as const), + value: scalarLiteralArbitrary(), + }), + }), + fc.record({ + type: fc.constant('simple' as const), + left: columnReferenceArbitrary(), + op: fc.constantFrom('IS', 'IS NOT'), + right: fc.record({ + type: fc.constant('literal' as const), + value: fc.constant(null), + }), + }), + fc.record({ + type: fc.constant('simple' as const), + left: columnReferenceArbitrary(), + op: fc.constantFrom('IN', 'NOT IN'), + right: fc.record({ + type: fc.constant('literal' as const), + value: fc.array(inLiteralArbitrary(), {maxLength: 6}), + }), + }), + ); +} + +function rowArbitrary() { + return fc.record({ + a: rowValueArbitrary(), + b: rowValueArbitrary(), + c: rowValueArbitrary(), + }); +} + +function columnReferenceArbitrary() { + return fc.record({ + type: fc.constant('column' as const), + name: fc.constantFrom('a', 'b', 'c'), + }); +} + +function scalarLiteralArbitrary(): fc.Arbitrary< + string | number | boolean | null +> { + return fc.oneof(inLiteralArbitrary(), fc.constant(null)); +} + +function rowValueArbitrary(): fc.Arbitrary< + string | number | boolean | null | undefined +> { + return fc.oneof( + inLiteralArbitrary(), + fc.constant(null), + fc.constant(undefined), + ); +} + +function inLiteralArbitrary(): fc.Arbitrary { + return fc.oneof( + fc.string({maxLength: 8}), + fc.integer({min: -20, max: 20}), + fc.boolean(), + ); +} diff --git a/packages/zql/src/planner/condition-normalizer.ts b/packages/zql/src/planner/condition-normalizer.ts index ae3a1f7bb0..c36b7151a8 100644 --- a/packages/zql/src/planner/condition-normalizer.ts +++ b/packages/zql/src/planner/condition-normalizer.ts @@ -109,13 +109,18 @@ function buildAnd(conditions: readonly Condition[]): Condition { const deduped = dedupe( flattened.filter(condition => !isAlwaysTrue(condition)), ); - switch (deduped.length) { + const intersected = intersectEquivalentAndPredicates(deduped); + if (intersected.some(isAlwaysFalse)) { + return FALSE; + } + + switch (intersected.length) { case 0: return TRUE; case 1: - return deduped[0]; + return intersected[0]; default: - return {type: 'and', conditions: deduped}; + return {type: 'and', conditions: intersected}; } } @@ -166,6 +171,149 @@ type InMergeGroup = { changed: boolean; }; +type DomainCandidate = { + readonly left: ValuePosition; + readonly include?: readonly InLiteralValue[] | undefined; + readonly exclude: readonly InLiteralValue[]; + readonly isAlreadyCompound: boolean; +}; + +type DomainGroup = { + readonly left: ValuePosition; + readonly firstIndex: number; + include: InLiteralValue[] | undefined; + readonly exclude: InLiteralValue[]; + readonly excludeKeys: Set; + changed: boolean; +}; + +function intersectEquivalentAndPredicates( + conditions: readonly Condition[], +): Condition[] { + const intersected: Array = [...conditions]; + const groups = new Map(); + + for (const [index, condition] of conditions.entries()) { + const candidate = getDomainCandidate(condition); + if (!candidate) { + continue; + } + + const key = stableStringify(candidate.left); + const group = groups.get(key); + if (!group) { + const nextGroup: DomainGroup = { + left: candidate.left, + firstIndex: index, + include: + candidate.include === undefined + ? undefined + : dedupeInLiteralValues(candidate.include), + exclude: [], + excludeKeys: new Set(), + changed: candidate.isAlreadyCompound, + }; + groups.set(key, nextGroup); + addExcludedValues(nextGroup, candidate.exclude); + continue; + } + + intersected[index] = undefined; + group.changed = true; + if (candidate.include !== undefined) { + group.include = + group.include === undefined + ? dedupeInLiteralValues(candidate.include) + : intersectInLiteralValues(group.include, candidate.include); + } + addExcludedValues(group, candidate.exclude); + } + + for (const group of groups.values()) { + if (!group.changed) { + continue; + } + intersected[group.firstIndex] = buildDomainCondition(group); + } + + return intersected.filter((condition): condition is Condition => !!condition); +} + +function getDomainCandidate(condition: Condition): DomainCandidate | undefined { + if (condition.type !== 'simple' || condition.right.type !== 'literal') { + return undefined; + } + + const {value} = condition.right; + switch (condition.op) { + case '=': + return isInLiteralValue(value) + ? { + left: condition.left, + include: [value], + exclude: [], + isAlreadyCompound: false, + } + : undefined; + case 'IN': + return Array.isArray(value) && value.every(isInLiteralValue) + ? { + left: condition.left, + include: value, + exclude: [], + isAlreadyCompound: true, + } + : undefined; + case '!=': + return isInLiteralValue(value) + ? { + left: condition.left, + exclude: [value], + isAlreadyCompound: false, + } + : undefined; + case 'NOT IN': + return Array.isArray(value) && + value.length > 0 && + value.every(isInLiteralValue) + ? { + left: condition.left, + exclude: value, + isAlreadyCompound: true, + } + : undefined; + default: + return undefined; + } +} + +function addExcludedValues( + group: DomainGroup, + values: readonly InLiteralValue[], +): void { + for (const value of values) { + const key = stableStringify(value); + if (group.excludeKeys.has(key)) { + group.changed = true; + continue; + } + group.excludeKeys.add(key); + group.exclude.push(value); + } +} + +function buildDomainCondition(group: DomainGroup): Condition { + if (group.include !== undefined) { + return buildInCondition( + group.left, + group.include.filter( + value => !group.excludeKeys.has(stableStringify(value)), + ), + ); + } + return buildNotInCondition(group.left, group.exclude); +} + function mergeEquivalentOrPredicates( conditions: readonly Condition[], ): Condition[] { @@ -239,6 +387,30 @@ function buildInCondition( } } +function buildNotInCondition( + left: ValuePosition, + values: readonly InLiteralValue[], +): Condition { + switch (values.length) { + case 0: + return TRUE; + case 1: + return { + type: 'simple', + left, + op: '!=', + right: {type: 'literal', value: values[0]}, + }; + default: + return { + type: 'simple', + left, + op: 'NOT IN', + right: {type: 'literal', value: values}, + }; + } +} + function getInMergeCandidate( condition: Condition, ): InMergeCandidate | undefined { @@ -295,6 +467,14 @@ function dedupeInLiteralValues( return deduped; } +function intersectInLiteralValues( + left: readonly InLiteralValue[], + right: readonly InLiteralValue[], +): InLiteralValue[] { + const rightValues = new Set(right.map(stableStringify)); + return left.filter(value => rightValues.has(stableStringify(value))); +} + function flatten( type: 'and' | 'or', conditions: readonly Condition[], diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts new file mode 100644 index 0000000000..6df96d2721 --- /dev/null +++ b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts @@ -0,0 +1,85 @@ +import type {QueryScenario} from '../../query-scenario.ts'; +import { + colName, + createEducationAppTables, + educationAppRelationships, + educationAppSchema, + educationAppTables, + relationshipName, + tableName, +} from '../education-app.ts'; + +const assignment = educationAppTables.assignment; +const assignmentToStudent = educationAppTables.assignment_to_student; +const assignmentToStudentRelationship = relationshipName( + educationAppRelationships.assignment, + 'assignment_to_student', +); + +export default { + name: 'child IN plus exclusion narrows before auto flipping', + schema: educationAppSchema, + seed: db => { + const tables = createEducationAppTables(db); + const assignment = tables.assignment; + const assignmentToStudent = tables.assignment_to_student; + + const assignmentStmt = db.prepare( + `INSERT INTO ${tableName(assignment)} (${colName(assignment, 'id')}, ${colName(assignment, 'teacher_id')}, ${colName(assignment, 'archived_at')}, ${colName(assignment, 'created_at')}) VALUES (?, ?, ?, ?)`, + ); + for (let i = 1; i <= 2_000; i++) { + assignmentStmt.run(i, 2, null, i); + } + + const membershipStmt = db.prepare( + `INSERT INTO ${tableName(assignmentToStudent)} (${colName(assignmentToStudent, 'assignment_id')}, ${colName(assignmentToStudent, 'student_id')}, ${colName(assignmentToStudent, 'created_at')}) VALUES (?, ?, ?)`, + ); + membershipStmt.run(101, 'student-1', 101); + membershipStmt.run(102, 'student-1', 102); + membershipStmt.run(1_500, 'student-2', 1_500); + }, + query: builder => + builder[assignment.name] + .whereExists(assignmentToStudentRelationship, q => + q.where(({and, cmp}) => + and( + cmp(colName(assignmentToStudent, 'student_id'), 'IN', [ + 'student-1', + 'student-2', + ]), + cmp(colName(assignmentToStudent, 'student_id'), '!=', 'student-2'), + ), + ), + ) + .orderBy(colName(assignment, 'created_at'), 'desc') + .orderBy(colName(assignment, 'id'), 'asc'), + expectations: { + optimizedAST: { + where: { + type: 'correlatedSubquery', + flip: true, + related: { + subquery: { + where: { + type: 'simple', + op: '=', + left: {type: 'column', name: 'student_id'}, + right: {type: 'literal', value: 'student-1'}, + }, + }, + }, + }, + }, + planDebug: ['flipped'], + sql: [ + { + table: 'assignment_to_student', + sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', + }, + { + table: 'assignment', + sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + }, + ], + }, +} satisfies QueryScenario; diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts new file mode 100644 index 0000000000..b04ab16fb4 --- /dev/null +++ b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts @@ -0,0 +1,50 @@ +import type {QueryScenario} from '../../query-scenario.ts'; +import { + colName, + createEducationAppTables, + educationAppSchema, + educationAppTables, + tableName, +} from '../education-app.ts'; + +const assignment = educationAppTables.assignment; + +export default { + name: 'contradictory same column parent filters collapse to false', + schema: educationAppSchema, + seed: db => { + const tables = createEducationAppTables(db); + const assignment = tables.assignment; + + const assignmentStmt = db.prepare( + `INSERT INTO ${tableName(assignment)} (${colName(assignment, 'id')}, ${colName(assignment, 'teacher_id')}, ${colName(assignment, 'archived_at')}, ${colName(assignment, 'created_at')}) VALUES (?, ?, ?, ?)`, + ); + for (let i = 1; i <= 100; i++) { + assignmentStmt.run(i, i % 2 === 0 ? 1 : 2, null, i); + } + }, + query: builder => + builder[assignment.name] + .where(({and, cmp}) => + and( + cmp(colName(assignment, 'teacher_id'), '=', 1), + cmp(colName(assignment, 'teacher_id'), '!=', 1), + ), + ) + .orderBy(colName(assignment, 'created_at'), 'desc') + .orderBy(colName(assignment, 'id'), 'asc'), + expectations: { + optimizedAST: { + where: { + type: 'or', + conditions: [], + }, + }, + sql: [ + { + table: 'assignment', + sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE FALSE ORDER BY "created_at" desc, "id" asc', + }, + ], + }, +} satisfies QueryScenario; diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/index.ts b/packages/zqlite/src/test/query-scenarios/scenarios/index.ts index 780f18f5ad..6a320c4c02 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/index.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/index.ts @@ -1,5 +1,7 @@ import type {QueryScenario} from '../../query-scenario.ts'; import type {educationAppSchema} from '../education-app.ts'; +import childDomainIntersection from './child-domain-intersection.ts'; +import contradictoryParentDomain from './contradictory-parent-domain.ts'; import duplicateExistsOrDedupe from './duplicate-exists-or-dedupe.ts'; import explicitFlipFalse from './explicit-flip-false.ts'; import explicitFlipTrue from './explicit-flip-true.ts'; @@ -20,6 +22,8 @@ export default [ studentMembershipMixedOr, studentMembershipSimpleExists, teacherFilterOrIn, + contradictoryParentDomain, + childDomainIntersection, teacherFilterWithMembership, explicitFlipFalse, explicitFlipTrue, From 7dd20954364343e2e08fe9c6008d577e14f47baa Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 09:49:59 -0400 Subject: [PATCH 03/13] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20model=20?= =?UTF-8?q?planner=20predicates=20as=20domains?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why: makes the query optimizer rules easier to audit and extend. * Replace parallel OR and AND merge bookkeeping with a shared column domain rewrite rule * Add coverage for overlapping IN predicate unions --- .../src/planner/condition-normalizer.test.ts | 15 + .../zql/src/planner/condition-normalizer.ts | 355 +++++++++--------- 2 files changed, 199 insertions(+), 171 deletions(-) diff --git a/packages/zql/src/planner/condition-normalizer.test.ts b/packages/zql/src/planner/condition-normalizer.test.ts index d51ed29e0a..a470ef7ed7 100644 --- a/packages/zql/src/planner/condition-normalizer.test.ts +++ b/packages/zql/src/planner/condition-normalizer.test.ts @@ -49,6 +49,21 @@ test('rewrites same-column OR equalities to IN', () => { ], }), ).toEqual(eq('status', 'active')); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + inCondition('status', 'IN', ['active', 'pending']), + inCondition('status', 'IN', ['pending', 'draft']), + ], + }), + ).toEqual({ + type: 'simple', + left: {type: 'column', name: 'status'}, + op: 'IN', + right: {type: 'literal', value: ['active', 'pending', 'draft']}, + }); }); test('normalizes degenerate IN predicates', () => { diff --git a/packages/zql/src/planner/condition-normalizer.ts b/packages/zql/src/planner/condition-normalizer.ts index c36b7151a8..5fef4d50c3 100644 --- a/packages/zql/src/planner/condition-normalizer.ts +++ b/packages/zql/src/planner/condition-normalizer.ts @@ -155,91 +155,103 @@ function buildOr(conditions: readonly Condition[]): Condition { } } -type InMergeCandidate = { - readonly left: ValuePosition; - readonly values: readonly InLiteralValue[]; - readonly isAlreadyIn: boolean; -}; - type InLiteralValue = string | number | boolean; -type InMergeGroup = { +type ColumnDomainBase = { readonly left: ValuePosition; - readonly firstIndex: number; - readonly values: InLiteralValue[]; - readonly valueKeys: Set; - changed: boolean; }; -type DomainCandidate = { - readonly left: ValuePosition; - readonly include?: readonly InLiteralValue[] | undefined; +// include undefined means the column can still take any literal value. +// exclude then removes known impossible values from that open domain. +type ColumnDomain = ColumnDomainBase & { + readonly include: readonly InLiteralValue[] | undefined; readonly exclude: readonly InLiteralValue[]; - readonly isAlreadyCompound: boolean; }; -type DomainGroup = { - readonly left: ValuePosition; +type PositiveColumnDomain = ColumnDomainBase & { + readonly include: readonly InLiteralValue[]; +}; + +type DomainPredicate = { + readonly domain: TDomain; + readonly changed: boolean; +}; + +type DomainGroup = { readonly firstIndex: number; - include: InLiteralValue[] | undefined; - readonly exclude: InLiteralValue[]; - readonly excludeKeys: Set; + domain: TDomain; changed: boolean; }; +type DomainRewrite = { + readonly extract: ( + condition: Condition, + ) => DomainPredicate | undefined; + readonly combine: (left: TDomain, right: TDomain) => TDomain; + readonly lower: (domain: TDomain) => Condition; +}; + function intersectEquivalentAndPredicates( conditions: readonly Condition[], ): Condition[] { - const intersected: Array = [...conditions]; - const groups = new Map(); + return rewriteColumnDomains(conditions, { + extract: columnDomainFromCondition, + combine: intersectColumnDomains, + lower: columnDomainToCondition, + }); +} +function mergeEquivalentOrPredicates( + conditions: readonly Condition[], +): Condition[] { + return rewriteColumnDomains(conditions, { + extract: positiveColumnDomainFromCondition, + combine: unionPositiveColumnDomains, + lower: positiveColumnDomainToCondition, + }); +} + +function rewriteColumnDomains( + conditions: readonly Condition[], + rewrite: DomainRewrite, +): Condition[] { + const rewritten: Array = [...conditions]; + const groups = new Map>(); for (const [index, condition] of conditions.entries()) { - const candidate = getDomainCandidate(condition); - if (!candidate) { + const predicate = rewrite.extract(condition); + if (!predicate) { continue; } - const key = stableStringify(candidate.left); + const key = domainKey(predicate.domain); const group = groups.get(key); if (!group) { - const nextGroup: DomainGroup = { - left: candidate.left, + groups.set(key, { firstIndex: index, - include: - candidate.include === undefined - ? undefined - : dedupeInLiteralValues(candidate.include), - exclude: [], - excludeKeys: new Set(), - changed: candidate.isAlreadyCompound, - }; - groups.set(key, nextGroup); - addExcludedValues(nextGroup, candidate.exclude); + domain: predicate.domain, + changed: predicate.changed, + }); continue; } - intersected[index] = undefined; + rewritten[index] = undefined; group.changed = true; - if (candidate.include !== undefined) { - group.include = - group.include === undefined - ? dedupeInLiteralValues(candidate.include) - : intersectInLiteralValues(group.include, candidate.include); - } - addExcludedValues(group, candidate.exclude); + group.domain = rewrite.combine(group.domain, predicate.domain); } for (const group of groups.values()) { if (!group.changed) { continue; } - intersected[group.firstIndex] = buildDomainCondition(group); + rewritten[group.firstIndex] = rewrite.lower(group.domain); } - return intersected.filter((condition): condition is Condition => !!condition); + return rewritten.filter((condition): condition is Condition => !!condition); } -function getDomainCandidate(condition: Condition): DomainCandidate | undefined { +function columnDomainFromCondition( + condition: Condition, +): DomainPredicate | undefined { if (condition.type !== 'simple' || condition.right.type !== 'literal') { return undefined; } @@ -248,119 +260,108 @@ function getDomainCandidate(condition: Condition): DomainCandidate | undefined { switch (condition.op) { case '=': return isInLiteralValue(value) - ? { - left: condition.left, - include: [value], - exclude: [], - isAlreadyCompound: false, - } + ? finiteColumnDomain(condition.left, [value]) : undefined; case 'IN': return Array.isArray(value) && value.every(isInLiteralValue) - ? { - left: condition.left, - include: value, - exclude: [], - isAlreadyCompound: true, - } + ? finiteColumnDomain(condition.left, value) : undefined; case '!=': return isInLiteralValue(value) - ? { - left: condition.left, - exclude: [value], - isAlreadyCompound: false, - } + ? excludedColumnDomain(condition.left, [value]) : undefined; case 'NOT IN': return Array.isArray(value) && value.length > 0 && value.every(isInLiteralValue) - ? { - left: condition.left, - exclude: value, - isAlreadyCompound: true, - } + ? excludedColumnDomain(condition.left, value) : undefined; default: return undefined; } } -function addExcludedValues( - group: DomainGroup, - values: readonly InLiteralValue[], -): void { - for (const value of values) { - const key = stableStringify(value); - if (group.excludeKeys.has(key)) { - group.changed = true; - continue; - } - group.excludeKeys.add(key); - group.exclude.push(value); +function positiveColumnDomainFromCondition( + condition: Condition, +): DomainPredicate | undefined { + const predicate = columnDomainFromCondition(condition); + if (!predicate || predicate.domain.include === undefined) { + return undefined; + } + if (predicate.domain.exclude.length > 0) { + return undefined; } + + return { + domain: { + left: predicate.domain.left, + include: predicate.domain.include, + }, + changed: predicate.changed, + }; } -function buildDomainCondition(group: DomainGroup): Condition { - if (group.include !== undefined) { - return buildInCondition( - group.left, - group.include.filter( - value => !group.excludeKeys.has(stableStringify(value)), - ), - ); - } - return buildNotInCondition(group.left, group.exclude); +function finiteColumnDomain( + left: ValuePosition, + values: readonly InLiteralValue[], +): DomainPredicate { + const include = literalSet(values); + return { + domain: {left, include, exclude: []}, + changed: include.length !== values.length, + }; } -function mergeEquivalentOrPredicates( - conditions: readonly Condition[], -): Condition[] { - const merged: Array = [...conditions]; - const groups = new Map(); +function excludedColumnDomain( + left: ValuePosition, + values: readonly InLiteralValue[], +): DomainPredicate { + const exclude = literalSet(values); + return { + domain: {left, include: undefined, exclude}, + changed: exclude.length !== values.length, + }; +} - for (const [index, condition] of conditions.entries()) { - const candidate = getInMergeCandidate(condition); - if (!candidate) { - continue; - } +function intersectColumnDomains( + left: ColumnDomain, + right: ColumnDomain, +): ColumnDomain { + return { + left: left.left, + include: intersectOptionalValueSets(left.include, right.include), + exclude: unionLiteralValues(left.exclude, right.exclude), + }; +} - const key = stableStringify(candidate.left); - let group = groups.get(key); - if (!group) { - group = { - left: candidate.left, - firstIndex: index, - values: [], - valueKeys: new Set(), - changed: candidate.isAlreadyIn, - }; - groups.set(key, group); - } else { - merged[index] = undefined; - group.changed = true; - } +function unionPositiveColumnDomains( + left: PositiveColumnDomain, + right: PositiveColumnDomain, +): PositiveColumnDomain { + return { + left: left.left, + include: unionLiteralValues(left.include, right.include), + }; +} - for (const value of candidate.values) { - const valueKey = stableStringify(value); - if (group.valueKeys.has(valueKey)) { - group.changed = true; - continue; - } - group.valueKeys.add(valueKey); - group.values.push(value); - } +function columnDomainToCondition(domain: ColumnDomain): Condition { + if (domain.include !== undefined) { + return buildInCondition( + domain.left, + subtractLiteralValues(domain.include, domain.exclude), + ); } + return buildNotInCondition(domain.left, domain.exclude); +} - for (const group of groups.values()) { - if (!group.changed) { - continue; - } - merged[group.firstIndex] = buildInCondition(group.left, group.values); - } +function positiveColumnDomainToCondition( + domain: PositiveColumnDomain, +): Condition { + return buildInCondition(domain.left, domain.include); +} - return merged.filter((condition): condition is Condition => !!condition); +function domainKey(domain: ColumnDomainBase): string { + return stableStringify(domain.left); } function buildInCondition( @@ -411,40 +412,6 @@ function buildNotInCondition( } } -function getInMergeCandidate( - condition: Condition, -): InMergeCandidate | undefined { - if (condition.type !== 'simple') { - return undefined; - } - - if (condition.op === '=') { - const value = - condition.right.type === 'literal' ? condition.right.value : undefined; - if (isInLiteralValue(value)) { - return { - left: condition.left, - values: [value], - isAlreadyIn: false, - }; - } - return undefined; - } - - if (condition.op === 'IN' && condition.right.type === 'literal') { - const value = condition.right.value; - if (Array.isArray(value) && value.every(isInLiteralValue)) { - return { - left: condition.left, - values: value, - isAlreadyIn: true, - }; - } - } - - return undefined; -} - function isInLiteralValue( value: LiteralValue | undefined, ): value is InLiteralValue { @@ -454,10 +421,14 @@ function isInLiteralValue( function dedupeInLiteralValues( values: readonly InLiteralValue[], ): InLiteralValue[] { + return literalSet(values); +} + +function literalSet(values: readonly InLiteralValue[]): InLiteralValue[] { const seen = new Set(); const deduped: InLiteralValue[] = []; for (const value of values) { - const key = stableStringify(value); + const key = literalValueKey(value); if (seen.has(key)) { continue; } @@ -467,12 +438,54 @@ function dedupeInLiteralValues( return deduped; } -function intersectInLiteralValues( +function intersectOptionalValueSets( + left: readonly InLiteralValue[] | undefined, + right: readonly InLiteralValue[] | undefined, +): readonly InLiteralValue[] | undefined { + if (left === undefined) { + return right; + } + if (right === undefined) { + return left; + } + return intersectLiteralValues(left, right); +} + +function unionLiteralValues( + left: readonly InLiteralValue[], + right: readonly InLiteralValue[], +): InLiteralValue[] { + const seen = new Set(left.map(literalValueKey)); + const union = [...left]; + for (const value of right) { + const key = literalValueKey(value); + if (seen.has(key)) { + continue; + } + seen.add(key); + union.push(value); + } + return union; +} + +function intersectLiteralValues( left: readonly InLiteralValue[], right: readonly InLiteralValue[], ): InLiteralValue[] { - const rightValues = new Set(right.map(stableStringify)); - return left.filter(value => rightValues.has(stableStringify(value))); + const rightValues = new Set(right.map(literalValueKey)); + return left.filter(value => rightValues.has(literalValueKey(value))); +} + +function subtractLiteralValues( + values: readonly InLiteralValue[], + excluded: readonly InLiteralValue[], +): InLiteralValue[] { + const excludedValues = new Set(excluded.map(literalValueKey)); + return values.filter(value => !excludedValues.has(literalValueKey(value))); +} + +function literalValueKey(value: InLiteralValue): string { + return stableStringify(value); } function flatten( From 5b3dd0f5c3f6fc795c15daf047edede9ca7578eb Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 10:02:40 -0400 Subject: [PATCH 04/13] =?UTF-8?q?=F0=9F=A7=AA=20test:=20harden=20planner?= =?UTF-8?q?=20normalization=20pathologies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why: pin the edge cases that can quietly reintroduce broad scans or dead child branches. * Absorb stricter OR branches even when the shared predicate is not common to every branch * Collapse impossible non-scalar EXISTS branches and add a scenario that proves the child scan disappears * Extend idempotence coverage to generated correlated subquery filters --- .../src/planner/condition-normalizer.test.ts | 108 ++++++++++++++++++ .../zql/src/planner/condition-normalizer.ts | 89 +++++++++++++-- .../impossible-exists-or-parent-filter.ts | 61 ++++++++++ .../test/query-scenarios/scenarios/index.ts | 2 + 4 files changed, 250 insertions(+), 10 deletions(-) create mode 100644 packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts diff --git a/packages/zql/src/planner/condition-normalizer.test.ts b/packages/zql/src/planner/condition-normalizer.test.ts index a470ef7ed7..31575419b5 100644 --- a/packages/zql/src/planner/condition-normalizer.test.ts +++ b/packages/zql/src/planner/condition-normalizer.test.ts @@ -199,6 +199,8 @@ test('factors common parent filters before merging child exists branches', () => test('absorbs redundant branches after factoring common predicates', () => { const active = eq('active', true); + const archived = eq('archived', false); + const pending = eq('status', 'pending'); expect( normalizeWhere({ @@ -212,6 +214,63 @@ test('absorbs redundant branches after factoring common predicates', () => { ], }), ).toEqual(active); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + active, + { + type: 'and', + conditions: [active, pending], + }, + archived, + ], + }), + ).toEqual({ + type: 'or', + conditions: [active, archived], + }); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + { + type: 'and', + conditions: [active, pending], + }, + { + type: 'and', + conditions: [active, pending, archived], + }, + ], + }), + ).toEqual({ + type: 'and', + conditions: [active, pending], + }); +}); + +test('collapses impossible exists subqueries', () => { + const active = eq('active', true); + + expect(normalizeWhere(exists(FALSE))).toEqual(FALSE); + expect(normalizeWhere(exists(FALSE, {op: 'NOT EXISTS'}))).toEqual(TRUE); + expect(normalizeWhere(exists(undefined, {subquery: {limit: 0}}))).toEqual( + FALSE, + ); + + expect( + normalizeWhere({ + type: 'or', + conditions: [active, exists(FALSE)], + }), + ).toEqual(active); + + expect(normalizeWhere(exists(FALSE, {scalar: true}))).toEqual( + exists(FALSE, {scalar: true}), + ); }); test('does not merge exists branches with different semantics', () => { @@ -315,6 +374,18 @@ test('is idempotent for generated simple filters', () => { ); }); +test('is idempotent for generated planner filters with correlated subqueries', () => { + fc.assert( + fc.property(plannerConditionArbitrary(), condition => { + const once = normalizePlannerAST({table: 'users', where: condition}); + const twice = normalizePlannerAST(once); + + expect(twice).toEqual(once); + }), + {numRuns: 1_000}, + ); +}); + function normalizeWhere(where: Condition): Condition | undefined { return normalizePlannerAST({table: 'users', where}).where; } @@ -392,6 +463,43 @@ function filterConditionArbitrary(): fc.Arbitrary { })).condition as fc.Arbitrary; } +function plannerConditionArbitrary(): fc.Arbitrary { + return fc.letrec(tie => ({ + condition: fc.oneof( + simpleConditionArbitrary(), + correlatedSubqueryConditionArbitrary(), + fc.record({ + type: fc.constant('and' as const), + conditions: fc.array(tie('condition'), {maxLength: 4}), + }), + fc.record({ + type: fc.constant('or' as const), + conditions: fc.array(tie('condition'), {maxLength: 4}), + }), + ), + })).condition as fc.Arbitrary; +} + +function correlatedSubqueryConditionArbitrary(): fc.Arbitrary { + return fc.record({ + type: fc.constant('correlatedSubquery' as const), + op: fc.constantFrom('EXISTS' as const, 'NOT EXISTS' as const), + flip: fc.option(fc.boolean(), {nil: undefined}), + scalar: fc.option(fc.boolean(), {nil: undefined}), + related: fc.record({ + correlation: fc.constant({ + parentField: ['id'], + childField: ['userId'], + } as const), + subquery: fc.record({ + table: fc.constant('posts'), + where: fc.option(filterConditionArbitrary(), {nil: undefined}), + limit: fc.option(fc.constantFrom(0, 1, 10), {nil: undefined}), + }), + }), + }); +} + function simpleConditionArbitrary(): fc.Arbitrary { return fc.oneof( fc.record({ diff --git a/packages/zql/src/planner/condition-normalizer.ts b/packages/zql/src/planner/condition-normalizer.ts index 5fef4d50c3..411230e17a 100644 --- a/packages/zql/src/planner/condition-normalizer.ts +++ b/packages/zql/src/planner/condition-normalizer.ts @@ -89,15 +89,15 @@ function normalizeSimpleCondition(condition: SimpleCondition): Condition { function normalizeCorrelatedSubquery( condition: CorrelatedSubqueryCondition, -): CorrelatedSubqueryCondition { +): Condition { const {[planIdSymbol]: _planId, ...conditionWithoutPlanId} = condition; - return { + return collapseImpossibleCorrelatedSubquery({ ...conditionWithoutPlanId, related: { ...condition.related, subquery: normalizePlannerAST(condition.related.subquery), }, - }; + }); } function buildAnd(conditions: readonly Condition[]): Condition { @@ -137,7 +137,9 @@ function buildOr(conditions: readonly Condition[]): Condition { return FALSE; } - const compacted = mergeEquivalentOrPredicates(deduped); + const compacted = absorbRedundantOrBranches( + mergeEquivalentOrPredicates(deduped), + ); const factored = factorCommonConjuncts(compacted); if (factored) { @@ -145,13 +147,18 @@ function buildOr(conditions: readonly Condition[]): Condition { } const merged = mergeSameRelationshipExists(compacted); - switch (merged.length) { + if (merged.some(isAlwaysTrue)) { + return TRUE; + } + + const surviving = merged.filter(condition => !isAlwaysFalse(condition)); + switch (surviving.length) { case 0: return FALSE; case 1: - return merged[0]; + return surviving[0]; default: - return {type: 'or', conditions: merged}; + return {type: 'or', conditions: surviving}; } } @@ -364,6 +371,22 @@ function domainKey(domain: ColumnDomainBase): string { return stableStringify(domain.left); } +function collapseImpossibleCorrelatedSubquery( + condition: CorrelatedSubqueryCondition, +): Condition { + if (condition.scalar === true) { + return condition; + } + const {subquery} = condition.related; + const subqueryCannotEmitRows = + subquery.limit === 0 || + (subquery.where !== undefined && isAlwaysFalse(subquery.where)); + if (!subqueryCannotEmitRows) { + return condition; + } + return condition.op === 'EXISTS' ? FALSE : TRUE; +} + function buildInCondition( left: ValuePosition, values: readonly InLiteralValue[], @@ -545,6 +568,52 @@ function factorCommonConjuncts( ]); } +function absorbRedundantOrBranches( + conditions: readonly Condition[], +): Condition[] { + const branches = conditions.map(condition => ({ + keys: + condition.type === 'and' + ? new Set(condition.conditions.map(conditionKey)) + : new Set([conditionKey(condition)]), + })); + const keep = conditions.map(() => true); + + for (const [candidateIndex, candidate] of branches.entries()) { + if (!keep[candidateIndex]) { + continue; + } + for (const [branchIndex, branch] of branches.entries()) { + if (candidateIndex === branchIndex || !keep[branchIndex]) { + continue; + } + if (!branchSubsumes(candidate.keys, branch.keys)) { + continue; + } + if ( + candidate.keys.size < branch.keys.size || + candidateIndex < branchIndex + ) { + keep[branchIndex] = false; + } + } + } + + return conditions.filter((_, index) => keep[index]); +} + +function branchSubsumes( + candidate: ReadonlySet, + branch: ReadonlySet, +): boolean { + for (const key of candidate) { + if (!branch.has(key)) { + return false; + } + } + return true; +} + function mergeSameRelationshipExists( conditions: readonly Condition[], ): Condition[] { @@ -590,9 +659,9 @@ function mergeSameRelationshipExists( function mergeExistsGroup( template: CorrelatedSubqueryCondition, filters: readonly Condition[], -): CorrelatedSubqueryCondition { +): Condition { const where = buildOr(filters); - return { + return collapseImpossibleCorrelatedSubquery({ ...template, related: { ...template.related, @@ -601,7 +670,7 @@ function mergeExistsGroup( where: isAlwaysTrue(where) ? undefined : where, }, }, - }; + }); } function mergeableExistsKey(condition: Condition): string | undefined { diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts new file mode 100644 index 0000000000..2570dc3450 --- /dev/null +++ b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts @@ -0,0 +1,61 @@ +import type {QueryScenario} from '../../query-scenario.ts'; +import { + colName, + createEducationAppTables, + educationAppRelationships, + educationAppSchema, + educationAppTables, + relationshipName, + tableName, +} from '../education-app.ts'; + +const assignment = educationAppTables.assignment; +const assignmentToStudent = educationAppTables.assignment_to_student; +const assignmentToStudentRelationship = relationshipName( + educationAppRelationships.assignment, + 'assignment_to_student', +); + +export default { + name: 'OR with impossible exists keeps only parent predicate', + schema: educationAppSchema, + seed: db => { + const tables = createEducationAppTables(db); + const assignment = tables.assignment; + + const assignmentStmt = db.prepare( + `INSERT INTO ${tableName(assignment)} (${colName(assignment, 'id')}, ${colName(assignment, 'teacher_id')}, ${colName(assignment, 'archived_at')}, ${colName(assignment, 'created_at')}) VALUES (?, ?, ?, ?)`, + ); + for (let i = 1; i <= 2_000; i++) { + assignmentStmt.run(i, i % 100 === 0 ? 1 : 2, null, i); + } + }, + query: builder => + builder[assignment.name] + .where(({cmp, exists, or}) => + or( + cmp(colName(assignment, 'teacher_id'), '=', 1), + exists(assignmentToStudentRelationship, q => + q.where(colName(assignmentToStudent, 'student_id'), 'IN', []), + ), + ), + ) + .orderBy(colName(assignment, 'created_at'), 'desc') + .orderBy(colName(assignment, 'id'), 'asc'), + expectations: { + optimizedAST: { + where: { + type: 'simple', + left: {type: 'column', name: 'teacher_id'}, + op: '=', + right: {type: 'literal', value: 1}, + }, + }, + sql: [ + { + table: 'assignment', + sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "teacher_id" = ? ORDER BY "created_at" desc, "id" asc', + }, + ], + }, +} satisfies QueryScenario; diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/index.ts b/packages/zqlite/src/test/query-scenarios/scenarios/index.ts index 6a320c4c02..8f7454dfb0 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/index.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/index.ts @@ -5,6 +5,7 @@ import contradictoryParentDomain from './contradictory-parent-domain.ts'; import duplicateExistsOrDedupe from './duplicate-exists-or-dedupe.ts'; import explicitFlipFalse from './explicit-flip-false.ts'; import explicitFlipTrue from './explicit-flip-true.ts'; +import impossibleExistsOrParentFilter from './impossible-exists-or-parent-filter.ts'; import mixedOrWithFalseAndMembership from './mixed-or-with-false-and-membership.ts'; import orWithFalseBranch from './or-with-false-branch.ts'; import parentOrExistsUnionRoots from './parent-or-exists-union-roots.ts'; @@ -27,6 +28,7 @@ export default [ teacherFilterWithMembership, explicitFlipFalse, explicitFlipTrue, + impossibleExistsOrParentFilter, orWithFalseBranch, mixedOrWithFalseAndMembership, twoStudentMembershipOr, From f3e4e479ea97744050f11ae204a039b8d5833a14 Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 10:32:50 -0400 Subject: [PATCH 05/13] fix(zql): execute sibling exists from selective roots --- packages/zql/src/builder/builder.ts | 305 +++++++++++- packages/zql/src/ivm/set-operators.test.ts | 351 ++++++++++++++ packages/zql/src/ivm/set-operators.ts | 452 ++++++++++++++++++ packages/zql/src/planner/planner-builder.ts | 6 +- .../src/query-planner-scenarios.test.ts | 3 + packages/zqlite/src/test/query-scenario.ts | 9 +- .../scenarios/parent-or-exists-union-roots.ts | 53 +- .../same-relationship-and-intersect.ts | 40 +- 8 files changed, 1123 insertions(+), 96 deletions(-) create mode 100644 packages/zql/src/ivm/set-operators.test.ts create mode 100644 packages/zql/src/ivm/set-operators.ts diff --git a/packages/zql/src/builder/builder.ts b/packages/zql/src/builder/builder.ts index 1df1cc2deb..0c9c1a25f2 100644 --- a/packages/zql/src/builder/builder.ts +++ b/packages/zql/src/builder/builder.ts @@ -30,6 +30,7 @@ import {Filter} from '../ivm/filter.ts'; import {FlippedJoin} from '../ivm/flipped-join.ts'; import {Join} from '../ivm/join.ts'; import type {Input, InputBase, Storage} from '../ivm/operator.ts'; +import {InputIntersection, InputUnion} from '../ivm/set-operators.ts'; import {Skip} from '../ivm/skip.ts'; import type {Source, SourceInput} from '../ivm/source.ts'; import {Take} from '../ivm/take.ts'; @@ -270,17 +271,39 @@ function buildPipelineInternal( assertNoNotExists(ast.where); } - const csqConditions = gatherCorrelatedSubqueryQueryConditions(ast.where); + const rootUnionBranches = getRootUnionBranches(ast); + if (rootUnionBranches) { + return applyRootUnionBranches( + ast, + rootUnionBranches, + delegate, + queryID, + name, + partitionKey, + ); + } + + const intersection = getSameRelationshipExistsIntersection( + ast.where, + delegate, + ); + const sourceWhere = intersection ? undefined : ast.where; + const csqConditions = intersection + ? [] + : gatherCorrelatedSubqueryQueryConditions(ast.where); const splitEditKeys: Set = partitionKey ? new Set(partitionKey) : new Set(); - const aliases = new Set(); for (const csq of csqConditions) { - aliases.add(csq.related.subquery.alias || ''); for (const key of csq.related.correlation.parentField) { splitEditKeys.add(key); } } + if (intersection) { + for (const key of intersection.related.correlation.parentField) { + splitEditKeys.add(key); + } + } if (ast.related) { for (const csq of ast.related) { for (const key of csq.correlation.parentField) { @@ -290,7 +313,7 @@ function buildPipelineInternal( } const conn = source.connect( must(ast.orderBy), - ast.where, + sourceWhere, splitEditKeys, delegate.debug, ); @@ -299,6 +322,15 @@ function buildPipelineInternal( end = delegate.decorateInput(end, `${name}:source(${ast.table})`); const {fullyAppliedFilters} = conn; + if (intersection) { + end = applySameRelationshipExistsIntersection( + intersection, + delegate, + end, + name, + ); + } + if (ast.start) { const skip = new Skip(end, ast.start); delegate.addEdge(end, skip); @@ -328,8 +360,8 @@ function buildPipelineInternal( } } - if (ast.where && (!fullyAppliedFilters || delegate.applyFiltersAnyway)) { - end = applyWhere(end, ast.where, delegate, name); + if (sourceWhere && (!fullyAppliedFilters || delegate.applyFiltersAnyway)) { + end = applyWhere(end, sourceWhere, delegate, name); } if (ast.limit !== undefined) { @@ -358,6 +390,246 @@ function buildPipelineInternal( return end; } +function applyRootUnionBranches( + ast: AST, + branches: readonly Condition[], + delegate: BuilderDelegate, + queryID: string, + name: string, + partitionKey?: CompoundKey, +): Input { + const inputs = branches.map((branch, index) => + buildPipelineInternal( + { + ...ast, + where: branch, + }, + delegate, + queryID, + `${name}:or-${index}`, + partitionKey, + ), + ); + + const union = new InputUnion(inputs); + for (const input of inputs) { + delegate.addEdge(input, union); + } + return delegate.decorateInput(union, `${name}:input-union`); +} + +function getRootUnionBranches(ast: AST): readonly Condition[] | undefined { + if ( + ast.where?.type !== 'or' || + ast.where.conditions.length < 2 || + ast.start !== undefined || + ast.limit !== undefined || + ast.related !== undefined + ) { + return undefined; + } + + const branches = ast.where.conditions; + if (!branches.some(conditionIncludesFlippedSubqueryAtAnyLevel)) { + return undefined; + } + + if (!branches.some(isNotAndDoesNotContainSubquery)) { + return undefined; + } + + if (branches.some(branch => branch.type === 'or')) { + return undefined; + } + + return branches; +} + +function applySameRelationshipExistsIntersection( + intersection: SameRelationshipExistsIntersection, + delegate: BuilderDelegate, + end: Input, + name: string, +): Input { + const {conditions, related} = intersection; + const childInputs = conditions.map((condition, index) => + buildPipelineInternal( + condition.related.subquery, + delegate, + '', + `${name}.${condition.related.subquery.alias}:intersect-${index}`, + related.correlation.childField, + ), + ); + const child = new InputIntersection( + childInputs, + related.correlation.childField, + ); + for (const childInput of childInputs) { + delegate.addEdge(childInput, child); + } + + const flippedJoin = new FlippedJoin({ + parent: end, + child, + parentKey: related.correlation.parentField, + childKey: related.correlation.childField, + relationshipName: must( + related.subquery.alias, + 'Subquery must have an alias', + ), + hidden: related.hidden ?? false, + system: related.system ?? 'client', + }); + delegate.addEdge(end, flippedJoin); + delegate.addEdge(child, flippedJoin); + return delegate.decorateInput( + flippedJoin, + `${name}:intersect-flipped-join(${related.subquery.alias})`, + ); +} + +type SameRelationshipExistsIntersection = { + readonly related: CorrelatedSubquery; + readonly conditions: readonly CorrelatedSubqueryCondition[]; +}; + +function getSameRelationshipExistsIntersection( + condition: Condition | undefined, + delegate: BuilderDelegate, +): SameRelationshipExistsIntersection | undefined { + if (condition?.type !== 'and') { + return undefined; + } + + const conditions = condition.conditions.map(getIntersectableExists); + if (conditions.some(candidate => candidate === undefined)) { + return undefined; + } + + const candidates = conditions as CorrelatedSubqueryCondition[]; + if (candidates.length < 2) { + return undefined; + } + if (!candidates.some(candidate => candidate.flip === true)) { + return undefined; + } + + const key = sameRelationshipExistsKey(candidates[0]); + if ( + !key || + candidates.some(candidate => sameRelationshipExistsKey(candidate) !== key) + ) { + return undefined; + } + + const childSource = delegate.getSource(candidates[0].related.subquery.table); + if (!childSource) { + return undefined; + } + + if ( + candidates.some( + candidate => + !isUniquePerCorrelationKey( + candidate, + childSource.tableSchema.primaryKey, + ), + ) + ) { + return undefined; + } + + return { + related: candidates[0].related, + conditions: candidates, + }; +} + +function getIntersectableExists( + condition: Condition, +): CorrelatedSubqueryCondition | undefined { + if (condition.type !== 'correlatedSubquery') { + return undefined; + } + if ( + condition.op !== 'EXISTS' || + condition.scalar === true || + condition.flip === false + ) { + return undefined; + } + + const {subquery} = condition.related; + if ( + subquery.related !== undefined || + subquery.start !== undefined || + subquery.limit !== undefined || + (subquery.where !== undefined && + !isNotAndDoesNotContainSubquery(subquery.where)) + ) { + return undefined; + } + return condition; +} + +function sameRelationshipExistsKey( + condition: CorrelatedSubqueryCondition, +): string | undefined { + const exists = getIntersectableExists(condition); + if (!exists) { + return undefined; + } + + const {related} = exists; + return JSON.stringify({ + system: related.system, + hidden: related.hidden, + correlation: related.correlation, + orderBy: related.subquery.orderBy, + subquery: { + schema: related.subquery.schema, + table: related.subquery.table, + }, + }); +} + +function isUniquePerCorrelationKey( + condition: CorrelatedSubqueryCondition, + childPrimaryKey: readonly string[], +): boolean { + const constrained = new Set(condition.related.correlation.childField); + collectEqualityConstrainedColumns( + condition.related.subquery.where, + constrained, + ); + return childPrimaryKey.every(key => constrained.has(key)); +} + +function collectEqualityConstrainedColumns( + condition: Condition | undefined, + constrained: Set, +): void { + if (!condition) { + return; + } + if (condition.type === 'simple') { + if ( + condition.op === '=' && + condition.left.type === 'column' && + condition.right.type === 'literal' + ) { + constrained.add(condition.left.name); + } + return; + } + if (condition.type === 'and') { + for (const child of condition.conditions) { + collectEqualityConstrainedColumns(child, constrained); + } + } +} + function applyWhere( input: Input, condition: Condition, @@ -420,19 +692,18 @@ function applyFilterWithFlips( const branches: Input[] = []; if (withoutFlipped.length > 0) { - branches.push( - buildFilterPipeline(end, delegate, filterInput => - applyOr( - filterInput, - { - type: 'or', - conditions: withoutFlipped, - }, - delegate, - name, - ), + const branch = buildFilterPipeline(end, delegate, filterInput => + applyOr( + filterInput, + { + type: 'or', + conditions: withoutFlipped, + }, + delegate, + name, ), ); + branches.push(branch); } for (const cond of withFlipped) { diff --git a/packages/zql/src/ivm/set-operators.test.ts b/packages/zql/src/ivm/set-operators.test.ts new file mode 100644 index 0000000000..6f2246a763 --- /dev/null +++ b/packages/zql/src/ivm/set-operators.test.ts @@ -0,0 +1,351 @@ +import {describe, expect, test} from 'vitest'; +import type {Row} from '../../../zero-protocol/src/data.ts'; +import {ChangeIndex} from './change-index.ts'; +import {ChangeType} from './change-type.ts'; +import { + makeAddChange, + makeChildChange, + makeEditChange, + makeRemoveChange, + type Change, +} from './change.ts'; +import type {Node} from './data.ts'; +import { + skipYields, + type FetchRequest, + type Input, + type Output, +} from './operator.ts'; +import type {SourceSchema} from './schema.ts'; +import {InputIntersection, InputUnion} from './set-operators.ts'; +import {consume, type Stream} from './stream.ts'; + +const schema: SourceSchema = { + tableName: 'assignment', + columns: { + id: {type: 'number'}, + value: {type: 'number'}, + }, + primaryKey: ['id'], + relationships: {}, + compareRows: (left, right) => (left.id as number) - (right.id as number), + isHidden: false, + sort: [['id', 'asc']], + system: 'client', +}; + +describe('InputUnion', () => { + test('fetch merges sorted inputs and deduplicates by primary key', () => { + const left = new MutableInput([ + node({id: 1, value: 10}), + node({id: 3, value: 30}), + ]); + const right = new MutableInput([ + node({id: 2, value: 20}), + node({id: 3, value: 30}), + ]); + + const union = new InputUnion([left, right]); + + expect(Array.from(skipYields(union.fetch({})), n => n.row.id)).toEqual([ + 1, 2, 3, + ]); + }); + + test('push emits one add, remove, or edit when multiple branches match', () => { + const left = new MutableInput([]); + const right = new MutableInput([]); + const union = new InputUnion([left, right]); + const sink = new RecordingOutput(); + union.setOutput(sink); + + left.rows = [node({id: 1, value: 10})]; + left.push(makeAddChange(node({id: 1, value: 10}))); + right.rows = [node({id: 1, value: 10})]; + right.push(makeAddChange(node({id: 1, value: 10}))); + + left.rows = [node({id: 1, value: 11})]; + left.push( + makeEditChange(node({id: 1, value: 11}), node({id: 1, value: 10})), + ); + right.rows = [node({id: 1, value: 11})]; + right.push( + makeEditChange(node({id: 1, value: 11}), node({id: 1, value: 10})), + ); + + left.rows = []; + left.push(makeRemoveChange(node({id: 1, value: 11}))); + right.rows = []; + right.push(makeRemoveChange(node({id: 1, value: 11}))); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.ADD, + ChangeType.EDIT, + ChangeType.REMOVE, + ]); + }); + + test('push emits primary key edits as remove and add', () => { + const left = new MutableInput([node({id: 1, value: 10})]); + const right = new MutableInput([]); + const union = new InputUnion([left, right]); + const sink = new RecordingOutput(); + union.setOutput(sink); + + left.rows = [node({id: 2, value: 10})]; + left.push( + makeEditChange(node({id: 2, value: 10}), node({id: 1, value: 10})), + ); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.REMOVE, + ChangeType.ADD, + ]); + expect(sink.changes.map(change => change[ChangeIndex.NODE].row.id)).toEqual( + [1, 2], + ); + }); + + test('push emits an edit when representative ownership moves earlier', () => { + const left = new MutableInput([]); + const right = new MutableInput([node({id: 1, value: 20})]); + const union = new InputUnion([left, right]); + const sink = new RecordingOutput(); + union.setOutput(sink); + + left.rows = [node({id: 1, value: 10})]; + left.push(makeAddChange(node({id: 1, value: 10}))); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.EDIT, + ]); + const [change] = sink.changes; + expect(change).toBeDefined(); + if (!change) { + throw new Error('Expected change'); + } + expect(change[ChangeIndex.TYPE]).toBe(ChangeType.EDIT); + if (change[ChangeIndex.TYPE] !== ChangeType.EDIT) { + throw new Error('Expected edit'); + } + expect(change[ChangeIndex.NODE].row).toEqual({ + id: 1, + value: 10, + }); + expect(change[ChangeIndex.OLD_NODE].row).toEqual({ + id: 1, + value: 20, + }); + }); + + test('push emits an edit when representative ownership moves later', () => { + const left = new MutableInput([node({id: 1, value: 10})]); + const right = new MutableInput([node({id: 1, value: 20})]); + const union = new InputUnion([left, right]); + const sink = new RecordingOutput(); + union.setOutput(sink); + + left.rows = []; + left.push(makeRemoveChange(node({id: 1, value: 10}))); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.EDIT, + ]); + const [change] = sink.changes; + expect(change).toBeDefined(); + if (!change) { + throw new Error('Expected change'); + } + expect(change[ChangeIndex.TYPE]).toBe(ChangeType.EDIT); + if (change[ChangeIndex.TYPE] !== ChangeType.EDIT) { + throw new Error('Expected edit'); + } + expect(change[ChangeIndex.NODE].row).toEqual({ + id: 1, + value: 20, + }); + expect(change[ChangeIndex.OLD_NODE].row).toEqual({ + id: 1, + value: 10, + }); + }); + + test('push forwards child changes from the earliest matching branch only', () => { + const left = new MutableInput([node({id: 1, value: 10})]); + const right = new MutableInput([node({id: 1, value: 10})]); + const union = new InputUnion([left, right]); + const sink = new RecordingOutput(); + union.setOutput(sink); + + left.push(childChange(node({id: 1, value: 10}))); + right.push(childChange(node({id: 1, value: 10}))); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.CHILD, + ]); + }); +}); + +describe('InputIntersection', () => { + test('fetch keeps rows whose key appears in every input', () => { + const left = new MutableInput([ + node({id: 1, value: 10}), + node({id: 2, value: 20}), + ]); + const right = new MutableInput([ + node({id: 2, value: 200}), + node({id: 3, value: 300}), + ]); + + const intersection = new InputIntersection([left, right], ['id']); + + expect(Array.from(skipYields(intersection.fetch({})), n => n.row)).toEqual([ + {id: 2, value: 20}, + ]); + }); + + test('fetch deduplicates rows by intersection key', () => { + const left = new MutableInput([ + node({id: 1, value: 10}), + node({id: 1, value: 11}), + node({id: 2, value: 20}), + ]); + const right = new MutableInput([node({id: 1, value: 100})]); + + const intersection = new InputIntersection([left, right], ['id']); + + expect(Array.from(skipYields(intersection.fetch({})), n => n.row)).toEqual([ + {id: 1, value: 10}, + ]); + }); + + test('push emits when a key enters or leaves the intersection', () => { + const left = new MutableInput([node({id: 1, value: 10})]); + const right = new MutableInput([]); + const intersection = new InputIntersection([left, right], ['id']); + const sink = new RecordingOutput(); + intersection.setOutput(sink); + + right.rows = [node({id: 1, value: 100})]; + right.push(makeAddChange(node({id: 1, value: 100}))); + + right.rows = []; + right.push(makeRemoveChange(node({id: 1, value: 100}))); + + expect(sink.changes).toHaveLength(2); + expect(sink.changes[0][ChangeIndex.TYPE]).toBe(ChangeType.ADD); + expect(sink.changes[0][ChangeIndex.NODE].row).toEqual({ + id: 1, + value: 10, + }); + expect(sink.changes[1][ChangeIndex.TYPE]).toBe(ChangeType.REMOVE); + expect(sink.changes[1][ChangeIndex.NODE].row).toEqual({ + id: 1, + value: 10, + }); + }); + + test('push emits remove and add when a non-primary branch moves keys', () => { + const left = new MutableInput([ + node({id: 1, value: 10}), + node({id: 2, value: 20}), + ]); + const right = new MutableInput([node({id: 1, value: 100})]); + const intersection = new InputIntersection([left, right], ['id']); + const sink = new RecordingOutput(); + intersection.setOutput(sink); + + right.rows = [node({id: 2, value: 100})]; + right.push( + makeEditChange(node({id: 2, value: 100}), node({id: 1, value: 100})), + ); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.REMOVE, + ChangeType.ADD, + ]); + expect(sink.changes.map(change => change[ChangeIndex.NODE].row)).toEqual([ + {id: 1, value: 10}, + {id: 2, value: 20}, + ]); + }); + + test('push forwards child changes from the representative branch only', () => { + const left = new MutableInput([node({id: 1, value: 10})]); + const right = new MutableInput([node({id: 1, value: 100})]); + const intersection = new InputIntersection([left, right], ['id']); + const sink = new RecordingOutput(); + intersection.setOutput(sink); + + right.push(childChange(node({id: 1, value: 100}))); + left.push(childChange(node({id: 1, value: 10}))); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.CHILD, + ]); + expect(sink.changes[0][ChangeIndex.NODE].row).toEqual({ + id: 1, + value: 10, + }); + }); +}); + +class MutableInput implements Input { + rows: Node[]; + #output: Output | undefined; + + constructor(rows: Node[]) { + this.rows = rows; + } + + setOutput(output: Output): void { + this.#output = output; + } + + getSchema(): SourceSchema { + return schema; + } + + *fetch(req: FetchRequest): Stream { + for (const row of this.rows) { + if (matchesConstraint(row.row, req.constraint)) { + yield row; + } + } + } + + push(change: Change): void { + consume(this.#output?.push(change, this) ?? []); + } + + destroy(): void {} +} + +class RecordingOutput implements Output { + readonly changes: Change[] = []; + + *push(change: Change): Stream<'yield'> { + this.changes.push(change); + } +} + +function node(row: Row): Node { + return {row, relationships: {}}; +} + +function childChange(parent: Node): Change { + return makeChildChange(parent, { + relationshipName: 'children', + change: makeAddChange(node({id: 99, value: 99})), + }); +} + +function matchesConstraint( + row: Row, + constraint: FetchRequest['constraint'], +): boolean { + if (!constraint) { + return true; + } + return Object.entries(constraint).every(([key, value]) => row[key] === value); +} diff --git a/packages/zql/src/ivm/set-operators.ts b/packages/zql/src/ivm/set-operators.ts new file mode 100644 index 0000000000..4e9375b744 --- /dev/null +++ b/packages/zql/src/ivm/set-operators.ts @@ -0,0 +1,452 @@ +import {areEqual} from '../../../shared/src/arrays.ts'; +import {assert} from '../../../shared/src/asserts.ts'; +import type {Writable} from '../../../shared/src/writable.ts'; +import type {CompoundKey} from '../../../zero-protocol/src/ast.ts'; +import type {Row, Value} from '../../../zero-protocol/src/data.ts'; +import {ChangeIndex} from './change-index.ts'; +import {ChangeType} from './change-type.ts'; +import { + makeAddChange, + makeEditChange, + makeRemoveChange, + type Change, +} from './change.ts'; +import type {Node} from './data.ts'; +import { + throwOutput, + type FetchRequest, + type Input, + type InputBase, + type Output, +} from './operator.ts'; +import type {SourceSchema} from './schema.ts'; +import {type Stream} from './stream.ts'; +import {mergeFetches} from './union-fan-in.ts'; + +export class InputUnion implements Input { + readonly #inputs: readonly Input[]; + readonly #inputIndexes: ReadonlyMap; + readonly #schema: SourceSchema; + #output: Output = throwOutput; + + constructor(inputs: readonly Input[]) { + assert(inputs.length > 0, 'InputUnion requires at least one input'); + this.#inputs = inputs; + this.#inputIndexes = new Map(inputs.map((input, index) => [input, index])); + this.#schema = mergeInputSchemas('input union', inputs); + for (const input of inputs) { + input.setOutput(this); + } + } + + setOutput(output: Output): void { + this.#output = output; + } + + getSchema(): SourceSchema { + return this.#schema; + } + + fetch(req: FetchRequest): Stream { + return mergeFetches( + this.#inputs.map(input => input.fetch(req)), + (left, right) => this.#schema.compareRows(left.row, right.row), + ); + } + + *push(change: Change, pusher: InputBase): Stream<'yield'> { + const pusherIndex = this.#inputIndexes.get(pusher); + assert(pusherIndex !== undefined, 'Pusher was not an input union input'); + + switch (change[ChangeIndex.TYPE]) { + case ChangeType.ADD: + yield* this.#pushAdd(change[ChangeIndex.NODE], pusher, pusherIndex); + return; + + case ChangeType.REMOVE: + yield* this.#pushRemove(change[ChangeIndex.NODE], pusher, pusherIndex); + return; + + case ChangeType.EDIT: { + const oldNode = change[ChangeIndex.OLD_NODE]; + const newNode = change[ChangeIndex.NODE]; + if ( + rowKey(oldNode.row, this.#schema.primaryKey) !== + rowKey(newNode.row, this.#schema.primaryKey) + ) { + yield* this.#pushRemove(oldNode, pusher, pusherIndex); + yield* this.#pushAdd(newNode, pusher, pusherIndex); + return; + } + + if (yield* this.#earlierInputHasMatch(pusherIndex, newNode)) { + return; + } + yield* this.#output.push(change, this); + return; + } + + case ChangeType.CHILD: + if ( + yield* this.#earlierInputHasMatch( + pusherIndex, + change[ChangeIndex.NODE], + ) + ) { + return; + } + yield* this.#output.push(change, this); + return; + } + } + + destroy(): void { + for (const input of this.#inputs) { + input.destroy(); + } + } + + *#pushAdd( + node: Node, + pusher: InputBase, + pusherIndex: number, + ): Generator<'yield'> { + const match = yield* this.#firstMatchingInputExcept(pusher, node); + if (!match) { + yield* this.#output.push(makeAddChange(node), this); + return; + } + if (pusherIndex < match.index && !rowsEqual(node.row, match.node.row)) { + yield* this.#output.push(makeEditChange(node, match.node), this); + } + } + + *#pushRemove( + node: Node, + pusher: InputBase, + pusherIndex: number, + ): Generator<'yield'> { + const match = yield* this.#firstMatchingInputExcept(pusher, node); + if (!match) { + yield* this.#output.push(makeRemoveChange(node), this); + return; + } + if (pusherIndex < match.index && !rowsEqual(node.row, match.node.row)) { + yield* this.#output.push(makeEditChange(match.node, node), this); + } + } + + *#firstMatchingInputExcept( + pusher: InputBase, + node: Node, + ): Generator< + 'yield', + {readonly index: number; readonly node: Node} | undefined + > { + const constraint = keyConstraint(node.row, this.#schema.primaryKey); + for (const [index, input] of this.#inputs.entries()) { + if (input === pusher) { + continue; + } + const matchingNode = yield* firstMatchingNode(input, constraint); + if (matchingNode) { + return {index, node: matchingNode}; + } + } + return undefined; + } + + *#earlierInputHasMatch( + pusherIndex: number, + node: Node, + ): Generator<'yield', boolean> { + const constraint = keyConstraint(node.row, this.#schema.primaryKey); + for (const input of this.#inputs.slice(0, pusherIndex)) { + if (yield* inputHasMatch(input, constraint)) { + return true; + } + } + return false; + } +} + +export class InputIntersection implements Input { + readonly #inputs: readonly Input[]; + readonly #key: CompoundKey; + readonly #schema: SourceSchema; + #output: Output = throwOutput; + + constructor(inputs: readonly Input[], key: CompoundKey) { + assert(inputs.length > 0, 'InputIntersection requires at least one input'); + this.#inputs = inputs; + this.#key = key; + this.#schema = firstInputSchema('input intersection', inputs); + for (const input of inputs) { + input.setOutput(this); + } + } + + setOutput(output: Output): void { + this.#output = output; + } + + getSchema(): SourceSchema { + return this.#schema; + } + + *fetch(req: FetchRequest): Stream { + const [first, ...rest] = this.#inputs; + const firstNodes: Node[] = []; + for (const node of first.fetch(req)) { + if (node === 'yield') { + yield node; + continue; + } + firstNodes.push(node); + } + + const matchingKeys: ReadonlySet[] = []; + for (const input of rest) { + const keys = new Set(); + for (const node of input.fetch(req)) { + if (node === 'yield') { + yield node; + continue; + } + keys.add(rowKey(node.row, this.#key)); + } + matchingKeys.push(keys); + } + + const yieldedKeys = new Set(); + for (const node of firstNodes) { + const key = rowKey(node.row, this.#key); + if (!yieldedKeys.has(key) && matchingKeys.every(keys => keys.has(key))) { + yieldedKeys.add(key); + yield node; + } + } + } + + *push(change: Change, pusher: InputBase): Stream<'yield'> { + assert(isInput(pusher), 'Expected pusher to be an input'); + assert(this.#inputs.includes(pusher), 'Pusher was not an input'); + + switch (change[ChangeIndex.TYPE]) { + case ChangeType.ADD: + yield* this.#pushAdd(change[ChangeIndex.NODE], pusher); + return; + + case ChangeType.REMOVE: + yield* this.#pushRemove(change[ChangeIndex.NODE], pusher); + return; + + case ChangeType.EDIT: { + const oldNode = change[ChangeIndex.OLD_NODE]; + const newNode = change[ChangeIndex.NODE]; + if (rowKey(oldNode.row, this.#key) !== rowKey(newNode.row, this.#key)) { + yield* this.#pushRemove(oldNode, pusher); + yield* this.#pushAdd(newNode, pusher); + return; + } + + if ( + pusher === this.#inputs[0] && + (yield* this.#allOtherInputsHaveMatch(pusher, newNode)) + ) { + yield* this.#output.push(change, this); + } + return; + } + + case ChangeType.CHILD: + if ( + pusher === this.#inputs[0] && + (yield* this.#allOtherInputsHaveMatch( + pusher, + change[ChangeIndex.NODE], + )) + ) { + yield* this.#output.push(change, this); + } + return; + } + } + + destroy(): void { + for (const input of this.#inputs) { + input.destroy(); + } + } + + *#pushAdd(node: Node, pusher: Input): Generator<'yield'> { + if (!(yield* this.#allOtherInputsHaveMatch(pusher, node))) { + return; + } + if (pusher === this.#inputs[0]) { + yield* this.#output.push(makeAddChange(node), this); + return; + } + const representative = yield* firstMatchingNode( + this.#inputs[0], + keyConstraint(node.row, this.#key), + ); + if (representative) { + yield* this.#output.push(makeAddChange(representative), this); + } + } + + *#pushRemove(node: Node, pusher: Input): Generator<'yield'> { + const constraint = keyConstraint(node.row, this.#key); + if ( + pusher !== this.#inputs[0] && + (yield* inputHasMatch(pusher, constraint)) + ) { + return; + } + if (!(yield* this.#allOtherInputsHaveMatch(pusher, node))) { + return; + } + if (pusher === this.#inputs[0]) { + yield* this.#output.push(makeRemoveChange(node), this); + return; + } + const representative = yield* firstMatchingNode( + this.#inputs[0], + constraint, + ); + if (representative) { + yield* this.#output.push(makeRemoveChange(representative), this); + } + } + + *#allOtherInputsHaveMatch( + pusher: InputBase, + node: Node, + ): Generator<'yield', boolean> { + const constraint = keyConstraint(node.row, this.#key); + for (const input of this.#inputs) { + if (input === pusher) { + continue; + } + if (!(yield* inputHasMatch(input, constraint))) { + return false; + } + } + return true; + } +} + +function mergeInputSchemas( + operatorName: string, + inputs: readonly Input[], +): SourceSchema { + const schema = { + ...firstInputSchema(operatorName, inputs), + relationships: { + ...inputs[0].getSchema().relationships, + }, + } satisfies Writable; + + const relationshipsFromBranches = new Set(); + for (const input of inputs.slice(1)) { + const inputSchema = input.getSchema(); + assertCompatibleSchema(operatorName, schema, inputSchema); + for (const [relationshipName, relationshipSchema] of Object.entries( + inputSchema.relationships, + )) { + if (relationshipName in schema.relationships) { + continue; + } + assert( + !relationshipsFromBranches.has(relationshipName), + `Relationship ${relationshipName} exists in multiple upstream inputs to ${operatorName}`, + ); + schema.relationships[relationshipName] = relationshipSchema; + relationshipsFromBranches.add(relationshipName); + } + } + + return schema; +} + +function firstInputSchema( + operatorName: string, + inputs: readonly Input[], +): SourceSchema { + const schema = inputs[0].getSchema(); + for (const input of inputs.slice(1)) { + assertCompatibleSchema(operatorName, schema, input.getSchema()); + } + return schema; +} + +function assertCompatibleSchema( + operatorName: string, + expected: SourceSchema, + actual: SourceSchema, +): void { + assert( + expected.tableName === actual.tableName, + `Table name mismatch in ${operatorName}: ${expected.tableName} !== ${actual.tableName}`, + ); + assert( + areEqual(expected.primaryKey, actual.primaryKey), + `Primary key mismatch in ${operatorName}`, + ); + assert( + expected.system === actual.system, + `System mismatch in ${operatorName}: ${expected.system} !== ${actual.system}`, + ); + assert( + JSON.stringify(expected.sort) === JSON.stringify(actual.sort), + `Sort mismatch in ${operatorName}`, + ); +} + +function keyConstraint( + row: Row, + key: readonly string[], +): Record { + return Object.fromEntries(key.map(column => [column, row[column]])); +} + +function rowKey(row: Row, key: readonly string[]): string { + return JSON.stringify(key.map(column => row[column])); +} + +function rowsEqual(left: Row, right: Row): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + +function isInput(input: InputBase): input is Input { + return 'fetch' in input; +} + +function* inputHasMatch( + input: Input, + constraint: Record, +): Generator<'yield', boolean> { + for (const node of input.fetch({constraint})) { + if (node === 'yield') { + yield node; + continue; + } + return true; + } + return false; +} + +function* firstMatchingNode( + input: Input, + constraint: Record, +): Generator<'yield', Node | undefined> { + for (const node of input.fetch({constraint})) { + if (node === 'yield') { + yield node; + continue; + } + return node; + } + return undefined; +} diff --git a/packages/zql/src/planner/planner-builder.ts b/packages/zql/src/planner/planner-builder.ts index 16153cd061..cb095565d6 100644 --- a/packages/zql/src/planner/planner-builder.ts +++ b/packages/zql/src/planner/planner-builder.ts @@ -334,10 +334,14 @@ function applyToCondition( planIdSymbol ]; const shouldFlip = planId !== undefined && flippedIds.has(planId); + const flip = + shouldFlip || condition.flip !== undefined + ? {flip: shouldFlip} + : undefined; return { ...condition, - flip: shouldFlip, + ...flip, related: { ...condition.related, subquery: { diff --git a/packages/zqlite/src/query-planner-scenarios.test.ts b/packages/zqlite/src/query-planner-scenarios.test.ts index fc8ad871d0..fb5390959f 100644 --- a/packages/zqlite/src/query-planner-scenarios.test.ts +++ b/packages/zqlite/src/query-planner-scenarios.test.ts @@ -45,4 +45,7 @@ function assertScenarioExpectations( if (expectations.sql) { expect(result.sql).toEqual(expectations.sql); } + if (expectations.rows) { + expect(result.rows).toEqual(expectations.rows); + } } diff --git a/packages/zqlite/src/test/query-scenario.ts b/packages/zqlite/src/test/query-scenario.ts index 5a80077c9b..f564d60945 100644 --- a/packages/zqlite/src/test/query-scenario.ts +++ b/packages/zqlite/src/test/query-scenario.ts @@ -4,6 +4,7 @@ import {computeZqlSpecs} from '../../../zero-cache/src/db/lite-tables.ts'; import type {LiteAndZqlSpec} from '../../../zero-cache/src/db/specs.ts'; import {CREATE_TABLE_METADATA_TABLE} from '../../../zero-cache/src/services/replicator/schema/table-metadata.ts'; import type {AST} from '../../../zero-protocol/src/ast.ts'; +import type {Row} from '../../../zero-protocol/src/data.ts'; import type {Schema} from '../../../zero-types/src/schema.ts'; import {buildPipeline} from '../../../zql/src/builder/builder.ts'; import {Debug} from '../../../zql/src/builder/debug-delegate.ts'; @@ -54,6 +55,7 @@ export type QueryScenarioExpectations = { readonly optimizedAST?: object; readonly planDebug?: readonly string[]; readonly sql?: readonly QueryScenarioSQL[]; + readonly rows?: readonly Row[]; }; export type QueryScenarioSQL = { @@ -66,6 +68,7 @@ export type QueryScenarioResult = { readonly optimizedAST: AST; readonly planDebug: string; readonly sql: readonly QueryScenarioSQL[]; + readonly rows: readonly Row[]; }; export function runQueryScenario( @@ -93,7 +96,10 @@ export function runQueryScenario( const input = buildPipeline(optimizedAST, delegate, 'query-scenario'); const sink = new Catch(input); - sink.fetch(); + const rows = sink + .fetch() + .filter(node => node !== 'yield') + .map(node => node.row); sink.destroy(); return { @@ -101,6 +107,7 @@ export function runQueryScenario( optimizedAST, planDebug: planDebugger.format(), sql: debug.queries, + rows, }; } diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts index 7712966184..bb7bf47170 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts @@ -17,45 +17,7 @@ const assignmentToStudentRelationship = relationshipName( ); export default { - name: 'parent OR exists could use union roots instead of broad parent scan', - knownFailure: { - reason: - 'A mixed OR with one parent predicate and one child exists can be served as a union of two selective roots. The planner currently keeps a broad parent root for the parent side and a flipped child root for the child side.', - current: ` -OR - assignment.teacher_id = 1 - exists assignment_to_student where student_id = student-1 - -Plan shape today: - -assignment all rows -assignment_to_student student-1 => assignment -`, - desired: ` -OR as two selective roots: - -assignment teacher_id = 1 -assignment_to_student student-1 => assignment - -Then merge rows by assignment id. -`, - currentSQL: [ - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" ORDER BY "created_at" desc, "id" asc', - }, - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', - }, - ], - engineIdea: - 'Add an OR union plan node that can execute each branch from its cheapest root, merge rows by primary key, and preserve the requested ordering after the union.', - }, + name: 'parent OR exists uses union roots instead of broad parent scan', schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); @@ -93,6 +55,19 @@ Then merge rows by assignment id. .orderBy(colName(assignment, 'created_at'), 'desc') .orderBy(colName(assignment, 'id'), 'asc'), expectations: { + optimizedAST: { + where: { + type: 'or', + conditions: [ + {}, + { + type: 'correlatedSubquery', + flip: true, + }, + ], + }, + }, + planDebug: ['FO ⋈ assignment_to_student: flipped'], sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts index 16fedad574..332cc26b5f 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts @@ -17,44 +17,7 @@ const assignmentToStudentRelationship = relationshipName( ); export default { - name: 'same relationship AND could intersect child scans before parent lookup', - knownFailure: { - reason: - 'Two required memberships over the same relationship can be intersected on assignment_id before loading parent rows. The planner currently chooses one child root, loads the parent, then probes the second child relationship.', - current: ` -AND - exists assignment_to_student where student_id = student-1 - exists assignment_to_student where student_id = student-2 - -Plan shape today: - -assignment_to_student student-1 => assignment => probe assignment_to_student student-2 -`, - desired: ` -AND as child intersection: - -assignment_to_student student-1 -assignment_to_student student-2 -both streams grouped by assignment_id -then assignment lookup -`, - currentSQL: [ - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND TRUE ORDER BY "created_at" desc, "id" asc', - }, - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "assignment_id" = ? AND "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - ], - engineIdea: - 'Teach join enumeration about sibling exists clauses that share the same relationship and correlation under AND. It could introduce an intersection node keyed by the child correlation fields before parent lookup.', - }, + name: 'same relationship AND intersects child scans before parent lookup', schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); @@ -109,5 +72,6 @@ then assignment lookup sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', }, ], + rows: [{id: 102, teacher_id: 2, archived_at: null, created_at: 102}], }, } satisfies QueryScenario; From dc9bfb507b7b3cfdd143568e1129b9eeb719803e Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 10:40:00 -0400 Subject: [PATCH 06/13] docs(zql): explain selective root operators --- packages/zql/src/builder/builder.ts | 106 ++++++++++++++++++++ packages/zql/src/ivm/set-operators.ts | 76 ++++++++++++++ packages/zql/src/planner/planner-builder.ts | 3 + packages/zqlite/src/test/query-scenario.ts | 4 + 4 files changed, 189 insertions(+) diff --git a/packages/zql/src/builder/builder.ts b/packages/zql/src/builder/builder.ts index 0c9c1a25f2..be500a074e 100644 --- a/packages/zql/src/builder/builder.ts +++ b/packages/zql/src/builder/builder.ts @@ -271,6 +271,37 @@ function buildPipelineInternal( assertNoNotExists(ast.where); } + // Two narrow physical rewrites run before the generic source/filter/join + // pipeline below. They are not boolean normalizations. They replace a broad + // parent scan with a plan that starts from selective roots once the planner + // has marked at least one EXISTS as flipped. + // + // OR shape: + // + // parent WHERE A OR EXISTS(child WHERE B) + // + // becomes: + // + // parent WHERE A child WHERE B + // \ / + // \ / + // InputUnion + // + // AND shape: + // + // parent WHERE EXISTS(child WHERE B) AND EXISTS(child WHERE C) + // + // becomes: + // + // child WHERE B child WHERE C + // \ / + // InputIntersection on child correlation key + // | + // FlippedJoin to parent + // + // The guards on each rewrite are intentionally strict. If a query needs + // limits, related rows, non unique child matches, or nested subquery + // semantics, it falls through to the older pipeline. const rootUnionBranches = getRootUnionBranches(ast); if (rootUnionBranches) { return applyRootUnionBranches( @@ -398,6 +429,19 @@ function applyRootUnionBranches( name: string, partitionKey?: CompoundKey, ): Input { + // Run every OR branch as its own root query, then merge by primary key. + // This is the physical equivalent of SQLite's multi-index OR strategy: + // + // OR + // teacher_id = 1 + // EXISTS(membership student_id = 'student-1') + // + // parent root: teacher_id = 1 + // child root: membership student_id = 'student-1' -> parent lookup + // union: sorted primary-key dedupe + // + // Each recursive branch keeps the same ordering and split-edit keys as the + // original AST, so the union can merge streams without re-sorting. const inputs = branches.map((branch, index) => buildPipelineInternal( { @@ -419,6 +463,9 @@ function applyRootUnionBranches( } function getRootUnionBranches(ast: AST): readonly Condition[] | undefined { + // This rewrite is only safe at the root of a plain query. start, limit, and + // related rows all observe the whole result stream, so they need a richer + // physical plan than "run branch pipelines, then union". if ( ast.where?.type !== 'or' || ast.where.conditions.length < 2 || @@ -430,14 +477,22 @@ function getRootUnionBranches(ast: AST): readonly Condition[] | undefined { } const branches = ast.where.conditions; + + // At least one branch must already be source driven by a flipped EXISTS. + // Otherwise a root union would just split a query that the existing source + // or filter pipeline can already handle. if (!branches.some(conditionIncludesFlippedSubqueryAtAnyLevel)) { return undefined; } + // There must also be at least one local parent branch. If every branch is a + // child branch, the existing UnionFanOut and UnionFanIn path handles it. if (!branches.some(isNotAndDoesNotContainSubquery)) { return undefined; } + // The normalizer flattens ORs before planning. If a nested OR survives here, + // keep the old path rather than inventing branch semantics locally. if (branches.some(branch => branch.type === 'or')) { return undefined; } @@ -451,6 +506,18 @@ function applySameRelationshipExistsIntersection( end: Input, name: string, ): Input { + // Build the child side before the parent lookup: + // + // assignment_to_student WHERE student_id = 'student-1' + // intersect by assignment_id + // assignment_to_student WHERE student_id = 'student-2' + // | + // assignment WHERE id = assignment_id + // + // The resulting InputIntersection emits child rows whose correlation key is + // present in every sibling EXISTS branch. FlippedJoin then performs the + // reduced parent lookup. This avoids loading a parent row after the first + // child scan only to probe the second child relationship row-by-row. const {conditions, related} = intersection; const childInputs = conditions.map((condition, index) => buildPipelineInternal( @@ -498,6 +565,15 @@ function getSameRelationshipExistsIntersection( condition: Condition | undefined, delegate: BuilderDelegate, ): SameRelationshipExistsIntersection | undefined { + // Detect a narrow, physical intersection opportunity: + // + // AND + // EXISTS(relationship R, child predicate A) + // EXISTS(relationship R, child predicate B) + // + // We only need one sibling to be planned as flipped. Once one branch is + // source-driven, intersecting all compatible siblings lets the runtime start + // with the child table and avoid parent-first probing. if (condition?.type !== 'and') { return undefined; } @@ -511,6 +587,10 @@ function getSameRelationshipExistsIntersection( if (candidates.length < 2) { return undefined; } + + // Respect explicit user intent. flip: false means "keep this semi-join". + // Undefined still means the planner may decide, so it can join a group where + // another sibling was chosen as flipped. if (!candidates.some(candidate => candidate.flip === true)) { return undefined; } @@ -528,6 +608,10 @@ function getSameRelationshipExistsIntersection( return undefined; } + // InputIntersection is a key-set operator. It emits one representative row + // per child correlation key, so each branch must be unique for that key. + // If a branch could return two child rows for the same parent, intersecting + // keys would no longer match the row-level EXISTS stream semantics. if ( candidates.some( candidate => @@ -549,6 +633,9 @@ function getSameRelationshipExistsIntersection( function getIntersectableExists( condition: Condition, ): CorrelatedSubqueryCondition | undefined { + // Only plain EXISTS branches participate. Nested subqueries, start, and + // limit can all make "does this key exist?" depend on more than the child + // predicate's key domain. if (condition.type !== 'correlatedSubquery') { return undefined; } @@ -582,6 +669,9 @@ function sameRelationshipExistsKey( } const {related} = exists; + // Include orderBy because SourceSchema.sort must match across every child + // input in the intersection. EXISTS does not care about child ordering + // semantically, but the runtime stream contract does. return JSON.stringify({ system: related.system, hidden: related.hidden, @@ -598,6 +688,19 @@ function isUniquePerCorrelationKey( condition: CorrelatedSubqueryCondition, childPrimaryKey: readonly string[], ): boolean { + // Prove: + // + // correlation child fields + literal equality filters cover child PK + // + // Example: + // + // child PK: [assignment_id, student_id] + // correlation key: [assignment_id] + // child predicate: student_id = 'student-1' + // + // The branch can now emit at most one membership row for each assignment_id, + // so intersecting by assignment_id is equivalent to intersecting row sets for + // EXISTS purposes. const constrained = new Set(condition.related.correlation.childField); collectEqualityConstrainedColumns( condition.related.subquery.where, @@ -610,6 +713,9 @@ function collectEqualityConstrainedColumns( condition: Condition | undefined, constrained: Set, ): void { + // This is intentionally conservative. Only literal equality predicates prove + // uniqueness. IN, OR, scalar subqueries, and nested EXISTS can still be + // optimized later, but they need a richer proof than this small helper. if (!condition) { return; } diff --git a/packages/zql/src/ivm/set-operators.ts b/packages/zql/src/ivm/set-operators.ts index 4e9375b744..bfae6ddf13 100644 --- a/packages/zql/src/ivm/set-operators.ts +++ b/packages/zql/src/ivm/set-operators.ts @@ -23,6 +23,32 @@ import type {SourceSchema} from './schema.ts'; import {type Stream} from './stream.ts'; import {mergeFetches} from './union-fan-in.ts'; +/** + * Physical OR over multiple independently optimized root pipelines. + * + * Conceptually: + * + * branch 0: parent WHERE teacher_id = 1 + * branch 1: child WHERE student_id = 'student-1' -> parent + * + * InputUnion + * / \ + * parent rows parent rows + * + * Fetch is a sorted merge by the output schema comparator. Duplicate primary + * keys are represented by the first branch in input order. Push has to preserve + * that same "earliest branch owns the row" rule, including handoffs: + * + * later owns row {id: 1, value: 20} + * earlier adds {id: 1, value: 10} + * + * The visible result did not gain a new primary key. Its representative row + * changed, so downstream receives EDIT(value 20 -> 10). + * + * This operator deliberately does not try to merge relationship payloads from + * duplicate rows. The builder only uses it for root queries without related + * rows, start, or limit. + */ export class InputUnion implements Input { readonly #inputs: readonly Input[]; readonly #inputIndexes: ReadonlyMap; @@ -111,6 +137,11 @@ export class InputUnion implements Input { pusher: InputBase, pusherIndex: number, ): Generator<'yield'> { + // ADD has three cases: + // + // no other branch has pk => new visible row, emit ADD + // later branch has pk => representative handoff, emit EDIT + // earlier branch has pk => still represented earlier, emit nothing const match = yield* this.#firstMatchingInputExcept(pusher, node); if (!match) { yield* this.#output.push(makeAddChange(node), this); @@ -126,6 +157,11 @@ export class InputUnion implements Input { pusher: InputBase, pusherIndex: number, ): Generator<'yield'> { + // REMOVE mirrors ADD: + // + // no other branch has pk => row disappeared, emit REMOVE + // later branch has pk => representative handoff, emit EDIT + // earlier branch has pk => still represented earlier, emit nothing const match = yield* this.#firstMatchingInputExcept(pusher, node); if (!match) { yield* this.#output.push(makeRemoveChange(node), this); @@ -143,6 +179,8 @@ export class InputUnion implements Input { 'yield', {readonly index: number; readonly node: Node} | undefined > { + // Search in input order because fetch uses input order as the tie-break for + // duplicate primary keys. Push must discover the same representative row. const constraint = keyConstraint(node.row, this.#schema.primaryKey); for (const [index, input] of this.#inputs.entries()) { if (input === pusher) { @@ -170,6 +208,26 @@ export class InputUnion implements Input { } } +/** + * Physical AND over same-relationship EXISTS branches. + * + * Conceptually: + * + * child WHERE student_id = 'student-1' -> keys {101, 102} + * child WHERE student_id = 'student-2' -> keys {102, 1500} + * + * InputIntersection on assignment_id + * | + * key {102} + * + * The output row is a representative child row from the first input for each + * intersected key. The following FlippedJoin uses that key to fetch the parent. + * + * The builder only creates this operator when every child branch is unique for + * the correlation key. That keeps "one representative per key" equivalent to + * EXISTS semantics. If a future caller wants arbitrary many rows per key, this + * operator would need row-bag semantics instead of key-set semantics. + */ export class InputIntersection implements Input { readonly #inputs: readonly Input[]; readonly #key: CompoundKey; @@ -195,6 +253,10 @@ export class InputIntersection implements Input { } *fetch(req: FetchRequest): Stream { + // Fetch rest branch key sets first, then stream first-branch + // representatives in first-branch order. yieldedKeys is the key-set part of + // the contract: even if the first branch has duplicate rows for a key, the + // intersection emits one representative key for the parent lookup. const [first, ...rest] = this.#inputs; const firstNodes: Node[] = []; for (const node of first.fetch(req)) { @@ -280,6 +342,9 @@ export class InputIntersection implements Input { } *#pushAdd(node: Node, pusher: Input): Generator<'yield'> { + // A key enters the intersection only when every other branch has that key. + // Non-first branches emit the first input's representative, because fetch + // always represents an intersected key with a first-input row. if (!(yield* this.#allOtherInputsHaveMatch(pusher, node))) { return; } @@ -297,6 +362,11 @@ export class InputIntersection implements Input { } *#pushRemove(node: Node, pusher: Input): Generator<'yield'> { + // A key leaves the intersection when this branch no longer has it and all + // other branches still do. For the first branch, removing the current + // representative is visible even if another first-branch row with the same + // key remains, because fetch would have emitted the removed representative + // before the change. const constraint = keyConstraint(node.row, this.#key); if ( pusher !== this.#inputs[0] && @@ -324,6 +394,8 @@ export class InputIntersection implements Input { pusher: InputBase, node: Node, ): Generator<'yield', boolean> { + // This tests key presence, not row equality. EXISTS only cares whether each + // branch can produce at least one child row for the parent correlation key. const constraint = keyConstraint(node.row, this.#key); for (const input of this.#inputs) { if (input === pusher) { @@ -341,6 +413,10 @@ function mergeInputSchemas( operatorName: string, inputs: readonly Input[], ): SourceSchema { + // The set operators sit between complete pipelines. They can only compose + // pipelines that expose the same row stream shape. Relationships are merged + // by name for the schema contract, but duplicate row relationship payloads + // are not merged by InputUnion. The builder avoids that case for root union. const schema = { ...firstInputSchema(operatorName, inputs), relationships: { diff --git a/packages/zql/src/planner/planner-builder.ts b/packages/zql/src/planner/planner-builder.ts index cb095565d6..0914bc022e 100644 --- a/packages/zql/src/planner/planner-builder.ts +++ b/packages/zql/src/planner/planner-builder.ts @@ -334,6 +334,9 @@ function applyToCondition( planIdSymbol ]; const shouldFlip = planId !== undefined && flippedIds.has(planId); + // Preserve explicit flip: false while avoiding a noisy flip: false on every + // ordinary planner-controlled EXISTS. The absence of flip means "planner may + // choose"; an explicit false means "do not flip this relationship". const flip = shouldFlip || condition.flip !== undefined ? {flip: shouldFlip} diff --git a/packages/zqlite/src/test/query-scenario.ts b/packages/zqlite/src/test/query-scenario.ts index f564d60945..4623303542 100644 --- a/packages/zqlite/src/test/query-scenario.ts +++ b/packages/zqlite/src/test/query-scenario.ts @@ -96,6 +96,10 @@ export function runQueryScenario( const input = buildPipeline(optimizedAST, delegate, 'query-scenario'); const sink = new Catch(input); + // SQL shape alone can be misleading because ScenarioDebug dedupes query text: + // the two student membership scans in an intersection scenario have the same + // SQL and different bind values. Row expectations prove the optimized + // physical shape still returns the intended result set. const rows = sink .fetch() .filter(node => node !== 'yield') From c1501ff487535652f7d586393a8d8ceced3b0874 Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 10:44:04 -0400 Subject: [PATCH 07/13] docs(zqlite): explain planner scenario SQL --- packages/zqlite/src/test/query-scenario.ts | 3 +++ .../query-scenarios/scenarios/child-domain-intersection.ts | 4 ++++ .../query-scenarios/scenarios/contradictory-parent-domain.ts | 4 ++++ .../query-scenarios/scenarios/duplicate-exists-or-dedupe.ts | 4 ++++ .../test/query-scenarios/scenarios/explicit-flip-false.ts | 4 ++++ .../src/test/query-scenarios/scenarios/explicit-flip-true.ts | 4 ++++ .../scenarios/impossible-exists-or-parent-filter.ts | 4 ++++ .../scenarios/mixed-or-with-false-and-membership.ts | 5 +++++ .../test/query-scenarios/scenarios/or-with-false-branch.ts | 4 ++++ .../scenarios/parent-or-exists-union-roots.ts | 4 ++++ .../scenarios/same-relationship-and-intersect.ts | 4 ++++ .../query-scenarios/scenarios/same-relationship-or-merge.ts | 4 ++++ .../scenarios/shared-parent-filter-or-factor.ts | 5 +++++ .../test/query-scenarios/scenarios/single-exists-child-or.ts | 4 ++++ .../query-scenarios/scenarios/student-membership-mixed-or.ts | 5 +++++ .../scenarios/student-membership-simple-exists.ts | 4 ++++ .../test/query-scenarios/scenarios/teacher-filter-or-in.ts | 4 ++++ .../scenarios/teacher-filter-with-membership.ts | 4 ++++ .../query-scenarios/scenarios/two-student-membership-or.ts | 4 ++++ 19 files changed, 78 insertions(+) diff --git a/packages/zqlite/src/test/query-scenario.ts b/packages/zqlite/src/test/query-scenario.ts index 4623303542..32160d3c60 100644 --- a/packages/zqlite/src/test/query-scenario.ts +++ b/packages/zqlite/src/test/query-scenario.ts @@ -54,6 +54,9 @@ export type QueryScenario = { export type QueryScenarioExpectations = { readonly optimizedAST?: object; readonly planDebug?: readonly string[]; + // Human readable optimizer story placed next to SQL expectations so each + // scenario explains why that physical shape is desirable. + readonly transformations: readonly string[]; readonly sql?: readonly QueryScenarioSQL[]; readonly rows?: readonly Row[]; }; diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts index 6df96d2721..57df9b0763 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts @@ -71,6 +71,10 @@ export default { }, }, planDebug: ['flipped'], + transformations: [ + 'First narrow the child predicate from student_id IN student 1 or student 2 plus student_id not student 2 down to just student 1.', + 'After that child domain is simplified, flip the EXISTS and scan membership before loading assignments.', + ], sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts index b04ab16fb4..d66ccdf67b 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts @@ -40,6 +40,10 @@ export default { conditions: [], }, }, + transformations: [ + 'The same parent column is required to be both teacher_id equals 1 and teacher_id not equals 1.', + 'Intersecting those domains leaves no possible assignment row, so the filter collapses to FALSE.', + ], sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts index 2bf5aa80cf..4962de68fc 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts @@ -67,6 +67,10 @@ export default { flip: true, }, }, + transformations: [ + 'The OR repeats the exact same EXISTS branch, so one copy is redundant.', + 'Deduplicate the relationship branch, then flip the single surviving membership predicate into one child scan.', + ], sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts index adaa4b4870..10ea4c299d 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts @@ -60,6 +60,10 @@ export default { ], }, }, + transformations: [ + 'The query explicitly says flip false, so the planner preserves that manual choice.', + 'Assignment remains the root stream and each candidate assignment probes membership by assignment_id and student_id.', + ], sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts index 93f1b23cdf..ec509e5b35 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts @@ -60,6 +60,10 @@ export default { ], }, }, + transformations: [ + 'The query explicitly says flip true, so membership becomes the root even though the dataset is small.', + 'Scan membership by student, then look up assignments by id and apply the archived parent filter.', + ], sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts index 2570dc3450..2a06aaead6 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts @@ -51,6 +51,10 @@ export default { right: {type: 'literal', value: 1}, }, }, + transformations: [ + 'The child branch cannot return rows because student_id IN empty set is always false.', + 'A false OR branch adds nothing, so the optimizer drops it and keeps the surviving teacher filter.', + ], sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts b/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts index a46727b02d..d0d473d050 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts @@ -78,6 +78,11 @@ export default { }, }, planDebug: ['flipped'], + transformations: [ + 'First remove the false OR branch because it cannot contribute rows.', + 'Keep the shared archived assignment filter on both paths, scan the teacher branch from assignments, and flip the membership branch from student rows.', + 'Union both assignment roots by primary key so the mixed OR does not degrade into a broad assignment scan.', + ], sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts b/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts index 3d3bb3960a..ecd721e78f 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts @@ -38,6 +38,10 @@ export default { right: {type: 'literal', value: 1}, }, }, + transformations: [ + 'An empty OR is always false, and false is an identity branch inside a larger OR.', + 'Drop the dead branch and plan the remaining teacher filter as the whole assignment predicate.', + ], sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts index bb7bf47170..810d280bf3 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts @@ -68,6 +68,10 @@ export default { }, }, planDebug: ['FO ⋈ assignment_to_student: flipped'], + transformations: [ + 'This mixed OR has two selective roots: assignments for the teacher branch and membership rows for the student branch.', + 'Run both roots separately, flip only the membership side, then union assignment rows by primary key.', + ], sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts index 332cc26b5f..92d7b011fe 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts @@ -62,6 +62,10 @@ export default { .orderBy(colName(assignment, 'created_at'), 'desc') .orderBy(colName(assignment, 'id'), 'asc'), expectations: { + transformations: [ + 'Two sibling EXISTS checks on the same relationship mean the assignment must appear in both child result sets.', + 'Scan membership for each student, intersect those streams by assignment_id, then load only assignments whose key survived both scans.', + ], sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts index cb8e607593..9e21ce4487 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts @@ -67,6 +67,10 @@ export default { flip: true, }, }, + transformations: [ + 'The sibling EXISTS branches use the same relationship and differ only by their child student filter.', + 'Merge them into one child student_id IN predicate so the planner flips one membership scan instead of running two equivalent relationship paths.', + ], sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts index 27fe620c37..c2248a5819 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts @@ -94,6 +94,11 @@ export default { ], }, }, + transformations: [ + 'Both OR branches repeat the same archived_at parent filter, so factor that predicate outside the OR.', + 'After factoring, the two membership branches are the same relationship and collapse into one student_id IN child filter.', + 'Flip that merged membership scan and apply archived_at during the assignment lookup.', + ], sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts index b05d7c565f..cb44d1d65d 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts @@ -58,6 +58,10 @@ export default { }, }, planDebug: ['flipped'], + transformations: [ + 'The OR lives inside one child EXISTS, so the student predicates can be merged before planning the join.', + 'Turn those child predicates into one student_id IN scan, flip the EXISTS, and load assignments by the correlated assignment key.', + ], sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts index 6320fed997..65751023ee 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts @@ -77,6 +77,11 @@ export default { }, }, planDebug: ['Best plan: Attempt 2', 'FO ⋈ assignment_to_student: flipped'], + transformations: [ + 'Keep the shared archived assignment filter on both paths.', + 'Run the teacher branch from assignment rows, then flip the selective membership branch so student rows become their own root.', + 'Union the assignment rows produced by both roots, using the assignment primary key to avoid duplicates.', + ], sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts index 0e679259ea..a2cc0b4f24 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts @@ -60,6 +60,10 @@ export default { }, }, planDebug: ['flipped'], + transformations: [ + 'The membership predicate is more selective than scanning assignments, so the EXISTS flips.', + 'Scan assignment_to_student by student first, then look up matching assignments and apply the archived filter during that parent lookup.', + ], sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts index 9f33082feb..7d922bd313 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts @@ -42,6 +42,10 @@ export default { right: {type: 'literal', value: [1, 2]}, }, }, + transformations: [ + 'Two equality checks on the same parent column describe one finite set.', + 'Normalize teacher_id equals 1 OR teacher_id equals 2 into one teacher_id IN filter so SQLite can use one compact assignment scan.', + ], sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts index 89a6c3c4b9..489f9e9e8e 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts @@ -61,6 +61,10 @@ export default { }, }, planDebug: ['semi'], + transformations: [ + 'The assignment side is already very selective because teacher_id and archived_at filter the parent table first.', + 'Keep the EXISTS as a semi join: scan those few assignments, then probe membership for each candidate assignment.', + ], sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts index 793badde92..7a032be3f0 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts @@ -78,6 +78,10 @@ export default { }, }, planDebug: ['flipped'], + transformations: [ + 'Both OR branches are the same relationship with the same assignment correlation.', + 'Merge the two child equality checks into one student_id IN filter, then flip to a single membership scan followed by assignment lookups.', + ], sql: [ { table: 'assignment_to_student', From 0912711864615b3f0849f5c0653c5d920e8520ac Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 10:46:54 -0400 Subject: [PATCH 08/13] refactor(zql): clarify intersectable exists guards --- packages/zql/src/builder/builder.ts | 81 +++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/packages/zql/src/builder/builder.ts b/packages/zql/src/builder/builder.ts index be500a074e..35f7de27e2 100644 --- a/packages/zql/src/builder/builder.ts +++ b/packages/zql/src/builder/builder.ts @@ -633,31 +633,47 @@ function getSameRelationshipExistsIntersection( function getIntersectableExists( condition: Condition, ): CorrelatedSubqueryCondition | undefined { - // Only plain EXISTS branches participate. Nested subqueries, start, and - // limit can all make "does this key exist?" depend on more than the child - // predicate's key domain. - if (condition.type !== 'correlatedSubquery') { - return undefined; - } - if ( - condition.op !== 'EXISTS' || - condition.scalar === true || - condition.flip === false - ) { - return undefined; - } + // The AND intersection rewrite is only valid for the simple shape below: + // + // EXISTS child + // where child filters have no nested EXISTS + // with no child related/start/limit + // + // Nested relationships, cursors, and limits can make "does this parent key + // exist?" depend on more than the child predicate's key domain. In that + // world, intersecting child key sets could skip rows that the original + // sibling EXISTS checks would have accepted. + return match(asCorrelatedSubqueryCondition(condition)) + .when(isPlainExistsBranch) + .when(hasIntersectableChildSubquery) + .value(); +} + +function asCorrelatedSubqueryCondition( + condition: Condition, +): CorrelatedSubqueryCondition | undefined { + return condition.type === 'correlatedSubquery' ? condition : undefined; +} + +function isPlainExistsBranch(condition: CorrelatedSubqueryCondition): boolean { + return ( + condition.op === 'EXISTS' && + condition.scalar !== true && + condition.flip !== false + ); +} +function hasIntersectableChildSubquery( + condition: CorrelatedSubqueryCondition, +): boolean { const {subquery} = condition.related; - if ( - subquery.related !== undefined || - subquery.start !== undefined || - subquery.limit !== undefined || - (subquery.where !== undefined && - !isNotAndDoesNotContainSubquery(subquery.where)) - ) { - return undefined; - } - return condition; + return ( + subquery.related === undefined && + subquery.start === undefined && + subquery.limit === undefined && + (subquery.where === undefined || + isNotAndDoesNotContainSubquery(subquery.where)) + ); } function sameRelationshipExistsKey( @@ -1171,3 +1187,22 @@ export function partitionBranches( } return [matched, notMatched] as const; } + +type Matcher = { + readonly when: (predicate: (value: T) => boolean) => Matcher; + readonly value: () => T | undefined; +}; + +// A tiny Effect Match inspired helper for linear eligibility checks. It keeps +// optimizer code shaped like "start with this candidate, then require these +// properties" without hiding the individual predicates behind a large helper. +function match(value: T | undefined): Matcher { + return { + when(predicate) { + return match(value !== undefined && predicate(value) ? value : undefined); + }, + value() { + return value; + }, + }; +} From d35795fb8c0723a1547ecfd1da77cb0a9bb77df4 Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 11:03:49 -0400 Subject: [PATCH 09/13] docs(zql): clarify planner scenarios --- packages/zql/src/builder/builder.ts | 66 ++++++------- packages/zql/src/ivm/set-operators.test.ts | 21 +++- packages/zql/src/ivm/set-operators.ts | 18 +++- packages/zql/src/ivm/union-fan-in.ts | 99 +++++++++++++++---- packages/zql/src/ivm/yield.fetch.test.ts | 81 ++++++++++++++- .../src/query-planner-scenarios.test.ts | 9 ++ packages/zqlite/src/test/query-scenario.ts | 32 +++--- .../scenarios/child-domain-intersection.ts | 13 ++- .../scenarios/contradictory-parent-domain.ts | 11 ++- .../scenarios/duplicate-exists-or-dedupe.ts | 12 ++- .../scenarios/explicit-flip-false.ts | 15 ++- .../scenarios/explicit-flip-true.ts | 12 ++- .../impossible-exists-or-parent-filter.ts | 11 ++- .../mixed-or-with-false-and-membership.ts | 17 +++- .../scenarios/or-with-false-branch.ts | 11 ++- .../scenarios/parent-or-exists-union-roots.ts | 15 ++- .../same-relationship-and-intersect.ts | 18 +++- .../scenarios/same-relationship-or-merge.ts | 12 ++- .../shared-parent-filter-or-factor.ts | 14 ++- .../scenarios/single-exists-child-or.ts | 13 ++- .../scenarios/student-membership-mixed-or.ts | 15 ++- .../student-membership-simple-exists.ts | 13 ++- .../scenarios/teacher-filter-or-in.ts | 11 ++- .../teacher-filter-with-membership.ts | 12 ++- .../scenarios/two-student-membership-or.ts | 12 ++- 25 files changed, 418 insertions(+), 145 deletions(-) diff --git a/packages/zql/src/builder/builder.ts b/packages/zql/src/builder/builder.ts index 35f7de27e2..1f5ba55eec 100644 --- a/packages/zql/src/builder/builder.ts +++ b/packages/zql/src/builder/builder.ts @@ -283,9 +283,12 @@ function buildPipelineInternal( // becomes: // // parent WHERE A child WHERE B - // \ / - // \ / - // InputUnion + // | | + // | lookup parent by FK + // | | + // parent rows parent rows + // \ / + // InputUnion // // AND shape: // @@ -299,9 +302,9 @@ function buildPipelineInternal( // | // FlippedJoin to parent // - // The guards on each rewrite are intentionally strict. If a query needs - // limits, related rows, non unique child matches, or nested subquery - // semantics, it falls through to the older pipeline. + // Each rewrite has its own strict guard below. If a query needs a shape the + // new physical operator cannot preserve, it falls through to the older + // generic pipeline. const rootUnionBranches = getRootUnionBranches(ast); if (rootUnionBranches) { return applyRootUnionBranches( @@ -595,10 +598,12 @@ function getSameRelationshipExistsIntersection( return undefined; } - const key = sameRelationshipExistsKey(candidates[0]); + const fingerprint = sameRelationshipExistsFingerprint(candidates[0]); if ( - !key || - candidates.some(candidate => sameRelationshipExistsKey(candidate) !== key) + !fingerprint || + candidates.some( + candidate => sameRelationshipExistsFingerprint(candidate) !== fingerprint, + ) ) { return undefined; } @@ -612,6 +617,13 @@ function getSameRelationshipExistsIntersection( // per child correlation key, so each branch must be unique for that key. // If a branch could return two child rows for the same parent, intersecting // keys would no longer match the row-level EXISTS stream semantics. + // + // child PK: [assignment_id, student_id] + // correlation key: [assignment_id] + // child filter: student_id = 'student-1' + // + // The correlation key plus the child filter covers the child PK, so this + // branch can produce at most one row for each assignment_id. if ( candidates.some( candidate => @@ -643,10 +655,17 @@ function getIntersectableExists( // exist?" depend on more than the child predicate's key domain. In that // world, intersecting child key sets could skip rows that the original // sibling EXISTS checks would have accepted. - return match(asCorrelatedSubqueryCondition(condition)) - .when(isPlainExistsBranch) - .when(hasIntersectableChildSubquery) - .value(); + const exists = asCorrelatedSubqueryCondition(condition); + if (!exists) { + return undefined; + } + if (!isPlainExistsBranch(exists)) { + return undefined; + } + if (!hasIntersectableChildSubquery(exists)) { + return undefined; + } + return exists; } function asCorrelatedSubqueryCondition( @@ -676,7 +695,7 @@ function hasIntersectableChildSubquery( ); } -function sameRelationshipExistsKey( +function sameRelationshipExistsFingerprint( condition: CorrelatedSubqueryCondition, ): string | undefined { const exists = getIntersectableExists(condition); @@ -1187,22 +1206,3 @@ export function partitionBranches( } return [matched, notMatched] as const; } - -type Matcher = { - readonly when: (predicate: (value: T) => boolean) => Matcher; - readonly value: () => T | undefined; -}; - -// A tiny Effect Match inspired helper for linear eligibility checks. It keeps -// optimizer code shaped like "start with this candidate, then require these -// properties" without hiding the individual predicates behind a large helper. -function match(value: T | undefined): Matcher { - return { - when(predicate) { - return match(value !== undefined && predicate(value) ? value : undefined); - }, - value() { - return value; - }, - }; -} diff --git a/packages/zql/src/ivm/set-operators.test.ts b/packages/zql/src/ivm/set-operators.test.ts index 6f2246a763..9dfbcf8e0d 100644 --- a/packages/zql/src/ivm/set-operators.test.ts +++ b/packages/zql/src/ivm/set-operators.test.ts @@ -52,6 +52,24 @@ describe('InputUnion', () => { ]); }); + test('fetch merges reversed inputs with the reversed comparator', () => { + const left = new MutableInput([ + node({id: 1, value: 10}), + node({id: 3, value: 30}), + ]); + const right = new MutableInput([ + node({id: 2, value: 20}), + node({id: 3, value: 30}), + node({id: 4, value: 40}), + ]); + + const union = new InputUnion([left, right]); + + expect( + Array.from(skipYields(union.fetch({reverse: true})), n => n.row.id), + ).toEqual([4, 3, 2, 1]); + }); + test('push emits one add, remove, or edit when multiple branches match', () => { const left = new MutableInput([]); const right = new MutableInput([]); @@ -307,7 +325,8 @@ class MutableInput implements Input { } *fetch(req: FetchRequest): Stream { - for (const row of this.rows) { + const rows = req.reverse ? this.rows.toReversed() : this.rows; + for (const row of rows) { if (matchesConstraint(row.row, req.constraint)) { yield row; } diff --git a/packages/zql/src/ivm/set-operators.ts b/packages/zql/src/ivm/set-operators.ts index bfae6ddf13..1f39009cd4 100644 --- a/packages/zql/src/ivm/set-operators.ts +++ b/packages/zql/src/ivm/set-operators.ts @@ -39,8 +39,15 @@ import {mergeFetches} from './union-fan-in.ts'; * keys are represented by the first branch in input order. Push has to preserve * that same "earliest branch owns the row" rule, including handoffs: * - * later owns row {id: 1, value: 20} - * earlier adds {id: 1, value: 10} + * before push: + * branch 0: empty + * branch 1: {id: 1, value: 20} + * output: {id: 1, value: 20} + * + * after branch 0 adds {id: 1, value: 10}: + * branch 0: {id: 1, value: 10} + * branch 1: {id: 1, value: 20} + * output: {id: 1, value: 10} * * The visible result did not gain a new primary key. Its representative row * changed, so downstream receives EDIT(value 20 -> 10). @@ -74,9 +81,14 @@ export class InputUnion implements Input { } fetch(req: FetchRequest): Stream { + const compareRows = req.reverse + ? (left: Node, right: Node) => + this.#schema.compareRows(right.row, left.row) + : (left: Node, right: Node) => + this.#schema.compareRows(left.row, right.row); return mergeFetches( this.#inputs.map(input => input.fetch(req)), - (left, right) => this.#schema.compareRows(left.row, right.row), + compareRows, ); } diff --git a/packages/zql/src/ivm/union-fan-in.ts b/packages/zql/src/ivm/union-fan-in.ts index e0bc801a53..4e7123214a 100644 --- a/packages/zql/src/ivm/union-fan-in.ts +++ b/packages/zql/src/ivm/union-fan-in.ts @@ -15,6 +15,7 @@ import { } from './operator.ts'; import { makeAddEmptyRelationships, + mergeEmpty, mergeRelationships, pushAccumulatedChanges, } from './push-accumulated.ts'; @@ -102,8 +103,19 @@ export class UnionFanIn implements Operator { fetch(req: FetchRequest): Stream { const iterables = this.#inputs.map(input => input.fetch(req)); - return mergeFetches(iterables, (l, r) => - this.#schema.compareRows(l.row, r.row), + const compareRows = req.reverse + ? (left: Node, right: Node) => + this.#schema.compareRows(right.row, left.row) + : (left: Node, right: Node) => + this.#schema.compareRows(left.row, right.row); + const relationshipNames = Object.keys(this.#schema.relationships); + return addEmptyRelationshipsToFetch( + mergeFetches( + iterables, + compareRows, + mergeNodeRelationships(relationshipNames), + ), + relationshipNames, ); } @@ -219,24 +231,58 @@ export class UnionFanIn implements Operator { } } +function mergeNodeRelationships(relationshipNames: string[]) { + return (left: Node, right: Node): Node => { + const relationships = { + ...right.relationships, + ...left.relationships, + }; + mergeEmpty(relationships, relationshipNames); + return { + row: left.row, + relationships, + }; + }; +} + +function* addEmptyRelationshipsToFetch( + fetch: Iterable, + relationshipNames: string[], +): Stream { + for (const node of fetch) { + if (node === 'yield') { + yield node; + continue; + } + const relationships = {...node.relationships}; + mergeEmpty(relationships, relationshipNames); + yield { + row: node.row, + relationships, + }; + } +} + export function* mergeFetches( fetches: Iterable[], comparator: (l: Node, r: Node) => number, + mergeEqual?: ((left: Node, right: Node) => Node) | undefined, ): IterableIterator { const iterators = fetches.map(i => i[Symbol.iterator]()); let threw = false; try { const current: (Node | null)[] = []; - let lastNodeYielded: Node | undefined; - for (let i = 0; i < iterators.length; i++) { + const advance = function* (i: number) { const iter = iterators[i]; let result = iter.next(); - // yield yields when initializing while (!result.done && result.value === 'yield') { yield result.value; result = iter.next(); } current[i] = result.done ? null : (result.value as Node); + }; + for (let i = 0; i < iterators.length; i++) { + yield* advance(i); } while (current.some(c => c !== null)) { const min = current.reduce( @@ -253,22 +299,35 @@ export function* mergeFetches( ); assert(min !== undefined, 'min is undefined'); - const [minNode, minIndex] = min; - const iter = iterators[minIndex]; - let result = iter.next(); - while (!result.done && result.value === 'yield') { - yield result.value; - result = iter.next(); - } - current[minIndex] = result.done ? null : (result.value as Node); - if ( - lastNodeYielded !== undefined && - comparator(lastNodeYielded, minNode) === 0 - ) { - continue; + let [nodeToYield, currentIndex] = min; + yield* advance(currentIndex); + + while (true) { + const equal = current.reduce( + ( + acc: [Node, number] | undefined, + c, + i, + ): [Node, number] | undefined => { + if (c === null || comparator(c, nodeToYield) !== 0) { + return acc; + } + return acc ?? [c, i]; + }, + undefined, + ); + if (!equal) { + break; + } + const [equalNode, equalIndex] = equal; + nodeToYield = mergeEqual + ? mergeEqual(nodeToYield, equalNode) + : nodeToYield; + currentIndex = equalIndex; + yield* advance(currentIndex); } - lastNodeYielded = minNode; - yield minNode; + + yield nodeToYield; } } catch (e) { threw = true; diff --git a/packages/zql/src/ivm/yield.fetch.test.ts b/packages/zql/src/ivm/yield.fetch.test.ts index 3f8252ff73..75645a14c4 100644 --- a/packages/zql/src/ivm/yield.fetch.test.ts +++ b/packages/zql/src/ivm/yield.fetch.test.ts @@ -51,6 +51,31 @@ class YieldSource implements Input { destroy(): void {} } +class StaticInput implements Input { + readonly #schema: SourceSchema; + readonly #nodes: readonly Node[]; + + constructor(schema: SourceSchema, nodes: readonly Node[]) { + this.#schema = schema; + this.#nodes = nodes; + } + + setOutput(_: Output): void {} + + getSchema(): SourceSchema { + return this.#schema; + } + + *fetch(req: FetchRequest): Stream { + const nodes = req.reverse ? this.#nodes.toReversed() : this.#nodes; + for (const node of nodes) { + yield node; + } + } + + destroy(): void {} +} + describe('Yield Propagation', () => { test('FilterStart/End propagates yield', () => { const source = new YieldSource(); @@ -160,13 +185,13 @@ describe('Yield Propagation', () => { "yield", "yield", "yield", + "yield", { "relationships": {}, "row": { "id": "1", }, }, - "yield", { "relationships": {}, "row": { @@ -400,3 +425,57 @@ describe('Yield Propagation', () => { expect(() => catchOp.fetch({})).toThrow(error); }); }); + +test('UnionFanIn fetch merges relationships from duplicate branch rows', () => { + const childSchema: SourceSchema = { + ...YIELD_SOURCE_SCHEMA_BASE, + tableName: 'child', + }; + const leftRelationship = () => + [{row: {id: 'left-child'}, relationships: {}}] as const; + const rightRelationship = () => + [{row: {id: 'right-child'}, relationships: {}}] as const; + const parent = new StaticInput(YIELD_SOURCE_SCHEMA_BASE, []); + const fanOut = new UnionFanOut(parent); + const left = new StaticInput( + { + ...YIELD_SOURCE_SCHEMA_BASE, + relationships: {leftRelationship: childSchema}, + }, + [ + { + row: {id: '1'}, + relationships: {leftRelationship}, + }, + ], + ); + const right = new StaticInput( + { + ...YIELD_SOURCE_SCHEMA_BASE, + relationships: {rightRelationship: childSchema}, + }, + [ + { + row: {id: '1'}, + relationships: {rightRelationship}, + }, + ], + ); + const fanIn = new UnionFanIn(fanOut, [left, right]); + const [node] = [...fanIn.fetch({})]; + + expect(node).not.toBe('yield'); + if (node === undefined || node === 'yield') { + throw new Error('Expected one fetched node'); + } + expect(Object.keys(node.relationships).sort()).toEqual([ + 'leftRelationship', + 'rightRelationship', + ]); + expect([...node.relationships.leftRelationship()]).toEqual([ + {row: {id: 'left-child'}, relationships: {}}, + ]); + expect([...node.relationships.rightRelationship()]).toEqual([ + {row: {id: 'right-child'}, relationships: {}}, + ]); +}); diff --git a/packages/zqlite/src/query-planner-scenarios.test.ts b/packages/zqlite/src/query-planner-scenarios.test.ts index fb5390959f..809673422f 100644 --- a/packages/zqlite/src/query-planner-scenarios.test.ts +++ b/packages/zqlite/src/query-planner-scenarios.test.ts @@ -7,6 +7,15 @@ import { import type {educationAppSchema} from './test/query-scenarios/education-app.ts'; import scenarios from './test/query-scenarios/scenarios/index.ts'; +// Scenario comments use a small visual legend: +// +// Before: the logical ZQL filter shape. +// After: the physical SQL scan order the optimized pipeline produces. +// +// A -> B +// +// means scan A first, then look up B. UNION and INTERSECT describe how parent +// row ids from separate roots are combined before rows reach the final result. for (const scenario of scenarios as readonly QueryScenario< typeof educationAppSchema >[]) { diff --git a/packages/zqlite/src/test/query-scenario.ts b/packages/zqlite/src/test/query-scenario.ts index 32160d3c60..ae00ebe348 100644 --- a/packages/zqlite/src/test/query-scenario.ts +++ b/packages/zqlite/src/test/query-scenario.ts @@ -21,19 +21,29 @@ import {createSQLiteCostModel} from '../sqlite-cost-model.ts'; import {newQueryDelegate} from './source-factory.ts'; type QuerySeen = { - readonly table: string; - readonly sql: string; + table: string; + sql: string; + calls: number; }; class ScenarioDebug extends Debug { readonly queries: QuerySeen[] = []; override initQuery(table: string, query: string): void { - if (!this.queries.some(q => q.table === table && q.sql === query)) { - this.queries.push({table, sql: query}); + const seen = this.queries.find(q => q.table === table && q.sql === query); + if (seen) { + seen.calls++; + } else { + this.queries.push({table, sql: query, calls: 1}); } super.initQuery(table, query); } + + compactQueries(): readonly QueryScenarioSQL[] { + return this.queries.map(({table, sql, calls}) => + calls === 1 ? {table, sql} : {table, sql, calls}, + ); + } } export type QueryScenario = { @@ -54,9 +64,6 @@ export type QueryScenario = { export type QueryScenarioExpectations = { readonly optimizedAST?: object; readonly planDebug?: readonly string[]; - // Human readable optimizer story placed next to SQL expectations so each - // scenario explains why that physical shape is desirable. - readonly transformations: readonly string[]; readonly sql?: readonly QueryScenarioSQL[]; readonly rows?: readonly Row[]; }; @@ -64,6 +71,7 @@ export type QueryScenarioExpectations = { export type QueryScenarioSQL = { readonly table: string; readonly sql: string; + readonly calls?: number | undefined; }; export type QueryScenarioResult = { @@ -99,10 +107,10 @@ export function runQueryScenario( const input = buildPipeline(optimizedAST, delegate, 'query-scenario'); const sink = new Catch(input); - // SQL shape alone can be misleading because ScenarioDebug dedupes query text: - // the two student membership scans in an intersection scenario have the same - // SQL and different bind values. Row expectations prove the optimized - // physical shape still returns the intended result set. + // SQL shape alone can be misleading because repeated query text can hide + // multiple physical scans with different bind values. ScenarioDebug keeps the + // SQL list readable by compacting repeated text into a calls count, so the + // intersection scenarios can prove both membership scans happen. const rows = sink .fetch() .filter(node => node !== 'yield') @@ -113,7 +121,7 @@ export function runQueryScenario( ast, optimizedAST, planDebug: planDebugger.format(), - sql: debug.queries, + sql: debug.compactQueries(), rows, }; } diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts index 57df9b0763..c1c358bca8 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts @@ -71,10 +71,14 @@ export default { }, }, planDebug: ['flipped'], - transformations: [ - 'First narrow the child predicate from student_id IN student 1 or student 2 plus student_id not student 2 down to just student 1.', - 'After that child domain is simplified, flip the EXISTS and scan membership before loading assignments.', - ], + // Before -> after: + // + // assignment + // EXISTS membership(student IN (1, 2) AND student != 2) + // + // membership(student = 1) -> assignment(id) + // + // Simplify the child domain first, then flip the selective EXISTS. sql: [ { table: 'assignment_to_student', @@ -83,6 +87,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 2, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts index d66ccdf67b..fbe22e8966 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts @@ -40,10 +40,13 @@ export default { conditions: [], }, }, - transformations: [ - 'The same parent column is required to be both teacher_id equals 1 and teacher_id not equals 1.', - 'Intersecting those domains leaves no possible assignment row, so the filter collapses to FALSE.', - ], + // Before -> after: + // + // teacher_id = 1 AND teacher_id != 1 + // + // FALSE + // + // The parent domain is empty, so no assignment can survive. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts index 4962de68fc..4c130368b5 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts @@ -67,10 +67,13 @@ export default { flip: true, }, }, - transformations: [ - 'The OR repeats the exact same EXISTS branch, so one copy is redundant.', - 'Deduplicate the relationship branch, then flip the single surviving membership predicate into one child scan.', - ], + // Before -> after: + // + // EXISTS membership(student = 1) OR EXISTS membership(student = 1) + // + // membership(student = 1) -> assignment(id) + // + // Duplicate OR branches collapse before the surviving EXISTS flips. sql: [ { table: 'assignment_to_student', @@ -79,6 +82,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts index 10ea4c299d..70b8b478c0 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts @@ -60,10 +60,16 @@ export default { ], }, }, - transformations: [ - 'The query explicitly says flip false, so the planner preserves that manual choice.', - 'Assignment remains the root stream and each candidate assignment probes membership by assignment_id and student_id.', - ], + // Before -> after: + // + // assignment(archived) + // EXISTS membership(student) with flip false + // + // assignment(archived) -> membership(assignment_id, student) + // + // Explicit flip false is user intent, so assignment remains the root. + // calls: 26 includes the repeated per-assignment membership probes plus + // the final probe used while exhausting the parent stream. sql: [ { table: 'assignment', @@ -72,6 +78,7 @@ export default { { table: 'assignment_to_student', sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "assignment_id" = ? AND "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', + calls: 26, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts index ec509e5b35..be49acd15f 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts @@ -60,10 +60,14 @@ export default { ], }, }, - transformations: [ - 'The query explicitly says flip true, so membership becomes the root even though the dataset is small.', - 'Scan membership by student, then look up assignments by id and apply the archived parent filter.', - ], + // Before -> after: + // + // assignment(archived) + // EXISTS membership(student) with flip true + // + // membership(student) -> assignment(id, archived) + // + // Explicit flip true makes membership the root. sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts index 2a06aaead6..d0da779f32 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts @@ -51,10 +51,13 @@ export default { right: {type: 'literal', value: 1}, }, }, - transformations: [ - 'The child branch cannot return rows because student_id IN empty set is always false.', - 'A false OR branch adds nothing, so the optimizer drops it and keeps the surviving teacher filter.', - ], + // Before -> after: + // + // teacher_id = 1 OR EXISTS membership(student IN []) + // + // teacher_id = 1 + // + // The impossible EXISTS branch is false, so OR keeps only the real branch. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts b/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts index d0d473d050..b04ee5ecf4 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts @@ -78,11 +78,17 @@ export default { }, }, planDebug: ['flipped'], - transformations: [ - 'First remove the false OR branch because it cannot contribute rows.', - 'Keep the shared archived assignment filter on both paths, scan the teacher branch from assignments, and flip the membership branch from student rows.', - 'Union both assignment roots by primary key so the mixed OR does not degrade into a broad assignment scan.', - ], + // Before -> after: + // + // assignment + // archived AND (teacher OR FALSE OR EXISTS membership(student)) + // + // assignment(archived), with teacher checked above the SQL scan + // for the parent branch + // UNION on assignment.id + // membership(student) -> assignment(id, archived) + // + // Drop FALSE, then plan the remaining mixed OR as two roots. sql: [ { table: 'assignment', @@ -95,6 +101,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND "archived_at" IS ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts b/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts index ecd721e78f..b42c412149 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts @@ -38,10 +38,13 @@ export default { right: {type: 'literal', value: 1}, }, }, - transformations: [ - 'An empty OR is always false, and false is an identity branch inside a larger OR.', - 'Drop the dead branch and plan the remaining teacher filter as the whole assignment predicate.', - ], + // Before -> after: + // + // teacher_id = 1 OR FALSE + // + // teacher_id = 1 + // + // A false branch adds no rows to an OR. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts index 810d280bf3..7d17e5313f 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts @@ -68,10 +68,16 @@ export default { }, }, planDebug: ['FO ⋈ assignment_to_student: flipped'], - transformations: [ - 'This mixed OR has two selective roots: assignments for the teacher branch and membership rows for the student branch.', - 'Run both roots separately, flip only the membership side, then union assignment rows by primary key.', - ], + // Before -> after: + // + // assignment + // teacher OR EXISTS membership(student) + // + // assignment(teacher) + // UNION on assignment.id + // membership(student) -> assignment(id) + // + // Mixed OR branches each keep the most selective root. sql: [ { table: 'assignment', @@ -84,6 +90,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts index 92d7b011fe..d0fbd9a750 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts @@ -62,14 +62,24 @@ export default { .orderBy(colName(assignment, 'created_at'), 'desc') .orderBy(colName(assignment, 'id'), 'asc'), expectations: { - transformations: [ - 'Two sibling EXISTS checks on the same relationship mean the assignment must appear in both child result sets.', - 'Scan membership for each student, intersect those streams by assignment_id, then load only assignments whose key survived both scans.', - ], + // Before -> after: + // + // EXISTS membership(student = 1) + // AND EXISTS membership(student = 2) + // + // membership(student = 1) + // INTERSECT on assignment_id + // membership(student = 2) + // -> assignment(id) + // + // AND means the assignment key must survive both child streams. + // The membership SQL has calls: 2 because the same SELECT text runs once + // for each student bind value before the assignment_id intersection. sql: [ { table: 'assignment_to_student', sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', + calls: 2, }, { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts index 9e21ce4487..b1fe004fb5 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts @@ -67,10 +67,13 @@ export default { flip: true, }, }, - transformations: [ - 'The sibling EXISTS branches use the same relationship and differ only by their child student filter.', - 'Merge them into one child student_id IN predicate so the planner flips one membership scan instead of running two equivalent relationship paths.', - ], + // Before -> after: + // + // EXISTS membership(student = 1) OR EXISTS membership(student = 2) + // + // membership(student IN (1, 2)) -> assignment(id) + // + // Sibling EXISTS branches over the same edge become one flipped scan. sql: [ { table: 'assignment_to_student', @@ -79,6 +82,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts index c2248a5819..23a8f72876 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts @@ -94,11 +94,14 @@ export default { ], }, }, - transformations: [ - 'Both OR branches repeat the same archived_at parent filter, so factor that predicate outside the OR.', - 'After factoring, the two membership branches are the same relationship and collapse into one student_id IN child filter.', - 'Flip that merged membership scan and apply archived_at during the assignment lookup.', - ], + // Before -> after: + // + // (archived AND EXISTS membership(student = 1)) + // OR (archived AND EXISTS membership(student = 2)) + // + // membership(student IN (1, 2)) -> assignment(id, archived) + // + // Factor the shared parent filter, merge the child domains, then flip. sql: [ { table: 'assignment_to_student', @@ -107,6 +110,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND "archived_at" IS ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts index cb44d1d65d..8ba88d0c69 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts @@ -58,10 +58,14 @@ export default { }, }, planDebug: ['flipped'], - transformations: [ - 'The OR lives inside one child EXISTS, so the student predicates can be merged before planning the join.', - 'Turn those child predicates into one student_id IN scan, flip the EXISTS, and load assignments by the correlated assignment key.', - ], + // Before -> after: + // + // assignment + // EXISTS membership(student = 1 OR student = 2) + // + // membership(student IN (1, 2)) -> assignment(id) + // + // Merge the child OR before choosing the flipped root. sql: [ { table: 'assignment_to_student', @@ -70,6 +74,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts index 65751023ee..9711a639e2 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts @@ -77,11 +77,15 @@ export default { }, }, planDebug: ['Best plan: Attempt 2', 'FO ⋈ assignment_to_student: flipped'], - transformations: [ - 'Keep the shared archived assignment filter on both paths.', - 'Run the teacher branch from assignment rows, then flip the selective membership branch so student rows become their own root.', - 'Union the assignment rows produced by both roots, using the assignment primary key to avoid duplicates.', - ], + // Before -> after: + // + // assignment + // archived AND (teacher OR EXISTS membership(student)) + // + // assignment(archived), with teacher checked above the SQL scan + // for the parent branch + // UNION on assignment.id + // membership(student) -> assignment(id, archived) sql: [ { table: 'assignment', @@ -94,6 +98,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND "archived_at" IS ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts index a2cc0b4f24..efebb733f2 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts @@ -60,10 +60,14 @@ export default { }, }, planDebug: ['flipped'], - transformations: [ - 'The membership predicate is more selective than scanning assignments, so the EXISTS flips.', - 'Scan assignment_to_student by student first, then look up matching assignments and apply the archived filter during that parent lookup.', - ], + // Before -> after: + // + // assignment(archived) + // EXISTS membership(student) + // + // membership(student) -> assignment(id, archived) + // + // The child predicate is selective, so membership becomes the root. sql: [ { table: 'assignment_to_student', @@ -72,6 +76,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND "archived_at" IS ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts index 7d922bd313..7a95d51205 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts @@ -42,10 +42,13 @@ export default { right: {type: 'literal', value: [1, 2]}, }, }, - transformations: [ - 'Two equality checks on the same parent column describe one finite set.', - 'Normalize teacher_id equals 1 OR teacher_id equals 2 into one teacher_id IN filter so SQLite can use one compact assignment scan.', - ], + // Before -> after: + // + // teacher_id = 1 OR teacher_id = 2 + // + // teacher_id IN (1, 2) + // + // Same-column equality ORs collapse into one finite domain. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts index 489f9e9e8e..200f1e5161 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts @@ -61,10 +61,14 @@ export default { }, }, planDebug: ['semi'], - transformations: [ - 'The assignment side is already very selective because teacher_id and archived_at filter the parent table first.', - 'Keep the EXISTS as a semi join: scan those few assignments, then probe membership for each candidate assignment.', - ], + // Before -> after: + // + // assignment(archived, teacher) + // EXISTS membership(student) + // + // assignment(archived, teacher) -> membership(assignment_id, student) + // + // The parent predicates are selective, so the semi join stays unflipped. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts index 7a032be3f0..1221c877b7 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts @@ -78,10 +78,13 @@ export default { }, }, planDebug: ['flipped'], - transformations: [ - 'Both OR branches are the same relationship with the same assignment correlation.', - 'Merge the two child equality checks into one student_id IN filter, then flip to a single membership scan followed by assignment lookups.', - ], + // Before -> after: + // + // EXISTS membership(student = 1) OR EXISTS membership(student = 2) + // + // membership(student IN (1, 2)) -> assignment(id) + // + // Same relationship, same correlation, one merged child domain. sql: [ { table: 'assignment_to_student', @@ -90,6 +93,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, From 758822b95de9a9c3e54114620b55539423b87ead Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 11:06:57 -0400 Subject: [PATCH 10/13] docs(zql): make optimizer comments generic --- packages/zql/src/builder/builder.ts | 144 ++++++++++++++------------ packages/zql/src/ivm/set-operators.ts | 109 ++++++++++--------- 2 files changed, 137 insertions(+), 116 deletions(-) diff --git a/packages/zql/src/builder/builder.ts b/packages/zql/src/builder/builder.ts index 1f5ba55eec..04abe28ddc 100644 --- a/packages/zql/src/builder/builder.ts +++ b/packages/zql/src/builder/builder.ts @@ -272,35 +272,39 @@ function buildPipelineInternal( } // Two narrow physical rewrites run before the generic source/filter/join - // pipeline below. They are not boolean normalizations. They replace a broad - // parent scan with a plan that starts from selective roots once the planner - // has marked at least one EXISTS as flipped. + // pipeline below. These do not change the meaning of the WHERE clause. They + // only choose a better place to start reading rows. // - // OR shape: + // OR example: // - // parent WHERE A OR EXISTS(child WHERE B) + // User query: + // issues where status = 'open' + // OR issue has label 'bug' // - // becomes: + // Optimized scan: + // scan issue(status = 'open') + // UNION by issue.id + // scan issue_label(label = 'bug') -> look up issue // - // parent WHERE A child WHERE B - // | | - // | lookup parent by FK - // | | - // parent rows parent rows - // \ / - // InputUnion + // The broad plan would scan all issues and ask "does either branch match?" + // for every row. The optimized plan starts from both selective doorways into + // issue rows, then dedupes by issue.id. // - // AND shape: + // AND example: // - // parent WHERE EXISTS(child WHERE B) AND EXISTS(child WHERE C) + // User query: + // issues where issue has label 'bug' + // AND issue has label 'urgent' // - // becomes: + // Optimized scan: + // scan issue_label(label = 'bug') -> issue ids + // INTERSECT issue ids + // scan issue_label(label = 'urgent') -> issue ids + // -> look up issue // - // child WHERE B child WHERE C - // \ / - // InputIntersection on child correlation key - // | - // FlippedJoin to parent + // The broad plan would load issues after the first label match, then probe + // the second label row-by-row. The optimized plan first finds the issue ids + // that appear in both child scans, then loads only those issues. // // Each rewrite has its own strict guard below. If a query needs a shape the // new physical operator cannot preserve, it falls through to the older @@ -432,16 +436,18 @@ function applyRootUnionBranches( name: string, partitionKey?: CompoundKey, ): Input { - // Run every OR branch as its own root query, then merge by primary key. - // This is the physical equivalent of SQLite's multi-index OR strategy: + // Run every OR branch as its own root query, then merge by primary key. A + // root is just the table/index we choose to scan first. This is the physical + // equivalent of SQLite's multi-index OR strategy: // - // OR - // teacher_id = 1 - // EXISTS(membership student_id = 'student-1') + // User query: + // issue.status = 'open' + // OR EXISTS(issue_label WHERE label = 'bug') // - // parent root: teacher_id = 1 - // child root: membership student_id = 'student-1' -> parent lookup - // union: sorted primary-key dedupe + // Optimized scan: + // issue WHERE status = 'open' + // UNION by issue.id + // issue_label WHERE label = 'bug' -> issue lookup // // Each recursive branch keeps the same ordering and split-edit keys as the // original AST, so the union can merge streams without re-sorting. @@ -481,9 +487,9 @@ function getRootUnionBranches(ast: AST): readonly Condition[] | undefined { const branches = ast.where.conditions; - // At least one branch must already be source driven by a flipped EXISTS. - // Otherwise a root union would just split a query that the existing source - // or filter pipeline can already handle. + // At least one branch must already be planned to start from a related table. + // Otherwise root union would just split a simple parent-table scan into + // smaller scans without buying us anything. if (!branches.some(conditionIncludesFlippedSubqueryAtAnyLevel)) { return undefined; } @@ -509,18 +515,22 @@ function applySameRelationshipExistsIntersection( end: Input, name: string, ): Input { - // Build the child side before the parent lookup: + // Build the child side before the parent lookup. In plain English: + // find the issue ids that satisfy every sibling EXISTS first, then load the + // issue rows for only those ids. // - // assignment_to_student WHERE student_id = 'student-1' - // intersect by assignment_id - // assignment_to_student WHERE student_id = 'student-2' - // | - // assignment WHERE id = assignment_id + // User query: + // EXISTS(issue_label WHERE label = 'bug') + // AND EXISTS(issue_label WHERE label = 'urgent') // - // The resulting InputIntersection emits child rows whose correlation key is - // present in every sibling EXISTS branch. FlippedJoin then performs the - // reduced parent lookup. This avoids loading a parent row after the first - // child scan only to probe the second child relationship row-by-row. + // Optimized scan: + // issue_label WHERE label = 'bug' + // INTERSECT by issue_id + // issue_label WHERE label = 'urgent' + // -> issue WHERE id = issue_id + // + // This avoids loading an issue after the first label match only to probe the + // second label relationship row-by-row. const {conditions, related} = intersection; const childInputs = conditions.map((condition, index) => buildPipelineInternal( @@ -613,17 +623,18 @@ function getSameRelationshipExistsIntersection( return undefined; } - // InputIntersection is a key-set operator. It emits one representative row - // per child correlation key, so each branch must be unique for that key. - // If a branch could return two child rows for the same parent, intersecting - // keys would no longer match the row-level EXISTS stream semantics. + // InputIntersection works with ids, not duplicate child rows: "which parent + // ids appeared in every related-table scan?" Because the operator keeps one + // child row for each parent id, each branch must prove it can produce at + // most one child row for the id it contributes. // - // child PK: [assignment_id, student_id] - // correlation key: [assignment_id] - // child filter: student_id = 'student-1' + // child table: issue_label + // child PK: [issue_id, label] + // parent id field: issue_id + // child filter: label = 'bug' // - // The correlation key plus the child filter covers the child PK, so this - // branch can produce at most one row for each assignment_id. + // issue_id plus label covers the child PK, so this branch can produce at + // most one issue_label row for each issue_id. if ( candidates.some( candidate => @@ -645,16 +656,16 @@ function getSameRelationshipExistsIntersection( function getIntersectableExists( condition: Condition, ): CorrelatedSubqueryCondition | undefined { - // The AND intersection rewrite is only valid for the simple shape below: + // The AND intersection rewrite is only valid for this simple shape: // - // EXISTS child - // where child filters have no nested EXISTS - // with no child related/start/limit + // EXISTS(related table) + // where related-table filters have no nested EXISTS + // with no related rows, cursor start, or limit inside the EXISTS // // Nested relationships, cursors, and limits can make "does this parent key - // exist?" depend on more than the child predicate's key domain. In that - // world, intersecting child key sets could skip rows that the original - // sibling EXISTS checks would have accepted. + // exist?" depend on more than the related table's filter. In that world, + // intersecting related-table ids could skip rows that the original sibling + // EXISTS checks would have accepted. const exists = asCorrelatedSubqueryCondition(condition); if (!exists) { return undefined; @@ -723,19 +734,20 @@ function isUniquePerCorrelationKey( condition: CorrelatedSubqueryCondition, childPrimaryKey: readonly string[], ): boolean { - // Prove: + // Prove that a child scan contributes at most one row per parent id: // - // correlation child fields + literal equality filters cover child PK + // parent id field + literal equality filters cover child PK // // Example: // - // child PK: [assignment_id, student_id] - // correlation key: [assignment_id] - // child predicate: student_id = 'student-1' + // child table: issue_label + // child PK: [issue_id, label] + // parent id field: issue_id + // child filter: label = 'bug' // - // The branch can now emit at most one membership row for each assignment_id, - // so intersecting by assignment_id is equivalent to intersecting row sets for - // EXISTS purposes. + // The branch can now emit at most one issue_label row for each issue_id, so + // intersecting by issue_id is equivalent to asking whether both EXISTS + // branches are true. const constrained = new Set(condition.related.correlation.childField); collectEqualityConstrainedColumns( condition.related.subquery.where, diff --git a/packages/zql/src/ivm/set-operators.ts b/packages/zql/src/ivm/set-operators.ts index 1f39009cd4..6639b309cc 100644 --- a/packages/zql/src/ivm/set-operators.ts +++ b/packages/zql/src/ivm/set-operators.ts @@ -24,20 +24,25 @@ import {type Stream} from './stream.ts'; import {mergeFetches} from './union-fan-in.ts'; /** - * Physical OR over multiple independently optimized root pipelines. + * Merges the results of an OR query that has more than one good starting point. * - * Conceptually: + * Example user query: * - * branch 0: parent WHERE teacher_id = 1 - * branch 1: child WHERE student_id = 'student-1' -> parent + * issues where status = 'open' + * OR issue has label 'bug' * - * InputUnion - * / \ - * parent rows parent rows + * Optimized scan: * - * Fetch is a sorted merge by the output schema comparator. Duplicate primary - * keys are represented by the first branch in input order. Push has to preserve - * that same "earliest branch owns the row" rule, including handoffs: + * branch 0: scan issue(status = 'open') + * branch 1: scan issue_label(label = 'bug') -> look up issue + * + * InputUnion + * / \ + * issue rows issue rows + * + * InputUnion streams those issue rows in final sort order and removes duplicate + * issue ids. If the same issue appears in both branches, the earlier branch is + * the visible copy. Push has to preserve that same rule, including handoffs: * * before push: * branch 0: empty @@ -49,8 +54,8 @@ import {mergeFetches} from './union-fan-in.ts'; * branch 1: {id: 1, value: 20} * output: {id: 1, value: 10} * - * The visible result did not gain a new primary key. Its representative row - * changed, so downstream receives EDIT(value 20 -> 10). + * The visible result did not gain a new primary key. The visible copy changed, + * so downstream receives EDIT(value 20 -> 10). * * This operator deliberately does not try to merge relationship payloads from * duplicate rows. The builder only uses it for root queries without related @@ -152,7 +157,7 @@ export class InputUnion implements Input { // ADD has three cases: // // no other branch has pk => new visible row, emit ADD - // later branch has pk => representative handoff, emit EDIT + // later branch has pk => visible-copy handoff, emit EDIT // earlier branch has pk => still represented earlier, emit nothing const match = yield* this.#firstMatchingInputExcept(pusher, node); if (!match) { @@ -172,7 +177,7 @@ export class InputUnion implements Input { // REMOVE mirrors ADD: // // no other branch has pk => row disappeared, emit REMOVE - // later branch has pk => representative handoff, emit EDIT + // later branch has pk => visible-copy handoff, emit EDIT // earlier branch has pk => still represented earlier, emit nothing const match = yield* this.#firstMatchingInputExcept(pusher, node); if (!match) { @@ -192,7 +197,7 @@ export class InputUnion implements Input { {readonly index: number; readonly node: Node} | undefined > { // Search in input order because fetch uses input order as the tie-break for - // duplicate primary keys. Push must discover the same representative row. + // duplicate primary keys. Push must discover the same visible copy. const constraint = keyConstraint(node.row, this.#schema.primaryKey); for (const [index, input] of this.#inputs.entries()) { if (input === pusher) { @@ -221,24 +226,31 @@ export class InputUnion implements Input { } /** - * Physical AND over same-relationship EXISTS branches. + * Keeps only parent ids that appear in every required related-table scan. * - * Conceptually: + * Example user query: * - * child WHERE student_id = 'student-1' -> keys {101, 102} - * child WHERE student_id = 'student-2' -> keys {102, 1500} + * issues where issue has label 'bug' + * AND issue has label 'urgent' * - * InputIntersection on assignment_id - * | - * key {102} + * Optimized scan: + * + * issue_label(label = 'bug') -> issue ids {10, 20} + * issue_label(label = 'urgent') -> issue ids {20, 30} * - * The output row is a representative child row from the first input for each - * intersected key. The following FlippedJoin uses that key to fetch the parent. + * InputIntersection on issue_id + * | + * issue id {20} + * | + * look up issue 20 * - * The builder only creates this operator when every child branch is unique for - * the correlation key. That keeps "one representative per key" equivalent to - * EXISTS semantics. If a future caller wants arbitrary many rows per key, this - * operator would need row-bag semantics instead of key-set semantics. + * The important idea is simple: do the cheap child-table lookups first, keep + * only ids found in every branch, then fetch the parent rows. The builder only + * creates this operator when each child branch can produce at most one row for + * a given parent id. That keeps "one row representing issue id 20" equivalent + * to EXISTS semantics. If a future caller wants arbitrary many rows per parent + * id, this operator would need to track duplicate child rows instead of just + * ids. */ export class InputIntersection implements Input { readonly #inputs: readonly Input[]; @@ -265,10 +277,10 @@ export class InputIntersection implements Input { } *fetch(req: FetchRequest): Stream { - // Fetch rest branch key sets first, then stream first-branch - // representatives in first-branch order. yieldedKeys is the key-set part of - // the contract: even if the first branch has duplicate rows for a key, the - // intersection emits one representative key for the parent lookup. + // Fetch the other branch id sets first, then stream rows from the first + // branch in its own order. yieldedKeys is the set part of the contract: + // even if the first branch has duplicate rows for an id, the intersection + // emits that id only once for the parent lookup. const [first, ...rest] = this.#inputs; const firstNodes: Node[] = []; for (const node of first.fetch(req)) { @@ -354,9 +366,9 @@ export class InputIntersection implements Input { } *#pushAdd(node: Node, pusher: Input): Generator<'yield'> { - // A key enters the intersection only when every other branch has that key. - // Non-first branches emit the first input's representative, because fetch - // always represents an intersected key with a first-input row. + // An id enters the intersection only when every other branch has that id. + // Non-first branches emit the first input's row, because fetch always uses + // the first branch as the visible copy for an intersected id. if (!(yield* this.#allOtherInputsHaveMatch(pusher, node))) { return; } @@ -364,21 +376,21 @@ export class InputIntersection implements Input { yield* this.#output.push(makeAddChange(node), this); return; } - const representative = yield* firstMatchingNode( + const visibleCopy = yield* firstMatchingNode( this.#inputs[0], keyConstraint(node.row, this.#key), ); - if (representative) { - yield* this.#output.push(makeAddChange(representative), this); + if (visibleCopy) { + yield* this.#output.push(makeAddChange(visibleCopy), this); } } *#pushRemove(node: Node, pusher: Input): Generator<'yield'> { - // A key leaves the intersection when this branch no longer has it and all + // An id leaves the intersection when this branch no longer has it and all // other branches still do. For the first branch, removing the current - // representative is visible even if another first-branch row with the same - // key remains, because fetch would have emitted the removed representative - // before the change. + // visible copy is visible even if another first-branch row with the same + // id remains, because fetch would have emitted the removed row before the + // change. const constraint = keyConstraint(node.row, this.#key); if ( pusher !== this.#inputs[0] && @@ -393,12 +405,9 @@ export class InputIntersection implements Input { yield* this.#output.push(makeRemoveChange(node), this); return; } - const representative = yield* firstMatchingNode( - this.#inputs[0], - constraint, - ); - if (representative) { - yield* this.#output.push(makeRemoveChange(representative), this); + const visibleCopy = yield* firstMatchingNode(this.#inputs[0], constraint); + if (visibleCopy) { + yield* this.#output.push(makeRemoveChange(visibleCopy), this); } } @@ -406,8 +415,8 @@ export class InputIntersection implements Input { pusher: InputBase, node: Node, ): Generator<'yield', boolean> { - // This tests key presence, not row equality. EXISTS only cares whether each - // branch can produce at least one child row for the parent correlation key. + // This tests id presence, not row equality. EXISTS only cares whether each + // branch can produce at least one related row for the parent id. const constraint = keyConstraint(node.row, this.#key); for (const input of this.#inputs) { if (input === pusher) { From 8d9b1d4f8057103370b24731d8165c60ae08b679 Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 11:11:34 -0400 Subject: [PATCH 11/13] docs(zql): improve optimizer ascii diagrams --- packages/zql/src/builder/builder.ts | 107 ++++++++++-------- packages/zql/src/ivm/set-operators.ts | 41 ++++--- .../src/query-planner-scenarios.test.ts | 14 ++- .../scenarios/child-domain-intersection.ts | 15 ++- .../scenarios/contradictory-parent-domain.ts | 13 ++- .../scenarios/duplicate-exists-or-dedupe.ts | 14 ++- .../scenarios/explicit-flip-false.ts | 8 +- .../scenarios/explicit-flip-true.ts | 8 +- .../impossible-exists-or-parent-filter.ts | 13 ++- .../mixed-or-with-false-and-membership.ts | 17 +-- .../scenarios/or-with-false-branch.ts | 13 ++- .../scenarios/parent-or-exists-union-roots.ts | 12 +- .../same-relationship-and-intersect.ts | 18 +-- .../scenarios/same-relationship-or-merge.ts | 14 ++- .../shared-parent-filter-or-factor.ts | 15 ++- .../scenarios/single-exists-child-or.ts | 12 +- .../scenarios/student-membership-mixed-or.ts | 13 ++- .../student-membership-simple-exists.ts | 8 +- .../scenarios/teacher-filter-or-in.ts | 10 +- .../teacher-filter-with-membership.ts | 8 +- .../scenarios/two-student-membership-or.ts | 14 ++- 21 files changed, 233 insertions(+), 154 deletions(-) diff --git a/packages/zql/src/builder/builder.ts b/packages/zql/src/builder/builder.ts index 04abe28ddc..c96a089e94 100644 --- a/packages/zql/src/builder/builder.ts +++ b/packages/zql/src/builder/builder.ts @@ -277,34 +277,39 @@ function buildPipelineInternal( // // OR example: // - // User query: - // issues where status = 'open' - // OR issue has label 'bug' + // Query: // - // Optimized scan: - // scan issue(status = 'open') - // UNION by issue.id - // scan issue_label(label = 'bug') -> look up issue + // issue + // |-- status = 'open' + // `-- OR has issue_label(label = 'bug') + // + // Scan plan: + // + // issue(status = 'open') -----------------------. + // +-- union issue.id + // issue_label(label = 'bug') -> issue(id) -------' // // The broad plan would scan all issues and ask "does either branch match?" - // for every row. The optimized plan starts from both selective doorways into - // issue rows, then dedupes by issue.id. + // for every row. This plan starts from both selective doorways into issue + // rows, then dedupes by issue.id. // // AND example: // - // User query: - // issues where issue has label 'bug' - // AND issue has label 'urgent' + // Query: + // + // issue + // |-- has issue_label(label = 'bug') + // `-- AND has issue_label(label = 'urgent') + // + // Scan plan: // - // Optimized scan: - // scan issue_label(label = 'bug') -> issue ids - // INTERSECT issue ids - // scan issue_label(label = 'urgent') -> issue ids - // -> look up issue + // issue_label(label = 'bug') -> ids {10, 20} --. + // +-- ids in both -> issue(id) + // issue_label(label = 'urgent') -> ids {20, 30} ----' // // The broad plan would load issues after the first label match, then probe - // the second label row-by-row. The optimized plan first finds the issue ids - // that appear in both child scans, then loads only those issues. + // the second label row-by-row. This plan first finds the issue ids that + // appear in both child scans, then loads only those issues. // // Each rewrite has its own strict guard below. If a query needs a shape the // new physical operator cannot preserve, it falls through to the older @@ -440,14 +445,16 @@ function applyRootUnionBranches( // root is just the table/index we choose to scan first. This is the physical // equivalent of SQLite's multi-index OR strategy: // - // User query: + // Query: + // // issue.status = 'open' // OR EXISTS(issue_label WHERE label = 'bug') // - // Optimized scan: - // issue WHERE status = 'open' - // UNION by issue.id - // issue_label WHERE label = 'bug' -> issue lookup + // Scan plan: + // + // issue(status = 'open') -----------------------. + // +-- union issue.id + // issue_label(label = 'bug') -> issue(id) -------' // // Each recursive branch keeps the same ordering and split-edit keys as the // original AST, so the union can merge streams without re-sorting. @@ -519,15 +526,17 @@ function applySameRelationshipExistsIntersection( // find the issue ids that satisfy every sibling EXISTS first, then load the // issue rows for only those ids. // - // User query: - // EXISTS(issue_label WHERE label = 'bug') - // AND EXISTS(issue_label WHERE label = 'urgent') + // Query: + // + // issue + // |-- EXISTS(issue_label WHERE label = 'bug') + // `-- EXISTS(issue_label WHERE label = 'urgent') // - // Optimized scan: - // issue_label WHERE label = 'bug' - // INTERSECT by issue_id - // issue_label WHERE label = 'urgent' - // -> issue WHERE id = issue_id + // Scan plan: + // + // issue_label(label = 'bug') -------------------. + // +-- intersect issue_id -> issue(id) + // issue_label(label = 'urgent') ----------------' // // This avoids loading an issue after the first label match only to probe the // second label relationship row-by-row. @@ -628,13 +637,17 @@ function getSameRelationshipExistsIntersection( // child row for each parent id, each branch must prove it can produce at // most one child row for the id it contributes. // - // child table: issue_label - // child PK: [issue_id, label] - // parent id field: issue_id - // child filter: label = 'bug' + // issue_label primary key: + // + // [issue_id, label] + // + // Branch filter: + // + // issue_id comes from the parent relationship + // label = 'bug' // - // issue_id plus label covers the child PK, so this branch can produce at - // most one issue_label row for each issue_id. + // Together those two values cover the full primary key, so this branch can + // produce at most one issue_label row for each issue_id. if ( candidates.some( candidate => @@ -740,14 +753,18 @@ function isUniquePerCorrelationKey( // // Example: // - // child table: issue_label - // child PK: [issue_id, label] - // parent id field: issue_id - // child filter: label = 'bug' + // issue_label primary key: + // + // [issue_id, label] + // + // Branch filter: + // + // issue_id comes from the parent relationship + // label = 'bug' // - // The branch can now emit at most one issue_label row for each issue_id, so - // intersecting by issue_id is equivalent to asking whether both EXISTS - // branches are true. + // Together those values identify one issue_label row. That makes + // intersecting by issue_id equivalent to asking whether both EXISTS branches + // are true. const constrained = new Set(condition.related.correlation.childField); collectEqualityConstrainedColumns( condition.related.subquery.where, diff --git a/packages/zql/src/ivm/set-operators.ts b/packages/zql/src/ivm/set-operators.ts index 6639b309cc..08a2650a84 100644 --- a/packages/zql/src/ivm/set-operators.ts +++ b/packages/zql/src/ivm/set-operators.ts @@ -26,19 +26,17 @@ import {mergeFetches} from './union-fan-in.ts'; /** * Merges the results of an OR query that has more than one good starting point. * - * Example user query: + * Query: * - * issues where status = 'open' - * OR issue has label 'bug' + * issue + * |-- status = 'open' + * `-- OR has issue_label(label = 'bug') * - * Optimized scan: + * Scan plan: * - * branch 0: scan issue(status = 'open') - * branch 1: scan issue_label(label = 'bug') -> look up issue - * - * InputUnion - * / \ - * issue rows issue rows + * branch 0: issue(status = 'open') ----------------. + * InputUnion -> issue rows + * branch 1: issue_label(label = 'bug') -> issue(id) -' * * InputUnion streams those issue rows in final sort order and removes duplicate * issue ids. If the same issue appears in both branches, the earlier branch is @@ -228,21 +226,20 @@ export class InputUnion implements Input { /** * Keeps only parent ids that appear in every required related-table scan. * - * Example user query: - * - * issues where issue has label 'bug' - * AND issue has label 'urgent' + * Query: * - * Optimized scan: + * issue + * |-- has issue_label(label = 'bug') + * `-- AND has issue_label(label = 'urgent') * - * issue_label(label = 'bug') -> issue ids {10, 20} - * issue_label(label = 'urgent') -> issue ids {20, 30} + * Scan plan: * - * InputIntersection on issue_id - * | - * issue id {20} - * | - * look up issue 20 + * issue_label(label = 'bug') -> ids {10, 20} --. + * InputIntersection -> id {20} + * issue_label(label = 'urgent') -> ids {20, 30} ----' + * | + * v + * issue(id = 20) * * The important idea is simple: do the cheap child-table lookups first, keep * only ids found in every branch, then fetch the parent rows. The builder only diff --git a/packages/zqlite/src/query-planner-scenarios.test.ts b/packages/zqlite/src/query-planner-scenarios.test.ts index 809673422f..04d302c098 100644 --- a/packages/zqlite/src/query-planner-scenarios.test.ts +++ b/packages/zqlite/src/query-planner-scenarios.test.ts @@ -7,15 +7,17 @@ import { import type {educationAppSchema} from './test/query-scenarios/education-app.ts'; import scenarios from './test/query-scenarios/scenarios/index.ts'; -// Scenario comments use a small visual legend: +// Scenario comments use one visual legend: // -// Before: the logical ZQL filter shape. -// After: the physical SQL scan order the optimized pipeline produces. +// Query: the logical ZQL filter the user wrote. +// Rewrite: pure filter simplification, when one happens. +// Scan plan: the physical SQL order the optimized pipeline produces. // -// A -> B +// A -> B means scan A first, then look up B. // -// means scan A first, then look up B. UNION and INTERSECT describe how parent -// row ids from separate roots are combined before rows reach the final result. +// left scan --. +// +-- union or intersect parent ids +// right scan -' for (const scenario of scenarios as readonly QueryScenario< typeof educationAppSchema >[]) { diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts index c1c358bca8..40cd6f188b 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts @@ -71,14 +71,21 @@ export default { }, }, planDebug: ['flipped'], - // Before -> after: + // Query: // // assignment - // EXISTS membership(student IN (1, 2) AND student != 2) + // `-- EXISTS membership(student IN [1, 2] AND student != 2) // - // membership(student = 1) -> assignment(id) + // Rewrite: + // + // student IN [1, 2] AND student != 2 + // | + // v + // student = 1 // - // Simplify the child domain first, then flip the selective EXISTS. + // Scan plan: + // + // membership(student = 1) -> assignment(id) sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts index fbe22e8966..2d6665529a 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts @@ -40,13 +40,18 @@ export default { conditions: [], }, }, - // Before -> after: + // Query: // - // teacher_id = 1 AND teacher_id != 1 + // teacher_id = 1 + // AND + // teacher_id != 1 // - // FALSE + // Rewrite: // - // The parent domain is empty, so no assignment can survive. + // [teacher 1] intersect [not teacher 1] + // | + // v + // FALSE sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts index 4c130368b5..0a9ae79f0d 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts @@ -67,13 +67,19 @@ export default { flip: true, }, }, - // Before -> after: + // Query: // - // EXISTS membership(student = 1) OR EXISTS membership(student = 1) + // EXISTS membership(student = 1) + // OR + // EXISTS membership(student = 1) // - // membership(student = 1) -> assignment(id) + // Rewrite: + // + // identical branches collapse to one // - // Duplicate OR branches collapse before the surviving EXISTS flips. + // Scan plan: + // + // membership(student = 1) -> assignment(id) sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts index 70b8b478c0..8fb12472e1 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts @@ -60,14 +60,16 @@ export default { ], }, }, - // Before -> after: + // Query: // // assignment(archived) - // EXISTS membership(student) with flip false + // `-- EXISTS membership(student), flip = false + // + // Scan plan: // // assignment(archived) -> membership(assignment_id, student) // - // Explicit flip false is user intent, so assignment remains the root. + // The user pinned the root at assignment, so we do not flip. // calls: 26 includes the repeated per-assignment membership probes plus // the final probe used while exhausting the parent stream. sql: [ diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts index be49acd15f..aa31a40396 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts @@ -60,14 +60,16 @@ export default { ], }, }, - // Before -> after: + // Query: // // assignment(archived) - // EXISTS membership(student) with flip true + // `-- EXISTS membership(student), flip = true + // + // Scan plan: // // membership(student) -> assignment(id, archived) // - // Explicit flip true makes membership the root. + // The user pinned the root at membership, so we start there. sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts index d0da779f32..35d88c51c3 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts @@ -51,13 +51,18 @@ export default { right: {type: 'literal', value: 1}, }, }, - // Before -> after: - // - // teacher_id = 1 OR EXISTS membership(student IN []) + // Query: // // teacher_id = 1 + // OR + // EXISTS membership(student IN []) + // + // Rewrite: // - // The impossible EXISTS branch is false, so OR keeps only the real branch. + // teacher_id = 1 OR FALSE + // | + // v + // teacher_id = 1 sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts b/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts index b04ee5ecf4..9c9dfa058f 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts @@ -78,17 +78,20 @@ export default { }, }, planDebug: ['flipped'], - // Before -> after: + // Query: // // assignment - // archived AND (teacher OR FALSE OR EXISTS membership(student)) + // `-- archived AND (teacher OR FALSE OR EXISTS membership(student)) // - // assignment(archived), with teacher checked above the SQL scan - // for the parent branch - // UNION on assignment.id - // membership(student) -> assignment(id, archived) + // Rewrite: // - // Drop FALSE, then plan the remaining mixed OR as two roots. + // remove FALSE from the OR + // + // Scan plan: + // + // assignment(archived), teacher checked in pipeline --. + // +-- union assignment.id + // membership(student) -> assignment(id, archived) ----' sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts b/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts index b42c412149..087e9974cb 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts @@ -38,13 +38,18 @@ export default { right: {type: 'literal', value: 1}, }, }, - // Before -> after: - // - // teacher_id = 1 OR FALSE + // Query: // // teacher_id = 1 + // OR + // FALSE + // + // Rewrite: // - // A false branch adds no rows to an OR. + // FALSE adds no rows + // | + // v + // teacher_id = 1 sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts index 7d17e5313f..5c5ccd6f08 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts @@ -68,16 +68,16 @@ export default { }, }, planDebug: ['FO ⋈ assignment_to_student: flipped'], - // Before -> after: + // Query: // // assignment - // teacher OR EXISTS membership(student) + // `-- teacher OR EXISTS membership(student) // - // assignment(teacher) - // UNION on assignment.id - // membership(student) -> assignment(id) + // Scan plan: // - // Mixed OR branches each keep the most selective root. + // assignment(teacher) -----------------------. + // +-- union assignment.id + // membership(student) -> assignment(id) ------' sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts index d0fbd9a750..4bd82ac1f8 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts @@ -62,17 +62,19 @@ export default { .orderBy(colName(assignment, 'created_at'), 'desc') .orderBy(colName(assignment, 'id'), 'asc'), expectations: { - // Before -> after: + // Query: // - // EXISTS membership(student = 1) - // AND EXISTS membership(student = 2) + // assignment + // |-- EXISTS membership(student = 1) + // `-- EXISTS membership(student = 2) // - // membership(student = 1) - // INTERSECT on assignment_id - // membership(student = 2) - // -> assignment(id) + // Scan plan: // - // AND means the assignment key must survive both child streams. + // membership(student = 1) -> assignment ids --. + // +-- intersect assignment_id -> assignment(id) + // membership(student = 2) -> assignment ids ---' + // + // The assignment id must survive both child streams. // The membership SQL has calls: 2 because the same SELECT text runs once // for each student bind value before the assignment_id intersection. sql: [ diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts index b1fe004fb5..94444fc503 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts @@ -67,13 +67,19 @@ export default { flip: true, }, }, - // Before -> after: + // Query: // - // EXISTS membership(student = 1) OR EXISTS membership(student = 2) + // EXISTS membership(student = 1) + // OR + // EXISTS membership(student = 2) // - // membership(student IN (1, 2)) -> assignment(id) + // Rewrite: // - // Sibling EXISTS branches over the same edge become one flipped scan. + // same relationship OR -> student IN [1, 2] + // + // Scan plan: + // + // membership(student IN [1, 2]) -> assignment(id) sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts index 23a8f72876..a3ffd8649e 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts @@ -94,14 +94,19 @@ export default { ], }, }, - // Before -> after: + // Query: // - // (archived AND EXISTS membership(student = 1)) - // OR (archived AND EXISTS membership(student = 2)) + // archived AND EXISTS membership(student = 1) + // OR + // archived AND EXISTS membership(student = 2) // - // membership(student IN (1, 2)) -> assignment(id, archived) + // Rewrite: // - // Factor the shared parent filter, merge the child domains, then flip. + // archived AND EXISTS membership(student IN [1, 2]) + // + // Scan plan: + // + // membership(student IN [1, 2]) -> assignment(id, archived) sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts index 8ba88d0c69..d7b4711eec 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts @@ -58,14 +58,18 @@ export default { }, }, planDebug: ['flipped'], - // Before -> after: + // Query: // // assignment - // EXISTS membership(student = 1 OR student = 2) + // `-- EXISTS membership(student = 1 OR student = 2) // - // membership(student IN (1, 2)) -> assignment(id) + // Rewrite: // - // Merge the child OR before choosing the flipped root. + // child OR -> student IN [1, 2] + // + // Scan plan: + // + // membership(student IN [1, 2]) -> assignment(id) sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts index 9711a639e2..bd1b29bd63 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts @@ -77,15 +77,16 @@ export default { }, }, planDebug: ['Best plan: Attempt 2', 'FO ⋈ assignment_to_student: flipped'], - // Before -> after: + // Query: // // assignment - // archived AND (teacher OR EXISTS membership(student)) + // `-- archived AND (teacher OR EXISTS membership(student)) // - // assignment(archived), with teacher checked above the SQL scan - // for the parent branch - // UNION on assignment.id - // membership(student) -> assignment(id, archived) + // Scan plan: + // + // assignment(archived), teacher checked in pipeline --. + // +-- union assignment.id + // membership(student) -> assignment(id, archived) ----' sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts index efebb733f2..a46216e168 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts @@ -60,14 +60,14 @@ export default { }, }, planDebug: ['flipped'], - // Before -> after: + // Query: // // assignment(archived) - // EXISTS membership(student) + // `-- EXISTS membership(student) // - // membership(student) -> assignment(id, archived) + // Scan plan: // - // The child predicate is selective, so membership becomes the root. + // membership(student) -> assignment(id, archived) sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts index 7a95d51205..55a90f9a7b 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts @@ -42,13 +42,15 @@ export default { right: {type: 'literal', value: [1, 2]}, }, }, - // Before -> after: + // Query: // - // teacher_id = 1 OR teacher_id = 2 + // teacher_id = 1 + // OR + // teacher_id = 2 // - // teacher_id IN (1, 2) + // Rewrite: // - // Same-column equality ORs collapse into one finite domain. + // teacher_id IN [1, 2] sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts index 200f1e5161..16dcd63548 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts @@ -61,14 +61,16 @@ export default { }, }, planDebug: ['semi'], - // Before -> after: + // Query: // // assignment(archived, teacher) - // EXISTS membership(student) + // `-- EXISTS membership(student) + // + // Scan plan: // // assignment(archived, teacher) -> membership(assignment_id, student) // - // The parent predicates are selective, so the semi join stays unflipped. + // The assignment filters are selective, so assignment stays the root. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts index 1221c877b7..542c0ae212 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts @@ -78,13 +78,19 @@ export default { }, }, planDebug: ['flipped'], - // Before -> after: + // Query: // - // EXISTS membership(student = 1) OR EXISTS membership(student = 2) + // EXISTS membership(student = 1) + // OR + // EXISTS membership(student = 2) // - // membership(student IN (1, 2)) -> assignment(id) + // Rewrite: // - // Same relationship, same correlation, one merged child domain. + // same relationship OR -> student IN [1, 2] + // + // Scan plan: + // + // membership(student IN [1, 2]) -> assignment(id) sql: [ { table: 'assignment_to_student', From 96d484441b482b9365155c4057764cb8ba2a4448 Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:38:46 -0400 Subject: [PATCH 12/13] docs(zqlite): clarify planner scenario traces --- .../src/query-planner-scenarios.test.ts | 7 ++-- .../scenarios/child-domain-intersection.ts | 33 +++++++++++----- .../scenarios/contradictory-parent-domain.ts | 26 ++++++++----- .../scenarios/duplicate-exists-or-dedupe.ts | 30 +++++++++++---- .../scenarios/explicit-flip-false.ts | 30 +++++++++++---- .../scenarios/explicit-flip-true.ts | 25 +++++++++--- .../impossible-exists-or-parent-filter.ts | 30 +++++++++++---- .../mixed-or-with-false-and-membership.ts | 38 ++++++++++++++----- .../scenarios/or-with-false-branch.ts | 27 ++++++++----- .../scenarios/parent-or-exists-union-roots.ts | 29 +++++++++++--- .../same-relationship-and-intersect.ts | 30 ++++++++++----- .../scenarios/same-relationship-or-merge.ts | 32 ++++++++++++---- .../shared-parent-filter-or-factor.ts | 35 +++++++++++++---- .../scenarios/single-exists-child-or.ts | 27 ++++++++++--- .../scenarios/student-membership-mixed-or.ts | 31 +++++++++++---- .../student-membership-simple-exists.ts | 23 ++++++++--- .../scenarios/teacher-filter-or-in.ts | 23 ++++++++--- .../teacher-filter-with-membership.ts | 25 +++++++++--- .../scenarios/two-student-membership-or.ts | 32 ++++++++++++---- 19 files changed, 394 insertions(+), 139 deletions(-) diff --git a/packages/zqlite/src/query-planner-scenarios.test.ts b/packages/zqlite/src/query-planner-scenarios.test.ts index 04d302c098..0668d1f3ac 100644 --- a/packages/zqlite/src/query-planner-scenarios.test.ts +++ b/packages/zqlite/src/query-planner-scenarios.test.ts @@ -9,9 +9,10 @@ import scenarios from './test/query-scenarios/scenarios/index.ts'; // Scenario comments use one visual legend: // -// Query: the logical ZQL filter the user wrote. -// Rewrite: pure filter simplification, when one happens. -// Scan plan: the physical SQL order the optimized pipeline produces. +// Submitted ZQL: the filter shape the user wrote. +// Naive plan: the straightforward parent first execution we avoid. +// Optimized plan: the physical SQL order this test expects. +// Intuition: why the optimized plan is cheaper or clearer. // // A -> B means scan A first, then look up B. // diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts index 40cd6f188b..71549affa9 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts @@ -71,21 +71,36 @@ export default { }, }, planDebug: ['flipped'], - // Query: + // Submitted ZQL: + // + // assignment.whereExists( + // assignment_to_student, + // student_id IN ['student-1', 'student-2'] + // AND student_id != 'student-2' + // ) + // + // Naive plan: // // assignment - // `-- EXISTS membership(student IN [1, 2] AND student != 2) + // `-- for each assignment, probe membership by assignment_id + // and evaluate both student predicates + // + // Optimized plan: // - // Rewrite: + // student_id IN ['student-1', 'student-2'] + // AND student_id != 'student-2' + // | + // v + // student_id = 'student-1' // - // student IN [1, 2] AND student != 2 - // | - // v - // student = 1 + // assignment_to_student(student_id = 'student-1') + // `-- fetch assignment by assignment_id // - // Scan plan: + // Intuition: // - // membership(student = 1) -> assignment(id) + // The child filter simplifies to one student before planning, so the + // planner starts from the small membership index instead of walking every + // assignment. sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts index 2d6665529a..eac5863b34 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts @@ -40,18 +40,26 @@ export default { conditions: [], }, }, - // Query: + // Submitted ZQL: // - // teacher_id = 1 - // AND - // teacher_id != 1 + // assignment.where(teacher_id = 1 AND teacher_id != 1) // - // Rewrite: + // Naive plan: // - // [teacher 1] intersect [not teacher 1] - // | - // v - // FALSE + // assignment + // `-- scan rows and test both predicates one row at a time + // + // Optimized plan: + // + // teacher_id = 1 AND teacher_id != 1 + // | + // v + // FALSE + // + // Intuition: + // + // No row can be both teacher 1 and not teacher 1, so the planner emits + // an empty query instead of scanning assignment. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts index 0a9ae79f0d..29e58646db 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts @@ -67,19 +67,33 @@ export default { flip: true, }, }, - // Query: + // Submitted ZQL: // - // EXISTS membership(student = 1) - // OR - // EXISTS membership(student = 1) + // assignment.where( + // EXISTS assignment_to_student(student_id = 'student-1') + // OR EXISTS assignment_to_student(student_id = 'student-1') + // ) // - // Rewrite: + // Naive plan: // - // identical branches collapse to one + // assignment + // |-- probe membership for student-1 + // `-- probe membership for student-1 again // - // Scan plan: + // Optimized plan: // - // membership(student = 1) -> assignment(id) + // identical EXISTS branches + // | + // v + // EXISTS membership(student_id = 'student-1') + // + // assignment_to_student(student_id = 'student-1') + // `-- fetch assignment by assignment_id + // + // Intuition: + // + // Running the same child lookup twice cannot add rows, so the planner + // keeps one branch and flips it to the membership index. sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts index 8fb12472e1..46298b07fb 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts @@ -60,18 +60,32 @@ export default { ], }, }, - // Query: + // Submitted ZQL: // - // assignment(archived) - // `-- EXISTS membership(student), flip = false + // assignment + // .where(archived_at IS null) + // .whereExists(assignment_to_student, student_id = 'student-1', { + // flip: false, + // }) // - // Scan plan: + // Naive plan: // - // assignment(archived) -> membership(assignment_id, student) + // assignment(archived_at IS null) + // `-- for each assignment, probe membership by assignment_id // - // The user pinned the root at assignment, so we do not flip. - // calls: 26 includes the repeated per-assignment membership probes plus - // the final probe used while exhausting the parent stream. + // Optimized plan: + // + // assignment(archived_at IS null) + // `-- probe assignment_to_student( + // assignment_id = current assignment.id, + // student_id = 'student-1' + // ) + // + // Intuition: + // + // The user explicitly pinned the parent as the root. The 26 calls are the + // repeated per assignment probes plus the final probe that exhausts the + // parent stream. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts index aa31a40396..5c90736537 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts @@ -60,16 +60,29 @@ export default { ], }, }, - // Query: + // Submitted ZQL: // - // assignment(archived) - // `-- EXISTS membership(student), flip = true + // assignment + // .where(archived_at IS null) + // .whereExists(assignment_to_student, student_id = 'student-1', { + // flip: true, + // }) // - // Scan plan: + // Naive plan: // - // membership(student) -> assignment(id, archived) + // assignment(archived_at IS null) + // `-- for each assignment, probe membership by assignment_id // - // The user pinned the root at membership, so we start there. + // Optimized plan: + // + // assignment_to_student(student_id = 'student-1') + // `-- fetch assignment by assignment_id + // `-- keep it only if archived_at IS null + // + // Intuition: + // + // The user explicitly asked to flip the EXISTS, so the child side is the + // root even before the cost model gets a vote. sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts index 35d88c51c3..59fd40ae18 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts @@ -51,18 +51,32 @@ export default { right: {type: 'literal', value: 1}, }, }, - // Query: + // Submitted ZQL: // - // teacher_id = 1 - // OR - // EXISTS membership(student IN []) + // assignment.where( + // teacher_id = 1 + // OR EXISTS assignment_to_student(student_id IN []) + // ) // - // Rewrite: + // Naive plan: + // + // assignment + // |-- keep rows where teacher_id = 1 + // `-- also ask membership for an empty student set + // + // Optimized plan: + // + // EXISTS membership(student_id IN []) -> FALSE // // teacher_id = 1 OR FALSE - // | - // v - // teacher_id = 1 + // | + // v + // teacher_id = 1 + // + // Intuition: + // + // An empty IN list cannot find a child row, so the OR falls back to the + // only branch that can return assignments. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts b/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts index 9c9dfa058f..72252ddf84 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts @@ -78,20 +78,40 @@ export default { }, }, planDebug: ['flipped'], - // Query: + // Submitted ZQL: // - // assignment - // `-- archived AND (teacher OR FALSE OR EXISTS membership(student)) + // assignment.where( + // archived_at IS null + // AND ( + // teacher_id = 1 + // OR FALSE + // OR EXISTS assignment_to_student(student_id = 'student-1') + // ) + // ) // - // Rewrite: + // Naive plan: // - // remove FALSE from the OR + // assignment(archived_at IS null) + // |-- check teacher_id = 1 + // |-- carry a FALSE branch that can never match + // `-- if needed, probe membership by assignment_id // - // Scan plan: + // Optimized plan: // - // assignment(archived), teacher checked in pipeline --. - // +-- union assignment.id - // membership(student) -> assignment(id, archived) ----' + // teacher_id = 1 OR FALSE OR EXISTS membership(...) + // | + // v + // teacher_id = 1 OR EXISTS membership(...) + // + // assignment(archived_at IS null, teacher_id = 1) ----------. + // +-- union + // assignment_to_student(student_id = 'student-1') -> parent -' + // `-- keep parent only if archived_at IS null + // + // Intuition: + // + // The FALSE branch disappears before costing, then the remaining OR gets + // the same split root treatment as the mixed parent and child case. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts b/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts index 087e9974cb..dce346b831 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts @@ -38,18 +38,27 @@ export default { right: {type: 'literal', value: 1}, }, }, - // Query: + // Submitted ZQL: // - // teacher_id = 1 - // OR - // FALSE + // assignment.where(teacher_id = 1 OR FALSE) // - // Rewrite: + // Naive plan: // - // FALSE adds no rows - // | - // v - // teacher_id = 1 + // assignment + // `-- check teacher_id = 1 + // `-- also carry a FALSE branch that can never match + // + // Optimized plan: + // + // teacher_id = 1 OR FALSE + // | + // v + // teacher_id = 1 + // + // Intuition: + // + // FALSE contributes no rows to an OR, so it should not make costing + // think this query is less selective than the real teacher filter. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts index 5c5ccd6f08..b23479b378 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts @@ -68,16 +68,33 @@ export default { }, }, planDebug: ['FO ⋈ assignment_to_student: flipped'], - // Query: + // Submitted ZQL: + // + // assignment.where( + // teacher_id = 1 + // OR EXISTS assignment_to_student(student_id = 'student-1') + // ) + // + // Naive plan: // // assignment - // `-- teacher OR EXISTS membership(student) + // |-- check teacher_id = 1 + // `-- if needed, probe membership by assignment_id + // + // Optimized plan: + // + // assignment(teacher_id = 1) -------------------------------. + // +-- union + // assignment_to_student(student_id = 'student-1') -> parent -' + // | + // v + // assignment ids // - // Scan plan: + // Intuition: // - // assignment(teacher) -----------------------. - // +-- union assignment.id - // membership(student) -> assignment(id) ------' + // Each OR branch gets its best root. Parent matches come from the + // teacher index, membership matches come from the child index, and the + // resulting assignment ids are merged. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts index 4bd82ac1f8..7a47a52e82 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts @@ -62,21 +62,31 @@ export default { .orderBy(colName(assignment, 'created_at'), 'desc') .orderBy(colName(assignment, 'id'), 'asc'), expectations: { - // Query: + // Submitted ZQL: // // assignment - // |-- EXISTS membership(student = 1) - // `-- EXISTS membership(student = 2) + // .whereExists(assignment_to_student, student_id = 'student-1') + // .whereExists(assignment_to_student, student_id = 'student-2') // - // Scan plan: + // Naive plan: // - // membership(student = 1) -> assignment ids --. - // +-- intersect assignment_id -> assignment(id) - // membership(student = 2) -> assignment ids ---' + // assignment + // |-- probe membership for student-1 + // `-- probe membership for student-2 + // + // Optimized plan: + // + // assignment_to_student(student_id = 'student-1') -> assignment_ids --. + // intersect + // assignment_to_student(student_id = 'student-2') -> assignment_ids --' + // | + // v + // fetch assignment + // + // Intuition: // - // The assignment id must survive both child streams. - // The membership SQL has calls: 2 because the same SELECT text runs once - // for each student bind value before the assignment_id intersection. + // The same assignment id must appear in both child streams. The SQL text + // has calls: 2 because it runs once per student before intersecting ids. sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts index 94444fc503..7f31c607ec 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts @@ -67,19 +67,35 @@ export default { flip: true, }, }, - // Query: + // Submitted ZQL: // - // EXISTS membership(student = 1) - // OR - // EXISTS membership(student = 2) + // assignment.where( + // EXISTS assignment_to_student(student_id = 'student-1') + // OR EXISTS assignment_to_student(student_id = 'student-2') + // ) // - // Rewrite: + // Naive plan: // - // same relationship OR -> student IN [1, 2] + // assignment + // |-- probe membership for student-1 + // `-- probe membership for student-2 // - // Scan plan: + // Optimized plan: // - // membership(student IN [1, 2]) -> assignment(id) + // EXISTS membership(student_id = 'student-1') + // OR + // EXISTS membership(student_id = 'student-2') + // | + // v + // EXISTS membership(student_id IN ['student-1', 'student-2']) + // + // assignment_to_student(student_id IN ['student-1', 'student-2']) + // `-- fetch assignment by assignment_id + // + // Intuition: + // + // Two EXISTS branches over the same relationship are one child domain. + // Merging them keeps the plan on the selective membership index. sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts index a3ffd8649e..43bab3adf0 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts @@ -94,19 +94,38 @@ export default { ], }, }, - // Query: + // Submitted ZQL: // - // archived AND EXISTS membership(student = 1) - // OR - // archived AND EXISTS membership(student = 2) + // assignment.where( + // (archived_at IS null + // AND EXISTS assignment_to_student(student_id = 'student-1')) + // OR + // (archived_at IS null + // AND EXISTS assignment_to_student(student_id = 'student-2')) + // ) // - // Rewrite: + // Naive plan: // - // archived AND EXISTS membership(student IN [1, 2]) + // assignment + // |-- branch 1 checks archived_at, then probes student-1 + // `-- branch 2 checks archived_at again, then probes student-2 // - // Scan plan: + // Optimized plan: // - // membership(student IN [1, 2]) -> assignment(id, archived) + // shared archived_at IS null + // | + // v + // archived_at IS null + // AND EXISTS membership(student_id IN ['student-1', 'student-2']) + // + // assignment_to_student(student_id IN ['student-1', 'student-2']) + // `-- fetch assignment by assignment_id + // `-- keep it only if archived_at IS null + // + // Intuition: + // + // The duplicate parent filter is factored once, and the two child + // branches become one membership index scan. sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts index d7b4711eec..f1c17c7c20 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts @@ -58,18 +58,33 @@ export default { }, }, planDebug: ['flipped'], - // Query: + // Submitted ZQL: + // + // assignment.whereExists( + // assignment_to_student, + // student_id = 'student-1' OR student_id = 'student-2' + // ) + // + // Naive plan: // // assignment - // `-- EXISTS membership(student = 1 OR student = 2) + // `-- for each assignment, probe membership by assignment_id + // and test both student equality branches + // + // Optimized plan: // - // Rewrite: + // student_id = 'student-1' OR student_id = 'student-2' + // | + // v + // student_id IN ['student-1', 'student-2'] // - // child OR -> student IN [1, 2] + // assignment_to_student(student_id IN ['student-1', 'student-2']) + // `-- fetch assignment by assignment_id // - // Scan plan: + // Intuition: // - // membership(student IN [1, 2]) -> assignment(id) + // A same column OR inside the child query is one IN lookup, and that + // lookup is selective enough to become the root. sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts index bd1b29bd63..70d77ef41e 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts @@ -77,16 +77,33 @@ export default { }, }, planDebug: ['Best plan: Attempt 2', 'FO ⋈ assignment_to_student: flipped'], - // Query: + // Submitted ZQL: // - // assignment - // `-- archived AND (teacher OR EXISTS membership(student)) + // assignment.where( + // archived_at IS null + // AND ( + // teacher_id = 1 + // OR EXISTS assignment_to_student(student_id = 'student-1') + // ) + // ) // - // Scan plan: + // Naive plan: // - // assignment(archived), teacher checked in pipeline --. - // +-- union assignment.id - // membership(student) -> assignment(id, archived) ----' + // assignment(archived_at IS null) + // |-- check teacher_id = 1 + // `-- if needed, probe membership by assignment_id + // + // Optimized plan: + // + // assignment(archived_at IS null, teacher_id = 1) ----------. + // +-- union + // assignment_to_student(student_id = 'student-1') -> parent -' + // `-- keep parent only if archived_at IS null + // + // Intuition: + // + // The shared archived filter stays on both branches, while the OR splits + // so each side can use its own selective index. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts index a46216e168..af999ae09d 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts @@ -60,14 +60,27 @@ export default { }, }, planDebug: ['flipped'], - // Query: + // Submitted ZQL: // - // assignment(archived) - // `-- EXISTS membership(student) + // assignment + // .where(archived_at IS null) + // .whereExists(assignment_to_student, student_id = 'student-1') // - // Scan plan: + // Naive plan: // - // membership(student) -> assignment(id, archived) + // assignment(archived_at IS null) + // `-- for each assignment, probe membership by assignment_id + // + // Optimized plan: + // + // assignment_to_student(student_id = 'student-1') + // `-- fetch assignment by assignment_id + // `-- keep it only if archived_at IS null + // + // Intuition: + // + // The student membership index is smaller than the live assignment set, + // so it is cheaper to start from the child row and look up its parent. sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts index 55a90f9a7b..d400e2e2e0 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts @@ -42,15 +42,26 @@ export default { right: {type: 'literal', value: [1, 2]}, }, }, - // Query: + // Submitted ZQL: // - // teacher_id = 1 - // OR - // teacher_id = 2 + // assignment.where(teacher_id = 1 OR teacher_id = 2) // - // Rewrite: + // Naive plan: // - // teacher_id IN [1, 2] + // assignment + // `-- test teacher_id = 1 + // `-- test teacher_id = 2 + // + // Optimized plan: + // + // teacher_id = 1 OR teacher_id = 2 + // | + // v + // teacher_id IN [1, 2] + // + // Intuition: + // + // Two equality checks on the same column are one indexed IN lookup. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts index 16dcd63548..0505d141ee 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts @@ -61,16 +61,29 @@ export default { }, }, planDebug: ['semi'], - // Query: + // Submitted ZQL: // - // assignment(archived, teacher) - // `-- EXISTS membership(student) + // assignment + // .where(archived_at IS null AND teacher_id = 1) + // .whereExists(assignment_to_student, student_id = 'student-1') // - // Scan plan: + // Naive plan: // - // assignment(archived, teacher) -> membership(assignment_id, student) + // assignment(archived_at IS null AND teacher_id = 1) + // `-- for each matching assignment, probe membership // - // The assignment filters are selective, so assignment stays the root. + // Optimized plan: + // + // assignment(archived_at IS null AND teacher_id = 1) + // `-- probe assignment_to_student( + // assignment_id = current assignment.id, + // student_id = 'student-1' + // ) + // + // Intuition: + // + // The parent filters are already selective enough, so staying on + // assignment avoids starting from membership and then fetching parents. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts index 542c0ae212..35614854f2 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts @@ -78,19 +78,35 @@ export default { }, }, planDebug: ['flipped'], - // Query: + // Submitted ZQL: // - // EXISTS membership(student = 1) - // OR - // EXISTS membership(student = 2) + // assignment.where( + // EXISTS assignment_to_student(student_id = 'student-1') + // OR EXISTS assignment_to_student(student_id = 'student-2') + // ) // - // Rewrite: + // Naive plan: // - // same relationship OR -> student IN [1, 2] + // assignment + // |-- probe membership for student-1 + // `-- probe membership for student-2 // - // Scan plan: + // Optimized plan: // - // membership(student IN [1, 2]) -> assignment(id) + // EXISTS membership(student_id = 'student-1') + // OR + // EXISTS membership(student_id = 'student-2') + // | + // v + // EXISTS membership(student_id IN ['student-1', 'student-2']) + // + // assignment_to_student(student_id IN ['student-1', 'student-2']) + // `-- fetch assignment by assignment_id + // + // Intuition: + // + // Both OR branches ask the same relationship for the same parent key, so + // they can share one child index scan. sql: [ { table: 'assignment_to_student', From 54a53ff725e0cc7d1a4c1c8e89f14fec6f87edfe Mon Sep 17 00:00:00 2001 From: Alp Karavil <46430499+Karavil@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:59:30 -0400 Subject: [PATCH 13/13] fix(zql): harden set operator planner rewrites --- packages/zql/src/builder/builder.ts | 29 ++++- packages/zql/src/ivm/set-operators.test.ts | 101 +++++++++++++++- packages/zql/src/ivm/set-operators.ts | 68 +++++------ packages/zql/src/ivm/strip-relationships.ts | 110 ++++++++++++++++++ .../src/planner/condition-normalizer.test.ts | 22 ++++ .../zql/src/planner/condition-normalizer.ts | 7 +- packages/zqlite/src/test/query-scenario.ts | 13 ++- 7 files changed, 299 insertions(+), 51 deletions(-) create mode 100644 packages/zql/src/ivm/strip-relationships.ts diff --git a/packages/zql/src/builder/builder.ts b/packages/zql/src/builder/builder.ts index c96a089e94..5039503eb6 100644 --- a/packages/zql/src/builder/builder.ts +++ b/packages/zql/src/builder/builder.ts @@ -33,6 +33,7 @@ import type {Input, InputBase, Storage} from '../ivm/operator.ts'; import {InputIntersection, InputUnion} from '../ivm/set-operators.ts'; import {Skip} from '../ivm/skip.ts'; import type {Source, SourceInput} from '../ivm/source.ts'; +import {StripRelationships} from '../ivm/strip-relationships.ts'; import {Take} from '../ivm/take.ts'; import {UnionFanIn} from '../ivm/union-fan-in.ts'; import {UnionFanOut} from '../ivm/union-fan-out.ts'; @@ -458,8 +459,8 @@ function applyRootUnionBranches( // // Each recursive branch keeps the same ordering and split-edit keys as the // original AST, so the union can merge streams without re-sorting. - const inputs = branches.map((branch, index) => - buildPipelineInternal( + const inputs = branches.map((branch, index) => { + const input = buildPipelineInternal( { ...ast, where: branch, @@ -468,8 +469,9 @@ function applyRootUnionBranches( queryID, `${name}:or-${index}`, partitionKey, - ), - ); + ); + return stripRootUnionBranchRelationships(input, delegate, name, index); + }); const union = new InputUnion(inputs); for (const input of inputs) { @@ -478,6 +480,25 @@ function applyRootUnionBranches( return delegate.decorateInput(union, `${name}:input-union`); } +function stripRootUnionBranchRelationships( + input: Input, + delegate: BuilderDelegate, + name: string, + index: number, +): Input { + // Root union is a set of parent rows, not a way to expose relationship + // payloads. A flipped EXISTS branch may temporarily attach its child rows so + // the branch can prove the EXISTS is true, but those rows are condition-only + // data. Strip them before the branch joins the union so every branch exposes + // the same plain parent-row schema. + if (Object.keys(input.getSchema().relationships).length === 0) { + return input; + } + const stripped = new StripRelationships(input); + delegate.addEdge(input, stripped); + return delegate.decorateInput(stripped, `${name}:or-${index}:strip-related`); +} + function getRootUnionBranches(ast: AST): readonly Condition[] | undefined { // This rewrite is only safe at the root of a plain query. start, limit, and // related rows all observe the whole result stream, so they need a richer diff --git a/packages/zql/src/ivm/set-operators.test.ts b/packages/zql/src/ivm/set-operators.test.ts index 9dfbcf8e0d..8afb5de982 100644 --- a/packages/zql/src/ivm/set-operators.test.ts +++ b/packages/zql/src/ivm/set-operators.test.ts @@ -19,6 +19,7 @@ import { import type {SourceSchema} from './schema.ts'; import {InputIntersection, InputUnion} from './set-operators.ts'; import {consume, type Stream} from './stream.ts'; +import {StripRelationships} from './strip-relationships.ts'; const schema: SourceSchema = { tableName: 'assignment', @@ -202,6 +203,57 @@ describe('InputUnion', () => { ChangeType.CHILD, ]); }); + + test('requires relationship-free branch streams', () => { + const related = new MutableInput([], { + ...schema, + relationships: {children: schema}, + }); + + expect(() => new InputUnion([related])).toThrow( + 'input union requires inputs without relationships', + ); + }); +}); + +describe('StripRelationships', () => { + test('removes relationship schema and fetched relationship payloads', () => { + const input = new MutableInput( + [ + node( + {id: 1, value: 10}, + { + children: () => [node({id: 99, value: 99})], + }, + ), + ], + { + ...schema, + relationships: {children: schema}, + }, + ); + + const stripped = new StripRelationships(input); + + expect(stripped.getSchema().relationships).toEqual({}); + expect( + Array.from(skipYields(stripped.fetch({})), node => node.relationships), + ).toEqual([{}]); + }); + + test('ignores relationship-only pushes', () => { + const input = new MutableInput([], { + ...schema, + relationships: {children: schema}, + }); + const stripped = new StripRelationships(input); + const sink = new RecordingOutput(); + stripped.setOutput(sink); + + input.push(childChange(node({id: 1, value: 10}))); + + expect(sink.changes).toEqual([]); + }); }); describe('InputIntersection', () => { @@ -237,6 +289,28 @@ describe('InputIntersection', () => { ]); }); + test('fetch streams the representative branch after reading other key sets', () => { + const events: string[] = []; + const left = new LoggingInput( + 'first', + [node({id: 1, value: 10}), node({id: 2, value: 20})], + events, + ); + const right = new LoggingInput( + 'other', + [node({id: 2, value: 200})], + events, + ); + const intersection = new InputIntersection([left, right], ['id']); + + const iter = intersection.fetch({})[Symbol.iterator](); + const result = iter.next(); + + expect(result.done).toBe(false); + expect(result.value).toEqual(node({id: 2, value: 20})); + expect(events).toEqual(['other:start', 'other:end', 'first:start']); + }); + test('push emits when a key enters or leaves the intersection', () => { const left = new MutableInput([node({id: 1, value: 10})]); const right = new MutableInput([]); @@ -310,10 +384,12 @@ describe('InputIntersection', () => { class MutableInput implements Input { rows: Node[]; + readonly #schema: SourceSchema; #output: Output | undefined; - constructor(rows: Node[]) { + constructor(rows: Node[], sourceSchema: SourceSchema = schema) { this.rows = rows; + this.#schema = sourceSchema; } setOutput(output: Output): void { @@ -321,7 +397,7 @@ class MutableInput implements Input { } getSchema(): SourceSchema { - return schema; + return this.#schema; } *fetch(req: FetchRequest): Stream { @@ -340,6 +416,23 @@ class MutableInput implements Input { destroy(): void {} } +class LoggingInput extends MutableInput { + readonly #label: string; + readonly #events: string[]; + + constructor(label: string, rows: Node[], events: string[]) { + super(rows); + this.#label = label; + this.#events = events; + } + + override *fetch(req: FetchRequest): Stream { + this.#events.push(`${this.#label}:start`); + yield* super.fetch(req); + this.#events.push(`${this.#label}:end`); + } +} + class RecordingOutput implements Output { readonly changes: Change[] = []; @@ -348,8 +441,8 @@ class RecordingOutput implements Output { } } -function node(row: Row): Node { - return {row, relationships: {}}; +function node(row: Row, relationships: Node['relationships'] = {}): Node { + return {row, relationships}; } function childChange(parent: Node): Change { diff --git a/packages/zql/src/ivm/set-operators.ts b/packages/zql/src/ivm/set-operators.ts index 08a2650a84..4f79c0feb3 100644 --- a/packages/zql/src/ivm/set-operators.ts +++ b/packages/zql/src/ivm/set-operators.ts @@ -55,9 +55,10 @@ import {mergeFetches} from './union-fan-in.ts'; * The visible result did not gain a new primary key. The visible copy changed, * so downstream receives EDIT(value 20 -> 10). * - * This operator deliberately does not try to merge relationship payloads from - * duplicate rows. The builder only uses it for root queries without related - * rows, start, or limit. + * This operator intentionally accepts only plain row streams with no + * relationships. The builder strips condition-only relationship payloads from + * branch pipelines before they enter the union. That keeps the schema honest: + * every node emitted by the union has the same shape that the union advertises. */ export class InputUnion implements Input { readonly #inputs: readonly Input[]; @@ -275,19 +276,18 @@ export class InputIntersection implements Input { *fetch(req: FetchRequest): Stream { // Fetch the other branch id sets first, then stream rows from the first - // branch in its own order. yieldedKeys is the set part of the contract: - // even if the first branch has duplicate rows for an id, the intersection - // emits that id only once for the parent lookup. + // branch in its own order. We intentionally materialize id sets for the + // non-representative branches, not full rows from the first branch: + // + // rest branches -> Set(parent id) + // | + // v + // first branch stream -> keep ids present in every Set + // + // yieldedKeys is the set part of the contract: even if the first branch + // has duplicate rows for an id, the intersection emits that id only once + // for the parent lookup. const [first, ...rest] = this.#inputs; - const firstNodes: Node[] = []; - for (const node of first.fetch(req)) { - if (node === 'yield') { - yield node; - continue; - } - firstNodes.push(node); - } - const matchingKeys: ReadonlySet[] = []; for (const input of rest) { const keys = new Set(); @@ -302,7 +302,11 @@ export class InputIntersection implements Input { } const yieldedKeys = new Set(); - for (const node of firstNodes) { + for (const node of first.fetch(req)) { + if (node === 'yield') { + yield node; + continue; + } const key = rowKey(node.row, this.#key); if (!yieldedKeys.has(key) && matchingKeys.every(keys => keys.has(key))) { yieldedKeys.add(key); @@ -431,34 +435,22 @@ function mergeInputSchemas( operatorName: string, inputs: readonly Input[], ): SourceSchema { - // The set operators sit between complete pipelines. They can only compose - // pipelines that expose the same row stream shape. Relationships are merged - // by name for the schema contract, but duplicate row relationship payloads - // are not merged by InputUnion. The builder avoids that case for root union. + // InputUnion dedupes duplicate parent rows by choosing one visible copy. It + // cannot honestly merge relationship payloads from multiple copies, so it + // refuses relationship-bearing inputs. Callers that use relationships as + // private filter evidence must strip them before constructing the union. const schema = { ...firstInputSchema(operatorName, inputs), - relationships: { - ...inputs[0].getSchema().relationships, - }, + relationships: {}, } satisfies Writable; - const relationshipsFromBranches = new Set(); - for (const input of inputs.slice(1)) { + for (const input of inputs) { const inputSchema = input.getSchema(); assertCompatibleSchema(operatorName, schema, inputSchema); - for (const [relationshipName, relationshipSchema] of Object.entries( - inputSchema.relationships, - )) { - if (relationshipName in schema.relationships) { - continue; - } - assert( - !relationshipsFromBranches.has(relationshipName), - `Relationship ${relationshipName} exists in multiple upstream inputs to ${operatorName}`, - ); - schema.relationships[relationshipName] = relationshipSchema; - relationshipsFromBranches.add(relationshipName); - } + assert( + Object.keys(inputSchema.relationships).length === 0, + `${operatorName} requires inputs without relationships`, + ); } return schema; diff --git a/packages/zql/src/ivm/strip-relationships.ts b/packages/zql/src/ivm/strip-relationships.ts new file mode 100644 index 0000000000..147b29cff8 --- /dev/null +++ b/packages/zql/src/ivm/strip-relationships.ts @@ -0,0 +1,110 @@ +import type {Writable} from '../../../shared/src/writable.ts'; +import {ChangeIndex} from './change-index.ts'; +import {ChangeType} from './change-type.ts'; +import { + makeAddChange, + makeEditChange, + makeRemoveChange, + type Change, +} from './change.ts'; +import type {Node} from './data.ts'; +import { + throwOutput, + type FetchRequest, + type Input, + type InputBase, + type Operator, + type Output, +} from './operator.ts'; +import type {SourceSchema} from './schema.ts'; +import {type Stream} from './stream.ts'; + +/** + * Removes relationship payloads from a row stream. + * + * Some physical plans use hidden relationships as proof that a filter matched. + * For example, a flipped EXISTS branch first joins from the child table back to + * the parent, which naturally attaches the matching child rows to the parent + * node. If that branch later feeds a root-level union, those child rows are + * only evidence for the branch. They are not part of the query result. + * + * This operator draws that boundary explicitly: + * + * branch proves EXISTS with child rows + * | + * v + * StripRelationships + * | + * v + * plain parent rows enter the union + */ +export class StripRelationships implements Operator { + readonly #input: Input; + readonly #schema: SourceSchema; + #output: Output = throwOutput; + + constructor(input: Input) { + this.#input = input; + this.#schema = { + ...input.getSchema(), + relationships: {}, + } satisfies Writable; + input.setOutput(this); + } + + setOutput(output: Output): void { + this.#output = output; + } + + getSchema(): SourceSchema { + return this.#schema; + } + + *fetch(req: FetchRequest): Stream { + for (const node of this.#input.fetch(req)) { + yield node === 'yield' ? node : stripRelationships(node); + } + } + + *push(change: Change, _pusher: InputBase): Stream<'yield'> { + switch (change[ChangeIndex.TYPE]) { + case ChangeType.ADD: + yield* this.#output.push( + makeAddChange(stripRelationships(change[ChangeIndex.NODE])), + this, + ); + return; + + case ChangeType.REMOVE: + yield* this.#output.push( + makeRemoveChange(stripRelationships(change[ChangeIndex.NODE])), + this, + ); + return; + + case ChangeType.EDIT: { + const oldNode = stripRelationships(change[ChangeIndex.OLD_NODE]); + const newNode = stripRelationships(change[ChangeIndex.NODE]); + if (JSON.stringify(oldNode.row) === JSON.stringify(newNode.row)) { + return; + } + yield* this.#output.push(makeEditChange(newNode, oldNode), this); + return; + } + + case ChangeType.CHILD: + return; + } + } + + destroy(): void { + this.#input.destroy(); + } +} + +function stripRelationships(node: Node): Node { + return { + row: node.row, + relationships: {}, + }; +} diff --git a/packages/zql/src/planner/condition-normalizer.test.ts b/packages/zql/src/planner/condition-normalizer.test.ts index 31575419b5..341dc24b61 100644 --- a/packages/zql/src/planner/condition-normalizer.test.ts +++ b/packages/zql/src/planner/condition-normalizer.test.ts @@ -157,6 +157,28 @@ test('merges OR exists branches over the same relationship', () => { ); }); +test('merges OR exists branches after implicit ordering is completed', () => { + expect( + normalizeWhere({ + type: 'or', + conditions: [ + exists(eq('title', 'hello'), {subquery: {orderBy: [['id', 'asc']]}}), + exists(eq('title', 'world'), {subquery: {orderBy: [['id', 'asc']]}}), + ], + }), + ).toEqual( + exists( + { + type: 'simple', + left: {type: 'column', name: 'title'}, + op: 'IN', + right: {type: 'literal', value: ['hello', 'world']}, + }, + {subquery: {orderBy: [['id', 'asc']]}}, + ), + ); +}); + test('preserves unrestricted exists when merging narrower exists branches', () => { expect( normalizeWhere({ diff --git a/packages/zql/src/planner/condition-normalizer.ts b/packages/zql/src/planner/condition-normalizer.ts index 411230e17a..36c68af21b 100644 --- a/packages/zql/src/planner/condition-normalizer.ts +++ b/packages/zql/src/planner/condition-normalizer.ts @@ -686,12 +686,15 @@ function mergeableExistsKey(condition: Condition): string | undefined { if ( subquery.related !== undefined || subquery.start !== undefined || - subquery.limit !== undefined || - subquery.orderBy !== undefined + subquery.limit !== undefined ) { return undefined; } + // EXISTS only asks whether at least one child row exists. Without related + // rows, start, or limit, child ordering cannot affect that answer. Ignore + // orderBy here so production callers that completed implicit primary-key + // ordering before planning still get the same logical EXISTS merge. const mergeShape: CorrelatedSubquery = { ...related, subquery: { diff --git a/packages/zqlite/src/test/query-scenario.ts b/packages/zqlite/src/test/query-scenario.ts index ae00ebe348..a1cd636ec8 100644 --- a/packages/zqlite/src/test/query-scenario.ts +++ b/packages/zqlite/src/test/query-scenario.ts @@ -98,14 +98,21 @@ export function runQueryScenario( const builder = createBuilder(scenario.schema); const ast = asQueryInternals(scenario.query(builder)).ast; - const planDebugger = new AccumulatorDebugger(); - const optimizedAST = planQueryOnce(ast, costModel, planDebugger); + const optimizedAST = planQueryOnce(ast, costModel, new AccumulatorDebugger()); const debug = new ScenarioDebug(); const delegate = newQueryDelegate(lc, testLogConfig, db, scenario.schema); delegate.debug = debug; - const input = buildPipeline(optimizedAST, delegate, 'query-scenario'); + const planDebugger = new AccumulatorDebugger(); + const input = buildPipeline( + ast, + delegate, + 'query-scenario', + costModel, + lc, + planDebugger, + ); const sink = new Catch(input); // SQL shape alone can be misleading because repeated query text can hide // multiple physical scans with different bind values. ScenarioDebug keeps the