diff --git a/packages/zql/src/builder/builder.ts b/packages/zql/src/builder/builder.ts index 1df1cc2deb..5039503eb6 100644 --- a/packages/zql/src/builder/builder.ts +++ b/packages/zql/src/builder/builder.ts @@ -30,8 +30,10 @@ import {Filter} from '../ivm/filter.ts'; import {FlippedJoin} from '../ivm/flipped-join.ts'; import {Join} from '../ivm/join.ts'; import type {Input, InputBase, Storage} from '../ivm/operator.ts'; +import {InputIntersection, InputUnion} from '../ivm/set-operators.ts'; import {Skip} from '../ivm/skip.ts'; import type {Source, SourceInput} from '../ivm/source.ts'; +import {StripRelationships} from '../ivm/strip-relationships.ts'; import {Take} from '../ivm/take.ts'; import {UnionFanIn} from '../ivm/union-fan-in.ts'; import {UnionFanOut} from '../ivm/union-fan-out.ts'; @@ -270,17 +272,82 @@ function buildPipelineInternal( assertNoNotExists(ast.where); } - const csqConditions = gatherCorrelatedSubqueryQueryConditions(ast.where); + // Two narrow physical rewrites run before the generic source/filter/join + // pipeline below. These do not change the meaning of the WHERE clause. They + // only choose a better place to start reading rows. + // + // OR example: + // + // Query: + // + // issue + // |-- status = 'open' + // `-- OR has issue_label(label = 'bug') + // + // Scan plan: + // + // issue(status = 'open') -----------------------. + // +-- union issue.id + // issue_label(label = 'bug') -> issue(id) -------' + // + // The broad plan would scan all issues and ask "does either branch match?" + // for every row. This plan starts from both selective doorways into issue + // rows, then dedupes by issue.id. + // + // AND example: + // + // Query: + // + // issue + // |-- has issue_label(label = 'bug') + // `-- AND has issue_label(label = 'urgent') + // + // Scan plan: + // + // issue_label(label = 'bug') -> ids {10, 20} --. + // +-- ids in both -> issue(id) + // issue_label(label = 'urgent') -> ids {20, 30} ----' + // + // The broad plan would load issues after the first label match, then probe + // the second label row-by-row. This plan first finds the issue ids that + // appear in both child scans, then loads only those issues. + // + // Each rewrite has its own strict guard below. If a query needs a shape the + // new physical operator cannot preserve, it falls through to the older + // generic pipeline. + const rootUnionBranches = getRootUnionBranches(ast); + if (rootUnionBranches) { + return applyRootUnionBranches( + ast, + rootUnionBranches, + delegate, + queryID, + name, + partitionKey, + ); + } + + const intersection = getSameRelationshipExistsIntersection( + ast.where, + delegate, + ); + const sourceWhere = intersection ? undefined : ast.where; + const csqConditions = intersection + ? [] + : gatherCorrelatedSubqueryQueryConditions(ast.where); const splitEditKeys: Set = partitionKey ? new Set(partitionKey) : new Set(); - const aliases = new Set(); for (const csq of csqConditions) { - aliases.add(csq.related.subquery.alias || ''); for (const key of csq.related.correlation.parentField) { splitEditKeys.add(key); } } + if (intersection) { + for (const key of intersection.related.correlation.parentField) { + splitEditKeys.add(key); + } + } if (ast.related) { for (const csq of ast.related) { for (const key of csq.correlation.parentField) { @@ -290,7 +357,7 @@ function buildPipelineInternal( } const conn = source.connect( must(ast.orderBy), - ast.where, + sourceWhere, splitEditKeys, delegate.debug, ); @@ -299,6 +366,15 @@ function buildPipelineInternal( end = delegate.decorateInput(end, `${name}:source(${ast.table})`); const {fullyAppliedFilters} = conn; + if (intersection) { + end = applySameRelationshipExistsIntersection( + intersection, + delegate, + end, + name, + ); + } + if (ast.start) { const skip = new Skip(end, ast.start); delegate.addEdge(end, skip); @@ -328,8 +404,8 @@ function buildPipelineInternal( } } - if (ast.where && (!fullyAppliedFilters || delegate.applyFiltersAnyway)) { - end = applyWhere(end, ast.where, delegate, name); + if (sourceWhere && (!fullyAppliedFilters || delegate.applyFiltersAnyway)) { + end = applyWhere(end, sourceWhere, delegate, name); } if (ast.limit !== undefined) { @@ -358,6 +434,393 @@ function buildPipelineInternal( return end; } +function applyRootUnionBranches( + ast: AST, + branches: readonly Condition[], + delegate: BuilderDelegate, + queryID: string, + name: string, + partitionKey?: CompoundKey, +): Input { + // Run every OR branch as its own root query, then merge by primary key. A + // root is just the table/index we choose to scan first. This is the physical + // equivalent of SQLite's multi-index OR strategy: + // + // Query: + // + // issue.status = 'open' + // OR EXISTS(issue_label WHERE label = 'bug') + // + // Scan plan: + // + // issue(status = 'open') -----------------------. + // +-- union issue.id + // issue_label(label = 'bug') -> issue(id) -------' + // + // Each recursive branch keeps the same ordering and split-edit keys as the + // original AST, so the union can merge streams without re-sorting. + const inputs = branches.map((branch, index) => { + const input = buildPipelineInternal( + { + ...ast, + where: branch, + }, + delegate, + queryID, + `${name}:or-${index}`, + partitionKey, + ); + return stripRootUnionBranchRelationships(input, delegate, name, index); + }); + + const union = new InputUnion(inputs); + for (const input of inputs) { + delegate.addEdge(input, union); + } + return delegate.decorateInput(union, `${name}:input-union`); +} + +function stripRootUnionBranchRelationships( + input: Input, + delegate: BuilderDelegate, + name: string, + index: number, +): Input { + // Root union is a set of parent rows, not a way to expose relationship + // payloads. A flipped EXISTS branch may temporarily attach its child rows so + // the branch can prove the EXISTS is true, but those rows are condition-only + // data. Strip them before the branch joins the union so every branch exposes + // the same plain parent-row schema. + if (Object.keys(input.getSchema().relationships).length === 0) { + return input; + } + const stripped = new StripRelationships(input); + delegate.addEdge(input, stripped); + return delegate.decorateInput(stripped, `${name}:or-${index}:strip-related`); +} + +function getRootUnionBranches(ast: AST): readonly Condition[] | undefined { + // This rewrite is only safe at the root of a plain query. start, limit, and + // related rows all observe the whole result stream, so they need a richer + // physical plan than "run branch pipelines, then union". + if ( + ast.where?.type !== 'or' || + ast.where.conditions.length < 2 || + ast.start !== undefined || + ast.limit !== undefined || + ast.related !== undefined + ) { + return undefined; + } + + const branches = ast.where.conditions; + + // At least one branch must already be planned to start from a related table. + // Otherwise root union would just split a simple parent-table scan into + // smaller scans without buying us anything. + if (!branches.some(conditionIncludesFlippedSubqueryAtAnyLevel)) { + return undefined; + } + + // There must also be at least one local parent branch. If every branch is a + // child branch, the existing UnionFanOut and UnionFanIn path handles it. + if (!branches.some(isNotAndDoesNotContainSubquery)) { + return undefined; + } + + // The normalizer flattens ORs before planning. If a nested OR survives here, + // keep the old path rather than inventing branch semantics locally. + if (branches.some(branch => branch.type === 'or')) { + return undefined; + } + + return branches; +} + +function applySameRelationshipExistsIntersection( + intersection: SameRelationshipExistsIntersection, + delegate: BuilderDelegate, + end: Input, + name: string, +): Input { + // Build the child side before the parent lookup. In plain English: + // find the issue ids that satisfy every sibling EXISTS first, then load the + // issue rows for only those ids. + // + // Query: + // + // issue + // |-- EXISTS(issue_label WHERE label = 'bug') + // `-- EXISTS(issue_label WHERE label = 'urgent') + // + // Scan plan: + // + // issue_label(label = 'bug') -------------------. + // +-- intersect issue_id -> issue(id) + // issue_label(label = 'urgent') ----------------' + // + // This avoids loading an issue after the first label match only to probe the + // second label relationship row-by-row. + const {conditions, related} = intersection; + const childInputs = conditions.map((condition, index) => + buildPipelineInternal( + condition.related.subquery, + delegate, + '', + `${name}.${condition.related.subquery.alias}:intersect-${index}`, + related.correlation.childField, + ), + ); + const child = new InputIntersection( + childInputs, + related.correlation.childField, + ); + for (const childInput of childInputs) { + delegate.addEdge(childInput, child); + } + + const flippedJoin = new FlippedJoin({ + parent: end, + child, + parentKey: related.correlation.parentField, + childKey: related.correlation.childField, + relationshipName: must( + related.subquery.alias, + 'Subquery must have an alias', + ), + hidden: related.hidden ?? false, + system: related.system ?? 'client', + }); + delegate.addEdge(end, flippedJoin); + delegate.addEdge(child, flippedJoin); + return delegate.decorateInput( + flippedJoin, + `${name}:intersect-flipped-join(${related.subquery.alias})`, + ); +} + +type SameRelationshipExistsIntersection = { + readonly related: CorrelatedSubquery; + readonly conditions: readonly CorrelatedSubqueryCondition[]; +}; + +function getSameRelationshipExistsIntersection( + condition: Condition | undefined, + delegate: BuilderDelegate, +): SameRelationshipExistsIntersection | undefined { + // Detect a narrow, physical intersection opportunity: + // + // AND + // EXISTS(relationship R, child predicate A) + // EXISTS(relationship R, child predicate B) + // + // We only need one sibling to be planned as flipped. Once one branch is + // source-driven, intersecting all compatible siblings lets the runtime start + // with the child table and avoid parent-first probing. + if (condition?.type !== 'and') { + return undefined; + } + + const conditions = condition.conditions.map(getIntersectableExists); + if (conditions.some(candidate => candidate === undefined)) { + return undefined; + } + + const candidates = conditions as CorrelatedSubqueryCondition[]; + if (candidates.length < 2) { + return undefined; + } + + // Respect explicit user intent. flip: false means "keep this semi-join". + // Undefined still means the planner may decide, so it can join a group where + // another sibling was chosen as flipped. + if (!candidates.some(candidate => candidate.flip === true)) { + return undefined; + } + + const fingerprint = sameRelationshipExistsFingerprint(candidates[0]); + if ( + !fingerprint || + candidates.some( + candidate => sameRelationshipExistsFingerprint(candidate) !== fingerprint, + ) + ) { + return undefined; + } + + const childSource = delegate.getSource(candidates[0].related.subquery.table); + if (!childSource) { + return undefined; + } + + // InputIntersection works with ids, not duplicate child rows: "which parent + // ids appeared in every related-table scan?" Because the operator keeps one + // child row for each parent id, each branch must prove it can produce at + // most one child row for the id it contributes. + // + // issue_label primary key: + // + // [issue_id, label] + // + // Branch filter: + // + // issue_id comes from the parent relationship + // label = 'bug' + // + // Together those two values cover the full primary key, so this branch can + // produce at most one issue_label row for each issue_id. + if ( + candidates.some( + candidate => + !isUniquePerCorrelationKey( + candidate, + childSource.tableSchema.primaryKey, + ), + ) + ) { + return undefined; + } + + return { + related: candidates[0].related, + conditions: candidates, + }; +} + +function getIntersectableExists( + condition: Condition, +): CorrelatedSubqueryCondition | undefined { + // The AND intersection rewrite is only valid for this simple shape: + // + // EXISTS(related table) + // where related-table filters have no nested EXISTS + // with no related rows, cursor start, or limit inside the EXISTS + // + // Nested relationships, cursors, and limits can make "does this parent key + // exist?" depend on more than the related table's filter. In that world, + // intersecting related-table ids could skip rows that the original sibling + // EXISTS checks would have accepted. + const exists = asCorrelatedSubqueryCondition(condition); + if (!exists) { + return undefined; + } + if (!isPlainExistsBranch(exists)) { + return undefined; + } + if (!hasIntersectableChildSubquery(exists)) { + return undefined; + } + return exists; +} + +function asCorrelatedSubqueryCondition( + condition: Condition, +): CorrelatedSubqueryCondition | undefined { + return condition.type === 'correlatedSubquery' ? condition : undefined; +} + +function isPlainExistsBranch(condition: CorrelatedSubqueryCondition): boolean { + return ( + condition.op === 'EXISTS' && + condition.scalar !== true && + condition.flip !== false + ); +} + +function hasIntersectableChildSubquery( + condition: CorrelatedSubqueryCondition, +): boolean { + const {subquery} = condition.related; + return ( + subquery.related === undefined && + subquery.start === undefined && + subquery.limit === undefined && + (subquery.where === undefined || + isNotAndDoesNotContainSubquery(subquery.where)) + ); +} + +function sameRelationshipExistsFingerprint( + condition: CorrelatedSubqueryCondition, +): string | undefined { + const exists = getIntersectableExists(condition); + if (!exists) { + return undefined; + } + + const {related} = exists; + // Include orderBy because SourceSchema.sort must match across every child + // input in the intersection. EXISTS does not care about child ordering + // semantically, but the runtime stream contract does. + return JSON.stringify({ + system: related.system, + hidden: related.hidden, + correlation: related.correlation, + orderBy: related.subquery.orderBy, + subquery: { + schema: related.subquery.schema, + table: related.subquery.table, + }, + }); +} + +function isUniquePerCorrelationKey( + condition: CorrelatedSubqueryCondition, + childPrimaryKey: readonly string[], +): boolean { + // Prove that a child scan contributes at most one row per parent id: + // + // parent id field + literal equality filters cover child PK + // + // Example: + // + // issue_label primary key: + // + // [issue_id, label] + // + // Branch filter: + // + // issue_id comes from the parent relationship + // label = 'bug' + // + // Together those values identify one issue_label row. That makes + // intersecting by issue_id equivalent to asking whether both EXISTS branches + // are true. + const constrained = new Set(condition.related.correlation.childField); + collectEqualityConstrainedColumns( + condition.related.subquery.where, + constrained, + ); + return childPrimaryKey.every(key => constrained.has(key)); +} + +function collectEqualityConstrainedColumns( + condition: Condition | undefined, + constrained: Set, +): void { + // This is intentionally conservative. Only literal equality predicates prove + // uniqueness. IN, OR, scalar subqueries, and nested EXISTS can still be + // optimized later, but they need a richer proof than this small helper. + if (!condition) { + return; + } + if (condition.type === 'simple') { + if ( + condition.op === '=' && + condition.left.type === 'column' && + condition.right.type === 'literal' + ) { + constrained.add(condition.left.name); + } + return; + } + if (condition.type === 'and') { + for (const child of condition.conditions) { + collectEqualityConstrainedColumns(child, constrained); + } + } +} + function applyWhere( input: Input, condition: Condition, @@ -420,19 +883,18 @@ function applyFilterWithFlips( const branches: Input[] = []; if (withoutFlipped.length > 0) { - branches.push( - buildFilterPipeline(end, delegate, filterInput => - applyOr( - filterInput, - { - type: 'or', - conditions: withoutFlipped, - }, - delegate, - name, - ), + const branch = buildFilterPipeline(end, delegate, filterInput => + applyOr( + filterInput, + { + type: 'or', + conditions: withoutFlipped, + }, + delegate, + name, ), ); + branches.push(branch); } for (const cond of withFlipped) { diff --git a/packages/zql/src/ivm/set-operators.test.ts b/packages/zql/src/ivm/set-operators.test.ts new file mode 100644 index 0000000000..8afb5de982 --- /dev/null +++ b/packages/zql/src/ivm/set-operators.test.ts @@ -0,0 +1,463 @@ +import {describe, expect, test} from 'vitest'; +import type {Row} from '../../../zero-protocol/src/data.ts'; +import {ChangeIndex} from './change-index.ts'; +import {ChangeType} from './change-type.ts'; +import { + makeAddChange, + makeChildChange, + makeEditChange, + makeRemoveChange, + type Change, +} from './change.ts'; +import type {Node} from './data.ts'; +import { + skipYields, + type FetchRequest, + type Input, + type Output, +} from './operator.ts'; +import type {SourceSchema} from './schema.ts'; +import {InputIntersection, InputUnion} from './set-operators.ts'; +import {consume, type Stream} from './stream.ts'; +import {StripRelationships} from './strip-relationships.ts'; + +const schema: SourceSchema = { + tableName: 'assignment', + columns: { + id: {type: 'number'}, + value: {type: 'number'}, + }, + primaryKey: ['id'], + relationships: {}, + compareRows: (left, right) => (left.id as number) - (right.id as number), + isHidden: false, + sort: [['id', 'asc']], + system: 'client', +}; + +describe('InputUnion', () => { + test('fetch merges sorted inputs and deduplicates by primary key', () => { + const left = new MutableInput([ + node({id: 1, value: 10}), + node({id: 3, value: 30}), + ]); + const right = new MutableInput([ + node({id: 2, value: 20}), + node({id: 3, value: 30}), + ]); + + const union = new InputUnion([left, right]); + + expect(Array.from(skipYields(union.fetch({})), n => n.row.id)).toEqual([ + 1, 2, 3, + ]); + }); + + test('fetch merges reversed inputs with the reversed comparator', () => { + const left = new MutableInput([ + node({id: 1, value: 10}), + node({id: 3, value: 30}), + ]); + const right = new MutableInput([ + node({id: 2, value: 20}), + node({id: 3, value: 30}), + node({id: 4, value: 40}), + ]); + + const union = new InputUnion([left, right]); + + expect( + Array.from(skipYields(union.fetch({reverse: true})), n => n.row.id), + ).toEqual([4, 3, 2, 1]); + }); + + test('push emits one add, remove, or edit when multiple branches match', () => { + const left = new MutableInput([]); + const right = new MutableInput([]); + const union = new InputUnion([left, right]); + const sink = new RecordingOutput(); + union.setOutput(sink); + + left.rows = [node({id: 1, value: 10})]; + left.push(makeAddChange(node({id: 1, value: 10}))); + right.rows = [node({id: 1, value: 10})]; + right.push(makeAddChange(node({id: 1, value: 10}))); + + left.rows = [node({id: 1, value: 11})]; + left.push( + makeEditChange(node({id: 1, value: 11}), node({id: 1, value: 10})), + ); + right.rows = [node({id: 1, value: 11})]; + right.push( + makeEditChange(node({id: 1, value: 11}), node({id: 1, value: 10})), + ); + + left.rows = []; + left.push(makeRemoveChange(node({id: 1, value: 11}))); + right.rows = []; + right.push(makeRemoveChange(node({id: 1, value: 11}))); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.ADD, + ChangeType.EDIT, + ChangeType.REMOVE, + ]); + }); + + test('push emits primary key edits as remove and add', () => { + const left = new MutableInput([node({id: 1, value: 10})]); + const right = new MutableInput([]); + const union = new InputUnion([left, right]); + const sink = new RecordingOutput(); + union.setOutput(sink); + + left.rows = [node({id: 2, value: 10})]; + left.push( + makeEditChange(node({id: 2, value: 10}), node({id: 1, value: 10})), + ); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.REMOVE, + ChangeType.ADD, + ]); + expect(sink.changes.map(change => change[ChangeIndex.NODE].row.id)).toEqual( + [1, 2], + ); + }); + + test('push emits an edit when representative ownership moves earlier', () => { + const left = new MutableInput([]); + const right = new MutableInput([node({id: 1, value: 20})]); + const union = new InputUnion([left, right]); + const sink = new RecordingOutput(); + union.setOutput(sink); + + left.rows = [node({id: 1, value: 10})]; + left.push(makeAddChange(node({id: 1, value: 10}))); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.EDIT, + ]); + const [change] = sink.changes; + expect(change).toBeDefined(); + if (!change) { + throw new Error('Expected change'); + } + expect(change[ChangeIndex.TYPE]).toBe(ChangeType.EDIT); + if (change[ChangeIndex.TYPE] !== ChangeType.EDIT) { + throw new Error('Expected edit'); + } + expect(change[ChangeIndex.NODE].row).toEqual({ + id: 1, + value: 10, + }); + expect(change[ChangeIndex.OLD_NODE].row).toEqual({ + id: 1, + value: 20, + }); + }); + + test('push emits an edit when representative ownership moves later', () => { + const left = new MutableInput([node({id: 1, value: 10})]); + const right = new MutableInput([node({id: 1, value: 20})]); + const union = new InputUnion([left, right]); + const sink = new RecordingOutput(); + union.setOutput(sink); + + left.rows = []; + left.push(makeRemoveChange(node({id: 1, value: 10}))); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.EDIT, + ]); + const [change] = sink.changes; + expect(change).toBeDefined(); + if (!change) { + throw new Error('Expected change'); + } + expect(change[ChangeIndex.TYPE]).toBe(ChangeType.EDIT); + if (change[ChangeIndex.TYPE] !== ChangeType.EDIT) { + throw new Error('Expected edit'); + } + expect(change[ChangeIndex.NODE].row).toEqual({ + id: 1, + value: 20, + }); + expect(change[ChangeIndex.OLD_NODE].row).toEqual({ + id: 1, + value: 10, + }); + }); + + test('push forwards child changes from the earliest matching branch only', () => { + const left = new MutableInput([node({id: 1, value: 10})]); + const right = new MutableInput([node({id: 1, value: 10})]); + const union = new InputUnion([left, right]); + const sink = new RecordingOutput(); + union.setOutput(sink); + + left.push(childChange(node({id: 1, value: 10}))); + right.push(childChange(node({id: 1, value: 10}))); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.CHILD, + ]); + }); + + test('requires relationship-free branch streams', () => { + const related = new MutableInput([], { + ...schema, + relationships: {children: schema}, + }); + + expect(() => new InputUnion([related])).toThrow( + 'input union requires inputs without relationships', + ); + }); +}); + +describe('StripRelationships', () => { + test('removes relationship schema and fetched relationship payloads', () => { + const input = new MutableInput( + [ + node( + {id: 1, value: 10}, + { + children: () => [node({id: 99, value: 99})], + }, + ), + ], + { + ...schema, + relationships: {children: schema}, + }, + ); + + const stripped = new StripRelationships(input); + + expect(stripped.getSchema().relationships).toEqual({}); + expect( + Array.from(skipYields(stripped.fetch({})), node => node.relationships), + ).toEqual([{}]); + }); + + test('ignores relationship-only pushes', () => { + const input = new MutableInput([], { + ...schema, + relationships: {children: schema}, + }); + const stripped = new StripRelationships(input); + const sink = new RecordingOutput(); + stripped.setOutput(sink); + + input.push(childChange(node({id: 1, value: 10}))); + + expect(sink.changes).toEqual([]); + }); +}); + +describe('InputIntersection', () => { + test('fetch keeps rows whose key appears in every input', () => { + const left = new MutableInput([ + node({id: 1, value: 10}), + node({id: 2, value: 20}), + ]); + const right = new MutableInput([ + node({id: 2, value: 200}), + node({id: 3, value: 300}), + ]); + + const intersection = new InputIntersection([left, right], ['id']); + + expect(Array.from(skipYields(intersection.fetch({})), n => n.row)).toEqual([ + {id: 2, value: 20}, + ]); + }); + + test('fetch deduplicates rows by intersection key', () => { + const left = new MutableInput([ + node({id: 1, value: 10}), + node({id: 1, value: 11}), + node({id: 2, value: 20}), + ]); + const right = new MutableInput([node({id: 1, value: 100})]); + + const intersection = new InputIntersection([left, right], ['id']); + + expect(Array.from(skipYields(intersection.fetch({})), n => n.row)).toEqual([ + {id: 1, value: 10}, + ]); + }); + + test('fetch streams the representative branch after reading other key sets', () => { + const events: string[] = []; + const left = new LoggingInput( + 'first', + [node({id: 1, value: 10}), node({id: 2, value: 20})], + events, + ); + const right = new LoggingInput( + 'other', + [node({id: 2, value: 200})], + events, + ); + const intersection = new InputIntersection([left, right], ['id']); + + const iter = intersection.fetch({})[Symbol.iterator](); + const result = iter.next(); + + expect(result.done).toBe(false); + expect(result.value).toEqual(node({id: 2, value: 20})); + expect(events).toEqual(['other:start', 'other:end', 'first:start']); + }); + + test('push emits when a key enters or leaves the intersection', () => { + const left = new MutableInput([node({id: 1, value: 10})]); + const right = new MutableInput([]); + const intersection = new InputIntersection([left, right], ['id']); + const sink = new RecordingOutput(); + intersection.setOutput(sink); + + right.rows = [node({id: 1, value: 100})]; + right.push(makeAddChange(node({id: 1, value: 100}))); + + right.rows = []; + right.push(makeRemoveChange(node({id: 1, value: 100}))); + + expect(sink.changes).toHaveLength(2); + expect(sink.changes[0][ChangeIndex.TYPE]).toBe(ChangeType.ADD); + expect(sink.changes[0][ChangeIndex.NODE].row).toEqual({ + id: 1, + value: 10, + }); + expect(sink.changes[1][ChangeIndex.TYPE]).toBe(ChangeType.REMOVE); + expect(sink.changes[1][ChangeIndex.NODE].row).toEqual({ + id: 1, + value: 10, + }); + }); + + test('push emits remove and add when a non-primary branch moves keys', () => { + const left = new MutableInput([ + node({id: 1, value: 10}), + node({id: 2, value: 20}), + ]); + const right = new MutableInput([node({id: 1, value: 100})]); + const intersection = new InputIntersection([left, right], ['id']); + const sink = new RecordingOutput(); + intersection.setOutput(sink); + + right.rows = [node({id: 2, value: 100})]; + right.push( + makeEditChange(node({id: 2, value: 100}), node({id: 1, value: 100})), + ); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.REMOVE, + ChangeType.ADD, + ]); + expect(sink.changes.map(change => change[ChangeIndex.NODE].row)).toEqual([ + {id: 1, value: 10}, + {id: 2, value: 20}, + ]); + }); + + test('push forwards child changes from the representative branch only', () => { + const left = new MutableInput([node({id: 1, value: 10})]); + const right = new MutableInput([node({id: 1, value: 100})]); + const intersection = new InputIntersection([left, right], ['id']); + const sink = new RecordingOutput(); + intersection.setOutput(sink); + + right.push(childChange(node({id: 1, value: 100}))); + left.push(childChange(node({id: 1, value: 10}))); + + expect(sink.changes.map(change => change[ChangeIndex.TYPE])).toEqual([ + ChangeType.CHILD, + ]); + expect(sink.changes[0][ChangeIndex.NODE].row).toEqual({ + id: 1, + value: 10, + }); + }); +}); + +class MutableInput implements Input { + rows: Node[]; + readonly #schema: SourceSchema; + #output: Output | undefined; + + constructor(rows: Node[], sourceSchema: SourceSchema = schema) { + this.rows = rows; + this.#schema = sourceSchema; + } + + setOutput(output: Output): void { + this.#output = output; + } + + getSchema(): SourceSchema { + return this.#schema; + } + + *fetch(req: FetchRequest): Stream { + const rows = req.reverse ? this.rows.toReversed() : this.rows; + for (const row of rows) { + if (matchesConstraint(row.row, req.constraint)) { + yield row; + } + } + } + + push(change: Change): void { + consume(this.#output?.push(change, this) ?? []); + } + + destroy(): void {} +} + +class LoggingInput extends MutableInput { + readonly #label: string; + readonly #events: string[]; + + constructor(label: string, rows: Node[], events: string[]) { + super(rows); + this.#label = label; + this.#events = events; + } + + override *fetch(req: FetchRequest): Stream { + this.#events.push(`${this.#label}:start`); + yield* super.fetch(req); + this.#events.push(`${this.#label}:end`); + } +} + +class RecordingOutput implements Output { + readonly changes: Change[] = []; + + *push(change: Change): Stream<'yield'> { + this.changes.push(change); + } +} + +function node(row: Row, relationships: Node['relationships'] = {}): Node { + return {row, relationships}; +} + +function childChange(parent: Node): Change { + return makeChildChange(parent, { + relationshipName: 'children', + change: makeAddChange(node({id: 99, value: 99})), + }); +} + +function matchesConstraint( + row: Row, + constraint: FetchRequest['constraint'], +): boolean { + if (!constraint) { + return true; + } + return Object.entries(constraint).every(([key, value]) => row[key] === value); +} diff --git a/packages/zql/src/ivm/set-operators.ts b/packages/zql/src/ivm/set-operators.ts new file mode 100644 index 0000000000..4f79c0feb3 --- /dev/null +++ b/packages/zql/src/ivm/set-operators.ts @@ -0,0 +1,538 @@ +import {areEqual} from '../../../shared/src/arrays.ts'; +import {assert} from '../../../shared/src/asserts.ts'; +import type {Writable} from '../../../shared/src/writable.ts'; +import type {CompoundKey} from '../../../zero-protocol/src/ast.ts'; +import type {Row, Value} from '../../../zero-protocol/src/data.ts'; +import {ChangeIndex} from './change-index.ts'; +import {ChangeType} from './change-type.ts'; +import { + makeAddChange, + makeEditChange, + makeRemoveChange, + type Change, +} from './change.ts'; +import type {Node} from './data.ts'; +import { + throwOutput, + type FetchRequest, + type Input, + type InputBase, + type Output, +} from './operator.ts'; +import type {SourceSchema} from './schema.ts'; +import {type Stream} from './stream.ts'; +import {mergeFetches} from './union-fan-in.ts'; + +/** + * Merges the results of an OR query that has more than one good starting point. + * + * Query: + * + * issue + * |-- status = 'open' + * `-- OR has issue_label(label = 'bug') + * + * Scan plan: + * + * branch 0: issue(status = 'open') ----------------. + * InputUnion -> issue rows + * branch 1: issue_label(label = 'bug') -> issue(id) -' + * + * InputUnion streams those issue rows in final sort order and removes duplicate + * issue ids. If the same issue appears in both branches, the earlier branch is + * the visible copy. Push has to preserve that same rule, including handoffs: + * + * before push: + * branch 0: empty + * branch 1: {id: 1, value: 20} + * output: {id: 1, value: 20} + * + * after branch 0 adds {id: 1, value: 10}: + * branch 0: {id: 1, value: 10} + * branch 1: {id: 1, value: 20} + * output: {id: 1, value: 10} + * + * The visible result did not gain a new primary key. The visible copy changed, + * so downstream receives EDIT(value 20 -> 10). + * + * This operator intentionally accepts only plain row streams with no + * relationships. The builder strips condition-only relationship payloads from + * branch pipelines before they enter the union. That keeps the schema honest: + * every node emitted by the union has the same shape that the union advertises. + */ +export class InputUnion implements Input { + readonly #inputs: readonly Input[]; + readonly #inputIndexes: ReadonlyMap; + readonly #schema: SourceSchema; + #output: Output = throwOutput; + + constructor(inputs: readonly Input[]) { + assert(inputs.length > 0, 'InputUnion requires at least one input'); + this.#inputs = inputs; + this.#inputIndexes = new Map(inputs.map((input, index) => [input, index])); + this.#schema = mergeInputSchemas('input union', inputs); + for (const input of inputs) { + input.setOutput(this); + } + } + + setOutput(output: Output): void { + this.#output = output; + } + + getSchema(): SourceSchema { + return this.#schema; + } + + fetch(req: FetchRequest): Stream { + const compareRows = req.reverse + ? (left: Node, right: Node) => + this.#schema.compareRows(right.row, left.row) + : (left: Node, right: Node) => + this.#schema.compareRows(left.row, right.row); + return mergeFetches( + this.#inputs.map(input => input.fetch(req)), + compareRows, + ); + } + + *push(change: Change, pusher: InputBase): Stream<'yield'> { + const pusherIndex = this.#inputIndexes.get(pusher); + assert(pusherIndex !== undefined, 'Pusher was not an input union input'); + + switch (change[ChangeIndex.TYPE]) { + case ChangeType.ADD: + yield* this.#pushAdd(change[ChangeIndex.NODE], pusher, pusherIndex); + return; + + case ChangeType.REMOVE: + yield* this.#pushRemove(change[ChangeIndex.NODE], pusher, pusherIndex); + return; + + case ChangeType.EDIT: { + const oldNode = change[ChangeIndex.OLD_NODE]; + const newNode = change[ChangeIndex.NODE]; + if ( + rowKey(oldNode.row, this.#schema.primaryKey) !== + rowKey(newNode.row, this.#schema.primaryKey) + ) { + yield* this.#pushRemove(oldNode, pusher, pusherIndex); + yield* this.#pushAdd(newNode, pusher, pusherIndex); + return; + } + + if (yield* this.#earlierInputHasMatch(pusherIndex, newNode)) { + return; + } + yield* this.#output.push(change, this); + return; + } + + case ChangeType.CHILD: + if ( + yield* this.#earlierInputHasMatch( + pusherIndex, + change[ChangeIndex.NODE], + ) + ) { + return; + } + yield* this.#output.push(change, this); + return; + } + } + + destroy(): void { + for (const input of this.#inputs) { + input.destroy(); + } + } + + *#pushAdd( + node: Node, + pusher: InputBase, + pusherIndex: number, + ): Generator<'yield'> { + // ADD has three cases: + // + // no other branch has pk => new visible row, emit ADD + // later branch has pk => visible-copy handoff, emit EDIT + // earlier branch has pk => still represented earlier, emit nothing + const match = yield* this.#firstMatchingInputExcept(pusher, node); + if (!match) { + yield* this.#output.push(makeAddChange(node), this); + return; + } + if (pusherIndex < match.index && !rowsEqual(node.row, match.node.row)) { + yield* this.#output.push(makeEditChange(node, match.node), this); + } + } + + *#pushRemove( + node: Node, + pusher: InputBase, + pusherIndex: number, + ): Generator<'yield'> { + // REMOVE mirrors ADD: + // + // no other branch has pk => row disappeared, emit REMOVE + // later branch has pk => visible-copy handoff, emit EDIT + // earlier branch has pk => still represented earlier, emit nothing + const match = yield* this.#firstMatchingInputExcept(pusher, node); + if (!match) { + yield* this.#output.push(makeRemoveChange(node), this); + return; + } + if (pusherIndex < match.index && !rowsEqual(node.row, match.node.row)) { + yield* this.#output.push(makeEditChange(match.node, node), this); + } + } + + *#firstMatchingInputExcept( + pusher: InputBase, + node: Node, + ): Generator< + 'yield', + {readonly index: number; readonly node: Node} | undefined + > { + // Search in input order because fetch uses input order as the tie-break for + // duplicate primary keys. Push must discover the same visible copy. + const constraint = keyConstraint(node.row, this.#schema.primaryKey); + for (const [index, input] of this.#inputs.entries()) { + if (input === pusher) { + continue; + } + const matchingNode = yield* firstMatchingNode(input, constraint); + if (matchingNode) { + return {index, node: matchingNode}; + } + } + return undefined; + } + + *#earlierInputHasMatch( + pusherIndex: number, + node: Node, + ): Generator<'yield', boolean> { + const constraint = keyConstraint(node.row, this.#schema.primaryKey); + for (const input of this.#inputs.slice(0, pusherIndex)) { + if (yield* inputHasMatch(input, constraint)) { + return true; + } + } + return false; + } +} + +/** + * Keeps only parent ids that appear in every required related-table scan. + * + * Query: + * + * issue + * |-- has issue_label(label = 'bug') + * `-- AND has issue_label(label = 'urgent') + * + * Scan plan: + * + * issue_label(label = 'bug') -> ids {10, 20} --. + * InputIntersection -> id {20} + * issue_label(label = 'urgent') -> ids {20, 30} ----' + * | + * v + * issue(id = 20) + * + * The important idea is simple: do the cheap child-table lookups first, keep + * only ids found in every branch, then fetch the parent rows. The builder only + * creates this operator when each child branch can produce at most one row for + * a given parent id. That keeps "one row representing issue id 20" equivalent + * to EXISTS semantics. If a future caller wants arbitrary many rows per parent + * id, this operator would need to track duplicate child rows instead of just + * ids. + */ +export class InputIntersection implements Input { + readonly #inputs: readonly Input[]; + readonly #key: CompoundKey; + readonly #schema: SourceSchema; + #output: Output = throwOutput; + + constructor(inputs: readonly Input[], key: CompoundKey) { + assert(inputs.length > 0, 'InputIntersection requires at least one input'); + this.#inputs = inputs; + this.#key = key; + this.#schema = firstInputSchema('input intersection', inputs); + for (const input of inputs) { + input.setOutput(this); + } + } + + setOutput(output: Output): void { + this.#output = output; + } + + getSchema(): SourceSchema { + return this.#schema; + } + + *fetch(req: FetchRequest): Stream { + // Fetch the other branch id sets first, then stream rows from the first + // branch in its own order. We intentionally materialize id sets for the + // non-representative branches, not full rows from the first branch: + // + // rest branches -> Set(parent id) + // | + // v + // first branch stream -> keep ids present in every Set + // + // yieldedKeys is the set part of the contract: even if the first branch + // has duplicate rows for an id, the intersection emits that id only once + // for the parent lookup. + const [first, ...rest] = this.#inputs; + const matchingKeys: ReadonlySet[] = []; + for (const input of rest) { + const keys = new Set(); + for (const node of input.fetch(req)) { + if (node === 'yield') { + yield node; + continue; + } + keys.add(rowKey(node.row, this.#key)); + } + matchingKeys.push(keys); + } + + const yieldedKeys = new Set(); + for (const node of first.fetch(req)) { + if (node === 'yield') { + yield node; + continue; + } + const key = rowKey(node.row, this.#key); + if (!yieldedKeys.has(key) && matchingKeys.every(keys => keys.has(key))) { + yieldedKeys.add(key); + yield node; + } + } + } + + *push(change: Change, pusher: InputBase): Stream<'yield'> { + assert(isInput(pusher), 'Expected pusher to be an input'); + assert(this.#inputs.includes(pusher), 'Pusher was not an input'); + + switch (change[ChangeIndex.TYPE]) { + case ChangeType.ADD: + yield* this.#pushAdd(change[ChangeIndex.NODE], pusher); + return; + + case ChangeType.REMOVE: + yield* this.#pushRemove(change[ChangeIndex.NODE], pusher); + return; + + case ChangeType.EDIT: { + const oldNode = change[ChangeIndex.OLD_NODE]; + const newNode = change[ChangeIndex.NODE]; + if (rowKey(oldNode.row, this.#key) !== rowKey(newNode.row, this.#key)) { + yield* this.#pushRemove(oldNode, pusher); + yield* this.#pushAdd(newNode, pusher); + return; + } + + if ( + pusher === this.#inputs[0] && + (yield* this.#allOtherInputsHaveMatch(pusher, newNode)) + ) { + yield* this.#output.push(change, this); + } + return; + } + + case ChangeType.CHILD: + if ( + pusher === this.#inputs[0] && + (yield* this.#allOtherInputsHaveMatch( + pusher, + change[ChangeIndex.NODE], + )) + ) { + yield* this.#output.push(change, this); + } + return; + } + } + + destroy(): void { + for (const input of this.#inputs) { + input.destroy(); + } + } + + *#pushAdd(node: Node, pusher: Input): Generator<'yield'> { + // An id enters the intersection only when every other branch has that id. + // Non-first branches emit the first input's row, because fetch always uses + // the first branch as the visible copy for an intersected id. + if (!(yield* this.#allOtherInputsHaveMatch(pusher, node))) { + return; + } + if (pusher === this.#inputs[0]) { + yield* this.#output.push(makeAddChange(node), this); + return; + } + const visibleCopy = yield* firstMatchingNode( + this.#inputs[0], + keyConstraint(node.row, this.#key), + ); + if (visibleCopy) { + yield* this.#output.push(makeAddChange(visibleCopy), this); + } + } + + *#pushRemove(node: Node, pusher: Input): Generator<'yield'> { + // An id leaves the intersection when this branch no longer has it and all + // other branches still do. For the first branch, removing the current + // visible copy is visible even if another first-branch row with the same + // id remains, because fetch would have emitted the removed row before the + // change. + const constraint = keyConstraint(node.row, this.#key); + if ( + pusher !== this.#inputs[0] && + (yield* inputHasMatch(pusher, constraint)) + ) { + return; + } + if (!(yield* this.#allOtherInputsHaveMatch(pusher, node))) { + return; + } + if (pusher === this.#inputs[0]) { + yield* this.#output.push(makeRemoveChange(node), this); + return; + } + const visibleCopy = yield* firstMatchingNode(this.#inputs[0], constraint); + if (visibleCopy) { + yield* this.#output.push(makeRemoveChange(visibleCopy), this); + } + } + + *#allOtherInputsHaveMatch( + pusher: InputBase, + node: Node, + ): Generator<'yield', boolean> { + // This tests id presence, not row equality. EXISTS only cares whether each + // branch can produce at least one related row for the parent id. + const constraint = keyConstraint(node.row, this.#key); + for (const input of this.#inputs) { + if (input === pusher) { + continue; + } + if (!(yield* inputHasMatch(input, constraint))) { + return false; + } + } + return true; + } +} + +function mergeInputSchemas( + operatorName: string, + inputs: readonly Input[], +): SourceSchema { + // InputUnion dedupes duplicate parent rows by choosing one visible copy. It + // cannot honestly merge relationship payloads from multiple copies, so it + // refuses relationship-bearing inputs. Callers that use relationships as + // private filter evidence must strip them before constructing the union. + const schema = { + ...firstInputSchema(operatorName, inputs), + relationships: {}, + } satisfies Writable; + + for (const input of inputs) { + const inputSchema = input.getSchema(); + assertCompatibleSchema(operatorName, schema, inputSchema); + assert( + Object.keys(inputSchema.relationships).length === 0, + `${operatorName} requires inputs without relationships`, + ); + } + + return schema; +} + +function firstInputSchema( + operatorName: string, + inputs: readonly Input[], +): SourceSchema { + const schema = inputs[0].getSchema(); + for (const input of inputs.slice(1)) { + assertCompatibleSchema(operatorName, schema, input.getSchema()); + } + return schema; +} + +function assertCompatibleSchema( + operatorName: string, + expected: SourceSchema, + actual: SourceSchema, +): void { + assert( + expected.tableName === actual.tableName, + `Table name mismatch in ${operatorName}: ${expected.tableName} !== ${actual.tableName}`, + ); + assert( + areEqual(expected.primaryKey, actual.primaryKey), + `Primary key mismatch in ${operatorName}`, + ); + assert( + expected.system === actual.system, + `System mismatch in ${operatorName}: ${expected.system} !== ${actual.system}`, + ); + assert( + JSON.stringify(expected.sort) === JSON.stringify(actual.sort), + `Sort mismatch in ${operatorName}`, + ); +} + +function keyConstraint( + row: Row, + key: readonly string[], +): Record { + return Object.fromEntries(key.map(column => [column, row[column]])); +} + +function rowKey(row: Row, key: readonly string[]): string { + return JSON.stringify(key.map(column => row[column])); +} + +function rowsEqual(left: Row, right: Row): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + +function isInput(input: InputBase): input is Input { + return 'fetch' in input; +} + +function* inputHasMatch( + input: Input, + constraint: Record, +): Generator<'yield', boolean> { + for (const node of input.fetch({constraint})) { + if (node === 'yield') { + yield node; + continue; + } + return true; + } + return false; +} + +function* firstMatchingNode( + input: Input, + constraint: Record, +): Generator<'yield', Node | undefined> { + for (const node of input.fetch({constraint})) { + if (node === 'yield') { + yield node; + continue; + } + return node; + } + return undefined; +} diff --git a/packages/zql/src/ivm/strip-relationships.ts b/packages/zql/src/ivm/strip-relationships.ts new file mode 100644 index 0000000000..147b29cff8 --- /dev/null +++ b/packages/zql/src/ivm/strip-relationships.ts @@ -0,0 +1,110 @@ +import type {Writable} from '../../../shared/src/writable.ts'; +import {ChangeIndex} from './change-index.ts'; +import {ChangeType} from './change-type.ts'; +import { + makeAddChange, + makeEditChange, + makeRemoveChange, + type Change, +} from './change.ts'; +import type {Node} from './data.ts'; +import { + throwOutput, + type FetchRequest, + type Input, + type InputBase, + type Operator, + type Output, +} from './operator.ts'; +import type {SourceSchema} from './schema.ts'; +import {type Stream} from './stream.ts'; + +/** + * Removes relationship payloads from a row stream. + * + * Some physical plans use hidden relationships as proof that a filter matched. + * For example, a flipped EXISTS branch first joins from the child table back to + * the parent, which naturally attaches the matching child rows to the parent + * node. If that branch later feeds a root-level union, those child rows are + * only evidence for the branch. They are not part of the query result. + * + * This operator draws that boundary explicitly: + * + * branch proves EXISTS with child rows + * | + * v + * StripRelationships + * | + * v + * plain parent rows enter the union + */ +export class StripRelationships implements Operator { + readonly #input: Input; + readonly #schema: SourceSchema; + #output: Output = throwOutput; + + constructor(input: Input) { + this.#input = input; + this.#schema = { + ...input.getSchema(), + relationships: {}, + } satisfies Writable; + input.setOutput(this); + } + + setOutput(output: Output): void { + this.#output = output; + } + + getSchema(): SourceSchema { + return this.#schema; + } + + *fetch(req: FetchRequest): Stream { + for (const node of this.#input.fetch(req)) { + yield node === 'yield' ? node : stripRelationships(node); + } + } + + *push(change: Change, _pusher: InputBase): Stream<'yield'> { + switch (change[ChangeIndex.TYPE]) { + case ChangeType.ADD: + yield* this.#output.push( + makeAddChange(stripRelationships(change[ChangeIndex.NODE])), + this, + ); + return; + + case ChangeType.REMOVE: + yield* this.#output.push( + makeRemoveChange(stripRelationships(change[ChangeIndex.NODE])), + this, + ); + return; + + case ChangeType.EDIT: { + const oldNode = stripRelationships(change[ChangeIndex.OLD_NODE]); + const newNode = stripRelationships(change[ChangeIndex.NODE]); + if (JSON.stringify(oldNode.row) === JSON.stringify(newNode.row)) { + return; + } + yield* this.#output.push(makeEditChange(newNode, oldNode), this); + return; + } + + case ChangeType.CHILD: + return; + } + } + + destroy(): void { + this.#input.destroy(); + } +} + +function stripRelationships(node: Node): Node { + return { + row: node.row, + relationships: {}, + }; +} diff --git a/packages/zql/src/ivm/union-fan-in.ts b/packages/zql/src/ivm/union-fan-in.ts index e0bc801a53..4e7123214a 100644 --- a/packages/zql/src/ivm/union-fan-in.ts +++ b/packages/zql/src/ivm/union-fan-in.ts @@ -15,6 +15,7 @@ import { } from './operator.ts'; import { makeAddEmptyRelationships, + mergeEmpty, mergeRelationships, pushAccumulatedChanges, } from './push-accumulated.ts'; @@ -102,8 +103,19 @@ export class UnionFanIn implements Operator { fetch(req: FetchRequest): Stream { const iterables = this.#inputs.map(input => input.fetch(req)); - return mergeFetches(iterables, (l, r) => - this.#schema.compareRows(l.row, r.row), + const compareRows = req.reverse + ? (left: Node, right: Node) => + this.#schema.compareRows(right.row, left.row) + : (left: Node, right: Node) => + this.#schema.compareRows(left.row, right.row); + const relationshipNames = Object.keys(this.#schema.relationships); + return addEmptyRelationshipsToFetch( + mergeFetches( + iterables, + compareRows, + mergeNodeRelationships(relationshipNames), + ), + relationshipNames, ); } @@ -219,24 +231,58 @@ export class UnionFanIn implements Operator { } } +function mergeNodeRelationships(relationshipNames: string[]) { + return (left: Node, right: Node): Node => { + const relationships = { + ...right.relationships, + ...left.relationships, + }; + mergeEmpty(relationships, relationshipNames); + return { + row: left.row, + relationships, + }; + }; +} + +function* addEmptyRelationshipsToFetch( + fetch: Iterable, + relationshipNames: string[], +): Stream { + for (const node of fetch) { + if (node === 'yield') { + yield node; + continue; + } + const relationships = {...node.relationships}; + mergeEmpty(relationships, relationshipNames); + yield { + row: node.row, + relationships, + }; + } +} + export function* mergeFetches( fetches: Iterable[], comparator: (l: Node, r: Node) => number, + mergeEqual?: ((left: Node, right: Node) => Node) | undefined, ): IterableIterator { const iterators = fetches.map(i => i[Symbol.iterator]()); let threw = false; try { const current: (Node | null)[] = []; - let lastNodeYielded: Node | undefined; - for (let i = 0; i < iterators.length; i++) { + const advance = function* (i: number) { const iter = iterators[i]; let result = iter.next(); - // yield yields when initializing while (!result.done && result.value === 'yield') { yield result.value; result = iter.next(); } current[i] = result.done ? null : (result.value as Node); + }; + for (let i = 0; i < iterators.length; i++) { + yield* advance(i); } while (current.some(c => c !== null)) { const min = current.reduce( @@ -253,22 +299,35 @@ export function* mergeFetches( ); assert(min !== undefined, 'min is undefined'); - const [minNode, minIndex] = min; - const iter = iterators[minIndex]; - let result = iter.next(); - while (!result.done && result.value === 'yield') { - yield result.value; - result = iter.next(); - } - current[minIndex] = result.done ? null : (result.value as Node); - if ( - lastNodeYielded !== undefined && - comparator(lastNodeYielded, minNode) === 0 - ) { - continue; + let [nodeToYield, currentIndex] = min; + yield* advance(currentIndex); + + while (true) { + const equal = current.reduce( + ( + acc: [Node, number] | undefined, + c, + i, + ): [Node, number] | undefined => { + if (c === null || comparator(c, nodeToYield) !== 0) { + return acc; + } + return acc ?? [c, i]; + }, + undefined, + ); + if (!equal) { + break; + } + const [equalNode, equalIndex] = equal; + nodeToYield = mergeEqual + ? mergeEqual(nodeToYield, equalNode) + : nodeToYield; + currentIndex = equalIndex; + yield* advance(currentIndex); } - lastNodeYielded = minNode; - yield minNode; + + yield nodeToYield; } } catch (e) { threw = true; diff --git a/packages/zql/src/ivm/yield.fetch.test.ts b/packages/zql/src/ivm/yield.fetch.test.ts index 3f8252ff73..75645a14c4 100644 --- a/packages/zql/src/ivm/yield.fetch.test.ts +++ b/packages/zql/src/ivm/yield.fetch.test.ts @@ -51,6 +51,31 @@ class YieldSource implements Input { destroy(): void {} } +class StaticInput implements Input { + readonly #schema: SourceSchema; + readonly #nodes: readonly Node[]; + + constructor(schema: SourceSchema, nodes: readonly Node[]) { + this.#schema = schema; + this.#nodes = nodes; + } + + setOutput(_: Output): void {} + + getSchema(): SourceSchema { + return this.#schema; + } + + *fetch(req: FetchRequest): Stream { + const nodes = req.reverse ? this.#nodes.toReversed() : this.#nodes; + for (const node of nodes) { + yield node; + } + } + + destroy(): void {} +} + describe('Yield Propagation', () => { test('FilterStart/End propagates yield', () => { const source = new YieldSource(); @@ -160,13 +185,13 @@ describe('Yield Propagation', () => { "yield", "yield", "yield", + "yield", { "relationships": {}, "row": { "id": "1", }, }, - "yield", { "relationships": {}, "row": { @@ -400,3 +425,57 @@ describe('Yield Propagation', () => { expect(() => catchOp.fetch({})).toThrow(error); }); }); + +test('UnionFanIn fetch merges relationships from duplicate branch rows', () => { + const childSchema: SourceSchema = { + ...YIELD_SOURCE_SCHEMA_BASE, + tableName: 'child', + }; + const leftRelationship = () => + [{row: {id: 'left-child'}, relationships: {}}] as const; + const rightRelationship = () => + [{row: {id: 'right-child'}, relationships: {}}] as const; + const parent = new StaticInput(YIELD_SOURCE_SCHEMA_BASE, []); + const fanOut = new UnionFanOut(parent); + const left = new StaticInput( + { + ...YIELD_SOURCE_SCHEMA_BASE, + relationships: {leftRelationship: childSchema}, + }, + [ + { + row: {id: '1'}, + relationships: {leftRelationship}, + }, + ], + ); + const right = new StaticInput( + { + ...YIELD_SOURCE_SCHEMA_BASE, + relationships: {rightRelationship: childSchema}, + }, + [ + { + row: {id: '1'}, + relationships: {rightRelationship}, + }, + ], + ); + const fanIn = new UnionFanIn(fanOut, [left, right]); + const [node] = [...fanIn.fetch({})]; + + expect(node).not.toBe('yield'); + if (node === undefined || node === 'yield') { + throw new Error('Expected one fetched node'); + } + expect(Object.keys(node.relationships).sort()).toEqual([ + 'leftRelationship', + 'rightRelationship', + ]); + expect([...node.relationships.leftRelationship()]).toEqual([ + {row: {id: 'left-child'}, relationships: {}}, + ]); + expect([...node.relationships.rightRelationship()]).toEqual([ + {row: {id: 'right-child'}, relationships: {}}, + ]); +}); diff --git a/packages/zql/src/planner/condition-normalizer.test.ts b/packages/zql/src/planner/condition-normalizer.test.ts new file mode 100644 index 0000000000..341dc24b61 --- /dev/null +++ b/packages/zql/src/planner/condition-normalizer.test.ts @@ -0,0 +1,594 @@ +import fc from 'fast-check'; +import {expect, test} from 'vitest'; +import type { + AST, + Condition, + CorrelatedSubqueryCondition, + LiteralValue, + SimpleOperator, +} from '../../../zero-protocol/src/ast.ts'; +import {createPredicate, type NoSubqueryCondition} from '../builder/filter.ts'; +import {normalizePlannerAST} from './condition-normalizer.ts'; + +const TRUE: Condition = {type: 'and', conditions: []}; +const FALSE: Condition = {type: 'or', conditions: []}; + +test('drops identity branches without losing real filters', () => { + const active = eq('active', true); + + expect(normalizeWhere({type: 'or', conditions: [active, FALSE]})).toEqual( + active, + ); + expect(normalizeWhere({type: 'and', conditions: [active, TRUE]})).toEqual( + active, + ); + expect(normalizeWhere({type: 'and', conditions: [active, FALSE]})).toEqual( + FALSE, + ); +}); + +test('rewrites same-column OR equalities to IN', () => { + expect( + normalizeWhere({ + type: 'or', + conditions: [eq('status', 'active'), eq('status', 'pending')], + }), + ).toEqual({ + type: 'simple', + left: {type: 'column', name: 'status'}, + op: 'IN', + right: {type: 'literal', value: ['active', 'pending']}, + }); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + eq('status', 'active'), + inCondition('status', 'IN', ['active']), + ], + }), + ).toEqual(eq('status', 'active')); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + inCondition('status', 'IN', ['active', 'pending']), + inCondition('status', 'IN', ['pending', 'draft']), + ], + }), + ).toEqual({ + type: 'simple', + left: {type: 'column', name: 'status'}, + op: 'IN', + right: {type: 'literal', value: ['active', 'pending', 'draft']}, + }); +}); + +test('normalizes degenerate IN predicates', () => { + const notInEmpty = inCondition('status', 'NOT IN', []); + + expect(normalizeWhere(inCondition('status', 'IN', []))).toEqual(FALSE); + expect(normalizeWhere(inCondition('status', 'IN', ['active']))).toEqual( + eq('status', 'active'), + ); + expect(normalizeWhere(notInEmpty)).toEqual(notInEmpty); + expect(normalizeWhere(inCondition('status', 'NOT IN', ['active']))).toEqual({ + type: 'simple', + left: {type: 'column', name: 'status'}, + op: '!=', + right: {type: 'literal', value: 'active'}, + }); +}); + +test('intersects same-column AND equalities and IN predicates', () => { + expect( + normalizeWhere({ + type: 'and', + conditions: [ + inCondition('status', 'IN', ['active', 'pending']), + eq('status', 'pending'), + ], + }), + ).toEqual(eq('status', 'pending')); + + expect( + normalizeWhere({ + type: 'and', + conditions: [ + inCondition('status', 'IN', ['active', 'pending']), + inCondition('status', 'IN', ['pending', 'draft']), + ], + }), + ).toEqual(eq('status', 'pending')); + + expect( + normalizeWhere({ + type: 'and', + conditions: [eq('status', 'active'), eq('status', 'pending')], + }), + ).toEqual(FALSE); +}); + +test('applies same-column AND exclusions', () => { + expect( + normalizeWhere({ + type: 'and', + conditions: [eq('status', 'active'), ne('status', 'active')], + }), + ).toEqual(FALSE); + + expect( + normalizeWhere({ + type: 'and', + conditions: [ + inCondition('status', 'IN', ['active', 'pending', 'draft']), + ne('status', 'pending'), + inCondition('status', 'NOT IN', ['draft']), + ], + }), + ).toEqual(eq('status', 'active')); + + expect( + normalizeWhere({ + type: 'and', + conditions: [ + inCondition('status', 'NOT IN', ['active']), + inCondition('status', 'NOT IN', ['pending']), + ], + }), + ).toEqual(inCondition('status', 'NOT IN', ['active', 'pending'])); +}); + +test('merges OR exists branches over the same relationship', () => { + expect( + normalizeWhere({ + type: 'or', + conditions: [exists(eq('title', 'hello')), exists(eq('title', 'world'))], + }), + ).toEqual( + exists({ + type: 'simple', + left: {type: 'column', name: 'title'}, + op: 'IN', + right: {type: 'literal', value: ['hello', 'world']}, + }), + ); +}); + +test('merges OR exists branches after implicit ordering is completed', () => { + expect( + normalizeWhere({ + type: 'or', + conditions: [ + exists(eq('title', 'hello'), {subquery: {orderBy: [['id', 'asc']]}}), + exists(eq('title', 'world'), {subquery: {orderBy: [['id', 'asc']]}}), + ], + }), + ).toEqual( + exists( + { + type: 'simple', + left: {type: 'column', name: 'title'}, + op: 'IN', + right: {type: 'literal', value: ['hello', 'world']}, + }, + {subquery: {orderBy: [['id', 'asc']]}}, + ), + ); +}); + +test('preserves unrestricted exists when merging narrower exists branches', () => { + expect( + normalizeWhere({ + type: 'or', + conditions: [exists(undefined), exists(eq('title', 'hello'))], + }), + ).toEqual(exists(undefined)); +}); + +test('factors common parent filters before merging child exists branches', () => { + const active = eq('active', true); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + { + type: 'and', + conditions: [active, exists(eq('title', 'hello'))], + }, + { + type: 'and', + conditions: [active, exists(eq('title', 'world'))], + }, + ], + }), + ).toEqual({ + type: 'and', + conditions: [ + active, + exists({ + type: 'simple', + left: {type: 'column', name: 'title'}, + op: 'IN', + right: {type: 'literal', value: ['hello', 'world']}, + }), + ], + }); +}); + +test('absorbs redundant branches after factoring common predicates', () => { + const active = eq('active', true); + const archived = eq('archived', false); + const pending = eq('status', 'pending'); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + active, + { + type: 'and', + conditions: [active, eq('status', 'pending')], + }, + ], + }), + ).toEqual(active); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + active, + { + type: 'and', + conditions: [active, pending], + }, + archived, + ], + }), + ).toEqual({ + type: 'or', + conditions: [active, archived], + }); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + { + type: 'and', + conditions: [active, pending], + }, + { + type: 'and', + conditions: [active, pending, archived], + }, + ], + }), + ).toEqual({ + type: 'and', + conditions: [active, pending], + }); +}); + +test('collapses impossible exists subqueries', () => { + const active = eq('active', true); + + expect(normalizeWhere(exists(FALSE))).toEqual(FALSE); + expect(normalizeWhere(exists(FALSE, {op: 'NOT EXISTS'}))).toEqual(TRUE); + expect(normalizeWhere(exists(undefined, {subquery: {limit: 0}}))).toEqual( + FALSE, + ); + + expect( + normalizeWhere({ + type: 'or', + conditions: [active, exists(FALSE)], + }), + ).toEqual(active); + + expect(normalizeWhere(exists(FALSE, {scalar: true}))).toEqual( + exists(FALSE, {scalar: true}), + ); +}); + +test('does not merge exists branches with different semantics', () => { + expect( + normalizeWhere({ + type: 'or', + conditions: [ + exists(eq('title', 'hello'), {op: 'NOT EXISTS'}), + exists(eq('title', 'world'), {op: 'NOT EXISTS'}), + ], + }), + ).toMatchObject({ + type: 'or', + conditions: [{op: 'NOT EXISTS'}, {op: 'NOT EXISTS'}], + }); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + exists(eq('title', 'hello'), {scalar: true}), + exists(eq('title', 'world'), {scalar: true}), + ], + }), + ).toMatchObject({type: 'or', conditions: [{scalar: true}, {scalar: true}]}); + + expect( + normalizeWhere({ + type: 'or', + conditions: [ + exists(eq('title', 'hello'), {subquery: {limit: 1}}), + exists(eq('title', 'world'), {subquery: {limit: 1}}), + ], + }), + ).toMatchObject({ + type: 'or', + conditions: [ + {related: {subquery: {limit: 1}}}, + {related: {subquery: {limit: 1}}}, + ], + }); +}); + +test('normalizes related subqueries recursively', () => { + expect( + normalizePlannerAST({ + table: 'users', + related: [ + { + correlation: { + parentField: ['id'], + childField: ['userId'], + }, + subquery: { + table: 'posts', + where: { + type: 'or', + conditions: [eq('title', 'hello'), eq('title', 'world')], + }, + }, + }, + ], + }).related?.[0].subquery.where, + ).toEqual({ + type: 'simple', + left: {type: 'column', name: 'title'}, + op: 'IN', + right: {type: 'literal', value: ['hello', 'world']}, + }); +}); + +test('preserves simple filter semantics under generated rows', () => { + fc.assert( + fc.property( + filterConditionArbitrary(), + rowArbitrary(), + (condition, row) => { + const normalized = normalizePlannerAST({ + table: 'users', + where: condition, + }).where as NoSubqueryCondition | undefined; + const before = createPredicate(condition)(row); + const after = normalized ? createPredicate(normalized)(row) : true; + + expect(after).toBe(before); + }, + ), + {numRuns: 1_000}, + ); +}); + +test('is idempotent for generated simple filters', () => { + fc.assert( + fc.property(filterConditionArbitrary(), condition => { + const once = normalizePlannerAST({table: 'users', where: condition}); + const twice = normalizePlannerAST(once); + + expect(twice).toEqual(once); + }), + {numRuns: 1_000}, + ); +}); + +test('is idempotent for generated planner filters with correlated subqueries', () => { + fc.assert( + fc.property(plannerConditionArbitrary(), condition => { + const once = normalizePlannerAST({table: 'users', where: condition}); + const twice = normalizePlannerAST(once); + + expect(twice).toEqual(once); + }), + {numRuns: 1_000}, + ); +}); + +function normalizeWhere(where: Condition): Condition | undefined { + return normalizePlannerAST({table: 'users', where}).where; +} + +function eq(name: string, value: LiteralValue): Condition { + return { + type: 'simple', + left: {type: 'column', name}, + op: '=', + right: {type: 'literal', value}, + }; +} + +function ne(name: string, value: LiteralValue): Condition { + return { + type: 'simple', + left: {type: 'column', name}, + op: '!=', + right: {type: 'literal', value}, + }; +} + +function inCondition( + name: string, + op: 'IN' | 'NOT IN', + values: readonly (string | number | boolean)[], +): Condition { + return { + type: 'simple', + left: {type: 'column', name}, + op, + right: {type: 'literal', value: values}, + }; +} + +function exists( + where: Condition | undefined, + options: { + readonly op?: 'EXISTS' | 'NOT EXISTS' | undefined; + readonly scalar?: boolean | undefined; + readonly subquery?: Partial | undefined; + } = {}, +): CorrelatedSubqueryCondition { + return { + type: 'correlatedSubquery', + op: options.op ?? 'EXISTS', + scalar: options.scalar, + related: { + correlation: { + parentField: ['id'], + childField: ['userId'], + }, + subquery: { + table: 'posts', + where, + ...options.subquery, + }, + }, + }; +} + +function filterConditionArbitrary(): fc.Arbitrary { + return fc.letrec(tie => ({ + condition: fc.oneof( + simpleConditionArbitrary(), + fc.record({ + type: fc.constant('and' as const), + conditions: fc.array(tie('condition'), {maxLength: 4}), + }), + fc.record({ + type: fc.constant('or' as const), + conditions: fc.array(tie('condition'), {maxLength: 4}), + }), + ), + })).condition as fc.Arbitrary; +} + +function plannerConditionArbitrary(): fc.Arbitrary { + return fc.letrec(tie => ({ + condition: fc.oneof( + simpleConditionArbitrary(), + correlatedSubqueryConditionArbitrary(), + fc.record({ + type: fc.constant('and' as const), + conditions: fc.array(tie('condition'), {maxLength: 4}), + }), + fc.record({ + type: fc.constant('or' as const), + conditions: fc.array(tie('condition'), {maxLength: 4}), + }), + ), + })).condition as fc.Arbitrary; +} + +function correlatedSubqueryConditionArbitrary(): fc.Arbitrary { + return fc.record({ + type: fc.constant('correlatedSubquery' as const), + op: fc.constantFrom('EXISTS' as const, 'NOT EXISTS' as const), + flip: fc.option(fc.boolean(), {nil: undefined}), + scalar: fc.option(fc.boolean(), {nil: undefined}), + related: fc.record({ + correlation: fc.constant({ + parentField: ['id'], + childField: ['userId'], + } as const), + subquery: fc.record({ + table: fc.constant('posts'), + where: fc.option(filterConditionArbitrary(), {nil: undefined}), + limit: fc.option(fc.constantFrom(0, 1, 10), {nil: undefined}), + }), + }), + }); +} + +function simpleConditionArbitrary(): fc.Arbitrary { + return fc.oneof( + fc.record({ + type: fc.constant('simple' as const), + left: columnReferenceArbitrary(), + op: fc.constantFrom('=', '!=', '<', '<=', '>', '>='), + right: fc.record({ + type: fc.constant('literal' as const), + value: scalarLiteralArbitrary(), + }), + }), + fc.record({ + type: fc.constant('simple' as const), + left: columnReferenceArbitrary(), + op: fc.constantFrom('IS', 'IS NOT'), + right: fc.record({ + type: fc.constant('literal' as const), + value: fc.constant(null), + }), + }), + fc.record({ + type: fc.constant('simple' as const), + left: columnReferenceArbitrary(), + op: fc.constantFrom('IN', 'NOT IN'), + right: fc.record({ + type: fc.constant('literal' as const), + value: fc.array(inLiteralArbitrary(), {maxLength: 6}), + }), + }), + ); +} + +function rowArbitrary() { + return fc.record({ + a: rowValueArbitrary(), + b: rowValueArbitrary(), + c: rowValueArbitrary(), + }); +} + +function columnReferenceArbitrary() { + return fc.record({ + type: fc.constant('column' as const), + name: fc.constantFrom('a', 'b', 'c'), + }); +} + +function scalarLiteralArbitrary(): fc.Arbitrary< + string | number | boolean | null +> { + return fc.oneof(inLiteralArbitrary(), fc.constant(null)); +} + +function rowValueArbitrary(): fc.Arbitrary< + string | number | boolean | null | undefined +> { + return fc.oneof( + inLiteralArbitrary(), + fc.constant(null), + fc.constant(undefined), + ); +} + +function inLiteralArbitrary(): fc.Arbitrary { + return fc.oneof( + fc.string({maxLength: 8}), + fc.integer({min: -20, max: 20}), + fc.boolean(), + ); +} diff --git a/packages/zql/src/planner/condition-normalizer.ts b/packages/zql/src/planner/condition-normalizer.ts new file mode 100644 index 0000000000..36c68af21b --- /dev/null +++ b/packages/zql/src/planner/condition-normalizer.ts @@ -0,0 +1,753 @@ +import type { + AST, + Condition, + CorrelatedSubquery, + CorrelatedSubqueryCondition, + LiteralValue, + SimpleCondition, + ValuePosition, +} from '../../../zero-protocol/src/ast.ts'; +import {planIdSymbol} from '../../../zero-protocol/src/ast.ts'; + +const TRUE: Condition = {type: 'and', conditions: []}; +const FALSE: Condition = {type: 'or', conditions: []}; + +export function normalizePlannerAST(ast: AST): AST { + return { + ...ast, + where: ast.where ? normalizeCondition(ast.where) : undefined, + related: ast.related?.map(related => ({ + ...related, + subquery: normalizePlannerAST(related.subquery), + })), + }; +} + +function normalizeCondition(condition: Condition): Condition { + switch (condition.type) { + case 'simple': + return normalizeSimpleCondition(condition); + case 'correlatedSubquery': + return normalizeCorrelatedSubquery(condition); + case 'and': + return buildAnd(condition.conditions.map(normalizeCondition)); + case 'or': + return buildOr(condition.conditions.map(normalizeCondition)); + } +} + +function normalizeSimpleCondition(condition: SimpleCondition): Condition { + if (condition.right.type !== 'literal') { + return condition; + } + + const value = condition.right.value; + if (!Array.isArray(value)) { + return condition; + } + + if (condition.op === 'IN') { + const values = dedupeInLiteralValues(value); + switch (values.length) { + case 0: + return FALSE; + case 1: + return { + ...condition, + op: '=', + right: {type: 'literal', value: values[0]}, + }; + default: + return { + ...condition, + right: {type: 'literal', value: values}, + }; + } + } + + if (condition.op === 'NOT IN') { + const values = dedupeInLiteralValues(value); + switch (values.length) { + case 0: + return condition; + case 1: + return { + ...condition, + op: '!=', + right: {type: 'literal', value: values[0]}, + }; + default: + return { + ...condition, + right: {type: 'literal', value: values}, + }; + } + } + + return condition; +} + +function normalizeCorrelatedSubquery( + condition: CorrelatedSubqueryCondition, +): Condition { + const {[planIdSymbol]: _planId, ...conditionWithoutPlanId} = condition; + return collapseImpossibleCorrelatedSubquery({ + ...conditionWithoutPlanId, + related: { + ...condition.related, + subquery: normalizePlannerAST(condition.related.subquery), + }, + }); +} + +function buildAnd(conditions: readonly Condition[]): Condition { + const flattened = flatten('and', conditions); + if (flattened.some(isAlwaysFalse)) { + return FALSE; + } + + const deduped = dedupe( + flattened.filter(condition => !isAlwaysTrue(condition)), + ); + const intersected = intersectEquivalentAndPredicates(deduped); + if (intersected.some(isAlwaysFalse)) { + return FALSE; + } + + switch (intersected.length) { + case 0: + return TRUE; + case 1: + return intersected[0]; + default: + return {type: 'and', conditions: intersected}; + } +} + +function buildOr(conditions: readonly Condition[]): Condition { + const flattened = flatten('or', conditions); + if (flattened.some(isAlwaysTrue)) { + return TRUE; + } + + const deduped = dedupe( + flattened.filter(condition => !isAlwaysFalse(condition)), + ); + if (deduped.length === 0) { + return FALSE; + } + + const compacted = absorbRedundantOrBranches( + mergeEquivalentOrPredicates(deduped), + ); + + const factored = factorCommonConjuncts(compacted); + if (factored) { + return normalizeCondition(factored); + } + + const merged = mergeSameRelationshipExists(compacted); + if (merged.some(isAlwaysTrue)) { + return TRUE; + } + + const surviving = merged.filter(condition => !isAlwaysFalse(condition)); + switch (surviving.length) { + case 0: + return FALSE; + case 1: + return surviving[0]; + default: + return {type: 'or', conditions: surviving}; + } +} + +type InLiteralValue = string | number | boolean; + +type ColumnDomainBase = { + readonly left: ValuePosition; +}; + +// include undefined means the column can still take any literal value. +// exclude then removes known impossible values from that open domain. +type ColumnDomain = ColumnDomainBase & { + readonly include: readonly InLiteralValue[] | undefined; + readonly exclude: readonly InLiteralValue[]; +}; + +type PositiveColumnDomain = ColumnDomainBase & { + readonly include: readonly InLiteralValue[]; +}; + +type DomainPredicate = { + readonly domain: TDomain; + readonly changed: boolean; +}; + +type DomainGroup = { + readonly firstIndex: number; + domain: TDomain; + changed: boolean; +}; + +type DomainRewrite = { + readonly extract: ( + condition: Condition, + ) => DomainPredicate | undefined; + readonly combine: (left: TDomain, right: TDomain) => TDomain; + readonly lower: (domain: TDomain) => Condition; +}; + +function intersectEquivalentAndPredicates( + conditions: readonly Condition[], +): Condition[] { + return rewriteColumnDomains(conditions, { + extract: columnDomainFromCondition, + combine: intersectColumnDomains, + lower: columnDomainToCondition, + }); +} + +function mergeEquivalentOrPredicates( + conditions: readonly Condition[], +): Condition[] { + return rewriteColumnDomains(conditions, { + extract: positiveColumnDomainFromCondition, + combine: unionPositiveColumnDomains, + lower: positiveColumnDomainToCondition, + }); +} + +function rewriteColumnDomains( + conditions: readonly Condition[], + rewrite: DomainRewrite, +): Condition[] { + const rewritten: Array = [...conditions]; + const groups = new Map>(); + for (const [index, condition] of conditions.entries()) { + const predicate = rewrite.extract(condition); + if (!predicate) { + continue; + } + + const key = domainKey(predicate.domain); + const group = groups.get(key); + if (!group) { + groups.set(key, { + firstIndex: index, + domain: predicate.domain, + changed: predicate.changed, + }); + continue; + } + + rewritten[index] = undefined; + group.changed = true; + group.domain = rewrite.combine(group.domain, predicate.domain); + } + + for (const group of groups.values()) { + if (!group.changed) { + continue; + } + rewritten[group.firstIndex] = rewrite.lower(group.domain); + } + + return rewritten.filter((condition): condition is Condition => !!condition); +} + +function columnDomainFromCondition( + condition: Condition, +): DomainPredicate | undefined { + if (condition.type !== 'simple' || condition.right.type !== 'literal') { + return undefined; + } + + const {value} = condition.right; + switch (condition.op) { + case '=': + return isInLiteralValue(value) + ? finiteColumnDomain(condition.left, [value]) + : undefined; + case 'IN': + return Array.isArray(value) && value.every(isInLiteralValue) + ? finiteColumnDomain(condition.left, value) + : undefined; + case '!=': + return isInLiteralValue(value) + ? excludedColumnDomain(condition.left, [value]) + : undefined; + case 'NOT IN': + return Array.isArray(value) && + value.length > 0 && + value.every(isInLiteralValue) + ? excludedColumnDomain(condition.left, value) + : undefined; + default: + return undefined; + } +} + +function positiveColumnDomainFromCondition( + condition: Condition, +): DomainPredicate | undefined { + const predicate = columnDomainFromCondition(condition); + if (!predicate || predicate.domain.include === undefined) { + return undefined; + } + if (predicate.domain.exclude.length > 0) { + return undefined; + } + + return { + domain: { + left: predicate.domain.left, + include: predicate.domain.include, + }, + changed: predicate.changed, + }; +} + +function finiteColumnDomain( + left: ValuePosition, + values: readonly InLiteralValue[], +): DomainPredicate { + const include = literalSet(values); + return { + domain: {left, include, exclude: []}, + changed: include.length !== values.length, + }; +} + +function excludedColumnDomain( + left: ValuePosition, + values: readonly InLiteralValue[], +): DomainPredicate { + const exclude = literalSet(values); + return { + domain: {left, include: undefined, exclude}, + changed: exclude.length !== values.length, + }; +} + +function intersectColumnDomains( + left: ColumnDomain, + right: ColumnDomain, +): ColumnDomain { + return { + left: left.left, + include: intersectOptionalValueSets(left.include, right.include), + exclude: unionLiteralValues(left.exclude, right.exclude), + }; +} + +function unionPositiveColumnDomains( + left: PositiveColumnDomain, + right: PositiveColumnDomain, +): PositiveColumnDomain { + return { + left: left.left, + include: unionLiteralValues(left.include, right.include), + }; +} + +function columnDomainToCondition(domain: ColumnDomain): Condition { + if (domain.include !== undefined) { + return buildInCondition( + domain.left, + subtractLiteralValues(domain.include, domain.exclude), + ); + } + return buildNotInCondition(domain.left, domain.exclude); +} + +function positiveColumnDomainToCondition( + domain: PositiveColumnDomain, +): Condition { + return buildInCondition(domain.left, domain.include); +} + +function domainKey(domain: ColumnDomainBase): string { + return stableStringify(domain.left); +} + +function collapseImpossibleCorrelatedSubquery( + condition: CorrelatedSubqueryCondition, +): Condition { + if (condition.scalar === true) { + return condition; + } + const {subquery} = condition.related; + const subqueryCannotEmitRows = + subquery.limit === 0 || + (subquery.where !== undefined && isAlwaysFalse(subquery.where)); + if (!subqueryCannotEmitRows) { + return condition; + } + return condition.op === 'EXISTS' ? FALSE : TRUE; +} + +function buildInCondition( + left: ValuePosition, + values: readonly InLiteralValue[], +): Condition { + switch (values.length) { + case 0: + return FALSE; + case 1: + return { + type: 'simple', + left, + op: '=', + right: {type: 'literal', value: values[0]}, + }; + default: + return { + type: 'simple', + left, + op: 'IN', + right: {type: 'literal', value: values}, + }; + } +} + +function buildNotInCondition( + left: ValuePosition, + values: readonly InLiteralValue[], +): Condition { + switch (values.length) { + case 0: + return TRUE; + case 1: + return { + type: 'simple', + left, + op: '!=', + right: {type: 'literal', value: values[0]}, + }; + default: + return { + type: 'simple', + left, + op: 'NOT IN', + right: {type: 'literal', value: values}, + }; + } +} + +function isInLiteralValue( + value: LiteralValue | undefined, +): value is InLiteralValue { + return value !== undefined && value !== null && !Array.isArray(value); +} + +function dedupeInLiteralValues( + values: readonly InLiteralValue[], +): InLiteralValue[] { + return literalSet(values); +} + +function literalSet(values: readonly InLiteralValue[]): InLiteralValue[] { + const seen = new Set(); + const deduped: InLiteralValue[] = []; + for (const value of values) { + const key = literalValueKey(value); + if (seen.has(key)) { + continue; + } + seen.add(key); + deduped.push(value); + } + return deduped; +} + +function intersectOptionalValueSets( + left: readonly InLiteralValue[] | undefined, + right: readonly InLiteralValue[] | undefined, +): readonly InLiteralValue[] | undefined { + if (left === undefined) { + return right; + } + if (right === undefined) { + return left; + } + return intersectLiteralValues(left, right); +} + +function unionLiteralValues( + left: readonly InLiteralValue[], + right: readonly InLiteralValue[], +): InLiteralValue[] { + const seen = new Set(left.map(literalValueKey)); + const union = [...left]; + for (const value of right) { + const key = literalValueKey(value); + if (seen.has(key)) { + continue; + } + seen.add(key); + union.push(value); + } + return union; +} + +function intersectLiteralValues( + left: readonly InLiteralValue[], + right: readonly InLiteralValue[], +): InLiteralValue[] { + const rightValues = new Set(right.map(literalValueKey)); + return left.filter(value => rightValues.has(literalValueKey(value))); +} + +function subtractLiteralValues( + values: readonly InLiteralValue[], + excluded: readonly InLiteralValue[], +): InLiteralValue[] { + const excludedValues = new Set(excluded.map(literalValueKey)); + return values.filter(value => !excludedValues.has(literalValueKey(value))); +} + +function literalValueKey(value: InLiteralValue): string { + return stableStringify(value); +} + +function flatten( + type: 'and' | 'or', + conditions: readonly Condition[], +): Condition[] { + const flattened: Condition[] = []; + for (const condition of conditions) { + if (condition.type === type) { + flattened.push(...condition.conditions); + } else { + flattened.push(condition); + } + } + return flattened; +} + +function factorCommonConjuncts( + conditions: readonly Condition[], +): Condition | undefined { + if (conditions.length < 2) { + return undefined; + } + + const branches = conditions.map(condition => + condition.type === 'and' ? [...condition.conditions] : [condition], + ); + const commonKeys = new Set(branches[0].map(conditionKey)); + for (const branch of branches.slice(1)) { + const branchKeys = new Set(branch.map(conditionKey)); + for (const key of [...commonKeys]) { + if (!branchKeys.has(key)) { + commonKeys.delete(key); + } + } + } + + if (commonKeys.size === 0) { + return undefined; + } + + const common: Condition[] = []; + const firstBranch = branches[0]; + for (const condition of firstBranch) { + if (commonKeys.has(conditionKey(condition))) { + common.push(condition); + } + } + + const remainingBranches = branches.map(branch => + branch.filter(condition => !commonKeys.has(conditionKey(condition))), + ); + + return buildAnd([ + ...common, + buildOr(remainingBranches.map(branch => buildAnd(branch))), + ]); +} + +function absorbRedundantOrBranches( + conditions: readonly Condition[], +): Condition[] { + const branches = conditions.map(condition => ({ + keys: + condition.type === 'and' + ? new Set(condition.conditions.map(conditionKey)) + : new Set([conditionKey(condition)]), + })); + const keep = conditions.map(() => true); + + for (const [candidateIndex, candidate] of branches.entries()) { + if (!keep[candidateIndex]) { + continue; + } + for (const [branchIndex, branch] of branches.entries()) { + if (candidateIndex === branchIndex || !keep[branchIndex]) { + continue; + } + if (!branchSubsumes(candidate.keys, branch.keys)) { + continue; + } + if ( + candidate.keys.size < branch.keys.size || + candidateIndex < branchIndex + ) { + keep[branchIndex] = false; + } + } + } + + return conditions.filter((_, index) => keep[index]); +} + +function branchSubsumes( + candidate: ReadonlySet, + branch: ReadonlySet, +): boolean { + for (const key of candidate) { + if (!branch.has(key)) { + return false; + } + } + return true; +} + +function mergeSameRelationshipExists( + conditions: readonly Condition[], +): Condition[] { + const merged: Array = [...conditions]; + const groups = new Map< + string, + { + readonly template: CorrelatedSubqueryCondition; + readonly firstIndex: number; + readonly filters: Condition[]; + } + >(); + + for (const [index, condition] of conditions.entries()) { + const key = mergeableExistsKey(condition); + if (!key || condition.type !== 'correlatedSubquery') { + continue; + } + + const group = groups.get(key); + const filter = condition.related.subquery.where ?? TRUE; + if (group) { + merged[index] = undefined; + group.filters.push(filter); + } else { + groups.set(key, { + template: condition, + firstIndex: index, + filters: [filter], + }); + } + } + + for (const group of groups.values()) { + merged[group.firstIndex] = mergeExistsGroup(group.template, group.filters); + } + + return dedupe( + merged.filter((condition): condition is Condition => !!condition), + ); +} + +function mergeExistsGroup( + template: CorrelatedSubqueryCondition, + filters: readonly Condition[], +): Condition { + const where = buildOr(filters); + return collapseImpossibleCorrelatedSubquery({ + ...template, + related: { + ...template.related, + subquery: { + ...template.related.subquery, + where: isAlwaysTrue(where) ? undefined : where, + }, + }, + }); +} + +function mergeableExistsKey(condition: Condition): string | undefined { + if (condition.type !== 'correlatedSubquery') { + return undefined; + } + if (condition.op !== 'EXISTS' || condition.scalar === true) { + return undefined; + } + + const {related} = condition; + const {subquery} = related; + if ( + subquery.related !== undefined || + subquery.start !== undefined || + subquery.limit !== undefined + ) { + return undefined; + } + + // EXISTS only asks whether at least one child row exists. Without related + // rows, start, or limit, child ordering cannot affect that answer. Ignore + // orderBy here so production callers that completed implicit primary-key + // ordering before planning still get the same logical EXISTS merge. + const mergeShape: CorrelatedSubquery = { + ...related, + subquery: { + schema: subquery.schema, + table: subquery.table, + alias: subquery.alias, + }, + }; + return stableStringify({ + op: condition.op, + flip: condition.flip, + scalar: condition.scalar, + related: mergeShape, + }); +} + +function dedupe(conditions: readonly Condition[]): Condition[] { + const seen = new Set(); + const deduped: Condition[] = []; + for (const condition of conditions) { + const key = conditionKey(condition); + if (seen.has(key)) { + continue; + } + seen.add(key); + deduped.push(condition); + } + return deduped; +} + +function conditionKey(condition: Condition): string { + return stableStringify(condition); +} + +function stableStringify(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map(stableStringify).join(',')}]`; + } + if (value && typeof value === 'object') { + const entries = Object.entries(value) + .filter(([, entry]) => entry !== undefined) + .sort(([left], [right]) => left.localeCompare(right)); + return `{${entries + .map(([key, entry]) => `${JSON.stringify(key)}:${stableStringify(entry)}`) + .join(',')}}`; + } + return JSON.stringify(value) ?? 'undefined'; +} + +function isAlwaysTrue(condition: Condition): boolean { + return condition.type === 'and' && condition.conditions.length === 0; +} + +function isAlwaysFalse(condition: Condition): boolean { + return condition.type === 'or' && condition.conditions.length === 0; +} diff --git a/packages/zql/src/planner/planner-builder.ts b/packages/zql/src/planner/planner-builder.ts index 29eea8882a..0914bc022e 100644 --- a/packages/zql/src/planner/planner-builder.ts +++ b/packages/zql/src/planner/planner-builder.ts @@ -9,6 +9,7 @@ import type { Disjunction, } from '../../../zero-protocol/src/ast.ts'; import {planIdSymbol} from '../../../zero-protocol/src/ast.ts'; +import {normalizePlannerAST} from './condition-normalizer.ts'; import type {ConnectionCostModel} from './planner-connection.ts'; import type {PlannerConstraint} from './planner-constraint.ts'; import type {PlanDebugger} from './planner-debug.ts'; @@ -314,9 +315,10 @@ export function planQuery( planDebugger?: PlanDebugger, lc?: LogContext, ): AST { - const plans = buildPlanGraph(ast, model, true); + const normalizedAST = normalizePlannerAST(ast); + const plans = buildPlanGraph(normalizedAST, model, true); planRecursively(plans, planDebugger, lc); - return applyPlansToAST(ast, plans); + return applyPlansToAST(normalizedAST, plans); } function applyToCondition( @@ -332,10 +334,17 @@ function applyToCondition( planIdSymbol ]; const shouldFlip = planId !== undefined && flippedIds.has(planId); + // Preserve explicit flip: false while avoiding a noisy flip: false on every + // ordinary planner-controlled EXISTS. The absence of flip means "planner may + // choose"; an explicit false means "do not flip this relationship". + const flip = + shouldFlip || condition.flip !== undefined + ? {flip: shouldFlip} + : undefined; return { ...condition, - flip: shouldFlip, + ...flip, related: { ...condition.related, subquery: { diff --git a/packages/zqlite/src/query-planner-scenarios.test.ts b/packages/zqlite/src/query-planner-scenarios.test.ts index fc8ad871d0..0668d1f3ac 100644 --- a/packages/zqlite/src/query-planner-scenarios.test.ts +++ b/packages/zqlite/src/query-planner-scenarios.test.ts @@ -7,6 +7,18 @@ import { import type {educationAppSchema} from './test/query-scenarios/education-app.ts'; import scenarios from './test/query-scenarios/scenarios/index.ts'; +// Scenario comments use one visual legend: +// +// Submitted ZQL: the filter shape the user wrote. +// Naive plan: the straightforward parent first execution we avoid. +// Optimized plan: the physical SQL order this test expects. +// Intuition: why the optimized plan is cheaper or clearer. +// +// A -> B means scan A first, then look up B. +// +// left scan --. +// +-- union or intersect parent ids +// right scan -' for (const scenario of scenarios as readonly QueryScenario< typeof educationAppSchema >[]) { @@ -45,4 +57,7 @@ function assertScenarioExpectations( if (expectations.sql) { expect(result.sql).toEqual(expectations.sql); } + if (expectations.rows) { + expect(result.rows).toEqual(expectations.rows); + } } diff --git a/packages/zqlite/src/test/query-scenario.ts b/packages/zqlite/src/test/query-scenario.ts index 5a80077c9b..a1cd636ec8 100644 --- a/packages/zqlite/src/test/query-scenario.ts +++ b/packages/zqlite/src/test/query-scenario.ts @@ -4,6 +4,7 @@ import {computeZqlSpecs} from '../../../zero-cache/src/db/lite-tables.ts'; import type {LiteAndZqlSpec} from '../../../zero-cache/src/db/specs.ts'; import {CREATE_TABLE_METADATA_TABLE} from '../../../zero-cache/src/services/replicator/schema/table-metadata.ts'; import type {AST} from '../../../zero-protocol/src/ast.ts'; +import type {Row} from '../../../zero-protocol/src/data.ts'; import type {Schema} from '../../../zero-types/src/schema.ts'; import {buildPipeline} from '../../../zql/src/builder/builder.ts'; import {Debug} from '../../../zql/src/builder/debug-delegate.ts'; @@ -20,19 +21,29 @@ import {createSQLiteCostModel} from '../sqlite-cost-model.ts'; import {newQueryDelegate} from './source-factory.ts'; type QuerySeen = { - readonly table: string; - readonly sql: string; + table: string; + sql: string; + calls: number; }; class ScenarioDebug extends Debug { readonly queries: QuerySeen[] = []; override initQuery(table: string, query: string): void { - if (!this.queries.some(q => q.table === table && q.sql === query)) { - this.queries.push({table, sql: query}); + const seen = this.queries.find(q => q.table === table && q.sql === query); + if (seen) { + seen.calls++; + } else { + this.queries.push({table, sql: query, calls: 1}); } super.initQuery(table, query); } + + compactQueries(): readonly QueryScenarioSQL[] { + return this.queries.map(({table, sql, calls}) => + calls === 1 ? {table, sql} : {table, sql, calls}, + ); + } } export type QueryScenario = { @@ -54,11 +65,13 @@ export type QueryScenarioExpectations = { readonly optimizedAST?: object; readonly planDebug?: readonly string[]; readonly sql?: readonly QueryScenarioSQL[]; + readonly rows?: readonly Row[]; }; export type QueryScenarioSQL = { readonly table: string; readonly sql: string; + readonly calls?: number | undefined; }; export type QueryScenarioResult = { @@ -66,6 +79,7 @@ export type QueryScenarioResult = { readonly optimizedAST: AST; readonly planDebug: string; readonly sql: readonly QueryScenarioSQL[]; + readonly rows: readonly Row[]; }; export function runQueryScenario( @@ -84,23 +98,38 @@ export function runQueryScenario( const builder = createBuilder(scenario.schema); const ast = asQueryInternals(scenario.query(builder)).ast; - const planDebugger = new AccumulatorDebugger(); - const optimizedAST = planQueryOnce(ast, costModel, planDebugger); + const optimizedAST = planQueryOnce(ast, costModel, new AccumulatorDebugger()); const debug = new ScenarioDebug(); const delegate = newQueryDelegate(lc, testLogConfig, db, scenario.schema); delegate.debug = debug; - const input = buildPipeline(optimizedAST, delegate, 'query-scenario'); + const planDebugger = new AccumulatorDebugger(); + const input = buildPipeline( + ast, + delegate, + 'query-scenario', + costModel, + lc, + planDebugger, + ); const sink = new Catch(input); - sink.fetch(); + // SQL shape alone can be misleading because repeated query text can hide + // multiple physical scans with different bind values. ScenarioDebug keeps the + // SQL list readable by compacting repeated text into a calls count, so the + // intersection scenarios can prove both membership scans happen. + const rows = sink + .fetch() + .filter(node => node !== 'yield') + .map(node => node.row); sink.destroy(); return { ast, optimizedAST, planDebug: planDebugger.format(), - sql: debug.queries, + sql: debug.compactQueries(), + rows, }; } diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts new file mode 100644 index 0000000000..71549affa9 --- /dev/null +++ b/packages/zqlite/src/test/query-scenarios/scenarios/child-domain-intersection.ts @@ -0,0 +1,116 @@ +import type {QueryScenario} from '../../query-scenario.ts'; +import { + colName, + createEducationAppTables, + educationAppRelationships, + educationAppSchema, + educationAppTables, + relationshipName, + tableName, +} from '../education-app.ts'; + +const assignment = educationAppTables.assignment; +const assignmentToStudent = educationAppTables.assignment_to_student; +const assignmentToStudentRelationship = relationshipName( + educationAppRelationships.assignment, + 'assignment_to_student', +); + +export default { + name: 'child IN plus exclusion narrows before auto flipping', + schema: educationAppSchema, + seed: db => { + const tables = createEducationAppTables(db); + const assignment = tables.assignment; + const assignmentToStudent = tables.assignment_to_student; + + const assignmentStmt = db.prepare( + `INSERT INTO ${tableName(assignment)} (${colName(assignment, 'id')}, ${colName(assignment, 'teacher_id')}, ${colName(assignment, 'archived_at')}, ${colName(assignment, 'created_at')}) VALUES (?, ?, ?, ?)`, + ); + for (let i = 1; i <= 2_000; i++) { + assignmentStmt.run(i, 2, null, i); + } + + const membershipStmt = db.prepare( + `INSERT INTO ${tableName(assignmentToStudent)} (${colName(assignmentToStudent, 'assignment_id')}, ${colName(assignmentToStudent, 'student_id')}, ${colName(assignmentToStudent, 'created_at')}) VALUES (?, ?, ?)`, + ); + membershipStmt.run(101, 'student-1', 101); + membershipStmt.run(102, 'student-1', 102); + membershipStmt.run(1_500, 'student-2', 1_500); + }, + query: builder => + builder[assignment.name] + .whereExists(assignmentToStudentRelationship, q => + q.where(({and, cmp}) => + and( + cmp(colName(assignmentToStudent, 'student_id'), 'IN', [ + 'student-1', + 'student-2', + ]), + cmp(colName(assignmentToStudent, 'student_id'), '!=', 'student-2'), + ), + ), + ) + .orderBy(colName(assignment, 'created_at'), 'desc') + .orderBy(colName(assignment, 'id'), 'asc'), + expectations: { + optimizedAST: { + where: { + type: 'correlatedSubquery', + flip: true, + related: { + subquery: { + where: { + type: 'simple', + op: '=', + left: {type: 'column', name: 'student_id'}, + right: {type: 'literal', value: 'student-1'}, + }, + }, + }, + }, + }, + planDebug: ['flipped'], + // Submitted ZQL: + // + // assignment.whereExists( + // assignment_to_student, + // student_id IN ['student-1', 'student-2'] + // AND student_id != 'student-2' + // ) + // + // Naive plan: + // + // assignment + // `-- for each assignment, probe membership by assignment_id + // and evaluate both student predicates + // + // Optimized plan: + // + // student_id IN ['student-1', 'student-2'] + // AND student_id != 'student-2' + // | + // v + // student_id = 'student-1' + // + // assignment_to_student(student_id = 'student-1') + // `-- fetch assignment by assignment_id + // + // Intuition: + // + // The child filter simplifies to one student before planning, so the + // planner starts from the small membership index instead of walking every + // assignment. + sql: [ + { + table: 'assignment_to_student', + sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', + }, + { + table: 'assignment', + sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 2, + }, + ], + }, +} satisfies QueryScenario; diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts new file mode 100644 index 0000000000..eac5863b34 --- /dev/null +++ b/packages/zqlite/src/test/query-scenarios/scenarios/contradictory-parent-domain.ts @@ -0,0 +1,70 @@ +import type {QueryScenario} from '../../query-scenario.ts'; +import { + colName, + createEducationAppTables, + educationAppSchema, + educationAppTables, + tableName, +} from '../education-app.ts'; + +const assignment = educationAppTables.assignment; + +export default { + name: 'contradictory same column parent filters collapse to false', + schema: educationAppSchema, + seed: db => { + const tables = createEducationAppTables(db); + const assignment = tables.assignment; + + const assignmentStmt = db.prepare( + `INSERT INTO ${tableName(assignment)} (${colName(assignment, 'id')}, ${colName(assignment, 'teacher_id')}, ${colName(assignment, 'archived_at')}, ${colName(assignment, 'created_at')}) VALUES (?, ?, ?, ?)`, + ); + for (let i = 1; i <= 100; i++) { + assignmentStmt.run(i, i % 2 === 0 ? 1 : 2, null, i); + } + }, + query: builder => + builder[assignment.name] + .where(({and, cmp}) => + and( + cmp(colName(assignment, 'teacher_id'), '=', 1), + cmp(colName(assignment, 'teacher_id'), '!=', 1), + ), + ) + .orderBy(colName(assignment, 'created_at'), 'desc') + .orderBy(colName(assignment, 'id'), 'asc'), + expectations: { + optimizedAST: { + where: { + type: 'or', + conditions: [], + }, + }, + // Submitted ZQL: + // + // assignment.where(teacher_id = 1 AND teacher_id != 1) + // + // Naive plan: + // + // assignment + // `-- scan rows and test both predicates one row at a time + // + // Optimized plan: + // + // teacher_id = 1 AND teacher_id != 1 + // | + // v + // FALSE + // + // Intuition: + // + // No row can be both teacher 1 and not teacher 1, so the planner emits + // an empty query instead of scanning assignment. + sql: [ + { + table: 'assignment', + sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE FALSE ORDER BY "created_at" desc, "id" asc', + }, + ], + }, +} satisfies QueryScenario; diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts index 8da1ae9d3d..29e58646db 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/duplicate-exists-or-dedupe.ts @@ -18,40 +18,6 @@ const assignmentToStudentRelationship = relationshipName( export default { name: 'duplicate exists OR could dedupe before planning', - knownFailure: { - reason: - 'A OR A is exactly A, but duplicate exists branches still reach join enumeration as separate correlated subqueries.', - current: ` -OR - exists assignment_to_student where student_id = student-1 - exists assignment_to_student where student_id = student-1 - -Plan shape today: - -two equivalent exists branches enter the planner -`, - desired: ` -Deduped predicate: - -exists assignment_to_student where student_id = student-1 - -Desired plan shape: - -assignment_to_student student-1 => assignment -`, - currentSQL: [ - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', - }, - ], - engineIdea: - 'Add canonical condition keys during boolean normalization. Use them to collapse duplicate OR branches and duplicate AND branches before costing.', - }, schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); @@ -101,6 +67,33 @@ assignment_to_student student-1 => assignment flip: true, }, }, + // Submitted ZQL: + // + // assignment.where( + // EXISTS assignment_to_student(student_id = 'student-1') + // OR EXISTS assignment_to_student(student_id = 'student-1') + // ) + // + // Naive plan: + // + // assignment + // |-- probe membership for student-1 + // `-- probe membership for student-1 again + // + // Optimized plan: + // + // identical EXISTS branches + // | + // v + // EXISTS membership(student_id = 'student-1') + // + // assignment_to_student(student_id = 'student-1') + // `-- fetch assignment by assignment_id + // + // Intuition: + // + // Running the same child lookup twice cannot add rows, so the planner + // keeps one branch and flips it to the membership index. sql: [ { table: 'assignment_to_student', @@ -109,6 +102,7 @@ assignment_to_student student-1 => assignment { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts index adaa4b4870..46298b07fb 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-false.ts @@ -60,6 +60,32 @@ export default { ], }, }, + // Submitted ZQL: + // + // assignment + // .where(archived_at IS null) + // .whereExists(assignment_to_student, student_id = 'student-1', { + // flip: false, + // }) + // + // Naive plan: + // + // assignment(archived_at IS null) + // `-- for each assignment, probe membership by assignment_id + // + // Optimized plan: + // + // assignment(archived_at IS null) + // `-- probe assignment_to_student( + // assignment_id = current assignment.id, + // student_id = 'student-1' + // ) + // + // Intuition: + // + // The user explicitly pinned the parent as the root. The 26 calls are the + // repeated per assignment probes plus the final probe that exhausts the + // parent stream. sql: [ { table: 'assignment', @@ -68,6 +94,7 @@ export default { { table: 'assignment_to_student', sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "assignment_id" = ? AND "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', + calls: 26, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts index 93f1b23cdf..5c90736537 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/explicit-flip-true.ts @@ -60,6 +60,29 @@ export default { ], }, }, + // Submitted ZQL: + // + // assignment + // .where(archived_at IS null) + // .whereExists(assignment_to_student, student_id = 'student-1', { + // flip: true, + // }) + // + // Naive plan: + // + // assignment(archived_at IS null) + // `-- for each assignment, probe membership by assignment_id + // + // Optimized plan: + // + // assignment_to_student(student_id = 'student-1') + // `-- fetch assignment by assignment_id + // `-- keep it only if archived_at IS null + // + // Intuition: + // + // The user explicitly asked to flip the EXISTS, so the child side is the + // root even before the cost model gets a vote. sql: [ { table: 'assignment_to_student', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts new file mode 100644 index 0000000000..59fd40ae18 --- /dev/null +++ b/packages/zqlite/src/test/query-scenarios/scenarios/impossible-exists-or-parent-filter.ts @@ -0,0 +1,87 @@ +import type {QueryScenario} from '../../query-scenario.ts'; +import { + colName, + createEducationAppTables, + educationAppRelationships, + educationAppSchema, + educationAppTables, + relationshipName, + tableName, +} from '../education-app.ts'; + +const assignment = educationAppTables.assignment; +const assignmentToStudent = educationAppTables.assignment_to_student; +const assignmentToStudentRelationship = relationshipName( + educationAppRelationships.assignment, + 'assignment_to_student', +); + +export default { + name: 'OR with impossible exists keeps only parent predicate', + schema: educationAppSchema, + seed: db => { + const tables = createEducationAppTables(db); + const assignment = tables.assignment; + + const assignmentStmt = db.prepare( + `INSERT INTO ${tableName(assignment)} (${colName(assignment, 'id')}, ${colName(assignment, 'teacher_id')}, ${colName(assignment, 'archived_at')}, ${colName(assignment, 'created_at')}) VALUES (?, ?, ?, ?)`, + ); + for (let i = 1; i <= 2_000; i++) { + assignmentStmt.run(i, i % 100 === 0 ? 1 : 2, null, i); + } + }, + query: builder => + builder[assignment.name] + .where(({cmp, exists, or}) => + or( + cmp(colName(assignment, 'teacher_id'), '=', 1), + exists(assignmentToStudentRelationship, q => + q.where(colName(assignmentToStudent, 'student_id'), 'IN', []), + ), + ), + ) + .orderBy(colName(assignment, 'created_at'), 'desc') + .orderBy(colName(assignment, 'id'), 'asc'), + expectations: { + optimizedAST: { + where: { + type: 'simple', + left: {type: 'column', name: 'teacher_id'}, + op: '=', + right: {type: 'literal', value: 1}, + }, + }, + // Submitted ZQL: + // + // assignment.where( + // teacher_id = 1 + // OR EXISTS assignment_to_student(student_id IN []) + // ) + // + // Naive plan: + // + // assignment + // |-- keep rows where teacher_id = 1 + // `-- also ask membership for an empty student set + // + // Optimized plan: + // + // EXISTS membership(student_id IN []) -> FALSE + // + // teacher_id = 1 OR FALSE + // | + // v + // teacher_id = 1 + // + // Intuition: + // + // An empty IN list cannot find a child row, so the OR falls back to the + // only branch that can return assignments. + sql: [ + { + table: 'assignment', + sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "teacher_id" = ? ORDER BY "created_at" desc, "id" asc', + }, + ], + }, +} satisfies QueryScenario; diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/index.ts b/packages/zqlite/src/test/query-scenarios/scenarios/index.ts index b878303c36..8f7454dfb0 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/index.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/index.ts @@ -1,8 +1,11 @@ import type {QueryScenario} from '../../query-scenario.ts'; import type {educationAppSchema} from '../education-app.ts'; +import childDomainIntersection from './child-domain-intersection.ts'; +import contradictoryParentDomain from './contradictory-parent-domain.ts'; import duplicateExistsOrDedupe from './duplicate-exists-or-dedupe.ts'; import explicitFlipFalse from './explicit-flip-false.ts'; import explicitFlipTrue from './explicit-flip-true.ts'; +import impossibleExistsOrParentFilter from './impossible-exists-or-parent-filter.ts'; import mixedOrWithFalseAndMembership from './mixed-or-with-false-and-membership.ts'; import orWithFalseBranch from './or-with-false-branch.ts'; import parentOrExistsUnionRoots from './parent-or-exists-union-roots.ts'; @@ -12,15 +15,20 @@ import sharedParentFilterOrFactor from './shared-parent-filter-or-factor.ts'; import singleExistsChildOr from './single-exists-child-or.ts'; import studentMembershipMixedOr from './student-membership-mixed-or.ts'; import studentMembershipSimpleExists from './student-membership-simple-exists.ts'; +import teacherFilterOrIn from './teacher-filter-or-in.ts'; import teacherFilterWithMembership from './teacher-filter-with-membership.ts'; import twoStudentMembershipOr from './two-student-membership-or.ts'; export default [ studentMembershipMixedOr, studentMembershipSimpleExists, + teacherFilterOrIn, + contradictoryParentDomain, + childDomainIntersection, teacherFilterWithMembership, explicitFlipFalse, explicitFlipTrue, + impossibleExistsOrParentFilter, orWithFalseBranch, mixedOrWithFalseAndMembership, twoStudentMembershipOr, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts b/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts index a46727b02d..72252ddf84 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/mixed-or-with-false-and-membership.ts @@ -78,6 +78,40 @@ export default { }, }, planDebug: ['flipped'], + // Submitted ZQL: + // + // assignment.where( + // archived_at IS null + // AND ( + // teacher_id = 1 + // OR FALSE + // OR EXISTS assignment_to_student(student_id = 'student-1') + // ) + // ) + // + // Naive plan: + // + // assignment(archived_at IS null) + // |-- check teacher_id = 1 + // |-- carry a FALSE branch that can never match + // `-- if needed, probe membership by assignment_id + // + // Optimized plan: + // + // teacher_id = 1 OR FALSE OR EXISTS membership(...) + // | + // v + // teacher_id = 1 OR EXISTS membership(...) + // + // assignment(archived_at IS null, teacher_id = 1) ----------. + // +-- union + // assignment_to_student(student_id = 'student-1') -> parent -' + // `-- keep parent only if archived_at IS null + // + // Intuition: + // + // The FALSE branch disappears before costing, then the remaining OR gets + // the same split root treatment as the mixed parent and child case. sql: [ { table: 'assignment', @@ -90,6 +124,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND "archived_at" IS ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts b/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts index 3d3bb3960a..dce346b831 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/or-with-false-branch.ts @@ -38,6 +38,27 @@ export default { right: {type: 'literal', value: 1}, }, }, + // Submitted ZQL: + // + // assignment.where(teacher_id = 1 OR FALSE) + // + // Naive plan: + // + // assignment + // `-- check teacher_id = 1 + // `-- also carry a FALSE branch that can never match + // + // Optimized plan: + // + // teacher_id = 1 OR FALSE + // | + // v + // teacher_id = 1 + // + // Intuition: + // + // FALSE contributes no rows to an OR, so it should not make costing + // think this query is less selective than the real teacher filter. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts index 7712966184..b23479b378 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/parent-or-exists-union-roots.ts @@ -17,45 +17,7 @@ const assignmentToStudentRelationship = relationshipName( ); export default { - name: 'parent OR exists could use union roots instead of broad parent scan', - knownFailure: { - reason: - 'A mixed OR with one parent predicate and one child exists can be served as a union of two selective roots. The planner currently keeps a broad parent root for the parent side and a flipped child root for the child side.', - current: ` -OR - assignment.teacher_id = 1 - exists assignment_to_student where student_id = student-1 - -Plan shape today: - -assignment all rows -assignment_to_student student-1 => assignment -`, - desired: ` -OR as two selective roots: - -assignment teacher_id = 1 -assignment_to_student student-1 => assignment - -Then merge rows by assignment id. -`, - currentSQL: [ - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" ORDER BY "created_at" desc, "id" asc', - }, - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', - }, - ], - engineIdea: - 'Add an OR union plan node that can execute each branch from its cheapest root, merge rows by primary key, and preserve the requested ordering after the union.', - }, + name: 'parent OR exists uses union roots instead of broad parent scan', schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); @@ -93,6 +55,46 @@ Then merge rows by assignment id. .orderBy(colName(assignment, 'created_at'), 'desc') .orderBy(colName(assignment, 'id'), 'asc'), expectations: { + optimizedAST: { + where: { + type: 'or', + conditions: [ + {}, + { + type: 'correlatedSubquery', + flip: true, + }, + ], + }, + }, + planDebug: ['FO ⋈ assignment_to_student: flipped'], + // Submitted ZQL: + // + // assignment.where( + // teacher_id = 1 + // OR EXISTS assignment_to_student(student_id = 'student-1') + // ) + // + // Naive plan: + // + // assignment + // |-- check teacher_id = 1 + // `-- if needed, probe membership by assignment_id + // + // Optimized plan: + // + // assignment(teacher_id = 1) -------------------------------. + // +-- union + // assignment_to_student(student_id = 'student-1') -> parent -' + // | + // v + // assignment ids + // + // Intuition: + // + // Each OR branch gets its best root. Parent matches come from the + // teacher index, membership matches come from the child index, and the + // resulting assignment ids are merged. sql: [ { table: 'assignment', @@ -105,6 +107,7 @@ Then merge rows by assignment id. { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts index 16fedad574..7a47a52e82 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-and-intersect.ts @@ -17,44 +17,7 @@ const assignmentToStudentRelationship = relationshipName( ); export default { - name: 'same relationship AND could intersect child scans before parent lookup', - knownFailure: { - reason: - 'Two required memberships over the same relationship can be intersected on assignment_id before loading parent rows. The planner currently chooses one child root, loads the parent, then probes the second child relationship.', - current: ` -AND - exists assignment_to_student where student_id = student-1 - exists assignment_to_student where student_id = student-2 - -Plan shape today: - -assignment_to_student student-1 => assignment => probe assignment_to_student student-2 -`, - desired: ` -AND as child intersection: - -assignment_to_student student-1 -assignment_to_student student-2 -both streams grouped by assignment_id -then assignment lookup -`, - currentSQL: [ - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND TRUE ORDER BY "created_at" desc, "id" asc', - }, - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "assignment_id" = ? AND "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - ], - engineIdea: - 'Teach join enumeration about sibling exists clauses that share the same relationship and correlation under AND. It could introduce an intersection node keyed by the child correlation fields before parent lookup.', - }, + name: 'same relationship AND intersects child scans before parent lookup', schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); @@ -99,15 +62,42 @@ then assignment lookup .orderBy(colName(assignment, 'created_at'), 'desc') .orderBy(colName(assignment, 'id'), 'asc'), expectations: { + // Submitted ZQL: + // + // assignment + // .whereExists(assignment_to_student, student_id = 'student-1') + // .whereExists(assignment_to_student, student_id = 'student-2') + // + // Naive plan: + // + // assignment + // |-- probe membership for student-1 + // `-- probe membership for student-2 + // + // Optimized plan: + // + // assignment_to_student(student_id = 'student-1') -> assignment_ids --. + // intersect + // assignment_to_student(student_id = 'student-2') -> assignment_ids --' + // | + // v + // fetch assignment + // + // Intuition: + // + // The same assignment id must appear in both child streams. The SQL text + // has calls: 2 because it runs once per student before intersecting ids. sql: [ { table: 'assignment_to_student', sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', + calls: 2, }, { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', }, ], + rows: [{id: 102, teacher_id: 2, archived_at: null, created_at: 102}], }, } satisfies QueryScenario; diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts index d093dfdcf1..7f31c607ec 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/same-relationship-or-merge.ts @@ -18,42 +18,6 @@ const assignmentToStudentRelationship = relationshipName( export default { name: 'same relationship OR could merge into one flipped membership scan', - knownFailure: { - reason: - 'The planner currently flips each sibling exists independently. The desired plan would rewrite the OR into one exists over the same relationship, then flip that single child scan.', - current: ` -OR - exists assignment_to_student where student_id = student-1 - exists assignment_to_student where student_id = student-2 - -Plan shape today: - -assignment_to_student student-1 => assignment -assignment_to_student student-2 => assignment -`, - desired: ` -exists assignment_to_student where: - student_id = student-1 - OR - student_id = student-2 - -Desired plan shape: - -assignment_to_student student-1 OR student-2 => assignment -`, - currentSQL: [ - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', - }, - ], - engineIdea: - 'Add a boolean normalization pass that recognizes OR siblings with the same relationship, same correlation, and same exists options. Merge the child filters under one child OR before join enumeration.', - }, schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); @@ -97,25 +61,50 @@ assignment_to_student student-1 OR student-2 => assignment .orderBy(colName(assignment, 'created_at'), 'desc') .orderBy(colName(assignment, 'id'), 'asc'), expectations: { - // Current optimized AST keeps an OR with two flipped correlated subqueries. - // That means the optimized query shape is effectively two sibling child - // scans, each filtering one student_id, followed by parent lookups. The - // desired optimized AST is one flipped correlated subquery whose child - // filter is student-1 OR student-2, so one child scan feeds parent lookups. optimizedAST: { where: { type: 'correlatedSubquery', flip: true, }, }, + // Submitted ZQL: + // + // assignment.where( + // EXISTS assignment_to_student(student_id = 'student-1') + // OR EXISTS assignment_to_student(student_id = 'student-2') + // ) + // + // Naive plan: + // + // assignment + // |-- probe membership for student-1 + // `-- probe membership for student-2 + // + // Optimized plan: + // + // EXISTS membership(student_id = 'student-1') + // OR + // EXISTS membership(student_id = 'student-2') + // | + // v + // EXISTS membership(student_id IN ['student-1', 'student-2']) + // + // assignment_to_student(student_id IN ['student-1', 'student-2']) + // `-- fetch assignment by assignment_id + // + // Intuition: + // + // Two EXISTS branches over the same relationship are one child domain. + // Merging them keeps the plan on the selective membership index. sql: [ { table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE ("student_id" = ? OR "student_id" = ?) ORDER BY "assignment_id" asc, "student_id" asc', + sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" IN (SELECT value FROM json_each(?)) ORDER BY "assignment_id" asc, "student_id" asc', }, { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts index b46f02a742..43bab3adf0 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/shared-parent-filter-or-factor.ts @@ -18,46 +18,6 @@ const assignmentToStudentRelationship = relationshipName( export default { name: 'shared parent filter across OR could factor before flipping child', - knownFailure: { - reason: - 'Both OR branches repeat the same parent predicate. Factoring that predicate out would leave one same relationship child OR that can flip as a single membership scan.', - current: ` -OR - AND - archived_at IS null - exists student-1 - AND - archived_at IS null - exists student-2 - -Plan shape today: - -student-1 => assignment with archived_at -student-2 => assignment with archived_at -`, - desired: ` -AND - archived_at IS null - exists child where: - student-1 OR student-2 - -Desired plan shape: - -assignment_to_student student-1 OR student-2 => assignment archived_at IS null -`, - currentSQL: [ - { - table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', - }, - { - table: 'assignment', - sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND ("archived_at" IS ? OR "archived_at" IS ?) ORDER BY "created_at" desc, "id" asc', - }, - ], - engineIdea: - 'Run a small boolean algebra normalization before join enumeration. Factor common simple parent predicates out of OR branches, then run the same relationship OR merge pass on the remaining child predicates.', - }, schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); @@ -123,7 +83,10 @@ assignment_to_student student-1 OR student-2 => assignment archived_at IS null related: { subquery: { where: { - type: 'or', + type: 'simple', + op: 'IN', + left: {type: 'column', name: 'student_id'}, + right: {type: 'literal', value: ['student-1', 'student-2']}, }, }, }, @@ -131,14 +94,47 @@ assignment_to_student student-1 OR student-2 => assignment archived_at IS null ], }, }, + // Submitted ZQL: + // + // assignment.where( + // (archived_at IS null + // AND EXISTS assignment_to_student(student_id = 'student-1')) + // OR + // (archived_at IS null + // AND EXISTS assignment_to_student(student_id = 'student-2')) + // ) + // + // Naive plan: + // + // assignment + // |-- branch 1 checks archived_at, then probes student-1 + // `-- branch 2 checks archived_at again, then probes student-2 + // + // Optimized plan: + // + // shared archived_at IS null + // | + // v + // archived_at IS null + // AND EXISTS membership(student_id IN ['student-1', 'student-2']) + // + // assignment_to_student(student_id IN ['student-1', 'student-2']) + // `-- fetch assignment by assignment_id + // `-- keep it only if archived_at IS null + // + // Intuition: + // + // The duplicate parent filter is factored once, and the two child + // branches become one membership index scan. sql: [ { table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE ("student_id" = ? OR "student_id" = ?) ORDER BY "assignment_id" asc, "student_id" asc', + sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" IN (SELECT value FROM json_each(?)) ORDER BY "assignment_id" asc, "student_id" asc', }, { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND "archived_at" IS ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts index 2882a14dbc..f1c17c7c20 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/single-exists-child-or.ts @@ -58,14 +58,42 @@ export default { }, }, planDebug: ['flipped'], + // Submitted ZQL: + // + // assignment.whereExists( + // assignment_to_student, + // student_id = 'student-1' OR student_id = 'student-2' + // ) + // + // Naive plan: + // + // assignment + // `-- for each assignment, probe membership by assignment_id + // and test both student equality branches + // + // Optimized plan: + // + // student_id = 'student-1' OR student_id = 'student-2' + // | + // v + // student_id IN ['student-1', 'student-2'] + // + // assignment_to_student(student_id IN ['student-1', 'student-2']) + // `-- fetch assignment by assignment_id + // + // Intuition: + // + // A same column OR inside the child query is one IN lookup, and that + // lookup is selective enough to become the root. sql: [ { table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE ("student_id" = ? OR "student_id" = ?) ORDER BY "assignment_id" asc, "student_id" asc', + sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" IN (SELECT value FROM json_each(?)) ORDER BY "assignment_id" asc, "student_id" asc', }, { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts index 6320fed997..70d77ef41e 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-mixed-or.ts @@ -77,6 +77,33 @@ export default { }, }, planDebug: ['Best plan: Attempt 2', 'FO ⋈ assignment_to_student: flipped'], + // Submitted ZQL: + // + // assignment.where( + // archived_at IS null + // AND ( + // teacher_id = 1 + // OR EXISTS assignment_to_student(student_id = 'student-1') + // ) + // ) + // + // Naive plan: + // + // assignment(archived_at IS null) + // |-- check teacher_id = 1 + // `-- if needed, probe membership by assignment_id + // + // Optimized plan: + // + // assignment(archived_at IS null, teacher_id = 1) ----------. + // +-- union + // assignment_to_student(student_id = 'student-1') -> parent -' + // `-- keep parent only if archived_at IS null + // + // Intuition: + // + // The shared archived filter stays on both branches, while the OR splits + // so each side can use its own selective index. sql: [ { table: 'assignment', @@ -89,6 +116,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND "archived_at" IS ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts index 0e679259ea..af999ae09d 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/student-membership-simple-exists.ts @@ -60,6 +60,27 @@ export default { }, }, planDebug: ['flipped'], + // Submitted ZQL: + // + // assignment + // .where(archived_at IS null) + // .whereExists(assignment_to_student, student_id = 'student-1') + // + // Naive plan: + // + // assignment(archived_at IS null) + // `-- for each assignment, probe membership by assignment_id + // + // Optimized plan: + // + // assignment_to_student(student_id = 'student-1') + // `-- fetch assignment by assignment_id + // `-- keep it only if archived_at IS null + // + // Intuition: + // + // The student membership index is smaller than the live assignment set, + // so it is cheaper to start from the child row and look up its parent. sql: [ { table: 'assignment_to_student', @@ -68,6 +89,7 @@ export default { { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? AND "archived_at" IS ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], }, diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts new file mode 100644 index 0000000000..d400e2e2e0 --- /dev/null +++ b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-or-in.ts @@ -0,0 +1,72 @@ +import type {QueryScenario} from '../../query-scenario.ts'; +import { + colName, + createEducationAppTables, + educationAppSchema, + educationAppTables, + tableName, +} from '../education-app.ts'; + +const assignment = educationAppTables.assignment; + +export default { + name: 'same column parent OR compacts to an IN filter', + schema: educationAppSchema, + seed: db => { + const tables = createEducationAppTables(db); + const assignment = tables.assignment; + + const assignmentStmt = db.prepare( + `INSERT INTO ${tableName(assignment)} (${colName(assignment, 'id')}, ${colName(assignment, 'teacher_id')}, ${colName(assignment, 'archived_at')}, ${colName(assignment, 'created_at')}) VALUES (?, ?, ?, ?)`, + ); + for (let i = 1; i <= 2_000; i++) { + assignmentStmt.run(i, i % 25 === 0 ? 1 : 2, null, i); + } + }, + query: builder => + builder[assignment.name] + .where(({cmp, or}) => + or( + cmp(colName(assignment, 'teacher_id'), '=', 1), + cmp(colName(assignment, 'teacher_id'), '=', 2), + ), + ) + .orderBy(colName(assignment, 'created_at'), 'desc') + .orderBy(colName(assignment, 'id'), 'asc'), + expectations: { + optimizedAST: { + where: { + type: 'simple', + left: {type: 'column', name: 'teacher_id'}, + op: 'IN', + right: {type: 'literal', value: [1, 2]}, + }, + }, + // Submitted ZQL: + // + // assignment.where(teacher_id = 1 OR teacher_id = 2) + // + // Naive plan: + // + // assignment + // `-- test teacher_id = 1 + // `-- test teacher_id = 2 + // + // Optimized plan: + // + // teacher_id = 1 OR teacher_id = 2 + // | + // v + // teacher_id IN [1, 2] + // + // Intuition: + // + // Two equality checks on the same column are one indexed IN lookup. + sql: [ + { + table: 'assignment', + sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "teacher_id" IN (SELECT value FROM json_each(?)) ORDER BY "created_at" desc, "id" asc', + }, + ], + }, +} satisfies QueryScenario; diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts index 89a6c3c4b9..0505d141ee 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/teacher-filter-with-membership.ts @@ -61,6 +61,29 @@ export default { }, }, planDebug: ['semi'], + // Submitted ZQL: + // + // assignment + // .where(archived_at IS null AND teacher_id = 1) + // .whereExists(assignment_to_student, student_id = 'student-1') + // + // Naive plan: + // + // assignment(archived_at IS null AND teacher_id = 1) + // `-- for each matching assignment, probe membership + // + // Optimized plan: + // + // assignment(archived_at IS null AND teacher_id = 1) + // `-- probe assignment_to_student( + // assignment_id = current assignment.id, + // student_id = 'student-1' + // ) + // + // Intuition: + // + // The parent filters are already selective enough, so staying on + // assignment avoids starting from membership and then fetching parents. sql: [ { table: 'assignment', diff --git a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts index 98068b47b8..35614854f2 100644 --- a/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts +++ b/packages/zqlite/src/test/query-scenarios/scenarios/two-student-membership-or.ts @@ -17,7 +17,7 @@ const assignmentToStudentRelationship = relationshipName( ); export default { - name: 'OR across two membership branches flips both branches', + name: 'OR across two membership branches merges and flips one child scan', schema: educationAppSchema, seed: db => { const tables = createEducationAppTables(db); @@ -63,28 +63,59 @@ export default { expectations: { optimizedAST: { where: { - type: 'or', - conditions: [ - { - type: 'correlatedSubquery', - flip: true, + type: 'correlatedSubquery', + flip: true, + related: { + subquery: { + where: { + type: 'simple', + op: 'IN', + left: {type: 'column', name: 'student_id'}, + right: {type: 'literal', value: ['student-1', 'student-2']}, + }, }, - { - type: 'correlatedSubquery', - flip: true, - }, - ], + }, }, }, planDebug: ['flipped'], + // Submitted ZQL: + // + // assignment.where( + // EXISTS assignment_to_student(student_id = 'student-1') + // OR EXISTS assignment_to_student(student_id = 'student-2') + // ) + // + // Naive plan: + // + // assignment + // |-- probe membership for student-1 + // `-- probe membership for student-2 + // + // Optimized plan: + // + // EXISTS membership(student_id = 'student-1') + // OR + // EXISTS membership(student_id = 'student-2') + // | + // v + // EXISTS membership(student_id IN ['student-1', 'student-2']) + // + // assignment_to_student(student_id IN ['student-1', 'student-2']) + // `-- fetch assignment by assignment_id + // + // Intuition: + // + // Both OR branches ask the same relationship for the same parent key, so + // they can share one child index scan. sql: [ { table: 'assignment_to_student', - sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" = ? ORDER BY "assignment_id" asc, "student_id" asc', + sql: 'SELECT "assignment_id","student_id","created_at" FROM "assignment_to_student" WHERE "student_id" IN (SELECT value FROM json_each(?)) ORDER BY "assignment_id" asc, "student_id" asc', }, { table: 'assignment', sql: 'SELECT "id","teacher_id","archived_at","created_at" FROM "assignment" WHERE "id" = ? ORDER BY "created_at" desc, "id" asc', + calls: 3, }, ], },