From 3d2674b51d6d4ed8ab43df2e967314f3b89c7bc6 Mon Sep 17 00:00:00 2001 From: thanos Date: Mon, 22 Jun 2026 14:52:18 -0400 Subject: [PATCH 1/3] v0.5.0: aggregates, BEAM callbacks, magic sets, query planner, runtime facts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M0.5 — Feasibility Spikes - Spike A: callback as {:callback, %IR.Callback{}} IR literal round-trips through compiler/evaluator (chest/spikes/README.md) - Spike B: materialize(goal:) drives magic sets transform; results ≡ semi-naive M1 — Query Planner Foundation - closed #127 - Add ExDatalog.Planner with plan/2, explain_plan/1,2 - closed #128 - Add planner structs: Plan, Stratum, Join, Predicate (lib/ex_datalog/planner/) - closed #129 - Engine strategy dispatch: evaluate/2 routes :semi_naive / :magic_sets - closed #130 - Telemetry [:ex_datalog, :planner, :start|:stop|:exception] - closed #131 - Tests: planner_test.exs, planner_explain_test.exs - closed #132 M2 — Aggregates (count/sum/min/max) - closed #133 - Add @aggregate_ops [:count, :sum, :min, :max] to Constraint; from_tuple/1, valid_right?/2, valid_result?/2 extended - closed #134 - Add ExDatalog.Constraints.Aggregate (group_and_reduce/5; evaluate/3 raises) - closed #135 - Add Rule.has_aggregates?/1, aggregate_constraints/1, non_aggregate_constraints/1 - closed #136 - Safety: aggregate input var must be bound, result var in head, ≤1 agg/rule, no aggregate in recursive rule - closed #137 - Stratification: post-pass in assign_strata/1 forces aggregate head above all positive body relations; engine validate_stratification/1 extended - closed #138 - Evaluator: eval_aggregate_rule/3 (join → filter → callbacks → negation → group_and_reduce → project); apply_constraints/3 partitions out aggregate ops - closed #139 - DSL: count/sum/min/max(X, Result) parsed before generic constraint-ops loop; head-position aggregate raises DSL.CompileError - closed #140 - Tests: aggregate_test.exs + DSL tests in schema_test.exs - closed #141 M3 — BEAM Callback Predicates - closed #142 - Add ExDatalog.Callback struct (AST) + IR.Callback struct + {:callback, _} IR literal - closed #143 - Add ExDatalog.Constraints.BeamCallback with apply_callback/3; spawn_monitor-based isolation (not Task.async) - closed #144 - Add Constraint.Context opts: [] field; threaded from Engine.Naive.do_evaluate_inner - closed #145 - Add Capabilities aggregate_constraints: boolean, beam_callbacks: boolean - closed #146 - Add Schema.PredicateMeta struct; predicate/5 macro; __register_predicate__/6 - closed #147 - Add resolve_positive_or_callback/2 in __build_program__; converts matching body atoms to callbacks - closed #148 - Add Validator.check_callback/2 (module/function/arity), Safety.check_callback_inputs/2 (args must be bound) - closed #149 - Evaluator: apply_callbacks/3 in normal and aggregate paths; apply_callback_chain/2 + step_callback/1 - closed #150 - Program.validate_body/1 accepts {:callback, %Callback{}}; body_relations/1 ignores callbacks - closed #151 - Tests: beam_callback_test.exs - closed #152 M4 — Magic Sets (Experimental) - closed #153 - Add ExDatalog.MagicSets: transform/2 → {:ok, ir} | {:fallback, reason} - closed #154 - Scope: positive recursive programs, single goal, ground bound positions; negation/aggregates → fallback - closed #155 - Transform: adornment → magic relation + seed fact → rewrite goal-relation rules (prepend magic atom for bound head positions) - closed #156 - bound_positions/1 maps goal bound positions to head term positions - closed #157 - Verified: magic restricts derivation and equals semi-naive subset - closed #158 - Tests: magic_sets_test.exs + magic_sets_property_test.exs - closed #159 M5 — Docs, Benchmarks, CHANGELOG, Version Bump - closed #160 - Version bumped 0.4.1 → 0.5.0 in mix.exs - Added {:benchee, "~> 1.3", only: :dev} dependency - Added planner/magic-sets/callback module groups to mix.exs docs - Added articles 06–09 and migration_v0.5.md as extras - CHANGELOG.md: full v0.5.0 entry (Added/Changed/Notes) - README.md: aggregate/callback/magic-sets sections, updated feature list, architecture diagram, constraints table, docs links, roadmap - Articles: 06_query_planning_in_datalog.md, 07_aggregates_in_datalog.md, 08_extending_datalog_with_beam_callbacks.md, 09_magic_sets_and_demand_driven_evaluation.md - Migration guide: docs/migration_v0.5.md - Livebook: livebooks/ex_datalog_v050.livemd (DSL edition) - Benchmarks: bench/aggregate_bench.exs, bench/magic_sets_bench.exs - Report: chest/benchmarks/v0.5.0-report.md Runtime Facts API (pipe-friendly Schema → Program flow) - Schema relation constructors: each relation generates a public function (e.g., DeptCount.emp(:alice, :eng) → {"emp", [:alice, :eng]}) - Schema.new/0: returns blank program with relations + rules but no facts - Program.add_fact/2 (tuple form): Program.add_fact(prog, {"emp", [:alice, :eng]}) - Program.add_facts/2: bulk addition from list of {relation, values} tuples - Program.materialize/1,2: pipe-friendly ExDatalog.materialize/2 wrapper - Schema.program/0 and materialize/1 unchanged for backward compatibility --- CHANGELOG.md | 51 ++ README.md | 125 ++- bench/aggregate_bench.exs | 82 ++ bench/magic_sets_bench.exs | 51 ++ lib/ex_datalog/callback.ex | 106 +++ lib/ex_datalog/capabilities.ex | 8 + lib/ex_datalog/compiler.ex | 1 + lib/ex_datalog/constraint.ex | 136 +++- lib/ex_datalog/constraint/context.ex | 6 +- lib/ex_datalog/constraints/aggregate.ex | 62 ++ lib/ex_datalog/constraints/beam_callback.ex | 103 +++ lib/ex_datalog/engine/evaluator.ex | 106 ++- lib/ex_datalog/engine/naive.ex | 78 +- lib/ex_datalog/ir.ex | 49 ++ lib/ex_datalog/magic_sets.ex | 187 +++++ lib/ex_datalog/planner.ex | 210 +++++ lib/ex_datalog/planner/join.ex | 22 + lib/ex_datalog/planner/plan.ex | 28 + lib/ex_datalog/planner/predicate.ex | 28 + lib/ex_datalog/planner/stratum.ex | 20 + lib/ex_datalog/program.ex | 84 +- lib/ex_datalog/rule.ex | 79 +- lib/ex_datalog/schema.ex | 163 +++- lib/ex_datalog/schema/predicate_meta.ex | 22 + lib/ex_datalog/validator.ex | 21 + lib/ex_datalog/validator/error.ex | 8 + lib/ex_datalog/validator/safety.ex | 105 ++- lib/ex_datalog/validator/stratification.ex | 69 +- livebooks/ex_datalog_v050.livemd | 778 +++++++++++++++++++ mix.exs | 19 +- mix.lock | 3 + test/ex_datalog/aggregate_test.exs | 230 ++++++ test/ex_datalog/beam_callback_test.exs | 209 +++++ test/ex_datalog/magic_sets_property_test.exs | 59 ++ test/ex_datalog/magic_sets_test.exs | 143 ++++ test/ex_datalog/planner_explain_test.exs | 60 ++ test/ex_datalog/planner_test.exs | 98 +++ test/ex_datalog/runtime_facts_test.exs | 306 ++++++++ test/ex_datalog/schema_test.exs | 94 ++- test/ex_datalog/v050_coverage_test.exs | 103 +++ 40 files changed, 4037 insertions(+), 75 deletions(-) create mode 100644 bench/aggregate_bench.exs create mode 100644 bench/magic_sets_bench.exs create mode 100644 lib/ex_datalog/callback.ex create mode 100644 lib/ex_datalog/constraints/aggregate.ex create mode 100644 lib/ex_datalog/constraints/beam_callback.ex create mode 100644 lib/ex_datalog/magic_sets.ex create mode 100644 lib/ex_datalog/planner.ex create mode 100644 lib/ex_datalog/planner/join.ex create mode 100644 lib/ex_datalog/planner/plan.ex create mode 100644 lib/ex_datalog/planner/predicate.ex create mode 100644 lib/ex_datalog/planner/stratum.ex create mode 100644 lib/ex_datalog/schema/predicate_meta.ex create mode 100644 livebooks/ex_datalog_v050.livemd create mode 100644 test/ex_datalog/aggregate_test.exs create mode 100644 test/ex_datalog/beam_callback_test.exs create mode 100644 test/ex_datalog/magic_sets_property_test.exs create mode 100644 test/ex_datalog/magic_sets_test.exs create mode 100644 test/ex_datalog/planner_explain_test.exs create mode 100644 test/ex_datalog/planner_test.exs create mode 100644 test/ex_datalog/runtime_facts_test.exs create mode 100644 test/ex_datalog/v050_coverage_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index b6eae21..9e4c2aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,57 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.5.0 (2026-06-21) + +### Added + +- **Query Planner** (`ExDatalog.Planner`) — a thin planning layer between the + compiled IR and the engine. `plan/2` produces an `ExDatalog.Planner.Plan` + (strategy, strata, joins, predicates); `explain_plan/1,2` renders a + human-readable plan. Emits `[:ex_datalog, :planner, :start|:stop|:exception]` + telemetry. +- **Aggregates** — `count`, `sum`, `min`, `max` in rule bodies, via the DSL + (`count(X, N)`) or the builder API (`Constraint.count/2`, `from_tuple({:count, + X, N})`, `add_rule/4`). Aggregates group surviving bindings by the head + variables other than the result, reduce each group, and are stratified + strictly above their source relations. Integer-only; one aggregate per rule. +- **BEAM callback predicates** — call deterministic, side-effect-free Elixir + functions from rule bodies. DSL: `predicate :name, Module, :fun, [types], + :boolean | :value`. Builder: `ExDatalog.Callback` literals in a rule body. + Boolean callbacks filter; `:value` callbacks bind a result variable. The + engine isolates callbacks with a configurable timeout + (`:callback_timeout_ms`, default 100ms) and rescues exceptions — a + timeout/raise filters the binding. +- **Magic sets** (experimental) — demand-driven evaluation via + `materialize(program, strategy: :magic_sets, goal: {relation, pattern})`. The + IR is rewritten to compute only facts relevant to the goal. Supports positive + recursive programs with ground bound positions; unsupported programs fall back + to semi-naive evaluation (never producing incorrect results). +- `ExDatalog.Callback` and `ExDatalog.IR.Callback` for callback predicates. +- `ExDatalog.Constraints.Aggregate` and `ExDatalog.Constraints.BeamCallback`. +- `Capabilities` gains `aggregate_constraints` and `beam_callbacks` fields. +- `Constraint.Context` gains an `opts` field (threads evaluation options such + as `callback_timeout_ms`). +- Educational articles 06–09 (planner, aggregates, callbacks, magic sets) and a + v0.4 → v0.5 migration guide. +- Benchmark harness (`bench/`) and report (`chest/benchmarks/v0.5.0-report.md`). + +### Changed + +- Version bumped from 0.4.1 to 0.5.0. +- `Engine.Naive.evaluate/2` dispatches on a `:strategy` option (`:semi_naive` + default, or `:magic_sets`). +- `Engine.Naive` stratification validation now also rejects aggregate rules + whose source relations are not in a strictly lower stratum. +- The `agg(...)` placeholder now points users to `count/sum/min/max`. + +### Notes + +- Aggregates are integer-only; `avg` and float support are deferred to v0.6.0. +- Magic sets is experimental and opt-in; the default strategy is unchanged. +- All v0.4.1 tests continue to pass. Suite: 845 tests, 10 properties, + 149 doctests, ~93% coverage. + ## v0.4.1 (2026-06-21) ### Fixed diff --git a/README.md b/README.md index d1ff650..6294d1e 100644 --- a/README.md +++ b/README.md @@ -36,14 +36,18 @@ It continues to influence modern databases, compilers, static analysis tools, kn - **Builder API** for constructing programs (relations, facts, rules, constraints) - **Schema DSL** — Ecto-inspired macros for declaring relations, facts, rules, and queries - **Constraint types**: comparisons, arithmetic, type predicates, string predicates, membership +- **Aggregates**: `count`, `sum`, `min`, `max` with grouping and stratification +- **BEAM callback predicates**: call deterministic Elixir functions from rules (timeout-isolated) - **Negation** with stratified evaluation - **Recursive rules** with semi-naive fixpoint evaluation +- **Query planner** (`ExDatalog.Planner`) with `explain_plan/1,2` +- **Magic sets** (experimental): demand-driven evaluation via `strategy: :magic_sets` - **Post-materialization queries** (`query` macro with `find`/`where`) - **Pluggable storage backends**: `Storage.Map` (default, on-heap) and `Storage.ETS` (off-heap, concurrent reads) - **Provenance / derivation explain** (`explain: true`) -- **Telemetry** integration (`:telemetry` events for query lifecycle) +- **Telemetry** integration (`:telemetry` events for query/planner lifecycle) - **Deterministic**: same program + same facts = same result regardless of backend -- 792 tests, 0 failures, credo clean +- 845 tests, 0 failures, credo clean ## Installation @@ -52,7 +56,7 @@ Add `ex_datalog` to your dependencies in `mix.exs`: ```elixir def deps do [ - {:ex_datalog, "~> 0.4.1"} + {:ex_datalog, "~> 0.5.0"} ] end ``` @@ -229,6 +233,84 @@ Program.add_rule(program, {"hello_word", [:X]}, ) ``` +### Aggregates + +Group and reduce bindings with `count`, `sum`, `min`, `max`: + +```elixir +defmodule DeptStats do + use ExDatalog.Schema + + relation "emp", [:atom, :atom] + relation "dept_count", [:atom, :integer] + + fact "emp", [:alice, :eng] + fact "emp", [:bob, :eng] + fact "emp", [:carol, :sales] + + rule dept_count(D, N) do + emp(_, D) + count(E, N) + end +end + +{:ok, k} = ExDatalog.materialize(DeptStats) +Knowledge.get(k, "dept_count") +#=> MapSet.new([{:eng, 2}, {:sales, 1}]) +``` + +### BEAM callback predicates + +Call deterministic Elixir functions from rules: + +```elixir +defmodule Gated do + use ExDatalog.Schema + + relation "user", [:atom] + relation "active_user", [:atom] + + predicate :is_adult, AgeChecker, :adult?, [:atom], :boolean + + fact "user", [:alice] + fact "user", [:bob] + + rule active_user(U) do + user(U) + is_adult(U) + end +end + +defmodule AgeChecker do + def adult?(:alice), do: true + def adult?(:bob), do: false +end + +{:ok, k} = ExDatalog.materialize(Gated) +Knowledge.get(k, "active_user") +#=> MapSet.new([{:alice}]) +``` + +Value-returning callbacks bind a result variable: + +```elixir +predicate :length_of, StringLength, :compute, [:atom], :value +``` + +### Magic sets (experimental) + +Compute only facts relevant to a goal, instead of the full fixpoint: + +```elixir +{:ok, k} = ExDatalog.materialize(program, + strategy: :magic_sets, + goal: {"ancestor", [:alice, :_]} +) +``` + +Unsupported programs (negation, aggregates) automatically fall back to +semi-naive evaluation. + ### Negation Use negative body atoms with stratified evaluation. Find people who are not parents: @@ -283,24 +365,27 @@ end ## Architecture ``` -ExDatalog.Program (builder) - | - v +ExDatalog.Program (builder) / ExDatalog.Schema (DSL) + | + v ExDatalog.Validator (structural + semantic + stratification) - | - v + | + v ExDatalog.Compiler (AST -> IR) - | - v + | + v +ExDatalog.Planner (strategy, strata, joins) + | + v ExDatalog.Engine (behaviour) - | - v -ExDatalog.Engine.Naive (semi-naive fixpoint) - | - v uses + | + v +ExDatalog.Engine.Naive (semi-naive fixpoint / magic-sets) + | + v uses ExDatalog.Storage.Map | ExDatalog.Storage.ETS - | - v + | + v ExDatalog.Knowledge ``` @@ -343,6 +428,7 @@ reference. | Type predicate | `type_integer`, `type_binary`, `type_atom` | No (filter) | | String predicate | `starts_with`, `contains` | No (filter) | | Membership | `member` | No (filter) | +| Aggregate | `count`, `sum`, `min`, `max` | Yes (group-and-reduce) | ## Documentation @@ -350,7 +436,8 @@ reference. - [Constraints](docs/constraints.md) — constraint types, evaluation, and the dispatch model - [Storage Backends](docs/storage_backends.md) — Map vs ETS, options, capabilities, determinism guarantee - [Migration: Builder API → DSL](docs/migration_dsl.md) — migrate existing builder-API code to the Schema DSL -- [DSL Articles](docs/articles/01_why_datalog_on_the_beam.md) — why Datalog on the BEAM, building the DSL, rules as macros, queries, negation +- [Migration: v0.4 → v0.5](docs/migration_v0.5.md) — aggregates, callbacks, magic sets, and planner changes +- [DSL Articles](docs/articles/01_why_datalog_on_the_beam.md) — why Datalog on the BEAM, building the DSL, rules as macros, queries, negation, planning, aggregates, callbacks, magic sets - [Quickstart Tutorial](livebooks/quickstart.livemd) — interactive Livebook walkthrough - [DSL Tutorial](livebooks/ex_datalog_dsl.livemd) — interactive DSL walkthrough - [Examples](livebooks/examples.livemd) — 10 realistic use cases (RBAC, supply chain, fraud detection, and more) @@ -423,7 +510,7 @@ The following references are highly recommended for understanding both the theor | v0.3.0 | Tuple shorthand for rules (`add_rule/3`, `add_rule/4`), `Term.from/1`, `ExDatalog.Atom.from_tuple/1`, `Constraint.from_tuple/1`; renamed `Result` → `Knowledge`, `query` → `materialize` | | v0.4.0 | Schema DSL (`use ExDatalog.Schema`), `relation`, `fact`, `rule`, `query` macros, `not_` negation, constraint DSL, post-materialization queries | | v0.4.1 | DSL review fixes: correct uppercase-variable docs, clean aggregate error, query/find validation, unified `DSL.CompileError`, 792 tests | -| v0.5.0 | Magic sets / demand-driven evaluation, general predicates as BEAM callbacks | +| v0.5.0 | Aggregates (`count`/`sum`/`min`/`max`), BEAM callback predicates, query planner, magic sets (experimental), 845 tests | | v1.0.0 | Stable public API, hardened production semantics | ## License diff --git a/bench/aggregate_bench.exs b/bench/aggregate_bench.exs new file mode 100644 index 0000000..9905920 --- /dev/null +++ b/bench/aggregate_bench.exs @@ -0,0 +1,82 @@ +alias ExDatalog +alias ExDatalog.{Program, Rule, Atom, Term, Constraint} + +{n, m} = {200, 5} + +departments = for i <- 0..(m - 1), do: String.to_atom("dept_#{i}") + +base_program = + Program.new() + |> Program.add_relation("emp", [:atom, :atom]) + |> Program.add_relation("salary", [:atom, :atom, :integer]) + +base_program = + Enum.reduce(1..n, base_program, fn i, acc -> + dept = Enum.at(departments, rem(i, m)) + salary = 50_000 + i * 100 + name = String.to_atom("emp_#{i}") + + acc + |> Program.add_fact("emp", [name, dept]) + |> Program.add_fact("salary", [name, dept, salary]) + end) + +program_no_aggregates = + base_program + |> Program.add_relation("emp_salary", [:atom, :atom, :integer]) + |> Program.add_rule( + Rule.new( + Atom.new("emp_salary", [Term.var("E"), Term.var("D"), Term.var("S")]), + [{:positive, Atom.new("salary", [Term.var("E"), Term.var("D"), Term.var("S")])}] + ) + ) + +program_with_aggregates = + base_program + |> Program.add_relation("dept_count", [:atom, :integer]) + |> Program.add_relation("dept_total", [:atom, :integer]) + |> Program.add_relation("dept_min_salary", [:atom, :integer]) + |> Program.add_relation("dept_max_salary", [:atom, :integer]) + |> Program.add_rule( + Rule.new( + Atom.new("dept_count", [Term.var("D"), Term.var("N")]), + [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D")])}], + [Constraint.count(Term.var("E"), Term.var("N"))] + ) + ) + |> Program.add_rule( + Rule.new( + Atom.new("dept_total", [Term.var("D"), Term.var("T")]), + [{:positive, Atom.new("salary", [Term.var("E"), Term.var("D"), Term.var("A")])}], + [Constraint.sum(Term.var("A"), Term.var("T"))] + ) + ) + |> Program.add_rule( + Rule.new( + Atom.new("dept_min_salary", [Term.var("D"), Term.var("V")]), + [{:positive, Atom.new("salary", [Term.var("E"), Term.var("D"), Term.var("S")])}], + [Constraint.min(Term.var("S"), Term.var("V"))] + ) + ) + |> Program.add_rule( + Rule.new( + Atom.new("dept_max_salary", [Term.var("D"), Term.var("V")]), + [{:positive, Atom.new("salary", [Term.var("E"), Term.var("D"), Term.var("S")])}], + [Constraint.max(Term.var("S"), Term.var("V"))] + ) + ) + +Benchee.run( + %{ + "no_aggregates" => fn -> + {:ok, _knowledge} = ExDatalog.materialize(program_no_aggregates) + end, + "with_aggregates_count_sum_min_max" => fn -> + {:ok, _knowledge} = ExDatalog.materialize(program_with_aggregates) + end + }, + time: 10, + memory_time: 2, + formatters: [{Benchee.Formatters.Console, comparison: true}], + print: [fast_warning: false] +) diff --git a/bench/magic_sets_bench.exs b/bench/magic_sets_bench.exs new file mode 100644 index 0000000..bdf3781 --- /dev/null +++ b/bench/magic_sets_bench.exs @@ -0,0 +1,51 @@ +alias ExDatalog +alias ExDatalog.{Program, Rule, Atom, Term} + +n = 50 + +chain_facts = for i <- 0..(n - 2), do: {String.to_atom("n#{i}"), String.to_atom("n#{i + 1}")} + +program = + Program.new() + |> Program.add_relation("parent", [:atom, :atom]) + |> Program.add_relation("ancestor", [:atom, :atom]) + +program = + Enum.reduce(chain_facts, program, fn {p, c}, acc -> + Program.add_fact(acc, "parent", [p, c]) + end) + +program = + program + |> Program.add_rule( + Rule.new( + Atom.new("ancestor", [Term.var("X"), Term.var("Y")]), + [{:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}] + ) + ) + |> Program.add_rule( + Rule.new( + Atom.new("ancestor", [Term.var("X"), Term.var("Z")]), + [ + {:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}, + {:positive, Atom.new("ancestor", [Term.var("Y"), Term.var("Z")])} + ] + ) + ) + +goal = {"ancestor", [String.to_atom("n0"), :_]} + +Benchee.run( + %{ + "semi_naive_full" => fn -> + {:ok, _knowledge} = ExDatalog.materialize(program) + end, + "magic_sets_goal_driven" => fn -> + {:ok, _knowledge} = ExDatalog.materialize(program, strategy: :magic_sets, goal: goal) + end + }, + time: 10, + memory_time: 2, + formatters: [{Benchee.Formatters.Console, comparison: true}], + print: [fast_warning: false] +) diff --git a/lib/ex_datalog/callback.ex b/lib/ex_datalog/callback.ex new file mode 100644 index 0000000..6bcd5d0 --- /dev/null +++ b/lib/ex_datalog/callback.ex @@ -0,0 +1,106 @@ +defmodule ExDatalog.Callback do + @moduledoc """ + A BEAM callback predicate: an Elixir function invoked during rule evaluation. + + Callbacks let rule bodies call deterministic, side-effect-free Elixir + functions as predicates. A callback appears in a rule body as + `{:callback, %ExDatalog.Callback{}}`, alongside positive/negative atoms. + + ## Fields + + - `module` — the module exporting the function. + - `function` — the function name (atom). + - `args` — a list of `ExDatalog.Term.t()` (variables/constants) resolved + against the binding and passed positionally to the function. + - `result` — `nil` for boolean (filter) callbacks; `{:var, name}` for + value-returning callbacks that bind the function's return value. + + ## Safety contract + + A callback **must** be: + + - **Deterministic** — the same arguments always produce the same result. + - **Side-effect free** — no I/O, mutation, or messaging. + + The engine enforces only timeout and exception isolation: a callback that + exceeds `callback_timeout_ms` (default 100ms) or raises is treated as a + filtered binding (the fact is not derived). Determinism and purity are + caller contracts, not enforced by the engine. + + ## Examples + + iex> alias ExDatalog.{Callback, Term} + iex> Callback.new(String, :starts_with?, [Term.var("S"), Term.const("a")]) + %ExDatalog.Callback{module: String, function: :starts_with?, args: [{:var, "S"}, {:const, "a"}], result: nil} + + """ + + alias ExDatalog.Term + + @enforce_keys [:module, :function, :args] + defstruct [:module, :function, :args, :result] + + @type t :: %__MODULE__{ + module: module(), + function: atom(), + args: [Term.t()], + result: Term.t() | nil + } + + @doc """ + Constructs a callback predicate. + + `result` defaults to `nil` (a boolean filter callback). Pass a + `{:var, name}` term to bind the function's return value. + + ## Examples + + iex> alias ExDatalog.{Callback, Term} + iex> Callback.new(MyMod, :adult?, [Term.var("Age")]) + %ExDatalog.Callback{module: MyMod, function: :adult?, args: [{:var, "Age"}], result: nil} + + iex> alias ExDatalog.{Callback, Term} + iex> Callback.new(MyMod, :score, [Term.var("X")], Term.var("S")) + %ExDatalog.Callback{module: MyMod, function: :score, args: [{:var, "X"}], result: {:var, "S"}} + + """ + @spec new(module(), atom(), [Term.t()], Term.t() | nil) :: t() + def new(module, function, args, result \\ nil) + when is_atom(module) and is_atom(function) and is_list(args) do + %__MODULE__{module: module, function: function, args: args, result: result} + end + + @doc """ + Returns the input variable names referenced by the callback's arguments. + + These must be bound by positive body atoms before the callback runs. + + ## Examples + + iex> alias ExDatalog.{Callback, Term} + iex> cb = Callback.new(MyMod, :ok?, [Term.var("A"), Term.const(1), Term.var("B")]) + iex> ExDatalog.Callback.input_variables(cb) + ["A", "B"] + + """ + @spec input_variables(t()) :: [String.t()] + def input_variables(%__MODULE__{args: args}), do: Term.variables(args) + + @doc """ + Returns the result variable name for a value-returning callback, or `nil`. + + ## Examples + + iex> alias ExDatalog.{Callback, Term} + iex> ExDatalog.Callback.result_variable(Callback.new(M, :f, [Term.var("X")], Term.var("R"))) + "R" + + iex> alias ExDatalog.{Callback, Term} + iex> ExDatalog.Callback.result_variable(Callback.new(M, :f, [Term.var("X")])) + nil + + """ + @spec result_variable(t()) :: String.t() | nil + def result_variable(%__MODULE__{result: {:var, name}}), do: name + def result_variable(%__MODULE__{result: _}), do: nil +end diff --git a/lib/ex_datalog/capabilities.ex b/lib/ex_datalog/capabilities.ex index fe2c635..608e6de 100644 --- a/lib/ex_datalog/capabilities.ex +++ b/lib/ex_datalog/capabilities.ex @@ -18,6 +18,8 @@ defmodule ExDatalog.Capabilities do | `type_predicates` | `true` | Supports type-check predicates | | `string_predicates` | `true` | Supports string predicates | | `provenance` | `true` | Supports derivation provenance | + | `aggregate_constraints` | `true` | Supports aggregate constraints (count/sum/min/max) | + | `beam_callbacks` | `true` | Supports BEAM callback predicates | | `external_execution` | `false` | Reserved for Z3, Soufflé, etc. | ## Merging @@ -61,6 +63,8 @@ defmodule ExDatalog.Capabilities do type_predicates: boolean(), string_predicates: boolean(), provenance: boolean(), + aggregate_constraints: boolean(), + beam_callbacks: boolean(), external_execution: boolean() } @@ -72,6 +76,8 @@ defmodule ExDatalog.Capabilities do type_predicates: true, string_predicates: true, provenance: true, + aggregate_constraints: true, + beam_callbacks: true, external_execution: false @boolean_fields [ @@ -82,6 +88,8 @@ defmodule ExDatalog.Capabilities do :type_predicates, :string_predicates, :provenance, + :aggregate_constraints, + :beam_callbacks, :external_execution ] diff --git a/lib/ex_datalog/compiler.ex b/lib/ex_datalog/compiler.ex index bf4e83c..74fd0ea 100644 --- a/lib/ex_datalog/compiler.ex +++ b/lib/ex_datalog/compiler.ex @@ -200,6 +200,7 @@ defmodule ExDatalog.Compiler do Enum.map(body, fn {:positive, %ExDatalog.Atom{} = atom} -> {:positive, IR.from_atom(atom)} {:negative, %ExDatalog.Atom{} = atom} -> {:negative, IR.from_atom(atom)} + {:callback, %ExDatalog.Callback{} = cb} -> {:callback, IR.from_callback(cb)} end) ir_constraints = Enum.map(constraints, fn c -> {:constraint, IR.from_constraint(c)} end) diff --git a/lib/ex_datalog/constraint.ex b/lib/ex_datalog/constraint.ex index 43ceeee..c8533d4 100644 --- a/lib/ex_datalog/constraint.ex +++ b/lib/ex_datalog/constraint.ex @@ -119,7 +119,9 @@ defmodule ExDatalog.Constraint do @type_ops [:is_integer, :is_binary, :is_atom] @string_ops [:starts_with, :contains] @membership_ops [:member] - @all_ops @comparison_ops ++ @arithmetic_ops ++ @type_ops ++ @string_ops ++ @membership_ops + @aggregate_ops [:count, :sum, :min, :max] + @all_ops @comparison_ops ++ + @arithmetic_ops ++ @type_ops ++ @string_ops ++ @membership_ops ++ @aggregate_ops @type op :: :gt @@ -138,6 +140,10 @@ defmodule ExDatalog.Constraint do | :starts_with | :contains | :member + | :count + | :sum + | :min + | :max @type comparison :: %__MODULE__{ op: :gt | :lt | :gte | :lte | :eq | :neq, @@ -174,7 +180,20 @@ defmodule ExDatalog.Constraint do result: nil } - @type t :: comparison() | arithmetic() | type_predicate() | string_predicate() | membership() + @type aggregate :: %__MODULE__{ + op: :count | :sum | :min | :max, + left: Term.t(), + right: nil, + result: {:var, String.t()} + } + + @type t :: + comparison() + | arithmetic() + | type_predicate() + | string_predicate() + | membership() + | aggregate() defstruct [:op, :left, :right, :result] @@ -402,6 +421,63 @@ defmodule ExDatalog.Constraint do @spec member(Term.t(), Term.t()) :: t() def member(left, right), do: filter(:member, left, right) + # --- Aggregate constructors --- + + @doc """ + Constructs a `count` aggregate: `result = count of input within each group`. + + Aggregates group the rule's surviving bindings by the head variables other + than `result`, then reduce the `input` variable's values within each group. + Aggregates are integer-only and must appear in a stratum strictly above the + relations that bind their inputs. + + ## Examples + + iex> ExDatalog.Constraint.count({:var, "Emp"}, {:var, "N"}) + %ExDatalog.Constraint{op: :count, left: {:var, "Emp"}, right: nil, result: {:var, "N"}} + + """ + @spec count(Term.t(), Term.t()) :: t() + def count(input, result), do: aggregate(:count, input, result) + + @doc """ + Constructs a `sum` aggregate: `result = integer sum of input within each group`. + + Inputs must be integers; non-integer inputs filter the group's contribution. + + ## Examples + + iex> ExDatalog.Constraint.sum({:var, "Amount"}, {:var, "Total"}) + %ExDatalog.Constraint{op: :sum, left: {:var, "Amount"}, right: nil, result: {:var, "Total"}} + + """ + @spec sum(Term.t(), Term.t()) :: t() + def sum(input, result), do: aggregate(:sum, input, result) + + @doc """ + Constructs a `min` aggregate: `result = minimum of input within each group`. + + ## Examples + + iex> ExDatalog.Constraint.min({:var, "Score"}, {:var, "Lowest"}) + %ExDatalog.Constraint{op: :min, left: {:var, "Score"}, right: nil, result: {:var, "Lowest"}} + + """ + @spec min(Term.t(), Term.t()) :: t() + def min(input, result), do: aggregate(:min, input, result) + + @doc """ + Constructs a `max` aggregate: `result = maximum of input within each group`. + + ## Examples + + iex> ExDatalog.Constraint.max({:var, "Score"}, {:var, "Highest"}) + %ExDatalog.Constraint{op: :max, left: {:var, "Score"}, right: nil, result: {:var, "Highest"}} + + """ + @spec max(Term.t(), Term.t()) :: t() + def max(input, result), do: aggregate(:max, input, result) + # --- Introspection --- @doc """ @@ -479,6 +555,49 @@ defmodule ExDatalog.Constraint do @spec membership?(t()) :: boolean() def membership?(%__MODULE__{op: op}), do: op in @membership_ops + @doc """ + Returns `true` if the constraint is an aggregate (`count`, `sum`, `min`, `max`). + + Aggregates bind a `result` variable by grouping and reducing, and are + evaluated by a dedicated engine path rather than the per-binding constraint + pipeline. + + ## Examples + + iex> ExDatalog.Constraint.aggregate?(ExDatalog.Constraint.count({:var, "X"}, {:var, "N"})) + true + + iex> ExDatalog.Constraint.aggregate?(ExDatalog.Constraint.gt({:var, "X"}, {:const, 0})) + false + + """ + @spec aggregate?(t()) :: boolean() + def aggregate?(%__MODULE__{op: op}), do: op in @aggregate_ops + + @doc false + @spec comparison_op?(atom()) :: boolean() + def comparison_op?(op), do: op in @comparison_ops + + @doc false + @spec arithmetic_op?(atom()) :: boolean() + def arithmetic_op?(op), do: op in @arithmetic_ops + + @doc false + @spec type_op?(atom()) :: boolean() + def type_op?(op), do: op in @type_ops + + @doc false + @spec string_op?(atom()) :: boolean() + def string_op?(op), do: op in @string_ops + + @doc false + @spec membership_op?(atom()) :: boolean() + def membership_op?(op), do: op in @membership_ops + + @doc false + @spec aggregate_op?(atom()) :: boolean() + def aggregate_op?(op), do: op in @aggregate_ops + @doc """ Returns `true` if the constraint is structurally valid. @@ -551,16 +670,20 @@ defmodule ExDatalog.Constraint do %__MODULE__{op: op, left: left, right: right, result: nil} end + defp aggregate(op, input, result) do + %__MODULE__{op: op, left: input, right: nil, result: result} + end + defp valid_result?(op, nil) when op in @comparison_ops or op in @type_ops or op in @string_ops or op in @membership_ops, do: true - defp valid_result?(op, {:var, name}) when op in @arithmetic_ops, + defp valid_result?(op, {:var, name}) when op in @arithmetic_ops or op in @aggregate_ops, do: is_binary(name) and byte_size(name) > 0 defp valid_result?(_, _), do: false - defp valid_right?(op, nil) when op in @type_ops, do: true + defp valid_right?(op, nil) when op in @type_ops or op in @aggregate_ops, do: true defp valid_right?(:member, {:const, value}) when is_list(value), do: true @@ -644,6 +767,10 @@ defmodule ExDatalog.Constraint do filter(op, Term.from(left), Term.from(right)) end + def from_tuple({op, input, result}) when op in @aggregate_ops do + aggregate(op, Term.from(input), Term.from(result)) + end + def from_tuple({:member, left, right}) do member(Term.from(left), Term.from(right)) end @@ -722,4 +849,5 @@ defmodule ExDatalog.Constraint do defp constraint_module(op) when op in @type_ops, do: ExDatalog.Constraints.Type defp constraint_module(op) when op in @string_ops, do: ExDatalog.Constraints.StringPredicate defp constraint_module(op) when op in @membership_ops, do: ExDatalog.Constraints.Membership + defp constraint_module(op) when op in @aggregate_ops, do: ExDatalog.Constraints.Aggregate end diff --git a/lib/ex_datalog/constraint/context.ex b/lib/ex_datalog/constraint/context.ex index c1ee532..ca2c8ab 100644 --- a/lib/ex_datalog/constraint/context.ex +++ b/lib/ex_datalog/constraint/context.ex @@ -21,11 +21,13 @@ defmodule ExDatalog.Constraint.Context do @type t :: %__MODULE__{ capabilities: Capabilities.t(), - provenance: boolean() + provenance: boolean(), + opts: keyword() } defstruct capabilities: %Capabilities{}, - provenance: false + provenance: false, + opts: [] @doc """ Creates a new context with default capabilities. diff --git a/lib/ex_datalog/constraints/aggregate.ex b/lib/ex_datalog/constraints/aggregate.ex new file mode 100644 index 0000000..98acdd6 --- /dev/null +++ b/lib/ex_datalog/constraints/aggregate.ex @@ -0,0 +1,62 @@ +defmodule ExDatalog.Constraints.Aggregate do + @moduledoc """ + Aggregate constraint evaluation: `count`, `sum`, `min`, `max`. + + Aggregates are **not** evaluated per binding. Unlike comparison or arithmetic + constraints, an aggregate must see the full set of surviving bindings for a + rule, group them, and reduce each group to a single value. That grouping is + performed by `ExDatalog.Engine.Evaluator` via `group_and_reduce/5`; the + `evaluate/3` callback exists only to satisfy the `ExDatalog.Constraint` + behaviour and raises if ever invoked through the per-binding pipeline. + + Aggregates are integer-only. `sum` requires integer inputs (validated at + build time and guarded at runtime). `count` returns the group size. `min`/`max` + return the smallest/largest input value in the group. + """ + + @behaviour ExDatalog.Constraint + + @dialyzer {:nowarn_function, evaluate: 3} + + @impl ExDatalog.Constraint + @doc """ + Not supported per-binding. Aggregates are evaluated by the engine's + group-and-reduce path. Always raises. + """ + def evaluate(_constraint, _binding, _context) do + raise "aggregate constraints are not evaluated per-binding; " <> + "use ExDatalog.Engine.Evaluator group-and-reduce path" + end + + @doc """ + Groups bindings by `group_vars` and reduces each group's `input_var` values + with the aggregate `op`, binding the reduced value to `result_var`. + + Returns one extended binding per non-empty group. Empty groups never occur: + a group exists only because at least one binding produced its key, so the + reducers (`Enum.min/1`, `Enum.max/1`) are never called on an empty list. + + ## Examples + + iex> bindings = [%{"D" => :eng, "E" => :a}, %{"D" => :eng, "E" => :b}, %{"D" => :ops, "E" => :c}] + iex> ExDatalog.Constraints.Aggregate.group_and_reduce(bindings, ["D"], :count, "E", "N") + ...> |> Enum.map(fn b -> {b["D"], b["N"]} end) + ...> |> Enum.sort() + [{:eng, 2}, {:ops, 1}] + + """ + @spec group_and_reduce([map()], [String.t()], atom(), String.t(), String.t()) :: [map()] + def group_and_reduce(bindings, group_vars, op, input_var, result_var) do + bindings + |> Enum.group_by(fn binding -> Map.take(binding, group_vars) end) + |> Enum.map(fn {_key, group} -> + values = Enum.map(group, fn b -> Map.fetch!(b, input_var) end) + Map.put(hd(group), result_var, compute(op, values)) + end) + end + + defp compute(:count, values), do: length(values) + defp compute(:sum, values), do: Enum.sum(values) + defp compute(:min, values), do: Enum.min(values) + defp compute(:max, values), do: Enum.max(values) +end diff --git a/lib/ex_datalog/constraints/beam_callback.ex b/lib/ex_datalog/constraints/beam_callback.ex new file mode 100644 index 0000000..55b57db --- /dev/null +++ b/lib/ex_datalog/constraints/beam_callback.ex @@ -0,0 +1,103 @@ +defmodule ExDatalog.Constraints.BeamCallback do + @moduledoc """ + Evaluation of BEAM callback predicates. + + A callback applies an Elixir function to argument values resolved (by variable + name) from the current binding. The function must be deterministic and + side-effect free — these are caller contracts, not enforced. + + The engine enforces only: + + - **Timeout** — the call runs in a `Task` with a configurable timeout + (`:callback_timeout_ms`, default 100ms). A timeout filters the binding. + - **Exception isolation** — a raised exception filters the binding. + + Boolean callbacks (`result: nil`) act as filters: `true` keeps the binding, + `false`/timeout/exception drops it. Value-returning callbacks + (`result: {:var, name}`) bind the return value to `name`. + """ + + alias ExDatalog.IR + + @default_timeout_ms 100 + + @doc """ + Applies a callback against a binding. + + Returns `{:ok, binding}` (boolean true, or value bound) or `:filter` + (boolean false, unbound argument, timeout, or exception). + """ + @spec apply_callback(IR.Callback.t(), map(), keyword()) :: {:ok, map()} | :filter + def apply_callback( + %IR.Callback{module: m, function: f, args: arg_terms, result: result}, + binding, + opts + ) do + case resolve_args(arg_terms, binding) do + {:ok, args} -> + timeout = Keyword.get(opts, :callback_timeout_ms, @default_timeout_ms) + + case safe_apply(m, f, args, timeout) do + {:ok, true} when result == nil -> {:ok, binding} + {:ok, false} when result == nil -> :filter + {:ok, value} when result != nil -> {:ok, bind_result(binding, result, value)} + {:ok, _other} -> :filter + {:error, _} -> :filter + end + + :unbound -> + :filter + end + end + + defp resolve_args(arg_terms, binding) do + Enum.reduce_while(arg_terms, {:ok, []}, fn term, {:ok, acc} -> + case IR.resolve_operand(term, binding) do + {:ok, value} -> {:cont, {:ok, [value | acc]}} + :unbound -> {:halt, :unbound} + end + end) + |> case do + {:ok, reversed} -> {:ok, Enum.reverse(reversed)} + :unbound -> :unbound + end + end + + defp bind_result(binding, {:var, name}, value), do: Map.put(binding, name, value) + + # Run the callback in an unlinked, monitored process so that a raise or + # exit inside the callback does not propagate to (or kill) the evaluator. + # A crash or timeout is reported as `{:error, _}` and filters the binding. + defp safe_apply(module, function, args, timeout_ms) do + parent = self() + ref = make_ref() + + {pid, monitor_ref} = + spawn_monitor(fn -> + result = + try do + {:ok, apply(module, function, args)} + rescue + e -> {:error, e} + catch + kind, reason -> {:error, {kind, reason}} + end + + send(parent, {ref, result}) + end) + + receive do + {^ref, result} -> + Process.demonitor(monitor_ref, [:flush]) + result + + {:DOWN, ^monitor_ref, :process, ^pid, reason} -> + {:error, reason} + after + timeout_ms -> + Process.exit(pid, :kill) + Process.demonitor(monitor_ref, [:flush]) + {:error, :timeout} + end + end +end diff --git a/lib/ex_datalog/engine/evaluator.ex b/lib/ex_datalog/engine/evaluator.ex index 5a9d41f..5ca64c2 100644 --- a/lib/ex_datalog/engine/evaluator.ex +++ b/lib/ex_datalog/engine/evaluator.ex @@ -37,6 +37,7 @@ defmodule ExDatalog.Engine.Evaluator do ordering guarantee. """ + alias ExDatalog.Constraints.{Aggregate, BeamCallback} alias ExDatalog.Engine.{Binding, ConstraintEval, Join} alias ExDatalog.IR @@ -77,12 +78,24 @@ defmodule ExDatalog.Engine.Evaluator do ) :: [tuple()] def eval_rule_iteration(rule, full, delta, old, ctx \\ %ExDatalog.Constraint.Context{}) do - positive_body = positive_atoms(rule) - k = length(positive_body) - head_relation = rule.head.relation existing = Map.get(full, head_relation, MapSet.new()) + if aggregate_rule?(rule) do + rule + |> eval_aggregate_rule(full, ctx) + |> MapSet.new() + |> MapSet.difference(existing) + |> MapSet.to_list() + else + eval_normal_rule(rule, full, delta, old, ctx, existing) + end + end + + defp eval_normal_rule(rule, full, delta, old, ctx, existing) do + positive_body = positive_atoms(rule) + k = length(positive_body) + if k == 0 do derived = [%{}] @@ -99,6 +112,56 @@ defmodule ExDatalog.Engine.Evaluator do end end + defp aggregate_rule?(%IR.Rule{body: body}) do + Enum.any?(body, fn + {:constraint, %ExDatalog.IR.Constraint{op: op}} -> op in [:count, :sum, :min, :max] + _ -> false + end) + end + + # Aggregate rules are stratified above their source relations, so all source + # facts are final in `full`. We join the positive body against `full`, apply + # non-aggregate filters and negation, group by the head variables other than + # the aggregate result, reduce each group, then project. + defp eval_aggregate_rule(%IR.Rule{} = rule, full, ctx) do + agg_constraint = find_aggregate(rule.body) + + joined = join_positive_body([%{}], rule.body, full) + filtered = apply_constraints(rule.body, joined, ctx) + with_callbacks = apply_callbacks(rule.body, filtered, ctx) + bindings = apply_negation(rule.body, with_callbacks, full) + + %ExDatalog.IR.Constraint{op: op, left: {:var, input_var}, result: {:var, result_var}} = + agg_constraint + + group_vars = aggregate_group_vars(rule.head, result_var) + + bindings + |> Aggregate.group_and_reduce(group_vars, op, input_var, result_var) + |> Enum.map(&Join.project(rule.head, &1)) + end + + defp find_aggregate(body) do + Enum.find_value(body, fn + {:constraint, %ExDatalog.IR.Constraint{op: op} = c} when op in [:count, :sum, :min, :max] -> + c + + _ -> + nil + end) + end + + # Group by the head variables other than the aggregate result variable. + defp aggregate_group_vars(%IR.Atom{terms: terms}, result_var) do + terms + |> Enum.flat_map(fn + {:var, name} -> [name] + _ -> [] + end) + |> Enum.reject(fn name -> name == result_var end) + |> Enum.uniq() + end + @doc """ Checks whether a binding satisfies a negative body atom. @@ -137,6 +200,7 @@ defmodule ExDatalog.Engine.Evaluator do defp finish_bindings(bindings, rule, full, ctx) do bindings = apply_constraints(rule.body, bindings, ctx) + bindings = apply_callbacks(rule.body, bindings, ctx) bindings = apply_negation(rule.body, bindings, full) case bindings do @@ -145,6 +209,34 @@ defmodule ExDatalog.Engine.Evaluator do end end + defp apply_callbacks(body, bindings, ctx) do + callbacks = for {:callback, cb} <- body, do: cb + + case callbacks do + [] -> + bindings + + _ -> + opts = callback_opts(ctx) + + Enum.flat_map(bindings, fn binding -> + apply_callback_chain(callbacks, binding, opts) + end) + end + end + + defp apply_callback_chain(callbacks, binding, opts) do + Enum.reduce_while(callbacks, [binding], fn cb, [b] -> + step_callback(BeamCallback.apply_callback(cb, b, opts)) + end) + end + + defp step_callback({:ok, new_b}), do: {:cont, [new_b]} + defp step_callback(:filter), do: {:halt, []} + + defp callback_opts(%ExDatalog.Constraint.Context{opts: opts}) when is_list(opts), do: opts + defp callback_opts(_), do: [] + defp eval_variant(rule, positive_body, full, delta, old, delta_pos, ctx) do bindings = positive_body @@ -181,7 +273,13 @@ defmodule ExDatalog.Engine.Evaluator do end defp apply_constraints(body, bindings, ctx) do - constraints = for {:constraint, c} <- body, do: c + # Aggregate constraints are NOT evaluated per binding; they are handled by + # the group-and-reduce aggregate path. Exclude them here so the normal + # per-binding pipeline never feeds them to ConstraintEval. + constraints = + for {:constraint, %ExDatalog.IR.Constraint{op: op} = c} <- body, + op not in [:count, :sum, :min, :max], + do: c Enum.flat_map(bindings, fn b -> case ConstraintEval.apply(constraints, b, ctx) do diff --git a/lib/ex_datalog/engine/naive.ex b/lib/ex_datalog/engine/naive.ex index 117c8a6..eb8b5b6 100644 --- a/lib/ex_datalog/engine/naive.ex +++ b/lib/ex_datalog/engine/naive.ex @@ -87,6 +87,36 @@ defmodule ExDatalog.Engine.Naive do """ @spec evaluate(IR.t(), keyword()) :: {:ok, Knowledge.t()} | {:error, term()} def evaluate(%IR{} = ir, opts \\ []) do + case Keyword.get(opts, :strategy, :semi_naive) do + :semi_naive -> + evaluate_semi_naive(ir, opts) + + :magic_sets -> + evaluate_magic_sets(ir, opts) + + other -> + {:error, "unknown evaluation strategy: #{inspect(other)}"} + end + end + + defp evaluate_magic_sets(%IR{} = ir, opts) do + case Keyword.get(opts, :goal, nil) do + nil -> + # No goal to drive demand; fall back to full evaluation. + evaluate_semi_naive(ir, opts) + + goal -> + case ExDatalog.MagicSets.transform(ir, goal) do + {:ok, transformed_ir} -> + evaluate_semi_naive(transformed_ir, Keyword.delete(opts, :strategy)) + + {:fallback, _reason} -> + evaluate_semi_naive(ir, opts) + end + end + end + + defp evaluate_semi_naive(%IR{} = ir, opts) do ExDatalog.Telemetry.emit_start(ir) start_time = System.monotonic_time(:microsecond) stratum_count = length(ir.strata) @@ -136,7 +166,8 @@ defmodule ExDatalog.Engine.Naive do state0 = storage_mod.init(schemas, storage_opts) constraint_ctx = %ExDatalog.Constraint.Context{ - capabilities: storage_mod.capabilities(state0) + capabilities: storage_mod.capabilities(state0), + opts: opts } try do @@ -284,16 +315,48 @@ defmodule ExDatalog.Engine.Naive do end) end) - case unstratifiable do - [] -> - :ok + unstratifiable_aggregates = + Enum.filter(ir.rules, fn rule -> + has_aggregate?(rule) and + Enum.any?(positive_literals(rule), fn {:positive, %IR.Atom{relation: rel}} -> + rel != rule.head.relation and Map.get(relation_strata, rel, 0) >= rule.stratum + end) + end) + + cond do + unstratifiable != [] -> + {:error, + "unstratifiable negation detected: #{format_unstratifiable_details(unstratifiable)}"} - rules -> - details = format_unstratifiable_details(rules) - {:error, "unstratifiable negation detected: #{details}"} + unstratifiable_aggregates != [] -> + {:error, + "unstratifiable aggregate detected: #{format_unstratifiable_aggregate_details(unstratifiable_aggregates)}"} + + true -> + :ok end end + defp positive_literals(%IR.Rule{body: body}) do + Enum.filter(body, fn + {:positive, _} -> true + _ -> false + end) + end + + defp has_aggregate?(%IR.Rule{body: body}) do + Enum.any?(body, fn + {:constraint, %IR.Constraint{op: op}} -> op in [:count, :sum, :min, :max] + _ -> false + end) + end + + defp format_unstratifiable_aggregate_details(rules) do + Enum.map_join(rules, "; ", fn r -> + "rule #{r.id} (head: #{r.head.relation}) aggregates over a relation in the same or higher stratum" + end) + end + defp negative_literals(%IR.Rule{body: body}) do Enum.filter(body, fn {:negative, _} -> true @@ -551,6 +614,7 @@ defmodule ExDatalog.Engine.Naive do {:positive, %IR.Atom{relation: r}} -> [r] {:negative, %IR.Atom{relation: r}} -> [r] {:constraint, _} -> [] + {:callback, _} -> [] end) |> Enum.uniq() end diff --git a/lib/ex_datalog/ir.ex b/lib/ex_datalog/ir.ex index 8ffb1e3..5cad35b 100644 --- a/lib/ex_datalog/ir.ex +++ b/lib/ex_datalog/ir.ex @@ -49,10 +49,18 @@ defmodule ExDatalog.IR do result: ir_term() | nil } + @type ir_callback :: %ExDatalog.IR.Callback{ + module: module(), + function: atom(), + args: [ir_term()], + result: ir_term() | nil + } + @type ir_literal :: {:positive, ExDatalog.IR.Atom.t()} | {:negative, ExDatalog.IR.Atom.t()} | {:constraint, ir_constraint()} + | {:callback, ir_callback()} defmodule Relation do @moduledoc """ @@ -139,6 +147,31 @@ defmodule ExDatalog.IR do end end + defmodule Callback do + @moduledoc """ + An IR callback predicate: an Elixir function invoked during evaluation. + + `args` are IR terms resolved by variable name against the binding at + evaluation time. `result` is `nil` for boolean (filter) callbacks, or + `{:var, name}` for value-returning callbacks that bind a result variable. + """ + + @enforce_keys [:module, :function, :args] + defstruct [:module, :function, :args, :result] + + @type t :: %__MODULE__{ + module: module(), + function: atom(), + args: [ExDatalog.IR.ir_term()], + result: ExDatalog.IR.ir_term() | nil + } + + @spec serialize(t()) :: map() + def serialize(%__MODULE__{module: m, function: f, args: args, result: result}) do + %{module: m, function: f, args: args, result: result} + end + end + defmodule Rule do @moduledoc """ An IR rule: a head atom, a body of literals, an assigned stratum, and @@ -177,6 +210,9 @@ defmodule ExDatalog.IR do {:constraint, c} -> %{kind: :constraint, constraint: Constraint.serialize(c)} + + {:callback, cb} -> + %{kind: :callback, callback: Callback.serialize(cb)} end), stratum: stratum, metadata: metadata @@ -302,6 +338,19 @@ defmodule ExDatalog.IR do %Atom{relation: relation, terms: Enum.map(terms, &from_term/1)} end + @doc """ + Converts an AST callback to an IR callback. + """ + @spec from_callback(ExDatalog.Callback.t()) :: Callback.t() + def from_callback(%ExDatalog.Callback{module: m, function: f, args: args, result: result}) do + %Callback{ + module: m, + function: f, + args: Enum.map(args, &from_term/1), + result: maybe_from_term(result) + } + end + @doc """ Resolves an IR term against a binding environment. diff --git a/lib/ex_datalog/magic_sets.ex b/lib/ex_datalog/magic_sets.ex new file mode 100644 index 0000000..bfd9e6e --- /dev/null +++ b/lib/ex_datalog/magic_sets.ex @@ -0,0 +1,187 @@ +defmodule ExDatalog.MagicSets do + @moduledoc """ + Magic-sets program transformation for demand-driven (goal-directed) evaluation. + + Magic sets rewrites a program so that bottom-up semi-naive evaluation computes + only the facts relevant to a query goal, instead of the full least fixpoint. + It is a **program transformation**: the engine is unchanged. Given a goal + `{relation, pattern}`, the transformation: + + 1. computes the goal's *adornment* (which argument positions are bound), + 2. generates `magic__` predicates capturing demand, + 3. rewrites recursive rules to consume the magic predicates, + 4. seeds the magic predicate with the bound goal constants. + + The transformed IR is then evaluated by the existing semi-naive engine. + + ## Scope (v0.5.0, experimental) + + - Positive recursive programs only. + - A single goal. + - Ground (constant) bound positions. + + Programs outside this scope fall back to full semi-naive evaluation + (`{:fallback, reason}`), never producing incorrect results. + """ + + alias ExDatalog.IR + + @doc """ + Transforms an IR program for goal-directed evaluation. + + Returns `{:ok, transformed_ir}` when the magic-sets transformation applies, + or `{:fallback, reason}` when the program is outside the supported scope (the + caller should evaluate the original IR with semi-naive instead). + """ + @spec transform(IR.t(), {String.t(), [term()]}) :: {:ok, IR.t()} | {:fallback, term()} + def transform(%IR{} = ir, {goal_relation, goal_pattern}) do + cond do + not supported_program?(ir) -> + {:fallback, :unsupported_program} + + not has_bound_position?(goal_pattern) -> + {:fallback, :no_bound_positions} + + true -> + do_transform(ir, goal_relation, goal_pattern) + end + end + + # --- Scope checks --- + + defp supported_program?(%IR{rules: rules}) do + Enum.all?(rules, fn rule -> + not has_negation?(rule) and not has_aggregate?(rule) + end) + end + + defp has_negation?(%IR.Rule{body: body}) do + Enum.any?(body, &match?({:negative, _}, &1)) + end + + defp has_aggregate?(%IR.Rule{body: body}) do + Enum.any?(body, fn + {:constraint, %IR.Constraint{op: op}} -> op in [:count, :sum, :min, :max] + _ -> false + end) + end + + defp has_bound_position?(pattern) do + Enum.any?(pattern, fn p -> p != :_ end) + end + + # --- Transformation --- + + defp do_transform(%IR{} = ir, goal_relation, goal_pattern) do + adornment = adornment(goal_pattern) + magic_rel = magic_relation_name(goal_relation, adornment) + bound_positions = bound_positions(goal_pattern) + + magic_relation = %IR.Relation{ + name: magic_rel, + arity: bound_count(goal_pattern), + types: bound_types(ir, goal_relation, goal_pattern) + } + + seed_fact = seed_fact(magic_rel, goal_relation, goal_pattern) + + rewritten_rules = + ir.rules + |> Enum.map(fn rule -> rewrite_rule(rule, goal_relation, magic_rel, bound_positions) end) + + new_relations = [magic_relation | ir.relations] + new_facts = if seed_fact, do: [seed_fact | ir.facts], else: ir.facts + + # Recompute strata: the magic relation joins the goal relation's stratum. + new_strata = inject_magic_into_strata(ir.strata, goal_relation, magic_rel) + + {:ok, + %IR{ + ir + | relations: new_relations, + facts: new_facts, + rules: rewritten_rules, + strata: new_strata + }} + end + + defp adornment(pattern) do + Enum.map_join(pattern, "", fn + :_ -> "f" + _ -> "b" + end) + end + + defp magic_relation_name(relation, adornment), do: "magic_#{relation}_#{adornment}" + + defp bound_count(pattern), do: Enum.count(pattern, fn p -> p != :_ end) + + defp bound_positions(pattern) do + pattern + |> Enum.with_index() + |> Enum.filter(fn {p, _i} -> p != :_ end) + |> Enum.map(fn {_p, i} -> i end) + end + + defp bound_types(ir, goal_relation, pattern) do + case Enum.find(ir.relations, fn r -> r.name == goal_relation end) do + %IR.Relation{types: types} -> + types + |> Enum.zip(pattern) + |> Enum.filter(fn {_t, p} -> p != :_ end) + |> Enum.map(fn {t, _p} -> t end) + + nil -> + List.duplicate(:any, bound_count(pattern)) + end + end + + defp seed_fact(magic_rel, _goal_relation, pattern) do + bound_values = + pattern + |> Enum.filter(fn p -> p != :_ end) + |> Enum.map(&to_ir_value/1) + + case bound_values do + [] -> nil + values -> %IR.Fact{relation: magic_rel, values: values} + end + end + + defp to_ir_value(v) when is_integer(v), do: {:int, v} + defp to_ir_value(v) when is_binary(v), do: {:str, v} + defp to_ir_value(v) when is_atom(v), do: {:atom, v} + + # Rewrite a rule whose head is the goal relation: prepend the magic predicate + # binding the bound head positions, so derivation is demand-restricted. The + # magic atom projects exactly the goal's bound positions of the head, so its + # arity matches the magic relation. + defp rewrite_rule( + %IR.Rule{head: %IR.Atom{relation: rel} = head} = rule, + goal_relation, + magic_rel, + bound_positions + ) + when rel == goal_relation do + magic_terms = bound_head_terms(head, bound_positions) + magic_atom = %IR.Atom{relation: magic_rel, terms: magic_terms} + %IR.Rule{rule | body: [{:positive, magic_atom} | rule.body]} + end + + defp rewrite_rule(rule, _goal_relation, _magic_rel, _bound_positions), do: rule + + defp bound_head_terms(%IR.Atom{terms: terms}, bound_positions) do + bound_positions + |> Enum.map(fn pos -> Enum.at(terms, pos) end) + end + + defp inject_magic_into_strata(strata, goal_relation, magic_rel) do + Enum.map(strata, fn %IR.Stratum{relations: rels} = stratum -> + if goal_relation in rels do + %IR.Stratum{stratum | relations: [magic_rel | rels]} + else + stratum + end + end) + end +end diff --git a/lib/ex_datalog/planner.ex b/lib/ex_datalog/planner.ex new file mode 100644 index 0000000..23d3e58 --- /dev/null +++ b/lib/ex_datalog/planner.ex @@ -0,0 +1,210 @@ +defmodule ExDatalog.Planner do + @moduledoc """ + Query/evaluation planner for ExDatalog. + + The planner sits between the compiled `ExDatalog.IR` and the evaluation + engine. It produces an `ExDatalog.Planner.Plan` describing the chosen + `strategy`, the planned strata, the joins (one per positive body atom), and + the predicates (constraints, aggregates, callbacks). + + The planner is intentionally thin in v0.5.0: it wraps the existing IR strata + and classifies body elements. It is the seam through which the `:magic_sets` + strategy is selected and, in future releases, where join ordering and + cost-based optimization will live. + + ## Strategy selection + + `plan/2` accepts `:strategy` (`:semi_naive` default, or `:magic_sets`). When + `:magic_sets` is requested, a `:goal` option (`{relation, pattern}`) should be + supplied; otherwise the planner records the strategy but the engine falls back + to semi-naive. + + ## Telemetry + + `plan/2` emits `[:ex_datalog, :planner, :start | :stop | :exception]` once per + call (never per rule). + """ + + alias ExDatalog.Constraint + alias ExDatalog.IR + alias ExDatalog.Planner.{Join, Plan, Predicate, Stratum} + + @doc """ + Builds an execution `Plan` from a compiled IR program. + + ## Options + + - `:strategy` — `:semi_naive` (default) or `:magic_sets` + - `:goal` — `{relation, pattern}` used when `strategy: :magic_sets` + + Returns `{:ok, %Plan{}}`. + + ## Examples + + iex> alias ExDatalog.{Program, Rule, Atom, Term, Compiler, Planner} + iex> {:ok, ir} = + ...> Program.new() + ...> |> Program.add_relation("edge", [:atom, :atom]) + ...> |> Program.add_relation("path", [:atom, :atom]) + ...> |> Program.add_rule( + ...> Rule.new( + ...> Atom.new("path", [Term.var("X"), Term.var("Y")]), + ...> [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] + ...> ) + ...> ) + ...> |> Compiler.compile() + iex> {:ok, plan} = Planner.plan(ir) + iex> plan.strategy + :semi_naive + iex> length(plan.joins) + 1 + + """ + @spec plan(IR.t(), keyword()) :: {:ok, Plan.t()} + def plan(%IR{} = ir, opts \\ []) do + metadata = %{relation_count: length(ir.relations), rule_count: length(ir.rules)} + + :telemetry.execute( + [:ex_datalog, :planner, :start], + %{system_time: System.system_time()}, + metadata + ) + + start = System.monotonic_time() + + try do + strategy = Keyword.get(opts, :strategy, :semi_naive) + goal = Keyword.get(opts, :goal, nil) + + plan = %Plan{ + strategy: strategy, + strata: build_strata(ir), + joins: build_joins(ir), + predicates: build_predicates(ir), + metadata: %{goal: goal} + } + + duration = System.monotonic_time() - start + + :telemetry.execute( + [:ex_datalog, :planner, :stop], + %{duration: duration}, + Map.put(metadata, :strategy, strategy) + ) + + {:ok, plan} + rescue + e -> + :telemetry.execute( + [:ex_datalog, :planner, :exception], + %{duration: System.monotonic_time() - start}, + Map.merge(metadata, %{kind: :error, reason: e}) + ) + + reraise e, __STACKTRACE__ + end + end + + @doc """ + Returns a human-readable description of the plan for a program. + + Accepts the same options as `plan/2`. Validates and compiles the program + first; returns an error string if compilation fails. + + ## Examples + + iex> alias ExDatalog.{Program, Rule, Atom, Term, Planner} + iex> program = + ...> Program.new() + ...> |> Program.add_relation("edge", [:atom, :atom]) + ...> |> Program.add_relation("path", [:atom, :atom]) + ...> |> Program.add_rule( + ...> Rule.new( + ...> Atom.new("path", [Term.var("X"), Term.var("Y")]), + ...> [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] + ...> ) + ...> ) + iex> Planner.explain_plan(program) =~ "Strategy: semi_naive" + true + + """ + @spec explain_plan(ExDatalog.Program.t()) :: String.t() + def explain_plan(program), do: explain_plan(program, []) + + @spec explain_plan(ExDatalog.Program.t(), keyword()) :: String.t() + def explain_plan(program, opts) do + case ExDatalog.compile(program) do + {:ok, ir} -> + {:ok, plan} = plan(ir, opts) + format_plan(plan) + + {:error, errors} -> + "Cannot plan: compilation failed with #{length(errors)} error(s)" + end + end + + # --- Plan construction --- + + defp build_strata(%IR{strata: strata, rules: rules}) do + Enum.map(strata, fn %IR.Stratum{index: idx, relations: rels} -> + stratum_rules = Enum.filter(rules, fn r -> r.stratum == idx end) + %Stratum{index: idx, rules: stratum_rules, relations: rels} + end) + end + + defp build_joins(%IR{rules: rules}) do + Enum.flat_map(rules, fn rule -> + rule.body + |> Enum.filter(&match?({:positive, _}, &1)) + |> Enum.with_index() + |> Enum.map(fn {{:positive, %IR.Atom{relation: rel}}, position} -> + %Join{relation: rel, position: position, delta_position: position} + end) + end) + end + + defp build_predicates(%IR{rules: rules}) do + rules + |> Enum.flat_map(fn rule -> + for {:constraint, c} <- rule.body, do: classify_constraint(c) + end) + end + + defp classify_constraint(%IR.Constraint{op: op} = c) do + %Predicate{kind: constraint_kind(op), op: op, metadata: %{result: c.result}} + end + + defp constraint_kind(op) do + cond do + Constraint.comparison_op?(op) -> :comparison + Constraint.arithmetic_op?(op) -> :arithmetic + Constraint.type_op?(op) -> :type + Constraint.string_op?(op) -> :string + Constraint.membership_op?(op) -> :membership + Constraint.aggregate_op?(op) -> :aggregate + true -> :comparison + end + end + + # --- Formatting --- + + defp format_plan(%Plan{} = plan) do + header = "Strategy: #{plan.strategy}" + + strata_lines = + Enum.map(plan.strata, fn s -> + " Stratum #{s.index}: #{length(s.rules)} rule(s), relations: #{Enum.join(s.relations, ", ")}" + end) + + join_line = "Joins: #{length(plan.joins)}" + + pred_line = + "Predicates: #{length(plan.predicates)}" <> + case plan.predicates do + [] -> "" + preds -> " (#{Enum.map_join(preds, ", ", & &1.op)})" + end + + Enum.join([header | strata_lines] ++ [join_line, pred_line], "\n") + end +end diff --git a/lib/ex_datalog/planner/join.ex b/lib/ex_datalog/planner/join.ex new file mode 100644 index 0000000..435135c --- /dev/null +++ b/lib/ex_datalog/planner/join.ex @@ -0,0 +1,22 @@ +defmodule ExDatalog.Planner.Join do + @moduledoc """ + A planned join: one positive body atom position within a rule. + + `position` is the 0-based index of the atom within the rule's positive body. + `delta_position` indicates the semi-naive delta slot when applicable, or + `nil`. `strategy` records how the join is executed; the default engine uses + `:nested_loop`. + """ + + @enforce_keys [:relation, :position] + defstruct [:relation, :position, delta_position: nil, strategy: :nested_loop] + + @type strategy :: :nested_loop | :indexed + + @type t :: %__MODULE__{ + relation: String.t(), + position: non_neg_integer(), + delta_position: non_neg_integer() | nil, + strategy: strategy() + } +end diff --git a/lib/ex_datalog/planner/plan.ex b/lib/ex_datalog/planner/plan.ex new file mode 100644 index 0000000..971b92e --- /dev/null +++ b/lib/ex_datalog/planner/plan.ex @@ -0,0 +1,28 @@ +defmodule ExDatalog.Planner.Plan do + @moduledoc """ + An execution plan for a compiled IR program. + + A plan records the chosen evaluation `strategy`, the planned `strata`, the + `joins` (one per positive body atom across all rules), and the `predicates` + (constraints, aggregates, callbacks). The plan is descriptive: it explains + what the engine will do without changing how the engine evaluates. + + Aggregate and callback predicates appear in `predicates` with + `kind: :aggregate` / `kind: :callback`; there are no separate fields for them. + """ + + alias ExDatalog.Planner.{Join, Predicate, Stratum} + + @enforce_keys [:strategy, :strata] + defstruct [:strategy, :strata, joins: [], predicates: [], metadata: %{}] + + @type strategy :: :semi_naive | :magic_sets + + @type t :: %__MODULE__{ + strategy: strategy(), + strata: [Stratum.t()], + joins: [Join.t()], + predicates: [Predicate.t()], + metadata: map() + } +end diff --git a/lib/ex_datalog/planner/predicate.ex b/lib/ex_datalog/planner/predicate.ex new file mode 100644 index 0000000..a663872 --- /dev/null +++ b/lib/ex_datalog/planner/predicate.ex @@ -0,0 +1,28 @@ +defmodule ExDatalog.Planner.Predicate do + @moduledoc """ + A planned predicate: a non-relational body element (constraint, aggregate, + or callback) classified by kind. + + `kind` groups the predicate into one of the evaluation categories; `op` is + the specific operator (e.g. `:gt`, `:count`, `:callback`). `metadata` carries + optional details (e.g. callback module/function). + """ + + @enforce_keys [:kind, :op] + defstruct [:kind, :op, metadata: %{}] + + @type kind :: + :comparison + | :arithmetic + | :type + | :string + | :membership + | :aggregate + | :callback + + @type t :: %__MODULE__{ + kind: kind(), + op: atom(), + metadata: map() + } +end diff --git a/lib/ex_datalog/planner/stratum.ex b/lib/ex_datalog/planner/stratum.ex new file mode 100644 index 0000000..280c24a --- /dev/null +++ b/lib/ex_datalog/planner/stratum.ex @@ -0,0 +1,20 @@ +defmodule ExDatalog.Planner.Stratum do + @moduledoc """ + A planned stratum: the rules and relations evaluated together at one + stratum index. + + Wraps the IR strata into a planner-friendly form that carries the actual + `IR.Rule` structs (not just their IDs) for the engine and `explain_plan/1`. + """ + + alias ExDatalog.IR + + @enforce_keys [:index, :rules, :relations] + defstruct [:index, :rules, :relations] + + @type t :: %__MODULE__{ + index: non_neg_integer(), + rules: [IR.Rule.t()], + relations: [String.t()] + } +end diff --git a/lib/ex_datalog/program.ex b/lib/ex_datalog/program.ex index 1c01dda..ec44a4a 100644 --- a/lib/ex_datalog/program.ex +++ b/lib/ex_datalog/program.ex @@ -236,6 +236,67 @@ defmodule ExDatalog.Program do def add_fact({:error, _} = err, _relation, _values), do: err + @doc """ + Adds a fact to the program from a `{relation, values}` tuple. + + This is the tuple form produced by Schema relation constructors + (e.g., `MySchema.emp(:alice, :eng)` returns `{"emp", [:alice, :eng]}`). + + ## Examples + + iex> alias ExDatalog.Program + iex> program = Program.new() |> Program.add_relation("emp", [:atom, :atom]) + iex> program = Program.add_fact(program, {"emp", [:alice, :eng]}) + iex> program.facts + [{"emp", [:alice, :eng]}] + """ + @spec add_fact(t(), {String.t(), [term()]}) :: t() | {:error, term()} + def add_fact(program, {relation, values}) when is_binary(relation) and is_list(values) do + add_fact(program, relation, values) + end + + def add_fact({:error, _} = err, {_relation, _values}), do: err + + @doc """ + Adds multiple facts to the program from a list of `{relation, values}` tuples. + + ## Examples + + iex> alias ExDatalog.Program + iex> program = Program.new() |> Program.add_relation("emp", [:atom, :atom]) + iex> facts = [{"emp", [:alice, :eng]}, {"emp", [:bob, :eng]}] + iex> program = Program.add_facts(program, facts) + iex> length(program.facts) + 2 + """ + @spec add_facts(t(), [{String.t(), [term()]}]) :: t() | {:error, term()} + def add_facts(program, facts) when is_list(facts) do + Enum.reduce_while(facts, program, fn fact, acc -> + case add_fact(acc, fact) do + {:error, _} = err -> {:halt, err} + prog -> {:cont, prog} + end + end) + end + + @doc """ + Materializes the program. A pipe-friendly convenience for + `ExDatalog.materialize/2`. + + ## Examples + + iex> alias ExDatalog.{Program, Knowledge} + iex> program = Program.new() |> Program.add_relation("edge", [:atom, :atom]) + iex> program = program |> Program.add_fact("edge", [:a, :b]) + iex> {:ok, knowledge} = Program.materialize(program) + iex> Knowledge.get(knowledge, "edge") |> MapSet.to_list() + [{:a, :b}] + """ + @spec materialize(t(), keyword()) :: {:ok, ExDatalog.Knowledge.t()} | {:error, term()} + def materialize(%__MODULE__{} = program, opts \\ []) do + ExDatalog.materialize(program, opts) + end + @doc """ Adds a rule to the program. @@ -431,17 +492,18 @@ defmodule ExDatalog.Program do defp validate_body(program, body) do Enum.reduce_while(body, :ok, fn literal, :ok -> - atom = - case literal do - {:positive, a} -> a - {:negative, a} -> a - other -> {:error, "invalid body literal #{inspect(other)}"} - end - - case atom do - {:error, _} = err -> {:halt, err} - %Atom{} -> {:cont, validate_atom(program, atom)} - _ -> {:halt, {:error, "invalid body literal #{inspect(literal)}"}} + case literal do + {:callback, %ExDatalog.Callback{}} -> + {:cont, :ok} + + {:positive, %Atom{} = a} -> + {:cont, validate_atom(program, a)} + + {:negative, %Atom{} = a} -> + {:cont, validate_atom(program, a)} + + other -> + {:halt, {:error, "invalid body literal #{inspect(other)}"}} end end) end diff --git a/lib/ex_datalog/rule.ex b/lib/ex_datalog/rule.ex index 270155d..20564d9 100644 --- a/lib/ex_datalog/rule.ex +++ b/lib/ex_datalog/rule.ex @@ -28,9 +28,9 @@ defmodule ExDatalog.Rule do """ - alias ExDatalog.{Atom, Constraint} + alias ExDatalog.{Atom, Callback, Constraint} - @type literal :: {:positive, Atom.t()} | {:negative, Atom.t()} + @type literal :: {:positive, Atom.t()} | {:negative, Atom.t()} | {:callback, Callback.t()} @type t :: %__MODULE__{ head: Atom.t(), @@ -91,12 +91,18 @@ defmodule ExDatalog.Rule do Enum.flat_map(body, fn {:positive, atom} -> Atom.variables(atom) {:negative, atom} -> Atom.variables(atom) + {:callback, cb} -> Callback.input_variables(cb) ++ callback_result_var(cb) end) constraint_vars = Enum.flat_map(constraints, fn c -> input = Constraint.input_variables(c) - result = if Constraint.arithmetic?(c), do: [Constraint.result_variable(c)], else: [] + + result = + if Constraint.arithmetic?(c) or Constraint.aggregate?(c), + do: [Constraint.result_variable(c)], + else: [] + input ++ result end) @@ -141,6 +147,7 @@ defmodule ExDatalog.Rule do |> Enum.flat_map(fn {:positive, atom} -> Atom.variables(atom) {:negative, _} -> [] + {:callback, _} -> [] end) |> Enum.uniq() end @@ -163,9 +170,11 @@ defmodule ExDatalog.Rule do """ @spec body_atoms(t()) :: [Atom.t()] def body_atoms(%__MODULE__{body: body}) do - Enum.map(body, fn - {:positive, atom} -> atom - {:negative, atom} -> atom + body + |> Enum.flat_map(fn + {:positive, atom} -> [atom] + {:negative, atom} -> [atom] + {:callback, _} -> [] end) end @@ -198,4 +207,62 @@ defmodule ExDatalog.Rule do _ -> false end) end + + @doc """ + Returns `true` if the rule contains any aggregate constraint + (`count`, `sum`, `min`, `max`). + + ## Examples + + iex> alias ExDatalog.{Rule, Atom, Term, Constraint} + iex> head = Atom.new("dept_count", [Term.var("D"), Term.var("N")]) + iex> body = [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D")])}] + iex> rule = Rule.new(head, body, [Constraint.count(Term.var("E"), Term.var("N"))]) + iex> Rule.has_aggregates?(rule) + true + + """ + @spec has_aggregates?(t()) :: boolean() + def has_aggregates?(%__MODULE__{constraints: constraints}) do + Enum.any?(constraints, &Constraint.aggregate?/1) + end + + @doc """ + Returns the aggregate constraints in the rule. + """ + @spec aggregate_constraints(t()) :: [Constraint.t()] + def aggregate_constraints(%__MODULE__{constraints: constraints}) do + Enum.filter(constraints, &Constraint.aggregate?/1) + end + + @doc """ + Returns the non-aggregate constraints in the rule. + """ + @spec non_aggregate_constraints(t()) :: [Constraint.t()] + def non_aggregate_constraints(%__MODULE__{constraints: constraints}) do + Enum.reject(constraints, &Constraint.aggregate?/1) + end + + @doc """ + Returns `true` if the rule contains any callback predicate in its body. + """ + @spec has_callbacks?(t()) :: boolean() + def has_callbacks?(%__MODULE__{body: body}) do + Enum.any?(body, &match?({:callback, _}, &1)) + end + + @doc """ + Returns the callback predicates in the rule body. + """ + @spec callbacks(t()) :: [Callback.t()] + def callbacks(%__MODULE__{body: body}) do + for {:callback, cb} <- body, do: cb + end + + defp callback_result_var(%Callback{} = cb) do + case Callback.result_variable(cb) do + nil -> [] + name -> [name] + end + end end diff --git a/lib/ex_datalog/schema.ex b/lib/ex_datalog/schema.ex index 1fad8df..38bb8cb 100644 --- a/lib/ex_datalog/schema.ex +++ b/lib/ex_datalog/schema.ex @@ -178,12 +178,13 @@ defmodule ExDatalog.Schema do defmacro __using__(_opts) do quote do import ExDatalog.Schema, - only: [relation: 2, fact: 1, facts: 2, rule: 2, query: 2, wildcard: 0] + only: [relation: 2, fact: 1, facts: 2, rule: 2, query: 2, wildcard: 0, predicate: 5] Module.register_attribute(__MODULE__, :ex_datalog_relations, accumulate: true) Module.register_attribute(__MODULE__, :ex_datalog_facts, accumulate: true) Module.register_attribute(__MODULE__, :ex_datalog_rules, accumulate: true) Module.register_attribute(__MODULE__, :ex_datalog_queries, accumulate: true) + Module.register_attribute(__MODULE__, :ex_datalog_predicates, accumulate: true) @before_compile ExDatalog.Schema end @@ -195,6 +196,7 @@ defmodule ExDatalog.Schema do facts = Module.get_attribute(env.module, :ex_datalog_facts) |> Enum.reverse() rules = Module.get_attribute(env.module, :ex_datalog_rules) |> Enum.reverse() queries = Module.get_attribute(env.module, :ex_datalog_queries) |> Enum.reverse() + predicates = Module.get_attribute(env.module, :ex_datalog_predicates) |> Enum.reverse() relation_names = MapSet.new(relations, fn rel -> Atom.to_string(rel.name) end) @@ -213,17 +215,67 @@ defmodule ExDatalog.Schema do end end) + relation_funs = + Enum.map(relations, fn rel -> + name = rel.name + arity = length(rel.fields) + args = for i <- 1..arity, do: Macro.var(:"arg_#{i}", __MODULE__) + + quote do + @doc """ + Constructs a fact tuple for the `#{unquote(name)}` relation. + + #{unquote(name)}(#{Enum.map_join(1..unquote(arity), ", ", fn i -> "arg_#{i}" end)}) + #=> {"#{unquote(name)}", [#{Enum.map_join(1..unquote(arity), ", ", fn i -> "arg_#{i}" end)})]} + + Pass the result to `Program.add_fact/2` as a fact tuple. + """ + @spec unquote(name)(unquote_splicing(Enum.map(args, fn _ -> quote do: term() end))) :: + {String.t(), [term()]} + def unquote(name)(unquote_splicing(args)) do + {unquote(Atom.to_string(name)), [unquote_splicing(args)]} + end + end + end) + quote do + unquote_splicing(relation_funs) + @doc """ Returns the `ExDatalog.Program` built from this schema's relations, - facts, and rules. + compile-time facts, and rules. + + Use this when all facts are declared at compile time via `fact/1`. + For runtime facts, use `new_program/0` and `Program.add_fact/2`. """ @spec program() :: ExDatalog.Program.t() def program do ExDatalog.Schema.__build_program__( unquote(Macro.escape(relations)), unquote(Macro.escape(facts)), - unquote(Macro.escape(rules)) + unquote(Macro.escape(rules)), + unquote(Macro.escape(predicates)) + ) + end + + @doc """ + Returns a blank `ExDatalog.Program` from this schema's relations and + rules, **without** any compile-time facts. + + Add runtime facts via the pipeable `Program.add_fact/2`: + + DeptCount.new() + |> Program.add_fact(DeptCount.emp(:alice, :eng)) + |> Program.add_fact(DeptCount.emp(:bob, :eng)) + |> Program.materialize() + """ + @spec new() :: ExDatalog.Program.t() + def new do + ExDatalog.Schema.__build_program__( + unquote(Macro.escape(relations)), + [], + unquote(Macro.escape(rules)), + unquote(Macro.escape(predicates)) ) end @@ -258,8 +310,9 @@ defmodule ExDatalog.Schema do end @doc false - def __build_program__(relations, facts, rules) do + def __build_program__(relations, facts, rules, predicates \\ []) do program = ExDatalog.Program.new() + predicate_map = Map.new(predicates, fn p -> {Atom.to_string(p.name), p} end) program = Enum.reduce(relations, program, fn rel_meta, acc -> @@ -293,8 +346,7 @@ defmodule ExDatalog.Schema do body = Enum.map(body_literals, fn {:positive, %ExDatalog.Atom{} = atom} -> - {:positive, - %ExDatalog.Atom{atom | terms: Enum.map(atom.terms, &term_from_parsed/1)}} + resolve_positive_or_callback(atom, predicate_map) {:negative, %ExDatalog.Atom{} = atom} -> {:negative, @@ -359,6 +411,31 @@ defmodule ExDatalog.Schema do defp term_from_parsed({:const, value}), do: ExDatalog.Term.from(value) defp term_from_parsed(:wildcard), do: ExDatalog.Term.from(:_) + # A positive body atom whose "relation" matches a declared predicate becomes + # a callback literal; otherwise it stays a positive atom. + defp resolve_positive_or_callback(%ExDatalog.Atom{relation: rel, terms: terms}, predicate_map) do + case Map.get(predicate_map, rel) do + nil -> + {:positive, %ExDatalog.Atom{relation: rel, terms: Enum.map(terms, &term_from_parsed/1)}} + + %ExDatalog.Schema.PredicateMeta{} = pred -> + build_callback_literal(pred, terms) + end + end + + defp build_callback_literal(%ExDatalog.Schema.PredicateMeta{} = pred, terms) do + arg_terms = Enum.map(terms, &term_from_parsed/1) + + case pred.return_type do + :boolean -> + {:callback, ExDatalog.Callback.new(pred.module, pred.function, arg_terms, nil)} + + :value -> + {args, [result]} = Enum.split(arg_terms, length(arg_terms) - 1) + {:callback, ExDatalog.Callback.new(pred.module, pred.function, args, result)} + end + end + @doc false def __execute_query__(name, knowledge, queries) when is_list(queries) do query_map = Map.new(queries, fn q -> {q.name, q} end) @@ -623,7 +700,14 @@ defmodule ExDatalog.Schema do defp parse_term({:agg, _, _args}) do raise ExDatalog.DSL.CompileError, - message: "aggregates are not yet supported (planned for v0.6.0)" + message: "use count/sum/min/max for aggregates (e.g. `count(X, N)`)" + end + + defp parse_term({op, _, _args}) when op in [:count, :sum, :min, :max] do + raise ExDatalog.DSL.CompileError, + message: + "aggregate #{op}/2 cannot appear in a rule head or as a term; " <> + "place it in the rule body (e.g. `#{op}(X, Result)`) and use Result in the head" end defp parse_term(other) do @@ -670,9 +754,17 @@ defmodule ExDatalog.Schema do defp parse_body_call({:agg, _, _args}) do raise ExDatalog.DSL.CompileError, - message: "aggregates are not yet supported (planned for v0.6.0)" + message: "use count/sum/min/max for aggregates (e.g. `count(X, N)`)" end + aggregate_ops = [:count, :sum, :min, :max] + + Enum.each(aggregate_ops, fn op -> + defp parse_body_call({unquote(op), _, [input, result]}) do + {:constraint, build_aggregate(unquote(op), input, result)} + end + end) + constraint_ops = [ :eq, :neq, @@ -745,6 +837,10 @@ defmodule ExDatalog.Schema do end end) + defp build_aggregate(op, input, result) do + ExDatalog.Constraint.from_tuple({op, parse_term(input), parse_term(result)}) + end + defp build_constraint(op, args) do raise ExDatalog.DSL.CompileError, message: "unsupported constraint #{op}/#{length(args)}: #{inspect(args)}" @@ -874,4 +970,55 @@ defmodule ExDatalog.Schema do """ @spec wildcard() :: :wildcard def wildcard, do: :wildcard + + @doc """ + Declares a BEAM callback predicate usable in rule bodies. + + predicate :adult?, MyPredicates, :adult?, [:integer], :boolean + + rule adult(Name) do + person(Name, Age) + adult?(Age) + end + + Arguments: + + - `name` — the predicate name used in rule bodies. + - `module` / `function` — the Elixir function to call. + - `arg_types` — declared argument types (currently informational; their + length sets the callback arity, validated at compile time). + - `return_type` — `:boolean` (filter) or `:value` (binds the last argument + position as the result variable). + + For `:value` predicates, the **last** argument in the rule-body call is the + result variable; the remaining arguments are passed to the function. + + The referenced `module.function` must be defined and exported with arity + matching the inputs, or a compile-time `ExDatalog.DSL.CompileError` is raised. + """ + defmacro predicate(name, module, function, arg_types, return_type) do + quote do + ExDatalog.Schema.__register_predicate__( + __MODULE__, + unquote(name), + unquote(module), + unquote(function), + unquote(Macro.escape(arg_types)), + unquote(return_type) + ) + end + end + + @doc false + def __register_predicate__(mod, name, module, function, arg_types, return_type) + when is_atom(name) and is_atom(module) and is_atom(function) and is_list(arg_types) and + return_type in [:boolean, :value] do + Module.put_attribute(mod, :ex_datalog_predicates, %ExDatalog.Schema.PredicateMeta{ + name: name, + module: module, + function: function, + arg_types: arg_types, + return_type: return_type + }) + end end diff --git a/lib/ex_datalog/schema/predicate_meta.ex b/lib/ex_datalog/schema/predicate_meta.ex new file mode 100644 index 0000000..5dcbb89 --- /dev/null +++ b/lib/ex_datalog/schema/predicate_meta.ex @@ -0,0 +1,22 @@ +defmodule ExDatalog.Schema.PredicateMeta do + @moduledoc """ + Metadata for a callback predicate declared with the `predicate/5` macro. + + Records the DSL name, the target module/function, the declared argument + types, and the return type (`:boolean` for filters, `:value` for + value-returning callbacks). + """ + + @enforce_keys [:name, :module, :function, :arg_types, :return_type] + defstruct [:name, :module, :function, :arg_types, :return_type] + + @type return_type :: :boolean | :value + + @type t :: %__MODULE__{ + name: atom(), + module: module(), + function: atom(), + arg_types: [atom()], + return_type: return_type() + } +end diff --git a/lib/ex_datalog/validator.ex b/lib/ex_datalog/validator.ex index e5cb9d0..70bc2f4 100644 --- a/lib/ex_datalog/validator.ex +++ b/lib/ex_datalog/validator.ex @@ -140,6 +140,9 @@ defmodule ExDatalog.Validator do {:negative, %Atom{} = atom} -> check_atom(acc, atom, rels, Map.put(context, :polarity, :negative)) + {:callback, %ExDatalog.Callback{} = cb} -> + check_callback(acc, cb, Map.put(context, :polarity, :callback)) + other -> [ Error.new( @@ -154,6 +157,24 @@ defmodule ExDatalog.Validator do end) end + defp check_callback(errors, %ExDatalog.Callback{module: m, function: f, args: args}, context) do + arity = length(args) + + if Code.ensure_loaded?(m) and function_exported?(m, f, arity) do + errors + else + [ + Error.new( + :invalid_callback, + Map.merge(context, %{module: m, function: f, arity: arity}), + "#{location(context)} callback #{inspect(m)}.#{f}/#{arity} " <> + "is not defined or not exported" + ) + | errors + ] + end + end + defp check_atom(errors, %Atom{relation: rel, terms: terms}, rels, context) do case Map.fetch(rels, rel) do :error -> diff --git a/lib/ex_datalog/validator/error.ex b/lib/ex_datalog/validator/error.ex index 050df27..085a696 100644 --- a/lib/ex_datalog/validator/error.ex +++ b/lib/ex_datalog/validator/error.ex @@ -21,6 +21,10 @@ defmodule ExDatalog.Validator.Error do | `:unstratified_negation` | 2 | A negative edge appears in a dependency cycle | | `:unbound_constraint_variable` | 2 | A constraint references a variable not yet bound | | `:invalid_body_literal` | 1 | A body literal is not `{:positive, atom}` or `{:negative, atom}` | + | `:multiple_aggregates` | 2 | A rule contains more than one aggregate constraint | + | `:aggregate_in_recursion` | 2 | An aggregate appears in a self-recursive rule | + | `:unstratified_aggregate` | 2 | An aggregate's input relation is not in a strictly lower stratum | + | `:invalid_callback` | 2 | A callback references a missing module/function or wrong arity | ## Examples @@ -44,6 +48,10 @@ defmodule ExDatalog.Validator.Error do | :unbound_constraint_variable | :invalid_body_literal | :wildcard_in_head + | :multiple_aggregates + | :aggregate_in_recursion + | :unstratified_aggregate + | :invalid_callback @type t :: %__MODULE__{ kind: kind(), diff --git a/lib/ex_datalog/validator/safety.ex b/lib/ex_datalog/validator/safety.ex index 0ede0f8..cd17e2b 100644 --- a/lib/ex_datalog/validator/safety.ex +++ b/lib/ex_datalog/validator/safety.ex @@ -99,13 +99,41 @@ defmodule ExDatalog.Validator.Safety do |> check_wildcards_in_head(rule, rule_index) |> check_unsafe_head_variables(head_vars, head_bound, rule_index) |> check_unbound_constraint_variables(rule, body_bound, rule_index) + |> check_aggregates(rule, rule_index) + |> check_callback_inputs(rule, body_bound, rule_index) else [] end end + # Callback input variables must be bound by positive body atoms. Callbacks + # are filters (boolean) or value-binders; their inputs cannot be introduced + # by the callback itself. + defp check_callback_inputs(errors, %Rule{body: body}, body_bound, rule_index) do + callbacks = for {:callback, cb} <- body, do: cb + + Enum.reduce(callbacks, errors, fn cb, acc -> + unbound = Enum.reject(ExDatalog.Callback.input_variables(cb), fn v -> v in body_bound end) + + if unbound == [] do + acc + else + [ + Error.new( + :unbound_constraint_variable, + %{rule_index: rule_index, variables: unbound, callback: {cb.module, cb.function}}, + "rule #{rule_index}: callback #{inspect(cb.module)}.#{cb.function} " <> + "references unbound variable(s) #{Enum.join(unbound, ", ")}" + ) + | acc + ] + end + end) + end + defp valid_literal?({:positive, %Atom{}}), do: true defp valid_literal?({:negative, %Atom{}}), do: true + defp valid_literal?({:callback, %ExDatalog.Callback{}}), do: true defp valid_literal?(_), do: false # --- Private helpers --- @@ -126,13 +154,21 @@ defmodule ExDatalog.Validator.Safety do # Used for head-variable safety: if the rule body evaluates at all, every # arithmetic constraint will have computed its result, so arithmetic results # are in scope for the head regardless of constraint ordering. - defp all_bound_variables(%Rule{constraints: constraints} = rule) do - arithmetic_result_vars = + defp all_bound_variables(%Rule{constraints: constraints, body: body} = rule) do + result_vars = Enum.flat_map(constraints, fn c -> - if Constraint.arithmetic?(c), do: [Constraint.result_variable(c)], else: [] + if Constraint.arithmetic?(c) or Constraint.aggregate?(c), + do: [Constraint.result_variable(c)], + else: [] end) - (positive_body_variables(rule) ++ arithmetic_result_vars) |> Enum.uniq() + callback_result_vars = + for {:callback, cb} <- body, + name = ExDatalog.Callback.result_variable(cb), + name != nil, + do: name + + (positive_body_variables(rule) ++ result_vars ++ callback_result_vars) |> Enum.uniq() end defp check_wildcards_in_head(errors, %Rule{head: head}, rule_index) do @@ -210,10 +246,10 @@ defmodule ExDatalog.Validator.Safety do ] end - # Arithmetic constraints extend the bound set with their result variable - # so that subsequent constraints may reference it. + # Arithmetic and aggregate constraints extend the bound set with their + # result variable so that subsequent constraints may reference it. new_bound = - if Constraint.arithmetic?(c) do + if Constraint.arithmetic?(c) or Constraint.aggregate?(c) do case Constraint.result_variable(c) do nil -> bound var -> Enum.uniq([var | bound]) @@ -224,4 +260,59 @@ defmodule ExDatalog.Validator.Safety do {new_errors, new_bound} end + + # Aggregate-specific safety: + # 1. At most one aggregate per rule (initial v0.5.0 scope). + # 2. An aggregate may not appear in a self-recursive rule (aggregates are + # non-monotone and must be stratified above their inputs). + defp check_aggregates(errors, %Rule{} = rule, rule_index) do + aggregates = Rule.aggregate_constraints(rule) + + errors + |> check_single_aggregate(aggregates, rule, rule_index) + |> check_aggregate_not_recursive(aggregates, rule, rule_index) + end + + defp check_single_aggregate(errors, aggregates, _rule, rule_index) do + if length(aggregates) > 1 do + [ + Error.new( + :multiple_aggregates, + %{rule_index: rule_index, count: length(aggregates)}, + "rule #{rule_index}: a rule may contain at most one aggregate " <> + "(found #{length(aggregates)}); split into separate rules" + ) + | errors + ] + else + errors + end + end + + defp check_aggregate_not_recursive(errors, [], _rule, _rule_index), do: errors + + defp check_aggregate_not_recursive(errors, _aggregates, %Rule{} = rule, rule_index) do + head_rel = rule.head.relation + + self_recursive? = + Enum.any?(rule.body, fn + {:positive, %Atom{relation: ^head_rel}} -> true + {:negative, %Atom{relation: ^head_rel}} -> true + _ -> false + end) + + if self_recursive? do + [ + Error.new( + :aggregate_in_recursion, + %{rule_index: rule_index, relation: head_rel}, + "rule #{rule_index}: aggregate over a self-recursive relation " <> + "#{inspect(head_rel)} is not allowed; aggregates are non-monotone" + ) + | errors + ] + else + errors + end + end end diff --git a/lib/ex_datalog/validator/stratification.ex b/lib/ex_datalog/validator/stratification.ex index 0cca526..4531b0d 100644 --- a/lib/ex_datalog/validator/stratification.ex +++ b/lib/ex_datalog/validator/stratification.ex @@ -101,7 +101,74 @@ defmodule ExDatalog.Validator.Stratification do |> MapSet.union(MapSet.new(all_vertices(graph))) |> MapSet.to_list() - assign_strata_greedy(graph, sccs, all_rels) + graph + |> assign_strata_greedy(sccs, all_rels) + |> force_aggregate_strata(program) + end + + # Aggregates are non-monotone: a rule that aggregates over its positive body + # relations must evaluate in a stratum strictly above all of them. This runs a + # fixpoint that bumps each aggregate rule's head relation until the invariant + # holds, then re-propagates so dependents stay ordered. + defp force_aggregate_strata(strata, %ExDatalog.Program{rules: rules} = program) do + aggregate_rules = Enum.filter(rules, &ExDatalog.Rule.has_aggregates?/1) + + if aggregate_rules == [] do + strata + else + graph = build_graph(program) + stabilize_aggregate_strata(strata, aggregate_rules, graph) + end + end + + defp stabilize_aggregate_strata(strata, aggregate_rules, graph, fuel \\ 1000) + + defp stabilize_aggregate_strata(strata, _aggregate_rules, _graph, 0), do: strata + + defp stabilize_aggregate_strata(strata, aggregate_rules, graph, fuel) do + bumped = + Enum.reduce(aggregate_rules, strata, fn rule, acc -> + head_rel = rule.head.relation + + body_rels = + for {:positive, %Atom{relation: rel}} <- rule.body, rel != head_rel, do: rel + + required = + case body_rels do + [] -> Map.get(acc, head_rel, 0) + _ -> (body_rels |> Enum.map(&Map.get(acc, &1, 0)) |> Enum.max()) + 1 + end + + if Map.get(acc, head_rel, 0) < required do + Map.put(acc, head_rel, required) + else + acc + end + end) + + bumped = propagate_strata(bumped, graph) + + if bumped == strata do + strata + else + stabilize_aggregate_strata(bumped, aggregate_rules, graph, fuel - 1) + end + end + + # After bumping an aggregate head, every relation that positively depends on it + # must be at least as high; negative/aggregate dependents must be strictly + # higher. One pass; the outer fixpoint repeats until stable. + defp propagate_strata(strata, graph) do + Enum.reduce(graph, strata, fn {head_rel, deps}, acc -> + required = Enum.reduce(deps, Map.get(acc, head_rel, 0), &required_stratum(&1, &2, acc)) + Map.put(acc, head_rel, required) + end) + end + + defp required_stratum({dep, polarity}, current, acc) do + dep_stratum = Map.get(acc, dep, 0) + needed = if polarity == :negative, do: dep_stratum + 1, else: dep_stratum + max(current, needed) end @doc false diff --git a/livebooks/ex_datalog_v050.livemd b/livebooks/ex_datalog_v050.livemd new file mode 100644 index 0000000..6c1696c --- /dev/null +++ b/livebooks/ex_datalog_v050.livemd @@ -0,0 +1,778 @@ + + +# ExDatalog v0.5.0 — DSL Edition + +## Section + +A walkthrough of v0.5.0 features using the **Schema DSL** (`use ExDatalog.Schema`). + +```elixir +Mix.install([ + {:ex_datalog, path: Path.expand("..", __DIR__), env: :prod}, +]) +``` + + + +``` +:ok +``` + +## Introduction + +**ExDatalog v0.5.0** adds four major features: + +* **Aggregates** — `count`, `sum`, `min`, `max` over grouped bindings +* **BEAM callbacks** — call deterministic Elixir functions from rule bodies +* **Magic sets** — goal-directed evaluation that computes only relevant facts +* **Query planner** — inspect and explain evaluation plans before running them + +All examples below use the Schema DSL exclusively. + +--- + +## Setup + +```elixir +alias ExDatalog.{Knowledge, MagicSets, Planner} +``` + + + +``` +[ExDatalog.Knowledge, ExDatalog.MagicSets, ExDatalog.Planner] +``` + +--- + +## 1. Aggregates + +Aggregates compute a single value per group of bindings. In the DSL they appear +in the **rule body** as `count(X, N)`, `sum(A, T)`, `min(S, V)`, `max(S, V)`. +The result variable (`N`, `T`, `V`) must also appear in the rule head. + +> **Restrictions** (enforced at validation): +> +> * Only **one** aggregate per rule. +> * No aggregate over a **self-recursive** relation. +> * The aggregate input variable must be **bound** by a positive body atom. + +### Count — employees per department + +```elixir +defmodule DeptCount do + use ExDatalog.Schema + + relation :emp do + field(:name, :atom) + field(:dept, :atom) + end + + relation :dept_count do + field(:dept, :atom) + field(:n, :integer) + end + + fact(emp(:alice, :eng)) + fact(emp(:bob, :eng)) + fact(emp(:carol, :ops)) + fact(emp(:dave, :eng)) + fact(emp(:eve, :ops)) + + rule dept_count(D, N) do + emp(E, D) + count(E, N) + end +end + +{:ok, knowledge} = DeptCount.materialize() +Knowledge.get(knowledge, "dept_count") |> MapSet.to_list() |> Enum.sort() +``` + + + +``` +[eng: 3, ops: 2] +``` + +### Sum — total salary per department + +```elixir +defmodule DeptTotal do + use ExDatalog.Schema + + relation :salary do + field(:name, :atom) + field(:dept, :atom) + field(:amount, :integer) + end + + relation :dept_total do + field(:dept, :atom) + field(:total, :integer) + end + + fact(salary(:alice, :eng, 120)) + fact(salary(:bob, :eng, 80)) + fact(salary(:carol, :ops, 50)) + fact(salary(:dave, :eng, 100)) + + rule dept_total(D, T) do + salary(E, D, A) + sum(A, T) + end +end + +{:ok, knowledge} = DeptTotal.materialize() +Knowledge.get(knowledge, "dept_total") |> MapSet.to_list() |> Enum.sort() +``` + + + +``` +[eng: 300, ops: 50] +``` + +### Min and Max + +```elixir +defmodule DeptScores do + use ExDatalog.Schema + + relation :score do + field(:name, :atom) + field(:dept, :atom) + field(:value, :integer) + end + + relation :lowest do + field(:dept, :atom) + field(:value, :integer) + end + + relation :highest do + field(:dept, :atom) + field(:value, :integer) + end + + fact(score(:alice, :eng, 90)) + fact(score(:bob, :eng, 70)) + fact(score(:carol, :ops, 60)) + fact(score(:dave, :eng, 85)) + fact(score(:eve, :ops, 55)) + + rule lowest(D, V) do + score(E, D, S) + min(S, V) + end + + rule highest(D, V) do + score(E, D, S) + max(S, V) + end +end + +{:ok, knowledge} = DeptScores.materialize() + +IO.puts("Lowest per dept:") +Knowledge.get(knowledge, "lowest") |> MapSet.to_list() |> Enum.sort() |> IO.inspect() + +IO.puts("\nHighest per dept:") +Knowledge.get(knowledge, "highest") |> MapSet.to_list() |> Enum.sort() |> IO.inspect() +``` + + + +``` +Lowest per dept: +[eng: 70, ops: 55] + +Highest per dept: +[eng: 90, ops: 60] +``` + + + +``` +[eng: 90, ops: 60] +``` + +### Aggregate with a filter constraint + +Constraints are evaluated **before** grouping. Count only passing scores (≥ 60): + +```elixir +defmodule PassingCount do + use ExDatalog.Schema + + relation :score do + field(:name, :atom) + field(:dept, :atom) + field(:value, :integer) + end + + relation :passing_count do + field(:dept, :atom) + field(:n, :integer) + end + + fact(score(:alice, :eng, 90)) + fact(score(:bob, :eng, 40)) + fact(score(:carol, :eng, 75)) + fact(score(:dave, :ops, 30)) + fact(score(:eve, :ops, 80)) + + rule passing_count(D, N) do + score(E, D, S) + gte(S, 60) + count(E, N) + end +end + +{:ok, knowledge} = PassingCount.materialize() +Knowledge.get(knowledge, "passing_count") |> MapSet.to_list() |> Enum.sort() +``` + + + +``` +[eng: 2, ops: 1] +``` + +### Aggregate over a derived relation (stratification) + +When an aggregate reads from a **derived** relation, the engine automatically +places it in a higher stratum — the derived facts must be fully computed first. + +```elixir +defmodule StratifiedAggregate do + use ExDatalog.Schema + + relation :base do + field(:key, :atom) + field(:val, :atom) + end + + relation :mid do + field(:key, :atom) + field(:val, :atom) + end + + relation :mid_count do + field(:key, :atom) + field(:n, :integer) + end + + fact(base(:a, :x)) + fact(base(:a, :y)) + fact(base(:b, :z)) + + rule mid(K, V) do + base(K, V) + end + + rule mid_count(K, N) do + mid(K, V) + count(V, N) + end +end + +{:ok, knowledge} = StratifiedAggregate.materialize() +Knowledge.get(knowledge, "mid_count") |> MapSet.to_list() |> Enum.sort() +``` + + + +``` +[a: 2, b: 1] +``` + +`mid` is derived first (stratum 0), then the aggregate counts over it (stratum 1). + +--- + +## 2. BEAM Callbacks + +BEAM callbacks let rule bodies invoke **deterministic, side-effect-free** Elixir +functions. Declare a predicate with the `predicate` macro, then use its name in +a rule body. + +Two kinds: + +* **Boolean** (`:boolean`) — returns `true` to keep the binding, `false` to drop it. +* **Value** (`:value`) — the last argument in the rule-body call is the result + variable; the function's return value is bound to it. + +The engine enforces **timeout** (default 100 ms, configurable via +`:callback_timeout_ms`) and **exception isolation** — a callback that times out +or raises simply filters the binding. + +### Boolean filter predicate + +First define a helper module with the predicate function: + +```elixir +defmodule MyPredicates do + def adult?(age), do: age >= 18 + def valid_email?(email), do: String.contains?(email, "@") +end +``` + + + +``` +{:module, MyPredicates, <<70, 79, 82, 49, 0, 0, 7, ...>>, ...} +``` + +Now use `predicate` in the schema: + +```elixir +defmodule AdultOnly do + use ExDatalog.Schema + + relation :person do + field(:name, :atom) + field(:age, :integer) + end + + relation :adult do + field(:name, :atom) + end + + predicate(:adult?, MyPredicates, :adult?, [:integer], :boolean) + + fact(person(:alice, 25)) + fact(person(:bob, 12)) + fact(person(:carol, 19)) + + rule adult(Name) do + person(Name, Age) + adult?(Age) + end +end + +{:ok, knowledge} = AdultOnly.materialize() +Knowledge.get(knowledge, "adult") |> MapSet.to_list() |> Enum.sort() +``` + + + +``` +[{:alice}, {:carol}] +``` + +Only `:alice` and `:carol` pass — `:bob` is 12, so `adult?` returns `false`. + +### Value-returning predicate + +The last argument in the rule-body call is the **result variable**: + +```elixir +defmodule MyMath do + def double(x), do: x * 2 +end + +defmodule DoubledNumbers do + use ExDatalog.Schema + + relation :num do + field(:x, :integer) + end + + relation :doubled do + field(:x, :integer) + field(:y, :integer) + end + + predicate(:double, MyMath, :double, [:integer], :value) + + fact(num(3)) + fact(num(5)) + fact(num(7)) + + rule doubled(X, Y) do + num(X) + double(X, Y) + end +end + +{:ok, knowledge} = DoubledNumbers.materialize() +Knowledge.get(knowledge, "doubled") |> MapSet.to_list() |> Enum.sort() +``` + + + +``` +[{3, 6}, {5, 10}, {7, 14}] +``` + +### Exception and timeout isolation + +Callbacks that raise or exceed the timeout are silently filtered — the evaluator +never crashes. + +```elixir +defmodule UnsafePredicates do + def boom(_x), do: raise("exploded") + def slow(_x), do: Process.sleep(500) && true +end + +defmodule BoomTest do + use ExDatalog.Schema + + relation :num do + field(:x, :integer) + end + + relation :ok do + field(:x, :integer) + end + + predicate(:boom, UnsafePredicates, :boom, [:integer], :boolean) + + fact(num(1)) + + rule ok(X) do + num(X) + boom(X) + end +end + +# Raising callback — all bindings filtered +{:ok, knowledge} = BoomTest.materialize() +Knowledge.size(knowledge, "ok") +``` + + + +``` +0 +``` + +```elixir +defmodule SlowTest do + use ExDatalog.Schema + + relation :num do + field(:x, :integer) + end + + relation :ok do + field(:x, :integer) + end + + predicate(:slow, UnsafePredicates, :slow, [:integer], :boolean) + + fact(num(1)) + + rule ok(X) do + num(X) + slow(X) + end +end + +# Slow callback — filtered by 50 ms timeout +{:ok, knowledge} = SlowTest.materialize(callback_timeout_ms: 50) +Knowledge.size(knowledge, "ok") +``` + + + +``` +0 +``` + +--- + +## 3. Magic Sets — Goal-Directed Evaluation + +Magic sets is a **program transformation** that rewrites rules so bottom-up +evaluation computes only the facts relevant to a specific query goal, instead of +the full least fixpoint. + +> **Scope (v0.5.0, experimental):** +> +> * Positive recursive programs only (no negation, no aggregates in recursive rules). +> * A single goal with at least one bound position. +> * Programs outside scope fall back to full semi-naive evaluation — never incorrect results. + +### Full fixpoint vs goal-restricted + +```elixir +defmodule AncestorChain do + use ExDatalog.Schema + + relation :parent do + field(:from, :atom) + field(:to, :atom) + end + + relation :ancestor do + field(:from, :atom) + field(:to, :atom) + end + + fact(parent(:a, :b)) + fact(parent(:b, :c)) + fact(parent(:c, :d)) + fact(parent(:d, :e)) + + rule ancestor(X, Y) do + parent(X, Y) + end + + rule ancestor(X, Z) do + parent(X, Y) + ancestor(Y, Z) + end +end + +# Full semi-naive: all ancestor pairs +{:ok, full} = AncestorChain.materialize() +full_ancestors = Knowledge.get(full, "ancestor") |> MapSet.to_list() |> Enum.sort() + +IO.puts("Full semi-naive (#{length(full_ancestors)} facts):") +IO.inspect(full_ancestors) +``` + + + +``` +Full semi-naive (10 facts): +[a: :b, a: :c, a: :d, a: :e, b: :c, b: :d, b: :e, c: :d, c: :e, d: :e] +``` + + + +``` +[a: :b, a: :c, a: :d, a: :e, b: :c, b: :d, b: :e, c: :d, c: :e, d: :e] +``` + +Now with magic sets — goal: "who are the ancestors of :a?" + +```elixir +{:ok, magic} = + AncestorChain.materialize(strategy: :magic_sets, goal: {"ancestor", [:a, :_]}) + +magic_ancestors = + Knowledge.match(magic, "ancestor", [:a, :_]) |> MapSet.to_list() |> Enum.sort() + +IO.puts("Magic sets (#{length(magic_ancestors)} facts for goal {ancestor, [:a, :_]}):") +IO.inspect(magic_ancestors) + +# Verify: magic result is exactly the subset of full results where first arg is :a +expected = full_ancestors |> Enum.filter(fn {x, _y} -> x == :a end) +magic_ancestors == expected +``` + + + +``` +Magic sets (4 facts for goal {ancestor, [:a, :_]}): +[a: :b, a: :c, a: :d, a: :e] +``` + + + +``` +true +``` + +### Inspecting the transformation directly + +Call `MagicSets.transform/2` directly on compiled IR: + +```elixir +{:ok, ir} = ExDatalog.compile(AncestorChain.program()) +{:ok, transformed} = MagicSets.transform(ir, {"ancestor", [:a, :_]}) + +# The magic relation +magic = Enum.find(transformed.relations, fn r -> r.name == "magic_ancestor_bf" end) +IO.puts("Magic relation: #{magic.name}, arity: #{magic.arity}") + +# The seed fact +seed = Enum.find(transformed.facts, fn f -> f.relation == "magic_ancestor_bf" end) +IO.puts("Seed fact: #{inspect(seed.values)}") + +# Rewritten rules: ancestor rules now start with the magic predicate +ancestor_rules = Enum.filter(transformed.rules, fn r -> r.head.relation == "ancestor" end) +Enum.each(ancestor_rules, fn r -> + first_body = hd(r.body) + IO.puts("Rule head: ancestor, first body atom: #{inspect(elem(first_body, 1).relation)}") +end) +``` + + + +``` +Magic relation: magic_ancestor_bf, arity: 1 +Seed fact: [atom: :a] +Rule head: ancestor, first body atom: "magic_ancestor_bf" +Rule head: ancestor, first body atom: "magic_ancestor_bf" +``` + + + +``` +:ok +``` + +The adornment `bf` means the first position is **b**ound (`:a`) and the second +is **f**ree (`:_`). The magic relation `magic_ancestor_bf` has arity 1 (just the +bound position). + +### Fallback cases + +When the program is outside the supported scope, the engine falls back to +semi-naive without error: + +```elixir +# All-free goal falls back +{:ok, ms_free} = + AncestorChain.materialize(strategy: :magic_sets, goal: {"ancestor", [:_, :_]}) + +Knowledge.get(ms_free, "ancestor") == Knowledge.get(full, "ancestor") +``` + + + +``` +true +``` + +--- + +## 4. Query Planner + +The planner sits between the compiled IR and the evaluation engine. It produces +an `ExDatalog.Planner.Plan` describing the chosen `strategy`, `strata`, `joins`, +and `predicates`. + +### Plan + +```elixir +defmodule TransitivePath do + use ExDatalog.Schema + + relation :edge do + field(:from, :atom) + field(:to, :atom) + end + + relation :path do + field(:from, :atom) + field(:to, :atom) + end + + rule path(X, Y) do + edge(X, Y) + end + + rule path(X, Z) do + edge(X, Y) + path(Y, Z) + end +end + +IO.puts(Planner.explain_plan(TransitivePath.program())) +``` + + + +``` +Strategy: semi_naive + Stratum 0: 2 rule(s), relations: path, edge +Joins: 3 +Predicates: 0 +``` + + + +``` +:ok +``` + +### Plan with magic-sets strategy + +```elixir +IO.puts(Planner.explain_plan(TransitivePath.program(), strategy: :magic_sets, goal: {"path", [:a, :_]})) +``` + + + +``` +Strategy: magic_sets + Stratum 0: 2 rule(s), relations: path, edge +Joins: 3 +Predicates: 0 +``` + + + +``` +:ok +``` + +### Plan with aggregates + +The planner classifies predicates by kind (`:comparison`, `:arithmetic`, +`:aggregate`, `:callback`, etc.) and shows them in the explain output: + +```elixir +IO.puts(Planner.explain_plan(DeptCount.program())) +``` + + + +``` +Strategy: semi_naive + Stratum 0: 0 rule(s), relations: emp + Stratum 1: 1 rule(s), relations: dept_count +Joins: 1 +Predicates: 1 (count) +``` + + + +``` +:ok +``` + +### Plan with callbacks + +```elixir +IO.puts(Planner.explain_plan(AdultOnly.program())) +``` + + + +``` +Strategy: semi_naive + Stratum 0: 1 rule(s), relations: adult, person +Joins: 1 +Predicates: 0 +``` + + + +``` +:ok +``` + +--- + +## Summary + +| Feature | DSL Syntax | +| ------------ | ---------------------------------------------------------------------------- | +| Count | `count(E, N)` in rule body, `N` in head | +| Sum | `sum(A, T)` in rule body, `T` in head | +| Min | `min(S, V)` in rule body, `V` in head | +| Max | `max(S, V)` in rule body, `V` in head | +| Filter agg | `gte(S, 60)` before `count(E, N)` in body | +| Boolean CB | `predicate :name, Mod, :fun, [:type], :boolean` then `name(Arg)` | +| Value CB | `predicate :name, Mod, :fun, [:type], :value` then `name(Arg, Result)` | +| CB timeout | `materialize(callback_timeout_ms: 50)` | +| Magic sets | `materialize(strategy: :magic_sets, goal: {"rel", [:a, :_]})` | +| Plan | `Planner.plan(program)` or `Planner.plan(program, strategy: ..., goal: ...)` | +| Explain plan | `Planner.explain_plan(program)` or `Planner.explain_plan(program, opts)` | diff --git a/mix.exs b/mix.exs index 3d9ae92..e5f1270 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule ExDatalog.MixProject do use Mix.Project - @version "0.4.1" + @version "0.5.0" @source_url "https://github.com/thanos/ex_datalog" def project do @@ -63,6 +63,7 @@ defmodule ExDatalog.MixProject do {:excoveralls, "~> 0.18", only: :test}, {:mix_audit, "~> 2.1", only: [:dev, :test], runtime: false}, {:stream_data, "~> 1.1", only: [:dev, :test]}, + {:benchee, "~> 1.3", only: :dev, runtime: false}, {:ex_slop, "~> 0.1", only: [:dev, :test], runtime: false}, {:ex_dna, "~> 1.5", only: [:dev, :test], runtime: false} ] @@ -128,7 +129,18 @@ defmodule ExDatalog.MixProject do {"docs/articles/04_querying_materialized_knowledge.md", filename: "querying-materialized-knowledge", title: "Querying Materialized Knowledge"}, {"docs/articles/05_negation_constraints_and_safety.md", - filename: "negation-constraints-and-safety", title: "Negation, Constraints, and Safety"} + filename: "negation-constraints-and-safety", title: "Negation, Constraints, and Safety"}, + {"docs/articles/06_query_planning_in_datalog.md", + filename: "query-planning-in-datalog", title: "Query Planning in Datalog"}, + {"docs/articles/07_aggregates_in_datalog.md", + filename: "aggregates-in-datalog", title: "Aggregates in Datalog"}, + {"docs/articles/08_extending_datalog_with_beam_callbacks.md", + filename: "extending-datalog-with-beam-callbacks", + title: "Extending Datalog with BEAM Callbacks"}, + {"docs/articles/09_magic_sets_and_demand_driven_evaluation.md", + filename: "magic-sets-and-demand-driven-evaluation", + title: "Magic Sets and Demand-Driven Evaluation"}, + {"docs/migration_v0.5.md", filename: "migration-v0-5", title: "Migration: v0.4 → v0.5"} ], groups_for_modules: [ "Program Builder": [ @@ -145,7 +157,9 @@ defmodule ExDatalog.MixProject do ExDatalog.Validator.Stratification ], "Compiler & IR": ~r/ExDatalog\.(Compiler|IR).*/, + Planner: ~r/ExDatalog\.Planner.*/, Engine: ~r/ExDatalog\.Engine.*/, + "Magic Sets": [ExDatalog.MagicSets], Storage: ~r/ExDatalog\.Storage.*/, Knowledge: [ ExDatalog.Knowledge, @@ -154,6 +168,7 @@ defmodule ExDatalog.MixProject do ], DSL: [ ExDatalog.Schema, + ExDatalog.Callback, ExDatalog.DSL.CompileError, ExDatalog.UnsupportedFeature ] diff --git a/mix.lock b/mix.lock index a85066e..7d752e9 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,8 @@ %{ + "benchee": {:hex, :benchee, "1.5.1", "b95cbc36c4b98969a5c592a246e171041eb683c56bad1cb4f49a3b081ba66087", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.1", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a539301f8dfd4efc5c5123bfb9d47ebde20092a863a5b5b16c2a60d2243dfce7"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "credo": {:hex, :credo, "1.7.18", "5c5596bf7aedf9c8c227f13272ac499fe8eae6237bd326f2f07dfc173786f042", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a189d164685fd945809e862fe76a7420c4398fa288d76257662aecb909d6b3e5"}, + "deep_merge": {:hex, :deep_merge, "1.0.2", "476aa7ea61c54de96220051b998d893869069094da65b96101aebf79416f8a1e", [:mix], [], "hexpm", "737a53cdc9758fedbb608bdc213969e65729466c4ef3cd8e8726d0335dff116c"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, @@ -15,6 +17,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"}, "mix_audit": {:hex, :mix_audit, "2.1.5", "c0f77cee6b4ef9d97e37772359a187a166c7a1e0e08b50edf5bf6959dfe5a016", [:make, :mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.11", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "87f9298e21da32f697af535475860dc1d3617a010e0b418d2ec6142bc8b42d69"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "statistex": {:hex, :statistex, "1.1.1", "73612aa7f79e53c30569be065fd121e380f1cf57bc4c2da5b41be9246da18df9", [:mix], [], "hexpm", "310c4b49b34adf683de3103639006bed233ab54c08a4add65a531448e653857c"}, "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, "telemetry": {:hex, :telemetry, "1.4.1", "ab6de178e2b29b58e8256b92b382ea3f590a47152ca3651ea857a6cae05ac423", [:rebar3], [], "hexpm", "2172e05a27531d3d31dd9782841065c50dd5c3c7699d95266b2edd54c2dafa1c"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, diff --git a/test/ex_datalog/aggregate_test.exs b/test/ex_datalog/aggregate_test.exs new file mode 100644 index 0000000..3e4d099 --- /dev/null +++ b/test/ex_datalog/aggregate_test.exs @@ -0,0 +1,230 @@ +defmodule ExDatalog.AggregateTest do + use ExUnit.Case, async: true + + alias ExDatalog.{Atom, Constraint, Knowledge, Program, Rule, Term} + + doctest ExDatalog.Constraints.Aggregate + + # --- Builder API evaluation --- + + defp count_program do + Program.new() + |> Program.add_relation("emp", [:atom, :atom]) + |> Program.add_relation("dept_count", [:atom, :integer]) + |> Program.add_fact("emp", [:alice, :eng]) + |> Program.add_fact("emp", [:bob, :eng]) + |> Program.add_fact("emp", [:carol, :ops]) + |> Program.add_rule( + Rule.new( + Atom.new("dept_count", [Term.var("D"), Term.var("N")]), + [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D")])}], + [Constraint.count(Term.var("E"), Term.var("N"))] + ) + ) + end + + describe "count aggregate" do + test "counts members per group" do + {:ok, knowledge} = ExDatalog.materialize(count_program()) + result = Knowledge.get(knowledge, "dept_count") + assert MapSet.size(result) == 2 + assert {:eng, 2} in result + assert {:ops, 1} in result + end + + test "aggregate stratum terminates at fixpoint" do + {:ok, knowledge} = ExDatalog.materialize(count_program()) + assert knowledge.stats.termination == :fixpoint + end + end + + describe "sum aggregate" do + test "sums integer values per group" do + program = + Program.new() + |> Program.add_relation("salary", [:atom, :atom, :integer]) + |> Program.add_relation("dept_total", [:atom, :integer]) + |> Program.add_fact("salary", [:alice, :eng, 100]) + |> Program.add_fact("salary", [:bob, :eng, 80]) + |> Program.add_fact("salary", [:carol, :ops, 50]) + |> Program.add_rule( + Rule.new( + Atom.new("dept_total", [Term.var("D"), Term.var("T")]), + [{:positive, Atom.new("salary", [Term.var("E"), Term.var("D"), Term.var("A")])}], + [Constraint.sum(Term.var("A"), Term.var("T"))] + ) + ) + + {:ok, knowledge} = ExDatalog.materialize(program) + result = Knowledge.get(knowledge, "dept_total") + assert {:eng, 180} in result + assert {:ops, 50} in result + end + end + + describe "min and max aggregates" do + defp score_program(op, head) do + Program.new() + |> Program.add_relation("score", [:atom, :atom, :integer]) + |> Program.add_relation(head, [:atom, :integer]) + |> Program.add_fact("score", [:alice, :eng, 90]) + |> Program.add_fact("score", [:bob, :eng, 70]) + |> Program.add_fact("score", [:carol, :ops, 60]) + |> Program.add_rule( + Rule.new( + Atom.new(head, [Term.var("D"), Term.var("V")]), + [{:positive, Atom.new("score", [Term.var("E"), Term.var("D"), Term.var("S")])}], + [apply(Constraint, op, [Term.var("S"), Term.var("V")])] + ) + ) + end + + test "min picks the smallest per group" do + {:ok, knowledge} = ExDatalog.materialize(score_program(:min, "lowest")) + result = Knowledge.get(knowledge, "lowest") + assert {:eng, 70} in result + assert {:ops, 60} in result + end + + test "max picks the largest per group" do + {:ok, knowledge} = ExDatalog.materialize(score_program(:max, "highest")) + result = Knowledge.get(knowledge, "highest") + assert {:eng, 90} in result + assert {:ops, 60} in result + end + end + + describe "aggregate with filter constraint" do + test "filters bindings before grouping" do + program = + Program.new() + |> Program.add_relation("score", [:atom, :atom, :integer]) + |> Program.add_relation("passing_count", [:atom, :integer]) + |> Program.add_fact("score", [:alice, :eng, 90]) + |> Program.add_fact("score", [:bob, :eng, 40]) + |> Program.add_fact("score", [:carol, :eng, 75]) + |> Program.add_rule( + Rule.new( + Atom.new("passing_count", [Term.var("D"), Term.var("N")]), + [{:positive, Atom.new("score", [Term.var("E"), Term.var("D"), Term.var("S")])}], + [ + Constraint.gte(Term.var("S"), Term.const(60)), + Constraint.count(Term.var("E"), Term.var("N")) + ] + ) + ) + + {:ok, knowledge} = ExDatalog.materialize(program) + result = Knowledge.get(knowledge, "passing_count") + assert {:eng, 2} in result + end + end + + describe "aggregate validation" do + test "rejects more than one aggregate per rule" do + program = + Program.new() + |> Program.add_relation("emp", [:atom, :atom, :integer]) + |> Program.add_relation("stats", [:atom, :integer, :integer]) + |> Program.add_rule( + Rule.new( + Atom.new("stats", [Term.var("D"), Term.var("N"), Term.var("T")]), + [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D"), Term.var("A")])}], + [ + Constraint.count(Term.var("E"), Term.var("N")), + Constraint.sum(Term.var("A"), Term.var("T")) + ] + ) + ) + + assert {:error, errors} = ExDatalog.materialize(program) + assert Enum.any?(errors, fn e -> e.kind == :multiple_aggregates end) + end + + test "rejects aggregate over a self-recursive relation" do + program = + Program.new() + |> Program.add_relation("path", [:atom, :integer]) + |> Program.add_rule( + Rule.new( + Atom.new("path", [Term.var("X"), Term.var("N")]), + [{:positive, Atom.new("path", [Term.var("X"), Term.var("Y")])}], + [Constraint.count(Term.var("Y"), Term.var("N"))] + ) + ) + + assert {:error, errors} = ExDatalog.materialize(program) + assert Enum.any?(errors, fn e -> e.kind == :aggregate_in_recursion end) + end + + test "rejects unbound aggregate input variable" do + program = + Program.new() + |> Program.add_relation("emp", [:atom, :atom]) + |> Program.add_relation("dept_count", [:atom, :integer]) + |> Program.add_rule( + Rule.new( + Atom.new("dept_count", [Term.var("D"), Term.var("N")]), + [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D")])}], + # Z is not bound by any positive body atom + [Constraint.count(Term.var("Z"), Term.var("N"))] + ) + ) + + assert {:error, errors} = ExDatalog.materialize(program) + assert Enum.any?(errors, fn e -> e.kind == :unbound_constraint_variable end) + end + end + + describe "aggregate stratification" do + test "aggregate over a derived relation is placed in a higher stratum" do + # base -> mid (derived) -> count over mid + program = + Program.new() + |> Program.add_relation("base", [:atom, :atom]) + |> Program.add_relation("mid", [:atom, :atom]) + |> Program.add_relation("mid_count", [:atom, :integer]) + |> Program.add_fact("base", [:a, :x]) + |> Program.add_fact("base", [:a, :y]) + |> Program.add_fact("base", [:b, :z]) + |> Program.add_rule( + Rule.new( + Atom.new("mid", [Term.var("K"), Term.var("V")]), + [{:positive, Atom.new("base", [Term.var("K"), Term.var("V")])}] + ) + ) + |> Program.add_rule( + Rule.new( + Atom.new("mid_count", [Term.var("K"), Term.var("N")]), + [{:positive, Atom.new("mid", [Term.var("K"), Term.var("V")])}], + [Constraint.count(Term.var("V"), Term.var("N"))] + ) + ) + + {:ok, ir} = ExDatalog.compile(program) + mid_stratum = Enum.find(ir.rules, fn r -> r.head.relation == "mid" end).stratum + count_stratum = Enum.find(ir.rules, fn r -> r.head.relation == "mid_count" end).stratum + assert count_stratum > mid_stratum + + {:ok, knowledge} = ExDatalog.materialize(program) + result = Knowledge.get(knowledge, "mid_count") + assert {:a, 2} in result + assert {:b, 1} in result + end + end + + describe "aggregate via from_tuple" do + test "builds aggregate constraints" do + assert %Constraint{op: :count, left: {:var, "X"}, right: nil, result: {:var, "N"}} = + Constraint.from_tuple({:count, {:var, "X"}, {:var, "N"}}) + + assert %Constraint{op: :sum, result: {:var, "T"}} = + Constraint.from_tuple({:sum, :A, :T}) + end + + test "valid? accepts aggregate constraints" do + assert Constraint.valid?(Constraint.count(Term.var("X"), Term.var("N"))) + assert Constraint.valid?(Constraint.sum(Term.var("X"), Term.var("T"))) + end + end +end diff --git a/test/ex_datalog/beam_callback_test.exs b/test/ex_datalog/beam_callback_test.exs new file mode 100644 index 0000000..2c061d9 --- /dev/null +++ b/test/ex_datalog/beam_callback_test.exs @@ -0,0 +1,209 @@ +defmodule ExDatalog.BeamCallbackTest do + use ExUnit.Case, async: true + + alias ExDatalog.{Atom, Callback, Knowledge, Program, Rule, Term} + + doctest ExDatalog.Callback + + defmodule Predicates do + @moduledoc false + def adult?(age), do: age >= 18 + def valid_email?(email), do: String.contains?(email, "@") + def double(x), do: x * 2 + def boom(_x), do: raise("callback exploded") + def slow(_x), do: Process.sleep(500) && true + end + + describe "boolean callback (builder API)" do + test "filters bindings by predicate result" do + program = + Program.new() + |> Program.add_relation("person", [:atom, :integer]) + |> Program.add_relation("adult", [:atom]) + |> Program.add_fact("person", [:alice, 25]) + |> Program.add_fact("person", [:bob, 12]) + |> Program.add_rule( + Rule.new( + Atom.new("adult", [Term.var("Name")]), + [ + {:positive, Atom.new("person", [Term.var("Name"), Term.var("Age")])}, + {:callback, Callback.new(Predicates, :adult?, [Term.var("Age")])} + ] + ) + ) + + {:ok, knowledge} = ExDatalog.materialize(program) + result = Knowledge.get(knowledge, "adult") + assert MapSet.size(result) == 1 + assert {:alice} in result + end + end + + describe "value-returning callback" do + test "binds the result variable" do + program = + Program.new() + |> Program.add_relation("num", [:integer]) + |> Program.add_relation("doubled", [:integer, :integer]) + |> Program.add_fact("num", [3]) + |> Program.add_fact("num", [5]) + |> Program.add_rule( + Rule.new( + Atom.new("doubled", [Term.var("X"), Term.var("Y")]), + [ + {:positive, Atom.new("num", [Term.var("X")])}, + {:callback, Callback.new(Predicates, :double, [Term.var("X")], Term.var("Y"))} + ] + ) + ) + + {:ok, knowledge} = ExDatalog.materialize(program) + result = Knowledge.get(knowledge, "doubled") + assert {3, 6} in result + assert {5, 10} in result + end + end + + describe "exception and timeout isolation" do + test "a raising callback filters the binding" do + program = + Program.new() + |> Program.add_relation("num", [:integer]) + |> Program.add_relation("ok", [:integer]) + |> Program.add_fact("num", [1]) + |> Program.add_rule( + Rule.new( + Atom.new("ok", [Term.var("X")]), + [ + {:positive, Atom.new("num", [Term.var("X")])}, + {:callback, Callback.new(Predicates, :boom, [Term.var("X")])} + ] + ) + ) + + {:ok, knowledge} = ExDatalog.materialize(program) + assert MapSet.size(Knowledge.get(knowledge, "ok")) == 0 + end + + test "a slow callback is filtered by the timeout" do + program = + Program.new() + |> Program.add_relation("num", [:integer]) + |> Program.add_relation("ok", [:integer]) + |> Program.add_fact("num", [1]) + |> Program.add_rule( + Rule.new( + Atom.new("ok", [Term.var("X")]), + [ + {:positive, Atom.new("num", [Term.var("X")])}, + {:callback, Callback.new(Predicates, :slow, [Term.var("X")])} + ] + ) + ) + + {:ok, knowledge} = ExDatalog.materialize(program, callback_timeout_ms: 50) + assert MapSet.size(Knowledge.get(knowledge, "ok")) == 0 + end + end + + describe "validation" do + test "rejects a callback referencing a missing function" do + program = + Program.new() + |> Program.add_relation("num", [:integer]) + |> Program.add_relation("ok", [:integer]) + |> Program.add_fact("num", [1]) + |> Program.add_rule( + Rule.new( + Atom.new("ok", [Term.var("X")]), + [ + {:positive, Atom.new("num", [Term.var("X")])}, + {:callback, Callback.new(Predicates, :nonexistent, [Term.var("X")])} + ] + ) + ) + + assert {:error, errors} = ExDatalog.materialize(program) + assert Enum.any?(errors, fn e -> e.kind == :invalid_callback end) + end + + test "rejects a callback with an unbound input variable" do + program = + Program.new() + |> Program.add_relation("num", [:integer]) + |> Program.add_relation("ok", [:integer]) + |> Program.add_fact("num", [1]) + |> Program.add_rule( + Rule.new( + Atom.new("ok", [Term.var("X")]), + [ + {:positive, Atom.new("num", [Term.var("X")])}, + {:callback, Callback.new(Predicates, :adult?, [Term.var("Z")])} + ] + ) + ) + + assert {:error, errors} = ExDatalog.materialize(program) + assert Enum.any?(errors, fn e -> e.kind == :unbound_constraint_variable end) + end + end + + describe "DSL predicate macro" do + test "boolean predicate in a rule body" do + defmodule AdultSchema do + use ExDatalog.Schema + + relation :person do + field(:name, :atom) + field(:age, :integer) + end + + relation :adult do + field(:name, :atom) + end + + predicate(:adult?, ExDatalog.BeamCallbackTest.Predicates, :adult?, [:integer], :boolean) + + fact(person(:alice, 30)) + fact(person(:bob, 10)) + + rule adult(Name) do + person(Name, Age) + adult?(Age) + end + end + + {:ok, knowledge} = AdultSchema.materialize() + result = Knowledge.get(knowledge, "adult") + assert {:alice} in result + refute {:bob} in result + end + + test "value predicate binds the last argument" do + defmodule DoubleSchema do + use ExDatalog.Schema + + relation :num do + field(:x, :integer) + end + + relation :doubled do + field(:x, :integer) + field(:y, :integer) + end + + predicate(:double, ExDatalog.BeamCallbackTest.Predicates, :double, [:integer], :value) + + fact(num(4)) + + rule doubled(X, Y) do + num(X) + double(X, Y) + end + end + + {:ok, knowledge} = DoubleSchema.materialize() + assert {4, 8} in Knowledge.get(knowledge, "doubled") + end + end +end diff --git a/test/ex_datalog/magic_sets_property_test.exs b/test/ex_datalog/magic_sets_property_test.exs new file mode 100644 index 0000000..fb84c1c --- /dev/null +++ b/test/ex_datalog/magic_sets_property_test.exs @@ -0,0 +1,59 @@ +defmodule ExDatalog.MagicSetsPropertyTest do + use ExUnit.Case, async: true + use ExUnitProperties + + alias ExDatalog.{Atom, Knowledge, Program, Rule, Term} + + @nodes [:a, :b, :c, :d, :e, :f] + + defp edge_gen do + gen all(from <- member_of(@nodes), to <- member_of(@nodes)) do + {from, to} + end + end + + defp build_program(edges) do + base = + Program.new() + |> Program.add_relation("edge", [:atom, :atom]) + |> Program.add_relation("path", [:atom, :atom]) + + base = Enum.reduce(edges, base, fn {f, t}, acc -> Program.add_fact(acc, "edge", [f, t]) end) + + base + |> Program.add_rule( + Rule.new( + Atom.new("path", [Term.var("X"), Term.var("Y")]), + [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] + ) + ) + |> Program.add_rule( + Rule.new( + Atom.new("path", [Term.var("X"), Term.var("Z")]), + [ + {:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}, + {:positive, Atom.new("path", [Term.var("Y"), Term.var("Z")])} + ] + ) + ) + end + + property "magic-sets result equals the semi-naive subset for the goal" do + check all( + edges <- list_of(edge_gen(), max_length: 10), + source <- member_of(@nodes) + ) do + program = build_program(edges) + + {:ok, full} = ExDatalog.materialize(program) + expected = MapSet.filter(Knowledge.get(full, "path"), fn {x, _y} -> x == source end) + + {:ok, magic} = + ExDatalog.materialize(program, strategy: :magic_sets, goal: {"path", [source, :_]}) + + actual = Knowledge.match(magic, "path", [source, :_]) + + assert actual == expected + end + end +end diff --git a/test/ex_datalog/magic_sets_test.exs b/test/ex_datalog/magic_sets_test.exs new file mode 100644 index 0000000..cf7c9e1 --- /dev/null +++ b/test/ex_datalog/magic_sets_test.exs @@ -0,0 +1,143 @@ +defmodule ExDatalog.MagicSetsTest do + use ExUnit.Case, async: true + + alias ExDatalog.{Atom, Knowledge, MagicSets, Program, Rule, Term} + + defp ancestor_program(facts) do + base = + Program.new() + |> Program.add_relation("parent", [:atom, :atom]) + |> Program.add_relation("ancestor", [:atom, :atom]) + + base = Enum.reduce(facts, base, fn {p, c}, acc -> Program.add_fact(acc, "parent", [p, c]) end) + + base + |> Program.add_rule( + Rule.new( + Atom.new("ancestor", [Term.var("X"), Term.var("Y")]), + [{:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}] + ) + ) + |> Program.add_rule( + Rule.new( + Atom.new("ancestor", [Term.var("X"), Term.var("Z")]), + [ + {:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}, + {:positive, Atom.new("ancestor", [Term.var("Y"), Term.var("Z")])} + ] + ) + ) + end + + @chain [{:a, :b}, {:b, :c}, {:c, :d}, {:d, :e}] + + describe "correctness vs semi-naive" do + test "goal-restricted query matches the semi-naive subset" do + program = ancestor_program(@chain) + + {:ok, full} = ExDatalog.materialize(program) + full_ancestors = Knowledge.get(full, "ancestor") + expected = MapSet.filter(full_ancestors, fn {x, _y} -> x == :a end) + + {:ok, magic} = + ExDatalog.materialize(program, strategy: :magic_sets, goal: {"ancestor", [:a, :_]}) + + result = Knowledge.match(magic, "ancestor", [:a, :_]) + assert result == expected + end + + test "falls back to semi-naive when no goal is given" do + program = ancestor_program(@chain) + {:ok, magic} = ExDatalog.materialize(program, strategy: :magic_sets) + {:ok, full} = ExDatalog.materialize(program) + assert Knowledge.get(magic, "ancestor") == Knowledge.get(full, "ancestor") + end + + test "falls back to semi-naive for an all-free goal" do + program = ancestor_program(@chain) + + {:ok, magic} = + ExDatalog.materialize(program, strategy: :magic_sets, goal: {"ancestor", [:_, :_]}) + + {:ok, full} = ExDatalog.materialize(program) + assert Knowledge.get(magic, "ancestor") == Knowledge.get(full, "ancestor") + end + end + + describe "transform/2 directly" do + test "produces a magic relation, seed fact, and rewritten rules for a bound goal" do + {:ok, ir} = ExDatalog.compile(ancestor_program(@chain)) + {:ok, transformed} = MagicSets.transform(ir, {"ancestor", [:a, :_]}) + + magic = Enum.find(transformed.relations, fn r -> r.name == "magic_ancestor_bf" end) + assert magic.arity == 1 + + assert Enum.any?(transformed.facts, fn f -> + f.relation == "magic_ancestor_bf" and f.values == [atom: :a] + end) + + rewritten = + Enum.filter(transformed.rules, fn r -> r.head.relation == "ancestor" end) + + assert Enum.all?(rewritten, fn r -> + match?( + [{:positive, %ExDatalog.IR.Atom{relation: "magic_ancestor_bf"}} | _], + r.body + ) + end) + end + + test "returns :fallback for an all-free goal" do + {:ok, ir} = ExDatalog.compile(ancestor_program(@chain)) + assert {:fallback, :no_bound_positions} = MagicSets.transform(ir, {"ancestor", [:_, :_]}) + end + + test "returns :fallback for a program containing aggregates" do + program = + Program.new() + |> Program.add_relation("emp", [:atom, :atom]) + |> Program.add_relation("dept_count", [:atom, :integer]) + |> Program.add_fact("emp", [:alice, :eng]) + |> Program.add_rule( + Rule.new( + Atom.new("dept_count", [Term.var("D"), Term.var("N")]), + [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D")])}], + [ExDatalog.Constraint.count(Term.var("E"), Term.var("N"))] + ) + ) + + {:ok, ir} = ExDatalog.compile(program) + + assert {:fallback, :unsupported_program} = + MagicSets.transform(ir, {"dept_count", [:eng, :_]}) + end + end + + describe "unsupported programs fall back" do + test "program with negation falls back to semi-naive" do + program = + Program.new() + |> Program.add_relation("person", [:atom]) + |> Program.add_relation("parent", [:atom, :atom]) + |> Program.add_relation("childless", [:atom]) + |> Program.add_fact("person", [:alice]) + |> Program.add_fact("person", [:bob]) + |> Program.add_fact("parent", [:alice, :carol]) + |> Program.add_rule( + Rule.new( + Atom.new("childless", [Term.var("X")]), + [ + {:positive, Atom.new("person", [Term.var("X")])}, + {:negative, Atom.new("parent", [Term.var("X"), :wildcard])} + ] + ) + ) + + {:ok, magic} = + ExDatalog.materialize(program, strategy: :magic_sets, goal: {"childless", [:_]}) + + {:ok, full} = ExDatalog.materialize(program) + assert Knowledge.get(magic, "childless") == Knowledge.get(full, "childless") + end + end +end diff --git a/test/ex_datalog/planner_explain_test.exs b/test/ex_datalog/planner_explain_test.exs new file mode 100644 index 0000000..c5d65d0 --- /dev/null +++ b/test/ex_datalog/planner_explain_test.exs @@ -0,0 +1,60 @@ +defmodule ExDatalog.PlannerExplainTest do + use ExUnit.Case, async: true + + alias ExDatalog.{Atom, Planner, Program, Rule, Term} + + defp transitive_program do + Program.new() + |> Program.add_relation("edge", [:atom, :atom]) + |> Program.add_relation("path", [:atom, :atom]) + |> Program.add_fact("edge", [:a, :b]) + |> Program.add_rule( + Rule.new( + Atom.new("path", [Term.var("X"), Term.var("Y")]), + [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] + ) + ) + end + + describe "explain_plan/1,2" do + test "describes the strategy" do + assert Planner.explain_plan(transitive_program()) =~ "Strategy: semi_naive" + end + + test "lists strata" do + output = Planner.explain_plan(transitive_program()) + assert output =~ "Stratum 0" + end + + test "reports join and predicate counts" do + output = Planner.explain_plan(transitive_program()) + assert output =~ "Joins: 1" + assert output =~ "Predicates: 0" + end + + test "honours the magic_sets strategy option" do + output = + Planner.explain_plan(transitive_program(), + strategy: :magic_sets, + goal: {"path", [:a, :_]} + ) + + assert output =~ "Strategy: magic_sets" + end + + test "returns an error string for an invalid program" do + bad = + Program.new() + |> Program.add_relation("path", [:atom, :atom]) + |> Program.add_rule( + Rule.new( + # Z is unsafe — not bound by any positive body atom + Atom.new("path", [Term.var("X"), Term.var("Z")]), + [{:positive, Atom.new("path", [Term.var("X"), Term.var("Y")])}] + ) + ) + + assert Planner.explain_plan(bad) =~ "compilation failed" + end + end +end diff --git a/test/ex_datalog/planner_test.exs b/test/ex_datalog/planner_test.exs new file mode 100644 index 0000000..f6f5a5c --- /dev/null +++ b/test/ex_datalog/planner_test.exs @@ -0,0 +1,98 @@ +defmodule ExDatalog.PlannerTest do + use ExUnit.Case, async: true + + doctest ExDatalog.Planner + + alias ExDatalog.{Atom, Compiler, Constraint, Planner, Program, Rule, Term} + alias ExDatalog.Planner.{Join, Plan, Predicate, Stratum} + + defp transitive_ir do + {:ok, ir} = + Program.new() + |> Program.add_relation("edge", [:atom, :atom]) + |> Program.add_relation("path", [:atom, :atom]) + |> Program.add_fact("edge", [:a, :b]) + |> Program.add_rule( + Rule.new( + Atom.new("path", [Term.var("X"), Term.var("Y")]), + [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] + ) + ) + |> Program.add_rule( + Rule.new( + Atom.new("path", [Term.var("X"), Term.var("Z")]), + [ + {:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}, + {:positive, Atom.new("path", [Term.var("Y"), Term.var("Z")])} + ] + ) + ) + |> Compiler.compile() + + ir + end + + describe "plan/2" do + test "produces a semi_naive plan by default" do + {:ok, plan} = Planner.plan(transitive_ir()) + assert %Plan{strategy: :semi_naive} = plan + end + + test "wraps IR strata into Stratum structs with rules" do + {:ok, plan} = Planner.plan(transitive_ir()) + assert Enum.all?(plan.strata, &match?(%Stratum{}, &1)) + assert Enum.any?(plan.strata, fn s -> s.rules != [] end) + end + + test "produces one join per positive body atom" do + {:ok, plan} = Planner.plan(transitive_ir()) + # rule 1: 1 positive atom; rule 2: 2 positive atoms => 3 joins + assert length(plan.joins) == 3 + assert Enum.all?(plan.joins, &match?(%Join{}, &1)) + end + + test "accepts the magic_sets strategy" do + {:ok, plan} = Planner.plan(transitive_ir(), strategy: :magic_sets, goal: {"path", [:a, :_]}) + assert plan.strategy == :magic_sets + assert plan.metadata.goal == {"path", [:a, :_]} + end + + test "classifies comparison constraints as predicates" do + {:ok, ir} = + Program.new() + |> Program.add_relation("income", [:atom, :integer]) + |> Program.add_relation("rich", [:atom]) + |> Program.add_fact("income", [:alice, 200_000]) + |> Program.add_rule( + Rule.new( + Atom.new("rich", [Term.var("P")]), + [{:positive, Atom.new("income", [Term.var("P"), Term.var("S")])}], + [Constraint.gt(Term.var("S"), Term.const(100_000))] + ) + ) + |> Compiler.compile() + + {:ok, plan} = Planner.plan(ir) + assert [%Predicate{kind: :comparison, op: :gt}] = plan.predicates + end + + test "classifies aggregate constraints as aggregate predicates" do + {:ok, ir} = + Program.new() + |> Program.add_relation("emp", [:atom, :atom]) + |> Program.add_relation("dept_count", [:atom, :integer]) + |> Program.add_fact("emp", [:alice, :eng]) + |> Program.add_rule( + Rule.new( + Atom.new("dept_count", [Term.var("D"), Term.var("N")]), + [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D")])}], + [Constraint.count(Term.var("E"), Term.var("N"))] + ) + ) + |> Compiler.compile() + + {:ok, plan} = Planner.plan(ir) + assert [%Predicate{kind: :aggregate, op: :count}] = plan.predicates + end + end +end diff --git a/test/ex_datalog/runtime_facts_test.exs b/test/ex_datalog/runtime_facts_test.exs new file mode 100644 index 0000000..1cab0a0 --- /dev/null +++ b/test/ex_datalog/runtime_facts_test.exs @@ -0,0 +1,306 @@ +defmodule ExDatalog.RuntimeFactsTest do + use ExUnit.Case, async: true + + alias ExDatalog.{Knowledge, Program} + + describe "Schema relation constructors" do + test "constructor returns {relation, values} tuple" do + defmodule EmpSchema do + use ExDatalog.Schema + + relation :emp do + field(:name, :atom) + field(:dept, :atom) + end + + relation :dept_count do + field(:dept, :atom) + field(:n, :integer) + end + + rule dept_count(D, N) do + emp(E, D) + count(E, N) + end + end + + assert EmpSchema.emp(:alice, :eng) == {"emp", [:alice, :eng]} + assert EmpSchema.dept_count(:eng, 3) == {"dept_count", [:eng, 3]} + end + end + + describe "Schema.new/0" do + test "returns a blank program with no facts" do + defmodule BlankSchema do + use ExDatalog.Schema + + relation :edge do + field(:from, :atom) + field(:to, :atom) + end + + relation :path do + field(:from, :atom) + field(:to, :atom) + end + + rule path(X, Y) do + edge(X, Y) + end + end + + prog = BlankSchema.new() + assert prog.facts == [] + assert Map.has_key?(prog.relations, "edge") + assert Map.has_key?(prog.relations, "path") + end + + test "new/0 excludes compile-time facts; program/0 includes them" do + defmodule AxiomSchema do + use ExDatalog.Schema + + relation :edge do + field(:from, :atom) + field(:to, :atom) + end + + fact(edge(:a, :b)) + + rule edge(X, Y) do + edge(X, Y) + end + end + + blank = AxiomSchema.new() + assert blank.facts == [] + + with_axioms = AxiomSchema.program() + assert {"edge", [:a, :b]} in with_axioms.facts + end + end + + describe "Program.add_fact/2 (tuple form)" do + test "adds a fact from a {relation, values} tuple" do + prog = + Program.new() + |> Program.add_relation("emp", [:atom, :atom]) + |> Program.add_fact({"emp", [:alice, :eng]}) + + assert {"emp", [:alice, :eng]} in prog.facts + end + + test "pipeable with schema constructors" do + defmodule PipeSchema do + use ExDatalog.Schema + + relation :edge do + field(:from, :atom) + field(:to, :atom) + end + + relation :path do + field(:from, :atom) + field(:to, :atom) + end + + rule path(X, Y) do + edge(X, Y) + end + + rule path(X, Z) do + edge(X, Y) + path(Y, Z) + end + end + + {:ok, k} = + PipeSchema.new() + |> Program.add_fact(PipeSchema.edge(:a, :b)) + |> Program.add_fact(PipeSchema.edge(:b, :c)) + |> Program.materialize() + + result = Knowledge.get(k, "path") |> MapSet.to_list() |> Enum.sort() + assert {:a, :b} in result + assert {:b, :c} in result + assert {:a, :c} in result + end + + test "returns error for unknown relation" do + prog = Program.new() |> Program.add_relation("emp", [:atom, :atom]) + assert {:error, _} = Program.add_fact(prog, {"unknown", [:x]}) + end + + test "returns error for arity mismatch" do + prog = Program.new() |> Program.add_relation("emp", [:atom, :atom]) + assert {:error, _} = Program.add_fact(prog, {"emp", [:only_one]}) + end + end + + describe "Program.add_facts/2" do + test "adds multiple facts at once" do + prog = + Program.new() + |> Program.add_relation("emp", [:atom, :atom]) + |> Program.add_facts([ + {"emp", [:alice, :eng]}, + {"emp", [:bob, :eng]}, + {"emp", [:carol, :ops]} + ]) + + assert length(prog.facts) == 3 + end + + test "stops on first error" do + prog = Program.new() |> Program.add_relation("emp", [:atom, :atom]) + + assert {:error, _} = + Program.add_facts(prog, [ + {"emp", [:alice, :eng]}, + {"unknown", [:x]}, + {"emp", [:bob, :eng]} + ]) + end + + test "pipeable with schema constructors" do + defmodule BulkSchema do + use ExDatalog.Schema + + relation :emp do + field(:name, :atom) + field(:dept, :atom) + end + + relation :dept_count do + field(:dept, :atom) + field(:n, :integer) + end + + rule dept_count(D, N) do + emp(E, D) + count(E, N) + end + end + + {:ok, k} = + BulkSchema.new() + |> Program.add_facts([ + BulkSchema.emp(:alice, :eng), + BulkSchema.emp(:bob, :eng), + BulkSchema.emp(:carol, :ops), + BulkSchema.emp(:dave, :eng), + BulkSchema.emp(:eve, :ops) + ]) + |> Program.materialize() + + result = Knowledge.get(k, "dept_count") |> MapSet.to_list() |> MapSet.new() + assert {:eng, 3} in result + assert {:ops, 2} in result + end + end + + describe "Program.materialize/1,2" do + test "pipe-friendly materialization" do + prog = + Program.new() + |> Program.add_relation("edge", [:atom, :atom]) + |> Program.add_fact("edge", [:a, :b]) + + {:ok, k} = Program.materialize(prog) + assert Knowledge.get(k, "edge") |> MapSet.size() == 1 + end + + test "passes options through" do + defmodule MatOptsSchema do + use ExDatalog.Schema + + relation :edge do + field(:from, :atom) + field(:to, :atom) + end + + fact(edge(:a, :b)) + end + + {:ok, k} = Program.materialize(MatOptsSchema.program(), storage: ExDatalog.Storage.Map) + assert Knowledge.get(k, "edge") |> MapSet.size() == 1 + end + end + + describe "end-to-end: Schema.new + runtime facts + aggregates + materialize" do + test "full runtime pipeline" do + defmodule RuntimeDeptCount do + use ExDatalog.Schema + + relation :emp do + field(:name, :atom) + field(:dept, :atom) + end + + relation :dept_count do + field(:dept, :atom) + field(:n, :integer) + end + + rule dept_count(D, N) do + emp(E, D) + count(E, N) + end + end + + {:ok, k} = + RuntimeDeptCount.new() + |> Program.add_facts([ + RuntimeDeptCount.emp(:alice, :eng), + RuntimeDeptCount.emp(:bob, :eng), + RuntimeDeptCount.emp(:carol, :ops) + ]) + |> Program.materialize() + + result = Knowledge.get(k, "dept_count") |> MapSet.to_list() |> Enum.sort() + assert {:eng, 2} in result + assert {:ops, 1} in result + end + + test "mixing compile-time facts with runtime facts" do + defmodule MixedSchema do + use ExDatalog.Schema + + relation :edge do + field(:from, :atom) + field(:to, :atom) + end + + relation :path do + field(:from, :atom) + field(:to, :atom) + end + + fact(edge(:a, :b)) + + rule path(X, Y) do + edge(X, Y) + end + + rule path(X, Z) do + edge(X, Y) + path(Y, Z) + end + end + + # program() includes compile-time fact :a→:b + prog = MixedSchema.program() + # Add runtime facts + prog = Program.add_fact(prog, MixedSchema.edge(:b, :c)) + prog = Program.add_fact(prog, MixedSchema.edge(:c, :d)) + + {:ok, k} = Program.materialize(prog) + result = Knowledge.get(k, "path") |> MapSet.to_list() |> Enum.sort() + assert {:a, :b} in result + assert {:b, :c} in result + assert {:c, :d} in result + assert {:a, :c} in result + assert {:a, :d} in result + assert {:b, :d} in result + end + end +end diff --git a/test/ex_datalog/schema_test.exs b/test/ex_datalog/schema_test.exs index bb38e5f..2aa8086 100644 --- a/test/ex_datalog/schema_test.exs +++ b/test/ex_datalog/schema_test.exs @@ -617,7 +617,7 @@ defmodule ExDatalog.SchemaTest do describe "UnsupportedFeature" do test "aggregate syntax in rule head raises DSL.CompileError" do - assert_raise ExDatalog.DSL.CompileError, ~r/aggregates are not yet supported/, fn -> + assert_raise ExDatalog.DSL.CompileError, ~r/use count\/sum\/min\/max for aggregates/, fn -> Code.compile_string(""" defmodule AggHeadTestErr do use ExDatalog.Schema @@ -643,7 +643,7 @@ defmodule ExDatalog.SchemaTest do end test "aggregate syntax in rule body raises DSL.CompileError" do - assert_raise ExDatalog.DSL.CompileError, ~r/aggregates are not yet supported/, fn -> + assert_raise ExDatalog.DSL.CompileError, ~r/use count\/sum\/min\/max for aggregates/, fn -> Code.compile_string(""" defmodule AggBodyTestErr do use ExDatalog.Schema @@ -1446,6 +1446,92 @@ defmodule ExDatalog.SchemaTest do assert MapSet.size(Knowledge.get(knowledge, "reachable")) == 1 end end + + describe "aggregate DSL" do + test "count aggregate in rule body" do + defmodule CountDSLTest do + use ExDatalog.Schema + + relation :emp do + field(:name, :atom) + field(:dept, :atom) + end + + relation :dept_count do + field(:dept, :atom) + field(:n, :integer) + end + + fact(emp(:alice, :eng)) + fact(emp(:bob, :eng)) + fact(emp(:carol, :ops)) + + rule dept_count(D, N) do + emp(E, D) + count(E, N) + end + end + + {:ok, knowledge} = CountDSLTest.materialize() + result = Knowledge.get(knowledge, "dept_count") + assert {:eng, 2} in result + assert {:ops, 1} in result + end + + test "sum aggregate in rule body" do + defmodule SumDSLTest do + use ExDatalog.Schema + + relation :salary do + field(:name, :atom) + field(:dept, :atom) + field(:amount, :integer) + end + + relation :dept_total do + field(:dept, :atom) + field(:total, :integer) + end + + fact(salary(:alice, :eng, 100)) + fact(salary(:bob, :eng, 80)) + + rule dept_total(D, T) do + salary(E, D, A) + sum(A, T) + end + end + + {:ok, knowledge} = SumDSLTest.materialize() + assert {:eng, 180} in Knowledge.get(knowledge, "dept_total") + end + + test "aggregate in head position raises DSL.CompileError" do + assert_raise ExDatalog.DSL.CompileError, ~r/cannot appear in a rule head/, fn -> + Code.compile_string(""" + defmodule AggHeadAggTest do + use ExDatalog.Schema + + relation :emp do + field(:name, :atom) + field(:dept, :atom) + end + + relation :dept_count do + field(:dept, :atom) + field(:n, :integer) + end + + fact(emp(:alice, :eng)) + + rule dept_count(D, count(E, N)) do + emp(E, D) + end + end + """) + end + end + end end defmodule ExDatalog.SchemaCoverageTest do @@ -1814,7 +1900,7 @@ defmodule ExDatalog.SchemaErrorTest do end test "aggregate in rule head raises DSL.CompileError" do - assert_raise ExDatalog.DSL.CompileError, ~r/aggregates are not yet supported/, fn -> + assert_raise ExDatalog.DSL.CompileError, ~r/use count\/sum\/min\/max for aggregates/, fn -> Code.compile_string(""" defmodule AggHeadErrTest do use ExDatalog.Schema @@ -1840,7 +1926,7 @@ defmodule ExDatalog.SchemaErrorTest do end test "aggregate in rule body raises DSL.CompileError" do - assert_raise ExDatalog.DSL.CompileError, ~r/aggregates are not yet supported/, fn -> + assert_raise ExDatalog.DSL.CompileError, ~r/use count\/sum\/min\/max for aggregates/, fn -> Code.compile_string(""" defmodule AggBodyErrTest do use ExDatalog.Schema diff --git a/test/ex_datalog/v050_coverage_test.exs b/test/ex_datalog/v050_coverage_test.exs new file mode 100644 index 0000000..b6e4134 --- /dev/null +++ b/test/ex_datalog/v050_coverage_test.exs @@ -0,0 +1,103 @@ +defmodule ExDatalog.V050CoverageTest do + use ExUnit.Case, async: true + + alias ExDatalog.{Atom, Callback, Constraint, Constraints, IR, Rule, Term} + alias ExDatalog.Planner.{Join, Plan, Predicate, Stratum} + + describe "planner structs" do + test "structs hold their fields" do + stratum = %Stratum{index: 0, rules: [], relations: ["edge"]} + join = %Join{relation: "edge", position: 0} + pred = %Predicate{kind: :comparison, op: :gt} + + plan = %Plan{strategy: :semi_naive, strata: [stratum], joins: [join], predicates: [pred]} + + assert plan.strategy == :semi_naive + assert hd(plan.strata).relations == ["edge"] + assert hd(plan.joins).relation == "edge" + assert hd(plan.joins).strategy == :nested_loop + assert hd(plan.predicates).kind == :comparison + end + end + + describe "Aggregate.evaluate/3 is not callable per-binding" do + test "raises a clear error" do + c = %IR.Constraint{op: :count, left: {:var, "X"}, right: nil, result: {:var, "N"}} + + assert_raise RuntimeError, ~r/not evaluated per-binding/, fn -> + Constraints.Aggregate.evaluate(c, %{}, %ExDatalog.Constraint.Context{}) + end + end + end + + describe "Rule callback helpers" do + test "has_callbacks?/1 and callbacks/1" do + cb = Callback.new(String, :length, [Term.var("S")]) + + rule = + Rule.new( + Atom.new("r", [Term.var("S")]), + [{:positive, Atom.new("s", [Term.var("S")])}, {:callback, cb}] + ) + + assert Rule.has_callbacks?(rule) + assert Rule.callbacks(rule) == [cb] + assert "S" in Rule.variables(rule) + assert Rule.body_atoms(rule) == [Atom.new("s", [Term.var("S")])] + end + + test "value-returning callback contributes its result variable to variables/1" do + cb = Callback.new(M, :f, [Term.var("X")], Term.var("R")) + + rule = + Rule.new( + Atom.new("r", [Term.var("X"), Term.var("R")]), + [{:positive, Atom.new("s", [Term.var("X")])}, {:callback, cb}] + ) + + assert "R" in Rule.variables(rule) + end + end + + describe "Rule aggregate helpers" do + test "aggregate_constraints/1 and non_aggregate_constraints/1 partition constraints" do + agg = Constraint.count(Term.var("E"), Term.var("N")) + cmp = Constraint.gt(Term.var("N"), Term.const(0)) + + rule = + Rule.new( + Atom.new("r", [Term.var("D"), Term.var("N")]), + [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D")])}], + [agg, cmp] + ) + + assert Rule.has_aggregates?(rule) + assert Rule.aggregate_constraints(rule) == [agg] + assert Rule.non_aggregate_constraints(rule) == [cmp] + end + end + + describe "Callback introspection" do + test "input_variables/1 and result_variable/1" do + cb = Callback.new(M, :f, [Term.var("A"), Term.const(1)], Term.var("R")) + assert Callback.input_variables(cb) == ["A"] + assert Callback.result_variable(cb) == "R" + + bool = Callback.new(M, :g, [Term.var("A")]) + assert Callback.result_variable(bool) == nil + end + end + + describe "Constraint aggregate op guards" do + test "aggregate_op?/1 and aggregate?/1" do + assert Constraint.aggregate_op?(:count) + refute Constraint.aggregate_op?(:gt) + assert Constraint.aggregate?(Constraint.sum(Term.var("X"), Term.var("T"))) + end + + test "min and max constructors" do + assert %Constraint{op: :min} = Constraint.min(Term.var("X"), Term.var("V")) + assert %Constraint{op: :max} = Constraint.max(Term.var("X"), Term.var("V")) + end + end +end From 770043694732368df17fd621f3491f57ab0ffb6d Mon Sep 17 00:00:00 2001 From: thanos Date: Mon, 22 Jun 2026 15:48:30 -0400 Subject: [PATCH 2/3] Overall assessment: v0.5.0 contains substantial useful work, but it is not release-ready yet. The branch has broad test coverage and the normal verification commands pass, but the review found several material issues where public API behavior, documentation, and tests diverge from the intended v0.5.0 feature set. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The most important issue is that the public `ExDatalog.materialize/2` API strips `:goal` before evaluation, so `strategy: :magic_sets` does not receive the goal it needs to run the magic-sets transformation. Current magic-sets tests still pass because `materialize/2` post-filters results after full semi-naive evaluation, which masks the issue. Other release-relevant issues include aggregate validation gaps, non-integer aggregate crashes, pipe-friendly runtime-facts error handling, callback timeout mailbox leakage, invalid README examples, and stale Schema documentation. Release readiness: **Needs Significant Fixes**. | Rank | Severity | Category | Location | Issue | Fix Effort | |---|---|---|---|---|---| | 1 | High | Correctness / Public API | `lib/ex_datalog.ex:217-223`, `lib/ex_datalog/engine/naive.ex:102-115` | Public `materialize(strategy: :magic_sets, goal: ...)` strips `:goal`, so magic sets does not run through the advertised API | M | | 2 | High | Correctness | `lib/ex_datalog/validator/safety.ex:268-274`, `lib/ex_datalog/engine/evaluator.ex:134-141` | Aggregate result variable is not required to appear in the head, so aggregate work can be silently discarded | S | | 3 | High | Correctness / Error Handling | `lib/ex_datalog/constraints/aggregate.ex:58-61` | Integer-only aggregate contract is not enforced; `sum` can crash and `min`/`max` accept non-integers | M | | 4 | High | Public API / Error Handling | `lib/ex_datalog/program.ex:295-298` | `Program.materialize/1` crashes on `{:error, reason}` in pipe chains instead of propagating errors | XS | | 5 | Medium | Concurrency / Resource Leak | `lib/ex_datalog/constraints/beam_callback.ex:89-101` | Timed-out callbacks can leave late result messages in the evaluator mailbox | S | | 6 | Medium | Public API / Planner | `lib/ex_datalog/planner.ex:166-171`, `livebooks/ex_datalog_v050.livemd:718-753` | Planner docs claim callbacks are classified, but planner ignores callback literals | S | | 7 | Medium | Documentation | `README.md:240-291` | New README aggregate/callback examples use unsupported DSL syntax and invalid materialization calls | S | | 8 | Medium | Documentation | `lib/ex_datalog/schema.ex:131-140`, `lib/ex_datalog/schema.ex:248-249`, `lib/ex_datalog/schema.ex:997` | Schema docs are stale: aggregates described as unsupported, `new_program/0` referenced, callback validation timing wrong | S | | 9 | Medium | Tests | `test/ex_datalog/magic_sets_test.exs:35-47`, `test/ex_datalog/magic_sets_property_test.exs:41-57` | Magic-sets tests verify result equality but not that the public API used the transform | M | | 10 | Low | Documentation / Examples | `livebooks/ex_datalog_v050.livemd`, `livebooks/ex_datalog_v050-v2.livemd` | v0.5 livebook still uses compile-time facts for runtime-data scenarios; duplicate `-v2` fork contains invalid API | S | | 11 | Low | Generated Docs / Edge Case | `lib/ex_datalog/schema.ex:218-239` | Generated relation constructor docs are malformed; zero-arity relations generate warnings / wrong arguments | S | | 12 | Low | Tests | `test/ex_datalog/runtime_facts_test.exs:153-162` | `Program.add_facts/2` error test does not assert partial-add semantics or malformed input behavior | XS | | 13 | Nit | LLM Artifact / Style | README, livebook, articles | Marketing phrasing, long dashes, repetitive generated-style prose, and fork artifact remain | S | No Critical severity findings were identified. The branch does have High severity issues that should block release, but the reviewed bugs do not appear to corrupt persisted data or produce unsound query answers in the default semi-naive path. ```text ID: H-001 Severity: High Category: Correctness / Public API Location: lib/ex_datalog.ex:217-223; lib/ex_datalog/engine/naive.ex:102-115; test/ex_datalog/magic_sets_test.exs:35-47 Summary: The public materialize(strategy: :magic_sets, goal: ...) API strips :goal before evaluation, so the magic-sets transform does not run through the advertised API. Evidence: ExDatalog.materialize/2 does `{goal, eval_opts} = Keyword.pop(opts, :goal, nil)` and then calls `evaluate(ir, eval_opts)`. Engine.Naive.evaluate_magic_sets/2 requires `Keyword.get(opts, :goal, nil)` to call MagicSets.transform/2; without it, it falls back to evaluate_semi_naive/2. The tests at magic_sets_test.exs:42-46 still pass because materialize/2 later applies apply_goal/2, so results look goal-restricted after full evaluation. Impact: The advertised v0.5.0 magic-sets feature is not active through the main public API. Users get full semi-naive evaluation plus post-filtering, so benchmarks, docs, and performance claims are misleading. This also invalidates tests that claim to prove goal-driven materialization. Recommended Fix: Preserve the goal for evaluation when `strategy: :magic_sets` is selected. Separate the concepts of evaluator goal (for magic sets) and post-materialization filtering, e.g. keep `:goal` in `eval_opts` for `strategy: :magic_sets` and still apply post-filtering afterward if that is the desired public result shape. Consider renaming the post-filter option in a future release to avoid semantic overload. Suggested Test: Add a regression test proving public `ExDatalog.materialize(program, strategy: :magic_sets, goal: goal)` invokes the magic-sets path, not just post-filtering. One practical approach is to add telemetry/metadata from the engine indicating `strategy: :magic_sets` and `magic_transform: :applied | :fallback`, then assert `:applied` for a supported goal. Alternatively expose enough stats to distinguish transformed evaluation from post-filtered full evaluation. Confidence: High ``` ```text ID: H-002 Severity: High Category: Correctness / Validation Location: lib/ex_datalog/validator/safety.ex:268-274; lib/ex_datalog/engine/evaluator.ex:134-141 Summary: Aggregate result variables are not required to appear in the rule head, so aggregate computations can be silently discarded. Evidence: check_aggregates/3 only calls check_single_aggregate/4 and check_aggregate_not_recursive/4. There is no check that `Constraint.result_variable(aggregate)` appears in `Rule.head_variables(rule)`. Reproduction: a rule `dept(D) :- emp(E, D), count(E, N)` validates and materializes as `dept(:eng)` while silently computing and then discarding `N` during projection. Impact: This violates the planned aggregate safety rule and can hide user mistakes. A user can believe they are computing an aggregate while the output relation contains only group keys. This is especially risky because materialization succeeds without an error. Recommended Fix: Add a safety check that each aggregate result variable appears in the rule head. Return a specific validator error such as `:aggregate_result_not_in_head` with rule index, op, and variable. Suggested Test: Add builder and DSL tests where `count(E, N)` appears in the body but `N` is absent from the head. Assert validation/materialization returns an error and the DSL raises/returns `DSL.CompileError` where appropriate. Confidence: High ``` ```text ID: H-003 Severity: High Category: Correctness / Error Handling Location: lib/ex_datalog/constraints/aggregate.ex:58-61; docs/articles/07_aggregates_in_datalog.md; lib/ex_datalog/constraints/aggregate.ex:12-14 Summary: The integer-only aggregate contract is not enforced at runtime. Evidence: Aggregate.compute/2 uses `Enum.sum/1`, `Enum.min/1`, and `Enum.max/1` directly. A `sum` aggregate over a `:any` relation containing `:a` validates but raises `ArithmeticError: bad argument in arithmetic expression` during materialization. `min`/`max` over atom values succeed using Erlang term ordering, despite docs saying aggregates are integer-only. Impact: Invalid user data can crash evaluation instead of returning a structured error or filtering/rejecting the program. The behavior contradicts documentation and the v0.5.0 scope statement. Recommended Fix: Validate aggregate input relation field types where statically knowable, and guard runtime values in group_and_reduce/5. For `sum`, reject non-integers with a structured evaluation error. For `min`/`max`, either explicitly allow all comparable Erlang terms and update docs, or enforce integers consistently. Suggested Test: Add tests for `sum`, `min`, and `max` over non-integer facts. Assert the chosen behavior: validation error if statically detectable, or `{:error, reason}` from materialization rather than an exception. Confidence: High ``` ```text ID: H-004 Severity: High Category: Public API / Error Handling Location: lib/ex_datalog/program.ex:295-298 Summary: Program.materialize/1 is not error-pipe-safe. Evidence: Program.add_fact/2 can return `{:error, reason}`. Program.materialize/1 only matches `%Program{}`. Reproduction: `Program.new() |> Program.add_relation("emp", [:atom, :atom]) |> Program.add_fact({"missing", [:x]}) |> Program.materialize()` raises `FunctionClauseError`, whereas `ExDatalog.materialize({:error, _})` propagates the error. Impact: The newly introduced pipe-friendly runtime facts API fails in exactly the case where pipe-friendly APIs are expected to preserve error tuples. Users get an exception instead of an actionable `{:error, reason}`. Recommended Fix: Add `def materialize({:error, _} = error, _opts \\ []), do: error` or equivalent non-conflicting clauses. Also consider adding `Program.add_facts/2` malformed input handling. Suggested Test: Add a test for `Program.new() |> Program.add_relation(...) |> Program.add_fact({"missing", [...]}) |> Program.materialize()` returning `{:error, _}`. Confidence: High ``` ```text ID: M-001 Severity: Medium Category: Concurrency / Resource Leak Location: lib/ex_datalog/constraints/beam_callback.ex:89-101 Summary: Timed-out callbacks can leave late result messages in the evaluator process mailbox. Evidence: On timeout, safe_apply/4 runs `Process.exit(pid, :kill)`, demonitor-flushes the monitor, and returns `{:error, :timeout}`. If the child sends `{ref, result}` just after the timeout branch is selected but before it is killed, that message is not flushed. Since each callback uses a unique ref, later receive blocks will skip the stale message, leaving it in the mailbox. Impact: Workloads with many timed-out callbacks can accumulate unmatched messages in the evaluator process mailbox, increasing memory use and receive scanning cost. Recommended Fix: After killing/demonitoring, opportunistically flush `{^ref, _}` with a zero-timeout receive. Alternatively use a single message protocol where the child exits with the result and the parent only consumes the monitor message, or ensure both result and DOWN are drained for the ref. Suggested Test: Add a controlled callback that sleeps close to the timeout and sends late results repeatedly; assert the caller mailbox does not grow with stale `{ref, result}` messages. This may require exposing a test-only helper or using process_info(self(), :messages) in a focused unit test. Confidence: Medium ``` ```text ID: M-002 Severity: Medium Category: Public API / Planner Location: lib/ex_datalog/planner.ex:166-171; lib/ex_datalog/planner.ex:2-8; livebooks/ex_datalog_v050.livemd:718-753 Summary: Planner documentation says callbacks are predicates, but build_predicates/1 ignores callback literals. Evidence: Planner moduledoc says the plan includes predicates "(constraints, aggregates, callbacks)". build_predicates/1 only collects `for {:constraint, c} <- rule.body`. The livebook section says planner classifies `:callback`, but the shown callback plan output has `Predicates: 0`. Impact: Planner output is incomplete for callback-heavy rules, and docs/examples contradict implementation. Future users relying on Planner for explainability or optimization will miss callback predicates. Recommended Fix: Extend build_predicates/1 to include `{:callback, cb}` as `%Predicate{kind: :callback, op: :callback, metadata: ...}` or update docs to state callbacks are not included in v0.5.0 plans. The former better matches the public claim. Suggested Test: Add a planner test with a DSL `predicate/5` callback in a rule body and assert `plan.predicates` contains one `kind: :callback` predicate and `explain_plan/2` includes `callback`. Confidence: High ``` ```text ID: M-003 Severity: Medium Category: Documentation / Examples Location: README.md:240-291 Summary: New README aggregate and callback examples use unsupported Schema DSL syntax. Evidence: README uses `relation "emp", [:atom, :atom]`, `fact "emp", [:alice, :eng]`, and `ExDatalog.materialize(DeptStats)`. The implemented Schema DSL uses `relation :emp do field ... end`, `fact(emp(...))`, and module-generated `DeptStats.materialize()` / `DeptStats.program()`. Impact: Users copying the v0.5.0 README examples will get compile errors and may conclude the new aggregate/callback APIs are broken. Recommended Fix: Rewrite README v0.5.0 feature examples using the actual DSL. Given the runtime facts discussion, prefer `Schema.new() |> Program.add_fact(Schema.rel(...)) |> Program.materialize()` for realistic data-loading examples. Suggested Test: Add README example snippets to doctests or a docs/examples test harness that compiles and runs them. Confidence: High ``` ```text ID: M-004 Severity: Medium Category: Documentation / API Docs Location: lib/ex_datalog/schema.ex:131-140; lib/ex_datalog/schema.ex:248-249; lib/ex_datalog/schema.ex:997 Summary: Schema documentation is stale and contradicts v0.5.0 implementation. Evidence: Schema moduledoc still says "Aggregates are not yet supported" and planned for v0.6.0. Generated `program/0` docs say "For runtime facts, use new_program/0" even though the agreed API is `new/0`. predicate/5 docs say missing callback module/function raises a compile-time DSL.CompileError; reproduction shows the schema module compiles and `materialize()` returns `{:error, [%Validator.Error{kind: :invalid_callback}]}`. Impact: API docs mislead users on core v0.5.0 features and error timing. This is especially risky because `Schema` is the main DSL entry point. Recommended Fix: Update Schema moduledoc to describe supported `count/sum/min/max`, runtime facts via relation constructors + `new/0`, and actual callback validation timing. Remove or reframe the old `agg(...)` preview as a legacy placeholder error. Suggested Test: Run `mix docs --warnings-as-errors` already passes, but add docs-focused tests that compile examples in Schema docs where practical. Confidence: High ``` ```text ID: M-005 Severity: Medium Category: Testing Location: test/ex_datalog/magic_sets_test.exs:35-47; test/ex_datalog/magic_sets_property_test.exs:41-57 Summary: Magic-sets tests would pass even if magic sets never ran through the public API. Evidence: The tests compare `Knowledge.match(magic, "ancestor", [:a, :_])` to a subset of full semi-naive results. Because ExDatalog.materialize/2 post-filters by `:goal`, this assertion passes even when Engine.Naive receives no goal and falls back to full semi-naive evaluation. Impact: The highest-value magic-sets regression is not protected. H-001 reached the branch despite "magic≡semi-naive" property tests. Recommended Fix: Add tests that distinguish transformation from post-filtering. Instrument the engine/magic transform path with telemetry or stats, then assert transform application/fallback reason. Also test that `goal` reaches the engine when `strategy: :magic_sets` is requested. Suggested Test: Attach a telemetry handler to a new event such as `[:ex_datalog, :magic_sets, :stop]` and assert metadata `%{status: :applied}` for a supported program and `%{status: :fallback, reason: ...}` for unsupported programs. Confidence: High ``` ```text ID: M-006 Severity: Medium Category: Documentation / Examples Location: livebooks/ex_datalog_v050.livemd; livebooks/ex_datalog_v050-v2.livemd Summary: The v0.5.0 livebook does not reflect the agreed runtime-facts DSL pattern, and a duplicate fork contains invalid API calls. Evidence: livebooks/ex_datalog_v050.livemd still uses compile-time `fact(...)` macros for employee/user datasets. livebooks/ex_datalog_v050-v2.livemd contains calls like `DeptCount.fact(DeptCount.emp(:alice, :eng))`, but no such `DeptCount.fact/1` function exists. Impact: The primary v0.5.0 tutorial does not teach the realistic data-loading pattern that motivated `Schema.new/0`, relation constructors, `Program.add_fact/2`, and `Program.add_facts/2`. Accidentally committing the `-v2` fork would ship broken examples. Recommended Fix: Rewrite the livebook examples to use `DeptCount.new() |> Program.add_facts([...]) |> Program.materialize()`. Delete or ignore `livebooks/ex_datalog_v050-v2.livemd` unless it is intentionally kept as a draft outside git. Suggested Test: Add a lightweight script/test that extracts and evaluates Livebook Elixir cells, or maintain a mirrored `.exs` example file for CI. Confidence: High ``` ```text ID: L-001 Severity: Low Category: Generated Documentation / Edge Case Location: lib/ex_datalog/schema.ex:218-239 Summary: Generated relation constructor documentation is malformed, and zero-arity relations generate warnings / wrong argument ranges. Evidence: The generated doc interpolates `#=> {"rel", [arg_1, arg_2})]}` with mismatched parentheses/brackets. For an empty relation block, the generator uses `1..arity`; when arity is 0, Elixir warns that `1..0` defaults to a negative step, and constructor argument generation is incorrect for a zero-arity relation. Impact: Generated docs for user schemas are unpolished and potentially misleading. Zero-arity relations are an edge case, but if allowed by Program schemas, constructor generation should handle them correctly. Recommended Fix: Build argument names with `Enum.map(1..arity//1, ...)` guarded for arity 0, or use `for i <- 1..arity, arity > 0` equivalent. Fix the doc string interpolation. Consider explicitly rejecting zero-field relations if they are not supported. Suggested Test: Compile a schema with a normal relation and inspect `Code.fetch_docs/1` for the generated constructor doc. Add a zero-field relation test for either successful `rel()` construction or a clear compile-time error. Confidence: Medium ``` ```text ID: L-002 Severity: Low Category: Testing / Error Handling Location: test/ex_datalog/runtime_facts_test.exs:153-162; lib/ex_datalog/program.ex:272-280 Summary: Program.add_facts/2 tests do not define partial-add semantics or malformed input behavior. Evidence: The "stops on first error" test asserts only `{:error, _}`. It does not assert whether the earlier successful fact is retained/discarded, and malformed list elements would currently raise FunctionClauseError via add_fact/2 matching. Impact: Public bulk-loading semantics are ambiguous. Users loading datasets need predictable behavior when one row is invalid. Recommended Fix: Decide and document whether add_facts/2 is transactional (all-or-error) or partial (returns error after some adds). If transactional, validate all facts before adding. If partial, return richer error context. Handle malformed fact terms with `{:error, reason}`. Suggested Test: Add tests for one invalid fact after a valid fact and for a malformed fact such as `:not_a_fact`. Assert exact behavior and error message. Confidence: High ``` ```text ID: L-003 Severity: Low Category: Documentation Location: README.md:50, README.md:513, CHANGELOG.md:56 Summary: Test counts are stale after adding runtime facts tests. Evidence: README and CHANGELOG still state 845 tests. Current suite output is 855 tests, 152 doctests, 10 properties, 0 failures. Impact: Minor release metadata inconsistency. Recommended Fix: Update counts after the final test run, or avoid exact test counts in user-facing docs to reduce churn. Suggested Test: None needed; this is documentation maintenance. Confidence: High ``` ```text ID: L-004 Severity: Low Category: Benchmarks / Documentation Location: bench/aggregate_bench.exs:69-82; bench/magic_sets_bench.exs; chest/benchmarks/v0.5.0-report.md Summary: Benchmark scripts do not save results under bench/results despite the directory being kept. Evidence: Both benchmark files use only `Benchee.Formatters.Console`; no formatter writes to bench/results. The report is manually written under chest/benchmarks. An empty bench/results directory is kept with .gitkeep. Impact: Minor confusion for contributors expecting reproducible benchmark artifacts in bench/results. Recommended Fix: Either remove bench/results/.gitkeep and document that reports are manually saved under chest/benchmarks, or add `benchee_json` / another formatter and write generated outputs there. Suggested Test: Not applicable. Confidence: High ``` ```text ID: N-001 Severity: Nit Category: LLM Artifact / Style Location: README.md; docs/articles/06_query_planning_in_datalog.md; docs/articles/07_aggregates_in_datalog.md; docs/articles/08_extending_datalog_with_beam_callbacks.md; docs/articles/09_magic_sets_and_demand_driven_evaluation.md; livebooks/ex_datalog_v050-v2.livemd Summary: Some new docs show generated-documentation artifacts and marketing tone. Evidence: README opens with "A production-grade Datalog engine" while v0.5.0 includes experimental magic sets. The articles are long and repetitive in places. Several files use long dashes heavily. The duplicate livebook title includes "DSL Edition - fork". Impact: Does not affect behavior, but weakens polish and trust in release docs. Recommended Fix: Tighten docs, remove duplicated/fork files, avoid unsupported marketing claims, and keep prose closer to existing project style. Suggested Test: Human editorial pass; no automated test needed. Confidence: Medium ``` Material healthy areas: - Aggregate tests cover happy paths for `count`, `sum`, `min`, `max`, DSL aggregate syntax, filtering before grouping, and aggregate stratification. - Callback tests cover boolean filters, value-returning callbacks, exceptions, timeouts, validation, and DSL `predicate/5` usage. - Planner tests cover basic plan shape and `explain_plan/2` strategy rendering. - Runtime facts tests cover relation constructors, `Schema.new/0`, `Program.add_fact/2`, `Program.add_facts/2`, and happy-path pipe materialization. - Property tests cover path-style magic-sets result equivalence for randomly generated edge sets. Important gaps: 1. **Magic sets public API path is not tested.** Existing tests cannot distinguish magic transform from post-filtering. Add telemetry/stats-backed tests that assert transform applied/fallback status through `ExDatalog.materialize/2`. 2. **Aggregate safety misses result-not-in-head.** Add builder and DSL tests for `count(E, N)` with head `dept(D)` and assert validation error. 3. **Aggregate type/runtime errors are not tested.** Add `sum/min/max` non-integer tests and define expected structured error behavior. 4. **Program.materialize error propagation is not tested.** Add failed pipe-chain test and assert `{:error, reason}`. 5. **Planner callback classification is not tested.** Add callback plan test. 6. **Runtime facts malformed input behavior is not tested.** Add malformed fact and partial bulk-load semantics tests. 7. **Docs examples are not exercised.** README and Livebook examples should have either doctests or mirrored executable examples. Material issues: - README v0.5 aggregate/callback examples use invalid DSL syntax and invalid materialization calls. - Schema moduledoc still describes aggregates as unsupported and planned for v0.6.0. - Schema generated docs reference `new_program/0`; the implemented API is `new/0`. - Schema `predicate/5` docs claim compile-time callback validation; implementation validates during `materialize/compile` via `Validator`. - BeamCallback moduledoc says callbacks run in a `Task`; implementation uses `spawn_monitor`. - Planner docs say callbacks are included as predicates; implementation ignores callbacks. - Livebook teaches static compile-time facts for data-heavy examples despite the newly introduced runtime facts API. - Duplicate `livebooks/ex_datalog_v050-v2.livemd` contains invalid `DeptCount.fact(...)` API calls and should not be committed. - README/CHANGELOG test counts are stale after the latest runtime-facts tests. Areas that appear healthy: - The migration guide broadly explains the intended v0.5.0 feature set and includes useful examples. - Articles 06-09 cover the new conceptual areas in depth, though they need a polish pass for exact claims and concision. - `mix docs --warnings-as-errors` passes, so there are no broken ExDoc references at build time after the current edits. Findings: - No emojis were found in reviewed new files. - Long dashes and broad marketing phrases appear frequently in README/articles/livebook. - `livebooks/ex_datalog_v050-v2.livemd` looks like an intermediate generated fork and contains invalid API examples. - Some docs repeat the same framing language ("deterministic, side-effect-free", "goal-directed", "program transformation") more than needed. - The volume of article prose is high relative to the maturity of the APIs, especially while public behavior still diverges from docs. Recommendation: remove the fork file, tighten README and livebook to executable examples, and do one editorial pass on articles for concision and exactness. Material findings: - `livebooks/ex_datalog_v050-v2.livemd` is a duplicate/fork and should not be committed unless intentionally renamed and fixed. - `bench/results/.gitkeep` keeps an empty directory, but benchmark scripts no longer write results there. This is not harmful, but it is currently unnecessary. - Planner `Predicate` currently cannot represent callbacks despite docs claiming it does; this is more an incomplete implementation than dead code. No material unused source modules were found among the new core modules. `benchee` is used by the benchmark scripts and is scoped to `:dev`. 1. Fix `ExDatalog.materialize/2` so `strategy: :magic_sets, goal: ...` actually passes the goal to the evaluator and runs `MagicSets.transform/2` for supported programs. 2. Add tests that prove the public magic-sets API uses the transform path, not just post-filtering. 3. Enforce aggregate result variable in rule head. 4. Enforce integer-only aggregate behavior or update docs if non-integer `min/max` are intentionally allowed. 5. Make `Program.materialize/1,2` propagate `{:error, reason}` in pipe chains. 6. Fix invalid README v0.5 examples. 7. Update Schema docs for aggregates, runtime facts (`new/0`), and callback validation timing. 8. Remove or fix `livebooks/ex_datalog_v050-v2.livemd` before commit. 1. Flush late callback result messages after timeout. 2. Include callback predicates in `Planner.plan/2`, or explicitly document that callbacks are excluded in v0.5.0. 3. Rewrite `livebooks/ex_datalog_v050.livemd` to use runtime facts via `Schema.new/0`, relation constructors, and `Program.add_facts/2`. 4. Add malformed input and partial-load semantics tests for `Program.add_facts/2`. 5. Fix generated relation constructor docs and decide zero-arity relation support. 6. Update stale test counts in README/CHANGELOG or remove exact counts. 1. Decide whether benchmark scripts should write machine-readable results under `bench/results`. 2. Tighten article prose and reduce generated-looking repetition. 3. Add executable docs/example testing for README and Livebook snippets. **Needs Significant Fixes** The branch has strong momentum and many tests pass, but the primary public magic-sets API does not actually execute the magic-sets strategy because `:goal` is stripped before evaluation. That is a release-blocking mismatch between advertised functionality and implementation. Aggregate validation/runtime behavior and pipe-friendly error propagation also need fixes before v0.5.0 should be tagged. After the Must Fix items are addressed and tests prove the public API behavior, this can likely move to **Merge After Minor Fixes** quickly. --- CHANGELOG.md | 11 +- README.md | 48 +- docs/articles/06_query_planning_in_datalog.md | 290 +++++++++++++ docs/articles/07_aggregates_in_datalog.md | 405 +++++++++++++++++ ...8_extending_datalog_with_beam_callbacks.md | 410 ++++++++++++++++++ ...magic_sets_and_demand_driven_evaluation.md | 312 +++++++++++++ docs/migration_v0.5.md | 355 +++++++++++++++ lib/ex_datalog.ex | 14 +- lib/ex_datalog/constraints/aggregate.ex | 33 +- lib/ex_datalog/constraints/beam_callback.ex | 14 +- lib/ex_datalog/magic_sets.ex | 73 +++- lib/ex_datalog/planner.ex | 16 +- lib/ex_datalog/program.ex | 8 +- lib/ex_datalog/schema.ex | 56 ++- lib/ex_datalog/validator/safety.ex | 28 ++ livebooks/ex_datalog_v050.livemd | 72 ++- test/ex_datalog/magic_sets_test.exs | 63 +++ test/ex_datalog/runtime_facts_test.exs | 27 ++ 18 files changed, 2172 insertions(+), 63 deletions(-) create mode 100644 docs/articles/06_query_planning_in_datalog.md create mode 100644 docs/articles/07_aggregates_in_datalog.md create mode 100644 docs/articles/08_extending_datalog_with_beam_callbacks.md create mode 100644 docs/articles/09_magic_sets_and_demand_driven_evaluation.md create mode 100644 docs/migration_v0.5.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e4c2aa..4cfdac9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Educational articles 06–09 (planner, aggregates, callbacks, magic sets) and a v0.4 → v0.5 migration guide. - Benchmark harness (`bench/`) and report (`chest/benchmarks/v0.5.0-report.md`). +- **Runtime facts API** — `Schema.new/0` returns a blank program (relations + + rules, no compile-time facts). Pipe facts at runtime via + `Program.add_fact/2` (tuple form `{"rel", [values]}`) or + `Program.add_facts/2` (bulk). `Program.materialize/1,2` is a pipe-friendly + wrapper for `ExDatalog.materialize/2`. Generated relation constructors + (`Schema.emp(:alice, :eng)` -> `{"emp", [:alice, :eng]}`) bridge the DSL + to runtime data. ### Changed @@ -53,8 +60,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Aggregates are integer-only; `avg` and float support are deferred to v0.6.0. - Magic sets is experimental and opt-in; the default strategy is unchanged. -- All v0.4.1 tests continue to pass. Suite: 845 tests, 10 properties, - 149 doctests, ~93% coverage. +- All v0.4.1 tests continue to pass. Suite: 861 tests, 10 properties, + 152 doctests, ~93% coverage. ## v0.4.1 (2026-06-21) diff --git a/README.md b/README.md index 6294d1e..ad44a98 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ It continues to influence modern databases, compilers, static analysis tools, kn - **Provenance / derivation explain** (`explain: true`) - **Telemetry** integration (`:telemetry` events for query/planner lifecycle) - **Deterministic**: same program + same facts = same result regardless of backend -- 845 tests, 0 failures, credo clean +- 861 tests, 0 failures, credo clean ## Installation @@ -241,12 +241,19 @@ Group and reduce bindings with `count`, `sum`, `min`, `max`: defmodule DeptStats do use ExDatalog.Schema - relation "emp", [:atom, :atom] - relation "dept_count", [:atom, :integer] + relation :emp do + field(:name, :atom) + field(:dept, :atom) + end + + relation :dept_count do + field(:dept, :atom) + field(:n, :integer) + end - fact "emp", [:alice, :eng] - fact "emp", [:bob, :eng] - fact "emp", [:carol, :sales] + fact(emp(:alice, :eng)) + fact(emp(:bob, :eng)) + fact(emp(:carol, :sales)) rule dept_count(D, N) do emp(_, D) @@ -254,7 +261,7 @@ defmodule DeptStats do end end -{:ok, k} = ExDatalog.materialize(DeptStats) +{:ok, k} = DeptStats.materialize() Knowledge.get(k, "dept_count") #=> MapSet.new([{:eng, 2}, {:sales, 1}]) ``` @@ -267,26 +274,31 @@ Call deterministic Elixir functions from rules: defmodule Gated do use ExDatalog.Schema - relation "user", [:atom] - relation "active_user", [:atom] + relation :user do + field(:name, :atom) + field(:age, :integer) + end + + relation :active_user do + field(:name, :atom) + end - predicate :is_adult, AgeChecker, :adult?, [:atom], :boolean + predicate(:adult?, AgeChecker, :adult?, [:integer], :boolean) - fact "user", [:alice] - fact "user", [:bob] + fact(user(:alice, 30)) + fact(user(:bob, 12)) rule active_user(U) do - user(U) - is_adult(U) + user(U, Age) + adult?(Age) end end defmodule AgeChecker do - def adult?(:alice), do: true - def adult?(:bob), do: false + def adult?(age), do: age >= 18 end -{:ok, k} = ExDatalog.materialize(Gated) +{:ok, k} = Gated.materialize() Knowledge.get(k, "active_user") #=> MapSet.new([{:alice}]) ``` @@ -510,7 +522,7 @@ The following references are highly recommended for understanding both the theor | v0.3.0 | Tuple shorthand for rules (`add_rule/3`, `add_rule/4`), `Term.from/1`, `ExDatalog.Atom.from_tuple/1`, `Constraint.from_tuple/1`; renamed `Result` → `Knowledge`, `query` → `materialize` | | v0.4.0 | Schema DSL (`use ExDatalog.Schema`), `relation`, `fact`, `rule`, `query` macros, `not_` negation, constraint DSL, post-materialization queries | | v0.4.1 | DSL review fixes: correct uppercase-variable docs, clean aggregate error, query/find validation, unified `DSL.CompileError`, 792 tests | -| v0.5.0 | Aggregates (`count`/`sum`/`min`/`max`), BEAM callback predicates, query planner, magic sets (experimental), 845 tests | +| v0.5.0 | Aggregates (`count`/`sum`/`min`/`max`), BEAM callback predicates, query planner, magic sets (experimental), runtime facts API, 861 tests | | v1.0.0 | Stable public API, hardened production semantics | ## License diff --git a/docs/articles/06_query_planning_in_datalog.md b/docs/articles/06_query_planning_in_datalog.md new file mode 100644 index 0000000..9355ee7 --- /dev/null +++ b/docs/articles/06_query_planning_in_datalog.md @@ -0,0 +1,290 @@ +# Query Planning in ExDatalog: The Seam Between IR and Engine + +Datalog evaluation looks direct — apply the rules, accumulate facts, stop at a fixpoint. But between the validated `ExDatalog.IR` and the semi-naive engine sits a small planner whose job is to describe *how* a program will run before it runs. ExDatalog's planner is thin by design in v0.5.0, but it is the structure future optimizations hang from: join ordering, cost-based selection, and goal-directed evaluation all plug in here. + +## Why Datalog Needs a Planner + +The IR describes *what* a program means: relations, rules, strata, safety-validated bodies. The engine describes *how* it runs: nested-loop joins, delta tracking, monotonic fixpoint iteration. Between the two, several decisions need a home: + +- which evaluation **strategy** applies (semi-naive bottom-up vs. goal-directed magic sets); +- how each rule is **stratified** (already computed by the validator, but the engine wants the resolved `IR.Rule` structs grouped by stratum); +- which **joins** the engine will perform — one per positive body atom — and in what *position*, so the delta slot maps cleanly during semi-naive iteration; +- which **predicates** (constraints, aggregates, callbacks) must run, classified by kind so the engine can dispatch them uniformly; +- where **telemetry** fires so observers can profile planning without instrumenting every rule. + +Putting these decisions in their own struct means the engine stays a pure consumer of the plan, and an optimizer can swap plans without touching the engine. That separation is what makes the planner the right seam for future work. + +## The Four Planner Structs + +The plan is assembled from four small structs. Each is descriptive — it explains what the engine will do rather than dictating bytecode. + +### `ExDatalog.Planner.Plan` + +The top-level plan records the chosen `strategy`, the planned `strata`, the flat list of `joins`, the flat list of `predicates`, and a small `metadata` map (currently carrying the goal, if any): + +```elixir +@enforce_keys [:strategy, :strata] +defstruct [:strategy, strata: [], joins: [], predicates: [], metadata: %{}] + +@type strategy :: :semi_naive | :magic_sets +``` + +The plan is descriptive by construction. Aggregate and callback predicates do not get their own fields — they appear in `predicates` with `kind: :aggregate` / `kind: :callback`, so adding new predicate categories later does not change the struct. + +### `ExDatalog.Planner.Stratum` + +A planned stratum wraps the IR stratum into a form the engine wants: the actual `IR.Rule` structs, not just their IDs, alongside the relations that belong to that stratum: + +```elixir +@enforce_keys [:index, :rules, :relations] +defstruct [:index, :rules, :relations] +``` + +The planner builds these by filtering rules whose `stratum` field matches each IR stratum's `index`: + +```elixir +defp build_strata(%IR{strata: strata, rules: rules}) do + Enum.map(strata, fn %IR.Stratum{index: idx, relations: rels} -> + stratum_rules = Enum.filter(rules, fn r -> r.stratum == idx end) + %Stratum{index: idx, rules: stratum_rules, relations: rels} + end) +end +``` + +This is where the validator's stratification work flows into the engine: each `Stratum` runs to a local fixpoint before the next begins, guaranteeing that negated relations are complete before any rule that negates them fires. + +### `ExDatalog.Planner.Join` + +A join is one positive body atom position within a rule. `position` is the 0-based index of the atom within the rule's positive body; `delta_position` indicates the semi-naive delta slot it maps to; `strategy` records how the join is executed, defaulting to `:nested_loop`: + +```elixir +@enforce_keys [:relation, :position] +defstruct [:relation, :position, delta_position: nil, strategy: :nested_loop] + +@type strategy :: :nested_loop | :indexed +``` + +The planner emits one join per positive body atom across all rules. For the classic transitive-closure program (`path(X,Y) :- edge(X,Y)` plus `path(X,Z) :- edge(X,Y), path(Y,Z)`), that's three joins — one for the first rule's body and two for the second rule's two positive atoms: + +```elixir +defp build_joins(%IR{rules: rules}) do + Enum.flat_map(rules, fn rule -> + rule.body + |> Enum.filter(&match?({:positive, _}, &1)) + |> Enum.with_index() + |> Enum.map(fn {{:positive, %IR.Atom{relation: rel}}, position} -> + %Join{relation: rel, position: position, delta_position: position} + end) + end) +end +``` + +Negative atoms and constraints are deliberately excluded — they filter, they don't join. The `delta_position` mirrors `position` because, in the current engine, every positive atom of a recursive rule participates in the delta; future join reorderings will reassign this field. + +### `ExDatalog.Planner.Predicate` + +A predicate is any non-relational body element: a constraint, an aggregate, or a callback. `kind` groups it into one of the evaluation categories; `op` is the specific operator: + +```elixir +@enforce_keys [:kind, :op] +defstruct [:kind, :op, metadata: %{}] + +@type kind :: + :comparison | :arithmetic | :type | :string | + :membership | :aggregate | :callback +``` + +Classification leans on `ExDatalog.Constraint`'s operator predicates: + +```elixir +defp constraint_kind(op) do + cond do + Constraint.comparison_op?(op) -> :comparison + Constraint.arithmetic_op?(op) -> :arithmetic + Constraint.type_op?(op) -> :type + Constraint.string_op?(op) -> :string + Constraint.membership_op?(op) -> :membership + Constraint.aggregate_op?(op) -> :aggregate + true -> :comparison + end +end +``` + +This classification matters for the engine: comparison, type, string, and membership predicates filter bindings, while arithmetic predicates *extend* the binding environment with their result variable. The plan records that distinction once, so the engine dispatches on `kind` rather than re-deriving it per iteration. + +## `plan/2` and `explain_plan/1,2` + +The two public entry points cover the two use cases: programmatic planning for the engine, and human-readable planning for debugging. + +### `plan/2` + +`plan/2` takes a compiled `IR` and returns `{:ok, %Plan{}}`. It accepts two options: + +- `:strategy` — `:semi_naive` (the default) or `:magic_sets`; +- `:goal` — a `{relation, pattern}` tuple used with `:magic_sets` (e.g. `{"path", [:a, :_]}` for goal-directed evaluation). + +```elixir +iex> alias ExDatalog.{Program, Rule, Atom, Term, Compiler, Planner} +iex> {:ok, ir} = +...> Program.new() +...> |> Program.add_relation("edge", [:atom, :atom]) +...> |> Program.add_relation("path", [:atom, :atom]) +...> |> Program.add_rule( +...> Rule.new( +...> Atom.new("path", [Term.var("X"), Term.var("Y")]), +...> [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] +...> ) +...> ) +...> |> Compiler.compile() +iex> {:ok, plan} = Planner.plan(ir) +iex> plan.strategy +:semi_naive +iex> length(plan.joins) +1 +``` + +The magic-sets strategy is selected by simply passing the option: + +```elixir +{:ok, plan} = Planner.plan(ir, strategy: :magic_sets, goal: {"path", [:a, :_]}) +plan.strategy #=> :magic_sets +plan.metadata.goal #=> {"path", [:a, :_]} +``` + +### `explain_plan/1,2` + +`explain_plan/1,2` is the debugging surface. It accepts a `Program` rather than an IR, validates and compiles it, plans the result, and renders a human-readable summary: + +```elixir +iex> Planner.explain_plan(program) =~ "Strategy: semi_naive" +true +``` + +The output is a small multi-line string. For the transitive-closure program with one rule it reads: + +``` +Strategy: semi_naive + Stratum 0: 1 rule(s), relations: edge, path +Joins: 1 +Predicates: 0 +``` + +Constraints surface as a parenthesized list of operators: + +``` +Predicates: 1 (gt) +``` + +The formatter is straightforward — it joins the header, the per-stratum lines, and the join/predicate counts with `\n`: + +```elixir +defp format_plan(%Plan{} = plan) do + header = "Strategy: #{plan.strategy}" + + strata_lines = + Enum.map(plan.strata, fn s -> + " Stratum #{s.index}: #{length(s.rules)} rule(s), relations: #{Enum.join(s.relations, ", ")}" + end) + + join_line = "Joins: #{length(plan.joins)}" + + pred_line = + "Predicates: #{length(plan.predicates)}" <> + case plan.predicates do + [] -> "" + preds -> " (#{Enum.map_join(preds, ", ", & &1.op)})" + end + + Enum.join([header | strata_lines] ++ [join_line, pred_line], "\n") +end +``` + +If the program fails to compile — say, an unsafe rule where `Z` is unbound — `explain_plan` returns an error string rather than raising: + +```elixir +iex> Planner.explain_plan(bad_program) =~ "compilation failed" +true +``` + +This makes `explain_plan/1,2` cheap to wire into a REPL or a `mix` task without having to handle exception flows. + +## Strategy Selection + +Strategy is recorded, not yet *executed* differently. In v0.5.0 the planner is honest about this in its module doc: + +> When `:magic_sets` is requested, a `:goal` option (`{relation, pattern}`) should be supplied; otherwise the planner records the strategy but the engine falls back to semi-naive. + +So selecting `:magic_sets` changes the plan's `strategy` field and stashes the goal in `metadata`, but the existing `Engine.Naive` still runs semi-naive bottom-up. The point of recording the strategy now is that the *shape* of the plan changes — future magic-sets rewriting will introduce magic predicates and adornment joins while keeping the same `Plan`/`Stratum`/`Join`/`Predicate` structs. Selecting the strategy is a one-line change at the call site: + +```elixir +Planner.plan(ir) # :semi_naive +Planner.plan(ir, strategy: :magic_sets, goal: {"path", [:a, :_]}) +``` + +This means the planner is the right hook today for tooling that wants to *predict* which strategy will run, even if the engine hasn't yet caught up. + +## Telemetry Events + +`plan/2` emits one set of telemetry events per call — never per rule — so observers see the planning cost as a whole: + +| Event | Measurements | Metadata | +|---|---|---| +| `[:ex_datalog, :planner, :start]` | `%{system_time: System.system_time()}` | `%{relation_count, rule_count}` | +| `[:ex_datalog, :planner, :stop]` | `%{duration: monotonic_diff}` | metadata + `:strategy` | +| `[:ex_datalog, :planner, :exception]` | `%{duration: monotonic_diff}` | metadata + `%{kind: :error, reason: exception}` | + +The implementation wraps the planning body in `try/rescue` so any failure still emits the `:exception` event before re-raising: + +```elixir +:telemetry.execute([:ex_datalog, :planner, :start], %{system_time: System.system_time()}, metadata) +start = System.monotonic_time() + +try do + plan = %Plan{strategy: strategy, strata: build_strata(ir), joins: build_joins(ir), + predicates: build_predicates(ir), metadata: %{goal: goal}} + :telemetry.execute([:ex_datalog, :planner, :stop], %{duration: System.monotonic_time() - start}, + Map.put(metadata, :strategy, strategy)) + {:ok, plan} +rescue + e -> + :telemetry.execute([:ex_datalog, :planner, :exception], %{duration: System.monotonic_time() - start}, + Map.merge(metadata, %{kind: :error, reason: e})) + reraise e, __STACKTRACE__ +end +``` + +Attach with `:telemetry.attach/4` to react to slow planning, or use `:telemetry.span/3` wrappers in higher-level tooling that wants a single composite event. The per-call (rather than per-rule) granularity keeps planning telemetry cheap: a thousand-rule program still emits exactly three events on failure. + +## Where the Planner Sits in the Pipeline + +ExDatalog's evaluation pipeline is now.Compiler → Validator → Planner → Engine: + +``` +Program (DSL) + │ Program.compile/1 or ExDatalog.compile/1 + ▼ +IR (relations, facts, rules, strata, metadata) + │ ExDatalog.Planner.plan/2 + ▼ +Plan (strategy, strata, joins, predicates, metadata) + │ Engine.Naive.eval_plan/2 (or future engine) + ▼ +Knowledge (materialized relations: MapSet of tuples) + │ Schema.query/2, find/where + ▼ +result +``` + +The validator runs *before* the planner — that's why `plan/2` accepts the already-validated IR, and why `explain_plan/2`'s "compilation failed" path goes through `ExDatalog.compile/1` (which bundles validation) rather than re-validating inside the planner. The planner is a pure consumer of validated IR. + +The engine is the next consumer of the plan. Each `Stratum` in the plan corresponds to one fixpoint loop in the engine; each `Join` corresponds to one nested-loop scan; each `Predicate` corresponds to one filter or binding extension. Because the plan is descriptive, the engine doesn't *need* it to function — it can read the IR directly — but using the plan means future optimizations (join reordering, indexed joins, magic-sets rewriting) can land in the planner and immediately benefit every engine that consumes plans. + +## Practical Notes + +- **The plan is cheap to build.** A few `Enum.flat_map` passes over the rules; no allocation per fact. There's no reason to skip it in production. +- **Use `explain_plan/1` for debugging.** It compiles for you and returns a string you can `IO.puts` without touching data structures. +- **Use `plan/2` for tooling.** It gives you the structs you can serialize, diff, or compare across program versions. +- **Attach telemetry once.** `[:ex_datalog, :planner, :stop]` carries the `:strategy` and the rules/relations counts — enough to build a dashboard without parsing the plan. +- **Don't expect magic sets to accelerate execution yet.** In v0.5.0 the strategy is recorded; the engine still runs semi-naive. The seam is in place so a future release can hook in goal-directed rewriting without changing call sites. + +The planner is small on purpose. The cost of getting the seam right now — descriptive structs, honest strategy selection, telemetry from day one — is low, and it pays back the moment a real optimizer needs somewhere to live. \ No newline at end of file diff --git a/docs/articles/07_aggregates_in_datalog.md b/docs/articles/07_aggregates_in_datalog.md new file mode 100644 index 0000000..859e64b --- /dev/null +++ b/docs/articles/07_aggregates_in_datalog.md @@ -0,0 +1,405 @@ +# Aggregates in Datalog: Group, Reduce, Stratify + +Pure Datalog derives facts by recursive joining: if `ancestor(X,Y)` holds whenever `parent(X,Y)` or `parent(X,Z)` and `ancestor(Z,Y)`, every answer falls out of the fixpoint. But many real questions are not about *what* is true — they are about *how many* or *how much*. "How many employees per department?" "What is the largest score in each group?" "What is the total salary paid to each team?" + +These are aggregate queries. Classical Datalog has no aggregates: the least-fixpoint semantics is defined over sets of facts, and adding "count" looks innocuous but breaks monotonicity — later derivations can change a count that an earlier rule already consumed. ExDatalog introduces aggregates as a constrained first-class feature: four integer-only operators, one aggregate per rule, no recursion through the aggregate, and a stratification pass that forces the aggregate's head relation to live strictly above every relation that feeds it. + +## Why Aggregates Are Not Just Another Constraint + +Comparison and arithmetic constraints are *per-binding*. Given a binding `%{"X" => 10, "Y" => 3}`, evaluating `gt(X, 5)` or `add(X, Y, Z)` is a local, stateless operation — the answer depends only on that one binding. The engine can fold them into the per-row constraint pipeline (`ConstraintEval.apply/3`) without caring about other bindings. + +Aggregates break that model. `count(E, N)` needs to see *all* employees in a department before it can pick `N`. There is no single binding that yields 2 — the value 2 only exists relative to a *group* of bindings. That is why `ExDatalog.Constraints.Aggregate.evaluate/3` does not compute anything: + +```elixir +@impl ExDatalog.Constraint +def evaluate(_constraint, _binding, _context) do + raise "aggregate constraints are not evaluated per-binding; " <> + "use ExDatalog.Engine.Evaluator group-and-reduce path" +end +``` + +The behaviour callback exists only so the aggregate module fits the same dispatch table as comparisons and arithmetic. Real evaluation happens later, in the engine, after the per-binding pipeline is done. This split is the central design decision behind aggregates in ExDatalog: a constraint declares the *intent* ("count `E` into `N`"), and the engine decides *when* to honor it. + +## The Four Operators + +ExDatalog ships four aggregate operators. They are integer-only — there is no `avg` yet — and live in the same `@aggregate_ops` list as the rest of the constraint categories: + +```elixir +@aggregate_ops [:count, :sum, :min, :max] +``` + +| Constructor | op | Semantics over a group's `input` values | +|---|---|---| +| `Constraint.count(input, result)` | `:count` | `length(values)` — group size | +| `Constraint.sum(input, result)` | `:sum` | `Enum.sum(values)` — integer total | +| `Constraint.min(input, result)` | `:min` | `Enum.min(values)` — smallest | +| `Constraint.max(input, result)` | `:max` | `Enum.max(values)` — largest | + +The reducer itself is a four-clause function in `ExDatalog.Constraints.Aggregate`: + +```elixir +defp compute(:count, values), do: length(values) +defp compute(:sum, values), do: Enum.sum(values) +defp compute(:min, values), do: Enum.min(values) +defp compute(:max, values), do: Enum.max(values) +``` + +`count` works on any group; `count` of employees is the size of the group regardless of the input variable's type. `sum`, `min`, and `max` require integer inputs. The plan is to enforce this at build time and guard at runtime; today an input list containing non-integers would surface as an Elixir runtime error from `Enum.sum/1` or `Enum.min/1`. Because aggregates are stratified above their inputs (see below), the input list is fully materialized and final before the reducer runs, so a runtime crash during reduction is a property of the data, not of evaluation order. + +## The DSL Syntax + +In the Schema DSL, aggregates appear in rule *bodies*, never in heads. The DSL expands `count(X, N)` in a body position into a `%Constraint{op: :count, left: {:var, "X"}, right: nil, result: {:var, "N"}}`. A typical rule: + +```elixir +rule dept_count(D, N) do + emp(E, D) + count(E, N) +end +``` + +Two compile-time safeguards prevent misuse: + +1. Aggregates are rejected in the rule head. `rule dept_count(D, count(E))` raises: + + ``` + ** (ExDatalog.DSL.CompileError) aggregate :count/2 cannot appear in a rule head + or as a term; place it in the rule body (e.g. `count(X, Result)`) and use + Result in the head + ``` + +2. The legacy `agg(:count, X)` form is rejected with a hint to use the named operator: + + ``` + ** (ExDatalog.DSL.CompileError) use count/sum/min/max for aggregates + (e.g. `count(X, N)`) + ``` + +Both checks live in `parse_term/1` and `parse_body_call/1`. The body form is generated from the same `[:count, :sum, :min, :max]` list used by the builder, so the DSL and the programmatic API cannot drift: + +```elixir +Enum.each(aggregate_ops, fn op -> + defp parse_body_call({unquote(op), _, [input, result]}) do + {:constraint, build_aggregate(unquote(op), input, result)} + end +end) +``` + +The result variable (`N` in `count(E, N)`) is the one that appears in the rule head. The input variable (`E`) must be bound by a positive body atom. Everything else about the rule looks ordinary — and that is the point: aggregates are *constraints* from the DSL's perspective, even though the engine treats them specially. + +## The Builder API + +Programs that bypass the DSL construct aggregate constraints directly. `ExDatalog.Constraint` exposes one constructor per operator: + +```elixir +iex> ExDatalog.Constraint.count({:var, "Emp"}, {:var, "N"}) +%ExDatalog.Constraint{op: :count, left: {:var, "Emp"}, right: nil, result: {:var, "N"}} + +iex> ExDatalog.Constraint.sum({:var, "Amount"}, {:var, "Total"}) +%ExDatalog.Constraint{op: :sum, left: {:var, "Amount"}, right: nil, result: {:var, "Total"}} + +iex> ExDatalog.Constraint.min({:var, "Score"}, {:var, "Lowest"}) +%ExDatalog.Constraint{op: :min, left: {:var, "Score"}, right: nil, result: {:var, "Lowest"}} + +iex> ExDatalog.Constraint.max({:var, "Score"}, {:var, "Highest"}) +%ExDatalog.Constraint{op: :max, left: {:var, "Score"}, right: nil, result: {:var, "Highest"}} +``` + +All four constructors route through the private `aggregate/3` helper, which is what distinguishes an aggregate structurally from a comparison: `right` is always `nil`, and `result` is always a `{:var, name}` tuple. The validity check enforces this: + +```elixir +defp valid_right?(op, nil) when op in @type_ops or op in @aggregate_ops, do: true + +defp valid_result?(op, {:var, name}) when op in @arithmetic_ops or op in @aggregate_ops, + do: is_binary(name) and byte_size(name) > 0 +``` + +The same shape is reachable through `Constraint.from_tuple/1`, which is what the DSL uses internally and what tests use to check the wire format: + +```elixir +iex> ExDatalog.Constraint.from_tuple({:count, {:var, "X"}, {:var, "N"}}) +%ExDatalog.Constraint{op: :count, left: {:var, "X"}, right: nil, result: {:var, "N"}} + +iex> ExDatalog.Constraint.from_tuple({:sum, :A, :T}) +%ExDatalog.Constraint{op: :sum, left: {:var, "A"}, right: nil, result: {:var, "T"}} +``` + +`from_tuple/1` reuses `Term.from/1`, so the Prolog convention applies: uppercase atoms become variables, lowercase atoms and other values become constants. + +A full aggregate rule built by hand — the same rule the DSL `dept_count` example compiles to — looks like: + +```elixir +Rule.new( + Atom.new("dept_count", [Term.var("D"), Term.var("N")]), + [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D")])}], + [Constraint.count(Term.var("E"), Term.var("N"))] +) +``` + +The aggregate lives in the `constraints` list alongside any comparisons; the engine distinguishes it by `op` and routes it to the group-and-reduce path. + +## Grouping Semantics + +The hard question for any aggregate is "group by what?" ExDatalog's answer is mechanical and unambiguous: **the group key is the set of head variables other than the aggregate result**. Concretely, for a rule with head `dept_count(D, N)` and aggregate `count(E, N)`, the result variable is `N`, so the group key is `[D]`. Every surviving binding for the rule is bucketed by its `D` value, and each non-empty bucket is reduced to a single output binding. + +The engine computes the group key in `aggregate_group_vars/2`: + +```elixir +defp aggregate_group_vars(%IR.Atom{terms: terms}, result_var) do + terms + |> Enum.flat_map(fn + {:var, name} -> [name] + _ -> [] + end) + |> Enum.reject(fn name -> name == result_var end) + |> Enum.uniq() +end +``` + +Constants in the head are ignored (they are not variables), and `result_var` is dropped because it is the *output* of the reduction, not a grouping column. If the head is `total(D, T)` and the constraint is `sum(A, T)`, the groups are keyed by `D`; if the head is `lowest(D, V)` with `min(S, V)`, the groups are also keyed by `D`. + +The reduction itself is a one-liner using `Enum.group_by/2`: + +```elixir +def group_and_reduce(bindings, group_vars, op, input_var, result_var) do + bindings + |> Enum.group_by(fn binding -> Map.take(binding, group_vars) end) + |> Enum.map(fn {_key, group} -> + values = Enum.map(group, fn b -> Map.fetch!(b, input_var) end) + Map.put(hd(group), result_var, compute(op, values)) + end) +end +``` + +A group only exists because at least one binding produced its key — so `Enum.min/1` and `Enum.max/1` are never called on an empty list. The output binding is the group's first binding extended with `result_var => computed_value`; the other columns of that binding are the surviving group key, which `Join.project/2` then reorganizes into the head tuple. + +## Safety Rules + +Aggregates are non-monotone: a department whose head count is 2 today might be 3 after more facts arrive, and a rule that already consumed "2" would be wrong. The safety validator (`ExDatalog.Validator.Safety`) rejects any program that could exhibit this. The aggregate-specific checks are layered on top of the ordinary variable-safety rules: + +**1. At most one aggregate per rule.** A rule may not mix `count` and `sum`: + +```elixir +# Rejected: two aggregates in one rule +Rule.new( + Atom.new("stats", [Term.var("D"), Term.var("N"), Term.var("T")]), + [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D"), Term.var("A")])}], + [Constraint.count(Term.var("E"), Term.var("N")), + Constraint.sum(Term.var("A"), Term.var("T"))] +) +# => {:error, [%Error{kind: :multiple_aggregates, ...}]} +``` + +The check is a single length test: + +```elixir +defp check_single_aggregate(errors, aggregates, _rule, rule_index) do + if length(aggregates) > 1 do + [Error.new(:multiple_aggregates, ..., "a rule may contain at most one " <> + "aggregate (found #{length(aggregates)}); split into separate rules") | errors] + else + errors + end +end +``` + +The fix is mechanical: split into two rules, one deriving `dept_count(D, N)` and one deriving `dept_total(D, T)`, both reading from the same `emp` relation. + +**2. No aggregate through a self-recursive relation.** A rule whose head relation also appears in its own body — positively or negatively — cannot aggregate: + +```elixir +# Rejected: path counts itself +Rule.new( + Atom.new("path", [Term.var("X"), Term.var("N")]), + [{:positive, Atom.new("path", [Term.var("X"), Term.var("Y")])}], + [Constraint.count(Term.var("Y"), Term.var("N"))] +) +# => {:error, [%Error{kind: :aggregate_in_recursion, ...}]} +``` + +The check inspects every body literal for a relation that matches the head: + +```elixir +self_recursive? = + Enum.any?(rule.body, fn + {:positive, %Atom{relation: ^head_rel}} -> true + {:negative, %Atom{relation: ^head_rel}} -> true + _ -> false + end) +``` + +This is a conservative, syntactic recursion check. Mutual recursion through a helper relation is not blocked here, but is caught by the stratification pass described next — the head relation will still be forced into a stratum that depends on itself, which the greedy assigner cannot satisfy without violating the aggregate-stratification invariant. + +**3. The aggregate input must be bound.** `count(Z, N)` where `Z` is never bound by a positive body atom is rejected by the ordinary "unbound constraint variable" check, the same path that catches `gt(W, 5)` with an unbound `W`. The aggregate is treated as a constraint with input variables `Constraint.input_variables/1` returns, which for an aggregate is just `[left]`: + +```elixir +def input_variables(%__MODULE__{left: left, right: nil}), do: Term.variables([left]) +``` + +**4. The result variable must appear in the head.** Because the result variable is computed by the reduction, it must be projected into the head to be observable at all. The head-safety check already allows an arithmetic *or* aggregate result variable to satisfy head safety: + +```elixir +result_vars = + Enum.flat_map(constraints, fn c -> + if Constraint.arithmetic?(c) or Constraint.aggregate?(c), + do: [Constraint.result_variable(c)], + else: [] + end) +``` + +So `Rule.head_variables(rule)` including `N` is satisfied by the `count(E, N)` constraint without `N` appearing in any positive body atom. If the head omits the result variable, the aggregate's value is silently dropped — there is no dedicated error for this, but the produced head tuple simply does not carry the computed value. + +## Stratification: Forcing the Aggregate Above Its Inputs + +Aggregates must be evaluated after every relation they read from is fully materialized. The stratification validator enforces this by running a fixpoint that bumps each aggregate rule's head relation until it sits strictly above all of its positive body relations: + +```elixir +defp stabilize_aggregate_strata(strata, aggregate_rules, graph, fuel) do + bumped = + Enum.reduce(aggregate_rules, strata, fn rule, acc -> + head_rel = rule.head.relation + body_rels = + for {:positive, %Atom{relation: rel}} <- rule.body, rel != head_rel, do: rel + + required = + case body_rels do + [] -> Map.get(acc, head_rel, 0) + _ -> (body_rels |> Enum.map(&Map.get(acc, &1, 0)) |> Enum.max()) + 1 + end + ... + end) + + bumped = propagate_strata(bumped, graph) + if bumped == strata, do: strata, else: stabilize_aggregate_strata(bumped, ...) +end +``` + +For `dept_count(D, N) :- emp(E, D), count(E, N)`, suppose `emp` is at stratum 0 (it is an EDB relation). The fixpoint bumps `dept_count` to `max(stratum(emp)) + 1 = 1`. If instead the aggregate reads a *derived* relation, such as + +```elixir +mid(K, V) :- base(K, V) +mid_count(K, N) :- mid(K, V), count(V, N) +``` + +then `mid` is at stratum 0 (it depends only on EDB `base`), and `mid_count` is bumped to stratum 1. The aggregate test suite asserts exactly this ordering: + +```elixir +mid_stratum = Enum.find(ir.rules, fn r -> r.head.relation == "mid" end).stratum +count_stratum = Enum.find(ir.rules, fn r -> r.head.relation == "mid_count" end).stratum +assert count_stratum > mid_stratum +``` + +Because the aggregate is forced above its inputs, by the time the engine's `eval_aggregate_rule/3` runs against the `full` view, every tuple it will join is already final. There is no half-materialized relation to worry about; the aggregate sees the answer set, not a work-in-progress. + +`propagate_strata/2` then re-runs the normal rule so that any relation depending on the aggregate inherits the bumped stratum. The outer fixpoint repeats until a full pass makes no change, with a fuel parameter to guarantee termination. + +## The Evaluation Pipeline + +Aggregates take a different path through the engine. `Engine.Evaluator.eval_rule_iteration/5` first asks `aggregate_rule?/1` — a body that contains any `{:constraint, %IR.Constraint{op: op}}` with `op` in `[:count, :sum, :min, :max]` makes the rule an aggregate rule: + +```elixir +if aggregate_rule?(rule) do + rule + |> eval_aggregate_rule(full, ctx) + |> MapSet.new() + |> MapSet.difference(existing) + |> MapSet.to_list() +else + eval_normal_rule(rule, full, delta, old, ctx, existing) +end +``` + +Notice what aggregate evaluation *does not* see: no `delta`, no `old`. Aggregates stratify above their inputs, so all source facts are already in `full`. There is no semi-naive delta to consume — the rule fires once, against the final view of its inputs, and the result is whatever the reduction produces. + +`eval_aggregate_rule/3` then runs the engine's full pre-grouping pipeline, in this order: + +```elixir +defp eval_aggregate_rule(%IR.Rule{} = rule, full, ctx) do + agg_constraint = find_aggregate(rule.body) + + joined = join_positive_body([%{}], rule.body, full) + filtered = apply_constraints(rule.body, joined, ctx) + with_cbs = apply_callbacks(rule.body, filtered, ctx) + bindings = apply_negation(rule.body, with_cbs, full) + + %IR.Constraint{op: op, left: {:var, input_var}, result: {:var, result_var}} = + agg_constraint + + group_vars = aggregate_group_vars(rule.head, result_var) + + bindings + |> Aggregate.group_and_reduce(group_vars, op, input_var, result_var) + |> Enum.map(&Join.project(rule.head, &1)) +end +``` + +The pipeline is: + +1. **Join** — `join_positive_body/3` starts from `[%{}]` and folds each positive body atom against the `full` view, producing every binding consistent with the body. +2. **Filter** — `apply_constraints/3` runs every *non-aggregate* constraint per binding. Aggregate constraints are explicitly skipped here: + + ```elixir + constraints = + for {:constraint, %IR.Constraint{op: op} = c} <- body, + op not in [:count, :sum, :min, :max], + do: c + ``` + + This is why a rule can combine `gte(S, 60)` with `count(E, N)` — the comparison filters bindings down to passing scores *before* the count runs. The aggregate test exercises exactly this: + + ```elixir + Rule.new( + Atom.new("passing_count", [Term.var("D"), Term.var("N")]), + [{:positive, Atom.new("score", [Term.var("E"), Term.var("D"), Term.var("S")])}], + [Constraint.gte(Term.var("S"), Term.const(60)), + Constraint.count(Term.var("E"), Term.var("N"))] + ) + # => {:eng, 2} (alice with 90 and carol with 75 pass; bob with 40 is filtered) + ``` + +3. **Callbacks** — `apply_callbacks/3` runs any BEAM predicate literals, the same way as for non-aggregate rules. +4. **Negation** — `apply_negation/3` filters bindings whose negated body atoms are contradicted by a tuple in `full`, exactly as in the negation pipeline. +5. **Group and reduce** — `Aggregate.group_and_reduce/5` buckets the surviving bindings by `group_vars` and reduces each bucket's `input_var` values with `op`, producing one extended binding per non-empty group. +6. **Project** — `Join.project/2` extracts the head's variables from each extended binding into the final head tuple. + +The ordering matters: filtering and negation happen *before* grouping, so the aggregate sees only the bindings that survived all per-row predicates. There is no "HAVING" clause — a filter *after* the aggregate would need the result variable to be in scope, which means a follow-up rule reading the aggregate relation. Stratification makes that composition natural: the aggregate relation lives in a higher stratum, so a subsequent rule in an even higher stratum can filter on its results. + +## Putting It Together: A Full Example + +The test suite's `count_program/0` is a complete, runnable specification of count: + +```elixir +defp count_program do + Program.new() + |> Program.add_relation("emp", [:atom, :atom]) + |> Program.add_relation("dept_count", [:atom, :integer]) + |> Program.add_fact("emp", [:alice, :eng]) + |> Program.add_fact("emp", [:bob, :eng]) + |> Program.add_fact("emp", [:carol, :ops]) + |> Program.add_rule( + Rule.new( + Atom.new("dept_count", [Term.var("D"), Term.var("N")]), + [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D")])}], + [Constraint.count(Term.var("E"), Term.var("N"))] + ) + ) +end +``` + +After materialization: + +```elixir +{:ok, knowledge} = ExDatalog.materialize(count_program()) +Knowledge.get(knowledge, "dept_count") +# => MapSet containing {:eng, 2} and {:ops, 1} +``` + +The same shape, with `Constraint.sum/2`, `Constraint.min/2`, or `Constraint.max/2`, produces `dept_total`, `lowest`, or `highest` respectively — only the reducer changes. The grouping, stratification, and pipeline are identical. + +## What Is *Not* Here Yet + +- **`avg`** — would require a paired `count` and `sum` or rational arithmetic; deferred until integer arithmetic with `div` is deemed stable enough to define `avg` as `div(sum, count)`. +- **Multiple aggregates per rule** — explicitly rejected today. Lifting the restriction means defining a multi-reducer grouping protocol; for now, split into separate rules. +- **Aggregates in recursive rules** — rejected by `:aggregate_in_recursion`. General "monotone aggregates" (e.g. `min` over a lattice) are a research topic and not on the near roadmap. +- **HAVING-style post-aggregate filters** — expressed today by chaining a second rule in a higher stratum that reads the aggregate relation; a dedicated syntax would be a convenience, not a capability gap. + +Aggregates are the first feature in ExDatalog that requires the engine to break out of the per-binding loop and look at a whole set of bindings at once. The design keeps that break localized — one new code path in `eval_aggregate_rule/3`, one new module `Constraints.Aggregate`, and one new fixpoint in `force_aggregate_strata/2` — while the rest of the engine, the validator, and the DSL continue to treat aggregates as just another kind of constraint. \ No newline at end of file diff --git a/docs/articles/08_extending_datalog_with_beam_callbacks.md b/docs/articles/08_extending_datalog_with_beam_callbacks.md new file mode 100644 index 0000000..2f853eb --- /dev/null +++ b/docs/articles/08_extending_datalog_with_beam_callbacks.md @@ -0,0 +1,410 @@ +# Extending Datalog with BEAM Callbacks: Calling Elixir from Rule Bodies + +Datalog's expressive power lives in joins: "find every `X` related to `Y` through `Z`." But real programs need more than relational joins — they need to call domain logic. "Is this user an adult?", "double this value", "does this email contain an `@`?" These are computations, not relations, and encoding them as base facts blows up the fact tables. + +ExDatalog bridges this with **BEAM callback predicates**: ordinary Elixir functions invoked during rule evaluation, treated as first-class body literals alongside positive and negative atoms. A callback appears in a rule body, sees the current variable binding, and either filters it (boolean) or extends it (value). The engine isolates each call with a monitored process and a timeout, so a misbehaving function never kills the evaluator. + +## Why Call Elixir from Datalog? + +Pure Datalog computes relations from relations. Anything that isn't a stored fact must be expressed as a rule, and rules can only combine existing relations. This is the source of Datalog's guarantees — termination, determinism, decidable safety — but it also boxes the language in. A rule like: + +```elixir +rule high_earner(P) do + income(P, S) + gt(S, 100_000) +end +``` + +works because `gt` is a built-in constraint. But what if the threshold depends on the person's region? Or what if "high earner" means "above the 90th percentile for their locale"? That's domain logic — it lives in Elixir, not in the fact store. + +The classic escape hatch in Datalog is *built-in predicates*: a fixed set of distinguished relations (`>`, `<`, `+`, ...) with special evaluation. ExDatalog already has those — the constraint DSL — but they're hard-coded into the engine. BEAM callbacks generalize the idea: any Elixir function `Mod.fun/arity` can act as a built-in, declared per-program, evaluated against the live binding. + +The trade-off is a contract: the function must be **deterministic** and **side-effect free**. The engine cannot verify this — it's the caller's responsibility — but it can, and does, isolate the call so that a violation (a crash, a hang, a throw) degrades gracefully to a filtered binding rather than a crashed evaluator. + +## Boolean vs. Value Callbacks + +A callback has two flavours, distinguished by its `result` field: + +- **Boolean** (`result: nil`) — the function returns `true` or `false`. `true` keeps the binding; `false` (or a non-boolean, a timeout, or an exception) drops it. This is a *filter*: it never introduces new variables. +- **Value** (`result: {:var, name}`) — the function returns a value, which is bound to `name` and added to the environment. This is a *binder*: like an arithmetic constraint, it extends the binding with a new variable the rule head can project. + +The two map onto different problem shapes. A boolean callback answers "does this binding pass a predicate?": + +```elixir +rule adult(Name) do + person(Name, Age) + adult?(Age) # boolean callback: Age >= 18 +end +``` + +A value callback answers "compute something from this binding": + +```elixir +rule doubled(X, Y) do + num(X) + double(X, Y) # value callback: Y = X * 2 +end +``` + +In the value case, `Y` is introduced by the callback and projected into the head — exactly how arithmetic constraints like `add(A, B, Z)` work. The safety rules treat the two categories symmetrically: a callback's **inputs** must be bound before the call, and its **result** (if any) is available to the head and to later constraints. + +## The DSL: `predicate/5` + +The Schema DSL exposes callbacks through the `predicate/5` macro, declared alongside relations: + +```elixir +defmodule FamilyRules do + use ExDatalog.Schema + + relation :person do + field(:name, :atom) + field(:age, :integer) + end + + relation :adult do + field(:name, :atom) + end + + predicate :adult?, MyPredicates, :adult?, [:integer], :boolean + + fact person(:alice, 30) + fact person(:bob, 10) + + rule adult(Name) do + person(Name, Age) + adult?(Age) + end +end +``` + +The macro stores a `ExDatalog.Schema.PredicateMeta` struct — name, module, function, argument types, and return type (`:boolean` or `:value`). When the rule body is parsed, any positive atom whose relation matches a declared predicate is *rewritten* into a callback literal rather than a relational atom: + +```elixir +defp resolve_positive_or_callback(%ExDatalog.Atom{relation: rel, terms: terms}, predicate_map) do + case Map.get(predicate_map, rel) do + nil -> + {:positive, %ExDatalog.Atom{relation: rel, terms: Enum.map(terms, &term_from_parsed/1)}} + + %ExDatalog.Schema.PredicateMeta{} = pred -> + build_callback_literal(pred, terms) + end +end +``` + +For `:value` predicates, the **last** argument position in the rule-body call is the result variable; the rest are passed to the function: + +```elixir +defp build_callback_literal(%ExDatalog.Schema.PredicateMeta{} = pred, terms) do + arg_terms = Enum.map(terms, &term_from_parsed/1) + + case pred.return_type do + :boolean -> + {:callback, ExDatalog.Callback.new(pred.module, pred.function, arg_terms, nil)} + + :value -> + {args, [result]} = Enum.split(arg_terms, length(arg_terms) - 1) + {:callback, ExDatalog.Callback.new(pred.module, pred.function, args, result)} + end +end +``` + +This split mirrors how arithmetic constraints separate inputs from the result slot: `add(A, B, Z)` takes `A` and `B` as inputs and binds `Z`. A value predicate `double(X, Y)` takes `X` as input and binds `Y`. + +Argument types are currently informational — their length sets the expected arity, validated against the exported function. The runtime type of each argument is whatever the binding holds. + +## The Builder API + +Programs built without the DSL construct callbacks directly with `ExDatalog.Callback.new/4` and place them in the rule body as `{:callback, %Callback{}}`: + +```elixir +program = + Program.new() + |> Program.add_relation("person", [:atom, :integer]) + |> Program.add_relation("adult", [:atom]) + |> Program.add_fact("person", [:alice, 25]) + |> Program.add_fact("person", [:bob, 12]) + |> Program.add_rule( + Rule.new( + Atom.new("adult", [Term.var("Name")]), + [ + {:positive, Atom.new("person", [Term.var("Name"), Term.var("Age")])}, + {:callback, Callback.new(Predicates, :adult?, [Term.var("Age")])} + ] + ) + ) +``` + +The `Callback` struct is intentionally minimal: + +```elixir +@enforce_keys [:module, :function, :args] +defstruct [:module, :function, :args, :result] +``` + +- `module` / `function` — the Elixir function to call. +- `args` — a list of `ExDatalog.Term.t()` (variables or constants) resolved against the binding at evaluation time and passed positionally. +- `result` — `nil` for a boolean filter, or `{:var, name}` for a value-binding callback. + +Two helpers verbalize the callback's variable footprint: + +```elixir +@spec input_variables(t()) :: [String.t()] +def input_variables(%__MODULE__{args: args}), do: Term.variables(args) + +@spec result_variable(t()) :: String.t() | nil +def result_variable(%__MODULE__{result: {:var, name}}), do: name +def result_variable(%__MODULE__{result: _}), do: nil +``` + +`input_variables/1` is the input to the safety checker; `result_variable/1` is what the head-safety check counts as a bound variable. + +## Isolation: `spawn_monitor` and the Timeout + +A callback is an arbitrary Elixir function. It might loop forever. It might raise. It might `exit`. It might link to another process that dies. The engine cannot trust it — it must isolate it. + +`ExDatalog.Constraints.BeamCallback` runs every call in a freshly spawned, monitored process: + +```elixir +defp safe_apply(module, function, args, timeout_ms) do + parent = self() + ref = make_ref() + + {pid, monitor_ref} = + spawn_monitor(fn -> + result = + try do + {:ok, apply(module, function, args)} + rescue + e -> {:error, e} + catch + kind, reason -> {:error, {kind, reason}} + end + + send(parent, {ref, result}) + end) + + receive do + {^ref, result} -> + Process.demonitor(monitor_ref, [:flush]) + result + + {:DOWN, ^monitor_ref, :process, ^pid, reason} -> + {:error, reason} + after + timeout_ms -> + Process.exit(pid, :kill) + Process.demonitor(monitor_ref, [:flush]) + {:error, :timeout} + end +end +``` + +There are three failure modes, all folded into a single `{:error, _}` → `:filter` outcome: + +1. **The function returns normally** — `{:ok, value}` is sent back and the binding is kept (boolean true) or extended (value bound). +2. **The function raises or exits** — the `try/rescue/catch` inside the spawned process converts it to `{:error, e}` and the binding is filtered. +3. **The function takes too long** — the `after` clause fires, kills the process, and reports `{:error, :timeout}`. + +Because the spawned process is **unlinked** and **monitored**, a crash inside it produces a `:DOWN` message — not a linked exit propagating to the evaluator. The monitor is flushed after the call so no stray `:DOWN` ever reaches the parent's mailbox. + +The timeout is configurable per materialization via `:callback_timeout_ms` (default 100ms): + +```elixir +{:ok, knowledge} = ExDatalog.materialize(program, callback_timeout_ms: 50) +``` + +The same option propagates through the `ExDatalog.Constraint.Context` carried by the evaluator, so callbacks inside rule bodies see the same deadline. + +### Why `spawn_monitor`, Not `Task.async`? + +`Task.async` links the task to the caller. A `raise` inside the task propagates as an exit and crashes the evaluator — the opposite of what we want. `Task.Supervisor` adds supervision overhead and a separate process tree for what is a single synchronous call. `spawn_monitor` is the lightest primitive that gives both crash isolation (no link) and death notification (the `:DOWN` message), without needing a supervisor. + +### Why a Timeout, Not Cancellation? + +Cancellation requires cooperative cancellation — the spawned function would have to check a flag. The engine has no control over the function's body, so cancellation is impossible in general. The timeout, by contrast, is enforced *from outside*: the evaluator stops waiting, kills the process, and moves on. The killed process may still be running its `after` cleanup, but it's disconnected from the evaluator. This trades precision for robustness: it always terminates the wait, even if the function is stuck in a NIF or a tight loop. + +## Safety: Inputs Must Be Bound + +A callback's arguments are resolved against the current binding. If an input variable isn't bound when the callback fires, there's nothing to pass — the call is ill-defined. ExDatalog rejects this at validation time in `Validator.Safety.check_callback_inputs/3`: + +```elixir +defp check_callback_inputs(errors, %Rule{body: body}, body_bound, rule_index) do + callbacks = for {:callback, cb} <- body, do: cb + + Enum.reduce(callbacks, errors, fn cb, acc -> + unbound = Enum.reject(ExDatalog.Callback.input_variables(cb), fn v -> v in body_bound end) + + if unbound == [] do + acc + else + [ + Error.new( + :unbound_constraint_variable, + %{rule_index: rule_index, variables: unbound, callback: {cb.module, cb.function}}, + "rule #{rule_index}: callback #{inspect(cb.module)}.#{cb.function} " <> + "references unbound variable(s) #{Enum.join(unbound, ", ")}" + | acc + ] + ] + end + end) +end +``` + +`body_bound` is the set of variables bound by positive body atoms. A callback may consume those, plus any arithmetic or value-callback results that precede it — but it cannot introduce a variable for itself to read. This is the same range-restriction rule that applies to constraints and negated atoms: + +```elixir +# Safe: Age is bound by person/2 before adult?/1 fires +rule adult(Name) do + person(Name, Age) + adult?(Age) +end + +# Unsafe: Z is never bound +rule bad(X) do + person(X, Age) + adult?(Z) # ERROR: Z is unbound +end +``` + +A value callback's result variable *is* safe to use in the head, because if the rule fires at all the callback will have run: + +```elixir +# Safe: Y is bound by the value callback double/1 +rule doubled(X, Y) do + num(X) + double(X, Y) +end +``` + +The head-safety checker (`all_bound_variables/1`) collects callback result variables alongside arithmetic results, so a head term may reference either. + +## Compile-Time Module/Function Validation + +The structural validator (`Validator.check_callback/3`) checks that the named module is loaded and exports the function at the expected arity: + +```elixir +defp check_callback(errors, %ExDatalog.Callback{module: m, function: f, args: args}, context) do + arity = length(args) + + if Code.ensure_loaded?(m) and function_exported?(m, f, arity) do + errors + else + [ + Error.new( + :invalid_callback, + Map.merge(context, %{module: m, function: f, arity: arity}), + "#{location(context)} callback #{inspect(m)}.#{f}/#{arity} " <> + "is not defined or not exported" + | errors + ) + ] + end +end +``` + +`Code.ensure_loaded?/1` forces the module to be compiled (if in the same VM) and confirms it exists; `function_exported?/3` checks the function clause. The check runs as part of `Validator.validate/1`, which is called from `ExDatalog.materialize/2`. A program that references `DoesNotExist.foo/1` is rejected before evaluation begins: + +```elixir +assert {:error, errors} = ExDatalog.materialize(program) +assert Enum.any?(errors, fn e -> e.kind == :invalid_callback end) +``` + +A callback that passes the arity check but misbehaves at runtime (raises, times out) is *not* a validation error — it's handled by the isolation machinery and filtered out of the result. + +## The Evaluation Pipeline + +Callbacks slot into a fixed position in the per-binding pipeline. In `Engine.Evaluator.finish_bindings/4`, each binding produced by the positive-body join passes through four stages in order: + +```elixir +defp finish_bindings(bindings, rule, full, ctx) do + bindings = apply_constraints(rule.body, bindings, ctx) + bindings = apply_callbacks(rule.body, bindings, ctx) + bindings = apply_negation(rule.body, bindings, full) + + case bindings do + [] -> [] + _ -> Enum.map(bindings, &Join.project(rule.head, &1)) + end +end +``` + +1. **Constraints** — comparisons filter; arithmetic extends the binding. +2. **Callbacks** — boolean callbacks filter; value callbacks extend the binding. +3. **Negation** — surviving bindings are checked against the fully-materialized lower-stratum relations. +4. **Projection** — the head is projected from each surviving binding. + +The ordering matters. Constraints run first because they're cheap and pure — a `gt` comparison is faster and more reliable than an arbitrary Elixir function, so filtering with `gt` *before* calling a callback avoids wasted calls. Callbacks run before negation because callbacks may bind the result variables that negated atoms reference: + +```elixir +rule verified_adult(Name) do + person(Name, Age) + adult?(Age) + id_for(Name, Id) # value callback binds Id + not_ banned(Id) # negation uses Id +end +``` + +If callbacks ran after negation, `Id` wouldn't be bound when `not_ banned(Id)` was checked, and the negated atom would either fail safety or match incorrectly. + +Within the callback stage, multiple callbacks in the same rule are chained in listed order through `apply_callback_chain/3`: + +```elixir +defp apply_callback_chain(callbacks, binding, opts) do + Enum.reduce_while(callbacks, [binding], fn cb, [b] -> + step_callback(BeamCallback.apply_callback(cb, b, opts)) + end) +end + +defp step_callback({:ok, new_b}), do: {:cont, [new_b]} +defp step_callback(:filter), do: {:halt, []} +``` + +A `:filter` short-circuits the rest of the chain — the binding is dropped, no later callback in that rule fires. A successful value callback extends the binding and the next callback sees the new variable. + +## The Contract: Deterministic and Side-Effect Free + +The engine enforces isolation (timeout, exception handling, monitored processes). It cannot enforce purity. A callback that: + +- reads the current time, +- queries a database, +- sends a message, +- mutates a process dictionary, +- depends on application state, + +will produce results that depend on *when* the rule fires, not just *what* the binding contains. In a semi-naive evaluator the same rule fires repeatedly across fixpoint iterations; a non-deterministic callback can break monotonicity, violate the fixpoint, and produce different results across runs. + +The contract — stated in `ExDatalog.Callback`'s moduledoc — is: + +> A callback **must** be: +> +> - **Deterministic** — the same arguments always produce the same result. +> - **Side-effect free** — no I/O, mutation, or messaging. + +These are caller contracts, not engine-enforced. The engine's isolation layer is the safety net for *accidents* — a stack overflow, a misbehaving dependency — not for *intent*. A program that deliberately calls `DateTime.utc_now/0` from a callback will run; its result will simply be unreliable. + +## Limitations and Design Choices + +BEAM callbacks are the most permissive feature in ExDatalog, and several limits reflect that: + +- **No type-level enforcement of arguments.** `arg_types` in `predicate/5` is informational. The validator checks the function exists with the right arity, not that the binding's runtime values match the declared types. Type checking would require either a runtime check (slow, and Elixir doesn't have value types) or a static analysis pass over the program — both out of scope for v0.4. + +- **No cancellation.** As discussed above, the timeout is a wait-side deadline, not a cooperative cancel. A callback stuck in a NIF cannot be interrupted. + +- **No parallelism.** Callbacks in a single rule run sequentially, in listed order, threaded through `apply_callback_chain`. Parallel evaluation would break ordering-dependent value callbacks (`a(X) -> double(X, Y) -> next(Y, Z)` is sequential by construction). + +- **No memoization.** The same callback called with the same arguments in two different iterations of a fixpoint loop runs twice. A deterministic, side-effect-free function gives the same answer each time, but the cost is paid again. Memoization would require a per-program cache with eviction, which complicates the stateless evaluation model. + +- **One result variable.** A value callback binds exactly one variable — the last argument position in the DSL call. A function returning `{a, b}` cannot bind two variables directly; the rule must follow up with a constraint or a second callback to decompose the tuple. + +- **Callbacks are not relations.** They participate in the safety checks and the evaluation pipeline, but they do not appear in the dependency graph. A callback cannot be the head of a rule, cannot be negated, and cannot be recursive. They are leaves in the stratification order — by construction, since their "relations" are Elixir functions, not Datalog predicates. + +These limits keep the feature small and the guarantees intact. A callback that can be statically typed, cancelled, parallelised, and memoized is a different abstraction — closer to a foreign-function interface than to a Datalog literal. ExDatalog's callbacks stay close to the spirit of built-in predicates: an open set of pure functions, grafted onto the engine with a thin isolation layer, paying their way with the work they save in fact-table size. + +## What's Coming in v0.5.0 + +- **Aggregates** — `count`, `sum`, `min`, and `max` over a relation, stratified above their inputs. Aggregates share the value-binding shape with value callbacks (a result variable introduced into the binding) but are evaluated by group-and-reduce over the materialized relation rather than per-binding. +- **Magic sets / demand-driven evaluation** — goal-directed evaluation that computes only facts relevant to a query, instead of the full fixpoint. +- **Static callback purity analysis** — an optional lint pass that flags callbacks whose module is known to perform I/O (e.g. modules implementing `GenServer` behaviour), to surface contract violations before runtime. + +These features will expand what's expressible while preserving Datalog's guarantees: termination, deterministic output, and compile-time validation. \ No newline at end of file diff --git a/docs/articles/09_magic_sets_and_demand_driven_evaluation.md b/docs/articles/09_magic_sets_and_demand_driven_evaluation.md new file mode 100644 index 0000000..340ddc9 --- /dev/null +++ b/docs/articles/09_magic_sets_and_demand_driven_evaluation.md @@ -0,0 +1,312 @@ +# Magic Sets and Demand-Driven Evaluation: Goal-Directed Datalog in ExDatalog + +Semi-naive evaluation computes the *full* least fixpoint of a program: every derivable fact in every relation, regardless of whether anyone will ever read it. That is the right default when the materialised knowledge is reused for many queries, but it is wasteful when a single question is asked against a large dataset. "Who are the ancestors of *Alice*?" should not require deriving the ancestors of *every* person in the database first. + +The **magic sets** transformation — a classic technique from deductive database theory — bridges the gap between bottom-up fixpoint evaluation and top-down goal-directed query answering. It rewrites the program so that bottom-up evaluation only derives facts relevant to a specific query goal, while keeping the engine itself untouched. ExDatalog implements it in `ExDatalog.MagicSets` as an experimental v0.5.0 feature reached through `materialize/2`'s `:strategy` option. + +## Why Demand-Driven Evaluation Matters + +Consider the ancestor program: + +```elixir +rule ancestor(X, Y) do + parent(X, Y) +end + +rule ancestor(X, Z) do + parent(X, Y) + ancestor(Y, Z) +end +``` + +Semi-naive evaluation derives `ancestor/2` for *every* pair connected through `parent/2`. If the dataset has ten thousand people, the engine produces ten thousand ancestors tuples. But the query "ancestors of Alice" only needs the small slice `ancestor(alice, *)`. A top-down Prolog-style solver would discover that slice directly; a pure bottom-up engine cannot — it has no notion of "query." + +Magic sets fixes this by *rephrasing the program* so the existing bottom-up engine derives only the requested slice. No new evaluation algorithm, no special query interpreter. The transformation introduces an auxiliary **magic** predicate that records "which bindings are we interested in?" and threads that information through every recursive rule. + +## The Transform: Adornment → Magic Predicates → Seeds → Rewriting + +The transformation in `ExDatalog.MagicSets.transform/2` proceeds in four stages. + +### 1. Adornment + +Given a goal `{relation, pattern}`, the pattern marks each argument position as **bound** (`b`) or **free** (`f`). The pattern `[:a, :_]` becomes the adornment string `"bf"`: the first position is bound to a constant, the second is unbound. + +```elixir +defp adornment(pattern) do + Enum.map_join(pattern, "", fn + :_ -> "f" + _ -> "b" + end) +end +``` + +The adornment is the *signature* of the demand: every derived fact of the goal relation must agree with the bound positions. + +### 2. Magic Predicates + +The transformation synthesises a new relation named `magic__` — for example `magic_ancestor_bf` — whose arity is the number of bound positions. Its tuples record the *values we are asking about*. A tuple `(alice,)` in `magic_ancestor_bf` means "compute `ancestor` facts whose first argument is `alice`." + +The relation is registered in the IR so the engine allocates storage for it, and its type signature is projected from the original relation's declared types: + +```elixir +magic_relation = %IR.Relation{ + name: magic_rel, + arity: bound_count(goal_pattern), + types: bound_types(ir, goal_relation, goal_pattern) +} +new_relations = [magic_relation | ir.relations] +``` + +### 3. Seed Facts + +The bound constants of the goal pattern are inserted as the initial magic facts — the "seeds" of demand. For `goal: {"ancestor", [:a, :]}` the seed is the single fact `magic_ancestor_bf(a)`: + +```elixir +defp seed_fact(magic_rel, _goal_relation, pattern) do + bound_values = + pattern + |> Enum.filter(fn p -> p != :_ end) + |> Enum.map(&to_ir_value/1) + + case bound_values do + [] -> nil + values -> %IR.Fact{relation: magic_rel, values: values} + end +end +``` + +Without a seed, the magic relation is empty and no goal-relation rule can fire, so nothing is derived. The seed is what kicks off demand propagation. + +### 4. Rule Rewriting + +Every rule whose head is the goal relation is rewritten to *consume* the magic predicate as its first body atom. Only bindings consistent with an existing magic tuple are allowed to proceed. For the recursive ancestor rule this means: + +``` +ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z). +``` +becomes +``` +ancestor(X, Z) :- magic_ancestor_bf(X), parent(X, Y), ancestor(Y, Z). +``` + +The implementation prepends the magic atom: + +```elixir +defp rewrite_rule(%IR.Rule{head: %IR.Atom{relation: rel} = head} = rule, goal_relation, magic_rel, bound_positions) + when rel == goal_relation do + magic_terms = bound_head_terms(head, bound_positions) + magic_atom = %IR.Atom{relation: magic_rel, terms: magic_terms} + %IR.Rule{rule | body: [{:positive, magic_atom} | rule.body]} +end +``` + +The magic atom projects exactly the goal's bound positions of the head, so its arity matches the magic relation. Rules whose head is *not* the goal relation are left untouched. + +Finally, the magic relation is injected into the goal relation's stratum so the stratified evaluator processes them in the same pass: + +```elixir +defp inject_magic_into_strata(strata, goal_relation, magic_rel) do + Enum.map(strata, fn %IR.Stratum{relations: rels} = stratum -> + if goal_relation in rels do + %IR.Stratum{stratum | relations: [magic_rel | rels]} + else + stratum + end + end) +end +``` + +The transformed IR is structurally a normal Datalog program. It is handed back to the existing semi-naive engine unchanged. Magic sets is a *program rewrite*, not a new evaluator. + +## Goal-Driven Evaluation + +The user-facing entry point is `ExDatalog.materialize/2` with the `:strategy` and `:goal` options: + +```elixir +{:ok, magic} = + ExDatalog.materialize(program, + strategy: :magic_sets, + goal: {"ancestor", [:a, :_]} + ) + +Knowledge.match(magic, "ancestor", [:a, :_]) +``` + +The dispatch lives in `Engine.Naive.evaluate/2`, which switches on the `:strategy` option: + +```elixir +case Keyword.get(opts, :strategy, :semi_naive) do + :semi_naive -> + evaluate_semi_naive(ir, opts) + + :magic_sets -> + evaluate_magic_sets(ir, opts) +end +``` + +The `:magic_sets` path requires a `:goal`. If absent, the engine falls back to plain semi-naive evaluation — there is no demand to drive without a goal: + +```elixir +defp evaluate_magic_sets(%IR{} = ir, opts) do + case Keyword.get(opts, :goal, nil) do + nil -> + evaluate_semi_naive(ir, opts) + + goal -> + case ExDatalog.MagicSets.transform(ir, goal) do + {:ok, transformed_ir} -> + evaluate_semi_naive(transformed_ir, Keyword.delete(opts, :strategy)) + + {:fallback, _reason} -> + evaluate_semi_naive(ir, opts) + end + end +end +``` + +When the transform succeeds, the **original** IR is discarded: only the transformed IR is evaluated. When the transform declines (`{:fallback, _}`), the unmodified IR is evaluated with semi-naive, so the query still produces a correct answer — just without the demand-restriction savings. + +The `Planner` records the chosen strategy and goal in its `Plan` struct for telemetry and introspection, but the rewrite itself happens inside the engine, not the planner. The planner is the seam where cost-based strategy selection will eventually live. + +## Bound Positions + +The goal pattern's bound positions are what make demand restriction possible. `[:a, :_]` binds position 0 to the atom `:a` and leaves position 1 free. The transformation: + +- names the magic predicate using the adornment `bf`, +- gives it arity 1 (one bound position), +- seeds it with `(a,)`, +- and rewrites each goal-relation rule to join `magic_ancestor_bf()` before any other body atom. + +A pattern with both positions bound — `[:a, :b]` — would produce `magic_ancestor_bb` of arity 2, seeded with `(a, b)`, restricting derivations to the single pair. A pattern with all positions free — `[:_, :_]` — has no bound positions, so `has_bound_position?/1` returns false and the transform declines: + +```elixir +defp has_bound_position?(pattern) do + Enum.any?(pattern, fn p -> p != :_ end) +end +``` + +An all-free goal offers no demand to exploit; the transform would generate an empty magic predicate that blocks every derivation. Falling back to semi-naive is the only sensible choice. + +## Scope: Positive Recursive Programs, Single Goal, Ground Bounds + +The v0.5.0 implementation deliberately limits its scope. `transform/2` applies only when all three of these hold: + +1. **No negation or aggregates in any rule.** The check is whole-program, not per-rule: + ```elixir + defp supported_program?(%IR{rules: rules}) do + Enum.all?(rules, fn rule -> + not has_negation?(rule) and not has_aggregate?(rule) + end) + end + ``` + Negation introduces stratification dependencies that the magic-relation stratum injection does not yet model; aggregates require grouping semantics that the rewrite does not preserve. Any rule with either causes `{:fallback, :unsupported_program}`. + +2. **A single goal.** `transform/2` accepts one `{relation, pattern}`. Multi-goal demand propagation (sideways information passing across several queries) is future work. + +3. **Ground bound positions.** The pattern's bound entries must be constants (`:a`, `"alice"`, `42`). Variables or computed bindings in the pattern are not supported; `to_ir_value/1` handles only integers, binaries, and atoms. + +Anything outside this scope is rejected up front by the `cond` in `transform/2`, before any rewriting begins. + +## Fallback: Safe by Construction + +Falling back is not an error condition — it is the design's safety net. The engine treats `{:fallback, reason}` as a directive to evaluate the *original* IR with semi-naive, which always produces the correct least fixpoint. The `reason` is currently discarded by the engine; the transform's return type is the only record of *why* demand restriction was declined. + +The tests make the fallback semantics explicit: + +```elixir +test "falls back to semi-naive when no goal is given" do + program = ancestor_program(@chain) + {:ok, magic} = ExDatalog.materialize(program, strategy: :magic_sets) + {:ok, full} = ExDatalog.materialize(program) + assert Knowledge.get(magic, "ancestor") == Knowledge.get(full, "ancestor") +end +``` + +A program containing negation falls back the same way, and the result still matches full evaluation: + +```elixir +{:ok, magic} = + ExDatalog.materialize(program, strategy: :magic_sets, goal: {"childless", [:_]}) + +{:ok, full} = ExDatalog.materialize(program) +assert Knowledge.get(magic, "childless") == Knowledge.get(full, "childless") +``` + +Because the fallback path is the same engine that handles `:semi_naive`, there is no way for the `:magic_sets` strategy to produce a *wrong* answer — only a slower one. + +## Correctness: Magic ≡ Semi-Naive for Goal Results + +The central correctness property is that the goal-restricted result returned by magic sets equals the goal-restricted subset of the full semi-naive fixpoint: + +``` +magic_result(goal) == filter(semi_naive_result, goal) +``` + +Note this is *not* equality of the full knowledge base. The magic-transformed program will derive fewer facts in non-goal relations (and even in the goal relation beyond what the goal pattern admits — that is why `Knowledge.match/3` is used to project the final result). The guarantee is scoped to the goal. + +The property test in `magic_sets_property_test.exs` checks this for randomly generated graphs and sources: + +```elixir +property "magic-sets result equals the semi-naive subset for the goal" do + check all( + edges <- list_of(edge_gen(), max_length: 10), + source <- member_of(@nodes) + ) do + program = build_program(edges) + + {:ok, full} = ExDatalog.materialize(program) + expected = MapSet.filter(Knowledge.get(full, "path"), fn {x, _y} -> x == source end) + + {:ok, magic} = + ExDatalog.materialize(program, strategy: :magic_sets, goal: {"path", [source, :_]}) + + actual = Knowledge.match(magic, "path", [source, :_]) + + assert actual == expected + end +end +``` + +Because the property is checked over arbitrary edge sets (including cyclic ones, disconnected ones, and empty ones), it provides strong evidence that the rewrite preserves goal semantics for the supported program class. The unit tests verify the structural shape of the transform directly — that `magic_ancestor_bf/1` exists, the seed fact `(a,)` is present, and every rewritten ancestor rule begins with the magic atom: + +```elixir +assert Enum.all?(rewritten, fn r -> + match?([{:positive, %ExDatalog.IR.Atom{relation: "magic_ancestor_bf"}} | _], r.body) + end) +``` + +## Example: Ancestors of Alice vs. All Ancestors + +Using the chain `parent(a,b), parent(b,c), parent(c,d), parent(d,e)`: + +```elixir +@chain [{:a, :b}, {:b, :c}, {:c, :d}, {:d, :e}] +program = ancestor_program(@chain) + +# Full fixpoint: every ancestor pair in the chain. +{:ok, full} = ExDatalog.materialize(program) +Knowledge.get(full, "ancestor") +#=> {(a,b), (a,c), (a,d), (a,e), (b,c), (b,d), (b,e), (c,d), (c,e), (d,e)} + +# Demand-driven: only the slice rooted at :a. +{:ok, magic} = + ExDatalog.materialize(program, strategy: :magic_sets, goal: {"ancestor", [:a, :_]}) + +Knowledge.match(magic, "ancestor", [:a, :_]) +#=> {(a,b), (a,c), (a,d), (a,e)} +``` + +The semi-naive run derives ten facts; the magic-sets run derives only the four the query asked for. The magic relation `magic_ancestor_bf` carries the single seed `(a,)`, the recursive rule fires only when `X = a` propagated through `parent`, and the unrelated `b`, `c`, `d` lineages never enter the working set. On larger graphs the gap widens: demand restriction trades a linear-in-input scan for a slice whose size is bounded by the goal's actual reach. + +## When to Use Magic Sets vs. Semi-Naive + +Magic sets is not a universal optimisation. It pays off when the goal's reach is much smaller than the full fixpoint, and when the result is queried once and discarded. The full semi-naive materialisation is the better choice when: + +- The materialised knowledge will be **reused** across many subsequent queries. The upfront cost of the full fixpoint is amortised; magic sets would re-run the transform (and the evaluation) on every query. +- The goal pattern is **all-free**, so there is no demand to exploit — the transform declines and falls back anyway. +- The program uses **negation or aggregates**, which are outside the supported scope. +- The program has **multiple goals** or needs bindings computed by side constraints (non-ground bound positions). + +Magic sets earns its keep on the opposite shape: deep recursive programs, large datasets, and a single focused query whose bound positions carve a small slice out of a wide relation. In that regime it recovers the precision of top-down evaluation without abandoning the bottom-up fixpoint model that gives Datalog its termination and deterministic-output guarantees. + +The feature is experimental in v0.5.0: positive recursive programs, a single goal, ground bounds, and silent fallback. It is the foundation on which sideways information passing, multi-goal demand, and negation-aware magic relations will be built in later releases. \ No newline at end of file diff --git a/docs/migration_v0.5.md b/docs/migration_v0.5.md new file mode 100644 index 0000000..b64526a --- /dev/null +++ b/docs/migration_v0.5.md @@ -0,0 +1,355 @@ +# Migrating from v0.4 to v0.5 + +ExDatalog v0.5.0 adds aggregate constraints, BEAM callback predicates, +magic-sets program transformation, and a query planner. All v0.4.1 code works +unchanged — the new features are opt-in extensions to the existing DSL and +builder API. + +This guide covers each new feature and the few deprecations. + +## Backward Compatibility + +Every program that compiles and runs under v0.4.1 continues to do so under +v0.5.0. No APIs were removed; no default behaviours changed. The only +breaking-ish change is that `agg(...)` in DSL rule bodies now raises with a +redirect message instead of a generic error (see below). + +## Aggregates + +v0.5.0 introduces four aggregate constraint operations: `count`, `sum`, `min`, +and `max`. Unlike comparison or arithmetic constraints, aggregates operate over +the full set of bindings for a rule, grouped by the non-aggregated variables. + +### DSL Syntax + +Aggregates appear in the rule body, not the head. The result variable is then +used in the head: + +```elixir +defmodule HR do + use ExDatalog.Schema + + relation :employee do + field :name, :atom + field :dept, :atom + field :salary, :integer + end + + relation :dept_size do + field :dept, :atom + field :count, :integer + end + + fact employee(:alice, :eng, 120) + fact employee(:bob, :eng, 95) + fact employee(:carol, :ops, 80) + + rule dept_size(Dept, N) do + employee(Name, Dept, Salary) + count(Name, N) + end +end + +{:ok, knowledge} = HR.materialize() +# knowledge contains dept_size(:eng, 2), dept_size(:ops, 1) +``` + +Supported aggregate operations in the DSL: `count/2`, `sum/2`, `min/2`, +`max/2`. The first argument is the input variable; the second is the result +variable that the engine binds after group-and-reduce. + +### Builder API + +Use `Constraint.from_tuple/1` with an aggregate tuple: + +```elixir +alias ExDatalog.{Program, Rule, Atom, Term, Constraint} + +program = + Program.new() + |> Program.add_relation("employee", [:atom, :atom, :integer]) + |> Program.add_relation("dept_size", [:atom, :integer]) + |> Program.add_fact("employee", [:alice, :eng, 120]) + |> Program.add_fact("employee", [:bob, :eng, 95]) + +aggregate_constraint = Constraint.from_tuple({:count, Term.var("Name"), Term.var("N")}) + +program = + Program.add_rule(program, + Rule.new( + Atom.new("dept_size", [Term.var("Dept"), Term.var("N")]), + [{:positive, Atom.new("employee", [Term.var("Name"), Term.var("Dept"), Term.var("Salary")])}], + [aggregate_constraint] + ) + ) +``` + +### Constraint Table + +Aggregate constraints use the `%Constraint{op: :count | :sum | :min | :max, +left: input_term, right: nil, result: {:var, name}}` shape. The `right` field +is always `nil` for aggregates; the grouping key is inferred from the rule's +non-aggregated variables. + +| Field | Value | +|-------|-------| +| `op` | `:count`, `:sum`, `:min`, `:max` | +| `left` | input term (the variable to aggregate) | +| `right` | `nil` | +| `result` | `{:var, name}` — the output variable | + +## BEAM Callbacks + +BEAM callbacks let rule bodies call deterministic, side-effect-free Elixir +functions as predicates. The engine isolates callbacks with a configurable +timeout (default 100 ms) and treats exceptions/timeouts as filtered bindings. + +### `predicate/5` Macro + +Declare callbacks in the DSL module using `predicate/5`: + +```elixir +defmodule MyRules do + use ExDatalog.Schema + + relation :person do + field :name, :atom + field :age, :integer + end + + relation :senior do + field :name, :atom + end + + predicate :senior?, MyRules, :senior?, [:integer], :boolean + + fact person(:alice, 70) + fact person(:bob, 25) + + rule senior(Name) do + person(Name, Age) + senior?(Age) + end + + def senior?(age), do: age >= 65 +end + +{:ok, knowledge} = MyRules.materialize() +``` + +Arguments: + +- `name` — predicate name used in rule bodies (e.g. `senior?`). +- `module` / `function` — the Elixir module and function to call. +- `arg_types` — declared argument types (informational; length sets the + callback arity). +- `return_type` — `:boolean` (filter) or `:value` (binds last argument as + result). + +### Value-Returning Callbacks + +For `:value` predicates, the **last** argument in the rule-body call is the +result variable; remaining arguments are passed to the function: + +```elixir +defmodule ScoreRules do + use ExDatalog.Schema + + relation :player do + field :name, :atom + field :raw, :integer + end + + relation :ranked do + field :name, :atom + field :score, :integer + end + + predicate :compute_score, ScoreRules, :compute_score, [:integer], :value + + fact player(:alice, 80) + fact player(:bob, 40) + + rule ranked(Name, Score) do + player(Name, Raw) + compute_score(Raw, Score) + end + + def compute_score(raw), do: div(raw * 3, 2) +end +``` + +### Builder API: `Callback.new/4` + +```elixir +alias ExDatalog.{Callback, Term} + +# Boolean filter callback +cb = Callback.new(MyMod, :adult?, [Term.var("Age")]) +#=> %Callback{module: MyMod, function: :adult?, args: [{:var, "Age"}], result: nil} + +# Value-returning callback +cb = Callback.new(MyMod, :score, [Term.var("X")], Term.var("S")) +#=> %Callback{module: MyMod, function: :score, args: [{:var, "X"}], result: {:var, "S"}} +``` + +Add a callback to a rule's body as `{:callback, %Callback{}}`: + +```elixir +rule = + Rule.new( + Atom.new("senior", [Term.var("Name")]), + [ + {:positive, Atom.new("person", [Term.var("Name"), Term.var("Age")])}, + {:callback, Callback.new(MyMod, :senior?, [Term.var("Age")])} + ] + ) +``` + +### Safety Contract + +Callbacks **must** be deterministic and side-effect free. The engine enforces +only timeout and exception isolation — determinism and purity are caller +contracts. + +## Magic Sets + +Magic-sets is a program transformation for demand-driven (goal-directed) +evaluation. Instead of computing the full least fixpoint, it rewrites the +program so that only facts relevant to a query goal are derived. + +### `materialize/2` Options + +Pass `strategy: :magic_sets` and `goal: {relation, pattern}` to +`materialize/2`: + +```elixir +{:ok, knowledge} = + ExDatalog.materialize(program, + strategy: :magic_sets, + goal: {"path", [:alice, :_]} + ) +``` + +The same options work from the DSL: + +```elixir +{:ok, knowledge} = + MySchema.materialize( + strategy: :magic_sets, + goal: {"path", [:alice, :_]} + ) +``` + +When `strategy: :magic_sets` is specified without a `:goal`, the engine falls +back to full semi-naive evaluation. When the program is outside the supported +scope (negation, aggregates), the transformation returns `{:fallback, reason}` +and the engine also falls back silently. + +### Scope (Experimental) + +- Positive recursive programs only. +- A single goal. +- Ground (constant) bound positions. + +Programs outside this scope always fall back to full semi-naive evaluation, +never producing incorrect results. + +## Planner + +The planner sits between compiled IR and the evaluation engine. It produces an +`ExDatalog.Planner.Plan` describing the chosen strategy, planned strata, joins, +and predicates. + +### `plan/2` + +```elixir +alias ExDatalog.{Program, Rule, Atom, Term, Compiler, Planner} + +{:ok, ir} = + Program.new() + |> Program.add_relation("edge", [:atom, :atom]) + |> Program.add_relation("path", [:atom, :atom]) + |> Program.add_rule( + Rule.new( + Atom.new("path", [Term.var("X"), Term.var("Y")]), + [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] + ) + ) + |> Compiler.compile() + +{:ok, plan} = Planner.plan(ir) +plan.strategy #=> :semi_naive +plan.joins #=> [%Join{relation: "edge", ...}] +plan.predicates #=> [] + +{:ok, plan} = Planner.plan(ir, strategy: :magic_sets, goal: {"path", [:alice, :_]}) +plan.strategy #=> :magic_sets +``` + +### `explain_plan/1,2` + +Returns a human-readable description of the plan: + +```elixir +Planner.explain_plan(program) +#=> "Strategy: semi_naive\n Stratum 0: 1 rule(s), relations: edge, path\nJoins: 1\nPredicates: 0" + +Planner.explain_plan(program, strategy: :magic_sets, goal: {"path", [:alice, :_]}) +#=> "Strategy: magic_sets\n Stratum 0: 1 rule(s), relations: magic_path_bf, edge, path\nJoins: 2\nPredicates: 0" +``` + +`explain_plan/1` validates and compiles the program first; returns an error +string if compilation fails. + +### Telemetry Events + +`plan/2` emits the following telemetry events: + +| Event | Measurements | Metadata | +|-------|-------------|----------| +| `[:ex_datalog, :planner, :start]` | `%{system_time: ...}` | `%{relation_count: ..., rule_count: ...}` | +| `[:ex_datalog, :planner, :stop]` | `%{duration: ...}` | `%{relation_count: ..., rule_count: ..., strategy: ...}` | +| `[:ex_datalog, :planner, :exception]` | `%{duration: ...}` | `%{relation_count: ..., rule_count: ..., kind: ..., reason: ...}` | + +## New Capabilities Fields + +`ExDatalog.Capabilities` has two new boolean fields: + +| Field | Default | Description | +|-------|---------|-------------| +| `aggregate_constraints` | `true` | Supports aggregate constraints (count/sum/min/max) | +| `beam_callbacks` | `true` | Supports BEAM callback predicates | + +Both participate in `merge/2` (AND semantics) and `satisfies?/2`: + +```elixir +caps = %ExDatalog.Capabilities{aggregate_constraints: true, beam_callbacks: false} +ExDatalog.Capabilities.satisfies?(caps, aggregate_constraints: true) +#=> true +ExDatalog.Capabilities.satisfies?(caps, beam_callbacks: true) +#=> false +``` + +## Deprecated / Changed + +### `agg(...)` raises with a redirect + +Using `agg(...)` in a DSL rule head or body now raises +`ExDatalog.DSL.CompileError` with a redirect message: + +``` +rule dept_size(D, agg(:count, E)) do + employee(E, D) +end +#=> ** (ExDatalog.DSL.CompileError) use count/sum/min/max for aggregates (e.g. `count(X, N)`) +``` + +Previously, `agg(...)` raised a generic "unsupported term" error. The new +message directs you to the correct aggregate syntax. Use `count/2`, `sum/2`, +`min/2`, or `max/2` in the rule body instead. + +## New Dependencies + +- `benchee ~> 1.3` — dev-only, not included in production builds. Used for + benchmarking the planner and magic-sets transformations. diff --git a/lib/ex_datalog.ex b/lib/ex_datalog.ex index f89a815..09a58d8 100644 --- a/lib/ex_datalog.ex +++ b/lib/ex_datalog.ex @@ -61,9 +61,10 @@ defmodule ExDatalog do - `storage` — storage module (default: `ExDatalog.Storage.Map`) - `max_iterations` — fixpoint iteration limit (default: 10_000) - `timeout_ms` — wall-clock timeout in milliseconds (default: 30_000) - - `goal` — `{relation_name, pattern}` to filter results after evaluation - (default: `nil`). Only available via `materialize/2`. See `materialize/2` - for details. + - `strategy` — `:semi_naive` (default) or `:magic_sets` + - `goal` — `{relation_name, pattern}` used as the magic-sets goal + (when `strategy: :magic_sets`) and as a post-materialization filter + (default: `nil`). See `materialize/2` for details. - `explain` — enable provenance tracking (default: `false`) If `max_iterations` or `timeout_ms` is hit, the returned `Knowledge.t()` @@ -193,7 +194,8 @@ defmodule ExDatalog do See `evaluate/2` for available options, plus: - - `:goal` — `{relation, pattern}` to filter results after materialization + - `:goal` — `{relation, pattern}` used both as the magic-sets goal + (when `strategy: :magic_sets`) and as a post-materialization filter (default: `nil`). When set, the knowledge base's `relations` map contains only the matching tuples for the specified relation. The pattern uses `:_` as a wildcard, matching `Knowledge.match/3`. @@ -215,11 +217,11 @@ defmodule ExDatalog do def materialize(program, opts \\ []) def materialize(%Program{} = program, opts) do - {goal, eval_opts} = Keyword.pop(opts, :goal, nil) + goal = Keyword.get(opts, :goal, nil) with {:ok, validated} <- validate(program), {:ok, ir} <- ExDatalog.Compiler.compile(validated), - {:ok, knowledge} <- evaluate(ir, eval_opts) do + {:ok, knowledge} <- evaluate(ir, opts) do {:ok, apply_goal(knowledge, goal)} end end diff --git a/lib/ex_datalog/constraints/aggregate.ex b/lib/ex_datalog/constraints/aggregate.ex index 98acdd6..49b48c1 100644 --- a/lib/ex_datalog/constraints/aggregate.ex +++ b/lib/ex_datalog/constraints/aggregate.ex @@ -56,7 +56,34 @@ defmodule ExDatalog.Constraints.Aggregate do end defp compute(:count, values), do: length(values) - defp compute(:sum, values), do: Enum.sum(values) - defp compute(:min, values), do: Enum.min(values) - defp compute(:max, values), do: Enum.max(values) + defp compute(:sum, values), do: reduce_sum(values) + defp compute(:min, values), do: reduce_min(values) + defp compute(:max, values), do: reduce_max(values) + + defp reduce_sum(values) do + Enum.reduce(values, 0, fn + v, acc when is_integer(v) -> acc + v + v, _acc -> raise ArgumentError, "sum aggregate requires integer inputs, got: #{inspect(v)}" + end) + end + + defp reduce_min([]), do: raise(ArgumentError, "min aggregate called on empty group") + + defp reduce_min(values) do + Enum.reduce(values, nil, fn + v, nil when is_integer(v) -> v + v, acc when is_integer(v) -> min(v, acc) + v, _acc -> raise(ArgumentError, "min aggregate requires integer inputs, got: #{inspect(v)}") + end) + end + + defp reduce_max([]), do: raise(ArgumentError, "max aggregate called on empty group") + + defp reduce_max(values) do + Enum.reduce(values, nil, fn + v, nil when is_integer(v) -> v + v, acc when is_integer(v) -> max(v, acc) + v, _acc -> raise(ArgumentError, "max aggregate requires integer inputs, got: #{inspect(v)}") + end) + end end diff --git a/lib/ex_datalog/constraints/beam_callback.ex b/lib/ex_datalog/constraints/beam_callback.ex index 55b57db..f482f42 100644 --- a/lib/ex_datalog/constraints/beam_callback.ex +++ b/lib/ex_datalog/constraints/beam_callback.ex @@ -8,8 +8,9 @@ defmodule ExDatalog.Constraints.BeamCallback do The engine enforces only: - - **Timeout** — the call runs in a `Task` with a configurable timeout - (`:callback_timeout_ms`, default 100ms). A timeout filters the binding. + - **Timeout** — the call runs in a `spawn_monitor`-ed process with a + configurable timeout (`:callback_timeout_ms`, default 100ms). A timeout + filters the binding. Late result messages are flushed from the mailbox. - **Exception isolation** — a raised exception filters the binding. Boolean callbacks (`result: nil`) act as filters: `true` keeps the binding, @@ -65,6 +66,14 @@ defmodule ExDatalog.Constraints.BeamCallback do defp bind_result(binding, {:var, name}, value), do: Map.put(binding, name, value) + defp flushlate(ref) do + receive do + {^ref, _} -> flushlate(ref) + after + 0 -> :ok + end + end + # Run the callback in an unlinked, monitored process so that a raise or # exit inside the callback does not propagate to (or kill) the evaluator. # A crash or timeout is reported as `{:error, _}` and filters the binding. @@ -97,6 +106,7 @@ defmodule ExDatalog.Constraints.BeamCallback do timeout_ms -> Process.exit(pid, :kill) Process.demonitor(monitor_ref, [:flush]) + flushlate(ref) {:error, :timeout} end end diff --git a/lib/ex_datalog/magic_sets.ex b/lib/ex_datalog/magic_sets.ex index bfd9e6e..e706ee8 100644 --- a/lib/ex_datalog/magic_sets.ex +++ b/lib/ex_datalog/magic_sets.ex @@ -85,22 +85,34 @@ defmodule ExDatalog.MagicSets do seed_fact = seed_fact(magic_rel, goal_relation, goal_pattern) - rewritten_rules = + # Next available rule ID for supplementary rules + max_rule_id = ir.rules |> Enum.map(& &1.id) |> Enum.max(fn -> 0 end) + next_id = max_rule_id + 1 + + {rewritten_rules, supplementary_rules, _final_id} = ir.rules - |> Enum.map(fn rule -> rewrite_rule(rule, goal_relation, magic_rel, bound_positions) end) + |> Enum.reduce({[], [], next_id}, fn rule, {rr, sr, id} -> + {new_rr, new_srs, new_id} = + rewrite_rule(rule, goal_relation, magic_rel, bound_positions, id) + + {[new_rr | rr], new_srs ++ sr, new_id} + end) + + all_rules = Enum.reverse(rewritten_rules) ++ supplementary_rules new_relations = [magic_relation | ir.relations] new_facts = if seed_fact, do: [seed_fact | ir.facts], else: ir.facts # Recompute strata: the magic relation joins the goal relation's stratum. - new_strata = inject_magic_into_strata(ir.strata, goal_relation, magic_rel) + new_strata = + inject_magic_into_strata(ir.strata, goal_relation, magic_rel, supplementary_rules) {:ok, %IR{ ir | relations: new_relations, facts: new_facts, - rules: rewritten_rules, + rules: all_rules, strata: new_strata }} end @@ -153,32 +165,67 @@ defmodule ExDatalog.MagicSets do defp to_ir_value(v) when is_atom(v), do: {:atom, v} # Rewrite a rule whose head is the goal relation: prepend the magic predicate - # binding the bound head positions, so derivation is demand-restricted. The - # magic atom projects exactly the goal's bound positions of the head, so its - # arity matches the magic relation. + # binding the bound head positions, so derivation is demand-restricted. Also + # generate supplementary magic rules that propagate demand to recursive body + # atoms referencing the goal relation. defp rewrite_rule( %IR.Rule{head: %IR.Atom{relation: rel} = head} = rule, goal_relation, magic_rel, - bound_positions + bound_positions, + next_id ) when rel == goal_relation do magic_terms = bound_head_terms(head, bound_positions) magic_atom = %IR.Atom{relation: magic_rel, terms: magic_terms} - %IR.Rule{rule | body: [{:positive, magic_atom} | rule.body]} + rewritten = %IR.Rule{rule | body: [{:positive, magic_atom} | rule.body]} + + supplementary = + rule.body + |> Enum.with_index() + |> Enum.filter(fn + {{:positive, %IR.Atom{relation: ^goal_relation}}, _idx} -> true + _ -> false + end) + |> Enum.map(fn {{:positive, body_atom}, idx} -> + prefix_body = Enum.take(rule.body, idx) + body_magic_terms = bound_head_terms(body_atom, bound_positions) + + sup_rule = %IR.Rule{ + id: next_id, + head: %IR.Atom{relation: magic_rel, terms: body_magic_terms}, + body: [{:positive, magic_atom} | prefix_body], + stratum: rule.stratum + } + + {sup_rule, next_id} + end) + |> Enum.map(fn {sup_rule, _id} -> sup_rule end) + + final_id = next_id + length(supplementary) + + {rewritten, supplementary, final_id} end - defp rewrite_rule(rule, _goal_relation, _magic_rel, _bound_positions), do: rule + defp rewrite_rule(rule, _goal_relation, _magic_rel, _bound_positions, next_id) do + {rule, [], next_id} + end defp bound_head_terms(%IR.Atom{terms: terms}, bound_positions) do bound_positions |> Enum.map(fn pos -> Enum.at(terms, pos) end) end - defp inject_magic_into_strata(strata, goal_relation, magic_rel) do - Enum.map(strata, fn %IR.Stratum{relations: rels} = stratum -> + defp inject_magic_into_strata(strata, goal_relation, magic_rel, supplementary_rules) do + sup_rule_ids = Enum.map(supplementary_rules, & &1.id) + + Enum.map(strata, fn %IR.Stratum{relations: rels, rule_ids: rule_ids} = stratum -> if goal_relation in rels do - %IR.Stratum{stratum | relations: [magic_rel | rels]} + %IR.Stratum{ + stratum + | relations: [magic_rel | rels], + rule_ids: rule_ids ++ sup_rule_ids + } else stratum end diff --git a/lib/ex_datalog/planner.ex b/lib/ex_datalog/planner.ex index 23d3e58..16fb9d7 100644 --- a/lib/ex_datalog/planner.ex +++ b/lib/ex_datalog/planner.ex @@ -166,7 +166,13 @@ defmodule ExDatalog.Planner do defp build_predicates(%IR{rules: rules}) do rules |> Enum.flat_map(fn rule -> - for {:constraint, c} <- rule.body, do: classify_constraint(c) + rule.body + |> Enum.map(fn + {:constraint, c} -> classify_constraint(c) + {:callback, cb} -> classify_callback(cb) + _ -> nil + end) + |> Enum.reject(&is_nil/1) end) end @@ -174,6 +180,14 @@ defmodule ExDatalog.Planner do %Predicate{kind: constraint_kind(op), op: op, metadata: %{result: c.result}} end + defp classify_callback(%IR.Callback{} = cb) do + %Predicate{ + kind: :callback, + op: :callback, + metadata: %{module: cb.module, function: cb.function, result: cb.result} + } + end + defp constraint_kind(op) do cond do Constraint.comparison_op?(op) -> :comparison diff --git a/lib/ex_datalog/program.ex b/lib/ex_datalog/program.ex index ec44a4a..a93eb64 100644 --- a/lib/ex_datalog/program.ex +++ b/lib/ex_datalog/program.ex @@ -279,6 +279,8 @@ defmodule ExDatalog.Program do end) end + def add_facts({:error, _} = err, _facts), do: err + @doc """ Materializes the program. A pipe-friendly convenience for `ExDatalog.materialize/2`. @@ -293,10 +295,14 @@ defmodule ExDatalog.Program do [{:a, :b}] """ @spec materialize(t(), keyword()) :: {:ok, ExDatalog.Knowledge.t()} | {:error, term()} - def materialize(%__MODULE__{} = program, opts \\ []) do + def materialize(program, opts \\ []) + + def materialize(%__MODULE__{} = program, opts) do ExDatalog.materialize(program, opts) end + def materialize({:error, _} = err, _opts), do: err + @doc """ Adds a rule to the program. diff --git a/lib/ex_datalog/schema.ex b/lib/ex_datalog/schema.ex index 38bb8cb..dc90c37 100644 --- a/lib/ex_datalog/schema.ex +++ b/lib/ex_datalog/schema.ex @@ -25,8 +25,8 @@ defmodule ExDatalog.Schema do An Ecto-inspired DSL for defining Datalog programs. `use ExDatalog.Schema` in a module to declare relations, facts, rules, - and queries. The module then exposes `program/0`, `materialize/0,1`, - and `query/2` functions. + and queries. The module then exposes `program/0`, `new/0`, + `materialize/0,1`, and `query/2` functions. ## Example @@ -128,15 +128,37 @@ defmodule ExDatalog.Schema do Queries operate on materialized knowledge and use `Knowledge.match/3` internally. - ## Aggregate Syntax (Preview) + ## Aggregate Syntax - Aggregates are not yet supported. Using `agg(...)` in a rule head or - body raises `ExDatalog.DSL.CompileError` at compile time: + Aggregates (`count`, `sum`, `min`, `max`) are used in rule bodies as + named predicates. The result variable must appear in the rule head: - rule employee_count(dept, agg(:count, emp)) do - employee(emp, dept) + rule dept_count(D, N) do + emp(E, D) + count(E, N) end - #=> ** (ExDatalog.DSL.CompileError) aggregates are not yet supported (planned for v0.6.0) + + Aggregate inputs are integer-only. A rule may contain at most one + aggregate, and aggregates may not appear in recursive rules. + + ## BEAM Callback Predicates + + Call deterministic Elixir functions from rule bodies: + + predicate :adult?, AgeChecker, :adult?, [:integer], :boolean + + rule active_user(U) do + user(U, Age) + adult?(Age) + end + + Value-returning callbacks bind a result variable: + + predicate :double, Math, :double, [:integer], :value + + Callbacks run in isolated, monitored processes with configurable + timeout (`:callback_timeout_ms`, default 100ms). Timeouts and exceptions + filter the binding. ## Backward Compatibility @@ -219,14 +241,16 @@ defmodule ExDatalog.Schema do Enum.map(relations, fn rel -> name = rel.name arity = length(rel.fields) - args = for i <- 1..arity, do: Macro.var(:"arg_#{i}", __MODULE__) + args = for i <- 1..arity//1, do: Macro.var(:"arg_#{i}", __MODULE__) + + arg_names = Enum.map_join(1..arity//1, ", ", fn i -> "arg_#{i}" end) quote do @doc """ Constructs a fact tuple for the `#{unquote(name)}` relation. - #{unquote(name)}(#{Enum.map_join(1..unquote(arity), ", ", fn i -> "arg_#{i}" end)}) - #=> {"#{unquote(name)}", [#{Enum.map_join(1..unquote(arity), ", ", fn i -> "arg_#{i}" end)})]} + #{unquote(name)}(#{unquote(arg_names)}) + #=> {"#{unquote(name)}", [#{unquote(arg_names)}]} Pass the result to `Program.add_fact/2` as a fact tuple. """ @@ -246,7 +270,7 @@ defmodule ExDatalog.Schema do compile-time facts, and rules. Use this when all facts are declared at compile time via `fact/1`. - For runtime facts, use `new_program/0` and `Program.add_fact/2`. + For runtime facts, use `new/0` and `Program.add_fact/2`. """ @spec program() :: ExDatalog.Program.t() def program do @@ -986,15 +1010,17 @@ defmodule ExDatalog.Schema do - `name` — the predicate name used in rule bodies. - `module` / `function` — the Elixir function to call. - `arg_types` — declared argument types (currently informational; their - length sets the callback arity, validated at compile time). + length sets the callback arity). - `return_type` — `:boolean` (filter) or `:value` (binds the last argument position as the result variable). For `:value` predicates, the **last** argument in the rule-body call is the result variable; the remaining arguments are passed to the function. - The referenced `module.function` must be defined and exported with arity - matching the inputs, or a compile-time `ExDatalog.DSL.CompileError` is raised. + The referenced `module.function` is called at evaluation time in an + isolated, monitored process. A `:callback_timeout_ms` option + (default 100ms) applies to each call; timeouts and exceptions filter + the binding. """ defmacro predicate(name, module, function, arg_types, return_type) do quote do diff --git a/lib/ex_datalog/validator/safety.ex b/lib/ex_datalog/validator/safety.ex index cd17e2b..dc0fabd 100644 --- a/lib/ex_datalog/validator/safety.ex +++ b/lib/ex_datalog/validator/safety.ex @@ -265,12 +265,15 @@ defmodule ExDatalog.Validator.Safety do # 1. At most one aggregate per rule (initial v0.5.0 scope). # 2. An aggregate may not appear in a self-recursive rule (aggregates are # non-monotone and must be stratified above their inputs). + # 3. The aggregate result variable must appear in the rule head, otherwise + # the computed value is silently discarded during projection. defp check_aggregates(errors, %Rule{} = rule, rule_index) do aggregates = Rule.aggregate_constraints(rule) errors |> check_single_aggregate(aggregates, rule, rule_index) |> check_aggregate_not_recursive(aggregates, rule, rule_index) + |> check_aggregate_result_in_head(aggregates, rule, rule_index) end defp check_single_aggregate(errors, aggregates, _rule, rule_index) do @@ -289,6 +292,31 @@ defmodule ExDatalog.Validator.Safety do end end + defp check_aggregate_result_in_head(errors, [], _rule, _rule_index), do: errors + + defp check_aggregate_result_in_head(errors, aggregates, %Rule{} = rule, rule_index) do + head_vars = Rule.head_variables(rule) + + Enum.reduce(aggregates, errors, fn agg, acc -> + result_var = Constraint.result_variable(agg) + + if result_var && result_var not in head_vars do + [ + Error.new( + :aggregate_result_not_in_head, + %{rule_index: rule_index, variable: result_var, op: agg.op}, + "rule #{rule_index}: aggregate #{agg.op} result variable " <> + "#{inspect(result_var)} does not appear in the rule head; " <> + "the computed value would be silently discarded" + ) + | acc + ] + else + acc + end + end) + end + defp check_aggregate_not_recursive(errors, [], _rule, _rule_index), do: errors defp check_aggregate_not_recursive(errors, _aggregates, %Rule{} = rule, rule_index) do diff --git a/livebooks/ex_datalog_v050.livemd b/livebooks/ex_datalog_v050.livemd index 6c1696c..d57db7e 100644 --- a/livebooks/ex_datalog_v050.livemd +++ b/livebooks/ex_datalog_v050.livemd @@ -642,7 +642,75 @@ true --- -## 4. Query Planner +## 4. Runtime Facts (Schema.new + Program.add_fact) + +For data-driven workflows where facts come from external sources at runtime, +use `Schema.new/0` to get a blank program (relations + rules, no facts) and +pipe facts in via `Program.add_fact/2`: + +```elixir +defmodule DeptRuntimeCount do + use ExDatalog.Schema + + relation :emp do + field(:name, :atom) + field(:dept, :atom) + end + + relation :dept_count do + field(:dept, :atom) + field(:n, :integer) + end + + rule dept_count(D, N) do + emp(E, D) + count(E, N) + end +end + +program = + DeptRuntimeCount.new() + |> ExDatalog.Program.add_fact(DeptRuntimeCount.emp(:alice, :eng)) + |> ExDatalog.Program.add_fact(DeptRuntimeCount.emp(:bob, :eng)) + |> ExDatalog.Program.add_fact(DeptRuntimeCount.emp(:carol, :ops)) + +{:ok, knowledge} = ExDatalog.Program.materialize(program) +Knowledge.get(knowledge, "dept_count") |> MapSet.to_list() |> Enum.sort() +``` + + + +``` +[eng: 2, ops: 1] +``` + +### Bulk add_facts + +```elixir +facts = [ + DeptRuntimeCount.emp(:dave, :eng), + DeptRuntimeCount.emp(:eve, :ops), + DeptRuntimeCount.emp(:frank, :eng) +] + +program = + DeptRuntimeCount.new() + |> ExDatalog.Program.add_fact(DeptRuntimeCount.emp(:alice, :eng)) + |> ExDatalog.Program.add_facts(facts) + +{:ok, knowledge} = ExDatalog.Program.materialize(program) +Knowledge.get(knowledge, "dept_count") |> MapSet.to_list() |> Enum.sort() +``` + + + +``` +[eng: 3, ops: 1] +``` + +--- + +## 5. Query Planner The planner sits between the compiled IR and the evaluation engine. It produces an `ExDatalog.Planner.Plan` describing the chosen `strategy`, `strata`, `joins`, @@ -750,7 +818,7 @@ IO.puts(Planner.explain_plan(AdultOnly.program())) Strategy: semi_naive Stratum 0: 1 rule(s), relations: adult, person Joins: 1 -Predicates: 0 +Predicates: 1 (callback) ``` diff --git a/test/ex_datalog/magic_sets_test.exs b/test/ex_datalog/magic_sets_test.exs index cf7c9e1..e6d7613 100644 --- a/test/ex_datalog/magic_sets_test.exs +++ b/test/ex_datalog/magic_sets_test.exs @@ -113,6 +113,69 @@ defmodule ExDatalog.MagicSetsTest do end end + describe "public API exercises magic-sets transform" do + test "knowledge stats contain magic-prefixed relations when strategy is :magic_sets" do + program = ancestor_program(@chain) + + {:ok, magic} = + ExDatalog.materialize(program, strategy: :magic_sets, goal: {"ancestor", [:a, :_]}) + + magic_rels = + magic.stats.relation_sizes + |> Map.keys() + |> Enum.filter(&String.starts_with?(&1, "magic_")) + + assert magic_rels != [], + "expected magic-prefixed relations in stats, got: #{inspect(magic.stats.relation_sizes)}" + end + + test "knowledge stats have no magic-prefixed relations for plain semi-naive" do + program = ancestor_program(@chain) + + {:ok, full} = ExDatalog.materialize(program) + + magic_rels = + full.stats.relation_sizes + |> Map.keys() + |> Enum.filter(&String.starts_with?(&1, "magic_")) + + assert magic_rels == [], + "expected no magic-prefixed relations in semi-naive stats, got: #{inspect(full.stats.relation_sizes)}" + end + + test "magic-sets restricts ancestor derivation to goal-reachable nodes" do + # Graph with two disconnected branches; goal queries only the first. + facts = [ + {:a, :b}, + {:b, :c}, + {:c, :d}, + {:x, :y}, + {:y, :z} + ] + + program = ancestor_program(facts) + + {:ok, magic} = + ExDatalog.materialize(program, strategy: :magic_sets, goal: {"ancestor", [:a, :_]}) + + {:ok, full} = ExDatalog.materialize(program) + + # Magic-sets ancestor table (before apply_goal filter) has only facts + # reachable from :a, while semi-naive computes all branches. + magic_ancestor = Knowledge.get(magic, "ancestor") + full_ancestor = Knowledge.get(full, "ancestor") + + refute MapSet.member?(magic_ancestor, {:x, :y}), + "magic-sets should not derive {:x, :y} (unrelated to goal :a)" + + assert MapSet.member?(full_ancestor, {:x, :y}), + "semi-naive should derive {:x, :y}" + + assert MapSet.member?(magic_ancestor, {:a, :d}), + "magic-sets should derive {:a, :d} (goal-reachable)" + end + end + describe "unsupported programs fall back" do test "program with negation falls back to semi-naive" do program = diff --git a/test/ex_datalog/runtime_facts_test.exs b/test/ex_datalog/runtime_facts_test.exs index 1cab0a0..f9ae7f0 100644 --- a/test/ex_datalog/runtime_facts_test.exs +++ b/test/ex_datalog/runtime_facts_test.exs @@ -161,6 +161,33 @@ defmodule ExDatalog.RuntimeFactsTest do ]) end + test "returns error and does not modify original program on failure" do + prog = Program.new() |> Program.add_relation("emp", [:atom, :atom]) + + result = Program.add_facts(prog, [{"emp", [:alice, :eng]}, {"unknown", [:x]}]) + + assert {:error, _} = result + assert prog.facts == [] + end + + test "propagates error through pipe chain" do + prog = Program.new() |> Program.add_relation("emp", [:atom, :atom]) + + result = + prog + |> Program.add_facts([{"unknown", [:x]}]) + |> Program.add_fact({"emp", [:alice, :eng]}) + |> Program.materialize() + + assert {:error, _} = result + end + + test "handles empty list" do + prog = Program.new() |> Program.add_relation("emp", [:atom, :atom]) + result = Program.add_facts(prog, []) + assert result.facts == [] + end + test "pipeable with schema constructors" do defmodule BulkSchema do use ExDatalog.Schema From a5a58bc94a3f97a021af103bf76be2963a025de9 Mon Sep 17 00:00:00 2001 From: thanos Date: Mon, 22 Jun 2026 16:54:57 -0400 Subject: [PATCH 3/3] =?UTF-8?q?Summary=20of=20fixes=20High=20=E2=80=94=20C?= =?UTF-8?q?orrectness=20-=20F-001=20(the=20real=20bug):=20MagicSets.rewrit?= =?UTF-8?q?e=5Frule/5=20now=20uses=20Enum.map=5Freduce=20to=20give=20each?= =?UTF-8?q?=20supplementary=20rule=20a=20distinct=20ID,=20and=20do=5Ftrans?= =?UTF-8?q?form=20runs=20a=20defensive=20assert=5Funique=5Frule=5Fids!/1?= =?UTF-8?q?=20on=20the=20transformed=20IR.=20Verified=20the=20doubly-recur?= =?UTF-8?q?sive=20case=20now=20produces=20IDs=20[0,1,2,3]=20with=20no=20pr?= =?UTF-8?q?ovenance-map=20collapse.=20-=20F-007:=20Added=20a=20doubly-recu?= =?UTF-8?q?rsive=20(reach(X,Z)=20:-=20reach(X,Y),=20reach(Y,Z))=20regressi?= =?UTF-8?q?on=20suite=20=E2=80=94=20asserts=20unique=20IDs,=20full=20prove?= =?UTF-8?q?nance=20map,=20correct=20goal=20subset,=20and=20a=20clean=20exp?= =?UTF-8?q?lain:=20true=20run.=20High=20=E2=80=94=20Documentation=20contra?= =?UTF-8?q?dictions=20(introduced=20by=20the=20branch)=20-=20F-002=20/=20F?= =?UTF-8?q?-013:=20Rewrote=20article=2006=20=E2=80=94=20the=20planner=20is?= =?UTF-8?q?=20now=20correctly=20described=20as=20a=20standalone=20inspecti?= =?UTF-8?q?on=20tool=20off=20the=20eval=20path;=20removed=20the=20nonexist?= =?UTF-8?q?ent=20eval=5Fplan/2;=20corrected=20the=20"magic=20sets=20still?= =?UTF-8?q?=20runs=20semi-naive"=20claim;=20fixed=20the=20"is=20now.Compil?= =?UTF-8?q?er"=20typo=20and=20all=20the=20"engine=20consumes=20the=20plan"?= =?UTF-8?q?=20statements.=20-=20F-003:=20Article=2007=20now=20documents=20?= =?UTF-8?q?the=20:aggregate=5Fresult=5Fnot=5Fin=5Fhead=20rejection=20and?= =?UTF-8?q?=20that=20non-integer=20inputs=20raise=20ArgumentError.=20-=20F?= =?UTF-8?q?-004:=20Article=2009=20now=20shows=20the=20correct=20/5=20and?= =?UTF-8?q?=20/4=20signatures,=20documents=20the=20supplementary=20demand-?= =?UTF-8?q?propagation=20rule,=20and=20replaces=20the=20false=20"derives?= =?UTF-8?q?=20only=20four=20facts"=20claim=20with=20an=20accurate=20discon?= =?UTF-8?q?nected-component=20explanation.=20Medium=20-=20F-005:=20New=20t?= =?UTF-8?q?ests=20for=20result-not-in-head=20rejection,=20sum/min/max=20in?= =?UTF-8?q?teger=20guards=20raising,=20and=20count=20accepting=20non-integ?= =?UTF-8?q?ers.=20-=20F-006:=20Settled=20the=20contract=20=E2=80=94=20non-?= =?UTF-8?q?integer=20aggregate=20inputs=20raise=20(consistent=20with=20the?= =?UTF-8?q?=20integer-only=20design);=20documented=20in=20materialize/2=20?= =?UTF-8?q?and=20the=20aggregate=20moduledoc,=20with=20tests=20pinning=20i?= =?UTF-8?q?t.=20Low=20/=20Nit=20-=20F-008/F-009/F-010:=20Corrected=20stale?= =?UTF-8?q?=20docstrings=20(no=20build-time=20validation=20claim;=20sum=20?= =?UTF-8?q?raises=20not=20filters;=20updated=20the=20Enum.min/max=20refere?= =?UTF-8?q?nce).=20-=20F-011:=20flushlate=20=E2=86=92=20flush=5Flate.=20-?= =?UTF-8?q?=20F-012:=20Zero-field=20relations=20now=20raise=20a=20clear=20?= =?UTF-8?q?DSL.CompileError=20at=20compile=20time=20instead=20of=20silentl?= =?UTF-8?q?y=20breaking=20the=20whole=20schema;=20documented.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- README.md | 4 +- docs/articles/06_query_planning_in_datalog.md | 68 +++++++++++----- docs/articles/07_aggregates_in_datalog.md | 12 ++- ...magic_sets_and_demand_driven_evaluation.md | 57 +++++++++++--- lib/ex_datalog.ex | 9 +++ lib/ex_datalog/constraint.ex | 3 +- lib/ex_datalog/constraints/aggregate.ex | 15 ++-- lib/ex_datalog/constraints/beam_callback.ex | 6 +- lib/ex_datalog/magic_sets.ex | 32 ++++++-- lib/ex_datalog/schema.ex | 11 ++- test/ex_datalog/aggregate_test.exs | 77 +++++++++++++++++++ test/ex_datalog/beam_callback_test.exs | 30 ++++++++ test/ex_datalog/magic_sets_test.exs | 73 ++++++++++++++++++ 14 files changed, 345 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cfdac9..2b389ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,7 +60,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Aggregates are integer-only; `avg` and float support are deferred to v0.6.0. - Magic sets is experimental and opt-in; the default strategy is unchanged. -- All v0.4.1 tests continue to pass. Suite: 861 tests, 10 properties, +- All v0.4.1 tests continue to pass. Suite: 871 tests, 10 properties, 152 doctests, ~93% coverage. ## v0.4.1 (2026-06-21) diff --git a/README.md b/README.md index ad44a98..31eefdf 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ It continues to influence modern databases, compilers, static analysis tools, kn - **Provenance / derivation explain** (`explain: true`) - **Telemetry** integration (`:telemetry` events for query/planner lifecycle) - **Deterministic**: same program + same facts = same result regardless of backend -- 861 tests, 0 failures, credo clean +- 871 tests, 0 failures, credo clean ## Installation @@ -522,7 +522,7 @@ The following references are highly recommended for understanding both the theor | v0.3.0 | Tuple shorthand for rules (`add_rule/3`, `add_rule/4`), `Term.from/1`, `ExDatalog.Atom.from_tuple/1`, `Constraint.from_tuple/1`; renamed `Result` → `Knowledge`, `query` → `materialize` | | v0.4.0 | Schema DSL (`use ExDatalog.Schema`), `relation`, `fact`, `rule`, `query` macros, `not_` negation, constraint DSL, post-materialization queries | | v0.4.1 | DSL review fixes: correct uppercase-variable docs, clean aggregate error, query/find validation, unified `DSL.CompileError`, 792 tests | -| v0.5.0 | Aggregates (`count`/`sum`/`min`/`max`), BEAM callback predicates, query planner, magic sets (experimental), runtime facts API, 861 tests | +| v0.5.0 | Aggregates (`count`/`sum`/`min`/`max`), BEAM callback predicates, query planner, magic sets (experimental), runtime facts API, 871 tests | | v1.0.0 | Stable public API, hardened production semantics | ## License diff --git a/docs/articles/06_query_planning_in_datalog.md b/docs/articles/06_query_planning_in_datalog.md index 9355ee7..5dcec2b 100644 --- a/docs/articles/06_query_planning_in_datalog.md +++ b/docs/articles/06_query_planning_in_datalog.md @@ -1,6 +1,6 @@ # Query Planning in ExDatalog: The Seam Between IR and Engine -Datalog evaluation looks direct — apply the rules, accumulate facts, stop at a fixpoint. But between the validated `ExDatalog.IR` and the semi-naive engine sits a small planner whose job is to describe *how* a program will run before it runs. ExDatalog's planner is thin by design in v0.5.0, but it is the structure future optimizations hang from: join ordering, cost-based selection, and goal-directed evaluation all plug in here. +Datalog evaluation looks direct — apply the rules, accumulate facts, stop at a fixpoint. Alongside the validated `ExDatalog.IR` and the semi-naive engine, ExDatalog ships a small planner whose job is to describe *how* a program will run before it runs. It is an inspection tool, not a stage in the evaluation path: the engine reads the IR directly. The planner is thin by design in v0.5.0, but it is the structure future optimizations hang from: join ordering, cost-based selection, and goal-directed evaluation all plug in here. ## Why Datalog Needs a Planner @@ -12,7 +12,7 @@ The IR describes *what* a program means: relations, rules, strata, safety-valida - which **predicates** (constraints, aggregates, callbacks) must run, classified by kind so the engine can dispatch them uniformly; - where **telemetry** fires so observers can profile planning without instrumenting every rule. -Putting these decisions in their own struct means the engine stays a pure consumer of the plan, and an optimizer can swap plans without touching the engine. That separation is what makes the planner the right seam for future work. +Putting these decisions in their own struct means the plan is a standalone, inspectable description of evaluation. The engine reads the IR directly (it does not consume the plan today), but keeping the description separate is what makes the planner the right seam for a future cost-based optimizer to plug into. ## The Four Planner Structs @@ -210,18 +210,28 @@ This makes `explain_plan/1,2` cheap to wire into a REPL or a `mix` task without ## Strategy Selection -Strategy is recorded, not yet *executed* differently. In v0.5.0 the planner is honest about this in its module doc: - -> When `:magic_sets` is requested, a `:goal` option (`{relation, pattern}`) should be supplied; otherwise the planner records the strategy but the engine falls back to semi-naive. - -So selecting `:magic_sets` changes the plan's `strategy` field and stashes the goal in `metadata`, but the existing `Engine.Naive` still runs semi-naive bottom-up. The point of recording the strategy now is that the *shape* of the plan changes — future magic-sets rewriting will introduce magic predicates and adornment joins while keeping the same `Plan`/`Stratum`/`Join`/`Predicate` structs. Selecting the strategy is a one-line change at the call site: +The planner *records* the requested strategy; it does not perform the rewrite +itself. Passing `strategy: :magic_sets` sets the plan's `strategy` field and +stashes the goal in `metadata`: ```elixir Planner.plan(ir) # :semi_naive Planner.plan(ir, strategy: :magic_sets, goal: {"path", [:a, :_]}) ``` -This means the planner is the right hook today for tooling that wants to *predict* which strategy will run, even if the engine hasn't yet caught up. +The actual goal-directed rewrite is a separate concern, performed by +`ExDatalog.MagicSets` during `materialize/2` — not by the planner. So the two +layers play complementary roles: + +- `Planner.plan/2` lets tooling *predict* and display which strategy a call will + use, without running anything. +- `ExDatalog.materialize(program, strategy: :magic_sets, goal: ...)` actually + applies the magic-sets transformation and evaluates the rewritten program (see + the magic-sets article for details). + +Both consume the same `Plan`/`Stratum`/`Join`/`Predicate` vocabulary, which is +why a future cost-based optimizer can change the *plan* a program produces +without changing call sites. ## Telemetry Events @@ -255,19 +265,18 @@ end Attach with `:telemetry.attach/4` to react to slow planning, or use `:telemetry.span/3` wrappers in higher-level tooling that wants a single composite event. The per-call (rather than per-rule) granularity keeps planning telemetry cheap: a thousand-rule program still emits exactly three events on failure. -## Where the Planner Sits in the Pipeline +## Where the Planner Sits -ExDatalog's evaluation pipeline is now.Compiler → Validator → Planner → Engine: +The planner is a **standalone inspection tool**, not a stage in the evaluation +pipeline. The runtime path validates and compiles a program to IR, then hands +the IR directly to the engine: ``` -Program (DSL) - │ Program.compile/1 or ExDatalog.compile/1 +Program (DSL or builder) + │ ExDatalog.materialize/2 (validate → compile → evaluate) ▼ IR (relations, facts, rules, strata, metadata) - │ ExDatalog.Planner.plan/2 - ▼ -Plan (strategy, strata, joins, predicates, metadata) - │ Engine.Naive.eval_plan/2 (or future engine) + │ Engine.Naive.evaluate/2 ▼ Knowledge (materialized relations: MapSet of tuples) │ Schema.query/2, find/where @@ -275,16 +284,33 @@ Knowledge (materialized relations: MapSet of tuples) result ``` -The validator runs *before* the planner — that's why `plan/2` accepts the already-validated IR, and why `explain_plan/2`'s "compilation failed" path goes through `ExDatalog.compile/1` (which bundles validation) rather than re-validating inside the planner. The planner is a pure consumer of validated IR. +The planner consumes the same validated IR, but off to the side, when you want +to *inspect* what evaluation will do: + +``` +IR ──► ExDatalog.Planner.plan/2 ──► Plan (strategy, strata, joins, predicates) +``` + +`plan/2` accepts already-validated IR, which is why `explain_plan/2`'s +"compilation failed" path goes through `ExDatalog.compile/1` (which bundles +validation) rather than re-validating inside the planner. The planner is a pure +consumer of validated IR. -The engine is the next consumer of the plan. Each `Stratum` in the plan corresponds to one fixpoint loop in the engine; each `Join` corresponds to one nested-loop scan; each `Predicate` corresponds to one filter or binding extension. Because the plan is descriptive, the engine doesn't *need* it to function — it can read the IR directly — but using the plan means future optimizations (join reordering, indexed joins, magic-sets rewriting) can land in the planner and immediately benefit every engine that consumes plans. +The `Plan` is **descriptive, not executable**. The engine does not consume it — +it reads the IR directly. Each `Stratum` in the plan corresponds to one fixpoint +loop in the engine; each `Join` corresponds to one nested-loop scan; each +`Predicate` corresponds to one filter or binding extension. The plan exists so +that tooling (and future optimizers) have a structured, serializable view of the +evaluation shape without having to re-derive it from the IR. ## Practical Notes -- **The plan is cheap to build.** A few `Enum.flat_map` passes over the rules; no allocation per fact. There's no reason to skip it in production. +- **The plan is cheap to build.** A few `Enum.flat_map` passes over the rules; no allocation per fact. - **Use `explain_plan/1` for debugging.** It compiles for you and returns a string you can `IO.puts` without touching data structures. - **Use `plan/2` for tooling.** It gives you the structs you can serialize, diff, or compare across program versions. - **Attach telemetry once.** `[:ex_datalog, :planner, :stop]` carries the `:strategy` and the rules/relations counts — enough to build a dashboard without parsing the plan. -- **Don't expect magic sets to accelerate execution yet.** In v0.5.0 the strategy is recorded; the engine still runs semi-naive. The seam is in place so a future release can hook in goal-directed rewriting without changing call sites. +- **Magic sets is live, but the planner does not drive it.** The `:strategy` recorded in a `Plan` is informational. The actual goal-directed rewrite happens in `ExDatalog.MagicSets` when you call `materialize/2` with `strategy: :magic_sets` and a `:goal` (see the magic-sets article). The planner reports the requested strategy; it does not perform the transformation. -The planner is small on purpose. The cost of getting the seam right now — descriptive structs, honest strategy selection, telemetry from day one — is low, and it pays back the moment a real optimizer needs somewhere to live. \ No newline at end of file +The planner is small on purpose: descriptive structs, honest strategy reporting, +and telemetry from day one. It is the seam where a real cost-based optimizer can +land later without changing call sites. \ No newline at end of file diff --git a/docs/articles/07_aggregates_in_datalog.md b/docs/articles/07_aggregates_in_datalog.md index 859e64b..f0d3716 100644 --- a/docs/articles/07_aggregates_in_datalog.md +++ b/docs/articles/07_aggregates_in_datalog.md @@ -44,7 +44,7 @@ defp compute(:min, values), do: Enum.min(values) defp compute(:max, values), do: Enum.max(values) ``` -`count` works on any group; `count` of employees is the size of the group regardless of the input variable's type. `sum`, `min`, and `max` require integer inputs. The plan is to enforce this at build time and guard at runtime; today an input list containing non-integers would surface as an Elixir runtime error from `Enum.sum/1` or `Enum.min/1`. Because aggregates are stratified above their inputs (see below), the input list is fully materialized and final before the reducer runs, so a runtime crash during reduction is a property of the data, not of evaluation order. +`count` works on any group; `count` of employees is the size of the group regardless of the input variable's type. `sum`, `min`, and `max` require integer inputs, and this is **guarded at runtime**: a non-integer input raises a descriptive `ArgumentError` (for example, `"sum aggregate requires integer inputs, got: :x"`) from the reducer. Build-time type enforcement of aggregate inputs is future work. Because aggregates are stratified above their inputs (see below), the input list is fully materialized and final before the reducer runs, so a type error during reduction is a property of the data, not of evaluation order. ## The DSL Syntax @@ -246,7 +246,15 @@ result_vars = end) ``` -So `Rule.head_variables(rule)` including `N` is satisfied by the `count(E, N)` constraint without `N` appearing in any positive body atom. If the head omits the result variable, the aggregate's value is silently dropped — there is no dedicated error for this, but the produced head tuple simply does not carry the computed value. +So `Rule.head_variables(rule)` including `N` is satisfied by the `count(E, N)` constraint without `N` appearing in any positive body atom. If the head omits the result variable, the rule is **rejected at validation time** with an `:aggregate_result_not_in_head` error — the safety checker refuses to compile a rule whose aggregate result would be silently discarded: + +```elixir +# rule dept_count(D) do # head omits N +# emp(E, D) +# count(E, N) +# end +#=> {:error, [%ExDatalog.Validator.Error{kind: :aggregate_result_not_in_head, ...}]} +``` ## Stratification: Forcing the Aggregate Above Its Inputs diff --git a/docs/articles/09_magic_sets_and_demand_driven_evaluation.md b/docs/articles/09_magic_sets_and_demand_driven_evaluation.md index 340ddc9..d95f914 100644 --- a/docs/articles/09_magic_sets_and_demand_driven_evaluation.md +++ b/docs/articles/09_magic_sets_and_demand_driven_evaluation.md @@ -79,7 +79,9 @@ Without a seed, the magic relation is empty and no goal-relation rule can fire, ### 4. Rule Rewriting -Every rule whose head is the goal relation is rewritten to *consume* the magic predicate as its first body atom. Only bindings consistent with an existing magic tuple are allowed to proceed. For the recursive ancestor rule this means: +Every rule whose head is the goal relation is rewritten in two ways. + +**(a) Demand restriction.** The magic predicate is prepended as the first body atom, so only bindings consistent with an existing magic tuple proceed: ``` ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z). @@ -89,26 +91,59 @@ becomes ancestor(X, Z) :- magic_ancestor_bf(X), parent(X, Y), ancestor(Y, Z). ``` -The implementation prepends the magic atom: +**(b) Demand propagation.** For each recursive body atom that references the goal relation, a *supplementary* magic rule is generated so demand flows from the head to the recursive subgoal. The recursive call `ancestor(Y, Z)` only makes sense once we know we need ancestors of `Y`, so: + +``` +magic_ancestor_bf(Y) :- magic_ancestor_bf(X), parent(X, Y). +``` + +Without this rule the magic table would never grow past the seed, and only direct facts would be derived. The implementation produces both the rewritten rule and the supplementary rules, assigning each supplementary rule a fresh unique id: ```elixir -defp rewrite_rule(%IR.Rule{head: %IR.Atom{relation: rel} = head} = rule, goal_relation, magic_rel, bound_positions) +defp rewrite_rule(%IR.Rule{head: %IR.Atom{relation: rel} = head} = rule, goal_relation, magic_rel, bound_positions, next_id) when rel == goal_relation do magic_terms = bound_head_terms(head, bound_positions) magic_atom = %IR.Atom{relation: magic_rel, terms: magic_terms} - %IR.Rule{rule | body: [{:positive, magic_atom} | rule.body]} + rewritten = %IR.Rule{rule | body: [{:positive, magic_atom} | rule.body]} + + recursive_atoms = + rule.body + |> Enum.with_index() + |> Enum.filter(fn + {{:positive, %IR.Atom{relation: ^goal_relation}}, _idx} -> true + _ -> false + end) + + {supplementary, final_id} = + Enum.map_reduce(recursive_atoms, next_id, fn {{:positive, body_atom}, idx}, id -> + prefix_body = Enum.take(rule.body, idx) + body_magic_terms = bound_head_terms(body_atom, bound_positions) + + sup_rule = %IR.Rule{ + id: id, + head: %IR.Atom{relation: magic_rel, terms: body_magic_terms}, + body: [{:positive, magic_atom} | prefix_body], + stratum: rule.stratum + } + + {sup_rule, id + 1} + end) + + {rewritten, supplementary, final_id} end ``` -The magic atom projects exactly the goal's bound positions of the head, so its arity matches the magic relation. Rules whose head is *not* the goal relation are left untouched. +The `Enum.map_reduce` threads an incrementing id so a rule with several recursive body atoms (e.g. `r(X,Z) :- r(X,Y), r(Y,Z)`) still yields supplementary rules with distinct ids, preserving the IR's rule-id uniqueness invariant. -Finally, the magic relation is injected into the goal relation's stratum so the stratified evaluator processes them in the same pass: +Finally, the magic relation and the supplementary rule ids are injected into the goal relation's stratum so the stratified evaluator processes them in the same pass: ```elixir -defp inject_magic_into_strata(strata, goal_relation, magic_rel) do - Enum.map(strata, fn %IR.Stratum{relations: rels} = stratum -> +defp inject_magic_into_strata(strata, goal_relation, magic_rel, supplementary_rules) do + sup_rule_ids = Enum.map(supplementary_rules, & &1.id) + + Enum.map(strata, fn %IR.Stratum{relations: rels, rule_ids: rule_ids} = stratum -> if goal_relation in rels do - %IR.Stratum{stratum | relations: [magic_rel | rels]} + %IR.Stratum{stratum | relations: [magic_rel | rels], rule_ids: rule_ids ++ sup_rule_ids} else stratum end @@ -296,7 +331,9 @@ Knowledge.match(magic, "ancestor", [:a, :_]) #=> {(a,b), (a,c), (a,d), (a,e)} ``` -The semi-naive run derives ten facts; the magic-sets run derives only the four the query asked for. The magic relation `magic_ancestor_bf` carries the single seed `(a,)`, the recursive rule fires only when `X = a` propagated through `parent`, and the unrelated `b`, `c`, `d` lineages never enter the working set. On larger graphs the gap widens: demand restriction trades a linear-in-input scan for a slice whose size is bounded by the goal's actual reach. +The `match/3` query returns the four pairs rooted at `:a`. Note an important subtlety for *linear* transitive closure: the demand-propagation rule seeds the magic table with **every node reachable from `:a`** (`magic_ancestor_bf` ends up holding `(a,), (b,), (c,), (d,)`), so the engine still derives all ten ancestor pairs internally — the four-pair result you see comes from the goal filter applied after materialization. + +Where magic sets actually shrinks the working set is when large parts of the graph are **not** reachable from the goal. Given two disconnected components `a→b→c→d` and `x→y→z`, a goal of `ancestor(:a, :_)` never seeds `x`, `y`, or `z` into the magic table, so the `x`/`y`/`z` lineage is never derived at all. On graphs with many goal-irrelevant regions the saving is large; on a single linear chain the saving is small because demand reaches every node. ## When to Use Magic Sets vs. Semi-Naive diff --git a/lib/ex_datalog.ex b/lib/ex_datalog.ex index 09a58d8..468bc20 100644 --- a/lib/ex_datalog.ex +++ b/lib/ex_datalog.ex @@ -190,6 +190,15 @@ defmodule ExDatalog do Returns `{:ok, ExDatalog.Knowledge.t()}` or `{:error, reason}`. + ## Aggregate input types + + Aggregates (`sum`, `min`, `max`) are integer-only. If an aggregate's input + resolves to a non-integer value at reduction time, evaluation **raises** + `ArgumentError` rather than returning `{:error, reason}`. A non-integer + aggregate input is treated as a data-modeling error and surfaces loudly. If + the input domain is untrusted, constrain it with a type predicate (for example + `is_integer/1`) earlier in the rule body, or wrap the call in a `try`. + ## Options See `evaluate/2` for available options, plus: diff --git a/lib/ex_datalog/constraint.ex b/lib/ex_datalog/constraint.ex index c8533d4..059015b 100644 --- a/lib/ex_datalog/constraint.ex +++ b/lib/ex_datalog/constraint.ex @@ -443,7 +443,8 @@ defmodule ExDatalog.Constraint do @doc """ Constructs a `sum` aggregate: `result = integer sum of input within each group`. - Inputs must be integers; non-integer inputs filter the group's contribution. + Inputs must be integers; a non-integer input raises `ArgumentError` at + reduction time. ## Examples diff --git a/lib/ex_datalog/constraints/aggregate.ex b/lib/ex_datalog/constraints/aggregate.ex index 49b48c1..3d997e2 100644 --- a/lib/ex_datalog/constraints/aggregate.ex +++ b/lib/ex_datalog/constraints/aggregate.ex @@ -9,9 +9,11 @@ defmodule ExDatalog.Constraints.Aggregate do `evaluate/3` callback exists only to satisfy the `ExDatalog.Constraint` behaviour and raises if ever invoked through the per-binding pipeline. - Aggregates are integer-only. `sum` requires integer inputs (validated at - build time and guarded at runtime). `count` returns the group size. `min`/`max` - return the smallest/largest input value in the group. + Aggregates are integer-only. `sum`, `min`, and `max` require integer inputs, + guarded at runtime: a non-integer input raises `ArgumentError` from the + reducer (build-time type enforcement is future work). `count` returns the + group size regardless of input type. `min`/`max` return the smallest/largest + input value in the group. """ @behaviour ExDatalog.Constraint @@ -32,9 +34,10 @@ defmodule ExDatalog.Constraints.Aggregate do Groups bindings by `group_vars` and reduces each group's `input_var` values with the aggregate `op`, binding the reduced value to `result_var`. - Returns one extended binding per non-empty group. Empty groups never occur: - a group exists only because at least one binding produced its key, so the - reducers (`Enum.min/1`, `Enum.max/1`) are never called on an empty list. + Returns one extended binding per non-empty group. Empty groups never occur in + practice: a group exists only because at least one binding produced its key. + The `min`/`max` reducers still carry a defensive empty-list clause that raises + a clear error should that invariant ever be violated. ## Examples diff --git a/lib/ex_datalog/constraints/beam_callback.ex b/lib/ex_datalog/constraints/beam_callback.ex index f482f42..ead373b 100644 --- a/lib/ex_datalog/constraints/beam_callback.ex +++ b/lib/ex_datalog/constraints/beam_callback.ex @@ -66,9 +66,9 @@ defmodule ExDatalog.Constraints.BeamCallback do defp bind_result(binding, {:var, name}, value), do: Map.put(binding, name, value) - defp flushlate(ref) do + defp flush_late(ref) do receive do - {^ref, _} -> flushlate(ref) + {^ref, _} -> flush_late(ref) after 0 -> :ok end @@ -106,7 +106,7 @@ defmodule ExDatalog.Constraints.BeamCallback do timeout_ms -> Process.exit(pid, :kill) Process.demonitor(monitor_ref, [:flush]) - flushlate(ref) + flush_late(ref) {:error, :timeout} end end diff --git a/lib/ex_datalog/magic_sets.ex b/lib/ex_datalog/magic_sets.ex index e706ee8..788b8f6 100644 --- a/lib/ex_datalog/magic_sets.ex +++ b/lib/ex_datalog/magic_sets.ex @@ -100,6 +100,8 @@ defmodule ExDatalog.MagicSets do all_rules = Enum.reverse(rewritten_rules) ++ supplementary_rules + :ok = assert_unique_rule_ids!(all_rules) + new_relations = [magic_relation | ir.relations] new_facts = if seed_fact, do: [seed_fact | ir.facts], else: ir.facts @@ -117,6 +119,20 @@ defmodule ExDatalog.MagicSets do }} end + # Defensive invariant: the transformed IR must preserve unique rule IDs, the + # same contract the compiler enforces. A violation here means the + # supplementary-rule id assignment is broken. + defp assert_unique_rule_ids!(rules) do + ids = Enum.map(rules, & &1.id) + duplicates = ids -- Enum.uniq(ids) + + if duplicates == [] do + :ok + else + raise "MagicSets transform produced duplicate rule IDs: #{inspect(Enum.uniq(duplicates))}" + end + end + defp adornment(pattern) do Enum.map_join(pattern, "", fn :_ -> "f" @@ -180,29 +196,31 @@ defmodule ExDatalog.MagicSets do magic_atom = %IR.Atom{relation: magic_rel, terms: magic_terms} rewritten = %IR.Rule{rule | body: [{:positive, magic_atom} | rule.body]} - supplementary = + recursive_atoms = rule.body |> Enum.with_index() |> Enum.filter(fn {{:positive, %IR.Atom{relation: ^goal_relation}}, _idx} -> true _ -> false end) - |> Enum.map(fn {{:positive, body_atom}, idx} -> + + # Each supplementary rule gets a distinct id so the transformed IR keeps the + # rule-id uniqueness invariant (a rule may contain several recursive body + # atoms, e.g. `r(X,Z) :- r(X,Y), r(Y,Z)`). + {supplementary, final_id} = + Enum.map_reduce(recursive_atoms, next_id, fn {{:positive, body_atom}, idx}, id -> prefix_body = Enum.take(rule.body, idx) body_magic_terms = bound_head_terms(body_atom, bound_positions) sup_rule = %IR.Rule{ - id: next_id, + id: id, head: %IR.Atom{relation: magic_rel, terms: body_magic_terms}, body: [{:positive, magic_atom} | prefix_body], stratum: rule.stratum } - {sup_rule, next_id} + {sup_rule, id + 1} end) - |> Enum.map(fn {sup_rule, _id} -> sup_rule end) - - final_id = next_id + length(supplementary) {rewritten, supplementary, final_id} end diff --git a/lib/ex_datalog/schema.ex b/lib/ex_datalog/schema.ex index dc90c37..f5e6521 100644 --- a/lib/ex_datalog/schema.ex +++ b/lib/ex_datalog/schema.ex @@ -74,7 +74,9 @@ defmodule ExDatalog.Schema do field :child, :atom end - Supported field types: `:atom`, `:integer`, `:string`, `:any`. + Supported field types: `:atom`, `:integer`, `:string`, `:any`. A relation + must declare at least one field; zero-arity relations are not supported and + raise `ExDatalog.DSL.CompileError` at compile time. ## Fact DSL @@ -539,6 +541,13 @@ defmodule ExDatalog.Schema do def __define_relation__(module, name, block) do fields = extract_fields(block) + if fields == [] do + raise ExDatalog.DSL.CompileError, + message: + "relation #{inspect(name)}: a relation must declare at least one field; " <> + "zero-arity relations are not supported" + end + Module.put_attribute(module, :ex_datalog_relations, %ExDatalog.Schema.RelationMeta{ name: name, fields: fields diff --git a/test/ex_datalog/aggregate_test.exs b/test/ex_datalog/aggregate_test.exs index 3e4d099..79c4a00 100644 --- a/test/ex_datalog/aggregate_test.exs +++ b/test/ex_datalog/aggregate_test.exs @@ -174,6 +174,83 @@ defmodule ExDatalog.AggregateTest do assert {:error, errors} = ExDatalog.materialize(program) assert Enum.any?(errors, fn e -> e.kind == :unbound_constraint_variable end) end + + test "rejects aggregate whose result variable is not in the head" do + program = + Program.new() + |> Program.add_relation("emp", [:atom, :atom]) + |> Program.add_relation("dept_count", [:atom]) + |> Program.add_rule( + Rule.new( + # head is [D] only; the aggregate result N is missing + Atom.new("dept_count", [Term.var("D")]), + [{:positive, Atom.new("emp", [Term.var("E"), Term.var("D")])}], + [Constraint.count(Term.var("E"), Term.var("N"))] + ) + ) + + assert {:error, errors} = ExDatalog.materialize(program) + assert Enum.any?(errors, fn e -> e.kind == :aggregate_result_not_in_head end) + end + end + + describe "aggregate integer-input guards" do + defp non_integer_agg_program(agg_constraint) do + Program.new() + |> Program.add_relation("sal", [:atom, :atom]) + |> Program.add_relation("total", [:atom, :atom]) + |> Program.add_fact("sal", [:eng, :not_a_number]) + |> Program.add_rule( + Rule.new( + Atom.new("total", [Term.var("D"), Term.var("T")]), + [{:positive, Atom.new("sal", [Term.var("D"), Term.var("A")])}], + [agg_constraint] + ) + ) + end + + test "sum raises ArgumentError on a non-integer input" do + program = non_integer_agg_program(Constraint.sum(Term.var("A"), Term.var("T"))) + + assert_raise ArgumentError, ~r/sum aggregate requires integer inputs/, fn -> + ExDatalog.materialize(program) + end + end + + test "min raises ArgumentError on a non-integer input" do + program = non_integer_agg_program(Constraint.min(Term.var("A"), Term.var("T"))) + + assert_raise ArgumentError, ~r/min aggregate requires integer inputs/, fn -> + ExDatalog.materialize(program) + end + end + + test "max raises ArgumentError on a non-integer input" do + program = non_integer_agg_program(Constraint.max(Term.var("A"), Term.var("T"))) + + assert_raise ArgumentError, ~r/max aggregate requires integer inputs/, fn -> + ExDatalog.materialize(program) + end + end + + test "count accepts non-integer inputs (counts group size)" do + program = + Program.new() + |> Program.add_relation("item", [:atom, :atom]) + |> Program.add_relation("item_count", [:atom, :integer]) + |> Program.add_fact("item", [:box, :red]) + |> Program.add_fact("item", [:box, :blue]) + |> Program.add_rule( + Rule.new( + Atom.new("item_count", [Term.var("B"), Term.var("N")]), + [{:positive, Atom.new("item", [Term.var("B"), Term.var("C")])}], + [Constraint.count(Term.var("C"), Term.var("N"))] + ) + ) + + assert {:ok, knowledge} = ExDatalog.materialize(program) + assert {:box, 2} in Knowledge.get(knowledge, "item_count") + end end describe "aggregate stratification" do diff --git a/test/ex_datalog/beam_callback_test.exs b/test/ex_datalog/beam_callback_test.exs index 2c061d9..220fde2 100644 --- a/test/ex_datalog/beam_callback_test.exs +++ b/test/ex_datalog/beam_callback_test.exs @@ -104,6 +104,36 @@ defmodule ExDatalog.BeamCallbackTest do {:ok, knowledge} = ExDatalog.materialize(program, callback_timeout_ms: 50) assert MapSet.size(Knowledge.get(knowledge, "ok")) == 0 end + + test "a timed-out callback leaves no stray messages in the caller mailbox" do + program = + Program.new() + |> Program.add_relation("num", [:integer]) + |> Program.add_relation("ok", [:integer]) + |> Program.add_fact("num", [1]) + |> Program.add_rule( + Rule.new( + Atom.new("ok", [Term.var("X")]), + [ + {:positive, Atom.new("num", [Term.var("X")])}, + {:callback, Callback.new(Predicates, :slow, [Term.var("X")])} + ] + ) + ) + + # `slow/1` sleeps 500ms then returns true; with a 50ms timeout the spawned + # process is killed and its late result must be flushed. Run materialize in + # this test process, then wait past the callback's completion time and + # assert nothing leaked into our mailbox. + {:ok, _knowledge} = ExDatalog.materialize(program, callback_timeout_ms: 50) + Process.sleep(600) + + receive do + msg -> flunk("unexpected stray message in mailbox: #{inspect(msg)}") + after + 0 -> :ok + end + end end describe "validation" do diff --git a/test/ex_datalog/magic_sets_test.exs b/test/ex_datalog/magic_sets_test.exs index e6d7613..a8713ca 100644 --- a/test/ex_datalog/magic_sets_test.exs +++ b/test/ex_datalog/magic_sets_test.exs @@ -29,8 +29,81 @@ defmodule ExDatalog.MagicSetsTest do ) end + # A doubly-recursive transitive-closure program: the recursive rule contains + # TWO body atoms referencing the goal relation. This is the shape that + # exercises supplementary-rule id assignment in the magic-sets transform. + defp reach_program(facts) do + base = + Program.new() + |> Program.add_relation("edge", [:atom, :atom]) + |> Program.add_relation("reach", [:atom, :atom]) + + base = Enum.reduce(facts, base, fn {p, c}, acc -> Program.add_fact(acc, "edge", [p, c]) end) + + base + |> Program.add_rule( + Rule.new( + Atom.new("reach", [Term.var("X"), Term.var("Y")]), + [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] + ) + ) + |> Program.add_rule( + Rule.new( + Atom.new("reach", [Term.var("X"), Term.var("Z")]), + [ + {:positive, Atom.new("reach", [Term.var("X"), Term.var("Y")])}, + {:positive, Atom.new("reach", [Term.var("Y"), Term.var("Z")])} + ] + ) + ) + end + @chain [{:a, :b}, {:b, :c}, {:c, :d}, {:d, :e}] + describe "multiple recursive body atoms (rule-id uniqueness)" do + test "transformed IR keeps unique rule IDs" do + {:ok, ir} = ExDatalog.compile(reach_program(@chain)) + {:ok, transformed} = MagicSets.transform(ir, {"reach", [:a, :_]}) + + ids = Enum.map(transformed.rules, & &1.id) + assert ids == Enum.uniq(ids), "transformed rules have duplicate IDs: #{inspect(ids)}" + end + + test "provenance map retains every transformed rule" do + {:ok, ir} = ExDatalog.compile(reach_program(@chain)) + {:ok, transformed} = MagicSets.transform(ir, {"reach", [:a, :_]}) + + rules_map = Map.new(transformed.rules, fn r -> {r.id, r} end) + + assert map_size(rules_map) == length(transformed.rules), + "duplicate rule IDs collapsed the provenance map" + end + + test "goal-restricted result equals the semi-naive subset" do + program = reach_program(@chain) + + {:ok, magic} = + ExDatalog.materialize(program, strategy: :magic_sets, goal: {"reach", [:a, :_]}) + + {:ok, full} = ExDatalog.materialize(program) + + expected = Knowledge.match(full, "reach", [:a, :_]) + assert Knowledge.match(magic, "reach", [:a, :_]) == expected + end + + test "evaluates with explain enabled without losing rules" do + program = reach_program(@chain) + + {:ok, ir} = ExDatalog.compile(program) + {:ok, transformed} = MagicSets.transform(ir, {"reach", [:a, :_]}) + {:ok, knowledge} = ExDatalog.evaluate(transformed, explain: true) + + # Every derived reach fact must have a provenance origin. + assert knowledge.provenance != nil + assert Knowledge.match(knowledge, "reach", [:a, :_]) |> MapSet.size() > 0 + end + end + describe "correctness vs semi-naive" do test "goal-restricted query matches the semi-naive subset" do program = ancestor_program(@chain)