diff --git a/.gitignore b/.gitignore index 4c5bcb7..1cf181c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,8 @@ ex_datalog-*.tar # Dialyzer PLT files. /priv/plts/ + +/livebook/*.exs +/livebook/*.md +/chest/ +.tool-versions diff --git a/CHANGELOG.md b/CHANGELOG.md index aae5ab7..3438c8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.0] - 2025-06-19 + +### Added + +- **Tuple shorthand for rules**: `Program.add_rule/3` and `Program.add_rule/4` accept + `{relation, [terms]}` tuples for heads, `{:polarity, {relation, [terms]}}` for body + literals, and `{:op, args...}` for constraints. Uppercase atoms (`:X`) become variables, + `:_` becomes a wildcard, lowercase atoms and other values become constants. +- `Term.from/1` — converts shorthand values to `Term.t()` following Prolog convention. +- `ExDatalog.Atom.from_tuple/1` — constructs an atom from `{"relation", [terms]}` shorthand. +- `Constraint.from_tuple/1` — constructs a constraint from operator tuples + like `{:neq, :A, :B}`, `{:add, :X, :Y, :Z}`, `{:is_integer, :V}`. + +### Changed + +- **`ExDatalog.Result` renamed to `ExDatalog.Knowledge`** — the struct returned by + `materialize/2` now reflects that it represents a materialized knowledge base, + not a query result. All references updated across source, tests, docs, and livebooks. +- `ExDatalog.query` (2-arity) renamed to `ExDatalog.materialize/2` — the top-level API function + now reflects that it runs the full fixpoint pipeline, not a single query. +- **Telemetry events renamed**: `[:ex_datalog, :query, :start|:stop|:exception]` → + `[:ex_datalog, :materialize, :start|:stop|:exception]`. +- `ExDatalog.validate/1` and `ExDatalog.compile/1` now pass through `{:error, _}` tuples + from the builder pipeline instead of raising `FunctionClauseError`. `materialize/2` + also passes through `{:error, _}` from a failed pipeline step. +- Livebook examples (`quickstart.livemd`, `examples.livemd`, `examples.exs`) converted + to tuple shorthand notation. README quickstart updated accordingly. + ## [0.2.0] - 2025-05-15 ### Added @@ -97,8 +125,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `ExDatalog.Engine.Join` — sequential-scan join (`join/3`), tuple matching (`match_tuple/3`), projection (`project/2`), indexed join (`join_indexed/4`, not yet wired into evaluator) - `ExDatalog.Engine.ConstraintEval` — constraint evaluation (comparison filters, arithmetic extensions) - `ExDatalog.Storage.Map` — default Map/MapSet-based storage backend - - `ExDatalog.Result` — result struct with relations, stats, and provenance fields - - Full pipeline: `ExDatalog.query/1` and `ExDatalog.query/2` public API + - `ExDatalog.Knowledge` — knowledge base struct with relations, stats, and provenance fields + - Full pipeline: `ExDatalog.materialize/2` public API - Phase 5: Negation and stratification - Negative body atoms (`{:negative, %IR.Atom{}}`) evaluated as filters against fully-materialised lower-stratum relations - Stratification validation rejects unstratifiable programs before evaluation diff --git a/README.md b/README.md index 624c9ca..ca276c9 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Add `ex_datalog` to your dependencies in `mix.exs`: ```elixir def deps do [ - {:ex_datalog, "~> 0.2.0"} + {:ex_datalog, "~> 0.3.0"} ] end ``` @@ -63,9 +63,9 @@ The classic Datalog example: compute all ancestors from parent facts. ```elixir alias ExDatalog -alias ExDatalog.{Program, Rule, Atom, Term} +alias ExDatalog.{Program, Knowledge} -{:ok, result} = +{:ok, knowledge} = Program.new() |> Program.add_relation("parent", [:atom, :atom]) |> Program.add_relation("ancestor", [:atom, :atom]) @@ -73,33 +73,52 @@ alias ExDatalog.{Program, Rule, Atom, Term} |> Program.add_fact("parent", [:bob, :carol]) |> Program.add_fact("parent", [:carol, :dave]) |> Program.add_rule( - Rule.new( - Atom.new("ancestor", [Term.var("X"), Term.var("Y")]), - [{:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}] - ) - ) + {"ancestor", [:X, :Y]}, + [{:positive, {"parent", [:X, :Y]}}] + ) |> Program.add_rule( - Rule.new( - Atom.new("ancestor", [Term.var("X"), Term.var("Z")]), - [ - {:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}, - {:positive, Atom.new("ancestor", [Term.var("Y"), Term.var("Z")])} - ] - ) - ) - |> ExDatalog.query() - -result.relations["ancestor"] + {"ancestor", [:X, :Z]}, + [ + {:positive, {"parent", [:X, :Y]}}, + {:positive, {"ancestor", [:Y, :Z]}} + ] + ) + |> ExDatalog.materialize() + +Knowledge.get(knowledge, "ancestor") #=> MapSet.new([{:alice, :bob}, {:bob, :carol}, {:carol, :dave}, #=> {:alice, :carol}, {:bob, :dave}, {:alice, :dave}]) ``` +### Shorthand rule notation + +`add_rule/3` and `add_rule/4` use a tuple-based shorthand that follows +Prolog convention: uppercase atoms become variables, `:_` becomes a wildcard, +and lowercase atoms/other values become constants. + +```elixir +# Base rule: ancestor(X,Y) :- parent(X,Y). +Program.add_rule(program, {"ancestor", [:X, :Y]}, [{:positive, {"parent", [:X, :Y]}}]) + +# With constraints — find high earners: +Program.add_rule(program, {"high_earner", [:X]}, [{:positive, {"income", [:X, :S]}}], [{:gt, :S, 100_000}]) + +# Negation — bachelors are males who are not married: +Program.add_rule(program, {"bachelor", [:X]}, [ + {:positive, {"male", [:X]}}, + {:negative, {"married", [:X, :_]}} +]) +``` + +The struct-based `add_rule/2` with `Rule.new/3` remains available for +full control over term types. + ### Arithmetic constraints Compute derived values in rules. Find numbers and their doubles: ```elixir -{:ok, result} = +{:ok, knowledge} = Program.new() |> Program.add_relation("number", [:integer]) |> Program.add_relation("doubled", [:integer, :integer]) @@ -107,15 +126,13 @@ Compute derived values in rules. Find numbers and their doubles: |> Program.add_fact("number", [2]) |> Program.add_fact("number", [3]) |> Program.add_rule( - Rule.new( - Atom.new("doubled", [Term.var("X"), Term.var("Y")]), - [{:positive, Atom.new("number", [Term.var("X")])}], - [Constraint.add(Term.var("X"), {:const, 2}, Term.var("Y"))] - ) - ) - |> ExDatalog.query() - -result.relations["doubled"] + {"doubled", [:X, :Y]}, + [{:positive, {"number", [:X]}}], + [{:add, :X, 2, :Y}] + ) + |> ExDatalog.materialize() + +Knowledge.get(knowledge, "doubled") #=> MapSet.new([{1, 3}, {2, 4}, {3, 5}]) ``` @@ -125,24 +142,21 @@ Filter bindings by Elixir type or list membership: ```elixir # Keep only integer values from a mixed-type relation -Rule.new( - Atom.new("int_value", [Term.var("X")]), - [{:positive, Atom.new("value", [Term.var("X")])}], - [Constraint.type_integer(Term.var("X"))] +Program.add_rule(program, {"int_value", [:N, :V]}, + [{:positive, {"value", [:N, :V]}}], + [{:is_integer, :V}] ) # Keep only "primary" colors -Rule.new( - Atom.new("primary_color", [Term.var("X")]), - [{:positive, Atom.new("color", [Term.var("X")])}], - [Constraint.member(Term.var("X"), {:const, [:red, :blue, :green]})] +Program.add_rule(program, {"primary_color", [:X]}, + [{:positive, {"color", [:X]}}], + [{:member, :X, [:red, :blue, :green]}] ) # Keep only strings that start with "hel" -Rule.new( - Atom.new("hello_word", [Term.var("X")]), - [{:positive, Atom.new("word", [Term.var("X")])}], - [Constraint.starts_with(Term.var("X"), {:const, "hel"})] +Program.add_rule(program, {"hello_word", [:X]}, + [{:positive, {"word", [:X]}}], + [{:starts_with, :X, "hel"}] ) ``` @@ -151,15 +165,10 @@ Rule.new( Use negative body atoms with stratified evaluation. Find people who are not parents: ```elixir -Program.add_rule( - Rule.new( - Atom.new("childless", [Term.var("X")]), - [ - {:positive, Atom.new("person", [Term.var("X")])}, - {:negative, Atom.new("parent", [Term.var("X"), Term.wildcard()])} - ] - ) -) +Program.add_rule(program, {"childless", [:X]}, [ + {:positive, {"person", [:X]}}, + {:negative, {"parent", [:X, :_]}} +]) ``` ### ETS backend @@ -168,11 +177,11 @@ For workloads exceeding ~100K facts, use the ETS backend for off-heap storage and reduced GC pressure: ```elixir -{:ok, result} = ExDatalog.query(program, storage: ExDatalog.Storage.ETS) +{:ok, knowledge} = ExDatalog.materialize(program, storage: ExDatalog.Storage.ETS) # Or with options: -{:ok, result} = - ExDatalog.query(program, +{:ok, knowledge} = + ExDatalog.materialize(program, storage: ExDatalog.Storage.ETS, storage_opts: [access: :public, write_concurrency: true] ) @@ -183,18 +192,18 @@ and reduced GC pressure: Track which rule derived each fact: ```elixir -{:ok, result} = ExDatalog.query(program, explain: true) -result.provenance.fact_origins +{:ok, knowledge} = ExDatalog.materialize(program, explain: true) +knowledge.provenance.fact_origins #=> %{"ancestor" => %{{:alice, :bob} => "rule_0", ...}, ...} ``` ### Telemetry ExDatalog emits `:telemetry` events at the start, end, and on exceptions -during query evaluation: +during materialization: ```elixir -:telemetry.attach("my-handler", [:ex_datalog, :query, :stop], &handle_stop/4, nil) +:telemetry.attach("my-handler", [:ex_datalog, :materialize, :stop], &handle_stop/4, nil) def handle_stop(_event, measurements, metadata, _config) do IO.puts("Query completed in #{measurements.duration} µs (#{measurements.iterations} iterations)") @@ -223,7 +232,7 @@ ExDatalog.Engine.Naive (semi-naive fixpoint) ExDatalog.Storage.Map | ExDatalog.Storage.ETS | v -ExDatalog.Result +ExDatalog.Knowledge ``` The `Storage` behaviour defines the contract for pluggable backends. @@ -271,6 +280,8 @@ reference. - [What is Datalog?](docs/what-is-datalog.md) — introduction, history, Prolog comparison, industry use cases, LLM integration - [Constraints](docs/constraints.md) — constraint types, evaluation, and the dispatch model - [Storage Backends](docs/storage_backends.md) — Map vs ETS, options, capabilities, determinism guarantee +- [Quickstart Tutorial](livebook/quickstart.livemd) — interactive Livebook walkthrough +- [Examples](livebook/examples.livemd) — 10 realistic use cases (RBAC, supply chain, fraud detection, and more) - [API reference](https://hexdocs.pm/ex_datalog) — full module and function documentation Generate docs locally: @@ -337,9 +348,9 @@ The following references are highly recommended for understanding both the theor | Version | Description | |---|---| -| v0.2.0 | ETS backend, constraint behaviour, type/string/membership predicates, capabilities, provenance, telemetry | -| v0.3.0 | Aggregation (`count`, `sum`, `min`, `max`), general predicates as deterministic BEAM callbacks | -| v0.4.0 | Magic sets / demand-driven evaluation, external solver adapter (experimental Z3/Soufflé) | +| v0.3.0 | Tuple shorthand for rules (`add_rule/3`, `add_rule/4`), `Term.from/1`, `ExDatalog.Atom.from_tuple/1`, `Constraint.from_tuple/1`; renamed `Result` → `Knowledge`, `query` → `materialize` | +| v0.4.0 | Sigil DSL (`~d`), aggregation (`count`, `sum`, `min`, `max`), general predicates as deterministic BEAM callbacks | +| v0.5.0 | Magic sets / demand-driven evaluation, external solver adapter (experimental Z3/Soufflé) | | v1.0.0 | Stable public API, hardened production semantics | ## License diff --git a/lib/ex_datalog.ex b/lib/ex_datalog.ex index 1d852f8..4b25b5e 100644 --- a/lib/ex_datalog.ex +++ b/lib/ex_datalog.ex @@ -11,31 +11,31 @@ defmodule ExDatalog do ## Quick Start - alias ExDatalog - alias ExDatalog.{Program, Rule, Atom, Term} - - {:ok, result} = - Program.new() - |> Program.add_relation("parent", [:atom, :atom]) - |> Program.add_relation("ancestor", [:atom, :atom]) - |> Program.add_fact("parent", [:alice, :bob]) - |> Program.add_fact("parent", [:bob, :carol]) - |> Program.add_rule( - Rule.new( - Atom.new("ancestor", [Term.var("X"), Term.var("Y")]), - [{:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}] - ) - ) - |> Program.add_rule( - Rule.new( - Atom.new("ancestor", [Term.var("X"), Term.var("Z")]), - [ - {:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}, - {:positive, Atom.new("ancestor", [Term.var("Y"), Term.var("Z")])} - ] - ) - ) - |> ExDatalog.query() + alias ExDatalog + alias ExDatalog.{Program, Rule, Atom, Term} + + {:ok, knowledge} = + Program.new() + |> Program.add_relation("parent", [:atom, :atom]) + |> Program.add_relation("ancestor", [:atom, :atom]) + |> Program.add_fact("parent", [:alice, :bob]) + |> Program.add_fact("parent", [:bob, :carol]) + |> Program.add_rule( + Rule.new( + Atom.new("ancestor", [Term.var("X"), Term.var("Y")]), + [{:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}] + ) + ) + |> Program.add_rule( + Rule.new( + Atom.new("ancestor", [Term.var("X"), Term.var("Z")]), + [ + {:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}, + {:positive, Atom.new("ancestor", [Term.var("Y"), Term.var("Z")])} + ] + ) + ) + |> ExDatalog.materialize() ## Pipeline @@ -45,17 +45,17 @@ defmodule ExDatalog do 2. `ExDatalog.Validator` — structural + semantic validation 3. `ExDatalog.Compiler` — AST to IR 4. `ExDatalog.Engine` — pluggable evaluation backend - 5. `ExDatalog.Result` — structured result with relation access + 5. `ExDatalog.Knowledge` — the knowledge base produced by evaluation Each step can be invoked individually: - {:ok, validated} = ExDatalog.validate(program) - {:ok, ir} = ExDatalog.compile(program) - {:ok, result} = ExDatalog.evaluate(ir, []) + {:ok, validated} = ExDatalog.validate(program) + {:ok, ir} = ExDatalog.compile(program) + {:ok, knowledge} = ExDatalog.evaluate(ir, []) ## Options - `query/2` and `evaluate/2` accept: + `materialize/2` and `evaluate/2` accept: - `engine` — backend module (default: `ExDatalog.Engine.Naive`) - `storage` — storage module (default: `ExDatalog.Storage.Map`) @@ -74,12 +74,12 @@ defmodule ExDatalog do ## Examples - iex> ExDatalog.new() - %ExDatalog.Program{relations: %{}, facts: [], rules: []} + iex> ExDatalog.new() + %ExDatalog.Program{relations: %{}, facts: [], rules: []} - iex> alias ExDatalog.Program - iex> ExDatalog.new() |> Program.add_relation("edge", [:atom, :atom]) - %ExDatalog.Program{relations: %{"edge" => %{arity: 2, types: [:atom, :atom]}}, facts: [], rules: []} + iex> alias ExDatalog.Program + iex> ExDatalog.new() |> Program.add_relation("edge", [:atom, :atom]) + %ExDatalog.Program{relations: %{"edge" => %{arity: 2, types: [:atom, :atom]}}, facts: [], rules: []} """ @spec new() :: Program.t() @@ -103,11 +103,11 @@ defmodule ExDatalog do ## Examples - iex> alias ExDatalog.Program - iex> program = Program.new() |> Program.add_relation("edge", [:atom, :atom]) - iex> {:ok, validated} = ExDatalog.validate(program) - iex> is_struct(validated, ExDatalog.Program) - true + iex> alias ExDatalog.Program + iex> program = Program.new() |> Program.add_relation("edge", [:atom, :atom]) + iex> {:ok, validated} = ExDatalog.validate(program) + iex> is_struct(validated, ExDatalog.Program) + true """ @spec validate(Program.t()) :: {:ok, Program.t()} | {:error, [Validator.Error.t()]} @@ -115,6 +115,8 @@ defmodule ExDatalog do Validator.validate(program) end + def validate({:error, _} = error), do: error + @doc """ Compiles a validated program to an engine-neutral IR. @@ -127,21 +129,21 @@ defmodule ExDatalog do ## Examples - iex> alias ExDatalog.{Program, Rule, Atom, Term} - iex> program = - ...> Program.new() - ...> |> Program.add_relation("edge", [:atom, :atom]) - ...> |> Program.add_relation("path", [:atom, :atom]) - ...> |> Program.add_rule( - ...> Rule.new( - ...> Atom.new("path", [Term.var("X"), Term.var("Y")]), - ...> [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] - ...> ) - ...> ) - iex> {:ok, ir} = ExDatalog.compile(program) - iex> length(ir.rules) == 1 and length(ir.relations) == 2 - true - true + iex> alias ExDatalog.{Program, Rule, Atom, Term} + iex> program = + ...> Program.new() + ...> |> Program.add_relation("edge", [:atom, :atom]) + ...> |> Program.add_relation("path", [:atom, :atom]) + ...> |> Program.add_rule( + ...> Rule.new( + ...> Atom.new("path", [Term.var("X"), Term.var("Y")]), + ...> [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] + ...> ) + ...> ) + iex> {:ok, ir} = ExDatalog.compile(program) + iex> length(ir.rules) == 1 and length(ir.relations) == 2 + true + true """ @spec compile(Program.t()) :: {:ok, ExDatalog.IR.t()} | {:error, [Validator.Error.t()]} @@ -149,10 +151,12 @@ defmodule ExDatalog do ExDatalog.Compiler.compile(program) end + def compile({:error, _} = error), do: error + @doc """ Evaluates a compiled IR program against a backend engine. - Returns `{:ok, ExDatalog.Result.t()}` or `{:error, reason}`. + Returns `{:ok, ExDatalog.Knowledge.t()}` or `{:error, reason}`. ## Options @@ -161,18 +165,18 @@ defmodule ExDatalog do - `:max_iterations` — fixpoint iteration limit (default: 10_000) - `:timeout_ms` — wall-clock timeout in ms (default: 30_000) """ - @spec evaluate(ExDatalog.IR.t(), keyword()) :: {:ok, ExDatalog.Result.t()} | {:error, term()} + @spec evaluate(ExDatalog.IR.t(), keyword()) :: {:ok, ExDatalog.Knowledge.t()} | {:error, term()} def evaluate(%ExDatalog.IR{} = ir, opts \\ []) do engine = Keyword.get(opts, :engine, ExDatalog.Engine.Naive) engine.evaluate(ir, opts) end @doc """ - One-shot: validate, compile, and evaluate a program. + One-shot: validate, compile, and materialize a program. Equivalent to `validate/1` → `compile/1` → `evaluate/2`. - Returns `{:ok, ExDatalog.Result.t()}` or `{:error, reason}`. + Returns `{:ok, ExDatalog.Knowledge.t()}` or `{:error, reason}`. ## Options @@ -180,22 +184,26 @@ defmodule ExDatalog do ## Examples - iex> alias ExDatalog.{Program, Rule, Atom, Term} - iex> program = - ...> Program.new() - ...> |> Program.add_relation("parent", [:atom, :atom]) - ...> |> Program.add_fact("parent", [:alice, :bob]) - iex> {:ok, result} = ExDatalog.query(program) - iex> ExDatalog.Result.size(result, "parent") - 1 + iex> alias ExDatalog.{Program, Rule, Atom, Term} + iex> program = + ...> Program.new() + ...> |> Program.add_relation("parent", [:atom, :atom]) + ...> |> Program.add_fact("parent", [:alice, :bob]) + iex> {:ok, knowledge} = ExDatalog.materialize(program) + iex> ExDatalog.Knowledge.size(knowledge, "parent") + 1 """ - @spec query(Program.t(), keyword()) :: - {:ok, ExDatalog.Result.t()} | {:error, [Validator.Error.t()] | term()} - def query(%Program{} = program, opts \\ []) do + @spec materialize(Program.t(), keyword()) :: + {:ok, ExDatalog.Knowledge.t()} | {:error, [Validator.Error.t()] | term()} + def materialize(program, opts \\ []) + + def materialize(%Program{} = program, opts) do with {:ok, validated} <- validate(program), {:ok, ir} <- ExDatalog.Compiler.compile(validated) do evaluate(ir, opts) end end + + def materialize({:error, _} = error, _opts), do: error end diff --git a/lib/ex_datalog/atom.ex b/lib/ex_datalog/atom.ex index 63aafb0..8065885 100644 --- a/lib/ex_datalog/atom.ex +++ b/lib/ex_datalog/atom.ex @@ -101,4 +101,33 @@ defmodule ExDatalog.Atom do end def valid?(_), do: false + + @doc """ + Constructs an atom from a shorthand tuple of the form `{relation, terms}`. + + Each term in the list is converted using `Term.from/1`, which follows the + Prolog convention: uppercase atoms become variables, lowercase atoms and + other values become constants, and `:_` becomes a wildcard. + + Accepts either a `{relation, terms}` tuple or an existing `%Atom{}` struct + (passed through unchanged). + + ## Examples + + iex> ExDatalog.Atom.from_tuple({"parent", [:X, :Y]}) + %ExDatalog.Atom{relation: "parent", terms: [{:var, "X"}, {:var, "Y"}]} + + iex> ExDatalog.Atom.from_tuple({"role", [:User, :_ ]}) + %ExDatalog.Atom{relation: "role", terms: [{:var, "User"}, :wildcard]} + + iex> ExDatalog.Atom.from_tuple({"value", [:X, 42]}) + %ExDatalog.Atom{relation: "value", terms: [{:var, "X"}, {:const, 42}]} + + """ + @spec from_tuple({String.t(), [Term.shorthand()]} | t()) :: t() + def from_tuple({relation, terms}) when is_binary(relation) and is_list(terms) do + new(relation, Enum.map(terms, &Term.from/1)) + end + + def from_tuple(%__MODULE__{} = atom), do: atom end diff --git a/lib/ex_datalog/constraint.ex b/lib/ex_datalog/constraint.ex index bcd0f28..43ceeee 100644 --- a/lib/ex_datalog/constraint.ex +++ b/lib/ex_datalog/constraint.ex @@ -570,6 +570,88 @@ defmodule ExDatalog.Constraint do defp valid_right?(_, _), do: false + @doc """ + Constructs a constraint from a shorthand tuple. + + Each term in the tuple is converted using `Term.from/1`, which follows the + Prolog convention: uppercase atoms become variables (`:A` → `{:var, "A"}`), + lowercase atoms and other values become constants, and `:_` becomes a wildcard. + + Accepts either a shorthand tuple or an existing `%Constraint{}` struct + (passed through unchanged). + + ### Comparison constraints (2 terms) + + {:gt, left, right} + {:lt, left, right} + {:gte, left, right} + {:lte, left, right} + {:eq, left, right} + {:neq, left, right} + + ### Arithmetic constraints (3 terms) + + {:add, left, right, result} + {:sub, left, right, result} + {:mul, left, right, result} + {:div, left, right, result} + + ### Type predicates (1 term) + + {:is_integer, term} + {:is_binary, term} + {:is_atom, term} + + ### String predicates (2 terms) + + {:starts_with, left, right} + {:contains, left, right} + + ### Membership (2 terms) + + {:member, left, right} + + ## Examples + + iex> ExDatalog.Constraint.from_tuple({:neq, :A, :B}) + %ExDatalog.Constraint{op: :neq, left: {:var, "A"}, right: {:var, "B"}, result: nil} + + iex> ExDatalog.Constraint.from_tuple({:add, :X, :Y, :Z}) + %ExDatalog.Constraint{op: :add, left: {:var, "X"}, right: {:var, "Y"}, result: {:var, "Z"}} + + iex> ExDatalog.Constraint.from_tuple({:gt, :S, 100_000}) + %ExDatalog.Constraint{op: :gt, left: {:var, "S"}, right: {:const, 100000}, result: nil} + + iex> ExDatalog.Constraint.from_tuple({:is_integer, :V}) + %ExDatalog.Constraint{op: :is_integer, left: {:var, "V"}, right: nil, result: nil} + + iex> ExDatalog.Constraint.from_tuple({:member, :Dept, [:engineering, :infra]}) + %ExDatalog.Constraint{op: :member, left: {:var, "Dept"}, right: {:const, [:engineering, :infra]}, result: nil} + + """ + @spec from_tuple(tuple() | t()) :: t() + def from_tuple(%__MODULE__{} = c), do: c + + def from_tuple({op, left, right, result}) when op in @arithmetic_ops do + arithmetic(op, Term.from(left), Term.from(right), Term.from(result)) + end + + def from_tuple({op, left, right}) when op in @comparison_ops do + comparison(op, Term.from(left), Term.from(right)) + end + + def from_tuple({op, left, right}) when op in @string_ops do + filter(op, Term.from(left), Term.from(right)) + end + + def from_tuple({:member, left, right}) do + member(Term.from(left), Term.from(right)) + end + + def from_tuple({op, term}) when op in @type_ops do + unary(op, Term.from(term)) + end + # --- Extensible constraint behaviour --- @doc """ diff --git a/lib/ex_datalog/engine.ex b/lib/ex_datalog/engine.ex index 86783d0..f6e266a 100644 --- a/lib/ex_datalog/engine.ex +++ b/lib/ex_datalog/engine.ex @@ -9,7 +9,7 @@ defmodule ExDatalog.Engine do @type ir :: ExDatalog.IR.t() @type opts :: keyword() - @type reply :: {:ok, ExDatalog.Result.t()} | {:error, term()} + @type reply :: {:ok, ExDatalog.Knowledge.t()} | {:error, term()} @callback evaluate(ir, opts) :: reply @callback name() :: String.t() diff --git a/lib/ex_datalog/engine/naive.ex b/lib/ex_datalog/engine/naive.ex index b27c539..be4450e 100644 --- a/lib/ex_datalog/engine/naive.ex +++ b/lib/ex_datalog/engine/naive.ex @@ -53,7 +53,7 @@ defmodule ExDatalog.Engine.Naive do alias ExDatalog.Engine.Evaluator alias ExDatalog.IR - alias ExDatalog.Result + alias ExDatalog.Knowledge @default_max_iterations 10_000 @default_timeout_ms 30_000 @@ -72,7 +72,7 @@ defmodule ExDatalog.Engine.Naive do Evaluates a compiled IR program to fixpoint. Accepts a compiled `IR.t()` struct (from `Compiler.compile/1`) and an - optional keyword list of options. Returns `{:ok, %Result{}}` on success + optional keyword list of options. Returns `{:ok, %Knowledge{}}` on success or `{:error, reason}` on failure. ## Options @@ -85,7 +85,7 @@ defmodule ExDatalog.Engine.Naive do See `#{inspect(__MODULE__)}` moduledoc for algorithm details. """ - @spec evaluate(IR.t(), keyword()) :: {:ok, Result.t()} | {:error, term()} + @spec evaluate(IR.t(), keyword()) :: {:ok, Knowledge.t()} | {:error, term()} def evaluate(%IR{} = ir, opts \\ []) do ExDatalog.Telemetry.emit_start(ir) start_time = System.monotonic_time(:microsecond) @@ -229,7 +229,7 @@ defmodule ExDatalog.Engine.Naive do nil end - %Result{ + %Knowledge{ relations: all_rels, stats: %{ iterations: total_iterations, diff --git a/lib/ex_datalog/explain.ex b/lib/ex_datalog/explain.ex index 5d14324..fec1969 100644 --- a/lib/ex_datalog/explain.ex +++ b/lib/ex_datalog/explain.ex @@ -1,8 +1,8 @@ defmodule ExDatalog.Explain do @moduledoc """ - Derivation tree explanation for Datalog query results. + Derivation tree explanation for Datalog knowledge. - When a query is executed with `explain: true`, the result includes provenance + When a program is materialized with `explain: true`, the knowledge base includes provenance data recording which rule derived each fact. This module reconstructs the derivation tree from that provenance data. @@ -20,14 +20,14 @@ defmodule ExDatalog.Explain do ## Usage - {:ok, result} = ExDatalog.query(program, explain: true) - {:ok, tree} = ExDatalog.Explain.explain(result, "ancestor", {:alice, :carol}) + {:ok, knowledge} = ExDatalog.materialize(program, explain: true) + {:ok, tree} = ExDatalog.Explain.explain(knowledge, "ancestor", {:alice, :carol}) The tree shows how the fact was derived, recursively expanding each derived body atom back to its own derivation. EDB facts terminate as `:base_fact`. """ - alias ExDatalog.{IR, Result} + alias ExDatalog.{IR, Knowledge} defmodule Node do @moduledoc """ @@ -76,18 +76,18 @@ defmodule ExDatalog.Explain do ...> [{:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}] ...> ) ...> ) - iex> {:ok, result} = ExDatalog.query(program, explain: true) - iex> {:ok, tree} = Explain.explain(result, "ancestor", {:alice, :bob}) + iex> {:ok, knowledge} = ExDatalog.materialize(program, explain: true) + iex> {:ok, tree} = Explain.explain(knowledge, "ancestor", {:alice, :bob}) iex> tree.rule_id 0 """ - @spec explain(Result.t(), String.t(), tuple()) :: + @spec explain(Knowledge.t(), String.t(), tuple()) :: {:ok, derivation()} | {:error, :no_provenance | :not_found} - def explain(%Result{provenance: nil}, _relation, _tuple) do + def explain(%Knowledge{provenance: nil}, _relation, _tuple) do {:error, :no_provenance} end - def explain(%Result{provenance: provenance}, relation, tuple) do + def explain(%Knowledge{provenance: provenance}, relation, tuple) do %{fact_origins: origins} = provenance build_tree(relation, tuple, origins, provenance.rules, %{}) end diff --git a/lib/ex_datalog/result.ex b/lib/ex_datalog/knowledge.ex similarity index 62% rename from lib/ex_datalog/result.ex rename to lib/ex_datalog/knowledge.ex index 462fa87..d9a4fbd 100644 --- a/lib/ex_datalog/result.ex +++ b/lib/ex_datalog/knowledge.ex @@ -1,10 +1,15 @@ -defmodule ExDatalog.Result do +defmodule ExDatalog.Knowledge do @moduledoc """ - Structured result from Datalog evaluation. + The complete knowledge base produced by Datalog evaluation. - Contains the derived fact sets for each relation, along with evaluation + Contains the derived fact sets for every relation, along with evaluation statistics (iteration count, duration, relation sizes). + The name reflects what this struct *is*: after applying all rules to all + facts until no new facts can be derived, you have a **knowledge base** — + every relation fully materialised, every derivable fact present. You then + *query* this knowledge with `get/2` and `match/3`. + When provenance tracking is enabled (via `explain: true`), the `provenance` field records which rule derived each fact. Base facts (EDB) are attributed as `:base`. This field is `nil` when provenance tracking is disabled, @@ -15,7 +20,7 @@ defmodule ExDatalog.Result do - `get/2` — all tuples for a relation. - `match/3` — tuples matching a pattern (`:_` for wildcard). - `size/2` — number of tuples in a relation. - - `relations/1` — list of all relation names in the result. + - `relations/1` — list of all relation names in the knowledge base. """ @type provenance :: %{ @@ -43,8 +48,8 @@ defmodule ExDatalog.Result do ## Examples - iex> result = %ExDatalog.Result{relations: %{"parent" => MapSet.new([{:alice, :bob}])}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{"parent" => 1}}} - iex> ExDatalog.Result.get(result, "parent") |> MapSet.to_list() + iex> knowledge = %ExDatalog.Knowledge{relations: %{"parent" => MapSet.new([{:alice, :bob}])}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{"parent" => 1}}} + iex> ExDatalog.Knowledge.get(knowledge, "parent") |> MapSet.to_list() [{:alice, :bob}] """ @@ -61,8 +66,8 @@ defmodule ExDatalog.Result do ## Examples - iex> result = %ExDatalog.Result{relations: %{"parent" => MapSet.new([{:alice, :bob}, {:carol, :dave}, {:alice, :carol}])}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{"parent" => 3}}} - iex> ExDatalog.Result.match(result, "parent", [:alice, :_]) |> MapSet.to_list() |> Enum.sort() + iex> knowledge = %ExDatalog.Knowledge{relations: %{"parent" => MapSet.new([{:alice, :bob}, {:carol, :dave}, {:alice, :carol}])}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{"parent" => 3}}} + iex> ExDatalog.Knowledge.match(knowledge, "parent", [:alice, :_]) |> MapSet.to_list() |> Enum.sort() [{:alice, :bob}, {:alice, :carol}] """ @@ -84,8 +89,8 @@ defmodule ExDatalog.Result do ## Examples - iex> result = %ExDatalog.Result{relations: %{"parent" => MapSet.new([{:alice, :bob}, {:carol, :dave}])}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{"parent" => 2}}} - iex> ExDatalog.Result.size(result, "parent") + iex> knowledge = %ExDatalog.Knowledge{relations: %{"parent" => MapSet.new([{:alice, :bob}, {:carol, :dave}])}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{"parent" => 2}}} + iex> ExDatalog.Knowledge.size(knowledge, "parent") 2 """ @@ -95,12 +100,12 @@ defmodule ExDatalog.Result do end @doc """ - Returns all relation names present in the result. + Returns all relation names present in the knowledge base. ## Examples - iex> result = %ExDatalog.Result{relations: %{"parent" => MapSet.new(), "ancestor" => MapSet.new()}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{}}} - iex> Enum.sort(ExDatalog.Result.relations(result)) + iex> knowledge = %ExDatalog.Knowledge{relations: %{"parent" => MapSet.new(), "ancestor" => MapSet.new()}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{}}} + iex> Enum.sort(ExDatalog.Knowledge.relations(knowledge)) ["ancestor", "parent"] """ diff --git a/lib/ex_datalog/program.ex b/lib/ex_datalog/program.ex index 2e3e72a..1c01dda 100644 --- a/lib/ex_datalog/program.ex +++ b/lib/ex_datalog/program.ex @@ -25,16 +25,16 @@ defmodule ExDatalog.Program do passes through unchanged. This means you can pipe freely and check for errors at the end: - {:ok, result} = + {:ok, knowledge} = Program.new() |> Program.add_relation("edge", [:atom, :atom]) |> Program.add_relation("path", [:atom, :atom]) |> Program.add_fact("edge", [:a, :b]) - |> Program.add_rule(...) - |> ExDatalog.query() + |> Program.add_rule({"path", [:X, :Y]}, [{:positive, {"edge", [:X, :Y]}}]) + |> ExDatalog.materialize() If `add_relation/3` fails, the `{:error, msg}` tuple flows through - `add_fact/3` and `add_rule/2` without raising, and `ExDatalog.query/1` + `add_fact/3` and `add_rule/3` without raising, and `ExDatalog.materialize/1` will detect the error struct and return `{:error, [msg]}`. Semantic validation (variable safety, stratification, constraint binding) @@ -48,6 +48,41 @@ defmodule ExDatalog.Program do (e.g., programs assembled by directly modifying the struct, which bypasses builder validation). + ## Shorthand rule notation + + `add_rule/3` and `add_rule/4` accept a more ergonomic tuple-based notation + that avoids the need for explicit `Rule.new/3`, `ExDatalog.Atom.new/2`, and + `Term.var/1` calls: + + - **Head** — `{"relation", [terms...]}` where each term follows the + Prolog convention: uppercase atoms become variables (`:X` → `{:var, "X"}`), + `:_` becomes a wildcard, lowercase atoms and other values become constants. + + - **Body** — `{:positive, {"rel", [terms...]}}` or + `{:negative, {"rel", [terms...]}}` for each literal. + + - **Constraints** — operator tuples like `{:neq, :A, :B}` for comparisons, + `{:add, :X, :Y, :Z}` for arithmetic, `{:is_integer, :V}` for type + predicates, `{:starts_with, :E, "prefix"}` for string predicates, and + `{:member, :X, [:a, :b]}` for membership. + + Program.add_rule(program, + {"ancestor", [:X, :Z]}, + [ + {:positive, {"parent", [:X, :Y]}}, + {:positive, {"ancestor", [:Y, :Z]}} + ] + ) + + Program.add_rule(program, + {"high_earner", [:X]}, + [{:positive, {"income", [:X, :S]}}], + [{:gt, :S, 100_000}] + ) + + The struct-based `add_rule/2` remains available for cases where you need + full control over term types. + ## Example iex> alias ExDatalog.{Program, Atom, Rule, Term} @@ -58,10 +93,8 @@ defmodule ExDatalog.Program do ...> |> Program.add_fact("parent", [:alice, :bob]) ...> |> Program.add_fact("parent", [:bob, :carol]) ...> |> Program.add_rule( - ...> Rule.new( - ...> Atom.new("ancestor", [Term.var("X"), Term.var("Y")]), - ...> [{:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}] - ...> ) + ...> {"ancestor", [:X, :Y]}, + ...> [{:positive, {"parent", [:X, :Y]}}] ...> ) iex> length(program.facts) == 2 true @@ -70,13 +103,20 @@ defmodule ExDatalog.Program do """ - alias ExDatalog.{Atom, Rule, Term} + alias ExDatalog.{Atom, Constraint, Rule, Term} @type relation_name :: String.t() @type ir_type :: :integer | :string | :atom | :any @type relation_schema :: %{arity: non_neg_integer(), types: [ir_type()]} @type fact_values :: [Term.value()] + @type head_shorthand :: {String.t(), [Term.shorthand()]} | Atom.t() + @type body_literal_shorthand :: + {:positive | :negative, {String.t(), [Term.shorthand()]}} + | {:positive | :negative, Atom.t()} + | Rule.literal() + @type constraint_shorthand :: tuple() | Constraint.t() + @type t :: %__MODULE__{ relations: %{relation_name() => relation_schema()}, facts: [{relation_name(), fact_values()}], @@ -236,6 +276,76 @@ defmodule ExDatalog.Program do def add_rule({:error, _} = err, %Rule{}), do: err + @doc """ + Adds a rule using shorthand notation for the head atom, body literals, + and constraints. + + This is a more ergonomic alternative to `add_rule/2` that avoids the need + for explicit `Rule.new/3`, `ExDatalog.Atom.new/2`, and `ExDatalog.Term.var/1` calls. + + The **head** is a tuple `{"relation", [terms...]}` where each term follows + the Prolog-inspired convention: + + - Uppercase atoms become logic variables (`:X` → `{:var, "X"}`) + - `:_` becomes a wildcard + - Lowercase atoms and other values become constants + + Each **body literal** is `{:positive, {"rel", [terms...]}}` or + `{:negative, {"rel", [terms...]}}`. You may also mix in structs like + `{:positive, ExDatalog.Atom.new(...)}`. + + Each **constraint** is an operator tuple like `{:neq, :A, :B}` or + `{:add, :X, :Y, :Z}`. You may also use existing `%Constraint{}` structs. + + Returns `{:error, reason}` if any structural check fails (same validation + as `add_rule/2`). + + ## Examples + + iex> alias ExDatalog.Program + iex> program = + ...> Program.new() + ...> |> Program.add_relation("parent", [:atom, :atom]) + ...> |> Program.add_relation("ancestor", [:atom, :atom]) + iex> result = Program.add_rule(program, + ...> {"ancestor", [:X, :Y]}, + ...> [{:positive, {"parent", [:X, :Y]}}] + ...> ) + iex> length(result.rules) == 1 + true + + iex> alias ExDatalog.Program + iex> program = + ...> Program.new() + ...> |> Program.add_relation("income", [:atom, :integer]) + ...> |> Program.add_relation("high_earner", [:atom]) + iex> result = Program.add_rule(program, + ...> {"high_earner", [:X]}, + ...> [{:positive, {"income", [:X, :S]}}], + ...> [{:gt, :S, 100_000}] + ...> ) + iex> length(result.rules) == 1 + true + + """ + @spec add_rule(t(), head_shorthand(), [body_literal_shorthand()], [constraint_shorthand()]) :: + t() | {:error, String.t()} + def add_rule(program, head, body, constraints \\ []) + + def add_rule(%__MODULE__{} = program, head, body, constraints) + when is_list(body) and is_list(constraints) do + rule = + Rule.new( + Atom.from_tuple(head), + Enum.map(body, &body_literal_from_tuple/1), + Enum.map(constraints, &Constraint.from_tuple/1) + ) + + add_rule(program, rule) + end + + def add_rule({:error, _} = err, _head, _body, _constraints), do: err + @doc """ Returns the schema for a relation, or `nil` if not defined. @@ -274,6 +384,16 @@ defmodule ExDatalog.Program do # --- Private helpers --- + defp body_literal_from_tuple({polarity, {_, _} = atom_tuple}) + when polarity in [:positive, :negative] do + {polarity, Atom.from_tuple(atom_tuple)} + end + + defp body_literal_from_tuple({polarity, %Atom{} = atom}) + when polarity in [:positive, :negative] do + {polarity, atom} + end + defp validate_atom(program, atom) do with :ok <- validate_atom_relation(atom, program), :ok <- validate_atom_arity(atom, program), diff --git a/lib/ex_datalog/telemetry.ex b/lib/ex_datalog/telemetry.ex index 207846a..470e03f 100644 --- a/lib/ex_datalog/telemetry.ex +++ b/lib/ex_datalog/telemetry.ex @@ -9,9 +9,9 @@ defmodule ExDatalog.Telemetry do | Event | When | Measurements | Metadata | |---|---|---|---| - | `[:ex_datalog, :query, :start]` | Before evaluation | `%{system_time: ...}` | `%{relation_count: ..., stratum_count: ...}` | - | `[:ex_datalog, :query, :stop]` | After evaluation | `%{duration: ..., iterations: ...}` | `%{relation_sizes: ..., stratum_count: ..., storage_type: ...}` | - | `[:ex_datalog, :query, :exception]` | On exception | `%{duration: ...}` | `%{kind: ..., reason: ..., stacktrace: ..., stratum_count: ...}` | + | `[:ex_datalog, :materialize, :start]` | Before evaluation | `%{system_time: ...}` | `%{relation_count: ..., stratum_count: ...}` | + | `[:ex_datalog, :materialize, :stop]` | After evaluation | `%{duration: ..., iterations: ...}` | `%{relation_sizes: ..., stratum_count: ..., storage_type: ...}` | + | `[:ex_datalog, :materialize, :exception]` | On exception | `%{duration: ...}` | `%{kind: ..., reason: ..., stacktrace: ..., stratum_count: ...}` | The `:start` event fires before evaluation begins. The `:stop` event fires after evaluation completes (success or error). The `:exception` event fires @@ -26,25 +26,25 @@ defmodule ExDatalog.Telemetry do alias ExDatalog.IR @doc """ - Returns the `[:ex_datalog, :query, :start]` event name. + Returns the `[:ex_datalog, :materialize, :start]` event name. """ - @spec query_start() :: [:ex_datalog | :query | :start, ...] - def query_start, do: [:ex_datalog, :query, :start] + @spec materialize_start() :: [:ex_datalog | :materialize | :start, ...] + def materialize_start, do: [:ex_datalog, :materialize, :start] @doc """ - Returns the `[:ex_datalog, :query, :stop]` event name. + Returns the `[:ex_datalog, :materialize, :stop]` event name. """ - @spec query_stop() :: [:ex_datalog | :query | :stop, ...] - def query_stop, do: [:ex_datalog, :query, :stop] + @spec materialize_stop() :: [:ex_datalog | :materialize | :stop, ...] + def materialize_stop, do: [:ex_datalog, :materialize, :stop] @doc """ - Returns the `[:ex_datalog, :query, :exception]` event name. + Returns the `[:ex_datalog, :materialize, :exception]` event name. """ - @spec query_exception() :: [:ex_datalog | :query | :exception, ...] - def query_exception, do: [:ex_datalog, :query, :exception] + @spec materialize_exception() :: [:ex_datalog | :materialize | :exception, ...] + def materialize_exception, do: [:ex_datalog, :materialize, :exception] @doc """ - Emits the `[:ex_datalog, :query, :start]` event. + Emits the `[:ex_datalog, :materialize, :start]` event. Called before evaluation begins. @@ -60,14 +60,14 @@ defmodule ExDatalog.Telemetry do @spec emit_start(IR.t()) :: :ok def emit_start(%IR{} = ir) do :telemetry.execute( - query_start(), + materialize_start(), %{system_time: System.monotonic_time()}, %{relation_count: length(ir.relations), stratum_count: length(ir.strata)} ) end @doc """ - Emits the `[:ex_datalog, :query, :stop]` event. + Emits the `[:ex_datalog, :materialize, :stop]` event. Called after evaluation completes (success or error). The `start_time` argument should be the `System.monotonic_time(:microsecond)` value captured @@ -93,14 +93,14 @@ defmodule ExDatalog.Telemetry do ) :: :ok def emit_stop(start_time, iterations, relation_sizes, stratum_count, storage_type) do :telemetry.execute( - query_stop(), + materialize_stop(), %{duration: System.monotonic_time(:microsecond) - start_time, iterations: iterations}, %{relation_sizes: relation_sizes, stratum_count: stratum_count, storage_type: storage_type} ) end @doc """ - Emits the `[:ex_datalog, :query, :exception]` event. + Emits the `[:ex_datalog, :materialize, :exception]` event. Called when an exception terminates evaluation. The `kind`, `reason`, and `stacktrace` should come from `__STACKTRACE__` inside a `rescue` or `catch`. @@ -119,7 +119,7 @@ defmodule ExDatalog.Telemetry do @spec emit_exception(integer(), Exception.kind(), term(), list(), non_neg_integer()) :: :ok def emit_exception(start_time, kind, reason, stacktrace, stratum_count) do :telemetry.execute( - query_exception(), + materialize_exception(), %{duration: System.monotonic_time(:microsecond) - start_time}, %{kind: kind, reason: reason, stacktrace: stacktrace, stratum_count: stratum_count} ) diff --git a/lib/ex_datalog/term.ex b/lib/ex_datalog/term.ex index 28dc7d2..c91c030 100644 --- a/lib/ex_datalog/term.ex +++ b/lib/ex_datalog/term.ex @@ -27,6 +27,7 @@ defmodule ExDatalog.Term do @type var_name :: String.t() @type value :: integer() | String.t() | atom() | list() @type t :: {:var, var_name()} | {:const, value()} | :wildcard + @type shorthand :: atom() | integer() | String.t() | list() | t() @doc """ Constructs a logic variable term. @@ -187,4 +188,74 @@ defmodule ExDatalog.Term do def variables(terms) when is_list(terms) do for {:var, name} <- terms, do: name end + + @doc """ + Converts a shorthand value to a term. + + Follows Prolog convention for distinguishing variables from constants: + + - **Uppercase atoms** become logic variables: `:A` → `{:var, "A"}`, + `:Pkg` → `{:var, "Pkg"}` + - **`:_`** becomes a wildcard: `:_` → `:wildcard` + - **Lowercase atoms** become constants: `:alice` → `{:const, :alice}` + - **Integers** become constants: `42` → `{:const, 42}` + - **Strings** become constants: `"hello"` → `{:const, "hello"}` + - **Lists** become constants: `[:a, :b]` → `{:const, [:a, :b]}` + - **Existing terms** pass through unchanged + + This is the inverse of writing out `Term.var/1`, `Term.const/1`, and + `Term.wildcard/0` explicitly, and is designed for use with the + tuple-shorthand forms of `Program.add_rule/3` and `Program.add_rule/4`. + + ## Examples + + iex> ExDatalog.Term.from(:A) + {:var, "A"} + + iex> ExDatalog.Term.from(:Pkg) + {:var, "Pkg"} + + iex> ExDatalog.Term.from(:_) + :wildcard + + iex> ExDatalog.Term.from(:alice) + {:const, :alice} + + iex> ExDatalog.Term.from(42) + {:const, 42} + + iex> ExDatalog.Term.from("hello") + {:const, "hello"} + + iex> ExDatalog.Term.from([:a, :b]) + {:const, [:a, :b]} + + iex> ExDatalog.Term.from({:var, "X"}) + {:var, "X"} + + iex> ExDatalog.Term.from({:const, :ok}) + {:const, :ok} + + iex> ExDatalog.Term.from(:wildcard) + :wildcard + + """ + @spec from(shorthand()) :: t() + def from({:var, _} = var), do: var + def from({:const, _} = const), do: const + def from(:wildcard), do: :wildcard + + def from(atom) when is_atom(atom) do + name = Kernel.to_string(atom) + + cond do + atom == :_ -> :wildcard + String.match?(name, ~r/^[A-Z]/) -> {:var, name} + true -> {:const, atom} + end + end + + def from(value) when is_integer(value), do: {:const, value} + def from(value) when is_binary(value), do: {:const, value} + def from(value) when is_list(value), do: {:const, value} end diff --git a/livebook/examples.livemd b/livebook/examples.livemd new file mode 100644 index 0000000..a9f285f --- /dev/null +++ b/livebook/examples.livemd @@ -0,0 +1,2462 @@ + + +# ExDatalog Examples + +```elixir +Mix.install([ + {:ex_datalog, path: Path.expand("..", __DIR__), env: :prod}, +]) +``` + +## Introduction + +Here are a dozen _"realistic"_ use cases for the ex_datalog library. Datalog shines in scenarios involving recursive graphs, hierarchical relationships, constraints, and pattern matching. + +## Setup + +```elixir +alias ExDatalog +alias ExDatalog.{Program, Knowledge} +``` + + + +``` +[ExDatalog.Program, ExDatalog.Knowledge] +``` + +--- + +## 1. Role-Based Access Control (RBAC) & Authorization + +**Use Case:** Modern applications often have complex authorization rules where roles inherit permissions from parent roles. Datalog can easily resolve the transitive closure of role inheritance to definitively answer _"Does user X have role Y?"_ + +```elixir +program = + Program.new() + |> Program.add_relation("role_parent", [:atom, :atom]) + |> Program.add_relation("user_direct_role", [:atom, :atom]) + |> Program.add_relation("has_role", [:atom, :atom]) + # A user has a role if assigned directly + |> Program.add_rule( + {"has_role", [:User, :Role]}, + [{:positive, {"user_direct_role", [:User, :Role]}}] + ) + # A user has a role if they have a parent role that grants it + |> Program.add_rule( + {"has_role", [:User, :Role]}, + [ + {:positive, {"has_role", [:User, :ParentRole]}}, + {:positive, {"role_parent", [:ParentRole, :Role]}} + ] + ) + +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "has_role" => %{arity: 2, types: [:atom, :atom]}, + "role_parent" => %{arity: 2, types: [:atom, :atom]}, + "user_direct_role" => %{arity: 2, types: [:atom, :atom]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "has_role", terms: [var: "User", var: "Role"]}, + body: [ + positive: %ExDatalog.Atom{relation: "has_role", terms: [var: "User", var: "ParentRole"]}, + positive: %ExDatalog.Atom{relation: "role_parent", terms: [var: "ParentRole", var: "Role"]} + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "has_role", terms: [var: "User", var: "Role"]}, + body: [ + positive: %ExDatalog.Atom{relation: "user_direct_role", terms: [var: "User", var: "Role"]} + ], + constraints: [] + } + ] +} +``` + +#### The Knowledge + +This dataset models a three-tier role hierarchy (admin, editor, viewer) and assigns three users to distinct levels. It demonstrates how high-level permissions naturally cascade down to include lower-level access rights. Here Alice inherits `:editor` and `:viewer` through `:admin` + +```elixir +{:ok, knowledge} = + program + |> Program.add_fact("role_parent", [:admin, :editor]) + |> Program.add_fact("role_parent", [:editor, :viewer]) + |> Program.add_fact("user_direct_role", [:alice, :admin]) + |> Program.add_fact("user_direct_role", [:bob, :editor]) + |> Program.add_fact("user_direct_role", [:charlie, :viewer]) + |> ExDatalog.materialize() +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "has_role" => MapSet.new([ + alice: :admin, + alice: :editor, + alice: :viewer, + bob: :editor, + bob: :viewer, + charlie: :viewer + ]), + "role_parent" => MapSet.new([admin: :editor, editor: :viewer]), + "user_direct_role" => MapSet.new([alice: :admin, bob: :editor, charlie: :viewer]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 4760, + iterations: 3, + relation_sizes: %{"has_role" => 6, "role_parent" => 2, "user_direct_role" => 3} + }, + provenance: nil + }} +``` + +Find all roles assigned to a specific user: + +```elixir +# Returns all roles (direct and inherited) for :alice +MapSet.new([alice: :admin, alice: :editor, alice: :viewer]) == + Knowledge.match(knowledge, "has_role", [:alice, :_]) +``` + + + +``` +true +``` + +Find all users who hold a specific privileged role: + +```elixir +# Useful for auditing who has admin access +MapSet.new([alice: :admin]) == Knowledge.match(knowledge, "has_role", [:_, :admin]) +``` + + + +``` +true +``` + +Verify a specific permission: + +```elixir +# Authorization check gate: Does Bob have the editor role? +MapSet.new([bob: :editor]) == Knowledge.match(knowledge, "has_role", [:bob, :editor]) +``` + + + +``` +true +``` + +--- + +## 2. Supply Chain / Bill of Materials (BOM) Tracking + +**Use Case:** Manufacturing and software supply chains need to know if a defective or vulnerable sub-component affects a finished product. This defines a structural BOM. An assembly contains a part if it uses it directly, or if it uses a sub-component that contains that part. + +```elixir +program = + Program.new() + |> Program.add_relation("direct_component", [:atom, :atom]) + |> Program.add_relation("contains_part", [:atom, :atom]) + # Base case: The assembly directly includes the part + |> Program.add_rule( + {"contains_part", [:Assembly, :Part]}, + [{:positive, {"direct_component", [:Assembly, :Part]}}] + ) + # Recursive case: The assembly includes a component that includes the part + |> Program.add_rule( + {"contains_part", [:Assembly, :SubPart]}, + [ + {:positive, {"contains_part", [:Assembly, :Component]}}, + {:positive, {"direct_component", [:Component, :SubPart]}} + ] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "contains_part" => %{arity: 2, types: [:atom, :atom]}, + "direct_component" => %{arity: 2, types: [:atom, :atom]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "contains_part", terms: [var: "Assembly", var: "SubPart"]}, + body: [ + positive: %ExDatalog.Atom{ + relation: "contains_part", + terms: [var: "Assembly", var: "Component"] + }, + positive: %ExDatalog.Atom{ + relation: "direct_component", + terms: [var: "Component", var: "SubPart"] + } + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "contains_part", terms: [var: "Assembly", var: "Part"]}, + body: [ + positive: %ExDatalog.Atom{ + relation: "direct_component", + terms: [var: "Assembly", var: "Part"] + } + ], + constraints: [] + } + ] +} +``` + +#### The Knowledge + +Let's model the classic manufacturing example of a bicycle. A hierarchical manufacturing tree where a finished bicycle requires top-level assemblies (like wheel_assembly), which are themselves composed of raw materials (like spokes and rim). + +```elixir +{:ok, knowledge} = + program + |> Program.add_fact("direct_component", [:bicycle, :frame_assembly]) + |> Program.add_fact("direct_component", [:bicycle, :wheel_assembly]) + |> Program.add_fact("direct_component", [:frame_assembly, :frame]) + |> Program.add_fact("direct_component", [:frame_assembly, :fork]) + |> Program.add_fact("direct_component", [:frame_assembly, :handlebars]) + |> Program.add_fact("direct_component", [:wheel_assembly, :tire]) + |> Program.add_fact("direct_component", [:wheel_assembly, :rim]) + |> Program.add_fact("direct_component", [:wheel_assembly, :spokes]) + |> ExDatalog.materialize() +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "contains_part" => MapSet.new([ + bicycle: :fork, + bicycle: :frame, + bicycle: :frame_assembly, + bicycle: :handlebars, + bicycle: :rim, + bicycle: :spokes, + bicycle: :tire, + bicycle: :wheel_assembly, + frame_assembly: :fork, + frame_assembly: :frame, + frame_assembly: :handlebars, + wheel_assembly: :rim, + wheel_assembly: :spokes, + wheel_assembly: :tire + ]), + "direct_component" => MapSet.new([ + bicycle: :frame_assembly, + bicycle: :wheel_assembly, + frame_assembly: :fork, + frame_assembly: :frame, + frame_assembly: :handlebars, + wheel_assembly: :rim, + wheel_assembly: :spokes, + wheel_assembly: :tire + ]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 98, + iterations: 2, + relation_sizes: %{"contains_part" => 14, "direct_component" => 8} + }, + provenance: nil + }} +``` + +**Full BOM Expansion (What is this made of?)**: +Extract the complete, flattened list of every single component and sub-assembly required to build a bicycle. + +```elixir +Knowledge.match(knowledge, "contains_part", [:bicycle, :_]) +``` + + + +``` +MapSet.new([ + bicycle: :fork, + bicycle: :frame, + bicycle: :frame_assembly, + bicycle: :handlebars, + bicycle: :rim, + bicycle: :spokes, + bicycle: :tire, + bicycle: :wheel_assembly +]) +``` + +**Where-Used Analysis (Where does this go?):** Find every assembly or finished product that relies on a specific raw material (e.g., if you are running low on spokes). + +```elixir +Knowledge.match(knowledge, "contains_part", [:_, :spokes]) +``` + + + +``` +MapSet.new([bicycle: :spokes, wheel_assembly: :spokes]) +``` + +**Direct Sub-components Only**: +Look at the immediate recipe for a specific sub-assembly without fully expanding the rest of the tree. + +```elixir +Knowledge.match(knowledge, "direct_component", [:frame_assembly, :_]) +``` + + + +``` +MapSet.new([frame_assembly: :fork, frame_assembly: :frame, frame_assembly: :handlebars]) +``` + +--- + +## 3. Social Network "Friend Recommendations" + +**Use Case:** Finding "Friends of Friends" to suggest as connections, but filtering out people who are already directly friends using ex_datalog's stratified negation. + +```elixir +program = + Program.new() + |> Program.add_relation("friend", [:atom, :atom]) + |> Program.add_relation("recommendation", [:atom, :atom]) + |> Program.add_rule( + {"recommendation", [:A, :C]}, + [ + {:positive, {"friend", [:A, :B]}}, + {:positive, {"friend", [:B, :C]}}, + {:negative, {"friend", [:A, :C]}} + ], + # Prevent recommending someone to themselves + [{:neq, :A, :C}] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "friend" => %{arity: 2, types: [:atom, :atom]}, + "recommendation" => %{arity: 2, types: [:atom, :atom]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "recommendation", terms: [var: "A", var: "C"]}, + body: [ + positive: %ExDatalog.Atom{relation: "friend", terms: [var: "A", var: "B"]}, + positive: %ExDatalog.Atom{relation: "friend", terms: [var: "B", var: "C"]}, + negative: %ExDatalog.Atom{relation: "friend", terms: [var: "A", var: "C"]} + ], + constraints: [ + %ExDatalog.Constraint{op: :neq, left: {:var, "A"}, right: {:var, "C"}, result: nil} + ] + } + ] +} +``` + +A small social graph of four users where relationships naturally overlap. It specifically includes a pre-existing friendship between Alice and Dave to test how negation successfully filters out redundant recommendations. + + + +#### The Knowledge + +```elixir +{:ok, knowledge} = + program + |> Program.add_fact("friend", [:alice, :bob]) + |> Program.add_fact("friend", [:bob, :charlie]) + |> Program.add_fact("friend", [:bob, :dave]) + # Alice is already friends with Dave, so he shouldn't be recommended + |> Program.add_fact("friend", [:alice, :dave]) + |> ExDatalog.materialize() + + +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "friend" => MapSet.new([alice: :bob, alice: :dave, bob: :charlie, bob: :dave]), + "recommendation" => MapSet.new([alice: :charlie]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 3108, + iterations: 1, + relation_sizes: %{"friend" => 4, "recommendation" => 1} + }, + provenance: nil + }} +``` + +Generate a recommendation feed for a user: + +```elixir +# Get all "friends of friends" to show to :alice +Knowledge.match(knowledge, "recommendation", [:alice, :_]) +``` + + + +``` +MapSet.new([alice: :charlie]) +``` + +Find out who is being recommended a specific person: + +```elixir +# Find all users who are seeing :dave in their suggested friends +Knowledge.match(knowledge, "recommendation", [:_, :dave]) +``` + + + +``` +MapSet.new([]) +``` + +Verify existing friendship status: + +```elixir +# Check if a direct connection already exists +Knowledge.match(knowledge, "friend", [:alice, :bob]) +``` + + + +``` +MapSet.new([alice: :bob]) +``` + +--- + +## 4. Code Taint Tracking / Static Security Analysis + +**Use Case:** Tracing the flow of untrusted data (taint) from user input sources down to sensitive execution sinks (like SQL queries or shell executions) to find vulnerabilities. + +```elixir +program = + Program.new() + |> Program.add_relation("source", [:atom]) + |> Program.add_relation("sink", [:atom]) + |> Program.add_relation("data_flow", [:atom, :atom]) + |> Program.add_relation("tainted", [:atom]) + |> Program.add_relation("vulnerability", [:atom]) + |> Program.add_relation("safe_flow", [:atom, :atom]) + |> Program.add_rule( + {"tainted", [:Node]}, + [{:positive, {"source", [:Node]}}] + ) + |> Program.add_rule( + {"tainted", [:To]}, + [ + {:positive, {"tainted", [:From]}}, + {:positive, {"data_flow", [:From, :To]}} + ] + ) + |> Program.add_rule( + {"vulnerability", [:Sink]}, + [ + {:positive, {"sink", [:Sink]}}, + {:positive, {"tainted", [:Sink]}} + ] + ) + |> Program.add_rule( + {"safe_flow", [:From, :To]}, + [ + # Look at every data flow we know about + {:positive, {"data_flow", [:From, :To]}}, + # Ensure the starting point is NOT infected + {:negative, {"tainted", [:From]}} + ] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "data_flow" => %{arity: 2, types: [:atom, :atom]}, + "safe_flow" => %{arity: 2, types: [:atom, :atom]}, + "sink" => %{arity: 1, types: [:atom]}, + "source" => %{arity: 1, types: [:atom]}, + "tainted" => %{arity: 1, types: [:atom]}, + "vulnerability" => %{arity: 1, types: [:atom]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "safe_flow", terms: [var: "From", var: "To"]}, + body: [ + positive: %ExDatalog.Atom{relation: "data_flow", terms: [var: "From", var: "To"]}, + negative: %ExDatalog.Atom{relation: "tainted", terms: [var: "From"]} + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "vulnerability", terms: [var: "Sink"]}, + body: [ + positive: %ExDatalog.Atom{relation: "sink", terms: [var: "Sink"]}, + positive: %ExDatalog.Atom{relation: "tainted", terms: [var: "Sink"]} + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "tainted", terms: [var: "To"]}, + body: [ + positive: %ExDatalog.Atom{relation: "tainted", terms: [var: "From"]}, + positive: %ExDatalog.Atom{relation: "data_flow", terms: [var: "From", var: "To"]} + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "tainted", terms: [var: "Node"]}, + body: [positive: %ExDatalog.Atom{relation: "source", terms: [var: "Node"]}], + constraints: [] + } + ] +} +``` + +Datalog rules are basically "if-then" logical statements, but written backwards: `Result :- Conditions`. + +In this specific example, we are doing Code Taint Tracking. In cybersecurity, "taint" means untrusted data (like a user typing into a web form). The goal of this program is to see if untrusted data ever touches a sensitive part of our application (like executing a SQL database query). + +Here is the translation of the Elixir `Rule.new` blocks into plain English logic. + +The Setup: The Facts (Relations) +Before the rules run, the engine knows about three basic facts (which we fed it in the test data): + +* `source(X)`: Node `X` is a place where untrusted data enters (e.g., user input). + +* `sink(X)`: Node `X` is a dangerous place to execute untrusted data (e.g., a database). + +* `data_flow(A, B)`: Data moves directly from node `A` to node `B`. + +#### Rule 1: The Base Case (Where does taint start?) + + + +```elixir + {"tainted", [:Node]}, + [{:positive, {"source", [:Node]}}] +``` + +Plain English: If a `Node` is a `source`, then that `Node` is `tainted`. + +How it works: This is the starting point. It looks at all our facts and says, "Ah, 'user_input' is a source. Therefore, I will now label 'user_input' as tainted." + +### Rule 2: The Recursive Case (How does taint spread?) + + + +```elixir + {"tainted", [:To]}, + [ + {:positive, {"tainted", [:From]}}, + {:positive, {"data_flow", [:From, :To]}} + ] +``` + +Plain English: If a From node is already tainted, AND data flows from From to To, then the To node also becomes tainted. + +How it works: This is the engine's superpower. Because it's a recursive rule, the engine will run it over and over until it stops finding new things. + +1. It knows `user_input` is tainted (from Rule 1). +2. It sees data flows from `user_input` to `parser`. So, `parser` becomes tainted. +3. It runs again! Now `parser` is tainted, and it sees data flows from `parser` to `sql_builder`. So, `sql_builder` becomes tainted. +4. It keeps spreading like an infection down the pipeline. + +#### Rule 3: The Vulnerability Check (Did something bad happen?) + + + +```elixir +{"vulnerability", [:Sink]}, +[ + {:positive, {"sink", [:Sink]}}, + {:positive, {"tainted", [:Sink]}} +] +``` + +Plain English: If a node is a `sink` (a sensitive execution point), AND that exact same node has been marked as `tainted`, then we have a `vulnerability`. + +How it works: After the "infection" finishes spreading in Rule 2, this rule checks the damage. It asks: "Did the taint ever reach our sensitive SQL query node?" If yes, it flags it as a vulnerability so the security team can fix it. + +#### Rule 4: Which data flows are safe? + + + +```elixir + {"safe_flow", [:From, :To]}, + [ + # Look at every data flow we know about + {:positive, {"data_flow", [:From, :To]}}, + # Ensure the starting point is NOT infected + {:negative, {"tainted", [:From]}} + ] +``` + +Plain English: We look at all known `data_flow` paths, and filter out any where the starting point has been marked as `tainted`. + +How it works: To do this, we have to follow Datalog's golden rule for negation: You cannot query for something that doesn't exist; you always have to start with a positive fact and filter it. + +Since we didn't define a master `node` list in the original program, the only place the engine knows `:safe_config` and `:logger` exist is inside the `data_flow` relation. + +So, instead of asking "Which nodes are safe?", we ask, "Which data flows are safe?" + + + + + +```mermaid +flowchart LR + %% Nodes and Data Flow + subgraph Tainted_Pipeline [Code Execution Pipeline] + direction LR + UI(["user_input (Source)"]) -->|data_flow| P["parser"] + P -->|data_flow| SB["sql_builder"] + SB -->|data_flow| SQ[["sql_query (Sink)"]] + end + + subgraph Safe_Pipeline [Safe Pipeline] + direction LR + SC(["safe_config (Not a Source)"]) -->|data_flow| L[["logger (Sink)"]] + end + + %% Visual Styling representing the Datalog output + %% Yellow represents nodes infected by the "tainted" rule + style UI fill:#ffeb3b,stroke:#f57f17,stroke-width:2px,color:#000 + style P fill:#ffeb3b,stroke:#f57f17,stroke-width:2px,color:#000 + style SB fill:#ffeb3b,stroke:#f57f17,stroke-width:2px,color:#000 + + %% Red represents the "vulnerability" rule firing + style SQ fill:#f44336,stroke:#b71c1c,stroke-width:4px,color:#fff + + %% Grey represents safe nodes untouched by taint + style SC fill:#f5f5f5,stroke:#9e9e9e,color:#000 + style L fill:#f5f5f5,stroke:#9e9e9e,color:#000 +``` + + + +This simulates an execution pipeline where untrusted user input flows through intermediate parser and builder functions before hitting a SQL execution sink. It models a classic SQL injection vulnerability pathway. + + + +#### The Knowledge + +```elixir +{:ok, knowledge} = + program + |> Program.add_fact("source", [:user_input]) + |> Program.add_fact("sink", [:sql_query]) + |> Program.add_fact("data_flow", [:user_input, :parser]) + |> Program.add_fact("data_flow", [:parser, :sql_builder]) + |> Program.add_fact("data_flow", [:sql_builder, :sql_query]) + |> Program.add_fact("data_flow", [:safe_config, :logger]) + |> ExDatalog.materialize() +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "data_flow" => MapSet.new([ + parser: :sql_builder, + safe_config: :logger, + sql_builder: :sql_query, + user_input: :parser + ]), + "safe_flow" => MapSet.new([safe_config: :logger]), + "sink" => MapSet.new([{:sql_query}]), + "source" => MapSet.new([{:user_input}]), + "tainted" => MapSet.new([{:parser}, {:sql_builder}, {:sql_query}, {:user_input}]), + "vulnerability" => MapSet.new([{:sql_query}]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 129, + iterations: 6, + relation_sizes: %{ + "data_flow" => 4, + "safe_flow" => 1, + "sink" => 1, + "source" => 1, + "tainted" => 4, + "vulnerability" => 1 + } + }, + provenance: nil + }} +``` + +List all vulnerable execution sinks: + +```elixir +# Get every sink that currently has untrusted data flowing into it +# Flagged because tainted data flows from user_input -> parser -> sql_builder -> sql_query +Knowledge.match(knowledge, "vulnerability", [:_]) +``` + + + +``` +MapSet.new([{:sql_query}]) +``` + +Check if a specific sensitive function is tainted: + +```elixir +# See whose's tainted +Knowledge.match(knowledge, "tainted", [:_]) +``` + + + +``` +MapSet.new([{:parser}, {:sql_builder}, {:sql_query}, {:user_input}]) +``` + +Trace the data flow into a specific node: + +```elixir +# Find upstream nodes passing data into the logger +Knowledge.match(knowledge, "data_flow", [:_, :system_logger]) +``` + + + +``` +MapSet.new([]) +``` + +Which data flows are safe? + +```elixir +Knowledge.match(knowledge, "safe_flow", [:_, :_]) +``` + + + +``` +MapSet.new([safe_config: :logger]) +``` + +It successfully looks at the 4 total data_flow connections, sees that user_input, parser, and sql_builder are in the tainted bucket, and filters them out—leaving only the untainted pipeline! + + + +--- + +## 5. Circular Fraud Ring Detection + +**Use Case:** Financial institutions need to detect cyclic money laundering loops, where funds move through a chain of accounts and eventually return to the originator. + +```elixir +program = + Program.new() + |> Program.add_relation("transfer", [:atom, :atom]) + |> Program.add_relation("money_path", [:atom, :atom]) + |> Program.add_relation("fraud_ring", [:atom]) + |> Program.add_rule( + {"money_path", [:A, :B]}, + [{:positive, {"transfer", [:A, :B]}}] + ) + |> Program.add_rule( + {"money_path", [:A, :C]}, + [ + {:positive, {"money_path", [:A, :B]}}, + {:positive, {"transfer", [:B, :C]}} + ] + ) + # If a path leads back to the sender, it's a cyclic loop + |> Program.add_rule( + {"fraud_ring", [:Account]}, + [{:positive, {"money_path", [:Account, :Account]}}] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "fraud_ring" => %{arity: 1, types: [:atom]}, + "money_path" => %{arity: 2, types: [:atom, :atom]}, + "transfer" => %{arity: 2, types: [:atom, :atom]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "fraud_ring", terms: [var: "Account"]}, + body: [ + positive: %ExDatalog.Atom{relation: "money_path", terms: [var: "Account", var: "Account"]} + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "money_path", terms: [var: "A", var: "C"]}, + body: [ + positive: %ExDatalog.Atom{relation: "money_path", terms: [var: "A", var: "B"]}, + positive: %ExDatalog.Atom{relation: "transfer", terms: [var: "B", var: "C"]} + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "money_path", terms: [var: "A", var: "B"]}, + body: [positive: %ExDatalog.Atom{relation: "transfer", terms: [var: "A", var: "B"]}], + constraints: [] + } + ] +} +``` + +A sequence of financial transfers that forms a closed, cyclic loop between three accounts (`A`, `B`, and `C`), alongside one isolated, innocent transfer. It provides the exact structure needed to trigger and isolate a money laundering ring. + + + +#### The Knowledge + +```elixir +{:ok, knowledge} = + program + |> Program.add_fact("transfer", [:acct_A, :acct_B]) + |> Program.add_fact("transfer", [:acct_A, :acct_C]) + |> Program.add_fact("transfer", [:acct_B, :acct_D]) + |> Program.add_fact("transfer", [:acct_C, :acct_D]) + |> Program.add_fact("transfer", [:acct_D, :acct_A]) # The loop closes here + |> Program.add_fact("transfer", [:acct_X, :acct_Y]) # Innocent transfer + |> ExDatalog.materialize() + + +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "fraud_ring" => MapSet.new([{:acct_A}, {:acct_B}, {:acct_C}, {:acct_D}]), + "money_path" => MapSet.new([ + acct_A: :acct_A, + acct_A: :acct_B, + acct_A: :acct_C, + acct_A: :acct_D, + acct_B: :acct_A, + acct_B: :acct_B, + acct_B: :acct_C, + acct_B: :acct_D, + acct_C: :acct_A, + acct_C: :acct_B, + acct_C: :acct_C, + acct_C: :acct_D, + acct_D: :acct_A, + acct_D: :acct_B, + acct_D: :acct_C, + acct_D: :acct_D, + acct_X: :acct_Y + ]), + "transfer" => MapSet.new([ + acct_A: :acct_B, + acct_A: :acct_C, + acct_B: :acct_D, + acct_C: :acct_D, + acct_D: :acct_A, + acct_X: :acct_Y + ]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 281, + iterations: 4, + relation_sizes: %{"fraud_ring" => 4, "money_path" => 17, "transfer" => 6} + }, + provenance: nil + }} +``` + +Identify all accounts flagged in a laundering loop: + +```elixir +# Extract the list of cyclic fraudulent accounts +Knowledge.match(knowledge, "fraud_ring", [:_]) +``` + + + +``` +MapSet.new([{:acct_A}, {:acct_B}, {:acct_C}, {:acct_D}]) +``` + +Trace all downstream recipients from a suspicious account: + +```elixir +# Find everywhere money went after hitting the suspect account +Knowledge.match(knowledge, "money_path", [:acct_A, :_]) +``` + + + +``` +MapSet.new([acct_A: :acct_A, acct_A: :acct_B, acct_A: :acct_C, acct_A: :acct_D]) +``` + +Why this result is awesome: +This perfectly illustrates the recursive power of Datalog: + +`acct_A: :acct_B`: The base case. Money moved directly from `A` to `B`. + +`acct_A: :acct_D`: The recursive case. The engine saw `A` went to `B`, and `A` went to `C`, then from `B` and `C` to `D` so it deduced `A` ultimately funded `D`. + +`acct_A: :acct_A`: The cyclic loop. The engine saw D sent the money back to A, completing the circle. This exact tuple is what triggers the **fraud_ring rule!** + + + +Find all accounts that funded a known mule: + +```elixir +# Look one step backwards to see who directly sent money to the mule +Knowledge.match(knowledge, "transfer", [:_, :acct_D]) +``` + + + +``` +MapSet.new([acct_B: :acct_D, acct_C: :acct_D]) +``` + +--- + +## 6. Network Routing & Outage Evasion + +**Use Case:** Dynamically determining if a server is reachable in a microservices mesh or physical network topology by plotting paths that entirely bypass offline nodes. + +```elixir +program = + Program.new() + |> Program.add_relation("link", [:atom, :atom]) + |> Program.add_relation("offline", [:atom]) + |> Program.add_relation("reachable", [:atom, :atom]) + |> Program.add_rule( + {"reachable", [:A, :B]}, + [ + {:positive, {"link", [:A, :B]}}, + {:negative, {"offline", [:B]}} + ] + ) + |> Program.add_rule( + {"reachable", [:A, :C]}, + [ + {:positive, {"reachable", [:A, :B]}}, + {:positive, {"link", [:B, :C]}}, + {:negative, {"offline", [:C]}} + ] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "link" => %{arity: 2, types: [:atom, :atom]}, + "offline" => %{arity: 1, types: [:atom]}, + "reachable" => %{arity: 2, types: [:atom, :atom]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "reachable", terms: [var: "A", var: "C"]}, + body: [ + positive: %ExDatalog.Atom{relation: "reachable", terms: [var: "A", var: "B"]}, + positive: %ExDatalog.Atom{relation: "link", terms: [var: "B", var: "C"]}, + negative: %ExDatalog.Atom{relation: "offline", terms: [var: "C"]} + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "reachable", terms: [var: "A", var: "B"]}, + body: [ + positive: %ExDatalog.Atom{relation: "link", terms: [var: "A", var: "B"]}, + negative: %ExDatalog.Atom{relation: "offline", terms: [var: "B"]} + ], + constraints: [] + } + ] +} +``` + +Let's build a network where a client talks to two load balancers (proxies), which both point to the same app server, which connects to a database (The Diamond Mesh). + + + +#### The Knowledge + +```elixir +{:ok, knowledge} = + program + |> Program.add_fact("link", [:client, :proxy_1]) + |> Program.add_fact("link", [:client, :proxy_2]) + |> Program.add_fact("link", [:proxy_1, :app_server]) + |> Program.add_fact("link", [:proxy_2, :app_server]) + |> Program.add_fact("link", [:app_server, :db]) + # Outage: The primary load balancer crashes + |> Program.add_fact("offline", [:proxy_1]) + |> ExDatalog.materialize() +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "link" => MapSet.new([ + app_server: :db, + client: :proxy_1, + client: :proxy_2, + proxy_1: :app_server, + proxy_2: :app_server + ]), + "offline" => MapSet.new([{:proxy_1}]), + "reachable" => MapSet.new([ + app_server: :db, + client: :app_server, + client: :db, + client: :proxy_2, + proxy_1: :app_server, + proxy_1: :db, + proxy_2: :app_server, + proxy_2: :db + ]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 234, + iterations: 3, + relation_sizes: %{"link" => 5, "offline" => 1, "reachable" => 8} + }, + provenance: nil + }} +``` + +If `:proxy_1` goes down, the path `client -> proxy_1 -> app_server` is dead. But because Datalog explores all possible logical branches, it will automatically find `client -> proxy_2 -> app_server` and use it to maintain the connection all the way to the `:db`. + + + +The "Evasion" Check: Can the client still reach the end of the line? + +```elixir +Knowledge.match(knowledge, "reachable", [:client, :_]) +``` + + + +``` +MapSet.new([client: :app_server, client: :db, client: :proxy_2]) +``` + +*(Notice that `:proxy_1` is completely missing from the client's reachable list, yet `:app_server` and `:db` are still successfully resolved!)* + + + +Dead Node Isolation - What can the dead node reach? +Because the node is offline, nothing should route through it, but what if we ask what it can reach? + +```elixir +Knowledge.match(knowledge, "reachable", [:proxy_1, :_]) +``` + + + +``` +MapSet.new([proxy_1: :app_server, proxy_1: :db]) +``` + +*(Wait, why does this happen? Because the rule only checks if the **destination** is offline! Since `app_server` is online, `proxy_1` can technically reach it. It's just that nobody can reach `proxy_1`.)* + + + +Verifying the broken link - Let's confirm the client was cut off from the dead proxy. + +```elixir +Knowledge.match(knowledge, "reachable", [:client, :proxy_1]) +``` + + + +``` +MapSet.new([]) +``` + +This dataset actually proves the Datalog engine is mapping an evasive path through a complex mesh! + + + +--- + +## 7. Financial Balance Calculation (Arithmetic Constraints) + +**Use Case:** Aggregating data, like calculating total purchasing power by combining a checking account balance with an approved credit limit. + +```elixir +program = + Program.new() + |> Program.add_relation("account_balance", [:atom, :integer]) + |> Program.add_relation("credit_limit", [:atom, :integer]) + |> Program.add_relation("cart_total", [:atom, :integer]) + |> Program.add_relation("purchasing_power", [:atom, :integer]) + |> Program.add_relation("checkout_approved", [:atom]) + |> Program.add_relation("low_cash_alert", [:atom]) + + # 1. Base Math: Purchasing Power = Balance + Credit + |> Program.add_rule( + {"purchasing_power", [:AccountID, :Total]}, + [ + {:positive, {"account_balance", [:AccountID, :Balance]}}, + {:positive, {"credit_limit", [:AccountID, :Limit]}} + ], + [{:add, :Balance, :Limit, :Total}] + ) + + # 2. Logic: Approve if Purchasing Power >= Cart Total + |> Program.add_rule( + {"checkout_approved", [:AccountID]}, + [ + {:positive, {"purchasing_power", [:AccountID, :PP]}}, + {:positive, {"cart_total", [:AccountID, :Cart]}} + ], + [{:gte, :PP, :Cart}] + ) + + # 3. Logic: Alert if actual cash is under $500 (ignoring credit) + |> Program.add_rule( + {"low_cash_alert", [:AccountID]}, + [{:positive, {"account_balance", [:AccountID, :Balance]}}], + [{:lt, :Balance, 500}] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "account_balance" => %{arity: 2, types: [:atom, :integer]}, + "cart_total" => %{arity: 2, types: [:atom, :integer]}, + "checkout_approved" => %{arity: 1, types: [:atom]}, + "credit_limit" => %{arity: 2, types: [:atom, :integer]}, + "low_cash_alert" => %{arity: 1, types: [:atom]}, + "purchasing_power" => %{arity: 2, types: [:atom, :integer]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "low_cash_alert", terms: [var: "AccountID"]}, + body: [ + positive: %ExDatalog.Atom{ + relation: "account_balance", + terms: [var: "AccountID", var: "Balance"] + } + ], + constraints: [ + %ExDatalog.Constraint{op: :lt, left: {:var, "Balance"}, right: {:const, 500}, result: nil} + ] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "checkout_approved", terms: [var: "AccountID"]}, + body: [ + positive: %ExDatalog.Atom{ + relation: "purchasing_power", + terms: [var: "AccountID", var: "PP"] + }, + positive: %ExDatalog.Atom{relation: "cart_total", terms: [var: "AccountID", var: "Cart"]} + ], + constraints: [ + %ExDatalog.Constraint{op: :gte, left: {:var, "PP"}, right: {:var, "Cart"}, result: nil} + ] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "purchasing_power", terms: [var: "AccountID", var: "Total"]}, + body: [ + positive: %ExDatalog.Atom{ + relation: "account_balance", + terms: [var: "AccountID", var: "Balance"] + }, + positive: %ExDatalog.Atom{relation: "credit_limit", terms: [var: "AccountID", var: "Limit"]} + ], + constraints: [%ExDatalog.Constraint{op: :add, left: {:var, ...}, ...}] + } + ] +} +``` + +### The Knowledge + +Let's feed it three distinct customer scenarios: + +```elixir +{:ok, knowledge} = + program +# Account 123: High roller, but trying to buy a $10k watch (Fails) + |> Program.add_fact("account_balance", [:acct_123, 1500]) + |> Program.add_fact("credit_limit", [:acct_123, 5000]) + |> Program.add_fact("cart_total", [:acct_123, 10000]) + + # Account 456: Broke in cash, but buying groceries on credit (Passes, but gets alert) + |> Program.add_fact("account_balance", [:acct_456, 200]) + |> Program.add_fact("credit_limit", [:acct_456, 1000]) + |> Program.add_fact("cart_total", [:acct_456, 800]) + + # Account 789: Debit only, buying a laptop (Passes, no alert) + |> Program.add_fact("account_balance", [:acct_789, 4000]) + |> Program.add_fact("credit_limit", [:acct_789, 0]) + |> Program.add_fact("cart_total", [:acct_789, 2000]) + |> ExDatalog.materialize() +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "account_balance" => MapSet.new([acct_123: 1500, acct_456: 200, acct_789: 4000]), + "cart_total" => MapSet.new([acct_123: 10000, acct_456: 800, acct_789: 2000]), + "checkout_approved" => MapSet.new([{:acct_456}, {:acct_789}]), + "credit_limit" => MapSet.new([acct_123: 5000, acct_456: 1000, acct_789: 0]), + "low_cash_alert" => MapSet.new([{:acct_456}]), + "purchasing_power" => MapSet.new([acct_123: 6500, acct_456: 1200, acct_789: 4000]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 4008, + iterations: 2, + relation_sizes: %{ + "account_balance" => 3, + "cart_total" => 3, + "checkout_approved" => 2, + "credit_limit" => 3, + "low_cash_alert" => 1, + "purchasing_power" => 3 + } + }, + provenance: nil + }} +``` + +Look up the total purchasing power for a specific user: + +```elixir +# Calculate checkout eligibility for account 123 +Knowledge.match(knowledge, "purchasing_power", [:acct_123, :_]) +``` + + + +``` +MapSet.new([acct_123: 6500]) +``` + +Find accounts hitting a specific threshold: + +```elixir +# Find accounts with exactly zero purchasing power +Knowledge.match(knowledge, "purchasing_power", [:_, 0]) +``` + + + +``` +MapSet.new([]) +``` + +Extract base credit limits for auditing: + +```elixir +# Retrieve the raw credit limits separate from cash balance +Knowledge.match(knowledge, "credit_limit", [:acct_123, :_]) +``` + + + +``` +MapSet.new([acct_123: 5000]) +``` + +Transaction Authorization: Identify all users whose shopping carts can be successfully routed to the payment processor. + +```elixir +Knowledge.match(knowledge, "checkout_approved", [:_]) +``` + + + +``` +MapSet.new([{:acct_456}, {:acct_789}]) +``` + +Overdraft / Low Funds Warning: Fetch the list of accounts that need an automated "Your cash balance is low" warning email. + +```elixir +Knowledge.match(knowledge, "low_cash_alert", [:_]) +``` + + + +``` +MapSet.new([{:acct_456}]) +``` + +*(Even though 456's checkout was approved via credit, their actual cash is under the $500 threshold).* + + + +Specific Account Health Check: Look up the exact computed purchasing power for a user who just called into customer support wondering why their card declined. + +```elixir +Knowledge.match(knowledge, "purchasing_power", [:acct_123, :_]) +``` + + + +``` +MapSet.new([acct_123: 6500]) +``` + +--- + +## 8. Package Manager Dependency Resolution + +**Use Case:** Computing all transitive dependencies required to install a library (exactly how Mix or Hex work under the hood). + +```elixir +program = + Program.new() + |> Program.add_relation("depends_on", [:atom, :atom]) + |> Program.add_relation("requires_package", [:atom, :atom]) + |> Program.add_rule( + {"requires_package", [:Pkg, :Dep]}, + [{:positive, {"depends_on", [:Pkg, :Dep]}}] + ) + |> Program.add_rule( + {"requires_package", [:Pkg, :TransDep]}, + [ + {:positive, {"depends_on", [:Pkg, :Dep]}}, + {:positive, {"requires_package", [:Dep, :TransDep]}} + ] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "depends_on" => %{arity: 2, types: [:atom, :atom]}, + "requires_package" => %{arity: 2, types: [:atom, :atom]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "requires_package", terms: [var: "Pkg", var: "TransDep"]}, + body: [ + positive: %ExDatalog.Atom{relation: "depends_on", terms: [var: "Pkg", var: "Dep"]}, + positive: %ExDatalog.Atom{ + relation: "requires_package", + terms: [var: "Dep", var: "TransDep"] + } + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "requires_package", terms: [var: "Pkg", var: "Dep"]}, + body: [positive: %ExDatalog.Atom{relation: "depends_on", terms: [var: "Pkg", var: "Dep"]}], + constraints: [] + } + ] +} +``` + +#### The Knowledge + +A linear software dependency tree where a high-level framework (`phoenix`) relies on a server (`plug`), which relies on a parser (`mime`). It mimics the exact resolution graph a package manager uses to install transitive requirements. + +```elixir +{:ok, knowledge} = + program + |> Program.add_fact("depends_on", [:phoenix, :plug]) + |> Program.add_fact("depends_on", [:plug, :mime]) + |> Program.add_fact("depends_on", [:ecto, :postgrex]) + |> ExDatalog.materialize() +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "depends_on" => MapSet.new([ecto: :postgrex, phoenix: :plug, plug: :mime]), + "requires_package" => MapSet.new([ecto: :postgrex, phoenix: :mime, phoenix: :plug, plug: :mime]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 61, + iterations: 2, + relation_sizes: %{"depends_on" => 3, "requires_package" => 4} + }, + provenance: nil + }} +``` + +Resolve the full installation manifest for a package: + +```elixir +# Get all direct and transitive dependencies needed to install :phoenix +Knowledge.match(knowledge, "requires_package", [:phoenix, :_]) +``` + + + +``` +MapSet.new([phoenix: :mime, phoenix: :plug]) +``` + +Perform a reverse dependency lookup (Who needs this?): + +```elixir +# Find every package in the ecosystem that relies on :telemetry +Knowledge.match(knowledge, "requires_package", [:_, :mime]) +``` + + + +``` +MapSet.new([phoenix: :mime, plug: :mime]) +``` + +Check direct dependency status: + +```elixir +# Check if :ecto lists :postgrex as a direct, top-level requirement +Knowledge.match(knowledge, "depends_on", [:ecto, :postgrex]) +``` + + + +``` +MapSet.new([ecto: :postgrex]) +``` + +--- + +## 9. Knowledge Graph Property Inheritance (Ontologies) + +**Use Case:** Modern AI systems, healthcare databases (like SNOMED CT), and semantic web applications use ontologies to model the world. If you know that a _"Golden Retriever is a Dog,"_ and a _"Dog is a Mammal,"_ Datalog can automatically infer that the Golden Retriever inherits all the biological traits of a Mammal without you having to explicitly store that data. + + + +This program defines a structural class hierarchy (`is_a`) and base properties (`has_property`), then uses recursion to allow sub-classes to inherit everything from their ancestors. + + + +#### The Program + +```elixir +program = + Program.new() + |> Program.add_relation("is_a", [:atom, :atom]) + |> Program.add_relation("has_property", [:atom, :atom]) + |> Program.add_relation("inherits_property", [:atom, :atom]) + + # Base case: An entity has a property if assigned directly + |> Program.add_rule( + {"inherits_property", [:Entity, :Prop]}, + [{:positive, {"has_property", [:Entity, :Prop]}}] + ) + + # Recursive case: An entity inherits properties from its super-class + |> Program.add_rule( + {"inherits_property", [:Entity, :Prop]}, + [ + {:positive, {"is_a", [:Entity, :SuperClass]}}, + {:positive, {"inherits_property", [:SuperClass, :Prop]}} + ] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "has_property" => %{arity: 2, types: [:atom, :atom]}, + "inherits_property" => %{arity: 2, types: [:atom, :atom]}, + "is_a" => %{arity: 2, types: [:atom, :atom]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "inherits_property", terms: [var: "Entity", var: "Prop"]}, + body: [ + positive: %ExDatalog.Atom{relation: "is_a", terms: [var: "Entity", var: "SuperClass"]}, + positive: %ExDatalog.Atom{ + relation: "inherits_property", + terms: [var: "SuperClass", var: "Prop"] + } + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "inherits_property", terms: [var: "Entity", var: "Prop"]}, + body: [ + positive: %ExDatalog.Atom{relation: "has_property", terms: [var: "Entity", var: "Prop"]} + ], + constraints: [] + } + ] +} +``` + +#### The Knowledge + +We build a simple biological taxonomy. Notice we never explicitly say a Golden Retriever breathes or is warm-blooded. + +```elixir +{:ok, knowledge} = + program + # The Taxonomy Tree + |> Program.add_fact("is_a", [:golden_retriever, :dog]) + |> Program.add_fact("is_a", [:dog, :mammal]) + |> Program.add_fact("is_a", [:mammal, :animal]) + + # The Base Properties + |> Program.add_fact("has_property", [:animal, :breathes_air]) + |> Program.add_fact("has_property", [:mammal, :warm_blooded]) + |> Program.add_fact("has_property", [:dog, :barks]) + |> ExDatalog.materialize() + +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "has_property" => MapSet.new([animal: :breathes_air, dog: :barks, mammal: :warm_blooded]), + "inherits_property" => MapSet.new([ + animal: :breathes_air, + dog: :barks, + dog: :breathes_air, + dog: :warm_blooded, + golden_retriever: :barks, + golden_retriever: :breathes_air, + golden_retriever: :warm_blooded, + mammal: :breathes_air, + mammal: :warm_blooded + ]), + "is_a" => MapSet.new([dog: :mammal, golden_retriever: :dog, mammal: :animal]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 107, + iterations: 4, + relation_sizes: %{"has_property" => 3, "inherits_property" => 9, "is_a" => 3} + }, + provenance: nil + }} +``` + +Returns `{:error, "arity mismatch for relation \"edge\": expected 2 values, got 1"}`. + +--- + + + +Complete Trait Resolution (What is this thing?): Extract every known fact about a specific leaf node by traversing all the way up the knowledge tree. + +```elixir +Knowledge.match(knowledge, "inherits_property", [:golden_retriever, :_]) +``` + + + +``` +MapSet.new([ + golden_retriever: :barks, + golden_retriever: :breathes_air, + golden_retriever: :warm_blooded +]) +``` + +Reverse Property Lookup (Who shares this trait?): +Find every entity in the database that possesses a specific trait, whether directly or through inheritance. + +```elixir +Knowledge.match(knowledge, "inherits_property", [:_, :warm_blooded]) +``` + + + +``` +MapSet.new([dog: :warm_blooded, golden_retriever: :warm_blooded, mammal: :warm_blooded]) +``` + +Direct vs. Derived Distinction: Sometimes you need to know what was explicitly asserted versus what was inferred. You can always just query the base facts. + +```elixir +# Correlate logins with a specific incident time +Knowledge.match(knowledge, "has_property", [:dog, :_]) +``` + + + +``` +MapSet.new([dog: :barks]) +``` + +--- + +## 10. Corporate Hierarchy Analysis + +Use Case: An HR system needs to retrieve the complete "reporting tree" for a C-Level executive to send out a department-wide announcement. + +```elixir +program = + Program.new() + |> Program.add_relation("manages", [:atom, :atom]) + |> Program.add_relation("in_reporting_chain", [:atom, :atom]) + |> Program.add_rule( + {"in_reporting_chain", [:Manager, :Employee]}, + [{:positive, {"manages", [:Manager, :Employee]}}] + ) + |> Program.add_rule( + {"in_reporting_chain", [:Executive, :Employee]}, + [ + {:positive, {"manages", [:Executive, :MidLevelManager]}}, + {:positive, {"in_reporting_chain", [:MidLevelManager, :Employee]}} + ] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "in_reporting_chain" => %{arity: 2, types: [:atom, :atom]}, + "manages" => %{arity: 2, types: [:atom, :atom]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{ + relation: "in_reporting_chain", + terms: [var: "Executive", var: "Employee"] + }, + body: [ + positive: %ExDatalog.Atom{ + relation: "manages", + terms: [var: "Executive", var: "MidLevelManager"] + }, + positive: %ExDatalog.Atom{ + relation: "in_reporting_chain", + terms: [var: "MidLevelManager", var: "Employee"] + } + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{ + relation: "in_reporting_chain", + terms: [var: "Manager", var: "Employee"] + }, + body: [ + positive: %ExDatalog.Atom{relation: "manages", terms: [var: "Manager", var: "Employee"]} + ], + constraints: [] + } + ] +} +``` + +#### Adding The Data + +A direct management chain linking a C-level executive down through middle management to two individual contributors. It provides a clean, top-down organizational chart to test deep recursive reporting structures. + +```elixir +{:ok, knowledge} = + program + |> Program.add_fact("manages", [:ceo, :vp_eng]) + |> Program.add_fact("manages", [:vp_eng, :eng_manager]) + |> Program.add_fact("manages", [:eng_manager, :alice]) + |> Program.add_fact("manages", [:eng_manager, :bob]) + |> ExDatalog.materialize() + +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "in_reporting_chain" => MapSet.new([ + ceo: :alice, + ceo: :bob, + ceo: :eng_manager, + ceo: :vp_eng, + eng_manager: :alice, + eng_manager: :bob, + vp_eng: :alice, + vp_eng: :bob, + vp_eng: :eng_manager + ]), + "manages" => MapSet.new([ + ceo: :vp_eng, + eng_manager: :alice, + eng_manager: :bob, + vp_eng: :eng_manager + ]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 145, + iterations: 3, + relation_sizes: %{"in_reporting_chain" => 9, "manages" => 4} + }, + provenance: nil + }} +``` + +#### Find the complete management chain above an employee: + +```elixir +# Find all managers (direct and indirect) for a specific IC +Knowledge.match(knowledge, "in_reporting_chain", [:_, :bob]) +``` + + + +``` +MapSet.new([ceo: :bob, eng_manager: :bob, vp_eng: :bob]) +``` + +#### Check direct reports: + +```elixir +# Verify if Alice is Bob's direct supervisor +Knowledge.match(knowledge, "manages", [:eng_manager, :bob]) +``` + + + +``` +MapSet.new([eng_manager: :bob]) +``` + +#### Get the complete downstream org chart for an executive: + +```elixir +# Find everyone who ultimately reports up to the CEO +Knowledge.match(knowledge, "in_reporting_chain", [:ceo, :_]) +``` + + + +``` +MapSet.new([ceo: :alice, ceo: :bob, ceo: :eng_manager, ceo: :vp_eng]) +``` + +##### I can't see the hierarchy? + +This question highlights a fundamental concept about how logic engines work: **Datalog is not a tree-builder; it is an edge-finder.** + +Datalog will always return a flat, mathematical set of tuples (facts). It does not natively output nested JSON, structs, or tree hierarchies. Its job is to find all the valid connections (the "edges" of the graph) as fast as possible. + +Once Datalog has extracted those connections, it hands the baton back to your application language. If you want to see a nested hierarchy, you use a few lines of standard Elixir to fold that flat data into a tree. + +Here is exactly how you do that. + +**Step 1: Query the Direct Edges** + +Instead of querying the recursive `in_reporting_chain` rule (which gives us the flattened "everyone under the CEO" list), we query the base `manages` relation to get the direct parent-child edges. + +```elixir +# Grab the flat tuples from the Datalog result +edges = knowledge.relations["manages"] |> Enum.to_list() +``` + + + +``` +[ceo: :vp_eng, eng_manager: :alice, eng_manager: :bob, vp_eng: :eng_manager] +``` + +**Step 2: Fold the Edges into a Tree with Elixir** + +We can write a quick Elixir script that groups these managers and their direct reports, and then recursively builds a nested map. + +```elixir +# 1. Group the edges into an adjacency list (manager => list of direct reports) +org_chart = + Enum.reduce(edges, %{}, fn {manager, report}, acc -> + Map.update(acc, manager, [report], fn existing -> [report | existing] end) + end) +# Result: %{ceo: [:vp_eng], eng_manager: [:alice, :bob], vp_eng: [:eng_manager]} + +# 2. Define a simple recursive builder +defmodule TreeBuilder do + def build(node, org_chart) do + case Map.get(org_chart, node) do + # If they manage no one, they are a leaf node + nil -> node + # If they manage people, recursively build their subtree + reports -> %{node => Enum.map(reports, &build(&1, org_chart))} + end + end +end + +# 3. Build the tree starting from the top +hierarchy = TreeBuilder.build(:ceo, org_chart) +``` + + + +``` +%{ceo: [%{vp_eng: [%{eng_manager: [:bob, :alice]}]}]} +``` + +--- + +## 11. Multi-Stage Cost Assessment Engine + +SAP’s allocation engines are legendary for their complexity, and modeling them in Datalog perfectly demonstrates how separating **business rules** (percentages, routing) from **execution logic** (the math and traversal) makes systems incredibly flexible. + +Instead of writing a massive Elixir `Enum.reduce` that hardcodes "Facilities costs go to IT, and then IT costs go to Product," we can write a pure Datalog rules engine. + +Here is how we model a **Multi-Stage Cost Assessment Engine** (combining your Document Splitting and Allocation rules). + +### The SAP Cost Allocation Engine + +This program takes raw financial documents posted to high-level cost centers (like "Facilities") and splits them down into granular departments using assessment percentages. It even handles **Level 2 (Transitive) Allocations**, where a receiving department (like IT) automatically re-allocates the costs it just received down to specific teams. + +```elixir +program = + Program.new() + # The raw invoice/journal entry + |> Program.add_relation("posted_cost", [:atom, :atom, :integer]) + # The percentage split rules (e.g., 40 = 40%) + |> Program.add_relation("assessment_rule", [:atom, :atom, :integer]) + # The outputs + |> Program.add_relation("direct_allocation", [:atom, :atom, :atom, :integer]) + |> Program.add_relation("transitive_allocation", [:atom, :atom, :atom, :integer]) + + # RULE 1: Direct Document Splitting (Level 1) + # Take a posted cost and split it according to the assessment rules. + |> Program.add_rule( + {"direct_allocation", [:Doc, :Sender, :Receiver, :AllocAmount]}, + [ + {:positive, {"posted_cost", [:Doc, :Sender, :Total]}}, + {:positive, {"assessment_rule", [:Sender, :Receiver, :Pct]}} + ], + [ + # Math: Amount = (Total * Pct) / 100 + {:mul, :Total, :Pct, :TempAmt}, + {:div, :TempAmt, 100, :AllocAmount} + ] + ) + + # RULE 2: Multi-Stage Iterative Flow (Level 2+) + # If a cost center receives an allocation, and has its OWN assessment rules, forward it. + |> Program.add_rule( + {"transitive_allocation", [:Doc, :Intermediate, :FinalReceiver, :FinalAmount]}, + [ + # Look for money that just arrived via Rule 1 + {:positive, {"direct_allocation", [:Doc, :OriginalSender, :Intermediate, :InterAmount]}}, + # Look to see if the receiver has a rule to pass it on + {:positive, {"assessment_rule", [:Intermediate, :FinalReceiver, :InterPct]}} + ], + [ + # Math: FinalAmount = (InterAmount * InterPct) / 100 + {:mul, :InterAmount, :InterPct, :TempAmt2}, + {:div, :TempAmt2, 100, :FinalAmount} + ] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "assessment_rule" => %{arity: 3, types: [:atom, :atom, :integer]}, + "direct_allocation" => %{arity: 4, types: [:atom, :atom, :atom, :integer]}, + "posted_cost" => %{arity: 3, types: [:atom, :atom, :integer]}, + "transitive_allocation" => %{arity: 4, types: [:atom, :atom, :atom, :integer]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{ + relation: "transitive_allocation", + terms: [var: "Doc", var: "Intermediate", var: "FinalReceiver", var: "FinalAmount"] + }, + body: [ + positive: %ExDatalog.Atom{ + relation: "direct_allocation", + terms: [var: "Doc", var: "OriginalSender", var: "Intermediate", var: "InterAmount"] + }, + positive: %ExDatalog.Atom{ + relation: "assessment_rule", + terms: [var: "Intermediate", var: "FinalReceiver", var: "InterPct"] + } + ], + constraints: [ + %ExDatalog.Constraint{ + op: :mul, + left: {:var, "InterAmount"}, + right: {:var, "InterPct"}, + result: {:var, "TempAmt2"} + }, + %ExDatalog.Constraint{ + op: :div, + left: {:var, "TempAmt2"}, + right: {:const, 100}, + result: {:var, "FinalAmount"} + } + ] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{ + relation: "direct_allocation", + terms: [var: "Doc", var: "Sender", var: "Receiver", var: "AllocAmount"] + }, + body: [ + positive: %ExDatalog.Atom{ + relation: "posted_cost", + terms: [var: "Doc", var: "Sender", var: "Total"] + }, + positive: %ExDatalog.Atom{ + relation: "assessment_rule", + terms: [var: "Sender", var: "Receiver", var: "Pct"] + } + ], + constraints: [%ExDatalog.Constraint{op: :mul, left: {:var, ...}, ...}, ...] + } + ] +} +``` + +### The Test Data (The Month-End Close) + +Let's simulate a $10,000 rent invoice hitting the Facilities cost center. Facilities splits its costs between IT and Sales. Then, IT re-allocates its portion of the rent down to the Engineering and Product teams. + +```elixir +{:ok, knowledge} = + program + # 1. The original invoice hits the general Facilities bucket + |> Program.add_fact("posted_cost", [:inv_001, :facilities, 10000]) + + # 2. Level 1 Assessment Rules (Facilities -> IT / Sales) + |> Program.add_fact("assessment_rule", [:facilities, :it_dept, 40]) # 40% + |> Program.add_fact("assessment_rule", [:facilities, :sales_dept, 60]) # 60% + + # 3. Level 2 Assessment Rules (IT -> Eng / Product) + |> Program.add_fact("assessment_rule", [:it_dept, :eng_team, 70]) # 70% + |> Program.add_fact("assessment_rule", [:it_dept, :product_team, 30]) # 30% + + |> ExDatalog.materialize() +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "assessment_rule" => MapSet.new([ + {:facilities, :it_dept, 40}, + {:facilities, :sales_dept, 60}, + {:it_dept, :eng_team, 70}, + {:it_dept, :product_team, 30} + ]), + "direct_allocation" => MapSet.new([ + {:inv_001, :facilities, :it_dept, 4000}, + {:inv_001, :facilities, :sales_dept, 6000} + ]), + "posted_cost" => MapSet.new([{:inv_001, :facilities, 10000}]), + "transitive_allocation" => MapSet.new([ + {:inv_001, :it_dept, :eng_team, 2800}, + {:inv_001, :it_dept, :product_team, 1200} + ]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 164, + iterations: 2, + relation_sizes: %{ + "assessment_rule" => 4, + "direct_allocation" => 2, + "posted_cost" => 1, + "transitive_allocation" => 2 + } + }, + provenance: nil + }} +``` + +### The Queries + +**1. Verify Document Splitting (Level 1)** +Let's see how the initial $10,000 was split out of Facilities. + +```elixir +Knowledge.match(knowledge, "direct_allocation", [:inv_001, :facilities, :_, :_]) +``` + + + +``` +MapSet.new([{:inv_001, :facilities, :it_dept, 4000}, {:inv_001, :facilities, :sales_dept, 6000}]) +``` + +**2. Verify Multi-Stage Assessment (Level 2)** +Now, let's look at the `transitive_allocation` rule to see how IT automatically flushed its $4,000 portion down to the delivery teams based on its own specific ratios. + +```elixir +Knowledge.match(knowledge, "transitive_allocation", [:inv_001, :it_dept, :_, :_]) +``` + + + +``` +MapSet.new([{:inv_001, :it_dept, :eng_team, 2800}, {:inv_001, :it_dept, :product_team, 1200}]) +``` + +![](files/CashFlowAllocation3.png) + + + +### Why this is powerful + +If a new department is spun up next month, or Sales decides they want to allocate their costs down to specific regional teams, **you do not touch the code**. You simply add new `assessment_rule` facts, and the Datalog engine automatically cascades the math. + +To help visualize exactly how this mathematical cascade works when dealing with complex enterprise splits, here is an interactive flow diagram of the data we just generated. + +--- + +One of the most notoriously difficult problems in SAP allocations is **circular loops** (e.g., IT charges HR for software licenses, but HR charges IT for recruiting costs). Given what you know about how Datalog evaluates recursively, how do you think our current rules engine would react if we accidentally created a loop in our `assessment_rule` facts? + +--- + +--- + +## 12. Static Call Graph Analysis (Dead Code Elimination) + +**Use Case:** A compiler needs to analyze the call graph of a program to figure out which functions are safely connected to the `main()` entry point. Any function that cannot be reached from the entry point is "dead code" and should be stripped out of the final compiled binary. + +This program uses recursive reachability to find the live code, and **stratified negation** to isolate the dead code. + +#### The Program + +```elixir +program = + Program.new() + |> Program.add_relation("calls", [:atom, :atom]) + |> Program.add_relation("entry_point", [:atom]) + |> Program.add_relation("function", [:atom]) + |> Program.add_relation("reachable", [:atom]) + |> Program.add_relation("dead_function", [:atom]) + + # 1. Base Reachability: Entry points are always reachable + |> Program.add_rule( + {"reachable", [:Func]}, + [{:positive, {"entry_point", [:Func]}}] + ) + + # 2. Recursive Reachability: If Caller is reachable, Callee is reachable + |> Program.add_rule( + {"reachable", [:Callee]}, + [ + {:positive, {"reachable", [:Caller]}}, + {:positive, {"calls", [:Caller, :Callee]}} + ] + ) + + # 3 & 4. Master List: Collect all known functions (either calling or being called) + |> Program.add_rule( + {"function", [:F]}, + [{:positive, {"calls", [:F, :_]}}] + ) + |> Program.add_rule( + {"function", [:F]}, + [{:positive, {"calls", [:_, :F]}}] + ) + + # 5. The Negation: Dead code is any known function that is NOT reachable + |> Program.add_rule( + {"dead_function", [:F]}, + [ + {:positive, {"function", [:F]}}, + {:negative, {"reachable", [:F]}} + ] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "calls" => %{arity: 2, types: [:atom, :atom]}, + "dead_function" => %{arity: 1, types: [:atom]}, + "entry_point" => %{arity: 1, types: [:atom]}, + "function" => %{arity: 1, types: [:atom]}, + "reachable" => %{arity: 1, types: [:atom]} + }, + facts: [], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "dead_function", terms: [var: "F"]}, + body: [ + positive: %ExDatalog.Atom{relation: "function", terms: [var: "F"]}, + negative: %ExDatalog.Atom{relation: "reachable", terms: [var: "F"]} + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "function", terms: [var: "F"]}, + body: [positive: %ExDatalog.Atom{relation: "calls", terms: [:wildcard, {:var, "F"}]}], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "function", terms: [var: "F"]}, + body: [positive: %ExDatalog.Atom{relation: "calls", terms: [{:var, "F"}, :wildcard]}], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "reachable", terms: [var: "Callee"]}, + body: [ + positive: %ExDatalog.Atom{relation: "reachable", terms: [var: "Caller"]}, + positive: %ExDatalog.Atom{relation: "calls", terms: [var: "Caller", var: "Callee"]} + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "reachable", terms: [var: "Func"]}, + body: [positive: %ExDatalog.Atom{relation: "entry_point", terms: [var: "Func"]}], + constraints: [] + } + ] +} +``` + +#### The Test Data + +We will simulate a program where `main` calls standard application logic, but there are a few legacy utility functions left sitting in the codebase that no longer connect back to the main app. + +```elixir +{:ok, knowledge} = + program + |> Program.add_fact("entry_point", [:main]) + + # The Live Code Path + |> Program.add_fact("calls", [:main, :init]) + |> Program.add_fact("calls", [:main, :render]) + |> Program.add_fact("calls", [:init, :load_config]) + |> Program.add_fact("calls", [:render, :draw_ui]) + + # The Dead Code (An orphaned island) + |> Program.add_fact("calls", [:deprecated_start, :legacy_helper]) + + # The Tricky One (Dead code that calls live code) + |> Program.add_fact("calls", [:old_util, :draw_ui]) + + |> ExDatalog.materialize() +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "calls" => MapSet.new([ + deprecated_start: :legacy_helper, + init: :load_config, + main: :init, + main: :render, + old_util: :draw_ui, + render: :draw_ui + ]), + "dead_function" => MapSet.new([{:deprecated_start}, {:legacy_helper}, {:old_util}]), + "entry_point" => MapSet.new([{:main}]), + "function" => MapSet.new([ + {:deprecated_start}, + {:draw_ui}, + {:init}, + {:legacy_helper}, + {:load_config}, + {:main}, + {:old_util}, + {:render} + ]), + "reachable" => MapSet.new([{:draw_ui}, {:init}, {:load_config}, {:main}, {:render}]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + duration_us: 181, + iterations: 4, + relation_sizes: %{ + "calls" => 6, + "dead_function" => 3, + "entry_point" => 1, + "function" => 8, + "reachable" => 5 + } + }, + provenance: nil + }} +``` + +**1. The Garbage Collector (Find all dead code)** +Find all functions that the compiler should delete from the final binary. + +```elixir +Knowledge.match(knowledge, "dead_function", [:_]) +``` + + + +``` +MapSet.new([{:deprecated_start}, {:legacy_helper}, {:old_util}]) +``` + +*(Notice how the engine correctly flags `:old_util` as dead. Even though it points to `:draw_ui` (which is alive), reachability is directional. Because nothing calls `:old_util`, it is dead).* + + + +**2. The Live Set (What is actually running?)** +Get the list of all functions safely anchored to the application root. + +```elixir +Knowledge.match(knowledge, "reachable", [:_]) +``` + + + +``` +MapSet.new([{:draw_ui}, {:init}, {:load_config}, {:main}, {:render}]) +``` + +**3. Discovering Orphans (Who is calling this?)** +If we see that a function is dead, we can query its callers to see *why* it's dead. Let's ask who calls the `legacy_helper`. + +```elixir +Knowledge.match(knowledge, "calls", [:_, :legacy_helper]) +``` + + + +``` +MapSet.new([deprecated_start: :legacy_helper]) +``` + +--- + +--- + +## Datalog as a source of knowledge for LLMS + +JSON is generally "easier" for LLMs to read and write, but Datalog is far better when you want an LLM to actually reason about relationships. LLMs are ultimately massive pattern-matching engines. The way an LLM process information is fundamentally tied to the volume of data it was trained on and how it's tokenization works. Because of that, both formats have distinct advantages depending on what you are asking them to do. + + + +Here is the breakdown of how LLMs handle both formats. + + + +If your goal is just to have an LLM summarize data, format output for a web app, or call an external API tool, JSON wins hands down. + +* __Massive Training Bias:__ LLMs have ingested billions of lines of JSON. Their internal weights are highly optimized to predict the closing bracket } of a JSON object and format key-value pairs perfectly. + +* __Hierarchical Understanding:__ JSON natively represents trees. When data is cleanly nested (like an organization chart or a configuration file), it maps very well to how its hold context in a prompt. + +* __Tool Calling:__ The entire ecosystem of LLM tool-calling (including how an LLM interact with external functions) is built on JSON schemas. + +__The downside of JSON:__ If your data is a highly interconnected graph (like the Code Taint Tracking or Fraud Ring examples we discussed), JSON becomes a nightmare. It forces you to use arbitrary ID strings to link objects, making the payload bloated, repetitive, and prone to "context window exhaustion" where an LLM might lose track of which ID belongs to what. + +### The Ultimate Architecture: The Hand-Off + +In the real world, the most effective way to use an LLM with a complex system isn't strictly one or the other—it is a division of labor. + +Because LLMs are great at generating code but sometimes struggle to reliably execute deep, multi-step logical deductions (especially with hundreds of facts), the best architecture looks like this: + +1. **You give the LLM:** Plain English questions and a JSON schema of your database. +2. **The LLM generates:** The Datalog `Rule` or `Query` to answer your question. +3. **Your app:** Takes the generated Datalog, runs it through `ex_datalog` (which is 100% mathematically deterministic and never hallucinates), and gets the answer. +4. **Your app:** Passes the flat result back to the LMM to summarize in a friendly sentence. + +If you were to build an AI agent for your Elixir application, would you be more interested in having an LLM write the Datalog rules dynamically based on user prompts, or having the LLM parse the final `Result.match` output into human-readable reports? diff --git a/livebook/files/Cash-Allocation-Flow.png b/livebook/files/Cash-Allocation-Flow.png new file mode 100644 index 0000000..613383c Binary files /dev/null and b/livebook/files/Cash-Allocation-Flow.png differ diff --git a/livebook/files/CashFlowAllocation3.png b/livebook/files/CashFlowAllocation3.png new file mode 100644 index 0000000..764f145 Binary files /dev/null and b/livebook/files/CashFlowAllocation3.png differ diff --git a/livebook/quickstart.livemd b/livebook/quickstart.livemd new file mode 100644 index 0000000..11f9420 --- /dev/null +++ b/livebook/quickstart.livemd @@ -0,0 +1,828 @@ + + +# ExDatalog Quickstart + +```elixir +Mix.install([ + {:ex_datalog, path: Path.expand("..", __DIR__), env: :prod}, +]) +``` + +## Introduction + +**ExDatalog** is a production-grade Datalog engine for Elixir. It uses bottom-up +semi-naive fixpoint evaluation with stratified negation, constraints, provenance +tracking, and pluggable storage backends. + +In this tutorial you will learn: + +* How to define **relations**, **facts**, and **rules** +* How to write **recursive** rules (transitive closure) +* How to use **negation** with stratification +* How to add **constraints** (comparisons, arithmetic, type checks, string predicates, membership) +* How to **query** the knowledge base with `Knowledge.get` and `Knowledge.match` +* How to switch **storage backends** +* How to use **provenance** to explain derived facts + +--- + +## Setup + +```elixir +alias ExDatalog +alias ExDatalog.{Program, Constraint, Knowledge} +``` + + + +``` +[ExDatalog.Program, ExDatalog.Constraint, ExDatalog.Knowledge] +``` + +--- + +## 1. Relations and Facts + +A Datalog program starts with **relations** (schemas) and **facts** (ground +tuples that are unconditionally true). + +```elixir +program = + Program.new() + |> Program.add_relation("parent", [:atom, :atom]) + |> Program.add_relation("ancestor", [:atom, :atom]) + |> Program.add_fact("parent", [:alice, :bob]) + |> Program.add_fact("parent", [:bob, :carol]) + |> Program.add_fact("parent", [:carol, :dave]) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "ancestor" => %{arity: 2, types: [:atom, :atom]}, + "parent" => %{arity: 2, types: [:atom, :atom]} + }, + facts: [{"parent", [:carol, :dave]}, {"parent", [:bob, :carol]}, {"parent", [:alice, :bob]}], + rules: [] +} +``` + +`add_relation/3` defines a named relation with its column types. `add_fact/3` +asserts a ground tuple. The arity of each fact must match its relation. + +--- + +## 2. Rules + +Every rule has a **head** (what's being derived) and a **body** (the conditions). +The `:-` separator reads as "if". All variables in the head must also appear in +a positive body literal — this is called **range restriction** and it guarantees +the engine can always bind every variable to a concrete value. + +The rule below reads: "X is an ancestor of Y *if* X is a parent of Y." +`ancestor` is the head, `parent(X, Y)` is the body, and `X` and `Y` are +**logic variables** — placeholders that the engine binds to actual values. + +### Recursive rule: transitive ancestry + +This is what makes Datalog powerful — a rule can reference its own head +relation. The engine evaluates rules **bottom-up**: it starts with the facts, +derives new facts by applying rules, and repeats until no new facts emerge +(the **fixpoint**). Recursion is safe because Datalog guarantees termination. + +`ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).` + +This reads: "X is an ancestor of Z *if* X is a parent of some Y and Y is an +ancestor of Z." The comma means "and" — all body literals must hold +simultaneously. On each iteration, newly derived `ancestor` facts from the +previous round feed back as `ancestor(Y, Z)` matches, extending the chain +one hop at a time until no new pairs are found. + +```elixir +program = + program + |> Program.add_rule( + {"ancestor", [:X, :Y]}, + [{:positive, {"parent", [:X, :Y]}}] + ) + |> Program.add_rule( + {"ancestor", [:X, :Z]}, + [ + {:positive, {"parent", [:X, :Y]}}, + {:positive, {"ancestor", [:Y, :Z]}} + ] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "ancestor" => %{arity: 2, types: [:atom, :atom]}, + "parent" => %{arity: 2, types: [:atom, :atom]} + }, + facts: [{"parent", [:carol, :dave]}, {"parent", [:bob, :carol]}, {"parent", [:alice, :bob]}], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "ancestor", terms: [var: "X", var: "Z"]}, + body: [ + positive: %ExDatalog.Atom{relation: "parent", terms: [var: "X", var: "Y"]}, + positive: %ExDatalog.Atom{relation: "ancestor", terms: [var: "Y", var: "Z"]} + ], + constraints: [] + }, + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "ancestor", terms: [var: "X", var: "Y"]}, + body: [positive: %ExDatalog.Atom{relation: "parent", terms: [var: "X", var: "Y"]}], + constraints: [] + } + ] +} +``` + +## 3. Materializing Knowledge + +```elixir +{:ok, knowledge} = ExDatalog.materialize(program) + +ancestors = Knowledge.get(knowledge, "ancestor") + +MapSet.to_list(ancestors) +|> Enum.sort() +``` + + + +``` +[alice: :bob, alice: :carol, alice: :dave, bob: :carol, bob: :dave, carol: :dave] +``` + +You should see all six ancestor facts — the three direct parent facts plus the +three transitive ones: + +* `{:alice, :bob}`, `{:bob, :carol}`, `{:carol, :dave}` — base +* `{:alice, :carol}`, `{:bob, :dave}` — one hop +* `{:alice, :dave}` — two hops + +### Knowledge API + +```elixir +# How many ancestor facts? +Knowledge.size(knowledge, "ancestor") + +# List all relation names +Knowledge.relations(knowledge) + +# Get all parent facts +Knowledge.get(knowledge, "parent") |> MapSet.to_list() + +# Pattern-match: find all ancestors of carol +Knowledge.match(knowledge, "ancestor", [:_, :carol]) |> MapSet.to_list() + +# Pattern-match: find everyone alice is an ancestor of +Knowledge.match(knowledge, "ancestor", [:alice, :_]) |> MapSet.to_list() +``` + + + +``` +[alice: :bob, alice: :carol, alice: :dave] +``` + +--- + +## 4. Negation + +Negation uses `{:negative, ...}` body literals. Datalog requires **stratified +negation** — no circular dependency through negation. + +`bachelor(X) :- male(X), not married(X, _).` + +```elixir +program = + Program.new() + |> Program.add_relation("male", [:atom]) + |> Program.add_relation("married", [:atom, :atom]) + |> Program.add_relation("bachelor", [:atom]) + |> Program.add_fact("male", [:alice]) + |> Program.add_fact("male", [:bob]) + |> Program.add_fact("married", [:alice, :carol]) + |> Program.add_rule( + {"bachelor", [:X]}, + [ + {:positive, {"male", [:X]}}, + {:negative, {"married", [:X, :_]}} + ] + ) + +{:ok, knowledge} = ExDatalog.materialize(program) + +Knowledge.get(knowledge, "bachelor") |> MapSet.to_list() +``` + + + +``` +[{:bob}] +``` + +Only `:bob` appears — `:alice` is excluded because she is married. The wildcard +`:_` in the shorthand notation matches any second argument. + +--- + +## 5. Constraints + +Constraints extend rules with comparisons, arithmetic, type checks, string +predicates, and membership tests. + +### Comparison Constraints + +`gt`, `lt`, `gte`, `lte`, `eq`, `neq` — filter bindings. + +Find employees earning more than 100,000: + +```elixir +program = + Program.new() + |> Program.add_relation("income", [:atom, :integer]) + |> Program.add_relation("high_earner", [:atom]) + |> Program.add_fact("income", [:alice, 120_000]) + |> Program.add_fact("income", [:bob, 80_000]) + |> Program.add_fact("income", [:carol, 150_000]) + |> Program.add_rule( + {"high_earner", [:X]}, + [{:positive, {"income", [:X, :S]}}], + [{:gt, :S, 100_000}] + ) + +{:ok, knowledge} = ExDatalog.materialize(program) + +Knowledge.get(knowledge, "high_earner") |> MapSet.to_list() +``` + + + +``` +[{:alice}, {:carol}] +``` + +### Arithmetic Constraints + +`add`, `sub`, `mul`, `div` — bind a result variable. Both inputs must be +bound before evaluation. + +Compute the sum of two numbers: + +```elixir +program = + Program.new() + |> Program.add_relation("pair", [:integer, :integer]) + |> Program.add_relation("sum", [:integer, :integer, :integer]) + |> Program.add_fact("pair", [3, 7]) + |> Program.add_rule( + {"sum", [:A, :B, :C]}, + [{:positive, {"pair", [:A, :B]}}], + [{:add, :A, :B, :C}] + ) + +{:ok, knowledge} = ExDatalog.materialize(program) + +Knowledge.get(knowledge, "sum") |> MapSet.to_list() +``` + + + +``` +[{3, 7, 10}] +``` + +> **Note:** Arithmetic constraints bind a *result* variable (`C` above). +> Both input variables (`A` and `B`) must already be bound by positive body +> atoms before the constraint fires. You cannot chain constraints where one +> constraint's result feeds into the next — use a separate rule instead. + +### Type Predicate Constraints + +`type_integer/1`, `type_binary/1`, `type_atom/1` — filter by Elixir type. + +```elixir +program = + Program.new() + |> Program.add_relation("value", [:atom, :any]) + |> Program.add_relation("int_value", [:atom, :integer]) + |> Program.add_fact("value", [:x, 42]) + |> Program.add_fact("value", [:y, "hello"]) + |> Program.add_fact("value", [:z, :ok]) + |> Program.add_rule( + {"int_value", [:N, :V]}, + [{:positive, {"value", [:N, :V]}}], + [{:is_integer, :V}] + ) + +{:ok, knowledge} = ExDatalog.materialize(program) + +Knowledge.get(knowledge, "int_value") |> MapSet.to_list() +``` + + + +``` +[x: 42] +``` + +Only `{:x, 42}` passes — `"hello"` and `:ok` are filtered out. + +### String Predicate Constraints + +`starts_with/2`, `contains/2` — filter string values. + +Find users whose email starts with `"admin."`: + +```elixir +program = + Program.new() + |> Program.add_relation("user", [:atom, :string]) + |> Program.add_relation("admin", [:atom]) + |> Program.add_fact("user", [:alice, "admin.alice@example.com"]) + |> Program.add_fact("user", [:bob, "bob@example.com"]) + |> Program.add_rule( + {"admin", [:X]}, + [{:positive, {"user", [:X, :E]}}], + [{:starts_with, :E, "admin."}] + ) +``` + + + +``` +%ExDatalog.Program{ + relations: %{ + "admin" => %{arity: 1, types: [:atom]}, + "user" => %{arity: 2, types: [:atom, :string]} + }, + facts: [{"user", [:bob, "bob@example.com"]}, {"user", [:alice, "admin.alice@example.com"]}], + rules: [ + %ExDatalog.Rule{ + head: %ExDatalog.Atom{relation: "admin", terms: [var: "X"]}, + body: [positive: %ExDatalog.Atom{relation: "user", terms: [var: "X", var: "E"]}], + constraints: [ + %ExDatalog.Constraint{ + op: :starts_with, + left: {:var, "E"}, + right: {:const, "admin."}, + result: nil + } + ] + } + ] +} +``` + +This rule reads: "X is an admin if there exists a user(X, E) where E starts +with `"admin."`". The body atom binds `X` to the username and `E` to the email, +then the constraint filters — only bindings where `E` begins with the prefix +survive. Alice passes (`"admin.alice@example.com"` starts with `"admin."`), +Bob does not (`"bob@example.com"` does not). + +```elixir +{:ok, knowledge} = ExDatalog.materialize(program) + +Knowledge.get(knowledge, "admin") |> MapSet.to_list() +``` + + + +``` +[{:alice}] +``` + +### Membership Constraint + +`member/2` — test whether a value belongs to a fixed list of values +(wrapped in `Term.const/1`). The right-hand side must be a compile-time +constant — you cannot pass a variable as the list. + +```elixir +program = + Program.new() + |> Program.add_relation("employee", [:atom, :atom]) + |> Program.add_relation("eng_employee", [:atom]) + |> Program.add_fact("employee", [:alice, :engineering]) + |> Program.add_fact("employee", [:bob, :sales]) + |> Program.add_fact("employee", [:carol, :engineering]) + |> Program.add_rule( + {"eng_employee", [:X]}, + [{:positive, {"employee", [:X, :Dept]}}], + [{:member, :Dept, [:engineering, :infra]}] + ) + +{:ok, knowledge} = ExDatalog.materialize(program) + +Knowledge.get(knowledge, "eng_employee") |> MapSet.to_list() +``` + + + +``` +[{:alice}, {:carol}] +``` + +--- + +## 6. A Practical Example — Network Reachability + +Computing **transitive closure** is one of Datalog's sweet spots. Transitive +closure means deriving every reachability relationship in a graph — not just +direct connections, but every indirect path too. If A links to B, and B links +to C, then A also reaches C (through B), even though there's no direct edge +from A to C. The "closure" means you keep following connections until nothing +new is discovered. + +You write the rule once and the engine derives all indirect connections +automatically via fixpoint iteration. In the example below, the `link` relation +only has direct edges (a→b, b→c, c→d, d→e), but the `reachable` rule derives +every path you can traverse through those edges — including a→c, a→d, a→e, +b→d, b→e, c→e. + +```elixir +program = + Program.new() + |> Program.add_relation("link", [:atom, :atom]) + |> Program.add_relation("reachable", [:atom, :atom]) + |> Program.add_relation("blocked", [:atom, :atom]) + |> Program.add_relation("allowed", [:atom, :atom]) + |> Program.add_relation("violation", [:atom, :atom]) + # Network topology + |> Program.add_fact("link", [:a, :b]) + |> Program.add_fact("link", [:b, :c]) + |> Program.add_fact("link", [:c, :d]) + |> Program.add_fact("link", [:d, :e]) + # Firewall: only certain links are explicitly allowed + |> Program.add_fact("blocked", [:c, :d]) + # Rules + |> Program.add_rule( + {"reachable", [:X, :Y]}, + [{:positive, {"link", [:X, :Y]}}] + ) + |> Program.add_rule( + {"reachable", [:X, :Z]}, + [ + {:positive, {"link", [:X, :Y]}}, + {:positive, {"reachable", [:Y, :Z]}} + ] + ) + |> Program.add_rule( + {"allowed", [:X, :Y]}, + [ + {:positive, {"reachable", [:X, :Y]}}, + {:negative, {"blocked", [:X, :Y]}} + ] + ) + +{:ok, knowledge} = ExDatalog.materialize(program) + +IO.puts("Reachable paths:") +Knowledge.get(knowledge, "reachable") |> MapSet.to_list() |> Enum.sort() |> IO.inspect() + +IO.puts("\nAllowed paths (reachable and not blocked):") +Knowledge.get(knowledge, "allowed") |> MapSet.to_list() |> Enum.sort() |> IO.inspect() + +IO.puts("\nBlocked paths:") +Knowledge.get(knowledge, "blocked") |> MapSet.to_list() |> Enum.sort() |> IO.inspect() +``` + + + +``` +Reachable paths: +[a: :b, a: :c, a: :d, a: :e, b: :c, b: :d, b: :e, c: :d, c: :e, d: :e] + +Allowed paths (reachable and not blocked): +[a: :b, a: :c, a: :d, a: :e, b: :c, b: :d, b: :e, c: :e, d: :e] + +Blocked paths: +[c: :d] +``` + + + +``` +[c: :d] +``` + +--- + +## 7. Storage Backends + +ExDatalog ships with two storage backends: + +* **`ExDatalog.Storage.Map`** (default) — on-heap, suitable for <100K facts. +* **`ExDatalog.Storage.ETS`** — off-heap, concurrent reads, suitable for >100K facts. + +```elixir +# Use the ETS backend for large fact sets +{:ok, knowledge_ets} = ExDatalog.materialize(program, storage: ExDatalog.Storage.ETS) + +# Knowledge bases are identical regardless of backend +Knowledge.get(knowledge_ets, "reachable") == Knowledge.get(knowledge, "reachable") +``` + + + +``` +true +``` + +Both backends guarantee **deterministic ordering** — identical programs produce +identical results. + +--- + +## 8. Provenance and Explain + +**Provenance tracking** records *how* each derived fact came to be. Without it, +the knowledge base tells you *what* is true but not *why*. With it, every +derived fact carries a trace of which rule produced it and which input facts +fed into that rule — forming a **derivation tree** you can inspect or debug. +Base facts (the ones you asserted with `add_fact`) are attributed as `:base_fact`, +while derived facts point to the rule ID that produced them. + +Enable provenance tracking with `explain: true`: + +```elixir +program = + Program.new() + |> Program.add_relation("parent", [:atom, :atom]) + |> Program.add_relation("ancestor", [:atom, :atom]) + |> Program.add_fact("parent", [:alice, :bob]) + |> Program.add_fact("parent", [:bob, :carol]) + |> Program.add_rule( + {"ancestor", [:X, :Y]}, + [{:positive, {"parent", [:X, :Y]}}] + ) + |> Program.add_rule( + {"ancestor", [:X, :Z]}, + [ + {:positive, {"parent", [:X, :Y]}}, + {:positive, {"ancestor", [:Y, :Z]}} + ] + ) + +{:ok, knowledge} = ExDatalog.materialize(program, explain: true) +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{ + "ancestor" => MapSet.new([alice: :bob, alice: :carol, bob: :carol]), + "parent" => MapSet.new([alice: :bob, bob: :carol]) + }, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :map, + indexed_lookup: false, + concurrent_reads: false, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + relation_sizes: %{"ancestor" => 3, "parent" => 2}, + duration_us: 97, + iterations: 2 + }, + provenance: %{ + rules: %{ + 0 => %ExDatalog.IR.Rule{ + id: 0, + head: %ExDatalog.IR.Atom{relation: "ancestor", terms: [var: "X", var: "Y"]}, + body: [positive: %ExDatalog.IR.Atom{relation: "parent", terms: [var: "X", var: "Y"]}], + stratum: 0, + metadata: %{} + }, + 1 => %ExDatalog.IR.Rule{ + id: 1, + head: %ExDatalog.IR.Atom{relation: "ancestor", terms: [var: "X", var: "Z"]}, + body: [ + positive: %ExDatalog.IR.Atom{relation: "parent", terms: [var: "X", var: "Y"]}, + positive: %ExDatalog.IR.Atom{relation: "ancestor", terms: [var: "Y", var: "Z"]} + ], + stratum: 0, + metadata: %{} + } + }, + fact_origins: %{ + "ancestor" => %{{:alice, :bob} => 0, {:alice, :carol} => 1, {:bob, :carol} => 0}, + "parent" => %{{:alice, :bob} => :base, {:bob, :carol} => :base} + } + } + }} +``` + +### Explain a base fact + +```elixir +ExDatalog.Explain.explain(knowledge, "ancestor", {:alice, :bob}) +``` + + + +``` +{:ok, %ExDatalog.Explain.Node{fact: {:alice, :bob}, rule_id: 0, children: [:base_fact, :base_fact]}} +``` + +Returns `:base_fact` — `{:alice, :bob}` was derived directly from the base +rule `ancestor(X,Y) :- parent(X,Y)`. + +### Explain a derived fact + +```elixir +ExDatalog.Explain.explain(knowledge, "ancestor", {:alice, :carol}) +``` + + + +``` +{:ok, + %ExDatalog.Explain.Node{ + fact: {:alice, :carol}, + rule_id: 1, + children: [ + :base_fact, + :base_fact, + %ExDatalog.Explain.Node{fact: {:alice, :bob}, rule_id: 0, children: [:base_fact, :base_fact]}, + %ExDatalog.Explain.Node{fact: {:alice, :carol}, rule_id: 1, children: []}, + %ExDatalog.Explain.Node{fact: {:bob, :carol}, rule_id: 0, children: [:base_fact, :base_fact]} + ] + }} +``` + +Returns a derivation tree showing which rule produced this fact and which +sub-facts it depended on. + +### Without provenance + +```elixir +{:ok, no_prov} = ExDatalog.materialize(program) + +ExDatalog.Explain.explain(no_prov, "ancestor", {:alice, :bob}) +``` + + + +``` +{:error, :no_provenance} +``` + +Returns `{:error, :no_provenance}` — provenance tracking was not enabled. + +--- + +## 9. Step-by-Step Pipeline + +You can also run the pipeline steps individually instead of using the +one-shot `materialize/2`: + +```elixir +program = + Program.new() + |> Program.add_relation("edge", [:atom, :atom]) + |> Program.add_relation("path", [:atom, :atom]) + |> Program.add_fact("edge", [:a, :b]) + |> Program.add_fact("edge", [:b, :c]) + |> Program.add_rule( + {"path", [:X, :Y]}, + [{:positive, {"edge", [:X, :Y]}}] + ) + |> Program.add_rule( + {"path", [:X, :Z]}, + [ + {:positive, {"edge", [:X, :Y]}}, + {:positive, {"path", [:Y, :Z]}} + ] + ) + +# Step 1: Validate +{:ok, validated} = ExDatalog.validate(program) + +# Step 2: Compile to IR +{:ok, ir} = ExDatalog.compile(validated) + +# Step 3: Evaluate +{:ok, knowledge} = ExDatalog.evaluate(ir, []) + +Knowledge.get(knowledge, "path") |> MapSet.to_list() |> Enum.sort() +``` + + + +``` +[a: :b, a: :c, b: :c] +``` + +This is useful for debugging — if validation fails, you get structured errors: + +```elixir +bad_program = + Program.new() + |> Program.add_relation("edge", [:atom, :atom]) + |> Program.add_fact("edge", [:only_one_arg]) + +ExDatalog.validate(bad_program) +``` + + + +``` +{:error, "arity mismatch for relation \"edge\": expected 2 values, got 1"} +``` + +Returns `{:error, "arity mismatch for relation \"edge\": expected 2 values, got 1"}`. + +--- + +## 10. Evaluation Options + +`ExDatalog.materialize/2` and `ExDatalog.evaluate/2` accept these options: + +| Option | Default | Description | +| ----------------- | ------------------------ | --------------------------------------- | +| `:engine` | `ExDatalog.Engine.Naive` | Evaluation backend | +| `:storage` | `ExDatalog.Storage.Map` | Storage backend | +| `:storage_opts` | `[]` | Storage options (e.g., ETS access mode) | +| `:max_iterations` | `10_000` | Fixpoint iteration limit | +| `:timeout_ms` | `30_000` | Wall-clock timeout in milliseconds | +| `:explain` | `false` | Enable provenance tracking | + +```elixir +{:ok, knowledge} = + ExDatalog.materialize(program, + storage: ExDatalog.Storage.ETS, + storage_opts: [access: :public], + max_iterations: 1_000, + timeout_ms: 5_000 + ) +``` + + + +``` +{:ok, + %ExDatalog.Knowledge{ + relations: %{"edge" => MapSet.new([a: :b, b: :c]), "path" => MapSet.new([a: :b, a: :c, b: :c])}, + stats: %{ + capabilities: %ExDatalog.Capabilities{ + storage_type: :ets, + indexed_lookup: true, + concurrent_reads: true, + arithmetic_constraints: true, + comparison_constraints: true, + type_predicates: true, + string_predicates: true, + provenance: true, + external_execution: false + }, + relation_sizes: %{"edge" => 2, "path" => 3}, + duration_us: 167, + iterations: 2 + }, + provenance: nil + }} +``` + +--- + +## Summary + +| Concept | API | +| ---------------- | ---------------------------------------------------------------- | +| Create program | `Program.new()` | +| Define relation | `Program.add_relation(program, name, types)` | +| Add fact | `Program.add_fact(program, name, values)` | +| Add rule | `Program.add_rule(program, {"rel", [terms]}, body, constraints)` | +| Variable | `:X` (uppercase atom in shorthand) | +| Constant | `:alice` or `42` (lowercase atom or value in shorthand) | +| Wildcard | `:_` in shorthand | +| Positive literal | `{:positive, {"rel", [terms]}}` | +| Negative literal | `{:negative, {"rel", [terms]}}` | +| Comparison | `{:gt, :X, 100_000}`, `{:neq, :A, :B}`, ... | +| Arithmetic | `{:add, :X, :Y, :Z}`, `{:sub, :A, :B, :C}`, ... | +| Type check | `{:is_integer, :V}`, `{:is_binary, :S}`, `{:is_atom, :A}` | +| String predicate | `{:starts_with, :E, "prefix"}`, `{:contains, :S, "sub"}` | +| Membership | `{:member, :X, [:a, :b, :c]}` | +| Materialize | `ExDatalog.materialize(program, opts)` | +| Get knowledge | `Knowledge.get(knowledge, relation)` | +| Pattern match | `Knowledge.match(knowledge, relation, pattern)` | +| Explain | `ExDatalog.Explain.explain(knowledge, relation, tuple)` | +| ETS backend | `ExDatalog.materialize(program, storage: ExDatalog.Storage.ETS)` | + +For more details, see the [API documentation](https://hexdocs.pm/ex_datalog). diff --git a/mix.exs b/mix.exs index 1ddc19d..cb20860 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule ExDatalog.MixProject do use Mix.Project - @version "0.2.0" + @version "0.3.0" @source_url "https://github.com/thanos/ex_datalog" def project do @@ -111,9 +111,11 @@ defmodule ExDatalog.MixProject do extras: [ "README.md", "CHANGELOG.md", - "docs/what-is-datalog.md", - "docs/constraints.md", - "docs/storage_backends.md" + {"docs/what-is-datalog.md", filename: "what-is-datalog", title: "What is Datalog?"}, + {"docs/constraints.md", filename: "constraints", title: "Constraints"}, + {"docs/storage_backends.md", filename: "storage-backends", title: "Storage Backends"}, + {"livebook/quickstart.livemd", filename: "quickstart", title: "Quickstart Tutorial"}, + {"livebook/examples.livemd", filename: "examples", title: "Examples"} ], groups_for_modules: [ "Program Builder": [ @@ -132,8 +134,8 @@ defmodule ExDatalog.MixProject do "Compiler & IR": ~r/ExDatalog\.(Compiler|IR).*/, Engine: ~r/ExDatalog\.Engine.*/, Storage: ~r/ExDatalog\.Storage.*/, - Results: [ - ExDatalog.Result, + Knowledge: [ + ExDatalog.Knowledge, ExDatalog.Explain, ExDatalog.Telemetry ] diff --git a/test/ex_datalog/atom_test.exs b/test/ex_datalog/atom_test.exs index 2dcd5e2..9a70cbb 100644 --- a/test/ex_datalog/atom_test.exs +++ b/test/ex_datalog/atom_test.exs @@ -98,4 +98,26 @@ defmodule ExDatalog.AtomTest do assert Atom.valid?(%Atom{relation: nil, terms: []}) == false end end + + describe "from_tuple/1" do + test "creates atom from shorthand tuple with variables" do + atom = Atom.from_tuple({"parent", [:X, :Y]}) + assert atom == %Atom{relation: "parent", terms: [{:var, "X"}, {:var, "Y"}]} + end + + test "creates atom from shorthand tuple with wildcard" do + atom = Atom.from_tuple({"role", [:User, :_]}) + assert atom == %Atom{relation: "role", terms: [{:var, "User"}, :wildcard]} + end + + test "creates atom from shorthand tuple with constants" do + atom = Atom.from_tuple({"value", [:X, 42]}) + assert atom == %Atom{relation: "value", terms: [{:var, "X"}, {:const, 42}]} + end + + test "passes through existing Atom struct" do + original = Atom.new("parent", [Term.var("X"), Term.var("Y")]) + assert Atom.from_tuple(original) == original + end + end end diff --git a/test/ex_datalog/constraint_test.exs b/test/ex_datalog/constraint_test.exs index 1af97dc..8c802d7 100644 --- a/test/ex_datalog/constraint_test.exs +++ b/test/ex_datalog/constraint_test.exs @@ -464,4 +464,76 @@ defmodule ExDatalog.ConstraintTest do assert :filter = Constraint.evaluate(c, %{"X" => :z}, %ExDatalog.Constraint.Context{}) end end + + describe "from_tuple/1" do + test "creates comparison constraint from shorthand" do + c = Constraint.from_tuple({:neq, :A, :B}) + assert c == Constraint.neq(Term.var("A"), Term.var("B")) + end + + test "creates gt constraint from shorthand with constant" do + c = Constraint.from_tuple({:gt, :S, 100_000}) + assert c == Constraint.gt(Term.var("S"), Term.const(100_000)) + end + + test "creates arithmetic constraint from shorthand" do + c = Constraint.from_tuple({:add, :X, :Y, :Z}) + assert c == Constraint.add(Term.var("X"), Term.var("Y"), Term.var("Z")) + end + + test "creates sub constraint from shorthand" do + c = Constraint.from_tuple({:sub, :X, :C, :Y}) + assert c == Constraint.sub(Term.var("X"), Term.var("C"), Term.var("Y")) + end + + test "creates type predicate from shorthand" do + c = Constraint.from_tuple({:is_integer, :V}) + assert c == Constraint.type_integer(Term.var("V")) + end + + test "creates string predicate from shorthand" do + c = Constraint.from_tuple({:starts_with, :E, "admin."}) + assert c == Constraint.starts_with(Term.var("E"), Term.const("admin.")) + end + + test "creates membership constraint from shorthand" do + c = Constraint.from_tuple({:member, :Dept, [:engineering, :infra]}) + assert c == Constraint.member(Term.var("Dept"), Term.const([:engineering, :infra])) + end + + test "passes through existing Constraint struct" do + original = Constraint.neq(@x, @y) + assert Constraint.from_tuple(original) == original + end + + test "creates all comparison ops from shorthand" do + for op <- [:gt, :lt, :gte, :lte, :eq, :neq] do + c = Constraint.from_tuple({op, :A, :B}) + assert c.op == op + assert c.left == {:var, "A"} + assert c.right == {:var, "B"} + assert c.result == nil + end + end + + test "creates all arithmetic ops from shorthand" do + for op <- [:add, :sub, :mul, :div] do + c = Constraint.from_tuple({op, :X, :Y, :Z}) + assert c.op == op + assert c.left == {:var, "X"} + assert c.right == {:var, "Y"} + assert c.result == {:var, "Z"} + end + end + + test "creates all type predicate ops from shorthand" do + for op <- [:is_integer, :is_binary, :is_atom] do + c = Constraint.from_tuple({op, :V}) + assert c.op == op + assert c.left == {:var, "V"} + assert c.right == nil + assert c.result == nil + end + end + end end diff --git a/test/ex_datalog/explain_test.exs b/test/ex_datalog/explain_test.exs index fa52d4d..e2ca55d 100644 --- a/test/ex_datalog/explain_test.exs +++ b/test/ex_datalog/explain_test.exs @@ -1,11 +1,11 @@ defmodule ExDatalog.ExplainTest do use ExUnit.Case, async: true - alias ExDatalog.{Atom, Explain, Program, Result, Rule, Term} + alias ExDatalog.{Atom, Explain, Knowledge, Program, Rule, Term} describe "explain/3 with no provenance" do test "returns error when provenance is nil" do - result = %Result{ + result = %Knowledge{ relations: %{"parent" => MapSet.new([{:alice, :bob}])}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{"parent" => 1}}, provenance: nil @@ -21,7 +21,7 @@ defmodule ExDatalog.ExplainTest do Program.new() |> Program.add_relation("parent", [:atom, :atom]) |> Program.add_fact("parent", [:alice, :bob]) - |> ExDatalog.query(explain: true) + |> ExDatalog.materialize(explain: true) assert {:ok, :base_fact} = Explain.explain(result, "parent", {:alice, :bob}) end @@ -38,7 +38,7 @@ defmodule ExDatalog.ExplainTest do [{:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])}] ) ) - |> ExDatalog.query(explain: true) + |> ExDatalog.materialize(explain: true) assert {:ok, tree} = Explain.explain(result, "ancestor", {:alice, :bob}) assert %Explain.Node{} = tree @@ -69,7 +69,7 @@ defmodule ExDatalog.ExplainTest do ] ) ) - |> ExDatalog.query(explain: true) + |> ExDatalog.materialize(explain: true) # Direct derivation: parent -> ancestor # The direct rule (single body atom) derives ancestor(alice, bob) @@ -93,7 +93,7 @@ defmodule ExDatalog.ExplainTest do Program.new() |> Program.add_relation("parent", [:atom, :atom]) |> Program.add_fact("parent", [:alice, :bob]) - |> ExDatalog.query(explain: true) + |> ExDatalog.materialize(explain: true) assert {:error, :not_found} = Explain.explain(result, "parent", {:charlie, :dave}) end @@ -116,7 +116,7 @@ defmodule ExDatalog.ExplainTest do ] ) ) - |> ExDatalog.query(explain: true) + |> ExDatalog.materialize(explain: true) assert {:ok, tree} = Explain.explain(result, "bachelor", {:bob}) assert tree.rule_id == 0 @@ -129,7 +129,7 @@ defmodule ExDatalog.ExplainTest do |> Program.add_relation("person", [:atom]) |> Program.add_fact("person", [:alice]) |> Program.add_fact("person", [:bob]) - |> ExDatalog.query(explain: true) + |> ExDatalog.materialize(explain: true) assert result.provenance != nil origins = result.provenance.fact_origins @@ -149,7 +149,7 @@ defmodule ExDatalog.ExplainTest do [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] ) ) - |> ExDatalog.query(explain: true) + |> ExDatalog.materialize(explain: true) origins = result.provenance.fact_origins assert origins["edge"][{:a, :b}] == :base diff --git a/test/ex_datalog/result_test.exs b/test/ex_datalog/knowledge_test.exs similarity index 71% rename from test/ex_datalog/result_test.exs rename to test/ex_datalog/knowledge_test.exs index 52abbf7..4e4b257 100644 --- a/test/ex_datalog/result_test.exs +++ b/test/ex_datalog/knowledge_test.exs @@ -1,62 +1,62 @@ -defmodule ExDatalog.ResultTest do +defmodule ExDatalog.KnowledgeTest do use ExUnit.Case, async: true - alias ExDatalog.Result + alias ExDatalog.Knowledge describe "get/2" do test "returns MapSet of tuples for a relation" do - result = %Result{ + knowledge = %Knowledge{ relations: %{"parent" => MapSet.new([{:alice, :bob}])}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{"parent" => 1}} } - assert Result.get(result, "parent") == MapSet.new([{:alice, :bob}]) + assert Knowledge.get(knowledge, "parent") == MapSet.new([{:alice, :bob}]) end test "returns empty MapSet for unknown relation" do - result = %Result{ + knowledge = %Knowledge{ relations: %{}, stats: %{iterations: 0, duration_us: 0, relation_sizes: %{}} } - assert Result.get(result, "unknown") == MapSet.new() + assert Knowledge.get(knowledge, "unknown") == MapSet.new() end end describe "match/3" do test "matches tuples by pattern" do - result = %Result{ + knowledge = %Knowledge{ relations: %{ "parent" => MapSet.new([{:alice, :bob}, {:alice, :carol}, {:bob, :dave}]) }, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{}} } - matched = Result.match(result, "parent", [:alice, :_]) + matched = Knowledge.match(knowledge, "parent", [:alice, :_]) assert MapSet.size(matched) == 2 assert {:alice, :bob} in matched assert {:alice, :carol} in matched end test "matches with all wildcards" do - result = %Result{ + knowledge = %Knowledge{ relations: %{"parent" => MapSet.new([{:alice, :bob}])}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{}} } - matched = Result.match(result, "parent", [:_, :_]) + matched = Knowledge.match(knowledge, "parent", [:_, :_]) assert MapSet.size(matched) == 1 end test "matches with exact values" do - result = %Result{ + knowledge = %Knowledge{ relations: %{ "parent" => MapSet.new([{:alice, :bob}, {:carol, :dave}]) }, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{}} } - matched = Result.match(result, "parent", [:alice, :bob]) + matched = Knowledge.match(knowledge, "parent", [:alice, :bob]) assert MapSet.size(matched) == 1 assert {:alice, :bob} in matched end @@ -64,23 +64,23 @@ defmodule ExDatalog.ResultTest do describe "size/2" do test "returns number of tuples" do - result = %Result{ + knowledge = %Knowledge{ relations: %{"parent" => MapSet.new([{:a, :b}, {:c, :d}])}, stats: %{iterations: 1, duration_us: 0, relation_sizes: %{}} } - assert Result.size(result, "parent") == 2 + assert Knowledge.size(knowledge, "parent") == 2 end end describe "relations/1" do test "returns sorted list of relation names" do - result = %Result{ + knowledge = %Knowledge{ relations: %{"z" => MapSet.new(), "a" => MapSet.new()}, stats: %{iterations: 0, duration_us: 0, relation_sizes: %{}} } - assert Result.relations(result) == ["a", "z"] + assert Knowledge.relations(knowledge) == ["a", "z"] end end end diff --git a/test/ex_datalog/program_test.exs b/test/ex_datalog/program_test.exs index 15d1cf2..114a858 100644 --- a/test/ex_datalog/program_test.exs +++ b/test/ex_datalog/program_test.exs @@ -2,7 +2,7 @@ defmodule ExDatalog.ProgramTest do use ExUnit.Case, async: true doctest ExDatalog.Program - alias ExDatalog.{Atom, Program, Rule, Term} + alias ExDatalog.{Atom, Constraint, Program, Rule, Term} defp base_program do Program.new() @@ -346,4 +346,220 @@ defmodule ExDatalog.ProgramTest do assert msg =~ "not defined" end end + + describe "add_rule/3 shorthand" do + test "adds a basic rule with tuple shorthand" do + program = + Program.new() + |> Program.add_relation("parent", [:atom, :atom]) + |> Program.add_relation("ancestor", [:atom, :atom]) + |> Program.add_rule( + {"ancestor", [:X, :Y]}, + [{:positive, {"parent", [:X, :Y]}}] + ) + + assert length(program.rules) == 1 + rule = hd(program.rules) + assert rule.head.relation == "ancestor" + assert rule.head.terms == [{:var, "X"}, {:var, "Y"}] + + assert rule.body == [ + {:positive, %Atom{relation: "parent", terms: [{:var, "X"}, {:var, "Y"}]}} + ] + + assert rule.constraints == [] + end + + test "adds a rule with negation using tuple shorthand" do + program = + Program.new() + |> Program.add_relation("person", [:atom]) + |> Program.add_relation("married", [:atom, :atom]) + |> Program.add_relation("bachelor", [:atom]) + |> Program.add_rule( + {"bachelor", [:X]}, + [ + {:positive, {"person", [:X]}}, + {:negative, {"married", [:X, :_]}} + ] + ) + + assert length(program.rules) == 1 + rule = hd(program.rules) + assert rule.head.relation == "bachelor" + assert length(rule.body) == 2 + assert elem(Enum.at(rule.body, 1), 0) == :negative + end + + test "returns error when head relation undefined" do + result = + Program.new() + |> Program.add_relation("parent", [:atom, :atom]) + |> Program.add_rule( + {"undefined", [:X]}, + [{:positive, {"parent", [:X]}}] + ) + + assert {:error, msg} = result + assert msg =~ "undefined relation" + end + + test "error propagates through shorthand add_rule" do + result = + Program.new() + |> Program.add_relation("", [:atom]) + |> Program.add_rule( + {"ancestor", [:X, :Y]}, + [{:positive, {"parent", [:X, :Y]}}] + ) + + assert {:error, msg} = result + assert msg =~ "non-empty string" + end + + test "lowercase atoms become constants in head" do + program = + Program.new() + |> Program.add_relation("likes", [:atom, :atom]) + |> Program.add_rule( + {"likes", [:alice, :X]}, + [{:positive, {"likes", [:alice, :X]}}] + ) + + rule = hd(program.rules) + assert rule.head.terms == [{:const, :alice}, {:var, "X"}] + end + + test "integers become constants in shorthand" do + program = + Program.new() + |> Program.add_relation("value", [:atom, :integer]) + |> Program.add_relation("big_value", [:atom, :integer]) + |> Program.add_rule( + {"big_value", [:X, 42]}, + [{:positive, {"value", [:X, 42]}}] + ) + + rule = hd(program.rules) + assert rule.head.terms == [{:var, "X"}, {:const, 42}] + end + end + + describe "add_rule/4 shorthand with constraints" do + test "adds a rule with comparison constraint" do + program = + Program.new() + |> Program.add_relation("income", [:atom, :integer]) + |> Program.add_relation("high_earner", [:atom]) + |> Program.add_rule( + {"high_earner", [:X]}, + [{:positive, {"income", [:X, :S]}}], + [{:gt, :S, 100_000}] + ) + + assert length(program.rules) == 1 + rule = hd(program.rules) + assert length(rule.constraints) == 1 + assert rule.constraints == [Constraint.gt(Term.var("S"), Term.const(100_000))] + end + + test "adds a rule with arithmetic constraint" do + program = + Program.new() + |> Program.add_relation("pair", [:integer, :integer]) + |> Program.add_relation("sum", [:integer, :integer, :integer]) + |> Program.add_rule( + {"sum", [:A, :B, :C]}, + [{:positive, {"pair", [:A, :B]}}], + [{:add, :A, :B, :C}] + ) + + rule = hd(program.rules) + assert rule.constraints == [Constraint.add(Term.var("A"), Term.var("B"), Term.var("C"))] + end + + test "adds a rule with neq constraint" do + program = + Program.new() + |> Program.add_relation("friend", [:atom, :atom]) + |> Program.add_relation("recommendation", [:atom, :atom]) + |> Program.add_rule( + {"recommendation", [:A, :C]}, + [ + {:positive, {"friend", [:A, :B]}}, + {:positive, {"friend", [:B, :C]}}, + {:negative, {"friend", [:A, :C]}} + ], + [{:neq, :A, :C}] + ) + + rule = hd(program.rules) + assert rule.constraints == [Constraint.neq(Term.var("A"), Term.var("C"))] + end + + test "adds a rule with type predicate constraint" do + program = + Program.new() + |> Program.add_relation("value", [:atom, :any]) + |> Program.add_relation("int_value", [:atom, :integer]) + |> Program.add_rule( + {"int_value", [:N, :V]}, + [{:positive, {"value", [:N, :V]}}], + [{:is_integer, :V}] + ) + + rule = hd(program.rules) + assert rule.constraints == [Constraint.type_integer(Term.var("V"))] + end + + test "adds a rule with membership constraint" do + program = + Program.new() + |> Program.add_relation("employee", [:atom, :atom]) + |> Program.add_relation("eng_employee", [:atom]) + |> Program.add_rule( + {"eng_employee", [:X]}, + [{:positive, {"employee", [:X, :Dept]}}], + [{:member, :Dept, [:engineering, :infra]}] + ) + + rule = hd(program.rules) + + assert rule.constraints == [ + Constraint.member(Term.var("Dept"), Term.const([:engineering, :infra])) + ] + end + + test "mixes shorthand and struct body literals" do + program = + Program.new() + |> Program.add_relation("parent", [:atom, :atom]) + |> Program.add_relation("ancestor", [:atom, :atom]) + |> Program.add_rule( + {"ancestor", [:X, :Y]}, + [ + {:positive, {"parent", [:X, :Y]}}, + {:positive, Atom.new("parent", [Term.var("X"), Term.var("Y")])} + ] + ) + + assert length(program.rules) == 1 + rule = hd(program.rules) + assert length(rule.body) == 2 + end + + test "error propagates through add_rule/4" do + result = + Program.new() + |> Program.add_relation("", [:atom]) + |> Program.add_rule( + {"ancestor", [:X, :Y]}, + [{:positive, {"parent", [:X, :Y]}}], + [{:neq, :X, :Y}] + ) + + assert {:error, msg} = result + assert msg =~ "non-empty string" + end + end end diff --git a/test/ex_datalog/telemetry_test.exs b/test/ex_datalog/telemetry_test.exs index 4154e75..8cf5249 100644 --- a/test/ex_datalog/telemetry_test.exs +++ b/test/ex_datalog/telemetry_test.exs @@ -4,16 +4,16 @@ defmodule ExDatalog.TelemetryTest do alias ExDatalog.{Atom, Program, Rule, Telemetry, Term} describe "event name functions" do - test "query_start/0 returns the correct event name" do - assert Telemetry.query_start() == [:ex_datalog, :query, :start] + test "materialize_start/0 returns the correct event name" do + assert Telemetry.materialize_start() == [:ex_datalog, :materialize, :start] end - test "query_stop/0 returns the correct event name" do - assert Telemetry.query_stop() == [:ex_datalog, :query, :stop] + test "materialize_stop/0 returns the correct event name" do + assert Telemetry.materialize_stop() == [:ex_datalog, :materialize, :stop] end - test "query_exception/0 returns the correct event name" do - assert Telemetry.query_exception() == [:ex_datalog, :query, :exception] + test "materialize_exception/0 returns the correct event name" do + assert Telemetry.materialize_exception() == [:ex_datalog, :materialize, :exception] end end @@ -32,7 +32,7 @@ defmodule ExDatalog.TelemetryTest do |> ExDatalog.compile() {measurements, metadata} = - capture_event(Telemetry.query_start(), fn -> + capture_event(Telemetry.materialize_start(), fn -> Telemetry.emit_start(ir) end) @@ -48,7 +48,7 @@ defmodule ExDatalog.TelemetryTest do relation_sizes = %{"edge" => 2} {measurements, metadata} = - capture_event(Telemetry.query_stop(), fn -> + capture_event(Telemetry.materialize_stop(), fn -> Telemetry.emit_stop(start_time, 3, relation_sizes, 1, :map) end) @@ -66,7 +66,7 @@ defmodule ExDatalog.TelemetryTest do stacktrace = [{:module, :function, 1, []}] {measurements, metadata} = - capture_event(Telemetry.query_exception(), fn -> + capture_event(Telemetry.materialize_exception(), fn -> Telemetry.emit_exception( start_time, :error, @@ -85,8 +85,8 @@ defmodule ExDatalog.TelemetryTest do end end - describe "integration: query emits telemetry events" do - test "successful query emits start and stop events" do + describe "integration: materialize emits telemetry events" do + test "successful materialize emits start and stop events" do program = Program.new() |> Program.add_relation("parent", [:atom, :atom]) @@ -100,10 +100,10 @@ defmodule ExDatalog.TelemetryTest do ) start_event = - capture_event(Telemetry.query_start(), fn -> + capture_event(Telemetry.materialize_start(), fn -> stop_event = - capture_event(Telemetry.query_stop(), fn -> - {:ok, _result} = ExDatalog.query(program) + capture_event(Telemetry.materialize_stop(), fn -> + {:ok, _knowledge} = ExDatalog.materialize(program) end) assert stop_event != nil @@ -137,9 +137,9 @@ defmodule ExDatalog.TelemetryTest do } start_event = - capture_event(Telemetry.query_start(), fn -> + capture_event(Telemetry.materialize_start(), fn -> stop_event = - capture_event(Telemetry.query_stop(), fn -> + capture_event(Telemetry.materialize_stop(), fn -> {:error, _} = ExDatalog.evaluate(ir) end) @@ -164,8 +164,8 @@ defmodule ExDatalog.TelemetryTest do ) start_event = - capture_event(Telemetry.query_start(), fn -> - {:error, _} = ExDatalog.query(program) + capture_event(Telemetry.materialize_start(), fn -> + {:error, _} = ExDatalog.materialize(program) end) assert start_event == nil @@ -173,15 +173,15 @@ defmodule ExDatalog.TelemetryTest do end describe "no overhead without handlers" do - test "query runs correctly when no telemetry handlers are attached" do + test "materialize runs correctly when no telemetry handlers are attached" do program = Program.new() |> Program.add_relation("edge", [:atom, :atom]) |> Program.add_fact("edge", [:a, :b]) |> Program.add_fact("edge", [:b, :c]) - {:ok, result} = ExDatalog.query(program) - assert MapSet.size(ExDatalog.Result.get(result, "edge")) == 2 + {:ok, knowledge} = ExDatalog.materialize(program) + assert MapSet.size(ExDatalog.Knowledge.get(knowledge, "edge")) == 2 end end diff --git a/test/ex_datalog/term_test.exs b/test/ex_datalog/term_test.exs index d1e34e0..bf1e4c7 100644 --- a/test/ex_datalog/term_test.exs +++ b/test/ex_datalog/term_test.exs @@ -166,4 +166,46 @@ defmodule ExDatalog.TermTest do assert Term.variables([]) == [] end end + + describe "from/1" do + test "converts uppercase atom to variable" do + assert Term.from(:A) == {:var, "A"} + end + + test "converts multi-char uppercase atom to variable" do + assert Term.from(:Pkg) == {:var, "Pkg"} + end + + test "converts underscore atom to wildcard" do + assert Term.from(:_) == :wildcard + end + + test "converts lowercase atom to constant" do + assert Term.from(:alice) == {:const, :alice} + end + + test "converts integer to constant" do + assert Term.from(42) == {:const, 42} + end + + test "converts string to constant" do + assert Term.from("hello") == {:const, "hello"} + end + + test "converts list to constant" do + assert Term.from([:a, :b]) == {:const, [:a, :b]} + end + + test "passes through existing var term" do + assert Term.from({:var, "X"}) == {:var, "X"} + end + + test "passes through existing const term" do + assert Term.from({:const, :alice}) == {:const, :alice} + end + + test "passes through existing wildcard" do + assert Term.from(:wildcard) == :wildcard + end + end end diff --git a/test/integration/engine_test.exs b/test/integration/engine_test.exs index a3fc5d1..532c12a 100644 --- a/test/integration/engine_test.exs +++ b/test/integration/engine_test.exs @@ -27,11 +27,11 @@ defmodule ExDatalog.IntegrationTest do ] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - ancestor = ExDatalog.Result.get(result, "ancestor") + ancestor = ExDatalog.Knowledge.get(result, "ancestor") assert MapSet.size(ancestor) == 6 assert {:alice, :bob} in ancestor @@ -55,10 +55,10 @@ defmodule ExDatalog.IntegrationTest do [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - path = ExDatalog.Result.get(result, "path") + path = ExDatalog.Knowledge.get(result, "path") assert MapSet.size(path) == 2 assert {:a, :b} in path assert {:b, :c} in path @@ -70,10 +70,10 @@ defmodule ExDatalog.IntegrationTest do |> Program.add_relation("person", [:atom]) |> Program.add_fact("person", [:alice]) |> Program.add_fact("person", [:bob]) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - person = ExDatalog.Result.get(result, "person") + person = ExDatalog.Knowledge.get(result, "person") assert MapSet.size(person) == 2 end @@ -95,10 +95,10 @@ defmodule ExDatalog.IntegrationTest do ] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - path3 = ExDatalog.Result.get(result, "path3") + path3 = ExDatalog.Knowledge.get(result, "path3") assert MapSet.size(path3) == 1 assert {:a, :b, :c} in path3 end @@ -120,10 +120,10 @@ defmodule ExDatalog.IntegrationTest do [Constraint.gt(Term.var("V"), Term.const(5))] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - big = ExDatalog.Result.get(result, "big_value") + big = ExDatalog.Knowledge.get(result, "big_value") assert MapSet.size(big) == 1 assert {:y, 10} in big end @@ -141,10 +141,10 @@ defmodule ExDatalog.IntegrationTest do [Constraint.add(Term.var("A"), Term.var("B"), Term.var("C"))] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - sums = ExDatalog.Result.get(result, "sum") + sums = ExDatalog.Knowledge.get(result, "sum") assert MapSet.size(sums) == 1 assert {3, 7, 10} in sums end @@ -164,16 +164,16 @@ defmodule ExDatalog.IntegrationTest do ] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - cycle = ExDatalog.Result.get(result, "cycle") + cycle = ExDatalog.Knowledge.get(result, "cycle") assert MapSet.size(cycle) == 1 assert {:a} in cycle end end - describe "end-to-end: Result API" do + describe "end-to-end: Knowledge API" do test "query with goal option" do result = Program.new() @@ -187,13 +187,13 @@ defmodule ExDatalog.IntegrationTest do [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - path = ExDatalog.Result.get(result, "path") + path = ExDatalog.Knowledge.get(result, "path") assert MapSet.size(path) == 2 - matched = ExDatalog.Result.match(result, "path", [:a, :_]) + matched = ExDatalog.Knowledge.match(result, "path", [:a, :_]) assert MapSet.size(matched) == 1 assert {:a, :b} in matched end @@ -241,10 +241,10 @@ defmodule ExDatalog.IntegrationTest do ] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - path = ExDatalog.Result.get(result, "path") + path = ExDatalog.Knowledge.get(result, "path") assert {:a, :b} in path assert {:b, :c} in path assert {:a, :c} in path @@ -273,10 +273,10 @@ defmodule ExDatalog.IntegrationTest do ] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - bachelors = ExDatalog.Result.get(result, "bachelor") + bachelors = ExDatalog.Knowledge.get(result, "bachelor") assert MapSet.size(bachelors) == 1 assert {:bob} in bachelors end @@ -299,10 +299,10 @@ defmodule ExDatalog.IntegrationTest do ] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - bachelors = ExDatalog.Result.get(result, "bachelor") + bachelors = ExDatalog.Knowledge.get(result, "bachelor") assert MapSet.size(bachelors) == 2 assert {:alice} in bachelors assert {:bob} in bachelors @@ -327,10 +327,10 @@ defmodule ExDatalog.IntegrationTest do ] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - filtered = ExDatalog.Result.get(result, "not_married_to_alice") + filtered = ExDatalog.Knowledge.get(result, "not_married_to_alice") assert MapSet.size(filtered) == 1 assert {:carol} in filtered end @@ -358,10 +358,10 @@ defmodule ExDatalog.IntegrationTest do ] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - verified = ExDatalog.Result.get(result, "verified") + verified = ExDatalog.Knowledge.get(result, "verified") assert MapSet.size(verified) == 1 assert {:alice} in verified end @@ -409,16 +409,16 @@ defmodule ExDatalog.IntegrationTest do ] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result - reachable = ExDatalog.Result.get(result, "reachable") + reachable = ExDatalog.Knowledge.get(result, "reachable") assert {:a, :b} in reachable assert {:b, :c} in reachable assert {:a, :c} in reachable - unreachable = ExDatalog.Result.get(result, "unreachable") + unreachable = ExDatalog.Knowledge.get(result, "unreachable") assert {:b, :a} in unreachable assert {:c, :a} in unreachable @@ -478,7 +478,7 @@ defmodule ExDatalog.IntegrationTest do [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] ) ) - |> ExDatalog.query() + |> ExDatalog.materialize() assert {:ok, result} = result assert result.provenance == nil @@ -496,7 +496,7 @@ defmodule ExDatalog.IntegrationTest do [{:positive, Atom.new("edge", [Term.var("X"), Term.var("Y")])}] ) ) - |> ExDatalog.query(explain: true) + |> ExDatalog.materialize(explain: true) assert result.provenance != nil assert result.provenance.fact_origins["edge"][{:a, :b}] == :base @@ -527,7 +527,7 @@ defmodule ExDatalog.IntegrationTest do ] ) ) - |> ExDatalog.query(explain: true) + |> ExDatalog.materialize(explain: true) assert {:ok, tree} = Explain.explain(result, "bachelor", {:bob}) assert %Explain.Node{} = tree