diff --git a/CHANGELOG.md b/CHANGELOG.md index ad46d6f..ce37e97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## Unreleased + +### Bug Fixes and Improvements + + * Count Unicode code points for `minLength` and `maxLength` without allocating a charlist for inputs larger than 256 bytes; retain standard conversion for inputs up to 256 bytes. Malformed UTF-8 now returns a validation error with `error_detail: "invalid_utf8"` instead of raising `UnicodeConversionError`. + * Compile each `patternProperties` regex once when the same schema also uses `additionalProperties`, with a bounded per-schema cache that preserves vocabulary gates, error behavior, and validation semantics. + * Reuse runtime pattern-match classification between sibling `patternProperties` and `additionalProperties` rules within one validation invocation, reducing repeated regex matching for wide objects while preserving rule order, annotations, and diagnostics. + * Adaptively merge evaluated keys from parent-dominated `anyOf` branches with balanced `MapSet` unions, avoiding repeated conversion of shared incoming evaluated keys while preserving full branch traversal, evaluated-key inputs, diagnostics, and legacy list-key behavior. + * Avoid rebuilding already-flat validation error lists at the public validation boundary, and fuse nested error flattening with legacy error formatting while preserving diagnostic order, values, and internal error grouping. + * Remove redundant input-key presence checks when validating `required`, while preserving declaration order, duplicate entries, non-object behavior, and diagnostic shape. + * Avoid rescanning built-in Draft 2020-12 resources after compilation, and reduce scope-scanner traversal intermediates while preserving resource identities, anchor targets, and traversal precedence. + ## v0.9.3 (2026-09-07) ### Bug Fixes and Improvements diff --git a/bench/compiler_regex_cache.exs b/bench/compiler_regex_cache.exs new file mode 100644 index 0000000..8b5df91 --- /dev/null +++ b/bench/compiler_regex_cache.exs @@ -0,0 +1,114 @@ +# Run from the repository root: +# mix run bench/compiler_regex_cache.exs +# mix run bench/compiler_regex_cache.exs --baseline d3864cf +# +# Loads only the selected compiler source in memory. Input construction, source +# loading, warmup, and explicit GC are outside the timed intervals. + +root = Path.expand("..", __DIR__) +compiler_path = Path.join(root, "lib/jsonschex/compiler.ex") + +{source, source_label} = + case System.argv() do + [] -> + {File.read!(compiler_path), "working tree (snapshot at script startup)"} + + ["--baseline", revision] -> + case System.cmd("git", ["--no-pager", "show", "#{revision}:lib/jsonschex/compiler.ex"], cd: root) do + {source, 0} -> {source, "git #{revision}:lib/jsonschex/compiler.ex"} + {output, status} -> raise "Cannot load baseline (exit #{status}): #{output}" + end + + _ -> + raise "Usage: mix run bench/compiler_regex_cache.exs [--baseline REVISION]" + end + +previous_options = Code.compiler_options(ignore_module_conflict: true) + +try do + Code.compile_string(source, compiler_path) +after + Code.compiler_options(previous_options) +end + +fingerprint = :crypto.hash(:sha256, source) |> Base.encode16(case: :lower) + +IO.puts("Compiler: #{source_label}; sha256=#{fingerprint}") +IO.puts("Elixir #{System.version()}, OTP #{System.otp_release()}, schedulers #{System.schedulers_online()}") +IO.puts("Median ms/op; five adaptive batches targeting 20 ms; construction, source loading, warmup and GC excluded") + +# Patterns use both ECMA property and whitespace transformations, so the sibling +# pass represents real regex preparation rather than only trivial Regex.compile/2 work. +defmodule CompilerRegexCacheBench do + alias JSONSchex.Types.Schema + + @widths [0, 1, 16, 32, 33, 128, 512] + + def cases do + Enum.flat_map(@widths, fn width -> + [ + {"patternProperties only width=#{width}", schema(width, :none)}, + {"patternProperties + additionalProperties=false width=#{width}", schema(width, false)}, + {"patternProperties + additionalProperties=true width=#{width}", schema(width, true)} + ] + end) + end + + def measure({label, schema}) do + compile = fn -> + {:ok, %Schema{}} = JSONSchex.compile(schema) + :ok + end + + compile.() + :erlang.garbage_collect() + {calibration_us, :ok} = :timer.tc(compile) + count = min(10_000, max(1, div(20_000, max(1, calibration_us)))) + + samples = + for _ <- 1..5 do + :erlang.garbage_collect() + + {elapsed_us, :ok} = + :timer.tc(fn -> + Enum.each(1..count, fn _ -> compile.() end) + end) + + elapsed_us / count / 1_000 + end + + median = samples |> Enum.sort() |> Enum.at(2) + + IO.puts( + "#{label}: median_ms=#{Float.round(median, 4)} " <> + "(count=#{count}; min_ms=#{Float.round(Enum.min(samples), 4)} " <> + "max_ms=#{Float.round(Enum.max(samples), 4)})" + ) + end + + defp schema(width, additional) do + patterns = pattern_map(width) + schema = %{"patternProperties" => patterns} + + case additional do + :none -> schema + value -> Map.put(schema, "additionalProperties", value) + end + end + + defp pattern_map(0), do: %{} + + defp pattern_map(width) do + Map.new(1..width, fn index -> + {"^\\p{Letter}" <> Integer.to_string(index) <> "\\s*$", true} + end) + end +end + +# A hard timeout prevents reporting an incomplete comparison as successful. +task = Task.async(fn -> Enum.each(CompilerRegexCacheBench.cases(), &CompilerRegexCacheBench.measure/1) end) + +case Task.yield(task, 50_000) || Task.shutdown(task, :brutal_kill) do + {:ok, :ok} -> :ok + _ -> raise "Benchmark exceeded the 50 s measurement budget; results are incomplete" +end diff --git a/bench/scope_scanner_traversal.exs b/bench/scope_scanner_traversal.exs new file mode 100644 index 0000000..0583836 --- /dev/null +++ b/bench/scope_scanner_traversal.exs @@ -0,0 +1,299 @@ +# Run from the repository root: +# mix run bench/scope_scanner_traversal.exs +# mix run bench/scope_scanner_traversal.exs --baseline 8bef311 +# +# Compares the selected production ScopeScanner traversal with a benchmark-only +# callback reducer. Inputs, source loading, result verification, warmup, and +# explicit garbage collection are outside timed intervals. Reclaimed words use +# VM-wide GC counters, so run this script in a dedicated VM. + +root = Path.expand("..", __DIR__) +source_paths = ["lib/jsonschex/schema_traversal.ex", "lib/jsonschex/scope_scanner.ex"] + +{sources, source_label} = + case System.argv() do + [] -> + { + Enum.map(source_paths, fn path -> {path, File.read!(Path.join(root, path))} end), + "working tree (snapshot at script startup)" + } + + ["--baseline", revision] -> + baseline_sources = + Enum.map(source_paths, fn path -> + case System.cmd("git", ["--no-pager", "show", "#{revision}:#{path}"], cd: root) do + {source, 0} -> {path, source} + {output, status} -> raise "Cannot load #{path} (exit #{status}): #{output}" + end + end) + + {baseline_sources, "git #{revision}"} + + _ -> + raise "Usage: mix run bench/scope_scanner_traversal.exs [--baseline REVISION]" + end + +previous_options = Code.compiler_options(ignore_module_conflict: true) + +try do + Enum.each(sources, fn {path, source} -> + Code.compile_string(source, Path.join(root, path)) + end) +after + Code.compiler_options(previous_options) +end + +fingerprint = + sources + |> Enum.map_join(fn {path, source} -> path <> <<0>> <> source end) + |> then(&:crypto.hash(:sha256, &1)) + |> Base.encode16(case: :lower) + +IO.puts("Scanner sources: #{source_label}; combined_sha256=#{fingerprint}") +IO.puts("Elixir #{System.version()}, OTP #{System.otp_release()}, ERTS #{:erlang.system_info(:version)}") +IO.puts("schedulers #{System.schedulers_online()}/#{System.schedulers()}; MIX_ENV=#{Mix.env()}") +IO.puts("Median ms/op, reductions/op, and reclaimed words/op; five adaptive batches targeting 20 ms") + +defmodule ScopeScannerReducerPrototype do + alias JSONSchex.URIUtil + + @single_schema_keywords [ + "additionalProperties", + "contains", + "contentSchema", + "items", + "not", + "propertyNames", + "unevaluatedItems", + "unevaluatedProperties", + "if", + "then", + "else" + ] + @schema_map_keywords [ + "dependentSchemas", + "patternProperties", + "properties", + "$defs", + "definitions" + ] + @schema_list_keywords ["allOf", "anyOf", "oneOf", "prefixItems"] + + def scan(schema), do: do_scan(schema, nil, {%{}, MapSet.new()}) + + defp do_scan(schema, base_uri, {registry, refs}) when is_map(schema) do + current_id = Map.get(schema, "$id") + new_base_uri = URIUtil.resolve(base_uri, current_id) || "" + + registry = + if current_id do + Map.put(registry, new_base_uri, schema) + else + registry + end + + registry = + registry + |> register_anchor(schema, "$anchor", new_base_uri) + |> register_anchor(schema, "$dynamicAnchor", new_base_uri) + + refs = + Enum.reduce(["$ref", "$dynamicRef"], refs, fn key, acc -> + case Map.get(schema, key) do + "#" <> _ = value -> MapSet.put(acc, value) + _ -> acc + end + end) + + reduce_scope_subschemas(schema, {registry, refs}, fn subschema, acc -> + do_scan(subschema, new_base_uri, acc) + end) + end + + defp do_scan(_value, _base_uri, acc), do: acc + + defp reduce_scope_subschemas(schema, acc, reducer) do + acc = + Enum.reduce(@schema_list_keywords, acc, fn keyword, inner_acc -> + reduce_schema_list(Map.get(schema, keyword), inner_acc, reducer) + end) + + acc = + Enum.reduce(@schema_map_keywords, acc, fn keyword, inner_acc -> + reduce_schema_map(Map.get(schema, keyword), inner_acc, reducer) + end) + + acc = + Enum.reduce(@single_schema_keywords, acc, fn keyword, inner_acc -> + reduce_schema(Map.get(schema, keyword), inner_acc, reducer) + end) + + acc = reduce_schema_map(Map.get(schema, "dependencies"), acc, reducer) + + case Map.get(schema, "items") do + items when is_list(items) -> reduce_schema_list(items, acc, reducer) + _ -> acc + end + end + + defp reduce_schema_map(value, acc, reducer) when is_map(value) do + value + |> Map.values() + |> Enum.reduce(acc, fn subschema, inner_acc -> + reduce_schema(subschema, inner_acc, reducer) + end) + end + + defp reduce_schema_map(_value, acc, _reducer), do: acc + + defp reduce_schema_list(value, acc, reducer) when is_list(value) do + Enum.reduce(value, acc, fn subschema, inner_acc -> + reduce_schema(subschema, inner_acc, reducer) + end) + end + + defp reduce_schema_list(_value, acc, _reducer), do: acc + + defp reduce_schema(subschema, acc, reducer) when is_map(subschema) or is_boolean(subschema), + do: reducer.(subschema, acc) + + defp reduce_schema(_value, acc, _reducer), do: acc + + defp register_anchor(acc, schema, keyword, base_uri) do + case Map.get(schema, keyword) do + anchor when is_binary(anchor) -> Map.put(acc, base_uri <> "#" <> anchor, schema) + _ -> acc + end + end +end + +defmodule ScopeScannerTraversalBench do + alias JSONSchex.Draft202012.Schemas + alias JSONSchex.ScopeScanner + + @base_uri "https://json-schema.org/draft/2020-12" + @family_uris [ + @base_uri <> "/schema", + @base_uri <> "/meta/core", + @base_uri <> "/meta/applicator", + @base_uri <> "/meta/unevaluated", + @base_uri <> "/meta/validation", + @base_uri <> "/meta/meta-data", + @base_uri <> "/meta/format-annotation", + @base_uri <> "/meta/format-assertion", + @base_uri <> "/meta/content" + ] + + def cases do + family = Enum.map(@family_uris, fn uri -> {:ok, schema} = Schemas.fetch(uri); schema end) + + [ + {"built-in family (9 resources)", family, &scan_family/2}, + {"wide properties (1024 schemas)", wide_schema(1_024), &scan_one/2}, + {"nested properties (depth 1024)", nested_schema(1_024), &scan_one/2}, + {"inactive extension values (4096 entries)", inactive_values_schema(4_096), &scan_one/2} + ] + end + + def measure({label, input, runner}) do + current = fn -> runner.(input, &ScopeScanner.scan/1) end + reducer = fn -> runner.(input, &ScopeScannerReducerPrototype.scan/1) end + + expected = current.() + unless reducer.() === expected, do: raise("prototype changed scanner output for #{label}") + + current.() + reducer.() + {calibration_us, ^expected} = :timer.tc(current) + count = min(1_000, max(1, div(20_000, max(1, calibration_us)))) + + current_stats = measure_implementation(current, count) + reducer_stats = measure_implementation(reducer, count) + + IO.puts(label) + print_stats(" production", current_stats, count) + print_stats(" reducer prototype", reducer_stats, count) + end + + def scan_family(schemas, scanner), do: Enum.map(schemas, scanner) + def scan_one(schema, scanner), do: scanner.(schema) + + defp measure_implementation(operation, count) do + times = + for _ <- 1..5 do + :erlang.garbage_collect() + + {elapsed_us, :ok} = + :timer.tc(fn -> + Enum.each(1..count, fn _ -> operation.() end) + end) + + elapsed_us / count / 1000 + end + + :erlang.garbage_collect() + {:reductions, before_reductions} = Process.info(self(), :reductions) + Enum.each(1..count, fn _ -> operation.() end) + {:reductions, after_reductions} = Process.info(self(), :reductions) + reductions_per_op = (after_reductions - before_reductions) / count + + :erlang.garbage_collect() + {_collections, before_reclaimed, _} = :erlang.statistics(:garbage_collection) + Enum.each(1..count, fn _ -> operation.() end) + :erlang.garbage_collect() + {_collections, after_reclaimed, _} = :erlang.statistics(:garbage_collection) + reclaimed_words_per_op = (after_reclaimed - before_reclaimed) / count + + {Enum.sort(times), reductions_per_op, reclaimed_words_per_op} + end + + defp print_stats(name, {times, reductions, reclaimed_words}, count) do + IO.puts( + "#{name}: median_ms=#{Float.round(Enum.at(times, 2), 4)} " <> + "min_ms=#{Float.round(hd(times), 4)} max_ms=#{Float.round(List.last(times), 4)} " <> + "reductions=#{Float.round(reductions, 1)} " <> + "reclaimed_words=#{Float.round(reclaimed_words, 1)} count=#{count}" + ) + end + + defp wide_schema(width) do + %{ + "$id" => "https://scanner.example/wide/root", + "properties" => + Map.new(1..width, fn index -> + {"property-#{index}", + %{ + "$id" => "child-#{index}", + "$anchor" => "anchor-#{index}", + "$ref" => "#local" + }} + end) + } + end + + defp nested_schema(depth) do + Enum.reduce(1..depth, %{"type" => "integer"}, fn index, child -> + %{ + "$id" => "level-#{index}/", + "$dynamicAnchor" => "node-#{index}", + "properties" => %{"child" => child} + } + end) + end + + defp inactive_values_schema(width) do + extensions = + Map.new(1..width, fn index -> + {"x-literal-#{index}", %{"$id" => "inactive-#{index}", "$anchor" => "ignored"}} + end) + + Map.put(extensions, "$id", "https://scanner.example/inactive/root") + end +end + +task = Task.async(fn -> Enum.each(ScopeScannerTraversalBench.cases(), &ScopeScannerTraversalBench.measure/1) end) + +case Task.yield(task, 60_000) || Task.shutdown(task, :brutal_kill) do + {:ok, :ok} -> :ok + _ -> raise "Benchmark exceeded the 60 s measurement budget; results are incomplete" +end diff --git a/bench/string_length.exs b/bench/string_length.exs new file mode 100644 index 0000000..31ae656 --- /dev/null +++ b/bench/string_length.exs @@ -0,0 +1,91 @@ +# Run from the repository root: mix run bench/string_length.exs +# Baseline: mix run bench/string_length.exs --baseline 7d103dc +# Only predicate source is pinned for a baseline; all other modules use the checkout. +root = Path.expand("..", __DIR__) +predicate_path = Path.join(root, "lib/jsonschex/compiler/predicates.ex") + +label = case System.argv() do + [] -> + "working tree" + + ["--baseline", revision] -> + {source, 0} = System.cmd("git", ["--no-pager", "show", "#{revision}:lib/jsonschex/compiler/predicates.ex"], cd: root) + previous_options = Code.compiler_options(ignore_module_conflict: true) + try do + Code.compile_string(source, predicate_path) + after + Code.compiler_options(previous_options) + end + "baseline #{revision}" + + _ -> + raise "Usage: mix run bench/string_length.exs [--baseline REVISION]" +end + +IO.puts("Predicates: #{label}; Elixir #{System.version()}, OTP #{System.otp_release()}, schedulers #{System.schedulers_online()}") +IO.puts("Median us/op; five adaptive batches targeting 20 ms (1..100000 calls); construction/compilation/GC excluded") + +defmodule StringLengthBench do + def measure({label, run}) do + run.() + :erlang.garbage_collect() + {calibration, _} = :timer.tc(fn -> Enum.each(1..10, fn _ -> run.() end) end) + iterations = min(100_000, max(1, div(200_000, max(1, calibration)))) + samples = for _ <- 1..5 do + :erlang.garbage_collect() + {us, _} = :timer.tc(fn -> Enum.each(1..iterations, fn _ -> run.() end) end) + us / iterations + end + median = samples |> Enum.sort() |> Enum.at(2) + IO.puts("#{label}: #{Float.round(median, 3)} us/op") + end +end + +# Include all encoding widths and multi-codepoint graphemes. Each unit's length +# is measured outside timing; validators always receive actual codepoint limits. +inputs = for {name, unit} <- [{"ASCII", "a"}, {"2-byte", "é"}, {"3-byte", "漢"}, {"4-byte", "😀"}, {"combining", "e\u0301"}], + size <- [16, 1000, 100_000] do + data = String.duplicate(unit, size) + count = length(String.to_charlist(unit)) * size + {:ok, valid_schema} = JSONSchex.compile(%{"minLength" => count, "maxLength" => count}) + {:ok, invalid_schema} = JSONSchex.compile(%{"minLength" => count + 1, "maxLength" => 0}) + {"#{name} codepoints=#{count}", data, count, valid_schema, invalid_schema} +end + +jobs = Enum.flat_map(inputs, fn {label, data, count, valid, invalid} -> + [ + {"#{label} predicate min pass", fn -> + :ok = JSONSchex.Compiler.Predicates.check_min_length(data, count) + end}, + {"#{label} predicate max pass", fn -> + :ok = JSONSchex.Compiler.Predicates.check_max_length(data, count) + end}, + {"#{label} validate both pass", fn -> + :ok = JSONSchex.validate(valid, data) + end}, + {"#{label} validate both fail", fn -> + {:error, [_, _]} = JSONSchex.validate(invalid, data) + end} + ] +end) + +# Old revisions raise on malformed UTF-8, so compare valid inputs across versions +# and measure the new structured-error path only on the current implementation. +invalid = String.duplicate("a", 10_000) <> <<0xFF>> +jobs = if System.argv() == [] do + {:ok, schema} = JSONSchex.compile(%{"minLength" => 0, "maxLength" => byte_size(invalid)}) + jobs ++ [{"malformed UTF-8 after 10000 ASCII codepoints (validate both errors)", fn -> + {:error, [first, second]} = JSONSchex.validate(schema, invalid) + "invalid_utf8" = first.context.error_detail + "invalid_utf8" = second.context.error_detail + end}] +else + IO.puts("Skipping malformed-input job: baseline versions use a different error contract") + jobs +end + +task = Task.async(fn -> Enum.each(jobs, &StringLengthBench.measure/1) end) +case Task.yield(task, 50_000) || Task.shutdown(task, :brutal_kill) do + {:ok, :ok} -> :ok + _ -> raise "String-length benchmark exceeded its 50 s measurement budget" +end diff --git a/bench/string_length_cutoff_explore.exs b/bench/string_length_cutoff_explore.exs new file mode 100644 index 0000000..64c3ec6 --- /dev/null +++ b/bench/string_length_cutoff_explore.exs @@ -0,0 +1,713 @@ +# Run from the repository root: +# mix run bench/string_length_cutoff_explore.exs --quick +# mix run bench/string_length_cutoff_explore.exs --extended +# +# Optional modes: +# mix run bench/string_length_cutoff_explore.exs --focus +# mix run bench/string_length_cutoff_explore.exs --all-raw +# +# --quick retains the exhaustive 0..256-byte generated scan with a bounded +# sampling configuration. --extended uses a sparse grid through 16 KiB and +# emits compact direct-counter results suitable for cutoff decisions. +# +# This explores the cutoff only. It deliberately duplicates the two existing +# counting implementations so neither benchmark path is selected by byte size. + +defmodule StringLengthCutoffExplore do + alias JSONSchex.Compiler.Predicates + + @production_cutoff 256 + @full_sizes Enum.to_list(0..256) + @focus_sizes [ + 0, + 1, + 16, + 30, + 31, + 32, + 33, + 46, + 47, + 48, + 49, + 62, + 63, + 64, + 65, + 66, + 79, + 80, + 81, + 95, + 96, + 97, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 255, + 256, + 257 + ] + @extended_sizes [ + 0, + 16, + 32, + 48, + 64, + 96, + 128, + 256, + 384, + 512, + 768, + 1_024, + 1_536, + 2_048, + 4_096, + 8_192, + 16_384 + ] + @families [ + {:ascii, "ASCII", "a"}, + {:two_byte, "2-byte", "é"}, + {:three_byte, "3-byte", "漢"}, + {:four_byte, "4-byte", "😀"}, + {:mixed, "mixed", "aé漢😀"} + ] + @quick_config %{ + calibration_iterations: 128, + warmup_iterations: 64, + sample_count: 3, + target_microseconds: 2_000, + max_iterations: 100_000 + } + @focus_config %{ + calibration_iterations: 256, + warmup_iterations: 128, + sample_count: 5, + target_microseconds: 5_000, + max_iterations: 100_000 + } + @extended_config %{ + calibration_iterations: 64, + warmup_iterations: 16, + sample_count: 5, + target_microseconds: 5_000, + max_iterations: 20_000 + } + @benchmark_timeout_ms 55_000 + + def main(argv) do + options = parse_args(argv) + config = options.config + cases = build_cases(options.sizes) + {valid_count, malformed_count} = validate_cases!(cases) + + IO.puts( + "String-length cutoff exploration (#{options.mode}); Elixir #{System.version()}, " <> + "OTP #{System.otp_release()}, schedulers #{System.schedulers_online()}" + ) + + IO.puts( + "Validated #{valid_count} valid exact-byte inputs and #{malformed_count} malformed inputs " <> + "against String.valid?/1 before timing." + ) + + IO.puts( + "Timing: median of #{config.sample_count} samples; each sample targets " <> + "#{config.target_microseconds} us; #{config.calibration_iterations} direct-call calibration " <> + "iterations; GC before each sample." + ) + + IO.puts( + "The timed paths are compiled tail-recursive loops invoked through :timer.tc/3; " <> + "no anonymous function runs once per counted string." + ) + + results = bounded_benchmark(cases, options) + + case options.mode do + :extended -> + print_extended_rows(results) + print_family_crossovers(results, options.sizes) + print_extended_cutoff_scores(results) + print_allocation_context() + + _ -> + if options.print_focus? do + print_focus_rows(results) + end + + if options.print_all? do + print_all_rows(results) + end + + print_family_crossovers(results, options.sizes) + + if options.sizes == @full_sizes do + print_cutoff_scores(results) + else + IO.puts("\nCutoff score table skipped: this mode does not contain every byte size from 0 through 256.") + end + end + end + + # This is the former short-string implementation, including its malformed + # UTF-8 result contract. + def original_charlist_count(binary) do + binary |> String.to_charlist() |> length() + rescue + UnicodeConversionError -> :error + end + + # This is the current long-string implementation, intentionally independent + # of the production cutoff so it can be timed at every input size. + def current_unrolled_count(binary), do: count_codepoints(binary, 0) + + def run_charlist(binary, iterations), do: repeat_charlist(binary, iterations, 0) + def run_unrolled(binary, iterations), do: repeat_unrolled(binary, iterations, 0) + def run_public_min(binary, minimum, iterations), do: repeat_public_min(binary, minimum, iterations, 0) + def run_benchmark(cases, config, public_sizes), do: benchmark(cases, config, public_sizes) + + defp bounded_benchmark(cases, options) do + task = Task.async(__MODULE__, :run_benchmark, [cases, options.config, options.public_sizes]) + + case Task.yield(task, @benchmark_timeout_ms) || Task.shutdown(task, :brutal_kill) do + {:ok, results} -> results + _ -> raise "Benchmark exceeded the #{@benchmark_timeout_ms} ms measurement budget" + end + end + + defp parse_args([]), do: quick_options(false) + defp parse_args(["--quick"]), do: quick_options(false) + defp parse_args(["--all-raw"]), do: quick_options(true) + + defp parse_args(["--focus"]) do + %{ + mode: :focus, + sizes: @focus_sizes, + config: @focus_config, + public_sizes: @focus_sizes, + print_focus?: true, + print_all?: false + } + end + + defp parse_args(["--extended"]) do + %{ + mode: :extended, + sizes: @extended_sizes, + config: @extended_config, + public_sizes: [], + print_focus?: false, + print_all?: false + } + end + + defp parse_args(_) do + raise "Usage: mix run bench/string_length_cutoff_explore.exs [--quick | --extended | --focus | --all-raw]" + end + + defp quick_options(print_all?) do + %{ + mode: :quick, + sizes: @full_sizes, + config: @quick_config, + public_sizes: if(print_all?, do: @focus_sizes, else: []), + print_focus?: print_all?, + print_all?: print_all? + } + end + + defp build_cases(sizes) do + for bytes <- sizes, {family, label, unit} <- @families do + data = exact_bytes(unit, bytes) + + %{ + family: family, + label: label, + bytes: bytes, + data: data, + codepoints: length(String.to_charlist(data)) + } + end + end + + # Exact arbitrary byte sizes require ASCII padding whenever a requested size + # is not divisible by the selected multibyte unit's encoded width. + defp exact_bytes(unit, bytes) do + whole_units = div(bytes, byte_size(unit)) + padding_bytes = rem(bytes, byte_size(unit)) + String.duplicate(unit, whole_units) <> String.duplicate("a", padding_bytes) + end + + defp validate_cases!(cases) do + Enum.each(cases, &validate_valid_case!/1) + + malformed = malformed_cases() + Enum.each(malformed, &validate_malformed_case!/1) + + {length(cases), length(malformed)} + end + + defp validate_valid_case!(%{label: label, bytes: bytes, data: data, codepoints: expected}) do + unless byte_size(data) == bytes do + raise "#{label} input has #{byte_size(data)} bytes, expected #{bytes}" + end + + unless String.valid?(data) do + raise "#{label} input at #{bytes} bytes is not valid UTF-8" + end + + charlist_count = original_charlist_count(data) + unrolled_count = current_unrolled_count(data) + + unless charlist_count == expected and unrolled_count == expected do + raise( + "#{label} input at #{bytes} bytes disagrees with String.to_charlist/1: " <> + "charlist=#{inspect(charlist_count)} unrolled=#{inspect(unrolled_count)} expected=#{expected}" + ) + end + + case Predicates.check_min_length(data, expected) do + :ok -> :ok + result -> raise "public check_min_length/2 rejected valid #{label} input: #{inspect(result)}" + end + end + + defp validate_malformed_case!({label, data}) do + if String.valid?(data) do + raise "#{label} unexpectedly passes String.valid?/1" + end + + charlist_count = original_charlist_count(data) + unrolled_count = current_unrolled_count(data) + + unless charlist_count == :error and unrolled_count == :error do + raise( + "#{label} malformed result mismatch: " <> + "charlist=#{inspect(charlist_count)} unrolled=#{inspect(unrolled_count)}" + ) + end + + case Predicates.check_min_length(data, 0) do + {:error, %{contrast: 0, input: ^data, error_detail: "invalid_utf8"}} -> :ok + result -> raise "#{label} public malformed result mismatch: #{inspect(result)}" + end + end + + defp malformed_cases do + [ + {"lone continuation", <<0x80>>}, + {"invalid leader", <<0xFF>>}, + {"overlong sequence", <<0xC0, 0x80>>}, + {"bad continuation", <<0xE2, 0x82, 0x41>>}, + {"surrogate encoding", <<0xED, 0xA0, 0x80>>}, + {"out-of-range scalar", <<0xF4, 0x90, 0x80, 0x80>>}, + {"truncated scalar", <<0xF0, 0x90, 0x80>>}, + {"255-byte malformed tail", String.duplicate("a", 254) <> <<0xFF>>}, + {"256-byte malformed tail", String.duplicate("a", 255) <> <<0xFF>>}, + {"257-byte malformed tail", String.duplicate("a", 256) <> <<0xFF>>}, + {"long mixed malformed tail", String.duplicate("aé漢😀", 7) <> <<0xFF>>} + ] + end + + defp benchmark(cases, config, public_sizes) do + for input <- cases do + benchmark_case(input, config, public_sizes) + end + end + + defp benchmark_case(%{bytes: bytes, data: data, codepoints: codepoints} = input, config, public_sizes) do + {charlist, unrolled} = + if rem(bytes, 2) == 0 do + {measure_charlist(data, codepoints, config), measure_unrolled(data, codepoints, config)} + else + unrolled = measure_unrolled(data, codepoints, config) + charlist = measure_charlist(data, codepoints, config) + {charlist, unrolled} + end + + public = + if bytes in public_sizes do + measure_public_min(data, codepoints, config) + else + nil + end + + Map.merge(input, %{charlist: charlist, unrolled: unrolled, public: public}) + end + + defp measure_charlist(binary, expected, config) do + warmup_iterations = config.warmup_iterations + expected_total = expected * warmup_iterations + + unless run_charlist(binary, warmup_iterations) == expected_total do + raise "charlist warmup did not preserve the expected count" + end + + :erlang.garbage_collect() + calibration_us = time_charlist(binary, expected, config.calibration_iterations) + iterations = iterations_for(calibration_us, config) + samples = charlist_samples(binary, expected, iterations, config.sample_count, []) + measurement(samples, iterations, config.sample_count) + end + + defp measure_unrolled(binary, expected, config) do + warmup_iterations = config.warmup_iterations + expected_total = expected * warmup_iterations + + unless run_unrolled(binary, warmup_iterations) == expected_total do + raise "unrolled warmup did not preserve the expected count" + end + + :erlang.garbage_collect() + calibration_us = time_unrolled(binary, expected, config.calibration_iterations) + iterations = iterations_for(calibration_us, config) + samples = unrolled_samples(binary, expected, iterations, config.sample_count, []) + measurement(samples, iterations, config.sample_count) + end + + defp measure_public_min(binary, minimum, config) do + warmup_iterations = config.warmup_iterations + + unless run_public_min(binary, minimum, warmup_iterations) == warmup_iterations do + raise "public predicate warmup did not preserve the expected result" + end + + :erlang.garbage_collect() + calibration_us = time_public_min(binary, minimum, config.calibration_iterations) + iterations = iterations_for(calibration_us, config) + samples = public_min_samples(binary, minimum, iterations, config.sample_count, []) + measurement(samples, iterations, config.sample_count) + end + + defp iterations_for(calibration_us, config) do + estimated = div(config.target_microseconds * config.calibration_iterations, max(calibration_us, 1)) + min(max(estimated, 1), config.max_iterations) + end + + defp charlist_samples(_binary, _expected, _iterations, 0, samples), do: Enum.reverse(samples) + + defp charlist_samples(binary, expected, iterations, remaining, samples) do + :erlang.garbage_collect() + microseconds = time_charlist(binary, expected, iterations) + sample = nanoseconds_per_operation(microseconds, iterations) + charlist_samples(binary, expected, iterations, remaining - 1, [sample | samples]) + end + + defp unrolled_samples(_binary, _expected, _iterations, 0, samples), do: Enum.reverse(samples) + + defp unrolled_samples(binary, expected, iterations, remaining, samples) do + :erlang.garbage_collect() + microseconds = time_unrolled(binary, expected, iterations) + sample = nanoseconds_per_operation(microseconds, iterations) + unrolled_samples(binary, expected, iterations, remaining - 1, [sample | samples]) + end + + defp public_min_samples(_binary, _minimum, _iterations, 0, samples), do: Enum.reverse(samples) + + defp public_min_samples(binary, minimum, iterations, remaining, samples) do + :erlang.garbage_collect() + microseconds = time_public_min(binary, minimum, iterations) + sample = nanoseconds_per_operation(microseconds, iterations) + public_min_samples(binary, minimum, iterations, remaining - 1, [sample | samples]) + end + + defp time_charlist(binary, expected, iterations) do + {microseconds, total} = :timer.tc(__MODULE__, :run_charlist, [binary, iterations]) + expected_total = expected * iterations + + unless total == expected_total do + raise "charlist timed loop returned #{inspect(total)}, expected #{expected_total}" + end + + microseconds + end + + defp time_unrolled(binary, expected, iterations) do + {microseconds, total} = :timer.tc(__MODULE__, :run_unrolled, [binary, iterations]) + expected_total = expected * iterations + + unless total == expected_total do + raise "unrolled timed loop returned #{inspect(total)}, expected #{expected_total}" + end + + microseconds + end + + defp time_public_min(binary, minimum, iterations) do + {microseconds, total} = :timer.tc(__MODULE__, :run_public_min, [binary, minimum, iterations]) + + unless total == iterations do + raise "public timed loop returned #{inspect(total)}, expected #{iterations}" + end + + microseconds + end + + defp nanoseconds_per_operation(microseconds, iterations), do: microseconds * 1_000.0 / iterations + + defp measurement(samples, iterations, sample_count) do + sorted = Enum.sort(samples) + + %{ + samples: samples, + median: Enum.at(sorted, div(sample_count, 2)), + iterations: iterations + } + end + + defp repeat_charlist(_binary, 0, total), do: total + + defp repeat_charlist(binary, remaining, total) do + count = original_charlist_count(binary) + repeat_charlist(binary, remaining - 1, total + count) + end + + defp repeat_unrolled(_binary, 0, total), do: total + + defp repeat_unrolled(binary, remaining, total) do + count = current_unrolled_count(binary) + repeat_unrolled(binary, remaining - 1, total + count) + end + + defp repeat_public_min(_binary, _minimum, 0, total), do: total + + defp repeat_public_min(binary, minimum, remaining, total) do + case Predicates.check_min_length(binary, minimum) do + :ok -> repeat_public_min(binary, minimum, remaining - 1, total + 1) + result -> raise "public predicate unexpectedly returned #{inspect(result)}" + end + end + + # Match the production long-input counter exactly: four decoded codepoints per + # recursive step, then a one-codepoint tail, an empty binary, or an error. + defp count_codepoints(<<_::utf8, _::utf8, _::utf8, _::utf8, rest::binary>>, count), + do: count_codepoints(rest, count + 4) + + defp count_codepoints(<<_codepoint::utf8, rest::binary>>, count), + do: count_codepoints(rest, count + 1) + + defp count_codepoints(<<>>, count), do: count + defp count_codepoints(_invalid, _count), do: :error + + defp print_focus_rows(results) do + IO.puts("\nFocused raw measurements (ns/op; samples are raw timed-loop samples)") + IO.puts( + "family,bytes,codepoints,charlist median [samples] n,unrolled median [samples] n," <> + "public check_min_length median [samples] n,winner" + ) + + results + |> Enum.filter(&(&1.bytes in @focus_sizes)) + |> Enum.each(fn result -> + IO.puts( + "#{result.label},#{result.bytes},#{result.codepoints}," <> + "#{measurement_text(result.charlist)},#{measurement_text(result.unrolled)}," <> + "#{measurement_text(result.public)},#{winner(result)}" + ) + end) + end + + defp print_all_rows(results) do + IO.puts("\nAll scan medians (ns/op; use the focused rows above for raw samples)") + IO.puts("family,bytes,codepoints,charlist median,unrolled median,winner") + + Enum.each(results, fn result -> + IO.puts( + "#{result.label},#{result.bytes},#{result.codepoints}," <> + "#{format_ns(result.charlist.median)},#{format_ns(result.unrolled.median)},#{winner(result)}" + ) + end) + end + + defp print_extended_rows(results) do + IO.puts("\nExtended direct-counter medians (ns/op; C/U means charlist/unrolled)") + IO.puts("bytes,ASCII,2-byte,3-byte,4-byte,mixed") + + Enum.each(@extended_sizes, fn bytes -> + measurements = + Enum.map(@families, fn {family, _label, _unit} -> + result = Enum.find(results, &(&1.bytes == bytes and &1.family == family)) + + "#{result.codepoints}cp #{format_ns(result.charlist.median)}/" <> + format_ns(result.unrolled.median) + end) + + IO.puts(Enum.join([Integer.to_string(bytes) | measurements], ",")) + end) + end + + defp print_extended_cutoff_scores(results) do + cutoffs = [-1, 64, 128, 256, 384, 512, 768, 1_024] + candidates = + Enum.map(cutoffs, fn cutoff -> + %{cutoff: cutoff, mean: strategy_score(results, cutoff) / length(results)} + end) + + best = Enum.min_by(candidates, & &1.mean) + + IO.puts("\nEqual-weight fixed-cutoff score on this sparse extended grid (mean ns/op; lower is better)") + IO.puts("cutoff,mean ns/op,from best") + + Enum.each(candidates, fn candidate -> + IO.puts( + "#{cutoff_label(candidate.cutoff)},#{format_ns(candidate.mean)}," <> + "#{format_percent(candidate.mean / best.mean - 1.0)}" + ) + end) + end + + defp print_allocation_context do + IO.puts( + "\nAllocation context: charlist counting allocates one list element per code point; " <> + "the unrolled counter avoids that allocation. This timing comparison does not measure heap words." + ) + end + + defp print_family_crossovers(results, sizes) do + IO.puts("\nPer-family crossover scan") + + IO.puts( + "family,first unrolled median win,first charlist median win,last charlist median win," <> + "stable charlist win through final selected byte,stable unrolled win through final selected byte" + ) + + Enum.each(@families, fn {family, label, _unit} -> + family_results = Enum.filter(results, &(&1.family == family)) + first_unrolled = first_unrolled_win(family_results) + first_charlist = first_charlist_win(family_results) + last_charlist = last_charlist_win(family_results) + stable_charlist = stable_charlist_win(family_results, sizes) + stable_unrolled = stable_unrolled_win(family_results, sizes) + + IO.puts( + "#{label},#{format_byte(first_unrolled)},#{format_byte(first_charlist)}," <> + "#{format_byte(last_charlist)},#{format_byte(stable_charlist)},#{format_byte(stable_unrolled)}" + ) + end) + end + + defp first_unrolled_win(results) do + results + |> Enum.find(fn result -> result.unrolled.median <= result.charlist.median end) + |> byte_or_nil() + end + + defp first_charlist_win(results) do + results + |> Enum.find(fn result -> result.charlist.median <= result.unrolled.median end) + |> byte_or_nil() + end + + defp last_charlist_win(results) do + results + |> Enum.filter(fn result -> result.charlist.median < result.unrolled.median end) + |> List.last() + |> byte_or_nil() + end + + defp stable_charlist_win(results, sizes) do + Enum.find(sizes, fn bytes -> + Enum.all?(results, fn result -> + result.bytes < bytes or result.charlist.median <= result.unrolled.median + end) + end) + end + + defp stable_unrolled_win(results, sizes) do + Enum.find(sizes, fn bytes -> + Enum.all?(results, fn result -> + result.bytes < bytes or result.unrolled.median <= result.charlist.median + end) + end) + end + + defp byte_or_nil(nil), do: nil + defp byte_or_nil(result), do: result.bytes + defp format_byte(nil), do: "none" + defp format_byte(bytes), do: Integer.to_string(bytes) + + defp print_cutoff_scores(results) do + candidates = + for cutoff <- [-1 | @full_sizes] do + %{cutoff: cutoff, mean: strategy_score(results, cutoff) / length(results)} + end + + best = Enum.min_by(candidates, & &1.mean) + current = Enum.find(candidates, &(&1.cutoff == @production_cutoff)) + charlist_only = Enum.find(candidates, &(&1.cutoff == 256)) + unrolled_only = Enum.find(candidates, &(&1.cutoff == -1)) + + IO.puts("\nEqual-weight fixed-cutoff score across all families and byte sizes (mean ns/op; lower is better)") + IO.puts("best measured cutoff: #{cutoff_label(best.cutoff)} at #{format_ns(best.mean)} ns/op") + IO.puts( + "current cutoff <= #{@production_cutoff}: #{format_ns(current.mean)} ns/op " <> + "(#{format_percent(current.mean / best.mean - 1.0)} from best)" + ) + IO.puts( + "charlist only: #{format_ns(charlist_only.mean)} ns/op " <> + "(#{format_percent(charlist_only.mean / best.mean - 1.0)} from best)" + ) + IO.puts( + "unrolled only: #{format_ns(unrolled_only.mean)} ns/op " <> + "(#{format_percent(unrolled_only.mean / best.mean - 1.0)} from best)" + ) + + selected_cutoffs = + [-1, 0, 16, 31, 32, 47, 48, 63, 64, 65, 80, 96, 128, 256, best.cutoff] + |> Enum.uniq() + |> Enum.sort() + + IO.puts("cutoff,mean ns/op,from best") + + Enum.each(selected_cutoffs, fn cutoff -> + candidate = Enum.find(candidates, &(&1.cutoff == cutoff)) + IO.puts( + "#{cutoff_label(cutoff)},#{format_ns(candidate.mean)}," <> + "#{format_percent(candidate.mean / best.mean - 1.0)}" + ) + end) + end + + defp strategy_score(results, cutoff) do + Enum.reduce(results, 0.0, fn result, total -> + measurement = if result.bytes <= cutoff, do: result.charlist, else: result.unrolled + total + measurement.median + end) + end + + defp winner(result) do + if result.charlist.median <= result.unrolled.median, do: "charlist", else: "unrolled" + end + + defp measurement_text(nil), do: "-" + + defp measurement_text(%{median: median, samples: samples, iterations: iterations}) do + raw = samples |> Enum.map(&format_ns/1) |> Enum.join(" ") + "#{format_ns(median)} [#{raw}] n=#{iterations}" + end + + defp format_ns(value), do: :erlang.float_to_binary(value, decimals: 2) + defp format_percent(value), do: :erlang.float_to_binary(value * 100.0, decimals: 2) <> "%" + defp cutoff_label(-1), do: "unrolled-only" + defp cutoff_label(cutoff), do: "<= #{cutoff}" +end + +StringLengthCutoffExplore.main(System.argv()) diff --git a/bench/string_length_cutoff_validation.exs b/bench/string_length_cutoff_validation.exs new file mode 100644 index 0000000..21a92a0 --- /dev/null +++ b/bench/string_length_cutoff_validation.exs @@ -0,0 +1,147 @@ +# Run from the repository root: +# mix run bench/string_length_cutoff_validation.exs --focus +# mix run bench/string_length_cutoff_validation.exs --cutoffs 64,256 --bytes 64,65,128,256,257 +# +# Compares fixed byte cutoffs through the public predicate and JSONSchex.validate/2. +# The predicate source is compiled in memory per candidate; no source file changes. +# `--focus` uses the current and proposed cutoffs at their relevant boundaries. + +root = Path.expand("..", __DIR__) +predicate_path = Path.join(root, "lib/jsonschex/compiler/predicates.ex") +predicate_source = File.read!(predicate_path) + +IO.puts( + "String-length cutoff validation; Elixir #{System.version()}, OTP #{System.otp_release()}, " <> + "schedulers #{System.schedulers_online()}" +) + + + +defmodule StringLengthCutoffValidation do + alias JSONSchex.Compiler.Predicates + + @default_cutoffs [64, 128, 256, 512] + @default_bytes [64, 65, 128, 256, 257, 384, 512, 513, 768] + @focus_cutoffs [64, 256] + @focus_bytes [64, 65, 128, 256, 257] + @default_timing %{sample_count: 3, target_microseconds: 5_000, max_iterations: 100_000} + @focus_timing %{sample_count: 3, target_microseconds: 5_000, max_iterations: 100_000} + @families [ + {:ascii, "ASCII", "a"}, + {:two_byte, "2-byte", "é"}, + {:three_byte, "3-byte", "漢"}, + {:four_byte, "4-byte", "😀"}, + {:mixed, "mixed", "aé漢😀"} + ] + + def run(source, path, argv) do + options = parse_args(argv) + cases = build_cases(options.bytes) + + IO.puts( + "Median us/op; #{options.timing.sample_count} adaptive batches targeting " <> + "#{options.timing.target_microseconds} us; input/schema construction, candidate compilation, " <> + "warmup and explicit GC excluded" + ) + + IO.puts( + "Cutoffs: #{Enum.join(options.cutoffs, ", ")}; bytes: #{Enum.join(options.bytes, ", ")}" + ) + + for cutoff <- options.cutoffs do + load_candidate!(source, path, cutoff) + + Enum.each(cases, fn %{label: label, bytes: bytes, codepoints: codepoints, data: data, schema: schema} -> + predicate = measure(fn -> :ok = Predicates.check_min_length(data, codepoints) end, options.timing) + validation = measure(fn -> :ok = JSONSchex.validate(schema, data) end, options.timing) + + IO.puts( + "cutoff=#{cutoff},#{label},bytes=#{bytes},codepoints=#{codepoints}," <> + "predicate_us=#{Float.round(predicate, 3)},validate_both_us=#{Float.round(validation, 3)}" + ) + end) + end + end + + defp parse_args(argv) do + {options, arguments, invalid} = + OptionParser.parse(argv, strict: [focus: :boolean, cutoffs: :string, bytes: :string]) + + if arguments != [] or invalid != [] do + raise "Usage: mix run bench/string_length_cutoff_validation.exs [--focus | --cutoffs 64,256 --bytes 64,65]" + end + + if Keyword.get(options, :focus, false) do + if Keyword.has_key?(options, :cutoffs) or Keyword.has_key?(options, :bytes) do + raise "--focus cannot be combined with --cutoffs or --bytes" + end + + %{cutoffs: @focus_cutoffs, bytes: @focus_bytes, timing: @focus_timing} + else + %{ + cutoffs: parse_integer_list(Keyword.get(options, :cutoffs), @default_cutoffs, "cutoffs"), + bytes: parse_integer_list(Keyword.get(options, :bytes), @default_bytes, "bytes"), + timing: @default_timing + } + end + end + + defp parse_integer_list(nil, default, _name), do: default + + defp parse_integer_list(value, _default, name) do + values = String.split(value, ",", trim: true) + + if values == [] do + raise "--#{name} must contain at least one non-negative integer" + end + + Enum.map(values, fn text -> + case Integer.parse(text) do + {integer, ""} when integer >= 0 -> integer + _ -> raise "--#{name} must contain comma-separated non-negative integers" + end + end) + end + + defp build_cases(bytes_list) do + for bytes <- bytes_list, {_family, label, unit} <- @families do + data = exact_bytes(unit, bytes) + codepoints = length(String.to_charlist(data)) + {:ok, schema} = JSONSchex.compile(%{"minLength" => codepoints, "maxLength" => codepoints}) + %{label: label, bytes: byte_size(data), codepoints: codepoints, data: data, schema: schema} + end + end + + defp exact_bytes(unit, bytes) do + String.duplicate(unit, div(bytes, byte_size(unit))) <> + String.duplicate("a", rem(bytes, byte_size(unit))) + end + + defp load_candidate!(source, path, cutoff) do + source = Regex.replace(~r/@charlist_byte_cutoff \d+/, source, "@charlist_byte_cutoff #{cutoff}") + previous_options = Code.compiler_options(ignore_module_conflict: true) + + try do + Code.compile_string(source, path) + after + Code.compiler_options(previous_options) + end + end + + defp measure(run, %{sample_count: sample_count, target_microseconds: target, max_iterations: max_iterations}) do + run.() + :erlang.garbage_collect() + {calibration, _} = :timer.tc(fn -> Enum.each(1..10, fn _ -> run.() end) end) + iterations = min(max_iterations, max(1, div(target * 10, max(1, calibration)))) + + samples = for _ <- 1..sample_count do + :erlang.garbage_collect() + {us, _} = :timer.tc(fn -> Enum.each(1..iterations, fn _ -> run.() end) end) + us / iterations + end + + samples |> Enum.sort() |> Enum.at(div(sample_count, 2)) + end +end + +StringLengthCutoffValidation.run(predicate_source, predicate_path, System.argv()) diff --git a/bench/validator_error_flattening.exs b/bench/validator_error_flattening.exs new file mode 100644 index 0000000..8509338 --- /dev/null +++ b/bench/validator_error_flattening.exs @@ -0,0 +1,191 @@ +# Run from the repository root: +# mix run bench/validator_error_flattening.exs +# mix run bench/validator_error_flattening.exs --baseline 9eab14f +# P9_BENCH_FILTER=unevaluatedProperties mix run bench/validator_error_flattening.exs +# +# P9 changes only the public Validator error boundary, so a baseline run reloads +# that source in memory. Schema/data construction, source loading, warmup, result +# verification, and explicit GC are outside timed intervals. + +root = Path.expand("..", __DIR__) +validator_path = "lib/jsonschex/validator.ex" + +{source, source_label} = + case System.argv() do + [] -> + path = Path.join(root, validator_path) + {File.read!(path), "working tree (snapshot at script startup)"} + + ["--baseline", revision] -> + case System.cmd("git", ["--no-pager", "show", "#{revision}:#{validator_path}"], cd: root) do + {source, 0} -> {source, "git #{revision}:#{validator_path}"} + {output, status} -> raise "Cannot load #{validator_path} (exit #{status}): #{output}" + end + + _ -> + raise "Usage: mix run bench/validator_error_flattening.exs [--baseline REVISION]" + end + +previous_options = Code.compiler_options(ignore_module_conflict: true) + +try do + Code.compile_string(source, Path.join(root, validator_path)) +after + Code.compiler_options(previous_options) +end + +fingerprint = source |> then(&:crypto.hash(:sha256, &1)) |> Base.encode16(case: :lower) + +IO.puts("Validator: #{source_label}; sha256=#{fingerprint}") +IO.puts("Elixir #{System.version()}, OTP #{System.otp_release()}, schedulers #{System.schedulers_online()}") + +IO.puts( + "Median ms/op and reductions/op; five adaptive batches targeting 20 ms; construction, source loading, verification, warmup and GC excluded" +) + +defmodule ValidatorErrorFlatteningBench do + @flat_widths [1, 16, 128, 1_024, 8_192] + @nested_depths [1, 8, 32, 128] + @branch_widths [4, 32, 128, 512] + @unevaluated_widths [16, 128, 1_024] + + def cases do + [valid_control(), single_error_control()] ++ + Enum.map(@flat_widths, &flat_items_case/1) ++ + Enum.map(@nested_depths, &nested_properties_case/1) ++ + Enum.flat_map(@branch_widths, &failed_applicator_cases/1) ++ + Enum.map(@unevaluated_widths, &unevaluated_properties_case/1) + end + + def measure({label, schema, data, expected_error_count}) do + expected_result = verify_result!(schema, data, expected_error_count) + + validate = fn -> + case JSONSchex.validate(schema, data) do + :ok -> :ok + {:error, _errors} -> :error + end + end + + ^expected_result = validate.() + :erlang.garbage_collect() + {calibration_us, ^expected_result} = :timer.tc(validate) + count = min(10_000, max(1, div(20_000, max(1, calibration_us)))) + + samples = + for _ <- 1..5 do + :erlang.garbage_collect() + + {elapsed_us, :ok} = + :timer.tc(fn -> + Enum.each(1..count, fn _ -> + ^expected_result = validate.() + end) + end) + + elapsed_us / count / 1_000 + end + + reductions = validation_reductions(validate, expected_result) + median = samples |> Enum.sort() |> Enum.at(2) + + IO.puts( + "#{label}: median_ms=#{Float.round(median, 4)} reductions=#{reductions} " <> + "errors=#{expected_error_count} (count=#{count}; " <> + "min_ms=#{Float.round(Enum.min(samples), 4)} " <> + "max_ms=#{Float.round(Enum.max(samples), 4)})" + ) + end + + defp valid_control do + {"valid primitive control", compile!(%{"type" => "integer"}), 1, 0} + end + + defp single_error_control do + {"single primitive error control", compile!(%{"type" => "integer"}), "bad", 1} + end + + defp flat_items_case(width) do + schema = compile!(%{"items" => %{"type" => "integer"}}) + data = Enum.map(1..width, &"bad-#{&1}") + {"flat item errors width=#{width}", schema, data, width} + end + + defp nested_properties_case(depth) do + width = 128 + leaf_schema = %{"items" => %{"allOf" => [%{"minimum" => 10}, %{"maximum" => 0}]}} + leaf_data = List.duplicate(5, width) + + {raw_schema, data} = + Enum.reduce(1..depth, {leaf_schema, leaf_data}, fn level, {schema, nested_data} -> + key = "level-#{level}" + {%{"properties" => %{key => schema}}, %{key => nested_data}} + end) + + {"nested property errors depth=#{depth} width=#{width}", compile!(raw_schema), data, 2 * width} + end + + defp failed_applicator_cases(width) do + branches = Enum.map(1..width, &%{"const" => &1}) + + Enum.map(["allOf", "anyOf", "oneOf"], fn keyword -> + schema = compile!(%{keyword => branches}) + {"all-failed #{keyword} branches=#{width}", schema, -1, width} + end) + end + + defp unevaluated_properties_case(width) do + data = Map.new(1..width, &{"field-#{&1}", &1}) + schema = compile!(%{"unevaluatedProperties" => false}) + {"false unevaluatedProperties width=#{width}", schema, data, width} + end + + defp verify_result!(schema, data, 0) do + case JSONSchex.validate(schema, data) do + :ok -> :ok + result -> raise "expected successful control, got: #{inspect(result)}" + end + end + + defp verify_result!(schema, data, expected_error_count) do + case JSONSchex.validate(schema, data) do + {:error, errors} when length(errors) == expected_error_count -> :error + {:error, errors} -> raise "expected #{expected_error_count} errors, got: #{length(errors)}" + :ok -> raise "expected #{expected_error_count} errors, validation passed" + end + end + + defp compile!(raw_schema) do + {:ok, schema} = JSONSchex.compile(raw_schema) + schema + end + + defp validation_reductions(validate, expected_result) do + :erlang.garbage_collect() + {:reductions, before} = Process.info(self(), :reductions) + ^expected_result = validate.() + {:reductions, after_validation} = Process.info(self(), :reductions) + after_validation - before + end +end + +cases = + case System.get_env("P9_BENCH_FILTER") do + nil -> + ValidatorErrorFlatteningBench.cases() + + filter -> + Enum.filter(ValidatorErrorFlatteningBench.cases(), fn {label, _schema, _data, _count} -> + String.contains?(label, filter) + end) + end + +task = + Task.async(fn -> + Enum.each(cases, &ValidatorErrorFlatteningBench.measure/1) + end) + +case Task.yield(task, 50_000) || Task.shutdown(task, :brutal_kill) do + {:ok, :ok} -> :ok + _ -> raise "Benchmark exceeded the 50 s measurement budget; results are incomplete" +end diff --git a/bench/validator_evaluated_keys_merge.exs b/bench/validator_evaluated_keys_merge.exs new file mode 100644 index 0000000..9bab099 --- /dev/null +++ b/bench/validator_evaluated_keys_merge.exs @@ -0,0 +1,317 @@ +# Run from the repository root: +# mix run bench/validator_evaluated_keys_merge.exs +# mix run bench/validator_evaluated_keys_merge.exs --baseline 56190fd +# +# P8 changes only evaluated-key merging in Keywords, so a baseline run reloads +# that source in memory. Schema/data construction, source loading, warmup, and +# explicit GC are outside timed intervals. + +root = Path.expand("..", __DIR__) +validator_path = "lib/jsonschex/validator/keywords.ex" + +{source, source_label} = + case System.argv() do + [] -> + path = Path.join(root, validator_path) + {File.read!(path), "working tree (snapshot at script startup)"} + + ["--baseline", revision] -> + case System.cmd("git", ["--no-pager", "show", "#{revision}:#{validator_path}"], cd: root) do + {source, 0} -> {source, "git #{revision}:#{validator_path}"} + {output, status} -> raise "Cannot load #{validator_path} (exit #{status}): #{output}" + end + + _ -> + raise "Usage: mix run bench/validator_evaluated_keys_merge.exs [--baseline REVISION]" + end + +previous_options = Code.compiler_options(ignore_module_conflict: true) + +try do + Code.compile_string(source, Path.join(root, validator_path)) +after + Code.compiler_options(previous_options) +end + +fingerprint = source |> then(&:crypto.hash(:sha256, &1)) |> Base.encode16(case: :lower) + +IO.puts("Validator keywords: #{source_label}; sha256=#{fingerprint}") +IO.puts("Elixir #{System.version()}, OTP #{System.otp_release()}, schedulers #{System.schedulers_online()}") + +IO.puts( + "Median ms/op and reductions/op; five adaptive batches targeting 20 ms; construction, source loading, warmup and GC excluded" +) + +defmodule ValidatorEvaluatedKeysMergeBench do + alias JSONSchex.Types.{Rule, Schema} + + @widths [1, 2, 4, 16, 64, 128] + @dominant_parent_width 16 + @non_dominant_branch_evaluated_key_width 16 + + def cases do + Enum.flat_map(@widths, fn width -> + [ + all_of_success_case(width), + all_of_discarded_evaluated_keys_case(width), + any_of_disjoint_case(width), + any_of_dominant_parent_case(width), + any_of_non_dominant_parent_case(width) + ] ++ + any_of_candidate_fallback_cases(width) ++ + any_of_late_candidate_fallback_cases(width) ++ + [ + dependent_schemas_success_case(width), + dependent_schemas_discarded_evaluated_keys_case(width), + legacy_dependencies_success_case(width) + ] + end) + end + + def measure({label, schema, data}) do + validate = fn -> + case JSONSchex.validate(schema, data) do + :ok -> :ok + {:error, _errors} -> :error + end + end + + expected = validate.() + :erlang.garbage_collect() + {calibration_us, ^expected} = :timer.tc(validate) + count = min(10_000, max(1, div(20_000, max(1, calibration_us)))) + + samples = + for _ <- 1..5 do + :erlang.garbage_collect() + + {elapsed_us, :ok} = + :timer.tc(fn -> + Enum.each(1..count, fn _ -> + ^expected = validate.() + end) + end) + + elapsed_us / count / 1_000 + end + + reductions = validation_reductions(validate, expected) + median = samples |> Enum.sort() |> Enum.at(2) + + IO.puts( + "#{label}: median_ms=#{Float.round(median, 4)} reductions=#{reductions} " <> + "(count=#{count}; min_ms=#{Float.round(Enum.min(samples), 4)} " <> + "max_ms=#{Float.round(Enum.max(samples), 4)})" + ) + end + + defp all_of_success_case(width) do + fields = keys("all-of", width) + schema = compile!(%{"allOf" => Enum.map(fields, &property_schema([&1]))}) + + {"allOf disjoint successful evaluated keys width=#{width}", schema, data_for(fields)} + end + + defp all_of_discarded_evaluated_keys_case(width) do + fields = keys("all-of", width) + schema = compile!(%{"allOf" => Enum.map(fields, &property_schema([&1])) ++ [false]}) + + {"allOf discarded successful evaluated keys width=#{width}", schema, data_for(fields)} + end + + defp any_of_disjoint_case(width) do + fields = keys("any-of", width) + schema = compile!(%{"anyOf" => Enum.map(fields, &property_schema([&1]))}) + + {"anyOf empty parent, disjoint evaluated keys width=#{width}", schema, data_for(fields)} + end + + defp any_of_dominant_parent_case(width) do + seed_fields = keys("seed", @dominant_parent_width) + branch_fields = keys("any-of", width) + + schema = + compile!(%{ + "properties" => Map.new(seed_fields, &{&1, true}), + "anyOf" => Enum.map(branch_fields, &property_schema([&1])) + }) + |> reorder_rules([:properties, :anyOf]) + + { + "anyOf parent=#{@dominant_parent_width}, one new evaluated key/branch width=#{width}", + schema, + data_for(seed_fields ++ branch_fields) + } + end + + defp any_of_non_dominant_parent_case(width) do + seed_fields = ["seed"] + + branches = + Enum.map(1..width, fn branch -> + keys("any-of-#{branch}", @non_dominant_branch_evaluated_key_width) + end) + + branch_fields = List.flatten(branches) + + schema = + compile!(%{ + "properties" => Map.new(seed_fields, &{&1, true}), + "anyOf" => Enum.map(branches, &property_schema/1) + }) + |> reorder_rules([:properties, :anyOf]) + + { + "anyOf parent=1, branch evaluated keys=#{@non_dominant_branch_evaluated_key_width} width=#{width}", + schema, + data_for(seed_fields ++ branch_fields) + } + end + + defp any_of_candidate_fallback_cases(width) when width >= 4 do + [any_of_candidate_fallback_case(width)] + end + + defp any_of_candidate_fallback_cases(_width), do: [] + + defp any_of_candidate_fallback_case(width) do + seed_fields = keys("seed", @dominant_parent_width) + + branches = + Enum.map(1..width, fn branch -> + keys("any-of-#{branch}", @non_dominant_branch_evaluated_key_width) + end) + + branch_fields = List.flatten(branches) + + schema = + compile!(%{ + "properties" => Map.new(seed_fields, &{&1, true}), + "anyOf" => Enum.map(branches, &property_schema/1) + }) + |> reorder_rules([:properties, :anyOf]) + + { + "anyOf candidate fallback parent=#{@dominant_parent_width}, branch evaluated keys=#{@non_dominant_branch_evaluated_key_width} width=#{width}", + schema, + data_for(seed_fields ++ branch_fields) + } + end + + defp any_of_late_candidate_fallback_cases(width) when width >= 4 do + [any_of_late_candidate_fallback_case(width)] + end + + defp any_of_late_candidate_fallback_cases(_width), do: [] + + defp any_of_late_candidate_fallback_case(width) do + seed_fields = keys("seed", @dominant_parent_width) + early_fields = keys("any-of-early", width - 1) + late_fields = keys("any-of-late", 8 * width) + + branches = + Enum.map(early_fields, &property_schema([&1])) ++ [property_schema(late_fields)] + + schema = + compile!(%{ + "properties" => Map.new(seed_fields, &{&1, true}), + "anyOf" => branches + }) + |> reorder_rules([:properties, :anyOf]) + + { + "anyOf late fallback parent=#{@dominant_parent_width}, early branches=#{width - 1}, late evaluated keys=#{8 * width}", + schema, + data_for(seed_fields ++ early_fields ++ late_fields) + } + end + + defp dependent_schemas_success_case(width) do + {dependencies, fields, trigger_fields} = dependent_schema_inputs(width) + schema = compile!(%{"dependentSchemas" => dependencies}) + + { + "dependentSchemas disjoint successful evaluated keys width=#{width}", + schema, + data_for(trigger_fields ++ fields) + } + end + + defp dependent_schemas_discarded_evaluated_keys_case(width) do + {dependencies, fields, trigger_fields} = dependent_schema_inputs(width) + schema = compile!(%{"dependentSchemas" => Map.put(dependencies, "failure", false)}) + + { + "dependentSchemas discarded successful evaluated keys width=#{width}", + schema, + data_for(["failure" | trigger_fields ++ fields]) + } + end + + defp legacy_dependencies_success_case(width) do + {dependencies, fields, trigger_fields} = dependent_schema_inputs(width) + schema = compile!(%{"dependencies" => dependencies}) + + { + "legacy dependencies disjoint successful evaluated keys width=#{width}", + schema, + data_for(trigger_fields ++ fields) + } + end + + defp dependent_schema_inputs(width) do + Enum.reduce(1..width, {%{}, [], []}, fn index, {dependencies, fields, triggers} -> + field = "dependent-value-#{index}" + trigger = "dependent-trigger-#{index}" + + { + Map.put(dependencies, trigger, property_schema([field])), + [field | fields], + [trigger | triggers] + } + end) + end + + defp property_schema(fields), do: %{"properties" => Map.new(fields, &{&1, true})} + defp data_for(fields), do: Map.new(fields, &{&1, true}) + defp keys(prefix, width), do: Enum.map(1..width, &"#{prefix}-#{&1}") + + defp compile!(raw_schema) do + {:ok, schema} = JSONSchex.compile(raw_schema) + schema + end + + defp reorder_rules(%Schema{rules: rules} = schema, names) do + selected = + Enum.map(names, fn name -> + case Enum.find(rules, &(&1.name == name)) do + %Rule{} = rule -> rule + nil -> raise "expected compiled #{inspect(name)} rule" + end + end) + + remaining = Enum.reject(rules, &(&1.name in names)) + %{schema | rules: selected ++ remaining} + end + + defp validation_reductions(validate, expected) do + :erlang.garbage_collect() + {:reductions, before} = Process.info(self(), :reductions) + ^expected = validate.() + {:reductions, after_validation} = Process.info(self(), :reductions) + after_validation - before + end +end + +task = + Task.async(fn -> + Enum.each( + ValidatorEvaluatedKeysMergeBench.cases(), + &ValidatorEvaluatedKeysMergeBench.measure/1 + ) + end) + +case Task.yield(task, 50_000) || Task.shutdown(task, :brutal_kill) do + {:ok, :ok} -> :ok + _ -> raise "Benchmark exceeded the 50 s measurement budget; results are incomplete" +end diff --git a/bench/validator_pattern_match_cache.exs b/bench/validator_pattern_match_cache.exs new file mode 100644 index 0000000..333c801 --- /dev/null +++ b/bench/validator_pattern_match_cache.exs @@ -0,0 +1,202 @@ +# Run from the repository root: +# mix run bench/validator_pattern_match_cache.exs +# mix run bench/validator_pattern_match_cache.exs --baseline 246fbaf +# +# Validation spans the validator orchestrator, rule dispatcher, and object-keyword +# reducers, so a baseline run reloads all three sources in memory. Input/schema +# construction, source loading, warmup, and explicit GC are outside timed intervals. + +root = Path.expand("..", __DIR__) +validator_paths = [ + "lib/jsonschex/validator/keywords.ex", + "lib/jsonschex/validator/rules.ex", + "lib/jsonschex/validator.ex" +] + +{sources, source_label} = + case System.argv() do + [] -> + { + Enum.map(validator_paths, fn path -> {Path.join(root, path), File.read!(Path.join(root, path))} end), + "working tree (snapshot at script startup)" + } + + ["--baseline", revision] -> + sources = + Enum.map(validator_paths, fn path -> + case System.cmd("git", ["--no-pager", "show", "#{revision}:#{path}"], cd: root) do + {source, 0} -> {Path.join(root, path), source} + {output, status} -> raise "Cannot load #{path} (exit #{status}): #{output}" + end + end) + + {sources, "git #{revision}: validator sources"} + + _ -> + raise "Usage: mix run bench/validator_pattern_match_cache.exs [--baseline REVISION]" + end + +previous_options = Code.compiler_options(ignore_module_conflict: true) + +try do + Enum.each(sources, fn {path, source} -> Code.compile_string(source, path) end) +after + Code.compiler_options(previous_options) +end + +fingerprint = + sources + |> Enum.map(&elem(&1, 1)) + |> IO.iodata_to_binary() + |> then(&:crypto.hash(:sha256, &1)) + |> Base.encode16(case: :lower) + +IO.puts("Validator: #{source_label}; sha256=#{fingerprint}") +IO.puts("Elixir #{System.version()}, OTP #{System.otp_release()}, schedulers #{System.schedulers_online()}") +IO.puts("Median ms/op and reductions/op; five adaptive batches targeting 20 ms; construction, source loading, warmup and GC excluded") + +defmodule ValidatorPatternMatchCacheBench do + alias JSONSchex.Types.{Rule, Schema} + + @widths [1, 2, 4, 8, 16, 32, 128, 512] + + def cases do + Enum.flat_map(@widths, fn width -> + [ + build_case(width, :pattern_only, :pattern_first), + build_case(width, :siblings_true, :pattern_first), + build_case(width, :siblings_true, :additional_first), + build_case(width, :siblings_false, :pattern_first), + build_case(width, :mostly_declared, :pattern_first) + ] + end) + end + + def measure({label, schema, data}) do + validate = fn -> + case JSONSchex.validate(schema, data) do + :ok -> :ok + {:error, _errors} -> :error + end + end + + expected = validate.() + :erlang.garbage_collect() + {calibration_us, ^expected} = :timer.tc(validate) + count = min(10_000, max(1, div(20_000, max(1, calibration_us)))) + + samples = + for _ <- 1..5 do + :erlang.garbage_collect() + + {elapsed_us, :ok} = + :timer.tc(fn -> + Enum.each(1..count, fn _ -> + ^expected = validate.() + end) + end) + + elapsed_us / count / 1_000 + end + + reductions = validation_reductions(validate, expected) + median = samples |> Enum.sort() |> Enum.at(2) + + IO.puts( + "#{label}: median_ms=#{Float.round(median, 4)} reductions=#{reductions} " <> + "(count=#{count}; min_ms=#{Float.round(Enum.min(samples), 4)} " <> + "max_ms=#{Float.round(Enum.max(samples), 4)})" + ) + end + + defp build_case(width, :pattern_only, _order) do + schema = %{"patternProperties" => pattern_map(width)} + {:ok, compiled} = JSONSchex.compile(schema) + {"patternProperties only, unmatched keys width=#{width}", compiled, unmatched_data(width)} + end + + defp build_case(width, :siblings_true, order) do + schema = %{"patternProperties" => pattern_map(width), "additionalProperties" => true} + {:ok, compiled} = JSONSchex.compile(schema) + + { + "siblings additionalProperties=true, unmatched keys, #{order} width=#{width}", + reorder_rules(compiled, order), + unmatched_data(width) + } + end + + defp build_case(width, :siblings_false, order) do + schema = %{"patternProperties" => pattern_map(width), "additionalProperties" => false} + {:ok, compiled} = JSONSchex.compile(schema) + + { + "siblings additionalProperties=false, unmatched keys, #{order} width=#{width}", + reorder_rules(compiled, order), + unmatched_data(width) + } + end + + defp build_case(width, :mostly_declared, order) do + properties = Map.new(1..width, fn index -> {"field-" <> Integer.to_string(index), true} end) + + schema = %{ + "properties" => properties, + "patternProperties" => pattern_map(width), + "additionalProperties" => true + } + + {:ok, compiled} = JSONSchex.compile(schema) + + { + "siblings mostly declared properties, #{order} width=#{width}", + reorder_rules(compiled, order), + unmatched_data(width) + } + end + + defp pattern_map(width) do + Map.new(1..width, fn index -> + {"^pattern-" <> Integer.to_string(index) <> "$", true} + end) + end + + defp unmatched_data(width) do + Map.new(1..width, fn index -> {"field-" <> Integer.to_string(index), index} end) + end + + defp reorder_rules(%Schema{rules: rules} = schema, :pattern_first) do + reorder_rules(schema, [:patternProperties, :additionalProperties], rules) + end + + defp reorder_rules(%Schema{rules: rules} = schema, :additional_first) do + reorder_rules(schema, [:additionalProperties, :patternProperties], rules) + end + + defp reorder_rules(%Schema{} = schema, names, rules) do + selected = + Enum.map(names, fn name -> + case Enum.find(rules, &(&1.name == name)) do + %Rule{} = rule -> rule + nil -> raise "expected compiled #{inspect(name)} rule" + end + end) + + %{schema | rules: selected ++ Enum.reject(rules, &(&1.name in names))} + end + + defp validation_reductions(validate, expected) do + :erlang.garbage_collect() + {:reductions, before} = Process.info(self(), :reductions) + ^expected = validate.() + {:reductions, after_validation} = Process.info(self(), :reductions) + after_validation - before + end +end + +task = Task.async(fn -> Enum.each(ValidatorPatternMatchCacheBench.cases(), &ValidatorPatternMatchCacheBench.measure/1) end) + +case Task.yield(task, 50_000) || Task.shutdown(task, :brutal_kill) do + {:ok, :ok} -> :ok + _ -> raise "Benchmark exceeded the 50 s measurement budget; results are incomplete" +end diff --git a/lib/jsonschex/compiler.ex b/lib/jsonschex/compiler.ex index a4564bc..fc0907a 100644 --- a/lib/jsonschex/compiler.ex +++ b/lib/jsonschex/compiler.ex @@ -331,7 +331,20 @@ defmodule JSONSchex.Compiler do end end + defp compile_keywords_list(%{"patternProperties" => _, "additionalProperties" => _} = keywords_map, base, vocabs, ctx) do + # Regexes are shared only by these sibling object keywords and only for + # this node. Enter this path after $defs compilation; the existing gate + # still controls whether either keyword prepares a regex. + compile_keywords_with_regex_cache(keywords_map, base, vocabs, ctx) + end defp compile_keywords_list(keywords_map, base, vocabs, ctx) do + # Retain the original keyword reducer for all other schemas, including + # patternProperties-only schemas, so the local optimization adds no + # per-keyword wrapper overhead to the common path. + compile_keywords_without_regex_cache(keywords_map, base, vocabs, ctx) + end + + defp compile_keywords_without_regex_cache(keywords_map, base, vocabs, ctx) do Enum.reduce_while(keywords_map, {:ok, []}, fn {k, v}, {:ok, acc} -> if keyword_allowed?(k, vocabs, ctx) do case compile_keyword({k, v}, keywords_map, base, vocabs, ctx) do @@ -345,6 +358,57 @@ defmodule JSONSchex.Compiler do end) end + defp compile_keywords_with_regex_cache(keywords_map, base, vocabs, ctx) do + Enum.reduce_while(keywords_map, {:ok, [], %{}}, fn {k, v}, {:ok, acc, cache} -> + if keyword_allowed?(k, vocabs, ctx) do + case compile_keyword_with_regex_cache({k, v}, keywords_map, base, vocabs, ctx, cache) do + {:ok, nil, updated_cache} -> {:cont, {:ok, acc, updated_cache}} + {:ok, rule, updated_cache} -> {:cont, {:ok, [rule | acc], updated_cache}} + {:error, error} -> {:halt, {:error, error}} + end + else + {:cont, {:ok, acc, cache}} + end + end) + |> case do + {:ok, rules, _cache} -> {:ok, rules} + {:error, _} = error -> error + end + end + + defp compile_keyword_with_regex_cache( + {"patternProperties", patterns}, + _full_schema, + base, + vocabs, + ctx, + regex_cache + ) + when is_map(regex_cache) do + with {:ok, compiled_patterns, updated_cache} <- + compile_pattern_properties(patterns, base, vocabs, ctx, regex_cache) do + {:ok, build_pattern_properties_rule(compiled_patterns), updated_cache} + end + end + + defp compile_keyword_with_regex_cache( + {"additionalProperties", sub_schema}, + full_schema, + base, + vocabs, + ctx, + regex_cache + ) + when is_map(regex_cache) do + compile_additional_properties(sub_schema, full_schema, base, vocabs, ctx, regex_cache) + end + + defp compile_keyword_with_regex_cache(keyword, full_schema, base, vocabs, ctx, regex_cache) do + with {:ok, rule} <- compile_keyword(keyword, full_schema, base, vocabs, ctx) do + {:ok, rule, regex_cache} + end + end + defp compile_unevaluted(_, nil, _base, _vocabs, _ctx), do: {:ok, nil} defp compile_unevaluted("unevaluatedProperties" = keyword, sub_schema, base, vocabs, ctx) do @@ -650,30 +714,13 @@ defmodule JSONSchex.Compiler do {:halt, {:error, %{error | path: path ++ [pattern]}}} {:error, {regex_term, _}} -> - {:halt, - {:error, - %Error{ - rule: :invalid_regex, - path: ["patternProperties", pattern], - context: %ErrorContext{ - contrast: "invalid_regex", - input: pattern, - error_detail: regex_term - } - }}} + {:halt, {:error, invalid_pattern_properties_regex_error(pattern, regex_term)}} end end) case result do - {:ok, compiled_patterns} -> - {:ok, - %Rule{ - name: :patternProperties, - params: compiled_patterns - }} - - error -> - error + {:ok, compiled_patterns} -> {:ok, build_pattern_properties_rule(compiled_patterns)} + error -> error end end @@ -681,23 +728,13 @@ defmodule JSONSchex.Compiler do raw_patterns = Map.keys(Map.get(full_schema, "patternProperties", %{})) regex_compilation = - Enum.reduce_while(raw_patterns, {:ok, []}, fn p, {:ok, acc} -> - case JSONSchex.Compiler.ECMARegex.compile(p) do - {:ok, r} -> - {:cont, {:ok, [r | acc]}} + Enum.reduce_while(raw_patterns, {:ok, []}, fn pattern, {:ok, acc} -> + case JSONSchex.Compiler.ECMARegex.compile(pattern) do + {:ok, regex} -> + {:cont, {:ok, [regex | acc]}} {:error, {regex_term, _}} -> - {:halt, - {:error, - %Error{ - rule: :invalid_regex, - path: ["patternProperties", p], - context: %ErrorContext{ - contrast: "invalid_regex", - input: p, - error_detail: regex_term - } - }}} + {:halt, {:error, invalid_pattern_properties_regex_error(pattern, regex_term)}} end end) @@ -890,6 +927,95 @@ defmodule JSONSchex.Compiler do defp compile_keyword(_, _, _base, _vocabs, _ctx), do: {:ok, nil} + defp compile_pattern_properties(patterns, _base, _vocabs, _ctx, regex_cache) + when map_size(patterns) == 0, + do: {:ok, nil, regex_cache} + + defp compile_pattern_properties(patterns, base, vocabs, ctx, regex_cache) do + Enum.reduce_while(patterns, {:ok, [], regex_cache}, fn {pattern, sub}, + {:ok, acc, cache} -> + case fetch_or_compile_pattern(pattern, cache) do + {:ok, regex, updated_cache} -> + case compile_schema_node(sub, base, vocabs, ctx) do + {:ok, compiled_sub} -> + {:cont, {:ok, [{regex, compiled_sub} | acc], updated_cache}} + + {:error, %Error{} = error} -> + path = error.path || [] + {:halt, {:error, %{error | path: path ++ [pattern]}}} + end + + {:error, {regex_term, _}} -> + {:halt, {:error, invalid_pattern_properties_regex_error(pattern, regex_term)}} + end + end) + end + + defp build_pattern_properties_rule(nil), do: nil + + defp build_pattern_properties_rule(compiled_patterns) do + %Rule{name: :patternProperties, params: compiled_patterns} + end + + defp compile_additional_properties(sub_schema, full_schema, base, vocabs, ctx, regex_cache) do + with {:ok, compiled_patterns, updated_cache} <- + compile_additional_pattern_regexes(full_schema, regex_cache), + {:ok, compiled_sub} <- compile_schema_node(sub_schema, base, vocabs, ctx) do + known_props_set = Map.keys(Map.get(full_schema, "properties", %{})) |> MapSet.new() + always_valid? = match?(%Schema{rules: []}, compiled_sub) + + {:ok, + %Rule{ + name: :additionalProperties, + params: %{ + schema: compiled_sub, + known_props: known_props_set, + patterns: compiled_patterns, + always_valid?: always_valid? + } + }, updated_cache} + end + end + + defp compile_additional_pattern_regexes(full_schema, regex_cache) do + raw_patterns = Map.keys(Map.get(full_schema, "patternProperties", %{})) + + Enum.reduce_while(raw_patterns, {:ok, [], regex_cache}, fn pattern, {:ok, acc, cache} -> + case fetch_or_compile_pattern(pattern, cache) do + {:ok, regex, updated_cache} -> + {:cont, {:ok, [regex | acc], updated_cache}} + + {:error, {regex_term, _}} -> + {:halt, {:error, invalid_pattern_properties_regex_error(pattern, regex_term)}} + end + end) + end + + defp fetch_or_compile_pattern(pattern, regex_cache) do + case Map.fetch(regex_cache, pattern) do + {:ok, regex} -> + {:ok, regex, regex_cache} + + :error -> + case JSONSchex.Compiler.ECMARegex.compile(pattern) do + {:ok, regex} -> {:ok, regex, Map.put(regex_cache, pattern, regex)} + {:error, _} = error -> error + end + end + end + + defp invalid_pattern_properties_regex_error(pattern, regex_term) do + %Error{ + rule: :invalid_regex, + path: ["patternProperties", pattern], + context: %ErrorContext{ + contrast: "invalid_regex", + input: pattern, + error_detail: regex_term + } + } + end + defp compile_optional_schema(nil, _base, _vocabs, _ctx), do: {:ok, nil} defp compile_optional_schema(schema, base, vocabs, ctx), diff --git a/lib/jsonschex/compiler/predicates.ex b/lib/jsonschex/compiler/predicates.ex index e5b120a..8d33e79 100644 --- a/lib/jsonschex/compiler/predicates.ex +++ b/lib/jsonschex/compiler/predicates.ex @@ -8,6 +8,10 @@ defmodule JSONSchex.Compiler.Predicates do alias JSONSchex.Types.ErrorContext + # A fixed compromise: it avoids the short non-ASCII regression from the + # unrolled counter while keeping charlist allocation bounded. + @charlist_byte_cutoff 256 + @doc """ Checks if data matches the specified JSON Schema type(s). """ @@ -74,17 +78,21 @@ defmodule JSONSchex.Compiler.Predicates do if data == const, do: :ok, else: {:error, %ErrorContext{contrast: const, input: data}} end - @doc "Checks `minLength`." + @doc "Checks `minLength`, returning an error for malformed UTF-8." def check_min_length(data, min) when is_binary(data) do - len = codepoint_length(data) - if len >= min, do: :ok, else: {:error, %ErrorContext{contrast: min, input: len}} + case codepoint_length(data) do + :error -> {:error, %ErrorContext{contrast: min, input: data, error_detail: "invalid_utf8"}} + len -> if len >= min, do: :ok, else: {:error, %ErrorContext{contrast: min, input: len}} + end end def check_min_length(_, _), do: :ok - @doc "Checks `maxLength`." + @doc "Checks `maxLength`, returning an error for malformed UTF-8." def check_max_length(data, max) when is_binary(data) do - len = codepoint_length(data) - if len <= max, do: :ok, else: {:error, %ErrorContext{contrast: max, input: len}} + case codepoint_length(data) do + :error -> {:error, %ErrorContext{contrast: max, input: data, error_detail: "invalid_utf8"}} + len -> if len <= max, do: :ok, else: {:error, %ErrorContext{contrast: max, input: len}} + end end def check_max_length(_, _), do: :ok @@ -158,9 +166,26 @@ defmodule JSONSchex.Compiler.Predicates do end defp unique_item_hash(item), do: :erlang.phash2(item) - defp codepoint_length(binary) do + # Keep the standard conversion for small and medium strings. The cutoff is + # measured across UTF-8 widths; ASCII alone would favor the unrolled path sooner. + defp codepoint_length(binary) when byte_size(binary) <= @charlist_byte_cutoff do binary |> String.to_charlist() |> length() - end + rescue + UnicodeConversionError -> :error + end + defp codepoint_length(binary), do: count_codepoints(binary, 0) + + # Count Unicode code points, not graphemes, without allocating a charlist. + # Scan to the end or an encoding error: diagnostics need the exact length of + # valid input, and permissive bounds must not silently accept malformed UTF-8. + # A small unroll reduces loop overhead for mixed-width UTF-8; the next clause + # handles the remaining one to three code points without padding or slicing. + defp count_codepoints(<<_::utf8, _::utf8, _::utf8, _::utf8, rest::binary>>, count), + do: count_codepoints(rest, count + 4) + defp count_codepoints(<<_codepoint::utf8, rest::binary>>, count), + do: count_codepoints(rest, count + 1) + defp count_codepoints(<<>>, count), do: count + defp count_codepoints(_invalid, _count), do: :error defp infer_type(v) when is_binary(v), do: "string" defp infer_type(v) when is_integer(v), do: "integer" diff --git a/lib/jsonschex/draft202012/schemas.ex b/lib/jsonschex/draft202012/schemas.ex index 42ae586..c6e7dfb 100644 --- a/lib/jsonschex/draft202012/schemas.ex +++ b/lib/jsonschex/draft202012/schemas.ex @@ -15,7 +15,6 @@ defmodule JSONSchex.Draft202012.Schemas do """ alias JSONSchex.Compiler - alias JSONSchex.ScopeScanner alias JSONSchex.Draft202012.Dialect @compiled_defs_cache_key {__MODULE__, :draft2020_12_compiled_defs} @@ -363,9 +362,8 @@ defmodule JSONSchex.Draft202012.Schemas do The returned map includes: - each built-in resource under its canonical absolute URI - - all compiled defs discovered within each resource - - all `$anchor` and `$dynamicAnchor` absolute entries discovered by scanning - the raw built-in resources + - all compiled defs, `$anchor`, and `$dynamicAnchor` entries discovered while + compiling each resource This allows built-in schemas to behave like a normal compiled defs registry during `$ref` and `$dynamicRef` resolution. @@ -403,18 +401,10 @@ defmodule JSONSchex.Draft202012.Schemas do {:ok, raw_schema} = fetch(family_uri) {:ok, compiled_schema} = Compiler.compile(raw_schema, base_uri: family_uri) - {scanned_defs, _refs} = ScopeScanner.scan(raw_schema) acc |> Map.put(family_uri, compiled_schema) |> Map.merge(compiled_schema.defs || %{}) - |> merge_scanned_anchor_entries(scanned_defs, compiled_schema) - end) - end - - defp merge_scanned_anchor_entries(defs, scanned_defs, compiled_schema) do - Enum.reduce(scanned_defs, defs, fn {scanned_uri, _raw_schema}, acc -> - Map.put_new(acc, scanned_uri, compiled_schema) end) end end diff --git a/lib/jsonschex/error_formatter.ex b/lib/jsonschex/error_formatter.ex index 1507359..96a6d14 100644 --- a/lib/jsonschex/error_formatter.ex +++ b/lib/jsonschex/error_formatter.ex @@ -59,6 +59,11 @@ defmodule JSONSchex.ErrorFormatter do "Value #{input} is greater than or equal to exclusive maximum #{max}" end + defp format_message(%Error{rule: rule, context: %{error_detail: "invalid_utf8"}}) + when rule in [:minLength, :maxLength] do + "Cannot check #{rule}: string is not valid UTF-8" + end + defp format_message(%Error{rule: :minLength, context: %{contrast: min, input: len}}) do "String length #{len} is less than minimum #{min}" end diff --git a/lib/jsonschex/schema_traversal.ex b/lib/jsonschex/schema_traversal.ex index 22554f6..1ba3aa6 100644 --- a/lib/jsonschex/schema_traversal.ex +++ b/lib/jsonschex/schema_traversal.ex @@ -46,28 +46,26 @@ defmodule JSONSchex.SchemaTraversal do def metadata_subschemas(schema), do: scope_subschemas(schema) @doc false - def scope_subschemas(schema) when is_map(schema) do - single_subschemas = schemas_at_keys(schema, @single_schema_keywords ++ @conditional_keywords) - map_subschemas = schema_map_subschemas(schema, @schema_map_keywords ++ @definition_keywords) - list_subschemas = schema_list_subschemas(schema, @schema_list_keywords) - dependency_subschemas = schema |> Map.get("dependencies", %{}) |> schema_map_values() - - legacy_items = - case Map.get(schema, "items") do - items when is_list(items) -> schema_list_values(items) - _ -> [] - end + def scope_subschemas(schema) do + schema + |> reduce_scope_subschemas([], fn subschema, acc -> [subschema | acc] end) + |> Enum.reverse() + end - Enum.concat([ - list_subschemas, - map_subschemas, - single_subschemas, - dependency_subschemas, - legacy_items - ]) + @doc false + def reduce_scope_subschemas(schema, acc, reducer) when is_map(schema) do + acc = reduce_schema_lists(schema, @schema_list_keywords, acc, reducer) + acc = reduce_schema_maps(schema, @schema_map_keywords ++ @definition_keywords, acc, reducer) + acc = reduce_schemas_at_keys(schema, @single_schema_keywords ++ @conditional_keywords, acc, reducer) + acc = reduce_schema_map(Map.get(schema, "dependencies"), acc, reducer) + + case Map.get(schema, "items") do + items when is_list(items) -> reduce_schema_list(items, acc, reducer) + _ -> acc + end end - def scope_subschemas(_schema), do: [] + def reduce_scope_subschemas(_schema, acc, _reducer), do: acc defp schemas_at_keys(schema, keywords) do keywords @@ -91,6 +89,47 @@ defmodule JSONSchex.SchemaTraversal do end) end + defp reduce_schema_lists(schema, keywords, acc, reducer) do + Enum.reduce(keywords, acc, fn keyword, inner_acc -> + reduce_schema_list(Map.get(schema, keyword), inner_acc, reducer) + end) + end + + defp reduce_schema_maps(schema, keywords, acc, reducer) do + Enum.reduce(keywords, acc, fn keyword, inner_acc -> + reduce_schema_map(Map.get(schema, keyword), inner_acc, reducer) + end) + end + + defp reduce_schemas_at_keys(schema, keywords, acc, reducer) do + Enum.reduce(keywords, acc, fn keyword, inner_acc -> + reduce_schema(Map.get(schema, keyword), inner_acc, reducer) + end) + end + + defp reduce_schema_map(value, acc, reducer) when is_map(value) do + value + |> Map.values() + |> Enum.reduce(acc, fn subschema, inner_acc -> + reduce_schema(subschema, inner_acc, reducer) + end) + end + + defp reduce_schema_map(_value, acc, _reducer), do: acc + + defp reduce_schema_list(value, acc, reducer) when is_list(value) do + Enum.reduce(value, acc, fn subschema, inner_acc -> + reduce_schema(subschema, inner_acc, reducer) + end) + end + + defp reduce_schema_list(_value, acc, _reducer), do: acc + + defp reduce_schema(subschema, acc, reducer) when is_map(subschema) or is_boolean(subschema), + do: reducer.(subschema, acc) + + defp reduce_schema(_value, acc, _reducer), do: acc + defp schema_map_values(value) when is_map(value) do value |> Map.values() diff --git a/lib/jsonschex/scope_scanner.ex b/lib/jsonschex/scope_scanner.ex index e735463..63fe5b5 100644 --- a/lib/jsonschex/scope_scanner.ex +++ b/lib/jsonschex/scope_scanner.ex @@ -72,9 +72,9 @@ defmodule JSONSchex.ScopeScanner do end end) - schema - |> SchemaTraversal.scope_subschemas() - |> Enum.reduce({registry, refs}, fn subschema, acc -> do_scan(subschema, new_base_uri, acc) end) + SchemaTraversal.reduce_scope_subschemas(schema, {registry, refs}, fn subschema, acc -> + do_scan(subschema, new_base_uri, acc) + end) end defp do_scan(_, _, acc), do: acc diff --git a/lib/jsonschex/validator.ex b/lib/jsonschex/validator.ex index 6d629e0..76d482d 100644 --- a/lib/jsonschex/validator.ex +++ b/lib/jsonschex/validator.ex @@ -18,11 +18,14 @@ defmodule JSONSchex.Validator do :ok """ - alias JSONSchex.Types.{Error, Schema, ValidationContext} + alias JSONSchex.Types.{Error, Rule, Schema, ValidationContext} alias JSONSchex.Validator.Rules @empty_mapset MapSet.new() + @typep validation_error_list :: + [Error.t() | {list(), atom(), map()} | validation_error_list()] + @doc """ Validates data against a compiled schema. @@ -53,21 +56,44 @@ defmodule JSONSchex.Validator do :ok {:error, errors} -> - flat_errors = List.flatten(errors) + {:error, flatten_and_format_errors(errors, data)} + end + end + + defp flatten_and_format_errors(errors, data) do + if flat_error_list?(errors) do + errors + else + flatten_and_format_errors(errors, data, []) + end + end - formatted_errors = - Enum.map(flat_errors, fn - {path, rule, context} when is_map(context) -> - %Error{path: path, rule: rule, context: context, value: data} + # Keyword reducers commonly return an already-flat list of Error structs. A + # linear shape check avoids rebuilding that list at the public boundary. + defp flat_error_list?([]), do: true + defp flat_error_list?([%Error{} | rest]), do: flat_error_list?(rest) + defp flat_error_list?(_errors), do: false - %Error{} = e -> - e - end) + defp flatten_and_format_errors([], _data, formatted_tail), do: formatted_tail - {:error, formatted_errors} - end + # Process the remaining siblings first so prepending the current leaf retains + # the depth-first, left-to-right order produced by List.flatten/1. + defp flatten_and_format_errors([error | rest], data, formatted_tail) do + formatted_tail = flatten_and_format_errors(rest, data, formatted_tail) + flatten_and_format_error(error, data, formatted_tail) + end + + defp flatten_and_format_error(errors, data, formatted_tail) when is_list(errors), + do: flatten_and_format_errors(errors, data, formatted_tail) + + defp flatten_and_format_error({path, rule, context}, data, formatted_tail) + when is_map(context) do + [%Error{path: path, rule: rule, context: context, value: data} | formatted_tail] end + defp flatten_and_format_error(%Error{} = error, _data, formatted_tail), + do: [error | formatted_tail] + @doc """ Recursive validation engine called by the rule dispatcher. @@ -81,7 +107,7 @@ defmodule JSONSchex.Validator do {:ok, MapSet.new()} """ @spec validate_entry(Schema.t(), term(), list(), ValidationContext.t(), term()) :: - {:ok, MapSet.t()} | {:error, list(Error.t()) | String.t()} + {:ok, MapSet.t()} | {:error, validation_error_list()} def validate_entry(schema, data, path, context, initial_evaluted \\ @empty_mapset) def validate_entry(%Schema{rules: [], source_id: nil}, _data, _path, _context, evaluated) do @@ -105,7 +131,7 @@ defmodule JSONSchex.Validator do end def validate_entry( - %Schema{rules: [rule1, rule2], source_id: nil}, + %Schema{rules: [rule1, rule2] = rules, source_id: nil}, data, path, context, @@ -113,6 +139,45 @@ defmodule JSONSchex.Validator do ) do ctx = {path, evaluated, context} + case two_rule_pattern_match_cache(rule1, rule2, data) do + nil -> + run_two_rules(rule1, rule2, data, path, context, evaluated, ctx) + cache -> + run_rules_with_pattern_cache(rules, data, path, context, evaluated, ctx, [], cache) + end + end + + def validate_entry( + %Schema{rules: rules, source_id: nil}, + data, + path, + context, + initial_evaluated + ) do + ctx = {path, initial_evaluated, context} + run_rules(rules, data, path, context, initial_evaluated, ctx, []) + end + + def validate_entry( + %Schema{rules: rules} = current_schema, + data, + path, + %ValidationContext{} = root_context, + initial_evaluated + ) do + current_context = update_context_if_needed(current_schema, root_context) + initial_ctx = {path, initial_evaluated, current_context} + run_rules(rules, data, path, current_context, initial_evaluated, initial_ctx, []) + end + + defp to_error_entry(errs, _path, _rule_name, _data) when is_list(errs), do: errs + + defp to_error_entry(err_ctx, path, rule_name, data) when is_map(err_ctx), + do: [%Error{path: path, rule: rule_name, context: err_ctx, value: data}] + + # Keep the specialized two-rule execution path for ordinary schemas. Only a + # direct pair of sibling object keywords enters the cache-aware runner. + defp run_two_rules(rule1, rule2, data, path, context, evaluated, ctx) do {eval1, ctx1, err1} = case Rules.apply(rule1, data, ctx) do :ok -> @@ -122,8 +187,8 @@ defmodule JSONSchex.Validator do new_eval = MapSet.union(evaluated, new_keys) {new_eval, {path, new_eval, context}, nil} - {:error, e} -> - {evaluated, ctx, to_error_entry(e, path, rule1.name, data)} + {:error, error} -> + {evaluated, ctx, to_error_entry(error, path, rule1.name, data)} end {eval2, err2} = @@ -134,46 +199,244 @@ defmodule JSONSchex.Validator do {:ok, new_keys} -> {MapSet.union(eval1, new_keys), nil} - {:error, e} -> - {eval1, to_error_entry(e, path, rule2.name, data)} + {:error, error} -> + {eval1, to_error_entry(error, path, rule2.name, data)} end case {err1, err2} do {nil, nil} -> {:ok, eval2} - {e, nil} -> {:error, e} - {nil, e} -> {:error, e} - {e1, e2} -> {:error, [e2 | [e1]]} + {error, nil} -> {:error, error} + {nil, error} -> {:error, error} + {error1, error2} -> {:error, [error2 | [error1]]} end end - def validate_entry( - %Schema{rules: rules, source_id: nil}, - data, - path, - context, - initial_evaluated - ) do - ctx = {path, initial_evaluated, context} - run_rules(rules, data, path, context, initial_evaluated, ctx, []) + + defp run_rules_with_pattern_cache([], _data, _path, _context, evaluated, _ctx, [], _cache) do + {:ok, evaluated} end - def validate_entry( - %Schema{rules: rules} = current_schema, - data, - path, - %ValidationContext{} = root_context, - initial_evaluated - ) do - current_context = update_context_if_needed(current_schema, root_context) + defp run_rules_with_pattern_cache([], _data, _path, _context, _evaluated, _ctx, errors, _cache) do + {:error, errors} + end - initial_ctx = {path, initial_evaluated, current_context} - run_rules(rules, data, path, current_context, initial_evaluated, initial_ctx, []) + defp run_rules_with_pattern_cache([rule | rest], data, path, context, evaluated, ctx, errors, cache) do + {result, updated_cache} = Rules.apply(rule, data, ctx, cache) + complete? = pattern_match_cache_complete?(rule, updated_cache) + + case result do + :ok -> + continue_cached_rules( + complete?, + rest, + data, + path, + context, + evaluated, + ctx, + errors, + updated_cache + ) + + {:ok, new_eval_keys} -> + cond do + MapSet.size(new_eval_keys) == 0 -> + continue_cached_rules( + complete?, + rest, + data, + path, + context, + evaluated, + ctx, + errors, + updated_cache + ) + + MapSet.size(evaluated) == 0 -> + new_ctx = {path, new_eval_keys, context} + + continue_cached_rules( + complete?, + rest, + data, + path, + context, + new_eval_keys, + new_ctx, + errors, + updated_cache + ) + + true -> + new_evaluated = MapSet.union(evaluated, new_eval_keys) + new_ctx = {path, new_evaluated, context} + + continue_cached_rules( + complete?, + rest, + data, + path, + context, + new_evaluated, + new_ctx, + errors, + updated_cache + ) + end + + {:error, new_errs} when is_list(new_errs) -> + continue_cached_rules( + complete?, + rest, + data, + path, + context, + evaluated, + ctx, + [new_errs | errors], + updated_cache + ) + + {:error, err_context} when is_map(err_context) -> + error = %Error{path: path, rule: rule.name, context: err_context, value: data} + + continue_cached_rules( + complete?, + rest, + data, + path, + context, + evaluated, + ctx, + [error | errors], + updated_cache + ) + end end - defp to_error_entry(errs, _path, _rule_name, _data) when is_list(errs), do: errs + defp continue_cached_rules(true, rest, data, path, context, evaluated, ctx, errors, _cache) do + run_rules(rest, data, path, context, evaluated, ctx, errors) + end - defp to_error_entry(err_ctx, path, rule_name, data) when is_map(err_ctx), - do: [%Error{path: path, rule: rule_name, context: err_ctx, value: data}] + defp continue_cached_rules(false, rest, data, path, context, evaluated, ctx, errors, cache) do + run_rules_with_pattern_cache(rest, data, path, context, evaluated, ctx, errors, cache) + end + + defp pattern_match_cache_complete?( + %Rule{name: :additionalProperties}, + %{source: :pattern_properties} + ), + do: true + + defp pattern_match_cache_complete?( + %Rule{name: :patternProperties}, + %{source: :additional_properties} + ), + do: true + + defp pattern_match_cache_complete?(_rule, _cache), do: false + + defp two_rule_pattern_match_cache( + %Rule{name: first_name} = first_rule, + %Rule{name: second_name} = second_rule, + data + ) + when first_name in [:patternProperties, :additionalProperties] and + second_name in [:patternProperties, :additionalProperties] and + first_name != second_name do + pattern_match_cache([first_rule, second_rule], data) + end + + defp two_rule_pattern_match_cache(_first_rule, _second_rule, _data), do: nil + + # A cache map cannot repay its setup work when the two sibling rules would + # repeat exactly one match check. All larger candidate workloads stay eligible. + defp pattern_match_cache( + [%Rule{name: :patternProperties, params: [_single_pattern]} | _rest], + data + ) + when is_map(data) and map_size(data) == 1, + do: nil + + defp pattern_match_cache( + [%Rule{name: :additionalProperties, params: %{patterns: [_single_pattern]}} | _rest], + data + ) + when is_map(data) and map_size(data) == 1, + do: nil + + # This cache records structural pattern matches only. It stays local to this + # validate_entry/5 call, so child schemas and later validations cannot reuse + # annotations, errors, reference state, or match facts from this object. + defp pattern_match_cache(rules, data) when is_map(data) and map_size(data) > 0 do + case sibling_pattern_rules(rules) do + {%Rule{params: compiled_patterns}, + %Rule{ + params: %{ + patterns: patterns, + known_props: %MapSet{} = known_props, + schema: _schema, + always_valid?: always_valid? + } + }} + when is_list(compiled_patterns) and is_list(patterns) and is_boolean(always_valid?) -> + if compiled_patterns != [] and + same_pattern_sequence?(compiled_patterns, patterns) and + has_additional_candidate?(data, known_props) do + %{source: nil, matches: %{}} + end + + _ -> + nil + end + end + + defp pattern_match_cache(_rules, _data), do: nil + + defp sibling_pattern_rules(rules) do + Enum.reduce_while(rules, {nil, nil}, fn + %Rule{name: :patternProperties} = rule, {nil, additional_rule} -> + {:cont, {rule, additional_rule}} + + %Rule{name: :patternProperties}, _ -> + # Duplicated + {:halt, nil} + + %Rule{name: :additionalProperties} = rule, {pattern_rule, nil} -> + {:cont, {pattern_rule, rule}} + + %Rule{name: :additionalProperties}, _ -> + # Duplicated + {:halt, nil} + + _rule, acc -> + {:cont, acc} + end) + |> case do + {%Rule{} = pattern_rule, %Rule{} = additional_rule} -> + {pattern_rule, additional_rule} + _ -> + nil + end + end + + defp same_pattern_sequence?([], []), do: true + + defp same_pattern_sequence?([{regex, _schema} | compiled_rest], [pattern | patterns]) + when regex === pattern do + same_pattern_sequence?(compiled_rest, patterns) + end + + defp same_pattern_sequence?(_, _), do: false + + defp has_additional_candidate?(data, known_props) do + if map_size(data) > MapSet.size(known_props) do + true + else + Enum.any?(data, fn {key, _value} -> not MapSet.member?(known_props, key) end) + end + end defp run_rules([], _data, _path, _context, evaluated, _ctx, []) do {:ok, evaluated} @@ -183,6 +446,33 @@ defmodule JSONSchex.Validator do {:error, errors} end + defp run_rules( + [%Rule{name: name} = rule | rest], + data, + path, + context, + evaluated, + ctx, + errors + ) + when name in [:patternProperties, :additionalProperties] do + case pattern_match_cache([rule | rest], data) do + nil -> + run_rules_without_pattern_match_cache( + [rule | rest], + data, + path, + context, + evaluated, + ctx, + errors + ) + + cache -> + run_rules_with_pattern_cache([rule | rest], data, path, context, evaluated, ctx, errors, cache) + end + end + defp run_rules([rule | rest], data, path, context, evaluated, ctx, errors) do case Rules.apply(rule, data, ctx) do :ok -> @@ -212,6 +502,86 @@ defmodule JSONSchex.Validator do end end + defp run_rules_without_pattern_match_cache([], _data, _path, _context, evaluated, _ctx, []) do + {:ok, evaluated} + end + + defp run_rules_without_pattern_match_cache( + [], + _data, + _path, + _context, + _evaluated, + _ctx, + errors + ) do + {:error, errors} + end + + defp run_rules_without_pattern_match_cache([rule | rest], data, path, context, evaluated, ctx, errors) do + case Rules.apply(rule, data, ctx) do + :ok -> + run_rules_without_pattern_match_cache(rest, data, path, context, evaluated, ctx, errors) + + {:ok, new_eval_keys} -> + cond do + MapSet.size(new_eval_keys) == 0 -> + run_rules_without_pattern_match_cache(rest, data, path, context, evaluated, ctx, errors) + + MapSet.size(evaluated) == 0 -> + new_ctx = {path, new_eval_keys, context} + + run_rules_without_pattern_match_cache( + rest, + data, + path, + context, + new_eval_keys, + new_ctx, + errors + ) + + true -> + new_evaluated = MapSet.union(evaluated, new_eval_keys) + new_ctx = {path, new_evaluated, context} + + run_rules_without_pattern_match_cache( + rest, + data, + path, + context, + new_evaluated, + new_ctx, + errors + ) + end + + {:error, new_errs} when is_list(new_errs) -> + run_rules_without_pattern_match_cache( + rest, + data, + path, + context, + evaluated, + ctx, + [new_errs | errors] + ) + + {:error, err_context} when is_map(err_context) -> + error = %Error{path: path, rule: rule.name, context: err_context, value: data} + + run_rules_without_pattern_match_cache( + rest, + data, + path, + context, + evaluated, + ctx, + [error | errors] + ) + end + end + defp update_context_if_needed(%{source_id: nil}, %ValidationContext{} = root_context) do root_context end diff --git a/lib/jsonschex/validator/keywords.ex b/lib/jsonschex/validator/keywords.ex index 4eb7ea4..257e433 100644 --- a/lib/jsonschex/validator/keywords.ex +++ b/lib/jsonschex/validator/keywords.ex @@ -280,7 +280,11 @@ defmodule JSONSchex.Validator.Keywords do Validates data against schemas in an `anyOf` array. """ def validate_anyOf(data, schemas, path, root, evaluated) do - reduce_anyOf(schemas, data, path, root, evaluated, 0, [], []) + if any_of_direct_merge_candidate?(schemas, evaluated) do + reduce_anyOf_with_shared_evaluated_keys(schemas, data, path, root, evaluated, 0, 0, [], []) + else + reduce_anyOf(schemas, data, path, root, evaluated, 0, [], []) + end end defp reduce_anyOf([], _data, _path, _root, _evaluated, count, merged_keys, _error_lists) @@ -316,6 +320,128 @@ defmodule JSONSchex.Validator.Keywords do end end + # The specialized reducer is reachable only for a sufficiently large incoming + # evaluated-key set and at least four branches. It preserves the input supplied + # to every child and selects direct merging only after all branches run. + defp any_of_direct_merge_candidate?([_, _, _, _ | _], evaluated), + do: MapSet.size(evaluated) >= 8 + + defp any_of_direct_merge_candidate?(_schemas, _evaluated), do: false + + defp reduce_anyOf_with_shared_evaluated_keys( + [], + _data, + _path, + _root, + evaluated, + count, + total_evaluated_key_count, + evaluated_key_sets, + _error_lists + ) + when count > 0 do + {:ok, + merge_any_of_evaluated_key_sets( + evaluated_key_sets, + evaluated, + count, + total_evaluated_key_count + )} + end + + defp reduce_anyOf_with_shared_evaluated_keys( + [], + _data, + _path, + _root, + _evaluated, + 0, + _total_evaluated_key_count, + _evaluated_key_sets, + error_lists + ) do + {:error, List.flatten(Enum.reverse(error_lists))} + end + + defp reduce_anyOf_with_shared_evaluated_keys( + [schema | rest], + data, + path, + root, + evaluated, + count, + total_evaluated_key_count, + evaluated_key_sets, + acc_errs + ) do + case Validator.validate_entry(schema, data, path, root, evaluated) do + {:ok, keys} -> + key_count = MapSet.size(keys) + + if key_count == 0 do + reduce_anyOf_with_shared_evaluated_keys( + rest, + data, + path, + root, + evaluated, + count + 1, + total_evaluated_key_count, + evaluated_key_sets, + acc_errs + ) + else + new_count = count + 1 + new_total_evaluated_key_count = total_evaluated_key_count + key_count + new_evaluated_key_sets = [keys | evaluated_key_sets] + + if any_of_incoming_evaluated_keys_dominate_successful_prefix?( + evaluated, + new_count, + new_total_evaluated_key_count + ) do + reduce_anyOf_with_shared_evaluated_keys( + rest, + data, + path, + root, + evaluated, + new_count, + new_total_evaluated_key_count, + new_evaluated_key_sets, + acc_errs + ) + else + reduce_anyOf( + rest, + data, + path, + root, + evaluated, + new_count, + evaluated_key_sets_to_lists(new_evaluated_key_sets), + acc_errs + ) + end + end + + {:error, errs} -> + new_errs = if count == 0, do: [errs | acc_errs], else: acc_errs + + reduce_anyOf_with_shared_evaluated_keys( + rest, + data, + path, + root, + evaluated, + count, + total_evaluated_key_count, + evaluated_key_sets, + new_errs + ) + end + end + @doc """ Validates data against schemas in a `oneOf` array. Exactly one schema must pass. @@ -427,6 +553,268 @@ defmodule JSONSchex.Validator.Keywords do def validate_pattern_properties(_, _, _, _), do: :ok + @doc false + def validate_pattern_properties( + data, + compiled_patterns, + path, + root, + %{source: source} = cache + ) + when is_map(data) and source in [nil, :pattern_properties] do + cache = if source == nil, do: %{cache | source: :pattern_properties}, else: cache + + reduce_patterns_with_match_cache( + compiled_patterns, + Map.to_list(data), + path, + root, + [], + [], + cache + ) + end + + def validate_pattern_properties( + data, + compiled_patterns, + path, + root, + %{source: :additional_properties} = cache + ) + when is_map(data) do + reduce_patterns_after_additional( + compiled_patterns, + Map.to_list(data), + path, + root, + 0, + [], + [], + cache + ) + end + + def validate_pattern_properties(data, compiled_patterns, path, root, cache) do + {validate_pattern_properties(data, compiled_patterns, path, root), cache} + end + + defp reduce_patterns_with_match_cache([], _data_list, _path, _root, [], [], cache), + do: {:ok, cache} + + defp reduce_patterns_with_match_cache([], _data_list, _path, _root, [], eval_keys, cache), + do: {{:ok, MapSet.new(eval_keys)}, cache} + + defp reduce_patterns_with_match_cache([], _data_list, _path, _root, errs, _eval_keys, cache), + do: {{:error, List.flatten(errs)}, cache} + + defp reduce_patterns_with_match_cache( + [{regex, schema} | rest_patterns], + data_list, + path, + root, + errs, + eval_keys, + cache + ) do + {errs, eval_keys, cache} = + reduce_pattern_data_with_match_cache(data_list, regex, schema, path, root, errs, eval_keys, cache) + + reduce_patterns_with_match_cache(rest_patterns, data_list, path, root, errs, eval_keys, cache) + end + + defp reduce_pattern_data_with_match_cache([], _regex, _schema, _path, _root, errs, eval_keys, cache), + do: {errs, eval_keys, cache} + + defp reduce_pattern_data_with_match_cache( + [{key, val} | rest], + regex, + schema, + path, + root, + errs, + eval_keys, + cache + ) do + if Regex.match?(regex, key) do + cache = mark_pattern_match(cache, key) + + case Validator.validate_entry(schema, val, [key | path], root) do + {:error, new_errs} -> + reduce_pattern_data_with_match_cache( + rest, + regex, + schema, + path, + root, + [new_errs | errs], + eval_keys, + cache + ) + + {:ok, _} -> + reduce_pattern_data_with_match_cache( + rest, + regex, + schema, + path, + root, + errs, + [key | eval_keys], + cache + ) + end + else + reduce_pattern_data_with_match_cache( + rest, + regex, + schema, + path, + root, + errs, + eval_keys, + cache + ) + end + end + + # additionalProperties ran first and already checked each non-property key up + # to its first match. Preserve pattern-major validation while skipping only + # those regex calls whose result is already known. + defp reduce_patterns_after_additional([], _data_list, _path, _root, _index, [], [], cache), + do: {:ok, cache} + + defp reduce_patterns_after_additional([], _data_list, _path, _root, _index, [], eval_keys, cache), + do: {{:ok, MapSet.new(eval_keys)}, cache} + + defp reduce_patterns_after_additional([], _data_list, _path, _root, _index, errs, _eval_keys, cache), + do: {{:error, List.flatten(errs)}, cache} + + defp reduce_patterns_after_additional( + [{regex, schema} | rest_patterns], + data_list, + path, + root, + index, + errs, + eval_keys, + cache + ) do + {errs, eval_keys} = + reduce_pattern_data_after_additional( + data_list, + regex, + schema, + path, + root, + index, + errs, + eval_keys, + cache + ) + + reduce_patterns_after_additional( + rest_patterns, + data_list, + path, + root, + index + 1, + errs, + eval_keys, + cache + ) + end + + defp reduce_pattern_data_after_additional( + [], + _regex, + _schema, + _path, + _root, + _index, + errs, + eval_keys, + _cache + ), + do: {errs, eval_keys} + + defp reduce_pattern_data_after_additional( + [{key, val} | rest], + regex, + schema, + path, + root, + index, + errs, + eval_keys, + cache + ) do + if pattern_matches_after_additional?(cache, key, index, regex) do + case Validator.validate_entry(schema, val, [key | path], root) do + {:error, new_errs} -> + reduce_pattern_data_after_additional( + rest, + regex, + schema, + path, + root, + index, + [new_errs | errs], + eval_keys, + cache + ) + + {:ok, _} -> + reduce_pattern_data_after_additional( + rest, + regex, + schema, + path, + root, + index, + errs, + [key | eval_keys], + cache + ) + end + else + reduce_pattern_data_after_additional( + rest, + regex, + schema, + path, + root, + index, + errs, + eval_keys, + cache + ) + end + end + + defp pattern_matches_after_additional?(%{matches: matches}, key, index, regex) do + case Map.fetch(matches, key) do + {:ok, :no_match} -> + false + + {:ok, {:first_match, ^index}} -> + true + + {:ok, {:first_match, first_index}} when index < first_index -> + false + + _ -> + Regex.match?(regex, key) + end + end + + defp mark_pattern_match(%{matches: matches} = cache, key) do + case matches do + %{^key => _match} -> cache + _ -> %{cache | matches: Map.put(matches, key, :matched)} + end + end + defp reduce_patterns([], _data_list, _path, _root, [], []), do: :ok defp reduce_patterns([], _data_list, _path, _root, [], eval_keys), @@ -469,6 +857,253 @@ defmodule JSONSchex.Validator.Keywords do def validate_additional_properties(_, _, _, _, _, _), do: :ok + @doc false + def validate_additional_properties_with_match_cache( + data, + schema, + known_props_set, + patterns, + path, + root, + %{source: source} = cache + ) + when is_map(data) and source in [nil, :additional_properties] do + cache = if source == nil, do: %{cache | source: :additional_properties}, else: cache + + reduce_additional_with_match_cache( + Map.to_list(data), + schema, + known_props_set, + patterns, + path, + root, + [], + [], + cache + ) + end + + def validate_additional_properties_with_match_cache( + data, + schema, + known_props_set, + _patterns, + path, + root, + %{source: :pattern_properties} = cache + ) + when is_map(data) do + reduce_additional_after_pattern( + Map.to_list(data), + schema, + known_props_set, + path, + root, + [], + [], + cache + ) + end + + def validate_additional_properties_with_match_cache( + data, + schema, + known_props_set, + patterns, + path, + root, + cache + ) do + {validate_additional_properties(data, schema, known_props_set, patterns, path, root), cache} + end + + defp reduce_additional_with_match_cache( + [], + _schema, + _known, + _patterns, + _path, + _root, + [], + [], + cache + ), + do: {:ok, cache} + + defp reduce_additional_with_match_cache( + [], + _schema, + _known, + _patterns, + _path, + _root, + [], + eval_keys, + cache + ), + do: {{:ok, MapSet.new(eval_keys)}, cache} + + defp reduce_additional_with_match_cache( + [], + _schema, + _known, + _patterns, + _path, + _root, + errs, + _eval_keys, + cache + ), + do: {{:error, List.flatten(errs)}, cache} + + defp reduce_additional_with_match_cache( + [{key, val} | rest], + schema, + known, + patterns, + path, + root, + errs, + eval_keys, + cache + ) do + if MapSet.member?(known, key) do + reduce_additional_with_match_cache( + rest, + schema, + known, + patterns, + path, + root, + errs, + eval_keys, + cache + ) + else + {classification, cache} = classify_additional_pattern(cache, key, patterns) + + case classification do + :no_match -> + case Validator.validate_entry(schema, val, [key | path], root) do + {:error, new_errs} -> + reduce_additional_with_match_cache( + rest, + schema, + known, + patterns, + path, + root, + [new_errs | errs], + eval_keys, + cache + ) + + {:ok, _} -> + reduce_additional_with_match_cache( + rest, + schema, + known, + patterns, + path, + root, + errs, + [key | eval_keys], + cache + ) + end + + {:first_match, _index} -> + reduce_additional_with_match_cache( + rest, + schema, + known, + patterns, + path, + root, + errs, + eval_keys, + cache + ) + end + end + end + + defp reduce_additional_after_pattern([], _schema, _known, _path, _root, [], [], cache), + do: {:ok, cache} + + defp reduce_additional_after_pattern([], _schema, _known, _path, _root, [], eval_keys, cache), + do: {{:ok, MapSet.new(eval_keys)}, cache} + + defp reduce_additional_after_pattern([], _schema, _known, _path, _root, errs, _eval_keys, cache), + do: {{:error, List.flatten(errs)}, cache} + + defp reduce_additional_after_pattern( + [{key, val} | rest], + schema, + known, + path, + root, + errs, + eval_keys, + %{matches: matches} = cache + ) do + cond do + MapSet.member?(known, key) -> + reduce_additional_after_pattern(rest, schema, known, path, root, errs, eval_keys, cache) + + Map.has_key?(matches, key) -> + reduce_additional_after_pattern(rest, schema, known, path, root, errs, eval_keys, cache) + + true -> + case Validator.validate_entry(schema, val, [key | path], root) do + {:error, new_errs} -> + reduce_additional_after_pattern( + rest, + schema, + known, + path, + root, + [new_errs | errs], + eval_keys, + cache + ) + + {:ok, _} -> + reduce_additional_after_pattern( + rest, + schema, + known, + path, + root, + errs, + [key | eval_keys], + cache + ) + end + end + end + + defp classify_additional_pattern(%{matches: matches} = cache, key, patterns) do + case Map.fetch(matches, key) do + {:ok, classification} -> + {classification, cache} + + :error -> + classification = first_matching_pattern(patterns, key, 0) + {classification, %{cache | matches: Map.put(matches, key, classification)}} + end + end + + defp first_matching_pattern([], _key, _index), do: :no_match + + defp first_matching_pattern([regex | rest], key, index) do + if Regex.match?(regex, key) do + {:first_match, index} + else + first_matching_pattern(rest, key, index + 1) + end + end + defp reduce_additional([], _schema, _known, _patterns, _path, _root, [], []), do: :ok defp reduce_additional([], _schema, _known, _patterns, _path, _root, [], eval_keys), @@ -524,6 +1159,90 @@ defmodule JSONSchex.Validator.Keywords do def collect_additional_keys(_, _, _), do: :ok + @doc false + def collect_additional_keys_with_match_cache( + data, + known_props_set, + patterns, + %{source: source} = cache + ) + when is_map(data) and source in [nil, :additional_properties] do + cache = if source == nil, do: %{cache | source: :additional_properties}, else: cache + + reduce_collect_additional_with_match_cache( + Map.to_list(data), + known_props_set, + patterns, + [], + cache + ) + end + + def collect_additional_keys_with_match_cache( + data, + known_props_set, + _patterns, + %{source: :pattern_properties} = cache + ) + when is_map(data) do + reduce_collect_additional_after_pattern(Map.to_list(data), known_props_set, [], cache) + end + + def collect_additional_keys_with_match_cache(data, known_props_set, patterns, cache) do + {collect_additional_keys(data, known_props_set, patterns), cache} + end + + defp reduce_collect_additional_with_match_cache([], _known, _patterns, [], cache), + do: {:ok, cache} + + defp reduce_collect_additional_with_match_cache([], _known, _patterns, eval_keys, cache), + do: {{:ok, MapSet.new(eval_keys)}, cache} + + defp reduce_collect_additional_with_match_cache( + [{key, _val} | rest], + known, + patterns, + eval_keys, + cache + ) do + if MapSet.member?(known, key) do + reduce_collect_additional_with_match_cache(rest, known, patterns, eval_keys, cache) + else + {classification, cache} = classify_additional_pattern(cache, key, patterns) + + case classification do + :no_match -> + reduce_collect_additional_with_match_cache(rest, known, patterns, [key | eval_keys], cache) + + {:first_match, _index} -> + reduce_collect_additional_with_match_cache(rest, known, patterns, eval_keys, cache) + end + end + end + + defp reduce_collect_additional_after_pattern([], _known, [], cache), do: {:ok, cache} + + defp reduce_collect_additional_after_pattern([], _known, eval_keys, cache), + do: {{:ok, MapSet.new(eval_keys)}, cache} + + defp reduce_collect_additional_after_pattern( + [{key, _val} | rest], + known, + eval_keys, + %{matches: matches} = cache + ) do + cond do + MapSet.member?(known, key) -> + reduce_collect_additional_after_pattern(rest, known, eval_keys, cache) + + Map.has_key?(matches, key) -> + reduce_collect_additional_after_pattern(rest, known, eval_keys, cache) + + true -> + reduce_collect_additional_after_pattern(rest, known, [key | eval_keys], cache) + end + end + defp reduce_collect_additional([], _known, _patterns, []), do: :ok defp reduce_collect_additional([], _known, _patterns, eval_keys), @@ -677,6 +1396,89 @@ defmodule JSONSchex.Validator.Keywords do end end + # Retain the existing flatten-and-rebuild semantics when direct union would + # otherwise change historical recursive handling of list-valued Elixir keys. + defp merge_evaluated_key_sets([]), do: @empty_mapset + + defp merge_evaluated_key_sets(evaluated_key_sets) do + evaluated_key_sets + |> Enum.map(&MapSet.to_list/1) + |> List.flatten() + |> MapSet.new() + end + + # Every successful anyOf child receives the same incoming evaluated keys. + # Direct pairwise union helps only when that known shared portion dominates. + # The fallback deliberately retains the historical recursive-flatten behavior. + defp merge_any_of_evaluated_key_sets( + evaluated_key_sets, + evaluated, + count, + total_evaluated_key_count + ) do + if any_of_shared_evaluated_keys_dominate?(evaluated, count, total_evaluated_key_count) do + merged_evaluated_keys = balanced_union_evaluated_key_sets(evaluated_key_sets) + + if list_free_evaluated_key_set?(merged_evaluated_keys) do + merged_evaluated_keys + else + merge_evaluated_key_sets(evaluated_key_sets) + end + else + merge_evaluated_key_sets(evaluated_key_sets) + end + end + + # The benchmark-derived gate requires known shared evaluated keys to make up + # at least two thirds of successful child evaluated-key memberships before + # unioning them. + defp any_of_shared_evaluated_keys_dominate?(evaluated, count, total_evaluated_key_count) do + count >= 4 and + any_of_incoming_evaluated_keys_dominate_successful_prefix?( + evaluated, + count, + total_evaluated_key_count + ) + end + + # Once the processed successful-branch prefix is not dominated by incoming + # evaluated keys, resume the original reducer rather than retaining raw + # MapSets in hopes that a later branch restores the average. That conservative + # choice keeps the fallback near its original allocation profile. + defp any_of_incoming_evaluated_keys_dominate_successful_prefix?( + evaluated, + count, + total_evaluated_key_count + ) do + shared_evaluated_key_count = MapSet.size(evaluated) * count + 3 * shared_evaluated_key_count >= 2 * total_evaluated_key_count + end + + defp evaluated_key_sets_to_lists(evaluated_key_sets), + do: Enum.map(evaluated_key_sets, &MapSet.to_list/1) + + defp list_free_evaluated_key_set?(evaluated_keys) do + Enum.all?(evaluated_keys, &(not is_list(&1))) + end + + defp balanced_union_evaluated_key_sets([]), do: @empty_mapset + defp balanced_union_evaluated_key_sets([evaluated_keys]), do: evaluated_keys + + defp balanced_union_evaluated_key_sets(evaluated_key_sets) do + evaluated_key_sets + |> union_evaluated_key_pairs([]) + |> balanced_union_evaluated_key_sets() + end + + defp union_evaluated_key_pairs([], acc), do: Enum.reverse(acc) + + defp union_evaluated_key_pairs([evaluated_keys], acc), + do: Enum.reverse([evaluated_keys | acc]) + + defp union_evaluated_key_pairs([left, right | rest], acc) do + union_evaluated_key_pairs(rest, [MapSet.union(left, right) | acc]) + end + @doc """ Validates content against a schema after decoding (`contentSchema`). Only active when `content_assertion: true`. diff --git a/lib/jsonschex/validator/rules.ex b/lib/jsonschex/validator/rules.ex index 077af31..bf693d0 100644 --- a/lib/jsonschex/validator/rules.ex +++ b/lib/jsonschex/validator/rules.ex @@ -116,11 +116,9 @@ defmodule JSONSchex.Validator.Rules do def apply(%Rule{name: :required, params: required}, data, {path, _, _}) do if is_map(data) do - if Enum.all?(required, &Map.has_key?(data, &1)) do - :ok - else - missing = Enum.reject(required, &Map.has_key?(data, &1)) - {:error, [%Error{path: path, rule: :required, context: %ErrorContext{contrast: missing}}]} + case Enum.reject(required, &Map.has_key?(data, &1)) do + [] -> :ok + missing -> {:error, [%Error{path: path, rule: :required, context: %ErrorContext{contrast: missing}}]} end else :ok @@ -326,6 +324,57 @@ defmodule JSONSchex.Validator.Rules do raise ArgumentError, "unsupported compiled rule: #{inspect(name)}" end + @doc false + @spec apply(Rule.t(), term(), validation_context(), map()) :: {result(), map()} + def apply( + %Rule{name: :patternProperties, params: compiled_patterns}, + data, + {path, _evaluated, root}, + cache + ) do + Keywords.validate_pattern_properties( + data, + compiled_patterns, + path, + root, + cache + ) + end + + def apply( + %Rule{ + name: :additionalProperties, + params: %{known_props: known_props, patterns: patterns, always_valid?: true} + }, + data, + _ctx, + cache + ) do + Keywords.collect_additional_keys_with_match_cache(data, known_props, patterns, cache) + end + + def apply( + %Rule{ + name: :additionalProperties, + params: %{schema: compiled_sub, known_props: known_props, patterns: patterns} + }, + data, + {path, _evaluated, root}, + cache + ) do + Keywords.validate_additional_properties_with_match_cache( + data, + compiled_sub, + known_props, + patterns, + path, + root, + cache + ) + end + + def apply(rule, data, ctx, cache), do: {__MODULE__.apply(rule, data, ctx), cache} + defp merge_dependency_results(:ok, :ok), do: :ok defp merge_dependency_results(:ok, {:ok, evaluated}), do: {:ok, evaluated} defp merge_dependency_results(:ok, {:error, errs}), do: {:error, errs} diff --git a/test/compiler_regex_cache_test.exs b/test/compiler_regex_cache_test.exs new file mode 100644 index 0000000..46f00f3 --- /dev/null +++ b/test/compiler_regex_cache_test.exs @@ -0,0 +1,136 @@ +defmodule JSONSchex.Test.CompilerRegexCacheTest do + use ExUnit.Case, async: true + + alias JSONSchex.Types.{Error, Schema} + + @draft2020_12 "https://json-schema.org/draft/2020-12/schema" + @core_vocabulary "https://json-schema.org/draft/2020-12/vocab/core" + + test "sibling rules retain compiled regex order and object validation semantics" do + schema = %{ + "properties" => %{"fixed" => %{"type" => "integer"}}, + "patternProperties" => %{ + "^x-" => %{"type" => "string"}, + "^count-" => %{"type" => "integer"} + }, + "additionalProperties" => false + } + + assert {:ok, compiled} = JSONSchex.compile(schema) + pattern_rule = rule(compiled, :patternProperties) + additional_rule = rule(compiled, :additionalProperties) + pattern_regexes = Enum.map(pattern_rule.params, &elem(&1, 0)) + + assert Enum.all?(pattern_regexes, &match?(%Regex{}, &1)) + assert pattern_regexes === additional_rule.params.patterns + assert additional_rule.params.known_props == MapSet.new(["fixed"]) + refute additional_rule.params.always_valid? + + assert :ok == + JSONSchex.validate(compiled, %{ + "fixed" => 1, + "x-name" => "alice", + "count-total" => 2 + }) + + assert {:error, pattern_errors} = JSONSchex.validate(compiled, %{"x-name" => 1}) + assert Enum.any?(pattern_errors, &(&1.rule == :type and &1.path == ["x-name"])) + refute Enum.any?(pattern_errors, &(&1.rule == :boolean_schema and &1.path == ["x-name"])) + + assert {:error, additional_errors} = JSONSchex.validate(compiled, %{"unmatched" => true}) + assert Enum.any?(additional_errors, &(&1.rule == :boolean_schema and &1.path == ["unmatched"])) + end + + test "always-valid additionalProperties retains evaluated-property annotations" do + schema = %{ + "patternProperties" => %{"^x-" => %{"type" => "string"}}, + "additionalProperties" => true, + "unevaluatedProperties" => false + } + + assert {:ok, compiled} = JSONSchex.compile(schema) + assert rule(compiled, :additionalProperties).params.always_valid? + + assert :ok == JSONSchex.validate(compiled, %{"x-name" => "alice", "other" => %{}}) + end + + test "inactive applicator vocabulary does not prepare sibling regexes" do + schema = %{ + "$schema" => @draft2020_12, + "$vocabulary" => %{@core_vocabulary => true}, + "patternProperties" => %{"[" => true}, + "additionalProperties" => false + } + + assert {:ok, %Schema{rules: []}} = JSONSchex.compile(schema) + end + + test "local definitions still fail before sibling pattern regex preparation" do + schema = %{ + "$defs" => %{"invalid" => %{"pattern" => "["}}, + "patternProperties" => %{"[" => true}, + "additionalProperties" => false + } + + assert {:error, %Error{} = error} = JSONSchex.compile(schema) + assert error.rule == :invalid_regex + assert error.path == ["$defs", "invalid", "pattern"] + assert List.starts_with?(error.context.error_detail, ~c"missing terminating ]") + end + + test "sibling child schema errors retain their existing path contracts" do + pattern_child_error = %{ + "patternProperties" => %{"^x-" => %{"minLength" => -1}}, + "additionalProperties" => false + } + + additional_child_error = %{ + "patternProperties" => %{"^x-" => true}, + "additionalProperties" => %{"minLength" => -1} + } + + assert {:error, %Error{} = pattern_error} = JSONSchex.compile(pattern_child_error) + assert pattern_error.path == ["minLength", "^x-"] + + assert {:error, %Error{} = additional_error} = JSONSchex.compile(additional_child_error) + assert additional_error.path == ["minLength"] + end + + @tag :performance + test "sibling regex compilation stays near patternProperties-only reductions" do + patterns_only = wide_pattern_schema(256, false) + siblings = wide_pattern_schema(256, true) + + # Load compiler and regex machinery before measuring process-local work. + assert {:ok, _} = JSONSchex.compile(patterns_only) + assert {:ok, _} = JSONSchex.compile(siblings) + + patterns_only_work = compile_reductions(patterns_only) + sibling_work = compile_reductions(siblings) + + # The pre-P6 compiler used about 1.8x work here because it compiled every + # pattern twice. The small allowance covers the additional rule itself. + assert sibling_work * 100 < patterns_only_work * 145, + "siblings used #{sibling_work} vs #{patterns_only_work} reductions" + end + + defp rule(%Schema{rules: rules}, name), do: Enum.find(rules, &(&1.name == name)) + + defp wide_pattern_schema(width, additional?) do + patterns = + Map.new(1..width, fn index -> + {Integer.to_string(index) <> "\\p{Letter}\\s", true} + end) + + schema = %{"patternProperties" => patterns} + if additional?, do: Map.put(schema, "additionalProperties", true), else: schema + end + + defp compile_reductions(schema) do + :erlang.garbage_collect() + {:reductions, before} = Process.info(self(), :reductions) + assert {:ok, %Schema{}} = JSONSchex.compile(schema) + {:reductions, after_compile} = Process.info(self(), :reductions) + after_compile - before + end +end diff --git a/test/draft202012_compiled_defs_registry_test.exs b/test/draft202012_compiled_defs_registry_test.exs new file mode 100644 index 0000000..9188240 --- /dev/null +++ b/test/draft202012_compiled_defs_registry_test.exs @@ -0,0 +1,100 @@ +defmodule JSONSchex.Test.Draft202012CompiledDefsRegistryTest do + use ExUnit.Case, async: true + + alias JSONSchex.{Compiler, ScopeScanner} + alias JSONSchex.Draft202012.Schemas + alias JSONSchex.Types.Schema + + @base_uri "https://json-schema.org/draft/2020-12" + @schema_uri @base_uri <> "/schema" + @family_uris [ + @schema_uri, + @base_uri <> "/meta/core", + @base_uri <> "/meta/applicator", + @base_uri <> "/meta/unevaluated", + @base_uri <> "/meta/validation", + @base_uri <> "/meta/meta-data", + @base_uri <> "/meta/format-annotation", + @base_uri <> "/meta/format-assertion", + @base_uri <> "/meta/content" + ] + + test "compiled family registry retains every canonical resource and dynamic meta anchor" do + registries = Enum.map(@family_uris, &Schemas.compiled_defs/1) + [registry | remaining_registries] = registries + + assert map_size(registry) == 26 + assert Enum.all?(remaining_registries, &(&1 === registry)) + + for uri <- @family_uris do + assert {:ok, raw_schema} = Schemas.fetch(uri) + assert %Schema{} = resource = Map.fetch!(registry, uri) + assert resource.source_id == uri + assert resource.raw === raw_schema + assert Map.fetch!(registry, uri <> "#meta") === resource + end + end + + test "compiled defs already contain every identity found by a second scope scan" do + for uri <- @family_uris do + assert {:ok, raw_schema} = Schemas.fetch(uri) + assert {:ok, compiled_schema} = Compiler.compile(raw_schema, base_uri: uri) + {scanned_defs, _refs} = ScopeScanner.scan(raw_schema) + + compiler_registry = + %{} + |> Map.put(uri, compiled_schema) + |> Map.merge(compiled_schema.defs || %{}) + + registry_with_second_scan = + Enum.reduce(scanned_defs, compiler_registry, fn {scanned_uri, _raw_schema}, acc -> + Map.put_new(acc, scanned_uri, compiled_schema) + end) + + assert registry_with_second_scan === compiler_registry + end + end + + test "combined family registry retains each local definition target" do + registry = Schemas.compiled_defs(@schema_uri) + + expected_sources = %{ + "#/$defs/anchorString" => @base_uri <> "/meta/core", + "#/$defs/nonNegativeInteger" => @base_uri <> "/meta/validation", + "#/$defs/nonNegativeIntegerDefault0" => @base_uri <> "/meta/validation", + "#/$defs/schemaArray" => @base_uri <> "/meta/applicator", + "#/$defs/simpleTypes" => @base_uri <> "/meta/validation", + "#/$defs/stringArray" => @base_uri <> "/meta/validation", + "#/$defs/uriReferenceString" => @base_uri <> "/meta/core", + "#/$defs/uriString" => @base_uri <> "/meta/core" + } + + for {pointer, source_id} <- expected_sources do + assert %Schema{source_id: ^source_id} = Map.fetch!(registry, pointer) + end + end + + test "built-in local refs and dynamic meta anchors remain valid without a loader" do + validation_uri = @base_uri <> "/meta/validation" + + assert {:ok, local_ref_schema} = + JSONSchex.compile(%{ + "$schema" => @schema_uri, + "$ref" => validation_uri <> "#/$defs/nonNegativeInteger" + }) + + assert :ok == JSONSchex.validate(local_ref_schema, 0) + assert {:error, [%{rule: :minimum, value: -1}]} = JSONSchex.validate(local_ref_schema, -1) + + assert {:ok, dynamic_ref_schema} = + JSONSchex.compile(%{ + "$schema" => @schema_uri, + "$dynamicRef" => validation_uri <> "#meta" + }) + + assert :ok == JSONSchex.validate(dynamic_ref_schema, %{"minLength" => 0}) + + assert {:error, [%{rule: :minimum, value: -1}]} = + JSONSchex.validate(dynamic_ref_schema, %{"minLength" => -1}) + end +end diff --git a/test/required_properties_test.exs b/test/required_properties_test.exs new file mode 100644 index 0000000..8e550c8 --- /dev/null +++ b/test/required_properties_test.exs @@ -0,0 +1,58 @@ +defmodule JSONSchex.Test.RequiredPropertiesTest do + use ExUnit.Case, async: true + + alias JSONSchex.Types.{Error, ErrorContext, Rule} + alias JSONSchex.Validator.Rules + + test "reports missing properties in required declaration order" do + required = ["missing-c", "present", "missing-a", "missing-b"] + assert {:ok, schema} = JSONSchex.compile(%{"required" => required}) + + assert {:error, [%Error{} = error]} = JSONSchex.validate(schema, %{"present" => true}) + assert error.rule == :required + assert error.context.contrast == ["missing-c", "missing-a", "missing-b"] + end + + test "direct rules preserve duplicate missing entries" do + rule = %Rule{name: :required, params: ["missing", "present", "missing"]} + context = {[], MapSet.new(), nil} + + assert {:error, [%Error{} = error]} = Rules.apply(rule, %{"present" => true}, context) + assert error.context.contrast == ["missing", "missing"] + end + + test "detects missing properties at the first, middle, and last positions" do + required = Enum.map(1..64, &"property-#{&1}") + assert {:ok, schema} = JSONSchex.compile(%{"required" => required}) + complete = Map.new(required, &{&1, true}) + + for missing <- [List.first(required), Enum.at(required, 32), List.last(required)] do + assert {:error, [%Error{} = error]} = JSONSchex.validate(schema, Map.delete(complete, missing)) + assert error.context.contrast == [missing] + end + end + + test "passes when every required property is present" do + required = Enum.map(1..64, &"property-#{&1}") + assert {:ok, schema} = JSONSchex.compile(%{"required" => required}) + + assert :ok == JSONSchex.validate(schema, Map.new(required, &{&1, nil})) + end + + test "remains inapplicable to non-object data and compatible with direct rules" do + rule = %Rule{name: :required, params: ["first", "second"]} + context = {[:parent], MapSet.new(), nil} + + assert :ok == Rules.apply(rule, [], context) + assert :ok == Rules.apply(%Rule{name: :required, params: []}, %{}, context) + + assert {:error, + [ + %Error{ + path: [:parent], + rule: :required, + context: %ErrorContext{contrast: ["first", "second"]} + } + ]} = Rules.apply(rule, %{}, context) + end +end diff --git a/test/schema_traversal_test.exs b/test/schema_traversal_test.exs new file mode 100644 index 0000000..8ec74bb --- /dev/null +++ b/test/schema_traversal_test.exs @@ -0,0 +1,39 @@ +defmodule JSONSchex.Test.SchemaTraversal do + use ExUnit.Case, async: true + + alias JSONSchex.{SchemaTraversal, ScopeScanner} + + test "scope reduction preserves Map.values traversal and duplicate-identity precedence" do + target_uri = "https://example.test/repeated" + + properties = + Map.new(1..40, fn index -> + {"property-#{index}", + %{ + "$id" => target_uri, + "$anchor" => "shared", + "const" => index + }} + end) + + expected = properties |> Map.values() |> List.last() + schema = %{"properties" => properties, "$ref" => target_uri} + + reduced = + schema + |> SchemaTraversal.reduce_scope_subschemas([], + fn subschema, acc -> + [subschema | acc] + end) + |> Enum.reverse() + + assert reduced === SchemaTraversal.scope_subschemas(schema) + + {registry, _refs} = ScopeScanner.scan(schema) + assert Map.fetch!(registry, target_uri) === expected + assert Map.fetch!(registry, target_uri <> "#shared") === expected + + assert {:ok, compiled} = JSONSchex.compile(schema) + assert Map.fetch!(compiled.defs, target_uri).raw === expected + end +end diff --git a/test/string_length_test.exs b/test/string_length_test.exs new file mode 100644 index 0000000..3461e18 --- /dev/null +++ b/test/string_length_test.exs @@ -0,0 +1,344 @@ +defmodule JSONSchex.Test.StringLengthTest do + use ExUnit.Case, async: true + + alias JSONSchex.Compiler.Predicates + alias JSONSchex.Types.{Error, ErrorContext} + + test "public min/max predicates count codepoints rather than bytes or graphemes" do + for {data, count} <- valid_strings() do + assert_predicate_bounds(data, bounds(count)) + end + end + + test "compiled min/max validation counts codepoints rather than bytes or graphemes" do + for {data, count} <- valid_strings() do + assert_compiled_bounds(data, bounds(count)) + assert {:ok, compiled} = JSONSchex.compile(%{"minLength" => count, "maxLength" => count}) + assert JSONSchex.validate(compiled, data) == :ok + end + end + + test "UTF-8 scalar boundaries, NUL, and noncharacters remain valid codepoints" do + # Cover encoding-width transitions and both sides of the surrogate gap. + boundaries = [0x00, 0x7F, 0x80, 0x7FF, 0x800, 0xD7FF, 0xE000, 0xFFFF, 0x10000, 0x10FFFF] + noncharacters = Enum.to_list(0xFDD0..0xFDEF) ++ + for plane <- 0..16, suffix <- [0xFFFE, 0xFFFF], do: plane * 0x10000 + suffix + codepoints = Enum.uniq(boundaries ++ noncharacters) + combined = List.to_string(codepoints) + + for {data, count} <- [{combined, length(codepoints)} | Enum.map(codepoints, &{<<&1::utf8>>, 1})] do + assert_predicate_bounds(data, bounds(count)) + assert_compiled_bounds(data, bounds(count)) + end + end + + test "integral float limits accepted by the compiler preserve exact contexts" do + for {data, count} <- valid_strings() ++ [{String.duplicate("aé€😀", 1024), 4096}] do + float_bounds = Enum.map(bounds(count), fn {rule, limit, result} -> + float_limit = limit * 1.0 + float_result = case result do + :ok -> :ok + {:error, context} -> {:error, %{context | contrast: float_limit}} + end + + {rule, float_limit, float_result} + end) + + assert_predicate_bounds(data, float_bounds) + assert_compiled_bounds(data, float_bounds) + end + end + + test "both failing constraints report the full long-string length, not a cap" do + data = String.duplicate("aé€😀", 4096) + count = 16384 + + for {min, max} <- [{count + 100, 1}, {(count + 100) * 1.0, 1.0}] do + min_context = %ErrorContext{contrast: min, input: count} + max_context = %ErrorContext{contrast: max, input: count} + + assert Predicates.check_min_length(data, min) === {:error, min_context} + assert Predicates.check_max_length(data, max) === {:error, max_context} + assert {:ok, compiled} = JSONSchex.compile(%{"minLength" => min, "maxLength" => max}) + assert {:error, errors} = JSONSchex.validate(compiled, data) + assert length(errors) == 2 + assert Enum.find(errors, &(&1.rule == :minLength)) === + %Error{rule: :minLength, path: [], value: data, context: min_context} + assert Enum.find(errors, &(&1.rule == :maxLength)) === + %Error{rule: :maxLength, path: [], value: data, context: max_context} + end + end + + test "length constraints remain no-ops for non-strings" do + # No type constraint: even contradictory lengths must ignore non-strings. + assert {:ok, compiled} = JSONSchex.compile(%{"minLength" => 10, "maxLength" => 0}) + + for data <- [nil, true, false, 0, 1.5, [], [97, 98], %{}, %{"a" => "text"}, :atom, {1, 2}, <<1::size(1)>>] do + assert Predicates.check_min_length(data, 10) == :ok + assert Predicates.check_max_length(data, 0) == :ok + assert JSONSchex.validate(compiled, data) == :ok + end + end + + test "public predicates return invalid_utf8 contexts for malformed UTF-8" do + for {_label, data} <- invalid_strings() do + for {rule, limit} <- invalid_bounds(data) do + assert check_length(rule, data, limit) === {:error, invalid_context(data, limit)} + end + end + end + + test "compiled validation reports malformed UTF-8 even with zero or loose bounds" do + for {_label, data} <- invalid_strings() do + for {rule, limit} <- invalid_bounds(data) do + assert {:ok, compiled} = JSONSchex.compile(%{Atom.to_string(rule) => limit}) + assert JSONSchex.validate(compiled, data) === + {:error, [%Error{rule: rule, path: [], value: data, context: invalid_context(data, limit)}]} + end + + max = byte_size(data) + 1 + assert {:ok, compiled} = JSONSchex.compile(%{"minLength" => 0, "maxLength" => max}) + assert {:error, errors} = JSONSchex.validate(compiled, data) + assert length(errors) == 2 + + for {rule, limit} <- [{:minLength, 0}, {:maxLength, max}] do + assert Enum.find(errors, &(&1.rule == rule)) === + %Error{rule: rule, path: [], value: data, context: invalid_context(data, limit)} + end + end + end + + test "malformed UTF-8 errors retain nested property and item paths" do + data = "aé€" <> <<0xFF>> + + for {rule, limit} <- [{:minLength, 0}, {:maxLength, 100}] do + schema = %{ + "properties" => %{ + "names" => %{"items" => %{Atom.to_string(rule) => limit}} + } + } + + assert {:ok, compiled} = JSONSchex.compile(schema) + assert JSONSchex.validate(compiled, %{"names" => ["valid", data]}) === + {:error, [%Error{rule: rule, path: [1, "names"], value: data, context: invalid_context(data, limit)}]} + end + end + + test "malformed UTF-8 length errors format safely at root and property paths" do + for {_label, data} <- invalid_strings(), + {rule, limit} <- [{:minLength, 0}, {:maxLength, byte_size(data) + 1}] do + length_schema = %{Atom.to_string(rule) => limit} + message = "Cannot check #{rule}: string is not valid UTF-8" + + for {schema, input, path, expected_message} <- [ + {length_schema, data, [], message}, + {%{"properties" => %{"name" => length_schema}}, %{"name" => data}, ["name"], "At /name: " <> message} + ] do + assert {:ok, compiled} = JSONSchex.compile(schema) + assert {:error, [error]} = JSONSchex.validate(compiled, input) + assert error === %Error{rule: rule, path: path, value: data, context: invalid_context(data, limit)} + formatted = JSONSchex.format_error(error) + assert formatted == expected_message + assert String.valid?(formatted) + assert :binary.match(formatted, data) == :nomatch + end + end + end + + test "all one-byte and two-byte binaries agree with the UTF-8 oracle" do + for first <- 0..255 do + assert_utf8_length_oracle(<>) + + for second <- 0..255 do + assert_utf8_length_oracle(<>) + end + end + end + + test "exact codepoint lengths span the 256-byte cutoff and longer mixed widths" do + for size <- [63, 64, 65, 127, 128, 129, 255, 256, 257, 1023, 1024, 1025], + unit <- ["a", "é", "€", "😀", "aé€😀", "😀€éa"] do + repetitions = div(size, byte_size(unit)) + padding = String.duplicate("a", rem(size, byte_size(unit))) + data = String.duplicate(unit, repetitions) <> padding + + assert byte_size(data) == size + assert String.valid?(data) + assert_utf8_length_oracle(data) + end + end + + test "restricted three-byte and four-byte leaders validate every second byte" do + # Later continuation boundaries distinguish malformed tails from restrictions + # on E0/ED/F0/F4's second byte (overlong, surrogate, and out-of-range scalars). + continuations = [0x7F, 0x80, 0xBF, 0xC0] + + for second <- 0..255 do + for leader <- [0xE0, 0xED], third <- continuations do + assert_utf8_length_oracle(<>) + end + + for leader <- [0xF0, 0xF4], third <- continuations, fourth <- continuations do + assert_utf8_length_oracle(<>) + end + end + end + + test "restricted and truncated sequences retain validation across the 256-byte cutoff" do + continuations = [0x7F, 0x80, 0xBF, 0xC0] + + # Build prefixes once; changing their codepoint widths also changes the + # remainder seen by the long counter's four-codepoint loop. + prefixes = for size <- 251..256, unit <- ["a", "é", "€", "😀", "aé€😀"] do + prefix = String.duplicate(unit, div(size, byte_size(unit))) <> + String.duplicate("a", rem(size, byte_size(unit))) + {size, prefix} + end + + for leader <- [0xE0, 0xED, 0xF0, 0xF4], second <- 0..255 do + tails = if leader in [0xE0, 0xED] do + for third <- continuations, do: <> + else + for third <- continuations, fourth <- continuations, + do: <> + end + + truncated = if leader in [0xE0, 0xED] do + [<>, <>] + else + [<>, <> | (for third <- continuations, do: <>)] + end + + for bytes <- tails ++ truncated, {size, prefix} <- prefixes, + size + byte_size(bytes) in [255, 256, 257] do + assert_utf8_length_oracle(prefix <> bytes) + end + end + end + + defp assert_utf8_length_oracle(data) do + # Only convert valid UTF-8: invalid inputs must preserve the binary, not a + # partially counted length, even when a zero bound could otherwise succeed. + if String.valid?(data) do + count = length(String.to_charlist(data)) + assert Predicates.check_min_length(data, count) === :ok + assert Predicates.check_max_length(data, count) === :ok + assert Predicates.check_min_length(data, count + 1) === + {:error, %ErrorContext{contrast: count + 1, input: count}} + + if count > 0 do + assert Predicates.check_max_length(data, count - 1) === + {:error, %ErrorContext{contrast: count - 1, input: count}} + end + else + expected = {:error, %ErrorContext{contrast: 0, input: data, error_detail: "invalid_utf8"}} + assert Predicates.check_min_length(data, 0) === expected + assert Predicates.check_max_length(data, 0) === expected + end + end + + defp valid_strings do + [ + {"", 0}, + {"a", 1}, + {"plain ASCII", 11}, + {"é", 1}, + {"€", 1}, + {"😀", 1}, + {"aé€😀", 4}, + {"e\u0301", 2}, + {"a\u0301\u0327", 3}, + {"\u{1F1FA}\u{1F1F8}", 2}, + {"\u{1F468}\u200D\u{1F469}\u200D\u{1F467}\u200D\u{1F466}", 7}, + {"\u{1F469}\u{1F3FD}\u200D\u{1F4BB}", 4} + ] + end + + defp bounds(count) do + [ + {:minLength, 0, :ok}, + {:minLength, count, :ok}, + {:minLength, count + 1, {:error, %ErrorContext{contrast: count + 1, input: count}}}, + {:maxLength, count, :ok}, + {:maxLength, count + 1, :ok} + ] ++ if count > 0 do + [ + {:maxLength, count - 1, {:error, %ErrorContext{contrast: count - 1, input: count}}}, + {:maxLength, 0, {:error, %ErrorContext{contrast: 0, input: count}}} + ] + else + [] + end + end + + defp assert_predicate_bounds(data, cases) do + for {rule, limit, expected} <- cases do + assert check_length(rule, data, limit) === expected + end + end + + defp assert_compiled_bounds(data, cases) do + for {rule, limit, expected} <- cases do + assert {:ok, compiled} = JSONSchex.compile(%{Atom.to_string(rule) => limit}) + result = JSONSchex.validate(compiled, data) + + case expected do + :ok -> assert result == :ok + {:error, context} -> + assert result === {:error, [%Error{rule: rule, path: [], value: data, context: context}]} + end + end + end + + defp check_length(:minLength, data, limit), do: Predicates.check_min_length(data, limit) + defp check_length(:maxLength, data, limit), do: Predicates.check_max_length(data, limit) + + defp invalid_strings do + malformed = [ + {:overlong_two, <<0xC0, 0x80>>}, + {:overlong_two_alt, <<0xC1, 0xBF>>}, + {:overlong_three, <<0xE0, 0x9F, 0xBF>>}, + {:overlong_four, <<0xF0, 0x8F, 0xBF, 0xBF>>}, + {:lone_continuation_low, <<0x80>>}, + {:lone_continuation_high, <<0xBF>>}, + {:bad_continuation_two, <<0xC2, 0x41>>}, + {:bad_continuation_three, <<0xE2, 0x82, 0x41>>}, + {:bad_continuation_four, <<0xF0, 0x90, 0x80, 0x41>>}, + {:surrogate_low, <<0xED, 0xA0, 0x80>>}, + {:surrogate_high, <<0xED, 0xBF, 0xBF>>}, + {:too_high, <<0xF4, 0x90, 0x80, 0x80>>}, + {:invalid_leader_f5, <<0xF5, 0x80, 0x80, 0x80>>}, + {:invalid_leader_fe, <<0xFE>>}, + {:invalid_leader_ff, <<0xFF>>}, + {:truncated_two, <<0xC2>>}, + {:truncated_three_one, <<0xE2>>}, + {:truncated_three_two, <<0xE2, 0x82>>}, + {:truncated_four_one, <<0xF0>>}, + {:truncated_four_two, <<0xF0, 0x90>>}, + {:truncated_four_three, <<0xF0, 0x90, 0x80>>} + ] + + # Preserve the full input in error contexts, including the valid prefix. + # Prefixes of one to three codepoints also exercise the unrolled loop's tail. + for prefix <- ["", "a", "aé", "aé€", "aé€😀", String.duplicate("aé€😀", 2048)], + {label, bytes} <- malformed do + {label, prefix <> bytes} + end + end + + defp invalid_bounds(data) do + # Exercise early success and failure paths, including minLength = 0. + [ + {:minLength, 0}, + {:minLength, 1}, + {:minLength, byte_size(data) + 1}, + {:maxLength, byte_size(data) + 1}, + {:maxLength, 0}, + {:maxLength, 1} + ] + end + + defp invalid_context(data, limit) do + %ErrorContext{contrast: limit, input: data, error_detail: "invalid_utf8"} + end +end diff --git a/test/validator_error_flattening_test.exs b/test/validator_error_flattening_test.exs new file mode 100644 index 0000000..a150f03 --- /dev/null +++ b/test/validator_error_flattening_test.exs @@ -0,0 +1,170 @@ +defmodule JSONSchex.Test.ValidatorErrorFlatteningTest do + use ExUnit.Case, async: true + + alias JSONSchex.Types.{Error, ErrorContext, Rule, Schema, ValidationContext} + + test "public validation preserves nested error order, paths, and local values" do + assert {:ok, schema} = + JSONSchex.compile(%{ + "properties" => %{ + "rows" => %{ + "items" => %{ + "allOf" => [ + %{"type" => "integer"}, + %{"minimum" => 10}, + %{"maximum" => 0} + ] + } + } + } + }) + + data = %{"rows" => ["bad", 5, 20]} + + assert {:error, errors} = JSONSchex.validate(schema, data) + + assert Enum.map(errors, fn %Error{rule: rule, path: path, value: value} -> + {rule, path, value} + end) == [ + {:maximum, [2, "rows"], 20}, + {:minimum, [1, "rows"], 5}, + {:maximum, [1, "rows"], 5}, + {:type, [0, "rows"], "bad"} + ] + end + + test "public validation does not change validate_entry error grouping" do + assert {:ok, compiled} = + JSONSchex.compile(%{ + "required" => ["missing"], + "properties" => %{ + "value" => %{ + "allOf" => [ + %{"type" => "string"}, + %{"minimum" => 10} + ] + } + } + }) + + schema = reorder_rules(compiled, [:required, :properties]) + data = %{"value" => 5} + + assert {:error, [property_errors, required_errors] = grouped_errors} = + validate_entry(schema, data) + + assert Enum.map(property_errors, & &1.rule) == [:type, :minimum] + assert Enum.map(required_errors, & &1.rule) == [:required] + + assert {:error, public_errors} = JSONSchex.validate(schema, data) + assert public_errors == List.flatten(grouped_errors) + end + + test "failed applicators preserve nested branch error order" do + branches = + 1..8 + |> Enum.chunk_every(2) + |> Enum.map(fn contrasts -> + %{"allOf" => Enum.map(contrasts, &%{"const" => &1})} + end) + + for keyword <- ["allOf", "anyOf", "oneOf"] do + assert {:ok, schema} = JSONSchex.compile(%{keyword => branches}) + + initial_evaluated = + if keyword == "anyOf", do: MapSet.new(1..8), else: MapSet.new() + + assert {:error, errors} = validate_entry(schema, -1, initial_evaluated) + assert Enum.map(errors, & &1.rule) == List.duplicate(:const, 8) + assert Enum.map(errors, & &1.context.contrast) == Enum.to_list(1..8) + end + end + + test "false unevaluated schemas retain rewritten errors and reverse traversal order" do + property_data = %{"a" => 1, "b" => 2, "c" => 3, "d" => 4} + assert {:ok, property_schema} = JSONSchex.compile(%{"unevaluatedProperties" => false}) + assert {:error, property_errors} = JSONSchex.validate(property_schema, property_data) + + expected_property_paths = + property_data + |> Map.to_list() + |> Enum.map(fn {key, _value} -> [key] end) + |> Enum.reverse() + + assert Enum.map(property_errors, & &1.path) == expected_property_paths + + Enum.each(property_errors, fn error -> + assert error.rule == :unevaluatedProperties + assert error.context == %ErrorContext{contrast: "not_allowed"} + assert error.value == nil + end) + + assert {:ok, item_schema} = JSONSchex.compile(%{"unevaluatedItems" => false}) + assert {:error, item_errors} = JSONSchex.validate(item_schema, [1, 2, 3, 4]) + assert Enum.map(item_errors, & &1.path) == [[3], [2], [1], [0]] + + Enum.each(item_errors, fn error -> + assert error.rule == :unevaluatedItems + assert error.context == %ErrorContext{contrast: "not_allowed"} + assert error.value == nil + end) + end + + test "ordinary unevaluated-property failures retain child rules and values" do + assert {:ok, schema} = + JSONSchex.compile(%{ + "unevaluatedProperties" => %{ + "allOf" => [ + %{"type" => "string"}, + %{"minLength" => 3} + ] + } + }) + + assert {:error, errors} = JSONSchex.validate(schema, %{"a" => 1, "b" => "x"}) + + assert Enum.map(errors, fn %Error{rule: rule, path: path, value: value} -> + {rule, path, value} + end) == [ + {:minLength, ["b"], "x"}, + {:type, ["a"], 1} + ] + end + + test "wide flat error results are flattened without losing entries" do + width = 4_096 + assert {:ok, schema} = JSONSchex.compile(%{"items" => %{"type" => "integer"}}) + + assert {:error, errors} = JSONSchex.validate(schema, List.duplicate("bad", width)) + assert length(errors) == width + assert hd(errors).path == [width - 1] + assert List.last(errors).path == [0] + assert Enum.all?(errors, &(&1.rule == :type and &1.value == "bad")) + end + + defp validate_entry(%Schema{} = schema, data, initial_evaluated \\ MapSet.new()) do + context = %ValidationContext{ + root_schema: schema, + source_id: schema.source_id, + raw: schema.raw, + scope_stack: if(schema.source_id, do: [schema.source_id], else: []) + } + + JSONSchex.Validator.validate_entry(schema, data, [], context, initial_evaluated) + end + + # Rule order is observable because the validator accumulates sibling failures + # in reverse execution order. Make that order explicit for the grouping test. + defp reorder_rules(%Schema{rules: rules} = schema, names) do + selected = + Enum.map(names, fn name -> + case Enum.find(rules, &(&1.name == name)) do + %Rule{} = rule -> rule + nil -> raise "expected compiled #{inspect(name)} rule" + end + end) + + remaining = Enum.reject(rules, &(&1.name in names)) + %{schema | rules: selected ++ remaining} + end +end diff --git a/test/validator_evaluated_keys_merge_test.exs b/test/validator_evaluated_keys_merge_test.exs new file mode 100644 index 0000000..512a4c1 --- /dev/null +++ b/test/validator_evaluated_keys_merge_test.exs @@ -0,0 +1,305 @@ +defmodule JSONSchex.Test.ValidatorEvaluatedKeysMergeTest do + use ExUnit.Case, async: true + + alias JSONSchex.Types.{Rule, Schema, ValidationContext} + + test "allOf unions overlapping and disjoint successful evaluated keys" do + assert {:ok, schema} = + JSONSchex.compile(%{ + "allOf" => [ + %{"properties" => %{"a" => true, "shared" => true}}, + %{"properties" => %{"b" => true, "shared" => true}} + ] + }) + + assert {:ok, evaluated} = validate_entry(schema, %{"a" => 1, "b" => 2, "shared" => 3}) + assert evaluated == MapSet.new(["a", "b", "shared"]) + end + + test "anyOf unions every successful branch with the same incoming evaluated keys" do + assert {:ok, compiled} = + JSONSchex.compile(%{ + "properties" => %{"seed" => true}, + "anyOf" => [ + %{"properties" => %{"a" => true, "shared" => true}}, + %{"properties" => %{"b" => true, "shared" => true}} + ] + }) + + schema = reorder_rules(compiled, [:properties, :anyOf]) + data = %{"seed" => 0, "a" => 1, "b" => 2, "shared" => 3} + + assert {:ok, evaluated} = validate_entry(schema, data) + assert evaluated == MapSet.new(["seed", "a", "b", "shared"]) + end + + test "direct anyOf merging does not pass earlier branch evaluated keys into a later branch" do + seed_keys = Enum.map(1..8, &"seed-#{&1}") + + assert {:ok, compiled} = + JSONSchex.compile(%{ + "properties" => Map.new(seed_keys, &{&1, true}), + "anyOf" => [ + %{"properties" => %{"a" => true}}, + %{ + "properties" => %{"b" => true}, + "unevaluatedProperties" => false + }, + true, + true, + true + ], + "unevaluatedProperties" => false + }) + + schema = reorder_rules(compiled, [:properties, :anyOf, :unevaluatedProperties]) + data = Map.merge(Map.new(seed_keys, &{&1, 0}), %{"a" => 1, "b" => 2}) + + assert {:error, errors} = JSONSchex.validate(schema, data) + assert [%{rule: :unevaluatedProperties, path: ["b"]}] = errors + end + + test "anyOf continues through later branches after an earlier success" do + parent = self() + seed_keys = Enum.map(1..8, &"seed-#{&1}") + + assert {:ok, compiled} = + JSONSchex.compile( + %{ + "properties" => Map.new(seed_keys, &{&1, true}), + "anyOf" => [ + %{"properties" => %{"first" => true}}, + %{"$ref" => "https://example.test/later.json"}, + true, + true + ] + }, + loader: fn _uri -> + send(parent, :later_any_of_branch) + {:ok, false} + end + ) + + schema = reorder_rules(compiled, [:properties, :anyOf]) + data = Map.put(Map.new(seed_keys, &{&1, true}), "first", true) + + refute_received :later_any_of_branch + assert :ok == JSONSchex.validate(schema, data) + assert_received :later_any_of_branch + end + + test "dependentSchemas unions all triggered successful evaluated keys" do + assert {:ok, schema} = + JSONSchex.compile(%{ + "dependentSchemas" => %{ + "trigger-a" => %{"properties" => %{"a" => true, "shared" => true}}, + "trigger-b" => %{"properties" => %{"b" => true, "shared" => true}} + } + }) + + data = %{ + "trigger-a" => true, + "trigger-b" => true, + "a" => 1, + "b" => 2, + "shared" => 3 + } + + assert {:ok, evaluated} = validate_entry(schema, data) + assert evaluated == MapSet.new(["a", "b", "shared"]) + end + + test "schema-valued legacy dependencies unions triggered evaluated keys" do + assert {:ok, schema} = + JSONSchex.compile(%{ + "dependencies" => %{ + "trigger-a" => %{"properties" => %{"a" => true, "shared" => true}}, + "trigger-b" => %{"properties" => %{"b" => true, "shared" => true}} + } + }) + + data = %{ + "trigger-a" => true, + "trigger-b" => true, + "a" => 1, + "b" => 2, + "shared" => 3 + } + + assert {:ok, evaluated} = validate_entry(schema, data) + assert evaluated == MapSet.new(["a", "b", "shared"]) + end + + test "empty successful evaluated-key results remain MapSets" do + for raw_schema <- [ + %{"allOf" => [true]}, + %{"anyOf" => [true, false]}, + %{"dependentSchemas" => %{"trigger" => %{"properties" => %{"value" => true}}}} + ] do + assert {:ok, schema} = JSONSchex.compile(raw_schema) + assert {:ok, evaluated} = validate_entry(schema, %{}) + assert evaluated == MapSet.new() + end + end + + test "anyOf merges dominant shared incoming evaluated keys" do + assert {:ok, schema} = + JSONSchex.compile(%{ + "anyOf" => [ + %{"properties" => %{"a" => true}}, + %{"properties" => %{"b" => true}}, + %{"properties" => %{"c" => true}}, + %{"properties" => %{"d" => true}} + ] + }) + + parent_evaluated_keys = MapSet.new(Enum.map(1..16, &"seed-#{&1}")) + data = %{"a" => 1, "b" => 2, "c" => 3, "d" => 4} + + assert {:ok, evaluated} = validate_entry(schema, data, parent_evaluated_keys) + assert evaluated == MapSet.union(parent_evaluated_keys, MapSet.new(["a", "b", "c", "d"])) + end + + test "anyOf preserves evaluated keys when incoming overlap falls back to bulk merging" do + branches = + Enum.map(1..4, fn branch -> + fields = Enum.map(1..8, &"branch-#{branch}-#{&1}") + %{"properties" => Map.new(fields, &{&1, true})} + end) + + branch_fields = + for branch <- 1..4, field <- 1..8 do + "branch-#{branch}-#{field}" + end + + assert {:ok, schema} = JSONSchex.compile(%{"anyOf" => branches}) + parent_evaluated_keys = MapSet.new(Enum.map(1..8, &"seed-#{&1}")) + + assert {:ok, evaluated} = + validate_entry(schema, Map.new(branch_fields, &{&1, true}), parent_evaluated_keys) + + assert evaluated == MapSet.union(parent_evaluated_keys, MapSet.new(branch_fields)) + end + + test "anyOf preserves evaluated keys when a later branch causes bulk fallback" do + early_fields = Enum.map(1..3, &"early-#{&1}") + late_fields = Enum.map(1..16, &"late-#{&1}") + + assert {:ok, schema} = + JSONSchex.compile(%{ + "anyOf" => + Enum.map(early_fields, &%{"properties" => %{&1 => true}}) ++ + [%{"properties" => Map.new(late_fields, &{&1, true})}] + }) + + parent_evaluated_keys = MapSet.new(Enum.map(1..8, &"seed-#{&1}")) + data = Map.new(early_fields ++ late_fields, &{&1, true}) + + assert {:ok, evaluated} = validate_entry(schema, data, parent_evaluated_keys) + + assert evaluated == + MapSet.union(parent_evaluated_keys, MapSet.new(early_fields ++ late_fields)) + end + + test "allOf and all-failed anyOf retain child error order" do + branches = [false, %{"type" => "string"}, %{"required" => ["missing"]}] + + for keyword <- ["allOf", "anyOf"] do + assert {:ok, schema} = JSONSchex.compile(%{keyword => branches}) + assert {:error, errors} = validate_entry(schema, %{}) + assert Enum.map(errors, & &1.rule) == [:boolean_schema, :type, :required] + end + + assert {:ok, direct_any_of} = + JSONSchex.compile(%{"anyOf" => branches ++ [false]}) + + parent_evaluated_keys = MapSet.new(Enum.map(1..8, &"seed-#{&1}")) + assert {:error, errors} = validate_entry(direct_any_of, %{}, parent_evaluated_keys) + assert Enum.map(errors, & &1.rule) == [:boolean_schema, :type, :required, :boolean_schema] + end + + test "dependentSchemas retains its existing reverse map-traversal error order" do + assert {:ok, schema} = + JSONSchex.compile(%{ + "dependentSchemas" => %{ + "first" => false, + "second" => %{"type" => "string"}, + "third" => %{"required" => ["missing"]} + } + }) + + [%Rule{name: :dependentSchemas, params: dependencies}] = + Enum.filter(schema.rules, &(&1.name == :dependentSchemas)) + + expected_rules = + dependencies + |> Map.to_list() + |> Enum.map(fn {_trigger, %Schema{rules: [%Rule{name: name} | _]}} -> name end) + |> Enum.reverse() + + assert {:error, errors} = + validate_entry(schema, %{"first" => true, "second" => true, "third" => true}) + + assert Enum.map(errors, & &1.rule) == expected_rules + end + + test "anyOf retains its existing list-valued evaluated-key behavior on the direct-merge fallback" do + assert {:ok, schema} = JSONSchex.compile(%{"anyOf" => [true, true, true, true]}) + parent_evaluated_keys = MapSet.new(Enum.map(1..8, &[&1])) + + assert {:ok, evaluated} = validate_entry(schema, %{}, parent_evaluated_keys) + assert evaluated == MapSet.union(parent_evaluated_keys, MapSet.new(1..8)) + end + + test "failed allOf and dependentSchemas discard successful child evaluated keys" do + assert {:ok, all_of_schema} = + JSONSchex.compile(%{ + "allOf" => [%{"properties" => %{"a" => true}}, false], + "unevaluatedProperties" => false + }) + + assert {:error, all_of_errors} = JSONSchex.validate(all_of_schema, %{"a" => 1}) + assert Enum.any?(all_of_errors, &(&1.rule == :boolean_schema)) + assert Enum.any?(all_of_errors, &(&1.rule == :unevaluatedProperties and &1.path == ["a"])) + + assert {:ok, dependent_schema} = + JSONSchex.compile(%{ + "dependentSchemas" => %{ + "trigger" => %{"properties" => %{"a" => true}}, + "failure" => false + }, + "unevaluatedProperties" => false + }) + + data = %{"trigger" => true, "failure" => true, "a" => 1} + assert {:error, dependent_errors} = JSONSchex.validate(dependent_schema, data) + assert Enum.any?(dependent_errors, &(&1.rule == :boolean_schema)) + assert Enum.any?(dependent_errors, &(&1.rule == :unevaluatedProperties and &1.path == ["a"])) + end + + defp validate_entry(schema, data, initial_evaluated_keys \\ MapSet.new()) + + defp validate_entry(%Schema{} = schema, data, initial_evaluated_keys) do + context = %ValidationContext{ + root_schema: schema, + source_id: schema.source_id, + raw: schema.raw, + scope_stack: if(schema.source_id, do: [schema.source_id], else: []) + } + + JSONSchex.Validator.validate_entry(schema, data, [], context, initial_evaluated_keys) + end + + defp reorder_rules(%Schema{rules: rules} = schema, names) do + selected = + Enum.map(names, fn name -> + case Enum.find(rules, &(&1.name == name)) do + %Rule{} = rule -> rule + nil -> raise "expected compiled #{inspect(name)} rule" + end + end) + + remaining = Enum.reject(rules, &(&1.name in names)) + %{schema | rules: selected ++ remaining} + end +end diff --git a/test/validator_pattern_match_cache_test.exs b/test/validator_pattern_match_cache_test.exs new file mode 100644 index 0000000..1693632 --- /dev/null +++ b/test/validator_pattern_match_cache_test.exs @@ -0,0 +1,224 @@ +defmodule JSONSchex.Test.ValidatorPatternMatchCacheTest do + use ExUnit.Case, async: true + + alias JSONSchex.Types.{Rule, Schema} + + test "cached classification preserves overlapping pattern diagnostics in both sibling orders" do + assert {:ok, compiled} = + JSONSchex.compile(%{ + "patternProperties" => %{ + "^x-" => %{"type" => "string"}, + "name$" => %{"minimum" => 2} + }, + "additionalProperties" => false + }) + + data = %{"x-name" => 1, "other" => true} + + for order <- [[:patternProperties, :additionalProperties], [:additionalProperties, :patternProperties]] do + cached = reorder_rules(compiled, order) + uncached = uncached_additional_patterns(cached) + cached_result = JSONSchex.validate(cached, data) + + assert cached_result === JSONSchex.validate(uncached, data) + assert {:error, errors} = cached_result + assert Enum.any?(errors, &(&1.rule == :type and &1.path == ["x-name"])) + assert Enum.any?(errors, &(&1.rule == :minimum and &1.path == ["x-name"])) + assert Enum.any?(errors, &(&1.rule == :boolean_schema and &1.path == ["other"])) + refute Enum.any?(errors, &(&1.rule == :boolean_schema and &1.path == ["x-name"])) + end + end + + test "structural matches remain separate from evaluated-property annotations in both sibling orders" do + assert {:ok, compiled} = + JSONSchex.compile(%{ + "patternProperties" => %{ + "^x-" => %{"type" => "string"}, + "^never-" => true + }, + "additionalProperties" => true, + "unevaluatedProperties" => false + }) + + for order <- [[:patternProperties, :additionalProperties], [:additionalProperties, :patternProperties]] do + cached = reorder_rules(compiled, order) + uncached = uncached_additional_patterns(cached) + invalid = %{"x-name" => 1} + cached_result = JSONSchex.validate(cached, invalid) + + assert cached_result === JSONSchex.validate(uncached, invalid) + assert {:error, errors} = cached_result + assert Enum.any?(errors, &(&1.rule == :type and &1.path == ["x-name"])) + assert Enum.any?(errors, &(&1.rule == :unevaluatedProperties and &1.path == ["x-name"])) + + assert :ok == JSONSchex.validate(cached, %{"x-name" => "ok", "other" => true}) + end + end + + test "cache selection does not run regex matching ahead of preceding rules" do + parent = self() + + assert {:ok, compiled} = + JSONSchex.compile( + %{ + "$ref" => "https://example.test/remote.json", + "patternProperties" => %{"^x-" => true, "^y-" => true}, + "additionalProperties" => true + }, + loader: fn _uri -> + send(parent, :runtime_loader_called) + {:error, :unavailable} + end + ) + + refute_received :runtime_loader_called + schema = reorder_rules(compiled, [:ref, :patternProperties, :additionalProperties]) + + assert_raise FunctionClauseError, fn -> + JSONSchex.validate(schema, %{1 => true}) + end + + assert_received :runtime_loader_called + end + + test "shared referenced schemas do not transfer failed annotations between object instances" do + raw_schema = %{ + "$id" => "https://example.test/root.json", + "$defs" => %{ + "node" => %{ + "$id" => "node.json", + "patternProperties" => %{ + "^x-" => %{"type" => "string"}, + "^never-" => true + }, + "additionalProperties" => true, + "unevaluatedProperties" => false + } + }, + "properties" => %{ + "left" => %{"$ref" => "#/$defs/node"}, + "right" => %{"$ref" => "#/$defs/node"} + } + } + + assert {:ok, compiled} = JSONSchex.compile(raw_schema) + + assert {:error, errors} = + JSONSchex.validate(compiled, %{ + "left" => %{"x-name" => 1}, + "right" => %{"x-name" => "ok", "other" => true} + }) + + assert Enum.any?(errors, &(&1.rule == :type and &1.path == ["x-name", "left"])) + + assert Enum.any?(errors, fn error -> + error.rule == :unevaluatedProperties and error.path == ["x-name", "left"] + end) + + refute Enum.any?(errors, fn error -> "right" in error.path end) + + assert :ok == + JSONSchex.validate(compiled, %{ + "left" => %{"x-name" => "left"}, + "right" => %{"x-name" => "right", "other" => true} + }) + end + + test "concurrent validations share no match state" do + assert {:ok, schema} = JSONSchex.compile(runtime_schema(8, true)) + + results = + Task.async_stream( + 1..64, + fn index -> + data = + if rem(index, 2) == 0 do + %{"pattern-1" => "value", "field" <> Integer.to_string(index) => index} + else + %{"pattern-2" => "value"} + end + + JSONSchex.validate(schema, data) + end, + max_concurrency: 8, + timeout: 5_000, + ordered: false + ) + |> Enum.to_list() + + assert Enum.all?(results, &(&1 == {:ok, :ok})) + end + + @tag :performance + test "sibling matching avoids a second full classification pass in reductions" do + assert {:ok, compiled} = JSONSchex.compile(runtime_schema(128, true)) + cached = reorder_rules(compiled, [:patternProperties, :additionalProperties]) + uncached = uncached_additional_patterns(cached) + data = Map.new(1..128, fn index -> {"field-" <> Integer.to_string(index), index} end) + + assert :ok == JSONSchex.validate(cached, data) + assert :ok == JSONSchex.validate(uncached, data) + + cached_work = validation_reductions(cached, data, 10) + uncached_work = validation_reductions(uncached, data, 10) + + assert cached_work * 100 < uncached_work * 85, + "cached used #{cached_work} vs uncached #{uncached_work} reductions" + end + + defp runtime_schema(width, additional_properties) do + patterns = + Map.new(1..width, fn index -> + {"^pattern-" <> Integer.to_string(index) <> "$", %{"type" => "string"}} + end) + + %{"patternProperties" => patterns, "additionalProperties" => additional_properties} + end + + defp reorder_rules(%Schema{rules: rules} = schema, names) do + selected = + Enum.map(names, fn name -> + case Enum.find(rules, &(&1.name == name)) do + %Rule{} = rule -> rule + nil -> raise "expected compiled #{inspect(name)} rule" + end + end) + + remaining = Enum.reject(rules, &(&1.name in names)) + %{schema | rules: selected ++ remaining} + end + + # Reverse only additionalProperties' regex list. It is equivalent for normal + # JSON property names because additionalProperties asks only whether any regex + # matches, but it intentionally fails the runtime cache's exact-sequence gate. + defp uncached_additional_patterns(%Schema{rules: rules} = schema) do + updated_rules = + Enum.map(rules, fn + %Rule{name: :additionalProperties, params: %{patterns: patterns} = params} = rule -> + reversed_patterns = Enum.reverse(patterns) + + if reversed_patterns === patterns do + raise "test requires at least two distinct patternProperties regexes" + end + + %{rule | params: %{params | patterns: reversed_patterns}} + + rule -> + rule + end) + + %{schema | rules: updated_rules} + end + + defp validation_reductions(schema, data, iterations) do + :erlang.garbage_collect() + {:reductions, before} = Process.info(self(), :reductions) + + Enum.each(1..iterations, fn _ -> + :ok = JSONSchex.validate(schema, data) + end) + + {:reductions, after_validation} = Process.info(self(), :reductions) + after_validation - before + end +end