Reuse already-compiled schema nodes where context matches

bench/compiler_cache.exs

@@ -0,0 +1,123 @@
+# Run from the repository root: mix run bench/compiler_cache.exs
+# Pin a baseline without changing files: mix run bench/compiler_cache.exs --baseline HEAD
+# No benchmark dependencies; input construction, source loading and GC are not timed.
+root = Path.expand("..", __DIR__)
+compiler_path = Path.join(root, "lib/jsonschex/compiler.ex")
+
+{source, source_label} =
+  case System.argv() do
+    [] ->
+      {File.read!(compiler_path), "working tree (snapshot at script startup)"}
+
+    ["--baseline", revision] ->
+      case System.cmd("git", ["--no-pager", "show", "#{revision}:lib/jsonschex/compiler.ex"], cd: root) do
+        {source, 0} -> {source, "git #{revision}:lib/jsonschex/compiler.ex"}
+        {output, status} -> raise "Cannot load baseline (exit #{status}): #{output}"
+      end
+
+    _ ->
+      raise "Usage: mix run bench/compiler_cache.exs [--baseline REVISION]"
+  end
+
+# Load immediately, before construction/warmup: later disk edits cannot change this VM.
+previous_options = Code.compiler_options(ignore_module_conflict: true)
+try do
+  Code.compile_string(source, compiler_path)
+after
+  Code.compiler_options(previous_options)
+end
+
+fingerprint = :crypto.hash(:sha256, source) |> Base.encode16(case: :lower)
+IO.puts("Compiler: #{source_label}; sha256=#{fingerprint}")
+IO.puts("Elixir #{System.version()}, OTP #{System.otp_release()}, ERTS #{:erlang.system_info(:version)}")
+IO.puts("OS #{inspect(:os.type())}; architecture #{:erlang.system_info(:system_architecture)}; schedulers #{System.schedulers_online()}/#{System.schedulers()}; MIX_ENV=#{Mix.env()}")
+IO.puts("Five batches; target 20 ms/batch; adaptive count 1..50; 50 s total measurement timeout")
+
+defmodule CompilerReuseBench do
+  # Each resource has its own absolute identity, including the terminal node.
+  def chain(keyword, depth) do
+    leaf = %{"$id" => "https://compiler-reuse.example/leaf", "type" => "integer"}
+
+    Enum.reduce(1..depth, leaf, fn index, child ->
+      %{
+        "$id" => "https://compiler-reuse.example/node/#{index}",
+        keyword => %{"child" => child}
+      }
+    end)
+  end
+
+  def wide_refs(width) do
+    # Exactly 20 schema nodes per definition: object plus 19 constrained properties.
+    definition = %{
+      "type" => "object",
+      "required" => ["field1"],
+      "properties" => Map.new(1..19, fn index ->
+        {"field#{index}", %{"type" => "string", "minLength" => 1, "maxLength" => 80}}
+      end)
+    }
+
+    # No root $id or base_uri: direct definitions and refs have matching context.
+    %{
+      "type" => "object",
+      "$defs" => Map.new(1..width, fn index -> {"def#{index}", definition} end),
+      "properties" => Map.new(1..width, fn index ->
+        {"value#{index}", %{"$ref" => "#/$defs/def#{index}"}}
+      end)
+    }
+  end
+
+  def measure({label, schema}) do
+    compile = fn ->
+      {:ok, %JSONSchex.Types.Schema{}} = JSONSchex.compile(schema)
+      :ok
+    end
+
+    compile.()
+    :erlang.garbage_collect()
+    {calibration_us, :ok} = :timer.tc(compile)
+    count = min(50, max(1, div(20_000, max(1, calibration_us))))
+
+    samples = for _ <- 1..5 do
+      :erlang.garbage_collect()
+      {elapsed_us, :ok} = :timer.tc(fn ->
+        Enum.each(1..count, fn _ -> compile.() end)
+      end)
+      elapsed_us / count / 1000
+    end
+
+    median = samples |> Enum.sort() |> Enum.at(2)
+    IO.puts("#{label}: median_ms=#{Float.round(median, 4)} (5 batches x #{count}; min_ms=#{Float.round(Enum.min(samples), 4)} max_ms=#{Float.round(Enum.max(samples), 4)})")
+  end
+end
+
+# Build all inputs before starting any measurement, including decoding the fixture.
+openapi =
+  Path.join(__DIR__, "priv/openapi_spec_schema.json")
+  |> File.read!()
+  |> Jason.decode!()
+  |> Map.fetch!("schema")
+
+ordinary = %{
+  "type" => "object",
+  "required" => ["name"],
+  "additionalProperties" => false,
+  "properties" => %{
+    "name" => %{"type" => "string", "minLength" => 1},
+    "age" => %{"type" => "integer", "minimum" => 0},
+    "tags" => %{"type" => "array", "items" => %{"type" => "string"}, "maxItems" => 10}
+  }
+}
+
+cases =
+  (for depth <- [100, 200, 400], do: {"properties+ids depth=#{depth}", CompilerReuseBench.chain("properties", depth)}) ++
+  (for depth <- [50, 100], do: {"defs+ids depth=#{depth}", CompilerReuseBench.chain("$defs", depth)}) ++
+  (for width <- [50, 100, 200], do: {"direct-def refs width=#{width} nodes/def=20", CompilerReuseBench.wide_refs(width)}) ++
+  [{"ordinary (no scope)", ordinary}, {"OpenAPI fixture", openapi}]
+
+# A hard timeout also bounds unexpectedly slow single compilations; never report
+# an incomplete set of samples as a successful benchmark run.
+task = Task.async(fn -> Enum.each(cases, &CompilerReuseBench.measure/1) end)
+case Task.yield(task, 50_000) || Task.shutdown(task, :brutal_kill) do
+  {:ok, :ok} -> :ok
+  _ -> raise "Benchmark exceeded the 50 s measurement budget; results are incomplete"
+end

lib/jsonschex/compiler.ex

@@ -25,6 +25,7 @@ defmodule JSONSchex.Compiler do
   alias JSONSchex.URIUtil
   alias JSONSchex.Compiler.Fragment
   alias JSONSchex.Compiler.Fragment.Bundle
+  alias JSONSchex.Compiler.Cache
 
   import JSONSchex.Types,
     only: [
@@ -100,8 +101,10 @@ defmodule JSONSchex.Compiler do
     with :ok <- Dialect.validate_required_vocabularies(raw_schema),
          {:ok, root_vocabs} <- resolve_dialect(raw_schema, loader, @default_vocabs_list),
          {:ok, root_compiled} <- compile_schema_node(raw_schema, init_base, root_vocabs, ctx) do
-      root_compiled = %{root_compiled | raw: context_document}
       {global_scopes, explicit_refs} = scan_context(context_document, init_base, scope_mode)
+      scope_cache = if map_size(global_scopes) == 0, do: %{}, else: Cache.build(root_compiled)
+      local_defs = root_compiled.defs
+      root_compiled = %{root_compiled | raw: context_document}
 
       full_defs =
         Enum.reduce_while(global_scopes, root_compiled.defs, fn {id, sub_raw}, acc_defs ->
@@ -115,9 +118,7 @@ defmodule JSONSchex.Compiler do
                 :ok ->
                   case resolve_dialect(sub_raw, loader, root_vocabs) do
                     {:ok, sub_vocabs} ->
-                      sub_raw
-                      |> Map.delete("$id")
-                      |> compile_schema_node(scope_resource_base(id), sub_vocabs, ctx)
+                      compile_scope(sub_raw, id, sub_vocabs, root_vocabs, ctx, scope_cache)
                       |> case do
                         {:ok, compiled_sub} ->
                           compiled_sub = %{compiled_sub | raw: sub_raw}
@@ -139,7 +140,7 @@ defmodule JSONSchex.Compiler do
         end)
 
       resolved_runtime_defs =
-        resolve_refs(context_document, MapSet.to_list(explicit_refs), root_vocabs, ctx, init_base)
+        resolve_refs(context_document, MapSet.to_list(explicit_refs), root_vocabs, ctx, init_base, local_defs)
 
       case merge_defs(full_defs, resolved_runtime_defs) do
         {:error, _} = error ->
@@ -151,6 +152,16 @@ defmodule JSONSchex.Compiler do
     end
   end
 
+  defp compile_scope(raw, id, vocabs, root_vocabs, ctx, scope_cache) do
+    base = scope_resource_base(id)
+    reusable = if vocabs === root_vocabs, do: Cache.fetch_scope(scope_cache, id, raw, base), else: nil
+
+    case reusable do
+      {:ok, compiled} -> {:ok, compiled}
+      nil -> compile_schema_node(Map.delete(raw, "$id"), base, vocabs, ctx)
+    end
+  end
+
   defp scope_resource_base(uri) when is_binary(uri) do
     {base, _fragment} = URIUtil.split_fragment(uri)
     base
@@ -193,11 +204,11 @@ defmodule JSONSchex.Compiler do
     {:ok, current_vocabs}
   end
 
-  defp resolve_refs(raw_schema, refs, vocabs, ctx, base_uri) do
+  defp resolve_refs(raw_schema, refs, vocabs, ctx, base_uri, local_defs) do
     ExJSONPointer.batch_resolve_reduce(raw_schema, refs, %{}, fn ref, result, acc ->
       case result do
         {:ok, fragment} ->
-          case compile_schema_node(fragment, base_uri, vocabs, ctx) do
+          case compile_ref_target(fragment, ref, base_uri, vocabs, ctx, local_defs) do
             {:ok, compiled_sub} ->
               compiled_sub = %{compiled_sub | raw: fragment}
 
@@ -213,6 +224,24 @@ defmodule JSONSchex.Compiler do
     end)
   end
 
+  # Only initial local definitions share this vocabulary and invocation context.
+  # Resolve the pointer first: escaped keys or containing-document definitions
+  # may select a different target than the local registry's spelling suggests.
+  defp compile_ref_target(fragment, ref, base_uri, vocabs, ctx, local_defs) when is_map(fragment) do
+    base = resolve_uri(base_uri, Map.get(fragment, "$id"))
+
+    case Map.get(local_defs, ref) do
+      %Schema{source_id: ^base, raw: raw} = compiled when raw === fragment ->
+        {:ok, compiled}
+
+      _ ->
+        compile_schema_node(fragment, base_uri, vocabs, ctx)
+    end
+  end
+
+  defp compile_ref_target(fragment, _ref, base_uri, vocabs, ctx, _local_defs),
+    do: compile_schema_node(fragment, base_uri, vocabs, ctx)
+
   defp compile_schema_node(true, _id, _vocabs, ctx) do
     {:ok,
      %Schema{

lib/jsonschex/compiler/cache.ex

@@ -0,0 +1,83 @@
+defmodule JSONSchex.Compiler.Cache do
+  @moduledoc false
+
+  alias JSONSchex.Types.{Rule, Schema}
+  alias JSONSchex.URIUtil
+
+  # This immutable cache belongs to one compilation and contains only the initial tree,
+  # whose nodes share the root vocabulary, loader and assertion options. Index
+  # identities, not raw schema maps: hashing descendant trees would itself turn
+  # nested-resource compilation into quadratic work.
+  def build(%Schema{} = schema), do: index_schema(schema, %{})
+
+  def fetch_scope(cache, id, raw, base) do
+    case Map.get(cache, id) do
+      %Schema{source_id: ^base, raw: candidate_raw} = candidate when candidate_raw === raw ->
+        if keyword_order_preserved?(raw), do: {:ok, candidate}, else: nil
+
+      _ ->
+        nil
+    end
+  end
+
+  # Scope compilation deletes $id before enumerating keywords. A map changing
+  # representation can reorder the remaining keys and hence ordered diagnostics.
+  defp keyword_order_preserved?(raw) do
+    keywords = Map.drop(raw, ["unevaluatedProperties", "unevaluatedItems"])
+    Enum.reject(Map.keys(keywords), &(&1 == "$id")) == Map.keys(Map.delete(keywords, "$id"))
+  end
+
+  defp index_schema(%Schema{raw: raw} = schema, index) when is_map(raw) do
+    index =
+      if is_binary(Map.get(raw, "$id")) and is_binary(schema.source_id) do
+        Map.put_new(index, schema.source_id, schema)
+      else
+        index
+      end
+
+    index = Enum.reduce(["$anchor", "$dynamicAnchor"], index, fn keyword, acc ->
+      case Map.get(raw, keyword) do
+        anchor when is_binary(anchor) ->
+          Map.put_new(acc, URIUtil.with_fragment(schema.source_id || "", anchor), schema)
+        _ ->
+          acc
+      end
+    end)
+
+    index = Enum.reduce(schema.defs, index, fn {_key, child}, acc -> index_schema(child, acc) end)
+    Enum.reduce(schema.rules, index, &index_rule/2)
+  end
+
+  defp index_schema(_schema, index), do: index
+
+  # Visit only compiler-owned schema slots, never arbitrary enum/const data or
+  # raw documents. Unrecognized future rule shapes safely miss reuse.
+  defp index_rule(%Rule{name: name, params: pairs}, index)
+       when name in [:properties, :patternProperties, :dependentSchemas] do
+    Enum.reduce(pairs, index, fn {_key, child}, acc -> index_schema(child, acc) end)
+  end
+
+  defp index_rule(%Rule{name: name, params: schemas}, index)
+       when name in [:allOf, :anyOf, :oneOf, :prefixItems] do
+    Enum.reduce(schemas, index, &index_schema/2)
+  end
+
+  defp index_rule(%Rule{name: name, params: schema}, index)
+       when name in [:not, :propertyNames], do: index_schema(schema, index)
+
+  defp index_rule(%Rule{name: name, params: %{schema: schema}}, index)
+       when name in [:contentSchema, :additionalProperties, :items, :contains,
+                     :unevaluatedProperties, :unevaluatedItems], do: index_schema(schema, index)
+
+  defp index_rule(%Rule{name: :if, params: branches}, index) do
+    Enum.reduce([:if, :then, :else], index, fn branch, acc ->
+      index_schema(Map.get(branches, branch), acc)
+    end)
+  end
+
+  defp index_rule(%Rule{name: :dependencies, params: %{schemas: schemas}}, index) do
+    Enum.reduce(schemas, index, fn {_key, child}, acc -> index_schema(child, acc) end)
+  end
+
+  defp index_rule(_rule, index), do: index
+end