Add hermetic template recording and replay

Gemfile

@@ -32,7 +32,6 @@ group :test do
 end
 
 group :spec do
-  # Using feature branch until https://github.com/Shopify/liquid-spec/pull/144 is merged
-  gem 'liquid-spec', github: 'Shopify/liquid-spec', branch: 'self-drop-env-lookup-specs'
+  gem 'liquid-spec', github: 'Shopify/liquid-spec'
   gem 'activesupport', require: false
 end

Rakefile

@@ -5,21 +5,42 @@ require 'rake/testtask'
 $LOAD_PATH.unshift(File.expand_path("../lib", __FILE__))
 require "liquid/version"
 
+template_recorder_test = "test/integration/template_recorder_test.rb"
+
 task(default: [:test, :rubocop])
 
 desc('run test suite with default parser')
 Rake::TestTask.new(:base_test) do |t|
   t.libs << 'lib' << 'test'
-  t.test_files = FileList['test/{integration,unit}/**/*_test.rb']
+  test_files = FileList['test/{integration,unit}/**/*_test.rb']
+  test_files.exclude(template_recorder_test) unless ENV["LIQUID_TEMPLATE_RECORDER_HOOKS"]
+  t.test_files = test_files
   t.verbose    = false
 end
 
 Rake::TestTask.new(:integration_test) do |t|
   t.libs << 'lib' << 'test'
-  t.test_files = FileList['test/integration/**/*_test.rb']
+  test_files = FileList['test/integration/**/*_test.rb']
+  test_files.exclude(template_recorder_test) unless ENV["LIQUID_TEMPLATE_RECORDER_HOOKS"]
+  t.test_files = test_files
   t.verbose    = false
 end
 
+desc('run template recorder tests with hooks enabled')
+task :template_recorder_test do
+  sh(
+    {
+      "LIQUID_PARSER_MODE" => "lax",
+      "LIQUID_TEMPLATE_RECORDER_HOOKS" => "1",
+    },
+    "bundle",
+    "exec",
+    "ruby",
+    "-Itest",
+    template_recorder_test,
+  )
+end
+
 desc('run test suite with warn error mode')
 task :warn_test do
   ENV['LIQUID_PARSER_MODE'] = 'warn'
@@ -59,6 +80,8 @@ task :test do
     Rake::Task['integration_test'].reenable
     Rake::Task['integration_test'].invoke
   end
+
+  Rake::Task['template_recorder_test'].invoke unless ENV["LIQUID_TEMPLATE_RECORDER_HOOKS"]
 end
 
 task(gem: :build)

docs/template_recorder.md

@@ -0,0 +1,78 @@
+# Recording and replaying renders
+
+`Liquid::TemplateRecorder` captures successful template renders so they can be
+replayed without the application's file system or Drop implementations.
+Recording does not wrap or replace assigns, so the recorded render has the same
+semantics as a normal render.
+
+The instrumentation hooks are disabled by default. Set
+`LIQUID_TEMPLATE_RECORDER_HOOKS=1` before the process loads Liquid. This is a
+boot-time setting; changing it after Liquid is loaded has no effect. Calling
+`Liquid::TemplateRecorder.record` while the hooks are disabled raises an error
+instead of silently producing an incomplete recording.
+
+```ruby
+Liquid::TemplateRecorder.record("render.json") do
+  template = Liquid::Template.parse(source)
+  template.render!(assigns)
+end
+
+replayer = Liquid::TemplateRecorder.replay_from("render.json", mode: :verify)
+replayer.render # raises if the output changed
+```
+
+A recording contains the root template, every parsed partial, partial contents,
+plain Hash/Array values resolved by the template, properties actually read from `Liquid::Drop` objects,
+filter-call diagnostics, engine options, and the rendered output. Drop instance
+variables are never inspected. An unsupported Ruby object raises
+`Liquid::TemplateRecorder::SerializationError` rather than silently producing a
+recording that cannot be replayed.
+
+## Storage formats
+
+A `.json` destination is written atomically after the recording block succeeds.
+It contains a session with every render performed by the block.
+
+A `.jsonl` destination is append-only. Each successful top-level render is one
+compact, self-contained JSON line. This is the recommended format for production
+sampling: a process failure can lose at most the render being written, writers
+are serialized with `flock`, and a recording can be replayed by index.
+
+A destination may instead be any writer object responding to `write(record)`. The
+writer receives one self-contained recording Hash per successful render. Liquid
+does not own or close injected writers, so applications can publish records to
+Kafka, object storage, or another transport without coupling that transport to
+the recorder.
+Pass `on_error:` to keep serialization or sink failures out of the render path;
+the callback receives the error and should not raise.
+
+```ruby
+Liquid::TemplateRecorder.record(kafka_writer) do
+  template.render!(assigns)
+end
+```
+
+```ruby
+Liquid::TemplateRecorder.replay_from("renders.jsonl")           # last render
+Liquid::TemplateRecorder.replay_from("renders.jsonl", index: 0) # first render
+Liquid::TemplateRecorder.records("renders.jsonl")               # inspect all
+```
+
+Compression is intentionally separate from the schema. In particular, one
+long-lived compressed stream makes appending, recovery, and selecting a render
+harder. Compress rotated `.jsonl` files with the storage system of your choice;
+a future compressed writer can use one independent frame per record without a
+schema change.
+
+Recording sessions are thread-local. Nested sessions in the same thread are
+rejected. Existing application register names and the one-argument
+`FileSystem#read_template_file` API remain unchanged.
+
+## Replay modes
+
+* `:compute` runs filters normally. Pass application filters with
+  `replayer.render(filters: MyFilters)`.
+* `:strict` returns each exact recorded filter result and rejects a changed
+  filter sequence. This can replay application-specific or nondeterministic
+  filters without loading their implementations.
+* `:verify` computes normally and raises when the final output differs.

lib/liquid.rb

@@ -90,3 +90,4 @@ module Liquid
 require 'liquid/usage'
 require 'liquid/registers'
 require 'liquid/template_factory'
+require "liquid/template_recorder"

lib/liquid/block_body.rb

@@ -84,10 +84,15 @@ def self.raise_missing_variable_terminator(token, parse_context)
 
     # @api private
     def self.render_node(context, output, node)
+      recorder = TemplateRecorder::HOOKS_ENABLED ? TemplateRecorder.current : nil
+      tag_call = recorder&.begin_tag_render(node, context)
+      output_start = output.length
       node.render_to_output_buffer(context, output)
     rescue => exc
       blank_tag = !node.instance_of?(Variable) && node.blank?
       rescue_render_node(context, output, node.line_number, exc, blank_tag)
+    ensure
+      recorder&.finish_tag_render(tag_call, output[output_start..]) if output_start
     end
 
     # @api private

lib/liquid/context.rb

@@ -228,6 +228,8 @@ def find_variable(key, raise_on_not_found: true)
 
       liquid_variable.context = self if variable != liquid_variable && liquid_variable.respond_to?(:context=)
 
+      recorder = @registers[TemplateRecorder::REGISTER_KEY] if defined?(TemplateRecorder) && TemplateRecorder::HOOKS_ENABLED
+      recorder&.emit_variable_read(key, liquid_variable)
       liquid_variable
     end
 

lib/liquid/drop.rb

@@ -37,11 +37,20 @@ def liquid_method_missing(method)
 
     # called by liquid to invoke a drop
     def invoke_drop(method_or_key)
-      if self.class.invokable?(method_or_key)
+      result = if self.class.invokable?(method_or_key)
         send(method_or_key)
       else
         liquid_method_missing(method_or_key)
       end
+
+      # A host application may assign its own object as a drop's context, and a
+      # drop can be invoked outside any render. Neither case has registers, and
+      # instrumentation must never turn either into a NoMethodError.
+      if defined?(TemplateRecorder) && TemplateRecorder::HOOKS_ENABLED && @context.respond_to?(:registers)
+        recorder = @context.registers[TemplateRecorder::REGISTER_KEY]
+      end
+      recorder&.emit_drop_read(self, method_or_key, result)
+      result
     end
 
     def key?(_name)

lib/liquid/partial_cache.rb

@@ -10,6 +10,10 @@ def self.load(template_name, context:, parse_context:)
 
       file_system = context.registers[:file_system]
       source      = file_system.read_template_file(template_name)
+      if defined?(TemplateRecorder) && TemplateRecorder::HOOKS_ENABLED
+        recorder = context.registers[TemplateRecorder::REGISTER_KEY]
+      end
+      recorder&.emit_file_read(template_name, source)
 
       parse_context.partial = true
 

lib/liquid/strainer_template.rb

@@ -48,13 +48,21 @@ def filter_methods
     end
 
     def invoke(method, *args)
-      if self.class.invokable?(method)
+      result = if self.class.invokable?(method)
         send(method, *args)
       elsif @context.strict_filters
         raise Liquid::UndefinedFilter, "undefined filter #{method}"
       else
         args.first
       end
+
+      # See Drop#invoke_drop: the context is not guaranteed to be a
+      # Liquid::Context, and instrumentation must not raise.
+      if defined?(TemplateRecorder) && TemplateRecorder::HOOKS_ENABLED && @context.respond_to?(:registers)
+        recorder = @context.registers[TemplateRecorder::REGISTER_KEY]
+      end
+      recorder&.emit_filter_call(method, args.first, args.drop(1), result)
+      result
     rescue ::ArgumentError => e
       raise Liquid::ArgumentError, e.message, e.backtrace
     end

lib/liquid/template.rb

@@ -105,6 +105,9 @@ def parse(source, options = {})
 
       tokenizer     = parse_context.new_tokenizer(source, start_line_number: @line_numbers && 1)
       @root         = Document.parse(tokenizer, parse_context)
+      if defined?(TemplateRecorder) && TemplateRecorder::HOOKS_ENABLED && TemplateRecorder.current
+        @template_recorder_source = source.dup.freeze
+      end
       self
     end
 
@@ -141,6 +144,10 @@ def errors
     def render(*args)
       return '' if @root.nil?
 
+      if defined?(TemplateRecorder) && TemplateRecorder::HOOKS_ENABLED
+        recording_session = TemplateRecorder.current
+      end
+      recording_assigns = args.first
       context = case args.first
       when Liquid::Context
         c = args.shift
@@ -180,6 +187,13 @@ def render(*args)
         context.add_filters(args.pop)
       end
 
+      recording = recording_session&.begin_render(self, recording_assigns, context)
+      if recording
+        recorder_registers = context.registers.static
+        previous_recorder = recorder_registers[TemplateRecorder::REGISTER_KEY]
+        recorder_registers[TemplateRecorder::REGISTER_KEY] = recording
+      end
+
       # Retrying a render resets resource usage
       context.resource_limits.reset
 
@@ -189,12 +203,43 @@ def render(*args)
 
       context.template_name ||= name
 
+      rendered_output = nil
+      render_succeeded = false
+      # A caller may hand in a buffer that already holds bytes this template did
+      # not produce, and may keep appending to it after this render returns. The
+      # recording must describe only what the template rendered, so remember
+      # where its output starts.
+      recorded_output_start = recording ? (output || '').bytesize : 0
       begin
         # render the nodelist.
-        @root.render_to_output_buffer(context, output || +'')
+        rendered_output = @root.render_to_output_buffer(context, output || +'')
+        render_succeeded = true
+        rendered_output
       rescue Liquid::MemoryError => e
-        context.handle_error(e)
+        rendered_output = context.handle_error(e)
+        # The caller still gets the error text, but a render that hit the memory
+        # limit is truncated by definition: recording it as a success would put
+        # a partial render into the corpus as if it were the specified output.
+        render_succeeded = false
+        recorded_output_start = 0
+        rendered_output
       ensure
+        if recording
+          if previous_recorder
+            recorder_registers[TemplateRecorder::REGISTER_KEY] = previous_recorder
+          else
+            recorder_registers.delete(TemplateRecorder::REGISTER_KEY)
+          end
+          # Slice off any caller-supplied prefix, and copy: the buffer is the
+          # caller's and may still be appended to, which would otherwise leak
+          # bytes into a recording that has already been taken.
+          recorded_output = if rendered_output.is_a?(String)
+            rendered_output.byteslice(recorded_output_start, rendered_output.bytesize - recorded_output_start).dup
+          else
+            rendered_output
+          end
+          recording_session.finish_render(recording, recorded_output, context, success: render_succeeded)
+        end
         @errors = context.errors
       end
     end