= 500
+
+ has_signal = unreachable || received_404 || system_error
+ return nil unless has_signal || policy_action.present?
+
+ {
+ unreachable: unreachable,
+ received_404: received_404,
+ system_error: system_error,
+ policy_action: policy_action
+ }.compact
+ end
+
+ private_class_method(
+ :fetch_for_mode,
+ :fetch_internal_or_legacy,
+ :fetch_with_shadow,
+ :run_shadow_compare,
+ :use_internal_fetch?,
+ :fetch_eligibility,
+ :legacy_fetch,
+ :internal_fetch,
+ :guarded_internal_fetch,
+ :fetch_blocked_result,
+ :log_internal_ineligible,
+ :log_internal_eligibility,
+ :log_shadow_skipped,
+ :log_fetch_path,
+ :annotate_ineligibility,
+ :log_fetch_outcome,
+ :structured_log_payload,
+ :detect_content_type,
+ :redirected?,
+ :response_contract,
+ :abort_structure?,
+ :control_structure?,
+ :extract_headers,
+ :normalize_header_key,
+ :extract_final_url,
+ :extract_redirect_chain,
+ :normalize_redirect_chain,
+ :extract_signals,
+ :extract_hints,
+ :looks_like_json?,
+ :looks_like_html?,
+ :canonical_wringer_signals
+ )
+ end
+end
diff --git a/app/services/distillator/fetch_shadow_comparator.rb b/app/services/distillator/fetch_shadow_comparator.rb
new file mode 100644
index 00000000..761c9a24
--- /dev/null
+++ b/app/services/distillator/fetch_shadow_comparator.rb
@@ -0,0 +1,111 @@
+require "digest/sha2"
+require "json"
+
+module Distillator
+ class FetchShadowComparator
+ WRINGER_KEYS = %i[
+ error_type
+ received_404
+ system_error
+ unreachable
+ ].freeze
+
+ def self.compare(url:, legacy:, internal:, logger: Rails.logger)
+ new(url: url, legacy: legacy, internal: internal, logger: logger).compare
+ rescue StandardError => e
+ logger.warn(
+ event: "distillator.fetch_shadow.compare_error",
+ url: url,
+ error_class: e.class.name,
+ error: e.message
+ )
+ nil
+ end
+
+ def initialize(url:, legacy:, internal:, logger:)
+ @url = url
+ @legacy = legacy || {}
+ @internal = internal || {}
+ @logger = logger || Rails.logger
+ end
+
+ def compare
+ mismatches = comparable_keys.filter_map do |key|
+ legacy_value = comparable(legacy).fetch(key)
+ internal_value = comparable(internal).fetch(key)
+ next if legacy_value == internal_value
+
+ { field: key, legacy: legacy_value, internal: internal_value }
+ end
+
+ logger.info(
+ event: "distillator.fetch_shadow.compare",
+ url: url,
+ matched: mismatches.empty?,
+ mismatches: mismatches
+ )
+
+ mismatches
+ rescue StandardError => e
+ logger.warn(
+ event: "distillator.fetch_shadow.compare_error",
+ url: url,
+ error_class: e.class.name,
+ error: e.message
+ )
+ nil
+ end
+
+ private
+
+ attr_reader :url, :legacy, :internal, :logger
+
+ def comparable_keys
+ comparable(legacy).keys
+ end
+
+ def comparable(response)
+ {
+ status: response[:status],
+ body_hash: body_hash(response[:body]),
+ headers: normalize_headers(response[:headers]),
+ final_url: response[:final_url],
+ redirect_chain: Array(response[:redirect_chain]).map(&:to_s),
+ wringer_error_type: wringer_value(response, :error_type),
+ wringer_received_404: wringer_value(response, :received_404),
+ wringer_system_error: wringer_value(response, :system_error),
+ wringer_unreachable: wringer_value(response, :unreachable)
+ }
+ end
+
+ def body_hash(body)
+ Digest::SHA256.hexdigest(JSON.generate(normalize_body(body)))
+ rescue StandardError
+ Digest::SHA256.hexdigest(body.to_s)
+ end
+
+ def normalize_body(value)
+ case value
+ when Hash
+ value.sort_by { |key, _| key.to_s }.to_h { |key, child| [key.to_s, normalize_body(child)] }
+ when Array
+ value.map { |child| normalize_body(child) }
+ else
+ value
+ end
+ end
+
+ def normalize_headers(headers)
+ return {} unless headers.is_a?(Hash)
+
+ headers.sort_by { |key, _| key.to_s }.to_h
+ end
+
+ def wringer_value(response, key)
+ wringer = response[:wringer]
+ return unless wringer.is_a?(Hash)
+
+ wringer[key] || wringer[key.to_s]
+ end
+ end
+end
diff --git a/app/services/distillator/graph_diff.rb b/app/services/distillator/graph_diff.rb
new file mode 100644
index 00000000..09a222d7
--- /dev/null
+++ b/app/services/distillator/graph_diff.rb
@@ -0,0 +1,116 @@
+require "json"
+
+module Distillator
+ class GraphDiff
+ Result = Struct.new(
+ :added,
+ :removed,
+ :same_count,
+ :added_count,
+ :removed_count,
+ :changed_literal_values,
+ :changed_uri_objects,
+ keyword_init: true
+ ) do
+ def to_h
+ {
+ added: added,
+ removed: removed,
+ same_count: same_count,
+ added_count: added_count,
+ removed_count: removed_count,
+ changed_literal_values: changed_literal_values,
+ changed_uri_objects: changed_uri_objects
+ }
+ end
+ end
+
+ def self.call(expected_graph:, actual_graph:)
+ new(expected_graph: expected_graph, actual_graph: actual_graph).call
+ end
+
+ def initialize(expected_graph:, actual_graph:)
+ @expected_graph = coerce_graph(expected_graph)
+ @actual_graph = coerce_graph(actual_graph)
+ end
+
+ def call
+ expected_lines = statement_lines(expected_graph)
+ actual_lines = statement_lines(actual_graph)
+
+ added = actual_lines - expected_lines
+ removed = expected_lines - actual_lines
+ same = expected_lines & actual_lines
+
+ Result.new(
+ added: added,
+ removed: removed,
+ same_count: same.count,
+ added_count: added.count,
+ removed_count: removed.count,
+ changed_literal_values: changed_objects(expected_graph, actual_graph, literal_only: true),
+ changed_uri_objects: changed_objects(expected_graph, actual_graph, uri_only: true)
+ )
+ end
+
+ private
+
+ attr_reader :expected_graph, :actual_graph
+
+ def coerce_graph(value)
+ return value if value.is_a?(RDF::Graph)
+
+ string = value.to_s
+ return load_jsonld_graph(string) if json?(string)
+
+ RDF::Graph.new << RDF::Reader.for(:nquads).new(string)
+ end
+
+ def load_jsonld_graph(string)
+ RDF::Graph.new << JSON::LD::API.toRdf(JSON.parse(string))
+ end
+
+ def json?(string)
+ JSON.parse(string)
+ true
+ rescue JSON::ParserError
+ false
+ end
+
+ def statement_lines(graph)
+ graph.statements.map { |statement| statement.to_ntriples.strip }.sort
+ end
+
+ def changed_objects(left_graph, right_graph, literal_only: false, uri_only: false)
+ left_index = index_by_subject_predicate(left_graph)
+ right_index = index_by_subject_predicate(right_graph)
+
+ (left_index.keys & right_index.keys).sort.map do |key|
+ left_objects = left_index[key]
+ right_objects = right_index[key]
+ next if left_objects == right_objects
+
+ changed = {
+ subject: key[0],
+ predicate: key[1],
+ expected: left_objects,
+ actual: right_objects
+ }
+
+ next if literal_only && !(left_objects + right_objects).all? { |obj| obj.start_with?("\"") }
+ next if uri_only && !(left_objects + right_objects).all? { |obj| obj.start_with?("<") }
+
+ changed
+ end.compact
+ end
+
+ def index_by_subject_predicate(graph)
+ graph.statements.each_with_object({}) do |statement, index|
+ key = [statement.subject.to_ntriples.strip, statement.predicate.to_ntriples.strip]
+ index[key] ||= []
+ index[key] << statement.object.to_ntriples.strip
+ index[key].sort!
+ end
+ end
+ end
+end
diff --git a/app/services/distillator/html_absolutizer.rb b/app/services/distillator/html_absolutizer.rb
new file mode 100644
index 00000000..ac85960f
--- /dev/null
+++ b/app/services/distillator/html_absolutizer.rb
@@ -0,0 +1,7 @@
+module Distillator
+ class HtmlAbsolutizer
+ def self.call(html:, base_url:)
+ Distillator::HtmlRewriter.absolute_src(html, base_url)
+ end
+ end
+end
diff --git a/app/services/distillator/html_rewriter.rb b/app/services/distillator/html_rewriter.rb
new file mode 100644
index 00000000..cd6199d5
--- /dev/null
+++ b/app/services/distillator/html_rewriter.rb
@@ -0,0 +1,94 @@
+require "cgi"
+require "nokogiri"
+require "uri"
+
+module Distillator
+ class HtmlRewriter
+ PLACEHOLDER_PREFIX = "__distillator_absolute_attr__".freeze
+
+ def self.absolute_src(html, url)
+ new(html, url).absolute_src
+ end
+
+ def initialize(html, url)
+ @html = html.to_s
+ @url = url.to_s
+ end
+
+ def absolute_src
+ fragment = Nokogiri::HTML::DocumentFragment.parse(html)
+ base_url = resolved_base_url(fragment)
+ passthroughs = {}
+
+ rewrite_attributes(fragment, "src", base_url, passthroughs: passthroughs)
+ rewrite_attributes(fragment, "href", base_url, passthroughs: passthroughs, exclude: ["base"])
+
+ restore_passthrough_values(fragment.to_html, passthroughs)
+ rescue StandardError => e
+ Rails.logger.error("Error in Distillator::HtmlRewriter.absolute_src: #{e.inspect}")
+ html
+ end
+
+ private
+
+ attr_reader :html, :url
+
+ def rewrite_attributes(fragment, attribute_name, base_url, passthroughs:, exclude: [])
+ fragment.css("[#{attribute_name}]").each do |node|
+ next if exclude.include?(node.name)
+
+ value = node[attribute_name]
+ next unless value.present?
+
+ if passthrough_value?(value)
+ placeholder = "#{PLACEHOLDER_PREFIX}#{passthroughs.length}__"
+ passthroughs[placeholder] = value
+ node[attribute_name] = placeholder
+ else
+ node[attribute_name] = absolute_url(value, base_url)
+ end
+ end
+ end
+
+ def resolved_base_url(fragment)
+ base_href = fragment.at_css("base[href]")&.[]("href")
+ return url if base_href.blank?
+
+ absolute_url(base_href, url)
+ end
+
+ def absolute_url(value, base_url = url)
+ value = value.to_s
+ return value if value.blank?
+ return value if passthrough_value?(value)
+
+ if value.start_with?("//")
+ scheme = URI.parse(base_url).scheme
+ return value unless scheme.present?
+
+ return "#{scheme}:#{value}"
+ end
+
+ URI.join(base_url, value).to_s
+ rescue StandardError
+ value
+ end
+
+ def passthrough_value?(value)
+ CGI.unescapeHTML(value).match?(/\s/) ||
+ value.match?(/%20/i) ||
+ value.start_with?("#") ||
+ value.start_with?("mailto:") ||
+ value.start_with?("tel:") ||
+ value.start_with?("data:")
+ end
+
+ def restore_passthrough_values(output, passthroughs)
+ rewritten = output.to_s
+ passthroughs.each do |placeholder, original|
+ rewritten = rewritten.gsub(placeholder, original)
+ end
+ rewritten
+ end
+ end
+end
diff --git a/app/services/distillator/inspection_links.rb b/app/services/distillator/inspection_links.rb
new file mode 100644
index 00000000..db1bdfbd
--- /dev/null
+++ b/app/services/distillator/inspection_links.rb
@@ -0,0 +1,55 @@
+module Distillator
+ class InspectionLinks
+ def self.call(url:, website: nil, payload: nil)
+ new(url: url, website: website, payload: payload).call
+ end
+
+ def initialize(url:, website: nil, payload: nil)
+ @url = url
+ @website = website
+ @payload = payload
+ end
+
+ def call
+ links = []
+ links << { label: "Source website", url: url } if url.present?
+ links << { label: "Compare", url: payload[:compare_url] } if payload[:compare_url].present?
+ links << { label: "Compare extracted statements", url: compare_extracted_statements_url } if compare_extracted_statements_url.present?
+ links << { label: payload[:label], url: payload[:active_cache_url] } if payload[:label].present? && payload[:active_cache_url].present?
+
+ Array(payload[:secondary_links]).each do |link|
+ next if link[:label].blank? || link[:url].blank?
+ next if link[:url] == payload[:compare_url]
+
+ links << { label: link[:label], url: link[:url] }
+ end
+
+ links << { label: "Webpage record", url: helpers.webpage_path(webpage) } if webpage.present?
+ links.uniq { |link| [link[:label], link[:url]] }
+ end
+
+ private
+
+ attr_reader :url, :website
+
+ def payload
+ @payload ||= Distillator::CacheLinkResolver.call(url: url, website: website)
+ end
+
+ def webpage
+ @webpage ||= website&.webpages&.find_by(url: url)
+ end
+
+ def helpers
+ Rails.application.routes.url_helpers
+ end
+
+ def compare_extracted_statements_url
+ return if url.blank?
+
+ params = { url: url }
+ params[:website_id] = website.id if website.present?
+ helpers.compare_extracted_statements_path(params)
+ end
+ end
+end
diff --git a/app/services/distillator/json_post_fetcher.rb b/app/services/distillator/json_post_fetcher.rb
new file mode 100644
index 00000000..84133da3
--- /dev/null
+++ b/app/services/distillator/json_post_fetcher.rb
@@ -0,0 +1,19 @@
+module Distillator
+ class JsonPostFetcher
+ def self.call(agent:, url:, scrape_options:)
+ agent.post(url, request_body(scrape_options), request_headers(scrape_options))
+ end
+
+ def self.request_headers(scrape_options)
+ headers = scrape_options[:headers] || scrape_options["headers"] || {}
+ normalized = headers.to_h.each_with_object({}) { |(key, value), memo| memo[key.to_s] = value }
+ return normalized if normalized.keys.any? { |key| key.downcase == "content-type" }
+
+ normalized.merge("Content-Type" => "application/json")
+ end
+
+ def self.request_body(scrape_options)
+ scrape_options[:request_body] || scrape_options["request_body"] || ""
+ end
+ end
+end
diff --git a/app/services/distillator/legacy_wringer_fetch.rb b/app/services/distillator/legacy_wringer_fetch.rb
new file mode 100644
index 00000000..455cf0b1
--- /dev/null
+++ b/app/services/distillator/legacy_wringer_fetch.rb
@@ -0,0 +1,99 @@
+module Distillator
+ class LegacyWringerFetch
+ def self.fetch(
+ url:,
+ render_js:,
+ scrape_options:,
+ client: nil,
+ agent: nil,
+ use_wringer: nil,
+ safe_wringer_call: nil,
+ logger: nil
+ )
+ return fetch_with_client(client, url: url, render_js: render_js, scrape_options: scrape_options) if client
+
+ client = Dsl::Support::WringerClient.new(
+ agent: agent || Mechanize.new,
+ render_js: render_js,
+ scrape_options: scrape_options,
+ use_wringer: use_wringer || ApplicationController.helpers.method(:use_wringer),
+ safe_wringer_call: safe_wringer_call || ApplicationController.helpers.method(:safe_wringer_call),
+ logger: logger || Rails.logger
+ )
+
+ client.fetch(url: url)
+ end
+
+ def self.fetch_wringer_backed(url:, render_js:, scrape_options:, agent:, use_wringer:, safe_wringer_call:, logger:)
+ agent = agent || Mechanize.new
+ use_wringer = use_wringer || ApplicationController.helpers.method(:use_wringer)
+ safe_wringer_call = safe_wringer_call || ApplicationController.helpers.method(:safe_wringer_call)
+ logger = logger || Rails.logger
+
+ raw_response = nil
+ safe_result = Distillator::FetchService.invoke_safe_wringer_call(safe_wringer_call) do
+ raw_response = fetch_wringer_response(
+ agent: agent,
+ use_wringer: use_wringer,
+ url: url,
+ render_js: render_js,
+ scrape_options: scrape_options
+ )
+ end
+
+ control = Distillator::FetchService.normalize_control_result(safe_result)
+ if control.present?
+ metadata = Distillator::FetchService.fetch_metadata(
+ Distillator::FetchService.normalize_fetch_result(raw_response, logger),
+ raw_response,
+ agent
+ )
+ return {
+ status: :abort,
+ body: control,
+ headers: metadata[:headers],
+ final_url: metadata[:final_url],
+ redirect_chain: metadata[:redirect_chain],
+ wringer: Distillator::FetchService.build_wringer_status(control, raw_response),
+ http_code: raw_response.respond_to?(:code) ? raw_response.code.to_i : nil,
+ raw_body: raw_response.respond_to?(:body) ? raw_response.body : nil
+ }
+ end
+
+ normalized = Distillator::FetchService.normalize_fetch_result(safe_result, logger)
+ metadata = Distillator::FetchService.fetch_metadata(normalized, raw_response, agent)
+
+ {
+ status: :ok,
+ body: normalized[:body],
+ headers: metadata[:headers],
+ final_url: metadata[:final_url],
+ redirect_chain: metadata[:redirect_chain],
+ wringer: Distillator::FetchService.build_wringer_status(safe_result, raw_response) ||
+ Distillator::FetchService.build_wringer_status(normalized, raw_response) || {},
+ http_code: normalized[:http_code],
+ raw_body: raw_response.respond_to?(:body) ? raw_response.body : normalized[:body]
+ }
+ end
+
+ def self.fetch_with_client(client, url:, render_js:, scrape_options:)
+ if client.is_a?(Dsl::Support::WringerClient)
+ client.fetch(url: url, render_js: render_js, scrape_options: scrape_options)
+ else
+ client.fetch(url: url)
+ end
+ end
+
+ def self.fetch_wringer_response(agent:, use_wringer:, url:, render_js:, scrape_options:)
+ wringer_url = use_wringer.call(url, render_js, scrape_options)
+
+ if agent.respond_to?(:get)
+ agent.get(wringer_url)
+ else
+ agent.get_file(wringer_url)
+ end
+ end
+
+ private_class_method :fetch_with_client, :fetch_wringer_response
+ end
+end
diff --git a/app/services/distillator/migration_comparator.rb b/app/services/distillator/migration_comparator.rb
new file mode 100644
index 00000000..a6eabeeb
--- /dev/null
+++ b/app/services/distillator/migration_comparator.rb
@@ -0,0 +1,348 @@
+require "digest/sha2"
+require "json"
+
+module Distillator
+ class MigrationComparator
+ MATCH = "MATCH".freeze
+ MISSING_STATEMENT = "MISSING_STATEMENT".freeze
+ EXTRA_STATEMENT = "EXTRA_STATEMENT".freeze
+ DIFFERENT_VALUE = "DIFFERENT_VALUE".freeze
+ PARSE_ERROR = "PARSE_ERROR".freeze
+
+ STATEMENT_VERDICT_PRECEDENCE = [
+ PARSE_ERROR,
+ DIFFERENT_VALUE,
+ MISSING_STATEMENT,
+ EXTRA_STATEMENT
+ ].freeze
+
+ STATEMENT_METADATA_KEYS = %w[
+ cache_changed
+ cache_refreshed
+ created_at
+ debug
+ duration_ms
+ generatedAt
+ probe
+ recorded_at
+ timing
+ trace
+ updated_at
+ ].freeze
+
+ Result = Struct.new(
+ :statement_verdict,
+ :statement_artifacts,
+ :export_verdict,
+ :export_artifacts,
+ :fetch_diagnostic_verdict,
+ :fetch_artifacts,
+ keyword_init: true
+ )
+
+ StatementComparison = Struct.new(:verdict, :artifacts, keyword_init: true)
+ ExportComparison = Struct.new(:verdict, :artifacts, keyword_init: true)
+ FetchComparison = Struct.new(:verdict, :artifacts, keyword_init: true)
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(
+ website:,
+ webpage:,
+ legacy_html:,
+ condenser_html:,
+ legacy_fetch: nil,
+ condenser_fetch: nil,
+ default_language: nil,
+ sources: nil,
+ refresh_helper: nil,
+ source_resolver: Statements::RefreshWebpageStatementsService,
+ export_service: ExportArtsdataService
+ )
+ @website = website
+ @webpage = webpage
+ @legacy_html = legacy_html
+ @condenser_html = condenser_html
+ @legacy_fetch = legacy_fetch
+ @condenser_fetch = condenser_fetch
+ @default_language = default_language || website.default_language
+ @sources = sources
+ @refresh_helper = refresh_helper || StatementsHelper.build_refresh_proxy(cookies: {})
+ @source_resolver = source_resolver
+ @export_service = export_service
+ end
+
+ def call
+ statement_result = compare_statements
+
+ if statement_result.verdict == MATCH
+ export_result = compare_export
+ Result.new(
+ statement_verdict: statement_result.verdict,
+ statement_artifacts: statement_result.artifacts,
+ export_verdict: export_result.verdict,
+ export_artifacts: export_result.artifacts,
+ fetch_diagnostic_verdict: nil,
+ fetch_artifacts: nil
+ )
+ else
+ fetch_result = compare_fetch_when_needed
+ Result.new(
+ statement_verdict: statement_result.verdict,
+ statement_artifacts: statement_result.artifacts,
+ export_verdict: nil,
+ export_artifacts: nil,
+ fetch_diagnostic_verdict: fetch_result&.verdict,
+ fetch_artifacts: fetch_result&.artifacts
+ )
+ end
+ end
+
+ def compare_statements
+ artifacts = resolved_sources.sort_by { |source| [source.property_id.to_i, source.id.to_i] }.map do |source|
+ compare_source(source)
+ end
+
+ StatementComparison.new(
+ verdict: overall_statement_verdict(artifacts),
+ artifacts: artifacts
+ )
+ end
+
+ def compare_export
+ actual = Distillator::ExportNormalizer.normalize(export_service.call(seedurl: website.seedurl))
+ expected = Distillator::ExportNormalizer.normalize(export_service.production_equivalent(seedurl: website.seedurl))
+
+ ExportComparison.new(
+ verdict: actual == expected ? MATCH : DIFFERENT_VALUE,
+ artifacts: {
+ actual: actual,
+ expected: expected
+ }
+ )
+ end
+
+ def compare_fetch_when_needed
+ return nil if legacy_fetch.nil? || condenser_fetch.nil?
+
+ legacy = normalize_fetch_artifact(legacy_fetch)
+ condenser = normalize_fetch_artifact(condenser_fetch)
+
+ FetchComparison.new(
+ verdict: legacy == condenser ? MATCH : DIFFERENT_VALUE,
+ artifacts: {
+ legacy: legacy,
+ condenser: condenser
+ }
+ )
+ end
+
+ private
+
+ attr_reader :website, :webpage, :legacy_html, :condenser_html, :legacy_fetch, :condenser_fetch,
+ :default_language, :sources, :refresh_helper, :source_resolver, :export_service
+
+ def resolved_sources
+ explicit_sources = Array(sources).presence
+ return explicit_sources if explicit_sources
+
+ source_resolver
+ .sources_for_webpage(webpage, default_language: default_language)
+ .select { |source| source.selected? }
+ end
+
+ def compare_source(source)
+ legacy = extract_value(source, html: legacy_html, side: :legacy)
+ condenser = extract_value(source, html: condenser_html, side: :condenser)
+
+ {
+ source_id: source.id,
+ property_id: source.property_id,
+ property_label: source.property.label,
+ language: source.language,
+ legacy: legacy[:value],
+ condenser: condenser[:value],
+ legacy_error: normalized_error(legacy[:error]),
+ condenser_error: normalized_error(condenser[:error]),
+ verdict: classify_statement_verdict(legacy, condenser)
+ }
+ end
+
+ def extract_value(source, html:, side:)
+ if html.blank?
+ return {
+ value: nil,
+ error: {
+ error: "#{side.to_s.humanize} HTML is not available for comparison",
+ error_type: "MissingComparisonHtml",
+ source: "migration_comparator"
+ }
+ }
+ end
+
+ runner = Dsl::Core::AlgorithmRunner.new(
+ url: webpage.url,
+ render_js: source.render_js,
+ scrape_options: refresh_helper.statement_scrape_options(
+ source: source,
+ webpage: webpage,
+ scrape_options: { force_scrape_every_hrs: 0 }
+ ),
+ tracer: Dsl::Tracing::NullTracer.new
+ )
+ runner.instance_variable_set(:@html, html)
+ runner.instance_variable_set(:@page, Nokogiri::HTML(html, nil, Encoding::UTF_8.to_s))
+ runner.define_singleton_method(:fetch_result_for) do |url:, render_js:, scrape_options:|
+ {
+ status: :abort,
+ body: [
+ "abort_update",
+ {
+ error: "Read-only migration comparison cannot fetch additional URLs",
+ error_type: "MigrationComparatorFetchUnsupported",
+ source: "migration_comparator",
+ step: "url"
+ }
+ ],
+ headers: {},
+ final_url: url,
+ redirect_chain: [url],
+ duration_ms: 0
+ }
+ end
+
+ raw = runner.run(source.algorithm_value)
+ return { value: nil, error: raw.second.to_h } if abort_structure?(raw)
+
+ formatted = refresh_helper.format_datatype(raw, source.property, webpage, statement_status: "initial")
+ return { value: nil, error: formatted.second.to_h } if abort_structure?(formatted)
+
+ normalized_value = normalize_statement_value(formatted)
+ { value: blank_value?(normalized_value) ? nil : normalized_value, error: nil }
+ rescue StandardError => error
+ {
+ value: nil,
+ error: {
+ error: error.message,
+ error_type: error.class.to_s,
+ source: "migration_comparator"
+ }
+ }
+ end
+
+ def overall_statement_verdict(artifacts)
+ verdicts = artifacts.map { |artifact| artifact[:verdict] }.uniq
+ return MATCH if verdicts == [MATCH]
+
+ STATEMENT_VERDICT_PRECEDENCE.find { |verdict| verdicts.include?(verdict) } || DIFFERENT_VALUE
+ end
+
+ def classify_statement_verdict(legacy, condenser)
+ return PARSE_ERROR if legacy[:error].present? || condenser[:error].present?
+ return MATCH if normalized_statement_value_equal?(legacy[:value], condenser[:value])
+ return MISSING_STATEMENT if blank_value?(condenser[:value]) && !blank_value?(legacy[:value])
+ return EXTRA_STATEMENT if blank_value?(legacy[:value]) && !blank_value?(condenser[:value])
+
+ DIFFERENT_VALUE
+ end
+
+ def normalized_statement_value_equal?(left, right)
+ normalize_statement_value(left) == normalize_statement_value(right)
+ end
+
+ def normalize_statement_value(value)
+ case value
+ when Hash
+ value
+ .reject { |key, _| STATEMENT_METADATA_KEYS.include?(key.to_s) }
+ .sort_by { |key, _| key.to_s }
+ .each_with_object({}) do |(key, child), out|
+ out[key.to_s] = normalize_statement_value(child)
+ end
+ when Array
+ value.map { |entry| normalize_statement_value(entry) }
+ .sort_by { |entry| JSON.generate(entry) }
+ when String
+ value.gsub(/\s+/, " ").strip
+ else
+ value
+ end
+ end
+
+ def normalized_error(error)
+ return nil unless error.respond_to?(:to_h)
+
+ error.to_h.slice("error", "error_type", "source", "step").transform_keys(&:to_s)
+ end
+
+ def blank_value?(value)
+ return true if value.blank?
+ return value.all? { |entry| blank_value?(entry) } if value.is_a?(Array)
+ return value.values.all? { |entry| blank_value?(entry) } if value.is_a?(Hash)
+
+ false
+ end
+
+ def abort_structure?(value)
+ value.is_a?(Array) && value.first == "abort_update" && value.second.respond_to?(:to_h)
+ end
+
+ def normalize_fetch_artifact(response)
+ {
+ status: response_value(response, :status).to_s,
+ http_code: extract_http_code(response),
+ final_url: response_value(response, :final_url).presence,
+ redirect_chain: Array(response_value(response, :redirect_chain)).map(&:to_s),
+ body_digest: body_digest(response_value(response, :body)),
+ body_length: body_length(response_value(response, :body)),
+ content_type: extract_content_type(response)
+ }
+ end
+
+ def extract_http_code(response)
+ response_value(response, :http_code) ||
+ response_value(response, :http_response_code) ||
+ nested_response_value(response, :wringer, :http_code)
+ end
+
+ def extract_content_type(response)
+ headers = response_value(response, :headers)
+ return unless headers.respond_to?(:[])
+
+ headers[:content_type] ||
+ headers["content_type"] ||
+ headers["Content-Type"] ||
+ headers[:"Content-Type"]
+ end
+
+ def body_digest(body)
+ Digest::SHA256.hexdigest(serialized_body(body))
+ end
+
+ def body_length(body)
+ serialized_body(body).bytesize
+ end
+
+ def serialized_body(body)
+ case body
+ when Hash, Array
+ JSON.generate(body)
+ else
+ body.to_s
+ end
+ end
+
+ def response_value(response, key)
+ response[key] || response[key.to_s]
+ end
+
+ def nested_response_value(response, parent_key, child_key)
+ parent = response_value(response, parent_key)
+ return unless parent.respond_to?(:[])
+
+ parent[child_key] || parent[child_key.to_s]
+ end
+ end
+end
diff --git a/app/services/distillator/migration_status.rb b/app/services/distillator/migration_status.rb
new file mode 100644
index 00000000..9e7ce5a8
--- /dev/null
+++ b/app/services/distillator/migration_status.rb
@@ -0,0 +1,43 @@
+module Distillator
+ class MigrationStatus
+ CHECKS = [
+ ["Fetch cache model present", -> { model_present? }],
+ ["Native refresh regression", -> { test_file_present?("test/integration/distillator_refresh_rdf_uri_cache_test.rb") }],
+ ["ResourceList validation", -> { test_file_present?("test/jobs/add_webpages_job_test.rb") }],
+ ["DSL golden fixtures", -> { test_file_present?("test/services/dsl/distillator_golden_fixture_test.rb") }],
+ ["Export invariance fixture", -> { test_file_present?("test/services/distillator/export_invariance_test.rb") }],
+ ["Render-JS fallback coverage", -> { test_file_present?("test/integration/distillator_render_js_export_test.rb") }],
+ ["JSON POST fallback coverage", -> { test_file_present?("test/integration/distillator_json_post_export_test.rb") }],
+ ["Manual link export coverage", -> { test_file_present?("test/integration/distillator_manual_link_export_test.rb") }]
+ ].freeze
+
+ def self.call
+ new.call
+ end
+
+ def call
+ {
+ checks: CHECKS.map { |label, fn| [label, fn.call] }.to_h,
+ legacy_bypasses: known_legacy_bypasses
+ }
+ end
+
+ private
+
+ def self.model_present?
+ defined?(Distillator::FetchCache).present?
+ end
+
+ def self.test_file_present?(path)
+ Rails.root.join(path).exist?
+ end
+
+ def known_legacy_bypasses
+ files = Dir[Rails.root.join("app/**/*.{rb}")].reject do |path|
+ path.include?("/services/wringer_client.rb") || path.end_with?("app/services/distillator/migration_status.rb")
+ end
+ matches = files.select { |path| File.read(path).include?("WringerClient.fetch(") }
+ { count: matches.count, files: matches.map { |path| Pathname(path).relative_path_from(Rails.root).to_s }.sort }
+ end
+ end
+end
diff --git a/app/services/distillator/native_fetch.rb b/app/services/distillator/native_fetch.rb
new file mode 100644
index 00000000..216b00f2
--- /dev/null
+++ b/app/services/distillator/native_fetch.rb
@@ -0,0 +1,308 @@
+module Distillator
+ class NativeFetch
+ BROWSER_USER_AGENT_ALIAS = "Mac Safari".freeze
+ HTTP_SERVER_ERROR_CODES = [500, 502, 503, 504].freeze
+ NETWORK_ERRORS = [
+ OpenSSL::SSL::SSLError,
+ Mechanize::Error,
+ SocketError,
+ Net::OpenTimeout,
+ Net::ReadTimeout,
+ Timeout::Error
+ ].freeze
+
+ def self.call(url:, render_js:, scrape_options:, agent: nil, logger: nil)
+ logger = logger || Rails.logger
+ if render_js
+ return Distillator::RenderedFetch.call(
+ url: url,
+ uri_key: scrape_options[:uri_key] || scrape_options["uri_key"] || Distillator::WringerUrlKey.call(url).uri_key,
+ iframe: iframe_request?(url: url, scrape_options: scrape_options),
+ agent: agent,
+ logger: logger
+ )
+ end
+
+ agent = configure_agent(agent || Mechanize.new)
+ log_request(logger, url: url, render_js: render_js, scrape_options: scrape_options)
+
+ raw_response = fetch_response(agent, url, scrape_options)
+ finalize_result(
+ url: url,
+ result: build_success_or_http_failure(raw_response, agent, logger),
+ scrape_options: scrape_options,
+ logger: logger
+ )
+ rescue Mechanize::ResponseCodeError => error
+ raw_response = response_from_error(error)
+ raise error unless raw_response
+
+ finalize_result(
+ url: url,
+ result: build_success_or_http_failure(raw_response, agent, logger),
+ scrape_options: scrape_options,
+ logger: logger
+ )
+ rescue OpenSSL::SSL::SSLError => error
+ retry_result = retry_with_ssl_verify_none(agent, url, scrape_options, logger)
+ return finalize_result(url: url, result: retry_result, scrape_options: scrape_options, logger: logger) if retry_result
+
+ finalize_result(
+ url: url,
+ result: failed_network_result(error),
+ scrape_options: scrape_options,
+ logger: logger
+ )
+ rescue *NETWORK_ERRORS => error
+ finalize_result(
+ url: url,
+ result: failed_network_result(error),
+ scrape_options: scrape_options,
+ logger: logger
+ )
+ end
+
+ def self.fetch(url:, render_js:, scrape_options:, agent: nil, logger: nil)
+ call(
+ url: url,
+ render_js: render_js,
+ scrape_options: scrape_options,
+ agent: agent,
+ logger: logger
+ )
+ end
+
+ def self.fetch_response(agent, url, scrape_options)
+ return fetch_post_response(agent, url, scrape_options) if Distillator::FetchService.json_post?(scrape_options) == true
+
+ if agent.respond_to?(:get)
+ agent.get(url)
+ else
+ agent.get_file(url)
+ end
+ end
+
+ def self.build_success_or_http_failure(raw_response, agent, logger)
+ normalized = Distillator::FetchService.normalize_fetch_result(raw_response, logger)
+ metadata = Distillator::FetchService.fetch_metadata(normalized, raw_response, agent)
+ control = Distillator::FetchService.normalize_control_result(normalized[:body])
+ http_code = normalized[:http_code].to_i
+
+ if control.present?
+ return control_result(control: control, metadata: metadata, raw_response: raw_response)
+ end
+
+ if http_code == 404 || HTTP_SERVER_ERROR_CODES.include?(http_code)
+ return http_failure_result(http_code: http_code, metadata: metadata, raw_response: raw_response)
+ end
+
+ {
+ status: :ok,
+ body: normalized[:body],
+ headers: metadata[:headers],
+ final_url: metadata[:final_url],
+ redirect_chain: metadata[:redirect_chain],
+ wringer: Distillator::FetchService.build_wringer_status(normalized, raw_response) || {},
+ http_code: http_code,
+ raw_body: normalized[:body]
+ }
+ end
+
+ def self.control_result(control:, metadata:, raw_response:)
+ {
+ status: :abort,
+ body: control,
+ headers: metadata[:headers],
+ final_url: metadata[:final_url],
+ redirect_chain: metadata[:redirect_chain],
+ wringer: Distillator::FetchService.build_wringer_status(control, raw_response),
+ http_code: nil,
+ raw_body: nil
+ }
+ end
+
+ def self.http_failure_result(http_code:, metadata:, raw_response:)
+ error_type = http_code == 404 ? "http_404" : "http_server_error"
+ retry_value = http_code == 404 ? false : true
+ control = [
+ "abort_update",
+ {
+ error: "HTTP #{http_code}",
+ error_type: error_type,
+ source: "native_fetch",
+ step: "url",
+ policy: {
+ action: "abort_update",
+ retry: retry_value,
+ cache: false
+ }
+ }
+ ]
+
+ {
+ status: :abort,
+ body: control,
+ headers: metadata[:headers],
+ final_url: metadata[:final_url],
+ redirect_chain: metadata[:redirect_chain],
+ wringer: Distillator::FetchService.build_wringer_status(control, raw_response),
+ http_code: http_code,
+ raw_body: raw_response.respond_to?(:body) ? raw_response.body : nil
+ }
+ end
+
+ def self.failed_network_result(error)
+ timeout = timeout_error?(error)
+ hints = timeout ? ["timeout"] : []
+ control = [
+ "abort_update",
+ {
+ error: error.message,
+ error_type: "NativeFetchError",
+ source: "native_fetch",
+ retry: true,
+ cache: false,
+ step: "url",
+ signals: { network_status: "failed", timeout: timeout },
+ hints: hints
+ }
+ ]
+
+ {
+ status: :abort,
+ body: control,
+ headers: {},
+ final_url: nil,
+ redirect_chain: [],
+ wringer: Distillator::FetchService.build_wringer_status(control),
+ http_code: nil,
+ raw_body: nil
+ }
+ end
+
+ def self.finalize_result(url:, result:, scrape_options:, logger:)
+ payload = Distillator::FetchService.enrich_wringer_metadata(url: url, result: result)
+ payload = annotate_request_metadata(payload, scrape_options: scrape_options)
+ log_response(logger, payload)
+ payload
+ end
+
+ def self.timeout_error?(error)
+ error.is_a?(Net::OpenTimeout) ||
+ error.is_a?(Net::ReadTimeout) ||
+ error.message.to_s.downcase.include?("timeout")
+ end
+
+ def self.response_from_error(error)
+ return error.page if error.respond_to?(:page) && error.page
+ return error.response if error.respond_to?(:response) && error.response
+
+ nil
+ end
+
+ def self.log_request(logger, url:, render_js:, scrape_options:)
+ logger.info(
+ event: "distillator.native_fetch.request",
+ url: url,
+ render_js: Distillator::BooleanParam.parse(render_js),
+ json_post: Distillator::FetchService.json_post?(scrape_options) == true
+ )
+ end
+
+ def self.fetch_post_response(agent, url, scrape_options)
+ Distillator::JsonPostFetcher.call(agent: agent, url: url, scrape_options: scrape_options)
+ end
+
+ def self.configure_agent(agent)
+ if agent.public_methods.include?(:user_agent_alias=)
+ agent.user_agent_alias = BROWSER_USER_AGENT_ALIAS
+ end
+ agent
+ end
+
+ def self.retry_with_ssl_verify_none(agent, url, scrape_options, logger)
+ return nil unless ssl_retry_supported?(agent)
+
+ set_ssl_verify_none(agent)
+ logger.warn(event: "distillator.native_fetch.ssl_retry", url: url, verify_mode: "VERIFY_NONE") if logger.respond_to?(:warn)
+
+ raw_response = fetch_response(agent, url, scrape_options)
+ result = build_success_or_http_failure(raw_response, agent, logger)
+ result[:wringer] = (result[:wringer] || {}).deep_dup
+ result[:wringer][:signals] = Distillator::FetchService.normalize_signals(result.dig(:wringer, :signals)).merge(ssl_verify_none_fallback: true)
+ result[:wringer][:hints] = (Distillator::FetchService.normalize_hints(result.dig(:wringer, :hints)) + ["ssl_verify_none_fallback"]).uniq
+ result
+ rescue *NETWORK_ERRORS
+ nil
+ end
+
+ def self.ssl_retry_supported?(agent)
+ ssl_http(agent).present?
+ end
+
+ def self.set_ssl_verify_none(agent)
+ ssl_http(agent).verify_mode = OpenSSL::SSL::VERIFY_NONE
+ end
+
+ def self.ssl_http(agent)
+ wrapped_agent = agent.agent
+ return unless wrapped_agent.respond_to?(:http)
+
+ wrapped_agent.http
+ rescue StandardError
+ nil
+ end
+
+ def self.annotate_request_metadata(result, scrape_options:)
+ payload = result.deep_dup
+ payload[:wringer] ||= {}
+ payload[:wringer][:signals] = Distillator::FetchService.normalize_signals(payload.dig(:wringer, :signals)).merge(
+ fetch_backend: "native",
+ request_method: Distillator::FetchService.json_post?(scrape_options) ? "POST" : "GET",
+ use_phantomjs: false,
+ phantomjs_iframe_extraction: false
+ )
+ payload
+ end
+
+ def self.log_response(logger, result)
+ logger.info(
+ event: "distillator.native_fetch.response",
+ status: result[:status],
+ http_code: result[:http_code],
+ final_url: result[:final_url],
+ redirect_chain: result[:redirect_chain] || [],
+ headers: result[:headers] || {},
+ signals: result.dig(:wringer, :signals) || {},
+ hints: result.dig(:wringer, :hints) || []
+ )
+ end
+
+ def self.iframe_request?(url:, scrape_options:)
+ return true if Distillator::BooleanParam.parse(scrape_options[:iframe] || scrape_options["iframe"])
+
+ Distillator::WringerUrlKey.call(url).uri_key.end_with?("iframe")
+ end
+
+ private_class_method(
+ :fetch_response,
+ :fetch_post_response,
+ :build_success_or_http_failure,
+ :control_result,
+ :http_failure_result,
+ :failed_network_result,
+ :finalize_result,
+ :timeout_error?,
+ :response_from_error,
+ :log_request,
+ :log_response,
+ :iframe_request?,
+ :configure_agent,
+ :retry_with_ssl_verify_none,
+ :ssl_retry_supported?,
+ :set_ssl_verify_none,
+ :ssl_http,
+ :annotate_request_metadata
+ )
+ end
+end
diff --git a/app/services/distillator/operator_next_action.rb b/app/services/distillator/operator_next_action.rb
new file mode 100644
index 00000000..706f8fe8
--- /dev/null
+++ b/app/services/distillator/operator_next_action.rb
@@ -0,0 +1,63 @@
+module Distillator
+ class OperatorNextAction
+ STALE_OR_UNKNOWN_STATUSES = %i[stale missing inconclusive not_evaluated blocked_by_fetch].freeze
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(website:, transition_status:, cache_link_payload: nil)
+ @website = website
+ @transition_status = transition_status
+ @cache_link_payload = cache_link_payload || {}
+ end
+
+ def call
+ return "Review blocker." if blockers?
+ return "Move to Shadow." if mode == :legacy
+ return "Run transition batch check." if transition_check_needed?
+ return "Activate after review." if mode == :shadow && review_activation_eligible?
+ return "Promote to Active." if mode == :shadow && ready?
+
+ if mode == :active && ready?
+ return "Open active cache." if active_cache_available?
+
+ return "Monitor."
+ end
+
+ "Monitor."
+ end
+
+ private
+
+ attr_reader :website, :transition_status, :cache_link_payload
+
+ def mode
+ Distillator::RolloutCopy.normalize(website&.distillator_mode)
+ end
+
+ def ready?
+ transition_status.status == :ready
+ end
+
+ def review_activation_eligible?
+ transition_status.respond_to?(:review_activation_eligible) && transition_status.review_activation_eligible
+ end
+
+ def blockers?
+ Array(transition_status.blockers).any?
+ end
+
+ def transition_check_needed?
+ return true if transition_status.status == :not_checked
+
+ [transition_status.fetch, transition_status.statements, transition_status.export].any? do |status|
+ STALE_OR_UNKNOWN_STATUSES.include?(status)
+ end
+ end
+
+ def active_cache_available?
+ cache_link_payload[:active_cache_url].present?
+ end
+ end
+end
diff --git a/app/services/distillator/phantomjs_fetcher.rb b/app/services/distillator/phantomjs_fetcher.rb
new file mode 100644
index 00000000..2c1f3634
--- /dev/null
+++ b/app/services/distillator/phantomjs_fetcher.rb
@@ -0,0 +1,173 @@
+require "json"
+
+module Distillator
+ class PhantomjsFetcher
+ PHANTOMJS_BASE_URL = "https://phantomjscloud.com/api/browser/v2".freeze
+
+ def self.call(url:, uri_key:, iframe:, headers: {}, timeout: nil, agent: nil, logger: nil)
+ logger = logger || Rails.logger
+
+ if ENV["PHANTOMJS_API_KEY"].blank?
+ logger.error("PHANTOMJS_API_KEY is not set for rendered fetch")
+ return direct_fallback_with_unavailable_renderer(url: url, iframe: iframe, agent: agent, logger: logger) if allow_direct_fallback?
+
+ return renderer_unavailable_result(url: url, iframe: iframe)
+ end
+
+ result = Distillator::NativeFetch.call(
+ url: request_url(url: url, iframe: iframe),
+ render_js: false,
+ scrape_options: {},
+ agent: agent,
+ logger: logger
+ )
+ result = extract_iframe_content(result) if iframe
+ result[:final_url] = url if result[:final_url].blank? || result[:final_url].to_s.include?(PHANTOMJS_BASE_URL)
+ annotate(result, iframe: iframe)
+ end
+
+ def self.request_url(url:, iframe:)
+ api_key = ENV.fetch("PHANTOMJS_API_KEY")
+ output_as_json = iframe ? "true" : "false"
+ request = "{url:%22#{url}%22,renderType:%22html%22,outputAsJson:#{output_as_json},requestSettings:{ignoreImages:true, waitInterval:2500 }}"
+
+ "#{PHANTOMJS_BASE_URL}/#{api_key}/?request=#{request}"
+ end
+
+ def self.extract_iframe_content(result)
+ return iframe_extraction_failure(result, "phantomjs_iframe_missing_response_body") unless result[:status] == :ok && result[:body].is_a?(String)
+
+ parsed = JSON.parse(result[:body])
+ iframe_html = parsed.dig("pageResponses", 0, "frameData", "childFrames", 0, "content")
+ return iframe_extraction_failure(result, "phantomjs_iframe_missing_child_content") if iframe_html.blank?
+
+ result.merge(
+ body: iframe_html,
+ raw_body: iframe_html,
+ wringer: merge_wringer_metadata(result[:wringer], signals: { phantomjs_iframe_extraction: true })
+ )
+ rescue JSON::ParserError, TypeError
+ iframe_extraction_failure(result, "phantomjs_iframe_malformed_json")
+ end
+
+ def self.annotate(result, fallback: nil, iframe: false)
+ payload = result.deep_dup
+ payload[:wringer] = merge_wringer_metadata(
+ payload[:wringer],
+ signals: {
+ renderer: "legacy_phantomjs",
+ renderer_unavailable: fallback.present?,
+ fetch_backend: fallback.present? ? "native" : "phantomjs",
+ request_method: "GET",
+ use_phantomjs: true,
+ phantomjs_iframe_extraction: iframe && payload.dig(:wringer, :signals, :phantomjs_iframe_extraction) != true ? false : payload.dig(:wringer, :signals, :phantomjs_iframe_extraction) == true
+ },
+ hints: ["legacy_phantomjs", ("phantomjs_unavailable" if fallback.present?)].compact
+ )
+ payload[:wringer][:signals][:renderer_fallback] = fallback if fallback.present?
+ payload
+ end
+
+ def self.renderer_unavailable_result(url:, iframe:)
+ {
+ status: :abort,
+ body: [
+ "abort_update",
+ {
+ error: "Legacy PhantomJS renderer is unavailable",
+ error_type: "DistillatorRendererUnavailable",
+ source: "phantomjs_fetcher",
+ retry: true,
+ cache: false,
+ step: "url",
+ signals: {
+ network_status: "failed",
+ renderer: "legacy_phantomjs",
+ renderer_unavailable: true,
+ renderer_fallback: "none",
+ use_phantomjs: true,
+ phantomjs_iframe_extraction: iframe
+ },
+ hints: ["legacy_phantomjs", "phantomjs_unavailable"]
+ }
+ ],
+ headers: {},
+ final_url: url,
+ redirect_chain: [],
+ wringer: {
+ error_type: "DistillatorRendererUnavailable",
+ source: "phantomjs_fetcher",
+ retry: true,
+ cache: false,
+ signals: {
+ network_status: "failed",
+ renderer: "legacy_phantomjs",
+ renderer_unavailable: true,
+ renderer_fallback: "none",
+ use_phantomjs: true,
+ phantomjs_iframe_extraction: iframe
+ },
+ hints: ["legacy_phantomjs", "phantomjs_unavailable"]
+ },
+ http_code: nil,
+ raw_body: nil
+ }
+ end
+
+ def self.direct_fallback_with_unavailable_renderer(url:, iframe:, agent:, logger:)
+ annotate(
+ Distillator::NativeFetch.call(
+ url: url,
+ render_js: false,
+ scrape_options: {},
+ agent: agent,
+ logger: logger
+ ),
+ fallback: "direct_url",
+ iframe: iframe
+ )
+ end
+
+ def self.allow_direct_fallback?
+ Distillator::BooleanParam.parse(ENV["DISTILLATOR_ALLOW_RENDERED_DIRECT_FALLBACK"])
+ end
+
+ def self.iframe_extraction_failure(result, hint)
+ {
+ status: :abort,
+ body: [
+ "abort_update",
+ {
+ error: hint.humanize,
+ error_type: "PhantomjsIframeExtractionError",
+ source: "phantomjs_fetcher",
+ retry: true,
+ cache: false,
+ step: "iframe",
+ signals: { phantomjs_iframe_extraction: false },
+ hints: [hint]
+ }
+ ],
+ headers: result[:headers] || {},
+ final_url: result[:final_url],
+ redirect_chain: result[:redirect_chain] || [],
+ wringer: merge_wringer_metadata(
+ result[:wringer],
+ signals: { phantomjs_iframe_extraction: false },
+ hints: [hint]
+ ),
+ http_code: nil,
+ raw_body: nil
+ }
+ end
+
+ def self.merge_wringer_metadata(wringer, signals: {}, hints: [])
+ payload = (wringer || {}).deep_dup
+ payload[:signals] = Distillator::FetchService.normalize_signals(payload[:signals]).merge(signals)
+ payload[:hints] = (Distillator::FetchService.normalize_hints(payload[:hints]) + hints).uniq
+ payload
+ end
+
+ private_class_method :annotate
+ end
+end
diff --git a/app/services/distillator/production_preflight.rb b/app/services/distillator/production_preflight.rb
new file mode 100644
index 00000000..3dfbe51b
--- /dev/null
+++ b/app/services/distillator/production_preflight.rb
@@ -0,0 +1,127 @@
+module Distillator
+ class ProductionPreflight
+ class Failure < StandardError; end
+
+ Entry = Struct.new(:label, :value, :ok, keyword_init: true) do
+ def to_output
+ "#{label}: #{value}"
+ end
+ end
+
+ Result = Struct.new(:entries, keyword_init: true) do
+ def ok?
+ entries.all?(&:ok)
+ end
+ end
+
+ REQUIRED_ROLLOUT_MODES = %w[legacy shadow active].freeze
+ STAGING_ROLLOUT_POLICY_MESSAGE = "Staging requires every website to be Shadow or Active.".freeze
+
+ def self.call
+ new.call
+ end
+
+ def call
+ Result.new(
+ entries: [
+ database_entry,
+ queue_adapter_entry,
+ rollout_modes_entry,
+ default_mode_entry,
+ staging_rollout_modes_entry,
+ cache_table_entry,
+ transition_report_route_entry,
+ compare_route_entry,
+ wringer_inspection_entry
+ ].compact
+ )
+ end
+
+ private
+
+ def database_entry
+ ActiveRecord::Base.connection_pool.with_connection do |connection|
+ connection.execute("SELECT 1")
+ end
+
+ Entry.new(label: "Database", value: "OK", ok: true)
+ rescue StandardError => e
+ Entry.new(label: "Database", value: "FAILED (#{e.message})", ok: false)
+ end
+
+ def queue_adapter_entry
+ adapter = ActiveJob::Base.queue_adapter_name.to_s.presence || "unknown"
+ Entry.new(label: "Queue adapter", value: adapter, ok: adapter != "unknown")
+ end
+
+ def rollout_modes_entry
+ modes = Website::DISTILLATOR_MODES
+ expected = REQUIRED_ROLLOUT_MODES.join("/")
+ if modes == REQUIRED_ROLLOUT_MODES
+ Entry.new(label: "Rollout modes", value: "#{expected} available", ok: true)
+ else
+ Entry.new(label: "Rollout modes", value: "FAILED (found #{modes.join('/')})", ok: false)
+ end
+ end
+
+ def default_mode_entry
+ mode = Distillator::FetchMode.rollout_resolution_object.rollout_mode.to_s
+ safe = mode == "legacy"
+ suffix = safe ? "" : " (unsafe)"
+
+ Entry.new(label: "Default mode", value: "#{mode}#{suffix}", ok: safe)
+ end
+
+ def staging_rollout_modes_entry
+ return unless Distillator::TransitionRuntime.staging?
+
+ invalid_scope = Distillator::TransitionRuntime.staging_invalid_rollout_mode_scope
+ invalid_count = invalid_scope.count
+
+ if invalid_count.zero?
+ Entry.new(
+ label: "Staging rollout modes",
+ value: "#{STAGING_ROLLOUT_POLICY_MESSAGE} OK",
+ ok: true
+ )
+ else
+ Entry.new(
+ label: "Staging rollout modes",
+ value: "#{STAGING_ROLLOUT_POLICY_MESSAGE} FAILED (#{invalid_count} invalid websites)",
+ ok: false
+ )
+ end
+ end
+
+ def cache_table_entry
+ ok = Distillator::FetchCache.table_exists?
+ Entry.new(label: "Cache table", value: ok ? "OK" : "FAILED (missing)", ok: ok)
+ rescue StandardError => e
+ Entry.new(label: "Cache table", value: "FAILED (#{e.message})", ok: false)
+ end
+
+ def transition_report_route_entry
+ path = Rails.application.routes.url_helpers.distillator_shadow_report_path
+ Entry.new(label: "Transition Report route", value: path.present? ? "OK" : "FAILED (missing)", ok: path.present?)
+ rescue StandardError => e
+ Entry.new(label: "Transition Report route", value: "FAILED (#{e.message})", ok: false)
+ end
+
+ def compare_route_entry
+ path = Rails.application.routes.url_helpers.condenser_cache_compare_path(uri: "http://example.org")
+ Entry.new(label: "Compare route", value: path.present? ? "OK" : "FAILED (missing)", ok: path.present?)
+ rescue StandardError => e
+ Entry.new(label: "Compare route", value: "FAILED (#{e.message})", ok: false)
+ end
+
+ def wringer_inspection_entry
+ base = ApplicationController.helpers.get_wringer_url_per_environment.to_s.strip
+ configured = base.present?
+ value = configured ? "configured" : "missing"
+
+ Entry.new(label: "Wringer inspection base", value: value, ok: configured)
+ rescue StandardError => e
+ Entry.new(label: "Wringer inspection base", value: "FAILED (#{e.message})", ok: false)
+ end
+ end
+end
diff --git a/app/services/distillator/promotion_readiness.rb b/app/services/distillator/promotion_readiness.rb
new file mode 100644
index 00000000..66a6cb80
--- /dev/null
+++ b/app/services/distillator/promotion_readiness.rb
@@ -0,0 +1,63 @@
+module Distillator
+ class PromotionReadiness
+ EVIDENCE_STALE_AFTER = 72.hours
+ LAVITRINE_EVIDENCE_STALE_AFTER = 24.hours
+ EXPORT_DIFF_STALE_AFTER = 7.days
+
+ Result = Struct.new(
+ :blockers,
+ :warnings,
+ :evidence_by_kind,
+ :safety,
+ :confidence,
+ :review_activation_eligible,
+ :manual_review_required,
+ :review_needed_fields,
+ :comparison_policy,
+ keyword_init: true
+ )
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(website:, cache: nil, evidence_by_kind: nil, now: Time.current)
+ @website = website
+ @cache = cache
+ @evidence_by_kind = evidence_by_kind
+ @now = now
+ end
+
+ def call
+ transition = transition_status
+ Result.new(
+ blockers: transition.blockers,
+ warnings: transition.warnings,
+ evidence_by_kind: evidence_by_kind,
+ safety: transition.safety,
+ confidence: transition.confidence,
+ review_activation_eligible: transition.review_activation_eligible,
+ manual_review_required: transition.manual_review_required,
+ review_needed_fields: transition.review_needed_fields,
+ comparison_policy: transition.comparison_policy
+ )
+ end
+
+ private
+
+ attr_reader :website, :cache, :now
+
+ def evidence_by_kind
+ @evidence_by_kind ||= website.respond_to?(:latest_transition_evidences_by_kind) ? website.latest_transition_evidences_by_kind : {}
+ end
+
+ def transition_status
+ @transition_status ||= Distillator::TransitionStatus.call(
+ website: website,
+ cache: cache,
+ evidence_by_kind: evidence_by_kind,
+ now: now
+ )
+ end
+ end
+end
diff --git a/app/services/distillator/refresh_runner.rb b/app/services/distillator/refresh_runner.rb
new file mode 100644
index 00000000..838fcbe3
--- /dev/null
+++ b/app/services/distillator/refresh_runner.rb
@@ -0,0 +1,18 @@
+module Distillator
+ class RefreshRunner
+ DEFAULT_SCRAPE_OPTIONS = { force_scrape_every_hrs: 1 }.freeze
+
+ def self.call(webpage:, refresh_helper:, scrape_options: {})
+ Statements::RefreshWebpageStatementsService.new(refresh_helper: refresh_helper).call(
+ webpage: webpage,
+ default_language: webpage.website.default_language,
+ scrape_options: normalized_scrape_options(scrape_options)
+ )
+ end
+
+ def self.normalized_scrape_options(scrape_options)
+ options = scrape_options.respond_to?(:to_h) ? scrape_options.to_h.symbolize_keys : {}
+ DEFAULT_SCRAPE_OPTIONS.merge(options.compact)
+ end
+ end
+end
diff --git a/app/services/distillator/rendered_fetch.rb b/app/services/distillator/rendered_fetch.rb
new file mode 100644
index 00000000..a5cc7772
--- /dev/null
+++ b/app/services/distillator/rendered_fetch.rb
@@ -0,0 +1,26 @@
+module Distillator
+ class RenderedFetch
+ def self.call(url:, uri_key:, iframe: false, headers: {}, timeout: nil, agent: nil, logger: nil)
+ renderer.call(
+ url: url,
+ uri_key: uri_key,
+ iframe: iframe,
+ headers: headers,
+ timeout: timeout,
+ agent: agent,
+ logger: logger || Rails.logger
+ )
+ end
+
+ def self.renderer
+ case ENV["DISTILLATOR_RENDERER"].to_s.presence || "legacy_phantomjs"
+ when "legacy_phantomjs"
+ Distillator::PhantomjsFetcher
+ else
+ Distillator::Renderers::DisabledRenderer
+ end
+ end
+
+ private_class_method :renderer
+ end
+end
diff --git a/app/services/distillator/renderers/disabled_renderer.rb b/app/services/distillator/renderers/disabled_renderer.rb
new file mode 100644
index 00000000..01111396
--- /dev/null
+++ b/app/services/distillator/renderers/disabled_renderer.rb
@@ -0,0 +1,41 @@
+module Distillator
+ module Renderers
+ class DisabledRenderer
+ def self.call(url:, uri_key:, iframe:, headers: {}, timeout: nil, agent: nil, logger: nil)
+ error = "Rendered fetch is disabled"
+ {
+ status: :abort,
+ body: ["abort_update", {
+ error: error,
+ error_type: "DistillatorRendererDisabled",
+ source: "distillator_renderer",
+ retry: false,
+ cache: false,
+ step: "url",
+ signals: {
+ network_status: "failed",
+ renderer: "disabled"
+ },
+ hints: ["rendered_fetch_disabled"]
+ }],
+ headers: {},
+ final_url: url,
+ redirect_chain: [],
+ wringer: {
+ error_type: "DistillatorRendererDisabled",
+ source: "distillator_renderer",
+ retry: false,
+ cache: false,
+ signals: {
+ network_status: "failed",
+ renderer: "disabled"
+ },
+ hints: ["rendered_fetch_disabled"]
+ },
+ http_code: nil,
+ raw_body: nil
+ }
+ end
+ end
+ end
+end
diff --git a/app/services/distillator/renderers/legacy_phantomjs_renderer.rb b/app/services/distillator/renderers/legacy_phantomjs_renderer.rb
new file mode 100644
index 00000000..a4cb251e
--- /dev/null
+++ b/app/services/distillator/renderers/legacy_phantomjs_renderer.rb
@@ -0,0 +1,19 @@
+require "json"
+
+module Distillator
+ module Renderers
+ class LegacyPhantomjsRenderer
+ def self.call(url:, uri_key:, iframe:, headers: {}, timeout: nil, agent: nil, logger: nil)
+ Distillator::PhantomjsFetcher.call(url: url, uri_key: uri_key, iframe: iframe, headers: headers, timeout: timeout, agent: agent, logger: logger)
+ end
+
+ def self.request_url(url:, iframe:)
+ Distillator::PhantomjsFetcher.request_url(url: url, iframe: iframe)
+ end
+
+ def self.extract_iframe_content(result)
+ Distillator::PhantomjsFetcher.extract_iframe_content(result)
+ end
+ end
+ end
+end
diff --git a/app/services/distillator/rollout_copy.rb b/app/services/distillator/rollout_copy.rb
new file mode 100644
index 00000000..c80c0dda
--- /dev/null
+++ b/app/services/distillator/rollout_copy.rb
@@ -0,0 +1,165 @@
+module Distillator
+ module RolloutCopy
+ PRODUCTION_STATE_UI = {
+ legacy: {
+ label: "Legacy",
+ css_class: "rollout-badge-legacy",
+ description: "Wringer serves production."
+ },
+ shadow: {
+ label: "Shadow",
+ css_class: "rollout-badge-shadow",
+ description: "Wringer serves production while Condenser is checked in the background."
+ },
+ active: {
+ label: "Active",
+ css_class: "rollout-badge-active",
+ description: "Condenser serves production while Wringer stays available for diagnostics."
+ }
+ }.freeze
+
+ DIAGNOSTIC_STATE_UI = {
+ replay: {
+ label: "Unknown",
+ css_class: "rollout-badge-replay",
+ description: "Diagnostic mode only. Do not treat this as a production state."
+ },
+ unknown: {
+ label: "Unknown",
+ css_class: "rollout-badge-unknown",
+ description: "Production mode could not be determined from the current context."
+ }
+ }.freeze
+
+ CACHE_ACTION_COPY = {
+ active_cache_label: "Open active cache",
+ condenser_cache_label: "Open Condenser cache",
+ legacy_inspection_label: "Inspect legacy Wringer",
+ compare_label: "Compare Condenser vs Wringer"
+ }.freeze
+
+ FORM_OPTION_LABELS = {
+ legacy: "Legacy",
+ shadow: "Shadow",
+ active: "Active"
+ }.freeze
+
+ INDEX_FILTER_OPTION_LABELS = {
+ all: "All production modes",
+ legacy: "Legacy",
+ shadow: "Shadow",
+ active: "Active",
+ replay: "Unknown",
+ unknown: "Unknown"
+ }.freeze
+
+ ROLLOUT_PANEL_TITLE = "Production mode".freeze
+
+ def self.label(mode)
+ state(mode)[:label]
+ end
+
+ def self.description(mode)
+ state(mode)[:description]
+ end
+
+ def self.css_class(mode)
+ state(mode)[:css_class]
+ end
+
+ def self.state(mode)
+ all_states.fetch(normalize(mode), DIAGNOSTIC_STATE_UI.fetch(:unknown))
+ end
+
+ def self.production_states
+ PRODUCTION_STATE_UI
+ end
+
+ def self.diagnostic_states
+ DIAGNOSTIC_STATE_UI
+ end
+
+ def self.active_cache_label
+ CACHE_ACTION_COPY.fetch(:active_cache_label)
+ end
+
+ def self.condenser_cache_label
+ CACHE_ACTION_COPY.fetch(:condenser_cache_label)
+ end
+
+ def self.legacy_inspection_label
+ CACHE_ACTION_COPY.fetch(:legacy_inspection_label)
+ end
+
+ def self.compare_label
+ CACHE_ACTION_COPY.fetch(:compare_label)
+ end
+
+ def self.website_form_options
+ [
+ [FORM_OPTION_LABELS.fetch(:legacy), "legacy"],
+ [FORM_OPTION_LABELS.fetch(:shadow), "shadow"],
+ [FORM_OPTION_LABELS.fetch(:active), "active"]
+ ]
+ end
+
+ def self.website_index_filter_options
+ [
+ [INDEX_FILTER_OPTION_LABELS.fetch(:all), nil],
+ [INDEX_FILTER_OPTION_LABELS.fetch(:legacy), "legacy"],
+ [INDEX_FILTER_OPTION_LABELS.fetch(:shadow), "shadow"],
+ [INDEX_FILTER_OPTION_LABELS.fetch(:active), "active"],
+ [INDEX_FILTER_OPTION_LABELS.fetch(:unknown), "unknown"]
+ ]
+ end
+
+ def self.rollout_panel_title
+ ROLLOUT_PANEL_TITLE
+ end
+
+ def self.active_backend_label(mode)
+ case normalize(mode)
+ when :active, :replay
+ "Condenser"
+ when :shadow, :legacy
+ "Wringer"
+ else
+ "Unknown"
+ end
+ end
+
+ def self.next_step(mode)
+ case normalize(mode)
+ when :active
+ "Monitor production and use Legacy if rollback is needed."
+ when :shadow
+ "Review diagnostics, then promote to Active when ready."
+ when :legacy
+ "Move to Shadow before promotion."
+ when :replay
+ "Use diagnostics only. Keep production on Legacy, Shadow, or Active."
+ else
+ "Confirm the production mode before making rollout decisions."
+ end
+ end
+
+ def self.normalize(mode)
+ case mode.to_s
+ when "legacy"
+ :legacy
+ when "shadow"
+ :shadow
+ when "active", "internal"
+ :active
+ when "replay"
+ :replay
+ else
+ :unknown
+ end
+ end
+
+ def self.all_states
+ @all_states ||= PRODUCTION_STATE_UI.merge(DIAGNOSTIC_STATE_UI).freeze
+ end
+ end
+end
diff --git a/app/services/distillator/rollout_event_recorder.rb b/app/services/distillator/rollout_event_recorder.rb
new file mode 100644
index 00000000..18d74860
--- /dev/null
+++ b/app/services/distillator/rollout_event_recorder.rb
@@ -0,0 +1,50 @@
+module Distillator
+ class RolloutEventRecorder
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(website:, from_mode:, to_mode:, actor: nil, reason: nil, readiness_snapshot: {}, event: nil)
+ @website = website
+ @from_mode = from_mode
+ @to_mode = to_mode
+ @actor = actor
+ @reason = reason
+ @readiness_snapshot = readiness_snapshot
+ @event = event
+ end
+
+ def call
+ return if from_mode.to_s == to_mode.to_s
+
+ Distillator::RolloutEvent.create!(
+ website: website,
+ from_mode: from_mode,
+ to_mode: to_mode,
+ actor: actor.presence,
+ reason: reason.presence,
+ readiness_snapshot: event_snapshot
+ )
+ end
+
+ private
+
+ attr_reader :website, :from_mode, :to_mode, :actor, :reason, :readiness_snapshot, :event
+
+ def event_snapshot
+ readiness_snapshot.to_h.merge(
+ "event" => event.presence || inferred_event
+ )
+ end
+
+ def inferred_event
+ return "rollout.rollback" if rollback?
+
+ "rollout.transition"
+ end
+
+ def rollback?
+ from_mode.to_s == "active" && to_mode.to_s == "legacy"
+ end
+ end
+end
diff --git a/app/services/distillator/rollout_resolution.rb b/app/services/distillator/rollout_resolution.rb
new file mode 100644
index 00000000..6d8315fc
--- /dev/null
+++ b/app/services/distillator/rollout_resolution.rb
@@ -0,0 +1,42 @@
+module Distillator
+ class RolloutResolution < Struct.new(
+ :execution_mode,
+ :rollout_mode,
+ :dispatch_mode,
+ :active_backend,
+ :source,
+ :requested_mode,
+ :website_id,
+ keyword_init: true
+ )
+ def self.from_mode(mode:, source:, requested_mode: nil, website_id: nil)
+ normalized_mode = canonical_mode(mode)
+
+ new(
+ execution_mode: normalized_mode,
+ rollout_mode: rollout_mode_for(normalized_mode),
+ dispatch_mode: dispatch_mode_for(normalized_mode),
+ active_backend: active_backend_for(normalized_mode),
+ source: source,
+ requested_mode: requested_mode&.to_sym,
+ website_id: website_id,
+ )
+ end
+
+ def self.canonical_mode(mode)
+ mode.to_sym == :internal ? :active : mode.to_sym
+ end
+
+ def self.rollout_mode_for(mode)
+ canonical_mode(mode)
+ end
+
+ def self.dispatch_mode_for(mode)
+ canonical_mode(mode) == :active ? :internal : canonical_mode(mode)
+ end
+
+ def self.active_backend_for(mode)
+ canonical_mode(mode) == :active ? :condenser : :wringer
+ end
+ end
+end
diff --git a/app/services/distillator/rollout_transition.rb b/app/services/distillator/rollout_transition.rb
new file mode 100644
index 00000000..8c0c502b
--- /dev/null
+++ b/app/services/distillator/rollout_transition.rb
@@ -0,0 +1,176 @@
+module Distillator
+ class RolloutTransition
+ Result = Struct.new(:success?, :website, :from_mode, :to_mode, :warnings, :blockers, :errors, keyword_init: true)
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(website:, to_mode:, actor: nil, reason: nil, force: false, override: false, review: false, attributes: {})
+ @website = website
+ @raw_to_mode = to_mode
+ @to_mode = normalize_mode(to_mode)
+ @actor = actor
+ @reason = normalize_reason(reason)
+ @force = force
+ @override = override
+ @review = review
+ @attributes = attributes.to_h.symbolize_keys.except(:distillator_mode)
+ end
+
+ def call
+ website.assign_attributes(attributes)
+ return failure(["Distillator mode is not included in the list"]) unless valid_target_mode?
+
+ if blocked_transition_errors.any?
+ blocked_transition_errors.each { |message| website.errors.add(:distillator_mode, message) }
+ return failure(blocked_transition_errors)
+ end
+
+ warnings = transition_warnings
+ from_mode = current_mode
+
+ website.distillator_mode = to_mode
+ if website.save
+ Distillator::RolloutEventRecorder.call(
+ website: website,
+ from_mode: from_mode,
+ to_mode: to_mode,
+ actor: actor,
+ reason: reason,
+ readiness_snapshot: readiness_snapshot(warnings),
+ event: rollout_event_name
+ )
+ return Result.new(success?: true, website: website, from_mode: from_mode, to_mode: to_mode, warnings: warnings, blockers: [], errors: [])
+ end
+
+ failure(website.errors.full_messages)
+ end
+
+ private
+
+ attr_reader :website, :to_mode, :actor, :reason, :force, :override, :review, :attributes, :raw_to_mode
+
+ def current_mode
+ website.distillator_mode.presence || "legacy"
+ end
+
+ def invalid_current_mode?
+ !Website::DISTILLATOR_MODES.include?(current_mode)
+ end
+
+ def valid_target_mode?
+ Website::DISTILLATOR_MODES.include?(to_mode)
+ end
+
+ def blocked_transition_errors
+ @blocked_transition_errors ||= begin
+ override_errors = explicit_override_errors
+ return override_errors if override_errors.any?
+ return [] if current_mode == to_mode
+ return [] if invalid_current_mode? && to_mode == "shadow"
+ return [] if current_mode == "legacy" && to_mode == "shadow"
+ return [] if current_mode == "shadow" && to_mode == "legacy"
+ return [] if current_mode == "active" && to_mode.in?(%w[legacy shadow])
+ return legacy_to_active_errors if current_mode == "legacy" && to_mode == "active"
+ return shadow_to_active_errors if current_mode == "shadow" && to_mode == "active"
+
+ ["Unsupported rollout transition"]
+ end
+ end
+
+ def legacy_to_active_errors
+ return [] if override_requested? && Distillator::TransitionRuntime.allow_active_override?
+
+ ["Direct legacy to active promotion is blocked"]
+ end
+
+ def shadow_to_active_errors
+ return review_to_active_errors if review_requested?
+ return [] if override_requested? && Distillator::TransitionRuntime.allow_active_override?
+ return [] if promotion_readiness.blockers.blank? && promotion_readiness.warnings.blank?
+
+ return promotion_readiness.blockers if promotion_readiness.blockers.any?
+ return promotion_readiness.warnings.map { |message| activation_error_message(message) } if promotion_readiness.warnings.any?
+
+ ["Cannot activate yet: checks are not complete."]
+ end
+
+ def transition_warnings
+ []
+ end
+
+ def promotion_readiness
+ @promotion_readiness ||= Distillator::PromotionReadiness.call(
+ website: website,
+ cache: Distillator::ShadowReportQuery.latest_cache_for_website(website),
+ evidence_by_kind: website.latest_transition_evidences_by_kind
+ )
+ end
+
+ def readiness_snapshot(warnings)
+ {
+ blockers: promotion_readiness.blockers,
+ warnings: warnings.presence || promotion_readiness.warnings,
+ cohort_key: website.distillator_primary_cohort_key,
+ override: override_requested?,
+ review_activation: review_requested?,
+ manual_review_required: promotion_readiness.respond_to?(:manual_review_required) ? promotion_readiness.manual_review_required : nil,
+ review_needed_fields: promotion_readiness.respond_to?(:review_needed_fields) ? promotion_readiness.review_needed_fields : [],
+ safety: promotion_readiness.respond_to?(:safety) ? promotion_readiness.safety : nil,
+ confidence: promotion_readiness.respond_to?(:confidence) ? promotion_readiness.confidence : nil,
+ comparison_policy: promotion_readiness.respond_to?(:comparison_policy) ? promotion_readiness.comparison_policy : nil
+ }
+ end
+
+ def explicit_override_errors
+ return [] unless override_requested?
+ return ["Activate anyway is not allowed in this runtime"] unless Distillator::TransitionRuntime.allow_active_override?
+ return ["Reason is required for Activate anyway"] if reason.blank?
+
+ []
+ end
+
+ def review_to_active_errors
+ return ["Reason is required for Activate after review"] if reason.blank?
+ return ["Activate after review is available only for review-needed parity differences."] unless promotion_readiness.respond_to?(:review_activation_eligible) && promotion_readiness.review_activation_eligible
+
+ []
+ end
+
+ def normalize_mode(mode)
+ return current_mode if mode.nil?
+
+ mode.to_s
+ end
+
+ def failure(errors)
+ Result.new(success?: false, website: website, from_mode: current_mode, to_mode: to_mode, warnings: [], blockers: blocked_transition_errors, errors: Array(errors))
+ end
+
+ def override_requested?
+ override || force
+ end
+
+ def review_requested?
+ review == true
+ end
+
+ def rollout_event_name
+ return "rollout.review_activate" if review_requested?
+ return "rollout.override" if override_requested?
+
+ nil
+ end
+
+ def activation_error_message(message)
+ normalized = message.to_s.sub(/\ANeeds review:\s*/i, "")
+ "Cannot activate yet: #{normalized.sub(/\.\z/, '')}."
+ end
+
+ def normalize_reason(value)
+ trimmed = value.to_s.strip
+ trimmed.presence
+ end
+ end
+end
diff --git a/app/services/distillator/shadow_report.rb b/app/services/distillator/shadow_report.rb
new file mode 100644
index 00000000..87dd7f43
--- /dev/null
+++ b/app/services/distillator/shadow_report.rb
@@ -0,0 +1,97 @@
+module Distillator
+ class ShadowReport
+ Result = Struct.new(:rows, :summary_counts, :global_summary_counts, :dashboard_counts, :blocker_counts, :page, :per_page, :total_count, :total_pages, keyword_init: true)
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(filters:, sort:, direction:, page:, per_page:)
+ @filters = filters
+ @sort = sort
+ @direction = direction
+ @page = page
+ @per_page = per_page
+ end
+
+ def call
+ query = Distillator::ShadowReportQuery.call(
+ filters: @filters,
+ sort: @sort,
+ direction: @direction,
+ page: @page,
+ per_page: @per_page
+ )
+
+ Result.new(
+ rows: query.records,
+ summary_counts: summary_counts(query.records),
+ global_summary_counts: summary_counts(query.records),
+ dashboard_counts: dashboard_counts(query.records),
+ blocker_counts: blocker_counts(query.records),
+ page: query.page,
+ per_page: query.per_page,
+ total_count: query.total_count,
+ total_pages: query.total_pages
+ )
+ end
+
+ private
+
+ def summary_counts(rows)
+ rows.each_with_object(
+ {
+ total: rows.length,
+ ready: 0,
+ review: 0,
+ blocked: 0,
+ not_checked: 0,
+ lavitrine_total: 0,
+ lavitrine_ready: 0,
+ lavitrine_review: 0,
+ lavitrine_blocked: 0,
+ lavitrine_not_checked: 0
+ }
+ ) do |row, counts|
+ counts[row.status] += 1 if counts.key?(row.status)
+ next unless row.cohort_key == Distillator::Cohorts::LavitrinePipeline.key
+
+ counts[:lavitrine_total] += 1
+ counts[:"lavitrine_#{row.status}"] += 1
+ end
+ end
+
+ def dashboard_counts(rows)
+ mode_counts = Website.group(:distillator_mode).count
+
+ {
+ legacy_sites: mode_counts.fetch("legacy", 0),
+ shadow_sites: mode_counts.fetch("shadow", 0),
+ active_sites: mode_counts.fetch("active", 0),
+ priority_sites: rows.count { |row| row.priority },
+ blocked_sites: rows.count { |row| row.status == :blocked },
+ promotable_sites: rows.count(&:promotable)
+ }
+ end
+
+ def blocker_counts(rows)
+ rows.each_with_object(
+ {
+ failed_fetch: 0,
+ missing_statement_evidence: 0,
+ missing_export_evidence: 0,
+ stale_evidence: 0,
+ redirect_cache_health_review: 0
+ }
+ ) do |row, counts|
+ messages = Array(row.blockers) + Array(row.warnings)
+
+ counts[:failed_fetch] += 1 if messages.any? { |message| message.include?("fetch check failed") }
+ counts[:missing_statement_evidence] += 1 if messages.any? { |message| message.include?("statements check is missing") }
+ counts[:missing_export_evidence] += 1 if messages.any? { |message| message.include?("export check is missing") }
+ counts[:stale_evidence] += 1 if messages.any? { |message| message.include?("check is stale") || message.include?("latest successful refresh is stale") }
+ counts[:redirect_cache_health_review] += 1 if messages.any? { |message| message.include?("fetch result redirected") }
+ end
+ end
+ end
+end
diff --git a/app/services/distillator/shadow_report_query.rb b/app/services/distillator/shadow_report_query.rb
new file mode 100644
index 00000000..b0e0ddcd
--- /dev/null
+++ b/app/services/distillator/shadow_report_query.rb
@@ -0,0 +1,326 @@
+require "will_paginate/collection"
+
+module Distillator
+ class ShadowReportQuery
+ FILTER_KEYS = %i[status recommendation safety confidence health_severity primary_issue_key term cohort mode promotable].freeze
+ STAGING_INVALID_FILTER = "invalid_on_staging".freeze
+ SORT_COLUMNS = %w[website status recommendation latest_attempt latest_successful_refresh issue_key].freeze
+ DEFAULT_SORT = "website".freeze
+ DEFAULT_DIRECTION = "asc".freeze
+ DEFAULT_LIMIT = 25
+ MAX_LIMIT = 100
+ DEFAULT_PER_PAGE = DEFAULT_LIMIT
+ MAX_PER_PAGE = MAX_LIMIT
+ RECOMMENDATION_ORDER = {
+ "blocked" => 0,
+ "review" => 1,
+ "ready" => 2,
+ "not_checked" => 3,
+ "unknown" => 3
+ }.freeze
+
+ CACHE_COLUMNS = %w[
+ id
+ uri_key
+ normalized_url
+ http_response_code
+ scrape_date
+ successful_refresh
+ signals
+ final_url
+ health_status
+ health_severity
+ redirected
+ network_status
+ primary_issue_key
+ primary_issue_label
+ primary_issue_severity
+ ].freeze
+
+ Result = Struct.new(:records, :all_records, :global_records, :page, :per_page, :total_count, :total_pages, keyword_init: true)
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(filters:, sort:, direction:, page:, per_page:)
+ @filters = filters.to_h.symbolize_keys
+ @sort = SORT_COLUMNS.include?(sort.to_s) ? sort.to_s : DEFAULT_SORT
+ @direction = %w[asc desc].include?(direction.to_s) ? direction.to_s : DEFAULT_DIRECTION
+ @page = page.to_i.positive? ? page.to_i : 1
+ normalized_per_page = per_page.to_i.positive? ? per_page.to_i : DEFAULT_LIMIT
+ @per_page = [normalized_per_page, MAX_LIMIT].min
+ end
+
+ def call
+ return default_result if default_report?
+
+ summaries = filtered_summaries
+ paginated = paginate_rows(summaries, total_count: summaries.length)
+
+ Result.new(
+ records: paginated.to_a,
+ all_records: summaries,
+ global_records: summaries,
+ page: paginated.current_page,
+ per_page: paginated.per_page,
+ total_count: paginated.total_entries,
+ total_pages: paginated.total_pages
+ )
+ end
+
+ private
+
+ attr_reader :filters, :sort, :direction, :page, :per_page
+
+ def default_result
+ scope = default_shadow_scope
+ rows = build_summaries(paginated_websites(scope))
+ total_count = scope.count
+ total_pages = (total_count.to_f / per_page).ceil
+ paginated = WillPaginate::Collection.create(page, per_page, total_count) do |pager|
+ pager.replace(rows)
+ end
+
+ Result.new(
+ records: paginated.to_a,
+ all_records: rows,
+ global_records: rows,
+ page: paginated.current_page,
+ per_page: paginated.per_page,
+ total_count: paginated.total_entries,
+ total_pages: total_pages
+ )
+ end
+
+ def filtered_summaries
+ @filtered_summaries ||= begin
+ rows = build_summaries(websites).select { |summary| include_summary?(summary) }
+ rows = rows.sort_by { |summary| sortable_value(summary) }
+ rows.reverse! if direction == "desc"
+ rows
+ end
+ end
+
+ def build_summaries(websites)
+ return [] if websites.empty?
+
+ evidence_by_website_id = Distillator::TransitionEvidence.latest_for_website_ids(websites.map(&:id))
+ latest_caches_by_website_id = latest_caches_by_website_id_for(websites)
+
+ websites.map do |website|
+ Distillator::ShadowSiteSummary.call(
+ website: website,
+ cache: latest_caches_by_website_id[website.id],
+ evidence_by_kind: evidence_by_website_id[website.id]
+ )
+ end
+ end
+
+ def websites
+ @websites ||= website_scope.includes(:webpages).order(:name).to_a
+ end
+
+ def latest_caches_by_website_id_for(websites)
+ lookup = {}
+
+ latest_caches_for(websites).each do |cache|
+ website = matched_website_for_cache(cache, websites)
+ next unless website
+ next if lookup.key?(website.id)
+
+ lookup[website.id] = cache
+ end
+
+ lookup
+ end
+
+ def latest_caches_for(websites)
+ urls = candidate_urls(websites)
+ keys = candidate_uri_keys(websites)
+ return [] if urls.empty? && keys.empty?
+
+ Distillator::FetchCache
+ .select(CACHE_COLUMNS.map { |column| "distillator_fetch_caches.#{column}" })
+ .where("normalized_url IN (:urls) OR final_url IN (:urls) OR uri_key IN (:keys)", urls: urls.presence || [""], keys: keys.presence || [""])
+ .order(Arel.sql("COALESCE(distillator_fetch_caches.scrape_date, distillator_fetch_caches.updated_at) DESC, distillator_fetch_caches.id DESC"))
+ .to_a
+ end
+
+ def matched_website_for_cache(cache, websites)
+ result = Distillator::CacheWebsiteMatcher.call(cache: cache, websites: websites)
+ result.website
+ end
+
+ def include_summary?(summary)
+ status_match?(summary) &&
+ safety_match?(summary) &&
+ confidence_match?(summary) &&
+ mode_match?(summary) &&
+ promotable_match?(summary) &&
+ health_severity_match?(summary) &&
+ primary_issue_key_match?(summary) &&
+ cohort_match?(summary) &&
+ term_match?(summary)
+ end
+
+ def mode_match?(summary)
+ return true if filters[:mode].blank?
+ return Distillator::TransitionRuntime.staging_rollout_mode_invalid?(summary.website.distillator_mode) if filters[:mode].to_s == STAGING_INVALID_FILTER
+
+ summary.website.distillator_mode.to_s == filters[:mode].to_s
+ end
+
+ def status_match?(summary)
+ requested = filters[:status].presence || filters[:recommendation].presence
+ return true if requested.blank?
+
+ summary.status.to_s == requested.to_s
+ end
+
+ def safety_match?(summary)
+ return true if filters[:safety].blank?
+
+ summary.safety.to_s == filters[:safety].to_s
+ end
+
+ def confidence_match?(summary)
+ return true if filters[:confidence].blank?
+
+ summary.confidence.to_s == filters[:confidence].to_s
+ end
+
+ def promotable_match?(summary)
+ requested = filters[:promotable].to_s
+ return true if requested.blank?
+
+ case requested
+ when "yes"
+ summary.promotable
+ when "no"
+ !summary.promotable
+ else
+ true
+ end
+ end
+
+ def health_severity_match?(summary)
+ return true if filters[:health_severity].blank?
+
+ summary.health_severity.to_s == filters[:health_severity].to_s
+ end
+
+ def primary_issue_key_match?(summary)
+ return true if filters[:primary_issue_key].blank?
+
+ summary.issue_key.to_s == filters[:primary_issue_key].to_s
+ end
+
+ def term_match?(summary)
+ return true if filters[:term].blank?
+
+ haystack = [summary.website.name, summary.website.seedurl, summary.cohort_label].compact.join(" ").downcase
+ haystack.include?(filters[:term].to_s.downcase)
+ end
+
+ def cohort_match?(summary)
+ return true if filters[:cohort].blank?
+
+ case filters[:cohort].to_s
+ when "other"
+ summary.cohort_key.blank?
+ else
+ summary.cohort_key.to_s == filters[:cohort].to_s
+ end
+ end
+
+ def sortable_value(summary)
+ case sort
+ when "status", "recommendation"
+ RECOMMENDATION_ORDER.fetch(summary.status.to_s, 99)
+ when "latest_attempt"
+ summary.latest_refresh || Time.at(0)
+ when "latest_successful_refresh"
+ summary.latest_successful_refresh || Time.at(0)
+ when "issue_key"
+ summary.issue_key.to_s.downcase
+ else
+ summary.website.name.to_s.downcase
+ end
+ end
+
+ def paginate_rows(rows, total_count:)
+ WillPaginate::Collection.create(page, per_page, total_count) do |pager|
+ pager.replace(rows[pager.offset, pager.per_page] || [])
+ end
+ end
+
+ def candidate_urls(websites)
+ websites.flat_map do |website|
+ website.webpages.flat_map do |webpage|
+ url = webpage.url.to_s
+ next [] if url.blank?
+
+ compact_urls = [url, url.split("#").first]
+ if url.end_with?("/")
+ compact_urls << url.chomp("/")
+ else
+ compact_urls << "#{url}/"
+ end
+ compact_urls.uniq
+ end
+ end.uniq
+ end
+
+ def candidate_uri_keys(websites)
+ websites.flat_map do |website|
+ website.webpages.filter_map do |webpage|
+ Distillator::WringerUrlKey.call(webpage.url).uri_key
+ rescue StandardError
+ nil
+ end
+ end.uniq
+ end
+
+ def website_scope
+ requested_mode = filters[:mode].presence || "shadow"
+ return Distillator::TransitionRuntime.staging_invalid_rollout_mode_scope if requested_mode == STAGING_INVALID_FILTER
+
+ Website.where(distillator_mode: requested_mode)
+ end
+
+ def default_shadow_scope
+ @default_shadow_scope ||= Website.where(distillator_mode: "shadow").order(:name)
+ end
+
+ def paginated_websites(scope)
+ scope.offset((page - 1) * per_page).limit(per_page).includes(:webpages).to_a
+ end
+
+ def default_report?
+ return false unless filters.except(:mode).compact_blank.empty?
+ return false unless filters[:mode].blank? || filters[:mode].to_s == "shadow"
+ return false unless sort == DEFAULT_SORT
+ return false unless direction == DEFAULT_DIRECTION
+
+ true
+ end
+
+ class << self
+ def latest_cache_for_website(website)
+ website_record = website.is_a?(Website) ? website : Website.find(website)
+ query = new(
+ filters: { mode: website_record.distillator_mode },
+ sort: DEFAULT_SORT,
+ direction: DEFAULT_DIRECTION,
+ page: 1,
+ per_page: 1
+ )
+
+ query.send(:latest_caches_for, [website_record]).find do |cache|
+ Distillator::CacheWebsiteMatcher.call(cache: cache, websites: [website_record]).website.present?
+ end
+ end
+ end
+ end
+end
diff --git a/app/services/distillator/shadow_site_detail.rb b/app/services/distillator/shadow_site_detail.rb
new file mode 100644
index 00000000..6256a9b7
--- /dev/null
+++ b/app/services/distillator/shadow_site_detail.rb
@@ -0,0 +1,518 @@
+module Distillator
+ class ShadowSiteDetail
+ Result = Struct.new(
+ :summary,
+ :transition_status,
+ :transition_evidence_by_kind,
+ :transition_evidence_explanations,
+ :latest_transition_evidence_checked_at,
+ :transition_check_requested_at,
+ :pending_transition_batch_check,
+ :primary_blocker,
+ :statement_failure_groups,
+ :url_matrix,
+ :root_cause,
+ :checked_scope,
+ :decision,
+ :rollout_notes,
+ :recent_rollout_events,
+ keyword_init: true
+ )
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(website:, cache: nil)
+ @website = website
+ @cache = cache
+ end
+
+ def call
+ Result.new(
+ summary: summary,
+ transition_status: transition_status,
+ transition_evidence_by_kind: transition_evidence_by_kind,
+ transition_evidence_explanations: transition_evidence_explanations,
+ latest_transition_evidence_checked_at: website.latest_transition_evidence_checked_at,
+ transition_check_requested_at: website.transition_check_requested_at,
+ pending_transition_batch_check: website.pending_transition_batch_check?,
+ primary_blocker: primary_blocker,
+ statement_failure_groups: statement_failure_groups,
+ url_matrix: url_matrix,
+ root_cause: root_cause,
+ checked_scope: checked_scope,
+ decision: decision,
+ rollout_notes: rollout_notes,
+ recent_rollout_events: website.rollout_events.order(created_at: :desc).limit(5)
+ )
+ end
+
+ private
+
+ attr_reader :website, :cache
+
+ def summary
+ @summary ||= Distillator::ShadowSiteSummary.call(website: website, cache: resolved_cache, include_cache_links: true)
+ end
+
+ def transition_status
+ @transition_status ||= Distillator::TransitionStatus.call(
+ website: website,
+ cache: resolved_cache,
+ evidence_by_kind: transition_evidence_by_kind
+ )
+ end
+
+ def transition_evidence_by_kind
+ @transition_evidence_by_kind ||= website.latest_transition_evidences_by_kind
+ end
+
+ def transition_evidence_explanations
+ @transition_evidence_explanations ||= transition_status.checks.map do |check|
+ evidence_key = evidence_key_for(check)
+ Distillator::TransitionEvidenceExplanation.call(
+ check_kind: evidence_key,
+ evidence: transition_evidence_by_kind[evidence_key],
+ website: website,
+ state: check.fetch(:state)
+ )
+ end
+ end
+
+ def checked_scope
+ @checked_scope ||= begin
+ fetch_details = transition_evidence_by_kind["fetch_parity"]&.details.to_h || {}
+ statement_details = transition_evidence_by_kind["statement_delta"]&.details.to_h || {}
+ export_details = transition_evidence_by_kind["export_diff"]&.details.to_h || {}
+ representative_urls = Array(statement_details["representative_webpages"] || export_details["representative_webpages"] || fetch_details["representative_webpages"]).compact
+ representative_count = (statement_details["representative_webpage_count"] || export_details["representative_webpage_count"] || fetch_details["representative_webpage_count"] || representative_urls.count).to_i
+ candidate_count = (statement_details["candidate_webpage_count"] || export_details["candidate_webpage_count"] || fetch_details["candidate_webpage_count"] || representative_count).to_i
+ publishable_count = (statement_details["publishable_event_page_count"] || export_details["publishable_event_page_count"] || fetch_details["publishable_event_page_count"] || 0).to_i
+
+ {
+ representative_webpage_count: representative_count,
+ candidate_webpage_count: candidate_count,
+ publishable_event_page_count: publishable_count,
+ representative_webpages: representative_urls,
+ selection_rule: statement_details["selection_rule"] || export_details["selection_rule"] || fetch_details["selection_rule"] || Distillator::TransitionCheck::SELECTION_RULE,
+ selected_candidate_tier_count: (statement_details["selected_candidate_tier_count"] || export_details["selected_candidate_tier_count"] || fetch_details["selected_candidate_tier_count"]).to_i,
+ statements_refreshed_count: (statement_details["statements_refreshed_count"] || 0).to_i,
+ statements_failed_count: (statement_details["statements_failed_count"] || transition_evidence_by_kind["statement_delta"]&.statement_delta || 0).to_i,
+ critical_statements_failed_count: integer_detail(statement_details, "critical_statements_failed_count"),
+ optional_statements_failed_count: integer_detail(statement_details, "optional_statements_failed_count"),
+ legacy_statement_failure: legacy_statement_failure_details?(statement_details),
+ statement_failure_reason: statement_details["reason"] || statement_details[:reason],
+ export_compared: export_details["export_compared"] == true,
+ export_basis: export_details["export_basis"].presence || "current export vs production-equivalent export",
+ sample_small: (
+ statement_details["sample_small"] == true ||
+ export_details["sample_small"] == true ||
+ fetch_details["sample_small"] == true ||
+ (publishable_count.positive? ? publishable_count : candidate_count) > representative_count
+ )
+ }
+ end
+ end
+
+ def url_matrix
+ @url_matrix ||= begin
+ fetch_rows = Array(transition_evidence_by_kind["fetch_parity"]&.details.to_h&.[]("representative_url_results") ||
+ transition_evidence_by_kind["fetch_parity"]&.details.to_h&.[](:representative_url_results))
+ statement_rows = Array(transition_evidence_by_kind["statement_delta"]&.details.to_h&.[]("representative_url_statement_results") ||
+ transition_evidence_by_kind["statement_delta"]&.details.to_h&.[](:representative_url_statement_results))
+ export_rows = Array(transition_evidence_by_kind["export_diff"]&.details.to_h&.[]("representative_url_export_results") ||
+ transition_evidence_by_kind["export_diff"]&.details.to_h&.[](:representative_url_export_results))
+ urls = checked_scope[:representative_webpages]
+ urls.map do |url|
+ fetch_row = symbolize_row(fetch_rows.find { |row| symbolize_row(row)[:url] == url })
+ fetch_row = fallback_fetch_row(url) if fetch_row.blank?
+ statement_row = symbolize_row(statement_rows.find { |row| symbolize_row(row)[:url] == url })
+ statement_row = fallback_statement_row(url) if statement_row.blank?
+ export_row = symbolize_row(export_rows.find { |row| symbolize_row(row)[:url] == url })
+ export_row = fallback_export_row(url) if export_row.blank?
+ {
+ url: url,
+ webpage_id: fetch_row[:webpage_id],
+ condenser_fetch_result: condenser_fetch_label(fetch_row),
+ legacy_lookup_result: legacy_lookup_label(fetch_row),
+ cache_comparison_result: cache_comparison_label(fetch_row),
+ statement_check_result: statement_check_label(statement_row),
+ export_impact: export_impact_label(export_row),
+ fetch_reason: fetch_row[:fetch_reason],
+ legacy_lookup_reason: fetch_row[:legacy_lookup_error] || fetch_row[:legacy_lookup_status],
+ cache_compare_reason: cache_compare_reason(fetch_row),
+ statement_reason: statement_row[:reason],
+ export_reason: export_row[:reason]
+ }
+ end
+ end
+ end
+
+ def root_cause
+ @root_cause ||= begin
+ fetch_rows = url_matrix.select do |row|
+ row[:fetch_reason].present? &&
+ row[:fetch_reason] != "ok" &&
+ row[:condenser_fetch_result] != "Passed"
+ end
+ legacy_rows = url_matrix.select { |row| row[:legacy_lookup_result] != "OK" }
+ compare_rows = url_matrix.select { |row| row[:cache_comparison_result].in?(%w[Failed Review Unknown Missing]) }
+
+ if primary_blocker&.key == "statement_delta"
+ {
+ failed_layer: "statements",
+ concrete_reason: primary_blocker.headline,
+ affected_url_count: checked_scope[:representative_webpage_count],
+ next_operator_action: transition_status.primary_action
+ }
+ elsif primary_blocker&.key == "export_diff"
+ {
+ failed_layer: "export",
+ concrete_reason: primary_blocker.headline,
+ affected_url_count: checked_scope[:representative_webpage_count],
+ next_operator_action: transition_status.primary_action
+ }
+ elsif fetch_rows.any?
+ {
+ failed_layer: "fetch",
+ concrete_reason: fetch_rows.first[:fetch_reason].to_s.humanize,
+ affected_url_count: fetch_rows.count,
+ next_operator_action: "Fetch or refresh Condenser cache for the affected URLs, then rerun the transition batch check."
+ }
+ elsif legacy_rows.any?
+ {
+ failed_layer: "legacy lookup",
+ concrete_reason: legacy_rows.first[:legacy_lookup_reason].to_s.humanize,
+ affected_url_count: legacy_rows.count,
+ next_operator_action: "Open the active Wringer cache or fix the legacy lookup configuration before rerunning the transition batch check."
+ }
+ elsif compare_rows.any?
+ {
+ failed_layer: "cache compare",
+ concrete_reason: compare_rows.first[:cache_compare_reason].to_s.humanize,
+ affected_url_count: compare_rows.count,
+ next_operator_action: "Compare Condenser vs Wringer for the affected URLs and review the parity differences."
+ }
+ else
+ {
+ failed_layer: "evidence missing",
+ concrete_reason: "No failing layer is currently recorded for the sampled URLs.",
+ affected_url_count: 0,
+ next_operator_action: transition_status.primary_action
+ }
+ end
+ end
+ end
+
+ def primary_blocker
+ @primary_blocker ||= transition_evidence_explanations.find { |explanation| explanation.severity == "blocker" && explanation.state != "passed" }
+ end
+
+ def statement_failure_groups
+ @statement_failure_groups ||= begin
+ details = transition_evidence_by_kind["statement_delta"]&.details.to_h || {}
+ failing_statements = statement_failure_entries(details)
+ statements_failed_count = (details["statements_failed_count"] || details[:statements_failed_count] || 0).to_i
+ reasons = Array(details["refresh_errors"] || details[:refresh_errors]).map(&:to_s)
+ normalized_reason = normalize_statement_failure_reason(reasons.first.presence || details["reason"] || details[:reason])
+
+ grouped = failing_statements.group_by do |statement|
+ [
+ statement[:severity].presence || "blocker",
+ normalized_reason,
+ statement[:source].presence || "Unknown source"
+ ]
+ end.map do |(severity, reason, source), entries|
+ {
+ severity: severity.presence || entries.first[:severity].presence || "blocker",
+ reason: reason,
+ source: source,
+ count: entries.count,
+ statement_ids: entries.map { |entry| entry[:id] }.compact
+ }
+ end
+
+ if grouped.blank? && statements_failed_count.positive?
+ grouped = [{
+ severity: legacy_statement_failure_details?(details) ? "unknown" : (details["optional_statements_failed_count"].to_i.positive? && details["critical_statements_failed_count"].to_i.zero? ? "warning" : "blocker"),
+ reason: normalized_reason,
+ source: nil,
+ count: statements_failed_count,
+ statement_ids: []
+ }]
+ end
+
+ grouped.sort_by { |group| [group[:severity] == "blocker" ? 0 : 1, -group[:count].to_i, group[:reason].to_s, group[:source].to_s] }
+ end
+ end
+
+ def decision
+ @decision ||= begin
+ action = if primary_blocker.present?
+ primary_blocker.next_action
+ elsif transition_status.review_activation_eligible
+ "Use the review checklist, then activate with a recorded reason if the content is acceptable."
+ elsif transition_status.status == :ready
+ "Promote to active when you are satisfied with the evidence."
+ else
+ transition_status.primary_action
+ end
+
+ {
+ label: decision_label,
+ why: decision_reasons,
+ evidence: transition_status.checks.map do |check|
+ evidence_key = evidence_key_for(check)
+ {
+ label: check.fetch(:label),
+ state: check.fetch(:state),
+ checked_at: transition_evidence_by_kind[evidence_key]&.checked_at
+ }
+ end,
+ recommended_action: action
+ }
+ end
+ end
+
+ def resolved_cache
+ @resolved_cache ||= cache || Distillator::ShadowReportQuery.latest_cache_for_website(website)
+ end
+
+ def rollout_notes
+ notes = [Distillator::RolloutCopy.description(website.distillator_mode)]
+ if website.lavitrine_pipeline?
+ notes << "La Vitrine pipeline requires statement equivalence before activation. Export confirmation and fetch diagnostics are supporting evidence."
+ else
+ notes << "Statement equivalence is the activation gate. Export confirmation and fetch diagnostics are supporting evidence."
+ end
+ notes
+ end
+
+ def decision_label
+ case transition_status.status
+ when :blocked
+ "Do not activate yet"
+ when :review
+ "Review before activating"
+ when :ready
+ "Safe to promote"
+ else
+ "Run checks before activating"
+ end
+ end
+
+ def decision_reasons
+ return [primary_blocker.headline] if primary_blocker.present?
+ return transition_status.warnings.presence if transition_status.warnings.any?
+
+ ["All sampled transition checks are currently passing."]
+ end
+
+ def evidence_key_for(check)
+ case check.fetch(:key).to_s
+ when "statement_equivalence"
+ "statement_delta"
+ when "export_confirmation"
+ "export_diff"
+ when "fetch_diagnostic"
+ "fetch_parity"
+ else
+ check.fetch(:key).to_s
+ end
+ end
+
+ def symbolize_row(row)
+ row.respond_to?(:to_h) ? row.to_h.symbolize_keys : {}
+ end
+
+ def condenser_fetch_label(row)
+ return "Missing" if row.blank?
+ if row[:fetch_status] == "failed"
+ return "Captcha" if row[:fetch_reason] == "captcha_detected"
+ return "Renderer unavailable" if %w[renderer_unavailable phantomjs_api_key_missing].include?(row[:fetch_reason].to_s)
+ return "Timed out" if row[:fetch_reason] == "transition_check_timeout_budget_exceeded"
+ return "Redirected" if row[:fetch_reason] == "redirect_to_listing"
+
+ return "Failed"
+ end
+
+ "Passed"
+ end
+
+ def legacy_lookup_label(row)
+ status = row[:legacy_lookup_status].to_s
+ return "OK" if status.blank? || status == "ok"
+ return "Missing config" if status == "missing_config"
+ return "Unreachable" if status == "unreachable"
+ return "Body omitted" if status == "body_omitted"
+
+ status.humanize
+ end
+
+ def cache_comparison_label(row)
+ case row[:comparison_status].to_s
+ when "passed", ""
+ "Passed"
+ when "metadata_only"
+ "Passed with metadata notes"
+ when "review"
+ "Review"
+ when "unknown", "not_performed"
+ "Unknown"
+ when "missing"
+ "Missing"
+ else
+ "Failed"
+ end
+ end
+
+ def statement_check_label(row)
+ case row[:status].to_s
+ when "passed"
+ "Passed"
+ when "warning"
+ "Warning"
+ when "failed"
+ "Failed"
+ when "blocked_by_fetch"
+ "Blocked by fetch"
+ when "inconclusive"
+ "Inconclusive"
+ else
+ transition_status.statements.to_s.humanize
+ end
+ end
+
+ def export_impact_label(row)
+ case row[:status].to_s
+ when "checked"
+ "Passed"
+ when "failed"
+ "Failed"
+ when "blocked_by_fetch"
+ "Blocked by fetch"
+ when "inconclusive"
+ "Inconclusive"
+ else
+ transition_status.export.to_s.humanize
+ end
+ end
+
+ def normalize_statement_failure_reason(raw_reason)
+ reason = raw_reason.to_s.strip
+ return "Statement failure" if reason.blank?
+ return "Blank DSL result" if reason.match?(/blank/i)
+ return "Captcha during URL step" if reason.match?(/captcha/i)
+ return "Invalid URL from json_url" if reason.match?(/invalid url/i) || reason.match?(/json_url/i)
+ return "Optional statement warning" if reason == "optional_statement_refresh_warning"
+ return "Critical statement failure" if reason.in?(%w[critical_statement_refresh_failed critical_and_optional_statement_refresh_failed])
+
+ reason.sub(/\AReason:\s*/i, "").strip.humanize
+ end
+
+ def statement_failure_entries(details)
+ explicit = []
+ explicit.concat(Array(details["critical_failing_statements"] || details[:critical_failing_statements]).map do |entry|
+ normalize_statement_failure_entry(entry, "blocker")
+ end)
+ explicit.concat(Array(details["optional_failing_statements"] || details[:optional_failing_statements]).map do |entry|
+ normalize_statement_failure_entry(entry, "warning")
+ end)
+ return explicit if explicit.any?
+
+ default_severity = legacy_statement_failure_details?(details) ? "unknown" : nil
+ Array(details["failing_statements"] || details[:failing_statements]).map do |entry|
+ normalize_statement_failure_entry(entry, default_severity)
+ end
+ end
+
+ def normalize_statement_failure_entry(entry, severity)
+ hash = entry.respond_to?(:to_h) ? entry.to_h.symbolize_keys : {}
+ hash[:severity] = severity.presence || hash[:severity].presence || "blocker"
+ hash
+ end
+
+ def cache_compare_reason(row)
+ return row[:comparison_status] if row[:comparison_status].present? && row[:comparison_status] != "passed"
+ return "metadata_only_difference" if row[:comparison_status] == "metadata_only"
+
+ "ok"
+ end
+
+ def fallback_fetch_row(url)
+ details = transition_evidence_by_kind["fetch_parity"]&.details.to_h || {}
+ {
+ url: url,
+ webpage_id: website.webpages.find_by(url: url)&.id,
+ fetch_status: transition_status.fetch == :failed ? "failed" : "passed",
+ fetch_reason: details["reason"] || details[:reason] || "ok",
+ legacy_lookup_status: details["legacy_lookup_status"] || details[:legacy_lookup_status] || "ok",
+ legacy_lookup_error: details["legacy_lookup_error"] || details[:legacy_lookup_error],
+ comparison_status: fallback_comparison_status(details)
+ }
+ end
+
+ def fallback_statement_row(url)
+ details = transition_evidence_by_kind["statement_delta"]&.details.to_h || {}
+ {
+ url: url,
+ status: case transition_status.statements
+ when :passed then "passed"
+ when :warning then "warning"
+ when :failed then "failed"
+ when :not_evaluated then "blocked_by_fetch"
+ when :inconclusive then "inconclusive"
+ else transition_status.statements.to_s
+ end,
+ reason: details["reason"] || details[:reason]
+ }
+ end
+
+ def fallback_export_row(url)
+ details = transition_evidence_by_kind["export_diff"]&.details.to_h || {}
+ {
+ url: url,
+ status: case transition_status.export
+ when :passed then "checked"
+ when :failed then "failed"
+ when :blocked_by_fetch then "blocked_by_fetch"
+ when :inconclusive then "inconclusive"
+ else transition_status.export.to_s
+ end,
+ reason: details["reason"] || details[:reason]
+ }
+ end
+
+ def fallback_comparison_status(details)
+ reason = details["reason"] || details[:reason]
+ return "failed" if reason == "cache_compare_blocking_regression"
+ return "missing" if reason == "cache_compare_missing"
+ return "unknown" if reason == "cache_compare_unknown"
+ return "review" if reason == "review_needed_difference"
+ return "metadata_only" if reason == "metadata_only_difference"
+
+ "passed"
+ end
+
+ def legacy_statement_failure_details?(details)
+ return false unless (details["statements_failed_count"] || details[:statements_failed_count] || 0).to_i.positive?
+
+ reason = details["reason"] || details[:reason]
+ return false if reason.to_s.in?(%w[fetch_failed_before_statement_refresh partial_fetch_failed_before_statement_refresh transition_check_timeout_budget_exceeded no_selected_statements optional_statement_refresh_warning])
+
+ !detail_key_present?(details, "critical_statements_failed_count") &&
+ !detail_key_present?(details, "optional_statements_failed_count")
+ end
+
+ def integer_detail(details, key)
+ return nil unless detail_key_present?(details, key)
+
+ (details[key.to_s] || details[key.to_sym]).to_i
+ end
+
+ def detail_key_present?(details, key)
+ details.key?(key.to_s) || details.key?(key.to_sym)
+ end
+ end
+end
diff --git a/app/services/distillator/shadow_site_summary.rb b/app/services/distillator/shadow_site_summary.rb
new file mode 100644
index 00000000..de48f50c
--- /dev/null
+++ b/app/services/distillator/shadow_site_summary.rb
@@ -0,0 +1,157 @@
+require "cgi"
+
+module Distillator
+ class ShadowSiteSummary
+ Result = Struct.new(
+ :website,
+ :cache,
+ :cohort_key,
+ :cohort_label,
+ :status,
+ :fetch_status,
+ :statements_status,
+ :export_status,
+ :safety,
+ :confidence,
+ :production_backend,
+ :testing_backend,
+ :health_summary,
+ :health_status,
+ :health_severity,
+ :latest_refresh,
+ :latest_successful_refresh,
+ :issue_key,
+ :blockers,
+ :warnings,
+ :cache_link_payload,
+ :last_checked,
+ :promotable,
+ :priority,
+ :mode_label,
+ :production_backend_label,
+ :readiness_label,
+ :severity,
+ :primary_blocker,
+ :primary_action,
+ keyword_init: true
+ )
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(website:, cache: nil, stale_after: Distillator::CacheHealth::STALE_AFTER, evidence_by_kind: nil, include_cache_links: false)
+ @website = website
+ @cache = cache
+ @stale_after = stale_after
+ @evidence_by_kind = evidence_by_kind
+ @include_cache_links = include_cache_links
+ end
+
+ def call
+ Result.new(
+ website: website,
+ cache: cache,
+ cohort_key: cohort_key,
+ cohort_label: cohort_label,
+ status: transition_status.status,
+ fetch_status: transition_status.fetch,
+ statements_status: transition_status.statements,
+ export_status: transition_status.export,
+ safety: transition_status.safety,
+ confidence: transition_status.confidence,
+ production_backend: production_backend,
+ testing_backend: :condenser,
+ health_summary: health_summary,
+ health_status: health_status,
+ health_severity: health_severity,
+ latest_refresh: cache&.scrape_date,
+ latest_successful_refresh: cache&.successful_refresh,
+ issue_key: cache&.primary_issue_key,
+ blockers: transition_status.blockers,
+ warnings: transition_status.warnings,
+ cache_link_payload: include_cache_links? ? cache_link_payload : nil,
+ last_checked: transition_status.last_checked,
+ promotable: transition_status.status == :ready,
+ priority: website.lavitrine_pipeline?,
+ mode_label: Distillator::RolloutCopy.label(website.distillator_mode),
+ production_backend_label: Distillator::RolloutCopy.active_backend_label(website.distillator_mode),
+ readiness_label: transition_status.readiness_label,
+ severity: transition_status.severity,
+ primary_blocker: transition_status.primary_blocker,
+ primary_action: operator_next_action
+ )
+ end
+
+ private
+
+ attr_reader :website, :cache, :stale_after, :evidence_by_kind
+
+ def health_summary
+ return "Unknown" unless evidence_exists?
+
+ label = cache.primary_issue_label.presence || cache.health_status.to_s.humanize
+ "#{cache.health_severity.to_s.humanize}: #{label}"
+ end
+
+ def health_status
+ return "unknown" unless evidence_exists?
+
+ cache.health_status.to_s.presence || "unknown"
+ end
+
+ def health_severity
+ return "unknown" unless evidence_exists?
+
+ cache.health_severity.to_s.presence || "unknown"
+ end
+
+ def cohort_key
+ website.respond_to?(:distillator_primary_cohort_key) ? website.distillator_primary_cohort_key : nil
+ end
+
+ def cohort_label
+ website.respond_to?(:distillator_primary_cohort_label) ? website.distillator_primary_cohort_label : nil
+ end
+
+ def evidence_exists?
+ cache.present?
+ end
+
+ def transition_status
+ @transition_status ||= Distillator::TransitionStatus.call(
+ website: website,
+ cache: cache,
+ evidence_by_kind: resolved_evidence_by_kind
+ )
+ end
+
+ def cache_link_payload
+ @cache_link_payload ||= Distillator::TransitionCheck.call(
+ website: website,
+ cache: cache,
+ evidence_by_kind: resolved_evidence_by_kind
+ ).cache_link_payload
+ end
+
+ def operator_next_action
+ Distillator::OperatorNextAction.call(
+ website: website,
+ transition_status: transition_status,
+ cache_link_payload: include_cache_links? ? cache_link_payload : nil
+ )
+ end
+
+ def production_backend
+ @production_backend ||= Distillator::FetchMode.rollout_resolution_object(website: website).active_backend
+ end
+
+ def include_cache_links?
+ @include_cache_links == true
+ end
+
+ def resolved_evidence_by_kind
+ evidence_by_kind || (website.respond_to?(:latest_transition_evidences_by_kind) ? website.latest_transition_evidences_by_kind : {})
+ end
+ end
+end
diff --git a/app/services/distillator/staging_rollout_repair.rb b/app/services/distillator/staging_rollout_repair.rb
new file mode 100644
index 00000000..2652ef16
--- /dev/null
+++ b/app/services/distillator/staging_rollout_repair.rb
@@ -0,0 +1,119 @@
+module Distillator
+ class StagingRolloutRepair
+ WebsiteSummary = Struct.new(:id, :name, :seedurl, :mode, keyword_init: true)
+
+ Result = Struct.new(
+ :dry_run,
+ :applied,
+ :invalid_count,
+ :repaired_count,
+ :websites,
+ :changed_websites,
+ :unchanged_websites,
+ :errors,
+ keyword_init: true
+ ) do
+ def success?
+ errors.blank?
+ end
+ end
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(apply: false, actor: "staging_rollout_repair", reason: "Staging rollout repair")
+ @apply = apply
+ @actor = actor
+ @reason = reason
+ end
+
+ def call
+ invalid_websites = scoped_invalid_websites
+ summaries = invalid_websites.map { |website| website_summary(website) }
+
+ return dry_run_result(summaries) unless apply?
+ return failure_result(summaries, "Staging rollout repair can only run on staging.") unless Distillator::TransitionRuntime.staging?
+
+ repaired_count = 0
+ errors = []
+ changed_websites = []
+ unchanged_websites = []
+
+ invalid_websites.each do |website|
+ result = Distillator::RolloutTransition.call(
+ website: website,
+ to_mode: "shadow",
+ actor: actor,
+ reason: reason
+ )
+
+ if result.success?
+ repaired_count += 1
+ changed_websites << website_summary(website.reload)
+ else
+ errors << "#{website.id} #{website.name}: #{result.errors.join(', ')}"
+ unchanged_websites << website_summary(website)
+ end
+ end
+
+ Result.new(
+ dry_run: false,
+ applied: true,
+ invalid_count: summaries.length,
+ repaired_count: repaired_count,
+ websites: summaries,
+ changed_websites: changed_websites,
+ unchanged_websites: unchanged_websites,
+ errors: errors
+ )
+ end
+
+ private
+
+ attr_reader :actor, :reason
+
+ def apply?
+ @apply == true
+ end
+
+ def scoped_invalid_websites
+ Distillator::TransitionRuntime.staging_invalid_rollout_mode_scope.order(:id).to_a
+ end
+
+ def website_summary(website)
+ WebsiteSummary.new(
+ id: website.id,
+ name: website.name,
+ seedurl: website.seedurl,
+ mode: website.distillator_mode.presence || "blank"
+ )
+ end
+
+ def dry_run_result(summaries)
+ Result.new(
+ dry_run: true,
+ applied: false,
+ invalid_count: summaries.length,
+ repaired_count: 0,
+ websites: summaries,
+ changed_websites: [],
+ unchanged_websites: summaries,
+ errors: []
+ )
+ end
+
+ def failure_result(summaries, error)
+ Result.new(
+ dry_run: false,
+ applied: false,
+ invalid_count: summaries.length,
+ repaired_count: 0,
+ websites: summaries,
+ changed_websites: [],
+ unchanged_websites: summaries,
+ errors: [error]
+ )
+ end
+ end
+end
diff --git a/app/services/distillator/transition_check.rb b/app/services/distillator/transition_check.rb
new file mode 100644
index 00000000..8780b96d
--- /dev/null
+++ b/app/services/distillator/transition_check.rb
@@ -0,0 +1,258 @@
+module Distillator
+ class TransitionCheck
+ PUBLISHABLE_SAMPLE_PERCENT = 0.20
+ MINIMUM_PUBLISHABLE_SAMPLE_SIZE = 5
+ MAXIMUM_PUBLISHABLE_SAMPLE_SIZE = 25
+ FALLBACK_REPRESENTATIVE_SAMPLE_SIZE = 3
+ SELECTION_RULE = "20% of publishable event pages, minimum 5, capped at 25, or all pages when fewer than 5 exist.".freeze
+
+ Result = Struct.new(
+ :website_id,
+ :website,
+ :mode,
+ :priority,
+ :active_backend,
+ :latest_cache_status,
+ :cache_present,
+ :compare_available,
+ :blocking_issues,
+ :warnings,
+ :promotable,
+ :status,
+ :fetch,
+ :statements,
+ :export,
+ :representative_webpage,
+ :representative_webpages,
+ :representative_url,
+ :representative_webpage_count,
+ :candidate_webpage_count,
+ :publishable_event_page_count,
+ :selected_candidate_tier_count,
+ :selection_rule,
+ :attempted_condenser_fetch,
+ :condenser_fetch_result,
+ :comparison,
+ :comparison_policy,
+ :cache,
+ :cache_link_payload,
+ :primary_action,
+ keyword_init: true
+ )
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(
+ website:,
+ cache: nil,
+ evidence_by_kind: nil,
+ run_fetch: false,
+ comparison_policy: :operator,
+ fetch_cache_store: Distillator::FetchCacheStore,
+ cache_compare: Distillator::CacheCompare
+ )
+ @website = website.is_a?(Website) ? website : Website.find(website)
+ @cache = cache
+ @evidence_by_kind = evidence_by_kind
+ @run_fetch = run_fetch == true
+ @comparison_policy = normalize_comparison_policy(comparison_policy)
+ @fetch_cache_store = fetch_cache_store
+ @cache_compare = cache_compare
+ end
+
+ def call
+ representative_webpages = representative_webpages()
+ representative_webpage = representative_webpages.first
+ fetch_attempt = fetch_attempt_for(representative_webpage)
+
+ Result.new(
+ website_id: website.id,
+ website: website,
+ mode: website.distillator_mode.to_sym,
+ priority: website.lavitrine_pipeline?,
+ active_backend: rollout_resolution.active_backend,
+ latest_cache_status: resolved_cache&.health_status.to_s.presence || "unknown",
+ cache_present: resolved_cache.present?,
+ compare_available: compare_available?,
+ blocking_issues: transition_status.blockers,
+ warnings: transition_status.warnings,
+ promotable: transition_status.status == :ready,
+ status: transition_status.status,
+ fetch: transition_status.fetch,
+ statements: transition_status.statements,
+ export: transition_status.export,
+ representative_webpage: representative_webpage,
+ representative_webpages: representative_webpages,
+ representative_url: representative_webpage&.url,
+ representative_webpage_count: representative_webpages.count,
+ candidate_webpage_count: candidate_webpage_count,
+ publishable_event_page_count: publishable_event_page_count,
+ selected_candidate_tier_count: selected_candidate_tier_count,
+ selection_rule: SELECTION_RULE,
+ attempted_condenser_fetch: fetch_attempt[:attempted],
+ condenser_fetch_result: fetch_attempt[:result],
+ comparison: fetch_attempt[:comparison],
+ comparison_policy: comparison_policy,
+ cache: resolved_cache,
+ cache_link_payload: cache_link_payload,
+ primary_action: primary_action
+ )
+ end
+
+ private
+
+ attr_reader :website, :evidence_by_kind, :fetch_cache_store, :cache_compare, :comparison_policy
+
+ def cache
+ @cache ||= Distillator::ShadowReportQuery.latest_cache_for_website(website)
+ end
+
+ def resolved_cache
+ @resolved_cache ||= cache
+ end
+
+ def transition_status
+ @transition_status ||= Distillator::TransitionStatus.call(
+ website: website,
+ cache: resolved_cache,
+ evidence_by_kind: evidence_by_kind || website.latest_transition_evidences_by_kind
+ )
+ end
+
+ def rollout_resolution
+ @rollout_resolution ||= Distillator::FetchMode.rollout_resolution_object(website: website)
+ end
+
+ def cache_link_payload
+ @cache_link_payload ||= begin
+ url = resolved_cache&.normalized_url.presence || website.webpages.first&.url.presence || website.seedurl
+ Distillator::CacheLinkResolver.call(url: url, website: website)
+ end
+ end
+
+ def compare_available?
+ Array(cache_link_payload[:secondary_links]).any? do |link|
+ link[:label] == Distillator::RolloutCopy.compare_label && link[:url].present?
+ end
+ end
+
+ def primary_action
+ Distillator::OperatorNextAction.call(
+ website: website,
+ transition_status: transition_status,
+ cache_link_payload: cache_link_payload
+ )
+ end
+
+ def representative_webpages
+ if publishable_event_page_count.positive?
+ publishable_representative_webpages
+ else
+ generic_representative_scope.limit(FALLBACK_REPRESENTATIVE_SAMPLE_SIZE).to_a
+ end
+ end
+
+ def candidate_webpage_count
+ website.webpages.transition_candidates.count
+ end
+
+ def publishable_event_page_count
+ @publishable_event_page_count ||= publishable_event_scope.count
+ end
+
+ def selected_candidate_tier_count
+ publishable_event_page_count.positive? ? publishable_event_page_count : generic_representative_scope.count
+ end
+
+ def publishable_representative_webpages
+ remaining = publishable_sample_size
+ pages = []
+
+ [future_publishable_event_scope, nil_archive_publishable_event_scope, past_publishable_event_scope].each do |scope|
+ break if remaining <= 0
+
+ selected = scope.limit(remaining).to_a
+ pages.concat(selected)
+ remaining -= selected.length
+ end
+
+ pages
+ end
+
+ def publishable_sample_size
+ count = publishable_event_page_count
+ return 0 if count.zero?
+
+ [count, [[(count * PUBLISHABLE_SAMPLE_PERCENT).ceil, MINIMUM_PUBLISHABLE_SAMPLE_SIZE].max, MAXIMUM_PUBLISHABLE_SAMPLE_SIZE].min].min
+ end
+
+ def publishable_event_scope
+ website.webpages.publishable.event_pages
+ end
+
+ def future_publishable_event_scope
+ publishable_event_scope
+ .where("archive_date >= ?", Time.current.beginning_of_day)
+ .reorder(archive_date: :asc, updated_at: :desc, id: :asc)
+ end
+
+ def past_publishable_event_scope
+ publishable_event_scope
+ .where.not(archive_date: nil)
+ .where("archive_date < ?", Time.current.beginning_of_day)
+ .reorder(archive_date: :desc, updated_at: :desc, id: :asc)
+ end
+
+ def nil_archive_publishable_event_scope
+ publishable_event_scope
+ .where(archive_date: nil)
+ .reorder(updated_at: :desc, created_at: :desc, id: :asc)
+ end
+
+ def generic_representative_scope
+ website.webpages.reorder(updated_at: :desc, created_at: :desc, id: :asc)
+ end
+
+ def run_fetch?
+ @run_fetch == true
+ end
+
+ def fetch_attempt_for(representative_webpage)
+ return { attempted: false, result: nil, comparison: nil } unless run_fetch?
+ return { attempted: false, result: nil, comparison: nil } unless representative_webpage.present?
+
+ # Transition checks force fresh Condenser evidence through the internal-only path.
+ # CacheCompare performs the separate legacy/Wringer lookup afterward.
+ fetch_result = fetch_cache_store.fetch(
+ uri: representative_webpage.url,
+ force_scrape: true,
+ mode: "internal",
+ website: website,
+ log_context: {
+ source: "transition_check",
+ website_id: website.id,
+ seedurl: website.seedurl
+ }
+ )
+
+ @resolved_cache = fetch_result.cache || cache
+
+ {
+ attempted: true,
+ result: fetch_result,
+ comparison: cache_compare.call(
+ uri: representative_webpage.url,
+ condenser_result: fetch_result,
+ comparison_policy: comparison_policy
+ )
+ }
+ end
+
+ def normalize_comparison_policy(value)
+ candidate = value.to_s.presence&.to_sym || :operator
+ Distillator::CacheCompare::POLICIES.include?(candidate) ? candidate : :operator
+ end
+ end
+end
diff --git a/app/services/distillator/transition_check_runner.rb b/app/services/distillator/transition_check_runner.rb
new file mode 100644
index 00000000..3be1a80e
--- /dev/null
+++ b/app/services/distillator/transition_check_runner.rb
@@ -0,0 +1,824 @@
+module Distillator
+ class TransitionCheckRunner
+ REQUEST_BUDGET_SECONDS = 20
+ TIMEOUT_GUARD_SECONDS = 3
+
+ Result = Struct.new(:website, :records, keyword_init: true) do
+ def flash_message
+ prefix = complete? ? "Transition check recorded:" : "Transition check incomplete:"
+ "#{prefix} #{summary_items.join(', ')}"
+ end
+
+ def complete?
+ summary_statuses.values.all? { |status| status == "checked" }
+ end
+
+ def summary_statuses
+ records.transform_values { |record| summary_status_for(record) }
+ end
+
+ private
+
+ def summary_status_for(record)
+ reason = record.details.to_h["reason"] || record.details.to_h[:reason]
+ return "not evaluated" if reason == "fetch_failed_before_statement_refresh"
+ return "blocked by fetch" if reason == "fetch_failed_before_export_comparison"
+ return "inconclusive" if %w[
+ no_selected_statements
+ export_diff_not_available
+ partial_fetch_failed_before_statement_refresh
+ partial_fetch_failed_before_export_comparison
+ transition_check_timeout_budget_exceeded
+ ].include?(reason)
+
+ case record.status.to_s
+ when "checked", "accepted"
+ "checked"
+ when "failed", "blocked", "rejected"
+ "failed"
+ else
+ "missing"
+ end
+ end
+
+ def summary_items
+ [
+ "fetch #{summary_statuses.fetch(:fetch_parity)}",
+ "statements #{summary_statuses.fetch(:statement_delta)}",
+ "export #{summary_statuses.fetch(:export_diff)}"
+ ]
+ end
+ end
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(
+ website:,
+ refresh_helper: nil,
+ refresh_runner: Distillator::RefreshRunner,
+ export_service: ExportArtsdataService,
+ transition_check_service: Distillator::TransitionCheck,
+ fetch_cache_store: Distillator::FetchCacheStore,
+ cache_compare: Distillator::CacheCompare,
+ budget_seconds: nil,
+ timeout_guard_seconds: TIMEOUT_GUARD_SECONDS,
+ clock: nil
+ )
+ @website = website.is_a?(Website) ? website : Website.find(website)
+ @refresh_helper = refresh_helper || StatementsHelper.build_refresh_proxy(cookies: {})
+ @refresh_runner = refresh_runner
+ @export_service = export_service
+ @transition_check_service = transition_check_service
+ @fetch_cache_store = fetch_cache_store
+ @cache_compare = cache_compare
+ @budget_seconds = budget_seconds
+ @timeout_guard_seconds = timeout_guard_seconds
+ @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+ @started_at = monotonic_now
+ @representative_result_cache = {}
+ end
+
+ def call
+ transition_check = transition_check_service.call(website: website, run_fetch: false)
+ representative_webpages = Array(transition_check.representative_webpages)
+ representative_url_results = collect_representative_url_results(representative_webpages, transition_check.comparison_policy)
+
+ Result.new(
+ website: website,
+ records: {
+ fetch_parity: record_fetch_check(transition_check, representative_url_results),
+ statement_delta: record_statement_check(transition_check, representative_webpages, representative_url_results),
+ export_diff: record_export_check(transition_check, representative_webpages, representative_url_results)
+ }
+ )
+ end
+
+ private
+
+ attr_reader :website, :refresh_helper, :refresh_runner, :export_service, :transition_check_service, :fetch_cache_store, :cache_compare,
+ :budget_seconds, :timeout_guard_seconds
+
+ def record_fetch_check(transition_check, representative_url_results)
+ latest_cache = representative_url_results.first&.dig(:cache) || transition_check.cache
+ status, details = fetch_check_payload(transition_check, representative_url_results)
+
+ Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: representative_url_results.first&.dig(:url).presence || transition_check.representative_url.presence || cache_or_seed_url(latest_cache),
+ check_kind: :fetch_parity,
+ status: status,
+ primary_issue_key: latest_cache&.primary_issue_key,
+ wringer_http_code: latest_cache&.http_response_code,
+ distillator_http_code: latest_cache&.http_response_code,
+ details: details
+ )
+ end
+
+ def record_statement_check(transition_check, representative_webpages, representative_url_results)
+ latest_cache = representative_url_results.first&.dig(:cache) || transition_check.cache
+ return record_missing_representatives(latest_cache, :statement_delta, transition_check) if representative_webpages.blank?
+
+ fetched_webpages, failed_fetch_results = partition_representative_webpages(representative_webpages, representative_url_results)
+ if timeout_budget_exceeded?
+ return pending_statement_timeout_record(latest_cache, transition_check, representative_webpages, fetched_webpages, failed_fetch_results)
+ end
+
+ statements_scope = representative_statement_scope(fetched_webpages)
+ if representative_url_results.all? { |result| result[:fetch_status] == "failed" }
+ return Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: cache_or_seed_url(latest_cache),
+ check_kind: :statement_delta,
+ status: :pending,
+ statement_delta: 0,
+ statement_count_delta_acceptable: nil,
+ details: scope_details(transition_check, representative_webpages).merge(
+ source: "statement_refresh",
+ representative_url_statement_results: representative_url_results.map do |result|
+ {
+ url: result[:url],
+ status: "blocked_by_fetch",
+ reason: result[:fetch_reason]
+ }
+ end,
+ statements_refreshed_count: 0,
+ statements_failed_count: 0,
+ failing_statement_ids: [],
+ failing_statements: [],
+ reason: "fetch_failed_before_statement_refresh"
+ )
+ )
+ end
+
+ if statements_scope.none?
+ return Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: cache_or_seed_url(latest_cache),
+ check_kind: :statement_delta,
+ status: :pending,
+ statement_delta: 0,
+ statement_count_delta_acceptable: nil,
+ details: scope_details(transition_check, representative_webpages).merge(
+ source: "statement_refresh",
+ statements_refreshed_count: 0,
+ statements_failed_count: 0,
+ failing_statement_ids: [],
+ failing_statements: [],
+ reason: "no_selected_statements"
+ )
+ )
+ end
+
+ refresh_errors = fetched_webpages.flat_map do |webpage|
+ refresh_runner.call(
+ webpage: webpage,
+ refresh_helper: refresh_helper,
+ scrape_options: {}
+ )
+ end
+ failing_statements = representative_problem_statements(fetched_webpages)
+ critical_failing_statements, optional_failing_statements = partition_statement_failures(failing_statements)
+ scope_statements = statements_scope.to_a
+ inferred_critical_failing_statements = if critical_failing_statements.any?
+ critical_failing_statements
+ elsif refresh_errors.present? && failing_statements.blank?
+ scope_statements.select { |statement| blocking_statement_failure?(statement) }
+ else
+ []
+ end
+ inferred_optional_failing_statements = if optional_failing_statements.any?
+ optional_failing_statements
+ elsif refresh_errors.present? && failing_statements.blank? && inferred_critical_failing_statements.blank?
+ scope_statements.reject { |statement| blocking_statement_failure?(statement) }
+ else
+ []
+ end
+ reported_failing_statements = if critical_failing_statements.any? || optional_failing_statements.any?
+ critical_failing_statements + optional_failing_statements
+ elsif refresh_errors.present?
+ scope_statements
+ else
+ []
+ end
+ statement_delta = inferred_critical_failing_statements.count
+ statement_results_by_url = representative_statement_results(
+ fetched_webpages,
+ failed_fetch_results,
+ refresh_errors,
+ inferred_critical_failing_statements,
+ inferred_optional_failing_statements
+ )
+ details = scope_details(transition_check, representative_webpages).merge(
+ source: "statement_refresh",
+ representative_url_statement_results: statement_results_by_url,
+ statements_refreshed_count: statements_scope.count,
+ statements_failed_count: [statement_delta, reported_failing_statements.count].max,
+ critical_statements_failed_count: inferred_critical_failing_statements.count,
+ optional_statements_failed_count: inferred_optional_failing_statements.count,
+ failing_statement_ids: reported_failing_statements.map(&:id),
+ failing_statements: failing_statement_details(reported_failing_statements),
+ critical_failing_statement_ids: inferred_critical_failing_statements.map(&:id),
+ critical_failing_statements: failing_statement_details(inferred_critical_failing_statements),
+ optional_failing_statement_ids: inferred_optional_failing_statements.map(&:id),
+ optional_failing_statements: failing_statement_details(inferred_optional_failing_statements)
+ )
+
+ if inferred_critical_failing_statements.any?
+ status = :failed
+ details[:reason] = inferred_optional_failing_statements.any? ? "critical_and_optional_statement_refresh_failed" : "critical_statement_refresh_failed"
+ elsif refresh_errors.present? || inferred_optional_failing_statements.any?
+ status = :warning
+ details[:reason] = "optional_statement_refresh_warning"
+ elsif failed_fetch_results.any?
+ status = :pending
+ details[:reason] = "partial_fetch_failed_before_statement_refresh"
+ elsif statement_delta.zero?
+ status = :checked
+ else
+ status = :failed
+ end
+ details[:refresh_errors] = compact_refresh_errors(refresh_errors) if refresh_errors.present?
+
+ Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: cache_or_seed_url(latest_cache),
+ check_kind: :statement_delta,
+ status: status,
+ statement_delta: statement_delta,
+ statement_count_delta_acceptable: inferred_critical_failing_statements.empty?,
+ details: details
+ )
+ rescue StandardError => error
+ Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: representative_webpages.first&.url.presence || cache_or_seed_url(latest_cache),
+ check_kind: :statement_delta,
+ status: :failed,
+ statement_count_delta_acceptable: false,
+ details: failure_details("statement_refresh_failed", error)
+ )
+ end
+
+ def record_export_check(transition_check, representative_webpages, representative_url_results)
+ latest_cache = representative_url_results.first&.dig(:cache) || transition_check.cache
+ return record_missing_representatives(latest_cache, :export_diff, transition_check) if representative_webpages.blank?
+ fetched_webpages, failed_fetch_results = partition_representative_webpages(representative_webpages, representative_url_results)
+
+ if timeout_budget_exceeded?
+ return pending_export_timeout_record(latest_cache, transition_check, representative_webpages, fetched_webpages, failed_fetch_results)
+ end
+
+ if representative_url_results.all? { |result| result[:fetch_status] == "failed" }
+ return Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: cache_or_seed_url(latest_cache),
+ check_kind: :export_diff,
+ status: :pending,
+ export_diff_status: "pending",
+ details: scope_details(transition_check, representative_webpages).merge(
+ source: "export_comparison",
+ export_compared: false,
+ export_basis: "current export vs production-equivalent export",
+ representative_url_export_results: representative_url_results.map do |result|
+ {
+ url: result[:url],
+ status: "blocked_by_fetch",
+ reason: result[:fetch_reason]
+ }
+ end,
+ reason: "fetch_failed_before_export_comparison"
+ )
+ )
+ end
+
+ actual = export_service.call(seedurl: website.seedurl)
+ expected = export_service.production_equivalent(seedurl: website.seedurl)
+ if Distillator::ExportNormalizer.blank_export?(actual) || Distillator::ExportNormalizer.blank_export?(expected)
+ return Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: cache_or_seed_url(latest_cache),
+ check_kind: :export_diff,
+ status: :pending,
+ export_diff_status: "pending",
+ details: scope_details(transition_check, representative_webpages).merge(
+ source: "export_comparison",
+ export_compared: false,
+ export_basis: "current export vs production-equivalent export",
+ representative_url_export_results: representative_export_results(representative_webpages, failed_fetch_results, "inconclusive", "export_diff_not_available"),
+ reason: "export_diff_not_available"
+ )
+ )
+ end
+
+ normalized_actual = Distillator::ExportNormalizer.normalize(actual)
+ normalized_expected = Distillator::ExportNormalizer.normalize(expected)
+ diff = Distillator::GraphDiff.call(expected_graph: normalized_expected, actual_graph: normalized_actual)
+ failed = diff.added_count.positive? || diff.removed_count.positive? || diff.changed_literal_values.any? || diff.changed_uri_objects.any?
+ status = failed ? :failed : :checked
+ partial_fetch_coverage = failed_fetch_results.any?
+ effective_status = partial_fetch_coverage ? :pending : status
+ effective_export_checked = partial_fetch_coverage ? false : (status == :checked)
+ effective_export_diff_status = partial_fetch_coverage ? "partial" : status.to_s
+
+ Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: cache_or_seed_url(latest_cache),
+ check_kind: :export_diff,
+ status: effective_status,
+ export_diff_checked: effective_export_checked,
+ export_diff_status: effective_export_diff_status,
+ export_diff_accepted: false,
+ rdf_added_count: diff.added_count,
+ rdf_removed_count: diff.removed_count,
+ details: scope_details(transition_check, representative_webpages).merge(
+ source: "export_comparison",
+ export_compared: !partial_fetch_coverage,
+ graph_diff_performed: true,
+ export_basis: "current export vs production-equivalent export",
+ representative_url_export_results: representative_export_results(representative_webpages, failed_fetch_results, status.to_s, status == :checked ? "ok" : "export_diff_detected"),
+ reason: partial_fetch_coverage ? "partial_fetch_failed_before_export_comparison" : nil,
+ changed_literal_values: diff.changed_literal_values.first(10),
+ changed_uri_objects: diff.changed_uri_objects.first(10)
+ ).compact
+ )
+ rescue StandardError => error
+ Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: cache_or_seed_url(latest_cache),
+ check_kind: :export_diff,
+ status: :failed,
+ export_diff_status: "failed",
+ details: scope_details(transition_check, representative_webpages).merge(
+ export_compared: false,
+ export_basis: "current export vs production-equivalent export"
+ ).merge(failure_details("export_generation_failed", error))
+ )
+ end
+
+ def fetch_check_payload(transition_check, representative_url_results)
+ cache = representative_url_results.first&.dig(:cache) || transition_check.cache
+ representative_webpages = Array(transition_check.representative_webpages)
+ details = scope_details(transition_check, representative_webpages).merge(
+ source: "transition_check",
+ attempted_condenser_fetch: representative_url_results.any? { |result| result[:attempted_condenser_fetch] },
+ condenser_fetch_success: representative_url_results.any? { |result| result[:fetch_status] == "passed" },
+ comparison_performed: representative_url_results.any? { |result| result[:comparison_performed] },
+ comparison_policy: transition_check.comparison_policy.to_s,
+ legacy_source: representative_url_results.first&.dig(:legacy_source),
+ legacy_lookup_status: representative_url_results.first&.dig(:legacy_lookup_status),
+ legacy_lookup_error: representative_url_results.first&.dig(:legacy_lookup_error),
+ condenser_source: representative_url_results.first&.dig(:condenser_source),
+ compare_missing: representative_url_results.first&.dig(:compare_missing),
+ compare_summary: representative_url_results.first&.dig(:compare_summary),
+ representative_url_results: representative_url_results.map { |result| serializable_representative_url_result(result) },
+ failed_layer: representative_failed_layer(representative_url_results),
+ affected_url_count: representative_affected_url_count(representative_url_results),
+ representative_urls_checked: representative_url_results.any?
+ ).compact
+
+ unless representative_webpages.present?
+ return [:pending, details.merge(reason: "no_representative_webpages", representative_urls_checked: false)]
+ end
+
+ unless cache.present?
+ return [:failed, details.merge(reason: "missing_cache")]
+ end
+
+ if representative_url_results.any? { |result| result[:fetch_status] == "failed" }
+ failed_result = representative_url_results.find { |result| result[:fetch_status] == "failed" }
+ return [:failed, details.merge(reason: failed_result[:fetch_reason] || "cache_health_failed")]
+ end
+
+ if representative_url_results.any? { |result| result[:legacy_lookup_status] == "missing_config" }
+ return [:checked, details.merge(reason: "legacy_lookup_missing_config", comparison_performed: false)]
+ end
+
+ if representative_url_results.any? { |result| result[:legacy_lookup_status] == "unreachable" }
+ return [:checked, details.merge(reason: "legacy_lookup_unreachable", comparison_performed: false)]
+ end
+
+ if representative_url_results.any? { |result| result[:legacy_lookup_status] == "body_omitted" }
+ return [:checked, details.merge(reason: "legacy_lookup_body_omitted", comparison_performed: false)]
+ end
+
+ if representative_url_results.any? { |result| result[:comparison_status] == "review" }
+ return [:checked, details.merge(reason: "review_needed_difference")]
+ end
+
+ if representative_url_results.any? { |result| result[:comparison_status] == "unknown" }
+ return [:failed, details.merge(reason: "cache_compare_unknown")]
+ end
+
+ if representative_url_results.any? { |result| result[:comparison_status] == "metadata_only" } &&
+ representative_url_results.none? { |result| %w[failed review unknown].include?(result[:comparison_status]) }
+ return [:checked, details.merge(reason: "metadata_only_difference")]
+ end
+
+ if representative_url_results.any? { |result| result[:comparison_status] == "missing" }
+ return [:failed, details.merge(reason: "cache_compare_missing")]
+ end
+
+ if representative_url_results.any? { |result| result[:comparison_status] == "failed" }
+ return [:failed, details.merge(reason: "cache_compare_blocking_regression")]
+ end
+
+ [:checked, details.merge(reason: "ok")]
+ end
+
+ def cache_or_seed_url(cache)
+ cache&.normalized_url.presence || website.webpages.first&.url.presence || website.seedurl
+ end
+
+ def fetch_failure_reason_from_result(result)
+ return "missing_condenser_attempt" unless result.present?
+ return "empty_body" if truthy?(cache_signal(result.cache, :empty_body))
+ return "captcha_detected" if captcha_result?(result)
+ return missing_renderer_reason(result) if missing_renderer_reason(result).present?
+
+ result.blocking_issue_key.presence ||
+ cache_signal(result.cache, :primary_issue_key).presence ||
+ cache_signal(result.cache, :network_status).presence ||
+ "cache_health_failed"
+ end
+
+ def cache_signal(cache, key)
+ return nil unless cache.present?
+
+ signals = (cache.signals || {}).to_h
+ return signals[key.to_s] if signals.key?(key.to_s)
+ return signals[key.to_sym] if signals.key?(key.to_sym)
+
+ nil
+ end
+
+ def truthy?(value)
+ value == true || value.to_s == "true" || value.to_s == "1"
+ end
+
+ def record_missing_representatives(latest_cache, check_kind, transition_check)
+ Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: cache_or_seed_url(latest_cache),
+ check_kind: check_kind,
+ status: :pending,
+ export_diff_status: check_kind == :export_diff ? "pending" : nil,
+ details: scope_details(transition_check, []).merge(
+ source: "transition_check",
+ export_compared: check_kind == :export_diff ? false : nil,
+ export_basis: check_kind == :export_diff ? "current export vs production-equivalent export" : nil,
+ statements_refreshed_count: check_kind == :statement_delta ? 0 : nil,
+ statements_failed_count: check_kind == :statement_delta ? 0 : nil,
+ reason: "no_representative_webpages"
+ ).compact
+ )
+ end
+
+ def representative_statement_scope(representative_webpages)
+ Statement
+ .selected_for_transition
+ .includes(:source, :webpage)
+ .where(webpage_id: representative_webpages.map(&:id))
+ end
+
+ def representative_problem_statements(representative_webpages)
+ representative_statement_scope(representative_webpages)
+ .select(&:transition_problem?)
+ end
+
+ def partition_statement_failures(statements)
+ Array(statements).partition { |statement| blocking_statement_failure?(statement) }
+ end
+
+ def blocking_statement_failure?(statement)
+ return false unless statement&.webpage&.rdfs_class&.name == "Event"
+
+ Webpage.publishable_required_property_labels.include?(statement.source&.property&.label)
+ end
+
+ def failing_statement_details(statements)
+ statements.map do |statement|
+ {
+ id: statement.id,
+ webpage_url: statement.webpage&.url,
+ source: [statement.source&.property&.label, statement.source&.language].compact.join(" / "),
+ severity: blocking_statement_failure?(statement) ? "blocker" : "warning"
+ }
+ end
+ end
+
+ def compact_refresh_errors(refresh_errors)
+ Array(refresh_errors).flat_map do |entry|
+ entry.to_h.flat_map do |label, messages|
+ next "#{label}: unknown error" if messages.blank?
+
+ Array(messages).map do |message|
+ detail = message.is_a?(Hash) ? message.to_h : message
+ "#{label}: #{detail.inspect}"
+ end
+ end
+ end.compact
+ end
+
+ def failure_details(reason, error)
+ {
+ source: "transition_check",
+ reason: reason,
+ error_class: error.class.name,
+ error_message: error.message
+ }
+ end
+
+ def scope_details(transition_check, representative_webpages)
+ publishable_event_page_count = transition_check.publishable_event_page_count.to_i
+ {
+ representative_webpages: representative_webpages.map(&:url),
+ representative_webpage_count: representative_webpages.count,
+ candidate_webpage_count: transition_check.candidate_webpage_count,
+ publishable_event_page_count: publishable_event_page_count,
+ selected_candidate_tier_count: transition_check.selected_candidate_tier_count,
+ selection_rule: transition_check.selection_rule,
+ sample_small: (publishable_event_page_count.positive? ? publishable_event_page_count : transition_check.candidate_webpage_count.to_i) > representative_webpages.count
+ }
+ end
+
+ def representative_url_result_for(webpage, comparison_policy)
+ cached = @representative_result_cache[webpage.url]
+ return cached.merge(url: webpage.url, webpage_id: webpage.id) if cached.present?
+
+ fetch_result = fetch_cache_store.fetch(
+ uri: webpage.url,
+ force_scrape: true,
+ mode: "internal",
+ website: website,
+ log_context: {
+ source: "transition_check",
+ website_id: website.id,
+ seedurl: website.seedurl
+ }
+ )
+ comparison = cache_compare.call(
+ uri: webpage.url,
+ condenser_result: fetch_result,
+ comparison_policy: comparison_policy
+ )
+
+ built = build_representative_url_result(webpage, fetch_result, comparison)
+ @representative_result_cache[webpage.url] = built
+ built
+ end
+
+ def build_representative_url_result(webpage, fetch_result, comparison)
+ fetch_status = fetch_result.transport_success? && fetch_result.content_success? ? "passed" : "failed"
+ {
+ url: webpage.url,
+ webpage_id: webpage.id,
+ cache: fetch_result.cache,
+ attempted_condenser_fetch: true,
+ condenser_fetch_success: fetch_status == "passed",
+ http_response_code: fetch_result.cache&.http_response_code,
+ cache_health_status: fetch_result.cache&.health_status,
+ stored_html: fetch_result.cache&.html.present?,
+ stored_body_bytes: fetch_result.cache&.body.to_s.bytesize,
+ fetch_status: fetch_status,
+ fetch_reason: fetch_status == "failed" ? fetch_failure_reason_from_result(fetch_result) : "ok",
+ comparison_performed: comparison.present?,
+ legacy_source: comparison&.dig(:legacy_source),
+ legacy_lookup_status: comparison&.dig(:legacy_lookup_status),
+ legacy_lookup_error: comparison&.dig(:legacy_lookup_error),
+ condenser_source: comparison&.dig(:condenser_source),
+ compare_missing: comparison&.dig(:missing),
+ compare_summary: comparison&.dig(:summary),
+ comparison_status: comparison_status(comparison)
+ }
+ end
+
+ def collect_representative_url_results(representative_webpages, comparison_policy)
+ collected = []
+
+ representative_webpages.each_with_index do |webpage, index|
+ if timeout_budget_exceeded?
+ return collected + representative_webpages.drop(index).map { |remaining| timed_out_representative_url_result(remaining) }
+ end
+
+ collected << representative_url_result_for(webpage, comparison_policy)
+ end
+
+ collected
+ end
+
+ def comparison_status(comparison)
+ return "not_performed" if comparison.blank?
+ return "missing" if comparison.dig(:missing, :legacy) || comparison.dig(:missing, :condenser)
+ return "failed" if comparison.dig(:summary, :blocking_regressions).present?
+ return "review" if comparison.dig(:summary, :review_needed_diffs).present?
+ return "unknown" if comparison.dig(:summary, :unknown_diffs).present?
+ return "metadata_only" if comparison.dig(:summary, :metadata_only_diffs).present?
+
+ "passed"
+ end
+
+ def serializable_representative_url_result(result)
+ result.except(:cache)
+ end
+
+ def representative_failed_layer(results)
+ return "fetch" if results.any? { |result| result[:fetch_status] == "failed" }
+ return "legacy_lookup" if results.any? { |result| %w[missing_config unreachable body_omitted].include?(result[:legacy_lookup_status]) }
+ return "cache_compare" if results.any? { |result| %w[failed review unknown missing].include?(result[:comparison_status]) }
+
+ nil
+ end
+
+ def representative_affected_url_count(results)
+ return results.count { |result| result[:fetch_status] == "failed" } if representative_failed_layer(results) == "fetch"
+ return results.count { |result| %w[missing_config unreachable body_omitted].include?(result[:legacy_lookup_status]) } if representative_failed_layer(results) == "legacy_lookup"
+ return results.count { |result| %w[failed review unknown missing].include?(result[:comparison_status]) } if representative_failed_layer(results) == "cache_compare"
+
+ 0
+ end
+
+ def representative_statement_results(representative_webpages, failed_fetch_results, refresh_errors, critical_failing_statements, optional_failing_statements)
+ critical_failing_ids = critical_failing_statements.map(&:id)
+ optional_failing_ids = optional_failing_statements.map(&:id)
+ rows = representative_webpages.map do |webpage|
+ webpage_statements = Statement.selected_for_transition.where(webpage_id: webpage.id).to_a
+ status =
+ if webpage_statements.empty?
+ "inconclusive"
+ elsif webpage_statements.any? { |statement| critical_failing_ids.include?(statement.id) }
+ "failed"
+ elsif refresh_errors.present? || webpage_statements.any? { |statement| optional_failing_ids.include?(statement.id) }
+ "warning"
+ else
+ "passed"
+ end
+
+ reason =
+ if webpage_statements.empty?
+ "no_selected_statements"
+ elsif status == "failed"
+ "critical_statement_refresh_failed"
+ elsif status == "warning"
+ "optional_statement_refresh_warning"
+ else
+ "ok"
+ end
+
+ {
+ url: webpage.url,
+ status: status,
+ reason: reason,
+ failing_statement_ids: webpage_statements.select { |statement| critical_failing_ids.include?(statement.id) || optional_failing_ids.include?(statement.id) }.map(&:id),
+ critical_failing_statement_ids: webpage_statements.select { |statement| critical_failing_ids.include?(statement.id) }.map(&:id),
+ optional_failing_statement_ids: webpage_statements.select { |statement| optional_failing_ids.include?(statement.id) }.map(&:id)
+ }
+ end
+ rows + failed_fetch_results.map do |result|
+ {
+ url: result[:url],
+ status: "blocked_by_fetch",
+ reason: result[:fetch_reason]
+ }
+ end
+ end
+
+ def representative_export_results(representative_webpages, failed_fetch_results, status, reason)
+ representative_webpages.map do |webpage|
+ { url: webpage.url, status: status, reason: reason }
+ end + failed_fetch_results.map do |result|
+ { url: result[:url], status: "blocked_by_fetch", reason: result[:fetch_reason] }
+ end
+ end
+
+ def partition_representative_webpages(representative_webpages, representative_url_results)
+ results_by_url = representative_url_results.index_by { |result| result[:url] }
+ fetched_webpages = representative_webpages.select { |webpage| results_by_url.fetch(webpage.url, {})[:fetch_status] != "failed" }
+ failed_fetch_results = representative_url_results.select { |result| result[:fetch_status] == "failed" }
+ [fetched_webpages, failed_fetch_results]
+ end
+
+ def pending_statement_timeout_record(latest_cache, transition_check, representative_webpages, fetched_webpages, failed_fetch_results)
+ Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: cache_or_seed_url(latest_cache),
+ check_kind: :statement_delta,
+ status: :pending,
+ statement_delta: 0,
+ statement_count_delta_acceptable: nil,
+ details: scope_details(transition_check, representative_webpages).merge(
+ source: "statement_refresh",
+ representative_url_statement_results: representative_statement_timeout_results(fetched_webpages, failed_fetch_results),
+ statements_refreshed_count: 0,
+ statements_failed_count: 0,
+ failing_statement_ids: [],
+ failing_statements: [],
+ reason: "transition_check_timeout_budget_exceeded"
+ )
+ )
+ end
+
+ def pending_export_timeout_record(latest_cache, transition_check, representative_webpages, fetched_webpages, failed_fetch_results)
+ Distillator::TransitionEvidenceRecorder.call(
+ website: website,
+ url: cache_or_seed_url(latest_cache),
+ check_kind: :export_diff,
+ status: :pending,
+ export_diff_checked: false,
+ export_diff_status: "pending",
+ details: scope_details(transition_check, representative_webpages).merge(
+ source: "export_comparison",
+ export_compared: false,
+ export_basis: "current export vs production-equivalent export",
+ representative_url_export_results: representative_export_timeout_results(fetched_webpages, failed_fetch_results),
+ reason: "transition_check_timeout_budget_exceeded"
+ )
+ )
+ end
+
+ def representative_statement_timeout_results(fetched_webpages, failed_fetch_results)
+ fetched_webpages.map do |webpage|
+ {
+ url: webpage.url,
+ status: "inconclusive",
+ reason: "transition_check_timeout_budget_exceeded"
+ }
+ end + failed_fetch_results.map do |result|
+ {
+ url: result[:url],
+ status: "blocked_by_fetch",
+ reason: result[:fetch_reason]
+ }
+ end
+ end
+
+ def representative_export_timeout_results(fetched_webpages, failed_fetch_results)
+ fetched_webpages.map do |webpage|
+ {
+ url: webpage.url,
+ status: "inconclusive",
+ reason: "transition_check_timeout_budget_exceeded"
+ }
+ end + failed_fetch_results.map do |result|
+ {
+ url: result[:url],
+ status: "blocked_by_fetch",
+ reason: result[:fetch_reason]
+ }
+ end
+ end
+
+ def timed_out_representative_url_result(webpage)
+ {
+ url: webpage.url,
+ webpage_id: webpage.id,
+ cache: nil,
+ attempted_condenser_fetch: false,
+ condenser_fetch_success: false,
+ http_response_code: nil,
+ cache_health_status: nil,
+ stored_html: false,
+ stored_body_bytes: 0,
+ fetch_status: "failed",
+ fetch_reason: "transition_check_timeout_budget_exceeded",
+ comparison_performed: false,
+ legacy_source: nil,
+ legacy_lookup_status: nil,
+ legacy_lookup_error: nil,
+ condenser_source: nil,
+ compare_missing: nil,
+ compare_summary: nil,
+ comparison_status: "not_performed"
+ }
+ end
+
+ def timeout_budget_exceeded?
+ return false unless budget_seconds.present?
+
+ elapsed = monotonic_now - @started_at
+ elapsed >= [budget_seconds.to_f - timeout_guard_seconds.to_f, 0].max
+ end
+
+ def monotonic_now
+ @clock.call.to_f
+ end
+
+ def captcha_result?(result)
+ issue_key = result.blocking_issue_key.presence || cache_signal(result.cache, :primary_issue_key).presence
+ return true if issue_key.to_s == "system_captcha"
+
+ Array(result.respond_to?(:hints) ? result.hints : nil).map(&:to_s).any? { |hint| hint.include?("captcha") }
+ end
+
+ def missing_renderer_reason(result)
+ return nil unless truthy?(cache_signal(result.cache, :renderer_unavailable))
+
+ if truthy?(cache_signal(result.cache, :use_phantomjs)) && ENV["PHANTOMJS_API_KEY"].blank?
+ "phantomjs_api_key_missing"
+ else
+ "renderer_unavailable"
+ end
+ end
+ end
+end
diff --git a/app/services/distillator/transition_evidence_explanation.rb b/app/services/distillator/transition_evidence_explanation.rb
new file mode 100644
index 00000000..e465a29f
--- /dev/null
+++ b/app/services/distillator/transition_evidence_explanation.rb
@@ -0,0 +1,381 @@
+module Distillator
+ class TransitionEvidenceExplanation
+ CHECK_LABELS = {
+ "fetch_parity" => "Fetch parity",
+ "statement_delta" => "Statements",
+ "export_diff" => "Export"
+ }.freeze
+ DEFAULT_SELECTION_RULE = Distillator::TransitionCheck::SELECTION_RULE
+
+ Result = Struct.new(
+ :key,
+ :check,
+ :state,
+ :severity,
+ :headline,
+ :details,
+ :next_action,
+ :links,
+ keyword_init: true
+ )
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(check_kind:, evidence:, website:, state:)
+ @check_kind = check_kind.to_s
+ @evidence = evidence
+ @website = website
+ @state = state.to_sym
+ end
+
+ def call
+ Result.new(
+ key: check_kind,
+ check: CHECK_LABELS.fetch(check_kind, check_kind.humanize),
+ state: state.to_s,
+ severity: severity,
+ headline: headline,
+ details: explanation_details,
+ next_action: next_action,
+ links: links
+ )
+ end
+
+ private
+
+ attr_reader :check_kind, :evidence, :website, :state
+
+ def severity
+ return "warning" if check_kind == "fetch_parity" && %w[legacy_lookup_missing_config legacy_lookup_unreachable legacy_lookup_body_omitted review_needed_difference].include?(reason)
+
+ case state
+ when :warning
+ "warning"
+ when :failed, :blocked_by_fetch, :not_evaluated
+ "blocker"
+ when :stale, :missing, :inconclusive
+ website.lavitrine_pipeline? ? "blocker" : "warning"
+ else
+ "ok"
+ end
+ end
+
+ def headline
+ case check_kind
+ when "fetch_parity"
+ fetch_headline
+ when "statement_delta"
+ statement_headline
+ when "export_diff"
+ export_headline
+ else
+ "#{check_kind.humanize} is #{state}."
+ end
+ end
+
+ def fetch_headline
+ return "No representative event webpages were available." if reason == "no_representative_webpages"
+ return "Latest Condenser attempt failed: empty body." if state == :failed && reason == "empty_body"
+ return "Transition check reached its runtime budget before all sampled URLs were fetched." if reason == "transition_check_timeout_budget_exceeded"
+ return "Captcha was detected while fetching one or more sampled URLs." if reason == "captcha_detected"
+ return "Rendered fetch is unavailable because PHANTOMJS_API_KEY is missing." if reason == "phantomjs_api_key_missing"
+ return "Rendered fetch is unavailable for one or more sampled URLs." if reason == "renderer_unavailable"
+ return "Condenser fetch passed, but legacy Wringer lookup is missing staging configuration." if reason == "legacy_lookup_missing_config"
+ return "Condenser fetch passed, but legacy Wringer lookup failed." if reason == "legacy_lookup_unreachable"
+ return "Condenser fetch passed, but legacy Wringer body was omitted from the comparison endpoint." if reason == "legacy_lookup_body_omitted"
+ return "Condenser and Wringer differ in fields that need manual review." if reason == "review_needed_difference"
+ return "Condenser and Wringer differ only in metadata fields." if reason == "metadata_only_difference"
+ return "Fresh Condenser evidence is missing for this comparison." if reason == "cache_compare_missing"
+ return "Condenser and Wringer have a blocking parity mismatch." if reason == "cache_compare_blocking_regression"
+ return "Condenser and Wringer comparison is still inconclusive." if reason == "cache_compare_unknown"
+ case state
+ when :passed
+ "Fetch parity passed."
+ when :failed
+ failed_layer == "cache_compare" ? "Condenser and Wringer comparison failed." : "Condenser fetch/cache failed for one or more sampled URLs."
+ when :stale
+ "Fetch parity is stale."
+ else
+ "Fetch parity has not been recorded yet."
+ end
+ end
+
+ def statement_headline
+ return "No representative event webpages were available." if reason == "no_representative_webpages"
+ return "Fetch failed before statements could be refreshed." if state == :not_evaluated || reason == "fetch_failed_before_statement_refresh"
+ return "Some representative URLs could not be fetched, so statement coverage is incomplete." if reason == "partial_fetch_failed_before_statement_refresh"
+ return "Transition check reached its runtime budget before statement coverage completed." if reason == "transition_check_timeout_budget_exceeded"
+ return "No selected statements were found for the representative webpages." if state == :inconclusive || reason == "no_selected_statements"
+ return "Legacy statement check failed before critical/optional classification was available." if legacy_generic_statement_failure?
+ return "Critical statements passed; optional statement refresh warnings need review." if state == :warning || reason == "optional_statement_refresh_warning"
+ return "Critical statement refresh failed for #{pluralize(critical_failing_statement_count, 'statement')}." if reason == "critical_statement_refresh_failed" && critical_failing_statement_count.positive?
+ return "Critical statement refresh failed for #{pluralize(critical_failing_statement_count, 'statement')}; optional statement warnings also need review." if reason == "critical_and_optional_statement_refresh_failed" && critical_failing_statement_count.positive?
+ return "Statement refresh found #{pluralize(failing_statement_count, 'failing statement')}." if state == :failed
+ return "Statements check is stale." if state == :stale
+ return "Statement check not yet recorded." if state == :missing
+
+ "Statements check passed."
+ end
+
+ def export_headline
+ return "Fetch failed before export could be compared." if state == :blocked_by_fetch || reason == "fetch_failed_before_export_comparison"
+ return "Some representative URLs could not be fetched, so export coverage is incomplete." if reason == "partial_fetch_failed_before_export_comparison"
+ return "Transition check reached its runtime budget before export coverage completed." if reason == "transition_check_timeout_budget_exceeded"
+ return "Export could not be generated." if reason == "export_generation_failed"
+ return "Export comparison is not available yet." if reason == "export_diff_not_available"
+ return "No representative event webpages were available." if reason == "no_representative_webpages"
+ return "Export comparison found #{pluralize(rdf_added_count, 'RDF statement')} added and #{pluralize(rdf_removed_count, 'RDF statement')} removed." if state == :failed
+ return "Export comparison is stale." if state == :stale
+ return "Export check not yet recorded." if state == :missing
+
+ "Export comparison passed."
+ end
+
+ def explanation_details
+ case check_kind
+ when "statement_delta"
+ statement_details
+ when "export_diff"
+ export_details
+ else
+ fetch_details
+ end
+ end
+
+ def fetch_details
+ details = []
+ details << "URL: #{evidence.url}" if evidence&.url.present?
+ details << "Failed layer: #{failed_layer.humanize}" if failed_layer.present?
+ details << "Affected sampled URLs: #{affected_url_count}" if affected_url_count.positive?
+ details << "Attempted Condenser fetch: #{evidence&.attempted_condenser_fetch? ? 'yes' : 'no'}" if evidence.present?
+ if details_hash.key?("condenser_fetch_success")
+ details << "Condenser fetch: #{details_hash['condenser_fetch_success'] ? 'passed' : 'failed'}"
+ end
+ details << "Issue: #{evidence.primary_issue_key}" if evidence&.primary_issue_key.present?
+ details << "Legacy source: #{details_hash['legacy_source']}" if details_hash["legacy_source"].present?
+ details << "Legacy lookup status: #{details_hash['legacy_lookup_status']}" if details_hash["legacy_lookup_status"].present?
+ details << "Legacy lookup error: #{details_hash['legacy_lookup_error']}" if details_hash["legacy_lookup_error"].present?
+ details << "Condenser source: #{details_hash['condenser_source']}" if details_hash["condenser_source"].present?
+ details << "Comparison policy: #{details_hash['comparison_policy']}" if details_hash["comparison_policy"].present?
+ details << "Compare performed: #{details_hash["comparison_performed"] ? 'yes' : 'no'}" if details_hash.key?("comparison_performed")
+ details << "Reason: #{reason.humanize}" if reason.present?
+ details
+ end
+
+ def statement_details
+ details = []
+ if legacy_generic_statement_failure?
+ details << "Legacy statement failures recorded: #{failing_statement_count}" if failing_statement_count.positive?
+ details << "Critical/optional split: not recorded"
+ else
+ details << "Critical statement failures: #{critical_failing_statement_count}" if critical_failing_statement_count.positive?
+ details << "Optional statement warnings: #{optional_failing_statement_count}" if optional_failing_statement_count.positive?
+ end
+ representative_webpages.each do |url|
+ details << "Webpage: #{url}"
+ end
+
+ critical_failing_statements.each do |statement|
+ details << "Critical statement ID: #{statement[:id]}"
+ details << "Critical source: #{statement[:source]}" if statement[:source].present?
+ details << "Critical webpage: #{statement[:webpage_url]}" if statement[:webpage_url].present? && !representative_webpages.include?(statement[:webpage_url])
+ end
+
+ optional_failing_statements.each do |statement|
+ details << "Statement ID: #{statement[:id]}"
+ details << "Source: #{statement[:source]}" if statement[:source].present?
+ details << "Webpage: #{statement[:webpage_url]}" if statement[:webpage_url].present? && !representative_webpages.include?(statement[:webpage_url])
+ end
+
+ refresh_errors.each do |error|
+ details << "Reason: #{error}"
+ end
+
+ if details.empty? && evidence&.statement_delta.present?
+ details << "Statement delta: #{evidence.statement_delta}"
+ end
+ details
+ end
+
+ def export_details
+ details = []
+ details << "URL: #{evidence.url}" if evidence&.url.present?
+ details << "RDF added: #{rdf_added_count}" unless rdf_added_count.nil?
+ details << "RDF removed: #{rdf_removed_count}" unless rdf_removed_count.nil?
+ details << "Reason: #{reason.humanize}" if reason.present?
+ details
+ end
+
+ def next_action
+ case check_kind
+ when "fetch_parity"
+ return "Increase runtime coverage or rerun the transition batch check outside the web request budget." if reason == "transition_check_timeout_budget_exceeded"
+ return "Resolve the captcha or use the direct inspection links for the affected URLs, then rerun the transition batch check." if reason == "captcha_detected"
+ return "Configure PHANTOMJS_API_KEY or switch away from rendered fetch for the affected URLs, then rerun the transition batch check." if reason == "phantomjs_api_key_missing"
+ return "Fix rendered fetch availability for the affected URLs, then rerun the transition batch check." if reason == "renderer_unavailable"
+ return "Configure the Wringer endpoint for staging, then rerun the transition batch check." if reason == "legacy_lookup_missing_config"
+ return "Fix the legacy Wringer endpoint, then rerun the transition batch check." if reason == "legacy_lookup_unreachable"
+ return "Verify the legacy Wringer body endpoint or compare using the legacy inspection link." if reason == "legacy_lookup_body_omitted"
+ return "Review the Condenser vs Wringer comparison for the affected URLs." if reason == "cache_compare_blocking_regression"
+ return "Re-run the cache comparison and verify the affected URLs." if %w[cache_compare_missing cache_compare_unknown].include?(reason)
+
+ state == :passed ? "No fetch action is needed right now." : "Fix the fetch/cache failure first, then rerun the transition batch check."
+ when "statement_delta"
+ if state == :passed
+ "No statement refresh action is needed right now."
+ elsif state == :not_evaluated || reason == "fetch_failed_before_statement_refresh"
+ "Fix the fetch/cache failure first, then rerun the transition batch check."
+ elsif reason == "transition_check_timeout_budget_exceeded"
+ "Rerun the transition batch check with enough runtime budget to finish statement coverage."
+ elsif reason == "partial_fetch_failed_before_statement_refresh"
+ "Fetch the missing representative URLs, then rerun the transition batch check to complete statement coverage."
+ elsif state == :inconclusive || reason == "no_selected_statements"
+ "Verify selected sources/statements for the sampled webpages."
+ elsif legacy_generic_statement_failure?
+ "Rerun the transition batch check to record current statement evidence with critical/optional classification."
+ elsif state == :warning || reason == "optional_statement_refresh_warning"
+ "Review the optional statement refresh warnings before activating."
+ elsif reason == "no_representative_webpages"
+ "Confirm this website has representative event webpages before activating."
+ else
+ "Open the statement trace and fix the source before activating."
+ end
+ when "export_diff"
+ if state == :passed
+ "No export action is needed right now."
+ elsif state == :blocked_by_fetch || reason == "fetch_failed_before_export_comparison"
+ "Fix the fetch/cache failure first, then rerun the transition batch check."
+ elsif reason == "transition_check_timeout_budget_exceeded"
+ "Rerun the transition batch check with enough runtime budget to finish export coverage."
+ elsif reason == "partial_fetch_failed_before_export_comparison"
+ "Fetch the missing representative URLs, then rerun the transition batch check to complete export coverage."
+ elsif reason == "export_generation_failed"
+ "Fix the export generation failure, then rerun the transition batch check."
+ else
+ "Review the export comparison and rerun the transition batch check before activating."
+ end
+ else
+ "Review this check before activating."
+ end
+ end
+
+ def links
+ case check_kind
+ when "statement_delta"
+ [{ label: "Statements", target: :statements }]
+ when "export_diff"
+ [{ label: "Transition report", target: :transition_report }]
+ else
+ [
+ { label: "Open failed cache result", target: :failed_cache_result },
+ { label: "Compare Condenser vs Wringer", target: :compare_cache },
+ { label: "Compare extracted statements", target: :compare_statements },
+ { label: "Open active Wringer cache", target: :active_wringer_cache }
+ ]
+ end
+ end
+
+ def details_hash
+ @details_hash ||= evidence&.details.to_h || {}
+ end
+
+ def reason
+ details_hash["reason"] || details_hash[:reason]
+ end
+
+ def representative_webpages
+ Array(details_hash["representative_webpages"] || details_hash[:representative_webpages]).compact
+ end
+
+ def count_representative_webpages
+ details_hash["representative_webpage_count"] || details_hash[:representative_webpage_count] || representative_webpages.count
+ end
+
+ def failing_statements
+ Array(details_hash["failing_statements"] || details_hash[:failing_statements]).map do |entry|
+ entry.respond_to?(:to_h) ? entry.to_h.symbolize_keys : {}
+ end
+ end
+
+ def critical_failing_statements
+ Array(details_hash["critical_failing_statements"] || details_hash[:critical_failing_statements]).map do |entry|
+ entry.respond_to?(:to_h) ? entry.to_h.symbolize_keys : {}
+ end
+ end
+
+ def optional_failing_statements
+ explicit = Array(details_hash["optional_failing_statements"] || details_hash[:optional_failing_statements]).map do |entry|
+ entry.respond_to?(:to_h) ? entry.to_h.symbolize_keys : {}
+ end
+ return explicit if explicit.any?
+
+ failing_statements.reject { |entry| entry[:severity].to_s == "blocker" }
+ end
+
+ def refresh_errors
+ Array(details_hash["refresh_errors"] || details_hash[:refresh_errors]).map(&:to_s)
+ end
+
+ def failing_statement_count
+ explicit_count = details_hash["statements_failed_count"] || details_hash[:statements_failed_count]
+ return explicit_count.to_i if explicit_count.present?
+ return failing_statements.count if failing_statements.any?
+
+ evidence&.statement_delta.to_i
+ end
+
+ def critical_failing_statement_count
+ explicit_count = details_hash["critical_statements_failed_count"] || details_hash[:critical_statements_failed_count]
+ return explicit_count.to_i if explicit_count.present?
+ return critical_failing_statements.count if critical_failing_statements.any?
+
+ evidence&.statement_delta.to_i
+ end
+
+ def optional_failing_statement_count
+ explicit_count = details_hash["optional_statements_failed_count"] || details_hash[:optional_statements_failed_count]
+ return explicit_count.to_i if explicit_count.present?
+
+ optional_failing_statements.count
+ end
+
+ def rdf_added_count
+ evidence&.rdf_added_count
+ end
+
+ def rdf_removed_count
+ evidence&.rdf_removed_count
+ end
+
+ def failed_layer
+ details_hash["failed_layer"] || details_hash[:failed_layer]
+ end
+
+ def affected_url_count
+ (details_hash["affected_url_count"] || details_hash[:affected_url_count] || 0).to_i
+ end
+
+ def pluralize(count, noun)
+ amount = count.to_i
+ return "0 #{noun}s" if amount.zero?
+ return "1 #{noun}" if amount == 1
+
+ "#{amount} #{noun}s"
+ end
+
+ def legacy_generic_statement_failure?
+ return false unless state == :failed && failing_statement_count.positive?
+
+ !statement_failure_split_recorded?
+ end
+
+ def statement_failure_split_recorded?
+ details_hash.key?("critical_statements_failed_count") ||
+ details_hash.key?(:critical_statements_failed_count) ||
+ details_hash.key?("optional_statements_failed_count") ||
+ details_hash.key?(:optional_statements_failed_count)
+ end
+ end
+end
diff --git a/app/services/distillator/transition_evidence_recorder.rb b/app/services/distillator/transition_evidence_recorder.rb
new file mode 100644
index 00000000..2ceb0f3f
--- /dev/null
+++ b/app/services/distillator/transition_evidence_recorder.rb
@@ -0,0 +1,60 @@
+module Distillator
+ class TransitionEvidenceRecorder
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(website:, url:, check_kind:, status:, checked_at: Time.current, cohort_key: nil, details: {}, **attributes)
+ @website = website
+ @url = url
+ @check_kind = check_kind
+ @status = status
+ @checked_at = checked_at
+ @cohort_key = cohort_key
+ @details = details
+ @attributes = attributes
+ end
+
+ def call
+ record = existing_record || Distillator::TransitionEvidence.new
+ record.assign_attributes(payload)
+ record.save!
+ record
+ end
+
+ private
+
+ attr_reader :website, :url, :check_kind, :status, :checked_at, :cohort_key, :details, :attributes
+
+ def payload
+ {
+ website: website,
+ url: normalized_url,
+ cohort_key: cohort_key.presence || inferred_cohort_key,
+ check_kind: check_kind.to_s,
+ status: status.to_s,
+ checked_at: checked_at,
+ details: details.to_h
+ }.merge(attributes.compact)
+ end
+
+ def normalized_url
+ Distillator::WringerUrlKey.call(url).normalized_url
+ rescue StandardError
+ url.to_s
+ end
+
+ def inferred_cohort_key
+ return unless website.respond_to?(:distillator_primary_cohort_key)
+
+ website.distillator_primary_cohort_key
+ end
+
+ def existing_record
+ Distillator::TransitionEvidence
+ .where(website: website, url: normalized_url, check_kind: check_kind.to_s)
+ .order(checked_at: :desc, id: :desc)
+ .first
+ end
+ end
+end
diff --git a/app/services/distillator/transition_runtime.rb b/app/services/distillator/transition_runtime.rb
new file mode 100644
index 00000000..0d27cceb
--- /dev/null
+++ b/app/services/distillator/transition_runtime.rb
@@ -0,0 +1,49 @@
+module Distillator
+ class TransitionRuntime
+ APP_NAME_KEYS = %w[HEROKU_APP_NAME APP_NAME HEROKU_PARENT_APP_NAME].freeze
+ STAGING_RUNTIME_VALUES = %w[staging].freeze
+ STAGING_ENV_VALUES = %w[staging].freeze
+ STAGING_ALLOWED_ROLLOUT_MODES = %w[shadow active].freeze
+
+ STAGING_APP_NAMES = %w[
+ footlight-condenser-test-c24c162bb7c8
+ ].freeze
+
+ def self.staging?
+ explicit_staging_runtime? || explicit_staging_environment? || known_staging_app?
+ end
+
+ def self.allow_active_override?
+ explicit_override_enabled? || Rails.env.test? || staging?
+ end
+
+ def self.known_staging_app?
+ app_names.any? { |name| STAGING_APP_NAMES.include?(name.to_s) }
+ end
+
+ def self.staging_invalid_rollout_mode_scope(scope = Website.all)
+ scope.where.not(distillator_mode: STAGING_ALLOWED_ROLLOUT_MODES)
+ .or(scope.where(distillator_mode: [nil, ""]))
+ end
+
+ def self.staging_rollout_mode_invalid?(mode)
+ !STAGING_ALLOWED_ROLLOUT_MODES.include?(mode.to_s)
+ end
+
+ def self.app_names
+ APP_NAME_KEYS.filter_map { |key| ENV[key].presence }
+ end
+
+ def self.explicit_override_enabled?
+ ENV["DISTILLATOR_ALLOW_ACTIVE_OVERRIDE"].to_s == "true"
+ end
+
+ def self.explicit_staging_runtime?
+ STAGING_RUNTIME_VALUES.include?(ENV["DISTILLATOR_RUNTIME"].to_s)
+ end
+
+ def self.explicit_staging_environment?
+ STAGING_ENV_VALUES.include?(ENV["RAILS_ENV"].to_s)
+ end
+ end
+end
diff --git a/app/services/distillator/transition_status.rb b/app/services/distillator/transition_status.rb
new file mode 100644
index 00000000..0ad280fd
--- /dev/null
+++ b/app/services/distillator/transition_status.rb
@@ -0,0 +1,565 @@
+module Distillator
+ class TransitionStatus
+ CHECK_RESULTS = %i[passed warning failed missing stale not_evaluated blocked_by_fetch inconclusive].freeze
+ STATUSES = %i[ready review blocked not_checked].freeze
+ SAFETY_LEVELS = %i[safe review unsafe unknown].freeze
+ CONFIDENCE_LEVELS = %i[high medium low not_checked].freeze
+
+ Result = Struct.new(
+ :status,
+ :safety,
+ :confidence,
+ :fetch,
+ :statements,
+ :export,
+ :checks,
+ :activation_recommendation,
+ :evidence_statuses,
+ :blockers,
+ :warnings,
+ :last_checked,
+ :readiness_label,
+ :severity,
+ :primary_blocker,
+ :primary_action,
+ :review_activation_eligible,
+ :manual_review_required,
+ :review_needed_fields,
+ :comparison_policy,
+ keyword_init: true
+ )
+
+ READINESS_LABELS = {
+ blocked: "Blocked",
+ review: "Needs review",
+ ready: "Ready",
+ not_checked: "Not checked"
+ }.freeze
+
+ STATUS_SEVERITIES = {
+ blocked: "high",
+ review: "medium",
+ ready: "ok",
+ not_checked: "unknown"
+ }.freeze
+
+ def self.call(...)
+ new(...).call
+ end
+
+ def initialize(website:, cache: nil, evidence_by_kind: nil, now: Time.current)
+ @website = website
+ @cache = cache
+ @evidence_by_kind = evidence_by_kind
+ @now = now
+ end
+
+ def call
+ Result.new(
+ status: overall_status,
+ safety: safety,
+ confidence: confidence,
+ fetch: fetch_status,
+ statements: statements_status,
+ export: export_status,
+ checks: checks,
+ activation_recommendation: activation_recommendation,
+ evidence_statuses: evidence_statuses,
+ blockers: blockers,
+ warnings: warnings,
+ last_checked: last_checked,
+ readiness_label: readiness_label,
+ severity: severity,
+ primary_blocker: primary_blocker,
+ primary_action: primary_action,
+ review_activation_eligible: review_activation_eligible?,
+ manual_review_required: manual_review_required?,
+ review_needed_fields: review_needed_fields,
+ comparison_policy: comparison_policy
+ )
+ end
+
+ def readiness_label
+ READINESS_LABELS.fetch(overall_status, READINESS_LABELS.fetch(:not_checked))
+ end
+
+ def severity
+ STATUS_SEVERITIES.fetch(overall_status, STATUS_SEVERITIES.fetch(:not_checked))
+ end
+
+ def primary_blocker
+ primary_blocker_reason.presence || blockers.first || warnings.first
+ end
+
+ def primary_action
+ activation_next_action
+ end
+
+ private
+
+ attr_reader :website, :cache, :now
+
+ def overall_status
+ case safety
+ when :unsafe
+ :blocked
+ when :review
+ :review
+ when :safe
+ :ready
+ else
+ :not_checked
+ end
+ end
+
+ def safety
+ return :unknown unless cache.present? || any_evidence_present?
+ return :unsafe if blockers.any?
+ return :review if warnings.any?
+
+ :safe
+ end
+
+ def confidence
+ return :not_checked unless cache.present? || any_evidence_present?
+ return :low if review_needed_difference? || legacy_lookup_missing_config? || legacy_lookup_unreachable? || legacy_lookup_body_omitted?
+ return :low if cache_compare_unknown?
+ return :low if [fetch_status, statements_status, export_status].any? { |status| %i[missing stale inconclusive not_evaluated blocked_by_fetch].include?(status) }
+ return :medium if statements_status == :warning
+ return :medium if metadata_only_difference?
+
+ :high
+ end
+
+ def blockers
+ reasons = []
+ reasons << fetch_failure_blocker if fetch_status == :failed
+ reasons << "Cannot activate yet: fetch check is stale." if fetch_status == :stale && lavitrine_pipeline?
+ reasons << "Cannot activate yet: statements check failed." if statements_status == :failed
+ reasons << "Cannot activate yet: statements could not be evaluated until fetch/cache is fixed." if statements_status == :not_evaluated && fetch_status != :failed
+ reasons << "Cannot activate yet: statements check is inconclusive." if statements_status == :inconclusive
+ reasons << "Cannot activate yet: statements check is missing." if lavitrine_pipeline? && statements_status == :missing
+ reasons << "Cannot activate yet: statements check is stale." if lavitrine_pipeline? && statements_status == :stale
+ reasons << "Cannot activate yet: export check failed." if export_status == :failed
+ reasons << "Cannot activate yet: export comparison could not be evaluated until fetch/cache is fixed." if export_status == :blocked_by_fetch && fetch_status != :failed
+ reasons << "Cannot activate yet: export check is inconclusive." if export_status == :inconclusive
+ reasons << "Cannot activate yet: export check is missing." if lavitrine_pipeline? && export_status == :missing
+ reasons << "Cannot activate yet: export check is stale." if lavitrine_pipeline? && export_status == :stale
+ reasons.uniq
+ end
+
+ def warnings
+ reasons = []
+ reasons << legacy_lookup_warning if legacy_lookup_warning.present?
+ reasons << "Needs review: Condenser and Wringer differ in fields that need manual verification." if review_needed_difference?
+ reasons << "Needs review: Condenser and Wringer comparison is still inconclusive." if cache_compare_unknown?
+ reasons << "Needs review: fetch check is stale." if fetch_status == :stale && !lavitrine_pipeline?
+ reasons << "Needs review: statements check is missing." if !lavitrine_pipeline? && statements_status == :missing
+ reasons << "Needs review: statements check is stale." if !lavitrine_pipeline? && statements_status == :stale
+ reasons << "Needs review: statements check is inconclusive." if !lavitrine_pipeline? && statements_status == :inconclusive
+ reasons << optional_statement_warning if optional_statement_warning.present?
+ reasons << "Needs review: export check is missing." if !lavitrine_pipeline? && export_status == :missing
+ reasons << "Needs review: export check is stale." if !lavitrine_pipeline? && export_status == :stale
+ reasons << "Needs review: export check is inconclusive." if !lavitrine_pipeline? && export_status == :inconclusive
+ reasons << "Needs review: export check is stale." if export_status == :stale && export_diff_evidence&.export_diff_accepted?
+ reasons << "Needs review: fetch result redirected." if redirect_changed?
+ reasons << "Needs review: latest successful refresh is stale." if stale_successful_refresh?
+ reasons.uniq
+ end
+
+ def fetch_status
+ evidence = fetch_parity_evidence
+ return fetch_status_from_evidence(evidence) if evidence.present?
+ return :missing unless cache.present?
+ return :failed if fetch_failed?
+ return :stale if fetch_evidence_stale?
+
+ :passed
+ end
+
+ def statements_status
+ evidence = statement_delta_evidence
+ return :missing if lavitrine_pipeline? && !evidence.present?
+ return signal_status("statement_count_delta_acceptable") unless evidence.present?
+ return :not_evaluated if fetch_prevented_statement_refresh?(evidence)
+ return :inconclusive if partial_fetch_prevented_statement_refresh?(evidence)
+ return :inconclusive if no_selected_statements?(evidence)
+ return :missing if evidence.status.to_s == "pending"
+ return :warning if evidence.status.to_s == "warning" || optional_statement_warning_evidence?(evidence)
+ return :failed if evidence.status.to_s.in?(%w[failed blocked rejected]) || evidence.acceptable_statement_delta? == false
+ return :stale if evidence.checked_at < now - evidence_stale_after
+ return :passed if evidence.acceptable_statement_delta?
+
+ :missing
+ end
+
+ def export_status
+ evidence = export_diff_evidence
+ return :missing if lavitrine_pipeline? && !evidence.present?
+ return export_status_from_cache unless evidence.present?
+ return :blocked_by_fetch if fetch_prevented_export_comparison?(evidence)
+ return :inconclusive if partial_fetch_prevented_export_comparison?(evidence)
+ return :inconclusive if export_diff_not_available?(evidence)
+ return :missing if evidence.status.to_s == "pending"
+ return :failed if evidence.status.to_s.in?(%w[failed blocked rejected]) || explicit_false?(evidence.export_diff_checked)
+ return :stale if export_evidence_stale?(evidence)
+ return :passed if evidence.export_diff_satisfied?
+
+ :missing
+ end
+
+ def signal_status(signal_key)
+ value = signal(signal_key)
+ return :missing if value.nil? || value.to_s == ""
+ return :failed if explicit_false?(value)
+
+ :passed
+ end
+
+ def fetch_status_from_evidence(evidence)
+ return :missing if evidence.status.to_s == "pending"
+ return :failed if evidence.status.to_s.in?(%w[failed blocked rejected])
+ return :stale if evidence.checked_at < now - evidence_stale_after
+ return :passed if evidence.status.to_s.in?(%w[checked accepted warning])
+
+ :missing
+ end
+
+ def export_status_from_cache
+ return :missing unless cache.present?
+
+ status = signal("export_diff_status").to_s
+ checked = signal("export_diff_checked")
+ accepted = signal("export_diff_accepted")
+ return :failed if %w[failed blocked rejected].include?(status)
+ return :passed if truthy?(checked) || truthy?(accepted) || %w[checked accepted].include?(status)
+
+ :missing
+ end
+
+ def fetch_failed?
+ latest_attempt_failed? || transport_failed? || content_failed? || blocking_issue? || last_good_preserved_failure?
+ end
+
+ def latest_attempt_failed?
+ %w[blocked network_failed attempt_failed content_rejected empty_body preserved_after_failure].include?(cache.health_status.to_s)
+ end
+
+ def transport_failed?
+ explicit_false?(signal("transport_success")) || cache.network_status.to_s == "failed"
+ end
+
+ def content_failed?
+ explicit_false?(signal("content_success"))
+ end
+
+ def blocking_issue?
+ cache.primary_issue_severity.to_s.in?(%w[blocked failed])
+ end
+
+ def last_good_preserved_failure?
+ truthy?(signal("last_good_preserved_failure"))
+ end
+
+ def fetch_evidence_stale?
+ return false unless cache&.successful_refresh.present?
+
+ cache.successful_refresh < now - evidence_stale_after
+ end
+
+ def stale_successful_refresh?
+ fetch_evidence_stale?
+ end
+
+ def redirect_changed?
+ return false unless cache.present?
+
+ cache.redirected == true ||
+ (cache.final_url.present? && cache.final_url.to_s != cache.normalized_url.to_s)
+ end
+
+ def export_evidence_stale?(evidence)
+ threshold = evidence.export_diff_accepted? ? Distillator::PromotionReadiness::EXPORT_DIFF_STALE_AFTER : evidence_stale_after
+ evidence.checked_at < now - threshold
+ end
+
+ def evidence_stale_after
+ lavitrine_pipeline? ? Distillator::PromotionReadiness::LAVITRINE_EVIDENCE_STALE_AFTER : Distillator::PromotionReadiness::EVIDENCE_STALE_AFTER
+ end
+
+ def last_checked
+ times = [cache&.scrape_date, cache&.successful_refresh]
+ times.concat(evidence_by_kind.values.compact.map(&:checked_at))
+ times.compact.max
+ end
+
+ def any_evidence_present?
+ evidence_by_kind.values.any?(&:present?)
+ end
+
+ def evidence_statuses
+ {
+ "fetch_parity" => transition_evidence_status(fetch_parity_evidence),
+ "statement_delta" => transition_evidence_status(statement_delta_evidence),
+ "export_diff" => transition_evidence_status(export_diff_evidence)
+ }
+ end
+
+ def checks
+ checks = [
+ {
+ key: "statement_equivalence",
+ label: "Statement equivalence",
+ state: statements_status,
+ role: "gate"
+ }
+ ]
+
+ if statements_status == :passed
+ checks << {
+ key: "export_confirmation",
+ label: "Export confirmation",
+ state: export_status,
+ role: "confirmation"
+ }
+ elsif fetch_status != :missing
+ checks << {
+ key: "fetch_diagnostic",
+ label: "Fetch diagnostic",
+ state: fetch_status,
+ role: "diagnostic"
+ }
+ end
+
+ checks
+ end
+
+ def activation_recommendation
+ {
+ label: activation_label,
+ reason: activation_reason,
+ next_action: activation_next_action
+ }
+ end
+
+ def activation_label
+ readiness_label
+ end
+
+ def activation_reason
+ return primary_blocker if primary_blocker.present?
+ return "All sampled transition checks are currently passing." if overall_status == :ready
+
+ "Run the transition batch check to record current sampled evidence."
+ end
+
+ def activation_next_action
+ return fetch_failure_next_action if fetch_status == :failed
+ return "Configure the Wringer endpoint for staging, then rerun the transition batch check." if legacy_lookup_missing_config?
+ return "Fix the legacy Wringer endpoint, then rerun the transition batch check." if legacy_lookup_unreachable?
+ return "Verify the legacy Wringer body endpoint or compare using the legacy inspection link." if legacy_lookup_body_omitted?
+ return "Review the checklist, then activate with a recorded reason." if review_activation_eligible?
+ return "Metadata notes only. Promote to active when you are satisfied with the evidence." if metadata_only_difference?
+ return "Review the unknown comparison results, then rerun the transition batch check if needed." if cache_compare_unknown?
+ return "Review the parity differences, then rerun the transition batch check if needed." if review_needed_difference?
+ return "Review the optional statement refresh warnings before activating." if statements_status == :warning
+ return "Verify selected sources/statements for the sampled webpages." if statements_status == :inconclusive
+ return "Fix the blocking check, then rerun the transition batch check." if blockers.any?
+ return "Review the warning and rerun the transition batch check if needed." if warnings.any?
+ return "Promote to active when you are satisfied with the evidence." if overall_status == :ready
+
+ "Run the transition batch check to record current sampled evidence."
+ end
+
+ def transition_evidence_status(evidence)
+ return :missing unless evidence.present?
+ return :not_evaluated if fetch_prevented_statement_refresh?(evidence)
+ return :blocked_by_fetch if fetch_prevented_export_comparison?(evidence)
+ return :inconclusive if partial_fetch_prevented_statement_refresh?(evidence) || partial_fetch_prevented_export_comparison?(evidence)
+ return :inconclusive if no_selected_statements?(evidence) || export_diff_not_available?(evidence)
+ return :missing if evidence.status.to_s == "pending"
+ return :warning if evidence.status.to_s == "warning"
+ return :failed if evidence.status.to_s.in?(%w[failed blocked rejected])
+ return :stale if evidence.checked_at < now - evidence_stale_after
+
+ :checked
+ end
+
+ def evidence_by_kind
+ @evidence_by_kind ||= website.respond_to?(:latest_transition_evidences_by_kind) ? website.latest_transition_evidences_by_kind : {}
+ end
+
+ def fetch_parity_evidence
+ evidence_by_kind["fetch_parity"]
+ end
+
+ def statement_delta_evidence
+ evidence_by_kind["statement_delta"]
+ end
+
+ def export_diff_evidence
+ evidence_by_kind["export_diff"]
+ end
+
+ def lavitrine_pipeline?
+ website.respond_to?(:lavitrine_pipeline?) && website.lavitrine_pipeline?
+ end
+
+ def signal(key)
+ return nil unless cache.present?
+
+ signals = (cache.signals || {}).to_h
+ return signals[key.to_s] if signals.key?(key.to_s)
+ return signals[key.to_sym] if signals.key?(key.to_sym)
+
+ nil
+ end
+
+ def truthy?(value)
+ value == true || value.to_s == "true" || value.to_s == "1"
+ end
+
+ def explicit_false?(value)
+ value == false || value.to_s == "false" || value.to_s == "0"
+ end
+
+ def legacy_lookup_warning
+ return "Needs review: legacy Wringer endpoint is not configured for this environment." if legacy_lookup_missing_config?
+ return "Needs review: legacy Wringer lookup failed during the latest transition batch check." if legacy_lookup_unreachable?
+ return "Needs review: legacy Wringer body was omitted from the comparison endpoint." if legacy_lookup_body_omitted?
+
+ nil
+ end
+
+ def legacy_lookup_missing_config?
+ fetch_parity_evidence&.detail_reason == "legacy_lookup_missing_config"
+ end
+
+ def legacy_lookup_unreachable?
+ fetch_parity_evidence&.detail_reason == "legacy_lookup_unreachable"
+ end
+
+ def legacy_lookup_body_omitted?
+ fetch_parity_evidence&.detail_reason == "legacy_lookup_body_omitted"
+ end
+
+ def review_needed_difference?
+ fetch_parity_evidence&.detail_reason == "review_needed_difference"
+ end
+
+ def metadata_only_difference?
+ fetch_parity_evidence&.detail_reason == "metadata_only_difference"
+ end
+
+ def cache_compare_unknown?
+ fetch_parity_evidence&.detail_reason == "cache_compare_unknown"
+ end
+
+ def review_activation_eligible?
+ overall_status == :review &&
+ blockers.blank? &&
+ fetch_status == :passed &&
+ statements_status == :passed &&
+ export_status == :passed &&
+ review_needed_difference?
+ end
+
+ def manual_review_required?
+ overall_status == :review &&
+ blockers.blank? &&
+ fetch_status == :passed &&
+ statements_status == :passed &&
+ export_status == :passed &&
+ legacy_lookup_body_omitted?
+ end
+
+ def review_needed_fields
+ Array(fetch_parity_evidence&.details.to_h&.dig("compare_summary", "review_needed_diffs") ||
+ fetch_parity_evidence&.details.to_h&.dig(:compare_summary, :review_needed_diffs)).map(&:to_s)
+ end
+
+ def comparison_policy
+ fetch_parity_evidence&.details.to_h&.[]("comparison_policy") ||
+ fetch_parity_evidence&.details.to_h&.[](:comparison_policy) ||
+ "operator"
+ end
+
+ def primary_blocker_reason
+ return fetch_failure_blocker if fetch_status == :failed
+ return "Cannot activate yet: fetch check is stale." if fetch_status == :stale && lavitrine_pipeline?
+ return "Cannot activate yet: statements could not be evaluated until fetch/cache is fixed." if statements_status == :not_evaluated
+ return "Cannot activate yet: statements check is inconclusive." if statements_status == :inconclusive
+ return "Cannot activate yet: statements check failed." if statements_status == :failed
+ return "Cannot activate yet: export comparison could not be evaluated until fetch/cache is fixed." if export_status == :blocked_by_fetch
+ return "Cannot activate yet: export check is inconclusive." if export_status == :inconclusive
+ return "Cannot activate yet: export check failed." if export_status == :failed
+ end
+
+ def fetch_failure_blocker
+ return "Cannot activate yet: cache comparison failed." if cache_compare_failure?
+ return "Cannot activate yet: cache comparison is missing." if cache_compare_missing?
+ return "Cannot activate yet: cache comparison is inconclusive." if cache_compare_unknown?
+
+ "Cannot activate yet: fetch check failed."
+ end
+
+ def fetch_failure_next_action
+ return "Review the compare page for the affected URLs, then rerun the transition batch check." if cache_compare_failure?
+ return "Re-run the comparison for the affected URLs, then rerun the transition batch check." if cache_compare_missing? || cache_compare_unknown?
+
+ "Fix fetch/cache first, then rerun the transition batch check."
+ end
+
+ def cache_compare_failure?
+ fetch_parity_evidence&.detail_reason == "cache_compare_blocking_regression"
+ end
+
+ def cache_compare_missing?
+ fetch_parity_evidence&.detail_reason == "cache_compare_missing"
+ end
+
+ def evidence_reason(evidence)
+ evidence&.details.to_h&.[]("reason") || evidence&.details.to_h&.[](:reason)
+ end
+
+ def fetch_prevented_statement_refresh?(evidence)
+ evidence_reason(evidence) == "fetch_failed_before_statement_refresh"
+ end
+
+ def partial_fetch_prevented_statement_refresh?(evidence)
+ evidence_reason(evidence).in?(%w[partial_fetch_failed_before_statement_refresh transition_check_timeout_budget_exceeded])
+ end
+
+ def no_selected_statements?(evidence)
+ evidence_reason(evidence) == "no_selected_statements"
+ end
+
+ def optional_statement_warning
+ return unless statements_status == :warning
+
+ "Critical statements passed; optional statement refresh warnings need review."
+ end
+
+ def optional_statement_warning_evidence?(evidence)
+ evidence_reason(evidence) == "optional_statement_refresh_warning" ||
+ (details_count(evidence, "optional_statements_failed_count") > 0 && details_count(evidence, "critical_statements_failed_count").zero?)
+ end
+
+ def details_count(evidence, key)
+ details = evidence&.details.to_h || {}
+ (details[key.to_s] || details[key.to_sym] || 0).to_i
+ end
+
+ def fetch_prevented_export_comparison?(evidence)
+ evidence_reason(evidence) == "fetch_failed_before_export_comparison"
+ end
+
+ def partial_fetch_prevented_export_comparison?(evidence)
+ evidence_reason(evidence).in?(%w[partial_fetch_failed_before_export_comparison transition_check_timeout_budget_exceeded])
+ end
+
+ def export_diff_not_available?(evidence)
+ evidence_reason(evidence) == "export_diff_not_available"
+ end
+ end
+end
diff --git a/app/services/distillator/url_safety_policy.rb b/app/services/distillator/url_safety_policy.rb
new file mode 100644
index 00000000..db4e5978
--- /dev/null
+++ b/app/services/distillator/url_safety_policy.rb
@@ -0,0 +1,127 @@
+require "ipaddr"
+require "resolv"
+require "uri"
+
+module Distillator
+ class UrlSafetyPolicy
+ Decision = Struct.new(:allowed, :reason, :message, :normalized_url, keyword_init: true) do
+ def allowed?
+ allowed
+ end
+ end
+
+ BLOCKED_IPV4_RANGES = [
+ "0.0.0.0/8",
+ "10.0.0.0/8",
+ "100.64.0.0/10",
+ "127.0.0.0/8",
+ "169.254.0.0/16",
+ "172.16.0.0/12",
+ "192.0.0.0/24",
+ "192.0.2.0/24",
+ "192.168.0.0/16",
+ "198.18.0.0/15",
+ "198.51.100.0/24",
+ "203.0.113.0/24",
+ "224.0.0.0/4",
+ "240.0.0.0/4"
+ ].map { |range| IPAddr.new(range) }.freeze
+
+ BLOCKED_IPV6_RANGES = [
+ "::/128",
+ "::1/128",
+ "fc00::/7",
+ "fe80::/10",
+ "ff00::/8"
+ ].map { |range| IPAddr.new(range) }.freeze
+
+ def self.check_url(url, resolver: Resolv)
+ new(resolver: resolver).check_url(url)
+ end
+
+ def self.check_response(response, resolver: Resolv)
+ new(resolver: resolver).check_response(response)
+ end
+
+ def initialize(resolver: Resolv)
+ @resolver = resolver
+ end
+
+ def check_url(url)
+ uri = parse_uri(url)
+ return blocked(:invalid_url, "Invalid URL: #{url}") unless uri
+ return blocked(:blocked_scheme, "Blocked URL scheme: #{uri.scheme}") unless %w[http https].include?(uri.scheme)
+ return blocked(:missing_host, "Missing URL host: #{url}") if uri.host.blank?
+ return blocked(:blocked_localhost, "Blocked localhost host: #{uri.host}") if localhost?(uri.host)
+
+ check_host(uri.host, normalized_url: uri.to_s)
+ end
+
+ def check_response(response)
+ Array(response[:redirect_chain]).each do |url|
+ decision = check_url(url)
+ return decision unless decision.allowed?
+ end
+
+ final_url = response[:final_url]
+ return allowed(nil) if final_url.blank?
+
+ check_url(final_url)
+ end
+
+ private
+
+ attr_reader :resolver
+
+ def check_host(host, normalized_url:)
+ literal_ip = parse_ip(host)
+ return check_ip(literal_ip, host, normalized_url: normalized_url) if literal_ip
+
+ addresses = resolver.getaddresses(host)
+ return blocked(:dns_resolution_failed, "Blocked URL host with no DNS resolution result: #{host}") if addresses.blank?
+
+ addresses.each do |address|
+ decision = check_ip(parse_ip(address), host, normalized_url: normalized_url)
+ return decision unless decision.allowed?
+ end
+
+ allowed(normalized_url)
+ rescue StandardError => error
+ blocked(:dns_resolution_error, "Blocked URL host after DNS resolution error for #{host}: #{error.class}")
+ end
+
+ def check_ip(ip, host, normalized_url:)
+ return blocked(:invalid_resolved_address, "Invalid resolved address for #{host}") unless ip
+
+ ranges = ip.ipv4? ? BLOCKED_IPV4_RANGES : BLOCKED_IPV6_RANGES
+ return blocked(:blocked_private_ip, "Blocked private or reserved address #{ip} for #{host}") if ranges.any? { |range| range.include?(ip) }
+
+ allowed(normalized_url)
+ end
+
+ def parse_uri(url)
+ URI.parse(url.to_s)
+ rescue URI::InvalidURIError
+ nil
+ end
+
+ def parse_ip(value)
+ IPAddr.new(value)
+ rescue IPAddr::InvalidAddressError
+ nil
+ end
+
+ def localhost?(host)
+ normalized = host.to_s.downcase
+ normalized == "localhost" || normalized.end_with?(".localhost")
+ end
+
+ def allowed(normalized_url)
+ Decision.new(allowed: true, reason: nil, message: nil, normalized_url: normalized_url)
+ end
+
+ def blocked(reason, message)
+ Decision.new(allowed: false, reason: reason, message: message, normalized_url: nil)
+ end
+ end
+end
diff --git a/app/services/distillator/webpage_removal_candidate.rb b/app/services/distillator/webpage_removal_candidate.rb
new file mode 100644
index 00000000..02469591
--- /dev/null
+++ b/app/services/distillator/webpage_removal_candidate.rb
@@ -0,0 +1,234 @@
+# frozen_string_literal: true
+
+module Distillator
+ # Determines whether a Webpage may be removed after a Distillator refresh.
+ #
+ # This replaces the old Wringer cleanup path:
+ #
+ # RefreshWebpageJob
+ # -> CcWringerHelper#wringer_received_404?
+ # -> live Wringer /websites.json?term=...
+ # -> delete based on Wringer-stored 404
+ #
+ # New Distillator cleanup path:
+ #
+ # RefreshWebpageJob
+ # -> Distillator::RefreshRunner
+ # -> Distillator::WebpageRemovalCandidate.call(webpage)
+ # -> Distillator::WringerUrlKey.call(webpage.url)
+ # -> Distillator::FetchCache.find_by(uri_key: ...)
+ # -> Distillator::WringerRules.find(primary_issue_key)
+ # -> Result(delete:, reason:, cache:, policy:)
+ #
+ # Issue state is produced earlier by the fetch/cache pipeline:
+ #
+ # FetchService
+ # -> WringerIssueSet / WringerSystemErrorMatcher
+ # -> config/wringer.yml
+ # -> FetchCacheStore
+ # -> CacheHealthMaterializer
+ # -> FetchCache issue fields
+ #
+ # Deletion is intentionally conservative. It requires:
+ # - a matching FetchCache row;
+ # - delete evidence from cache.delete_candidate or signals;
+ # - YAML policy.delete == true;
+ # - YAML policy.retry != true;
+ # - no successful_refresh at or after scrape_date.
+ #
+ # This service is read-only. It does not fetch, call Wringer, parse HTML,
+ # refresh records, or destroy webpages. The caller owns mutation:
+ #
+ # result = Distillator::WebpageRemovalCandidate.call(webpage)
+ # webpage.destroy if result.delete?
+ #
+ class WebpageRemovalCandidate
+ # Lightweight result object returned by .call.
+ #
+ # Fields:
+ # - delete: Boolean; true only when the caller may delete the webpage.
+ # - reason: Symbol explaining the decision.
+ # - cache: The Distillator::FetchCache row used for the decision, if any.
+ # - policy: The matched YAML policy hash, if any.
+ Result = Struct.new(:delete, :reason, :cache, :policy, keyword_init: true) do
+ def delete?
+ delete == true
+ end
+ end
+
+ # Evaluate deletion candidacy for a Webpage object or URL string.
+ #
+ # @param webpage_or_url [Webpage, String] object responding to #url, or a URL.
+ # @param include_fragment [Boolean] whether URI-key generation should include URL fragments.
+ # @return [Result]
+ def self.call(webpage_or_url, include_fragment: false)
+ new(webpage_or_url, include_fragment: include_fragment).call
+ end
+
+ # @param webpage_or_url [Webpage, String]
+ # @param include_fragment [Boolean]
+ def initialize(webpage_or_url, include_fragment:)
+ @webpage_or_url = webpage_or_url
+ @include_fragment = include_fragment
+ end
+
+ # Main decision entrypoint.
+ #
+ # Decision order matters:
+ # - Missing cache means no evidence, so no delete.
+ # - Missing delete policy means no delete, even if stale materialized data says otherwise.
+ # - Retryable issues are preserved for later refresh attempts.
+ # - Newer/same successful refresh prevents deleting valid content.
+ #
+ # @return [Result]
+ def call
+ return result(false, :missing_cache) unless cache
+
+ policy = rule_policy
+
+ return result(false, :not_delete_candidate, policy: policy) unless delete_policy?(policy)
+ return result(false, :retryable_issue, policy: policy) if retryable_issue?(policy)
+ return result(false, :newer_successful_refresh, policy: policy) if newer_successful_refresh?
+
+ result(true, :delete_candidate, policy: policy)
+ end
+
+ private
+
+ attr_reader :webpage_or_url, :include_fragment
+
+ # Build a Result with the current cache attached.
+ #
+ # @param delete [Boolean]
+ # @param reason [Symbol]
+ # @param policy [Hash]
+ # @return [Result]
+ def result(delete, reason, policy: {})
+ Result.new(delete: delete, reason: reason, cache: cache, policy: policy)
+ end
+
+ # Find the cache row by Wringer-compatible URI key.
+ #
+ # Invalid URLs are treated as "no usable evidence", not as errors.
+ #
+ # @return [Distillator::FetchCache, nil]
+ def cache
+ @cache ||= Distillator::FetchCache.find_by(uri_key: uri_key)
+ rescue URI::InvalidURIError, Addressable::URI::InvalidURIError
+ nil
+ end
+
+ # Compute the Wringer-compatible URI key for the source URL.
+ #
+ # @return [String]
+ def uri_key
+ @uri_key ||= Distillator::WringerUrlKey.call(url, include_fragment: include_fragment).uri_key
+ end
+
+ # Resolve a URL from either a Webpage-like object or a raw string.
+ #
+ # @return [String]
+ def url
+ webpage_or_url.respond_to?(:url) ? webpage_or_url.url : webpage_or_url.to_s
+ end
+
+ # Resolve the YAML policy for the cache's primary issue.
+ #
+ # Prefer the materialized column, but fall back to signals for partially
+ # materialized rows created during migration.
+ #
+ # @return [Hash]
+ def rule_policy
+ rule = Distillator::WringerRules.find(primary_issue_key)&.last
+ policy = rule && (rule["policy"] || rule[:policy])
+
+ policy.respond_to?(:to_h) ? policy.to_h : {}
+ end
+
+ # Primary issue key used to look up config/wringer.yml policy.
+ #
+ # @return [String, nil]
+ def primary_issue_key
+ cache&.primary_issue_key.presence || signals[:primary_issue_key].presence
+ end
+
+ # A delete decision requires both:
+ # - cache/signal evidence that this row was marked as a delete candidate; and
+ # - current YAML policy allowing deletion.
+ #
+ # This protects against stale cache rows after YAML policy changes.
+ #
+ # @param policy [Hash]
+ # @return [Boolean]
+ def delete_policy?(policy)
+ delete_candidate_evidence? && policy_value(policy, :delete) == true
+ end
+
+ # Detect delete-candidate evidence from materialized columns or signals.
+ #
+ # @return [Boolean]
+ def delete_candidate_evidence?
+ cache.delete_candidate == true || truthy_signal?(:primary_issue_delete)
+ end
+
+ # Retryable issues should not delete pages. Examples include waiting rooms,
+ # captcha, Cloudflare, Akamai, timeout, and other transient fetch failures.
+ #
+ # @param policy [Hash]
+ # @return [Boolean]
+ def retryable_issue?(policy)
+ policy_value(policy, :retry) == true
+ end
+
+ # Prevent deletion when the latest scrape was successful.
+ #
+ # successful_refresh is the latest successful content update.
+ # scrape_date is the latest scrape attempt.
+ #
+ # If successful_refresh >= scrape_date, the latest scrape attempt either
+ # succeeded or has been superseded by a successful refresh, so deleting would
+ # be unsafe.
+ #
+ # @return [Boolean]
+ def newer_successful_refresh?
+ return false if cache.successful_refresh.blank?
+ return false if cache.scrape_date.blank?
+
+ cache.successful_refresh >= cache.scrape_date
+ end
+
+ # Cache signals as an indifferent-access hash.
+ #
+ # @return [ActiveSupport::HashWithIndifferentAccess]
+ def signals
+ @signals ||= (cache&.signals || {}).to_h.with_indifferent_access
+ end
+
+ # Interpret a signal boolean conservatively.
+ #
+ # Accepts true and common string/number encodings used in serialized JSON.
+ #
+ # @param key [Symbol, String]
+ # @return [Boolean]
+ def truthy_signal?(key)
+ value = signals[key]
+ value == true || value.to_s == "true" || value.to_s == "1"
+ end
+
+ # Safely read string-keyed or symbol-keyed policy values.
+ #
+ # This intentionally preserves explicit false values.
+ #
+ # @param policy [Hash]
+ # @param key [Symbol, String]
+ # @return [Object, nil]
+ def policy_value(policy, key)
+ return nil unless policy.respond_to?(:[])
+
+ return policy[key.to_s] if policy.respond_to?(:key?) && policy.key?(key.to_s)
+ return policy[key.to_sym] if policy.respond_to?(:key?) && policy.key?(key.to_sym)
+
+ nil
+ end
+ end
+end
\ No newline at end of file
diff --git a/app/services/distillator/website_webpage_summary.rb b/app/services/distillator/website_webpage_summary.rb
new file mode 100644
index 00000000..cbb6a77a
--- /dev/null
+++ b/app/services/distillator/website_webpage_summary.rb
@@ -0,0 +1,74 @@
+module Distillator
+ class WebsiteWebpageSummary
+ CLASS_BUCKETS = %w[Event Person Place ResourceList WebPage].freeze
+
+ def self.for_websites(website_ids)
+ ids = Array(website_ids).map(&:to_i).uniq
+ return {} if ids.empty?
+
+ summaries = ids.index_with { empty_summary }
+
+ populate_totals!(summaries, ids)
+ populate_class_counts!(summaries, ids)
+ populate_publishable_counts!(summaries, ids)
+
+ summaries
+ end
+
+ def self.empty_summary
+ {
+ total: 0,
+ public_urls: 0,
+ internal_uris: 0,
+ by_class: CLASS_BUCKETS.index_with { 0 }.merge("Other" => 0),
+ publishable: 0,
+ not_publishable: 0
+ }
+ end
+
+ def self.populate_totals!(summaries, website_ids)
+ counts = Webpage.where(website_id: website_ids)
+ .group(:website_id)
+ .pluck(
+ :website_id,
+ Arel.sql("COUNT(*)"),
+ Arel.sql("SUM(CASE WHEN #{Webpage.public_source_url_sql} THEN 1 ELSE 0 END)"),
+ Arel.sql("SUM(CASE WHEN #{Webpage.public_source_url_sql} THEN 0 ELSE 1 END)")
+ )
+
+ counts.each do |website_id, total, public_urls, internal_uris|
+ summary = summaries.fetch(website_id)
+ summary[:total] = total.to_i
+ summary[:public_urls] = public_urls.to_i
+ summary[:internal_uris] = internal_uris.to_i
+ end
+ end
+
+ def self.populate_class_counts!(summaries, website_ids)
+ Webpage.left_outer_joins(:rdfs_class)
+ .where(website_id: website_ids)
+ .group(:website_id, "rdfs_classes.name")
+ .count
+ .each do |(website_id, class_name), count|
+ bucket = CLASS_BUCKETS.include?(class_name) ? class_name : "Other"
+ summaries.fetch(website_id)[:by_class][bucket] += count.to_i
+ end
+ end
+
+ def self.populate_publishable_counts!(summaries, website_ids)
+ Webpage.where(website_id: website_ids)
+ .publishable
+ .group(:website_id)
+ .count
+ .each do |website_id, count|
+ summaries.fetch(website_id)[:publishable] = count.to_i
+ end
+
+ summaries.each_value do |summary|
+ summary[:not_publishable] = summary[:total] - summary[:publishable]
+ end
+ end
+
+ private_class_method :populate_totals!, :populate_class_counts!, :populate_publishable_counts!
+ end
+end
diff --git a/app/services/distillator/wringer_endpoint.rb b/app/services/distillator/wringer_endpoint.rb
new file mode 100644
index 00000000..f38004c1
--- /dev/null
+++ b/app/services/distillator/wringer_endpoint.rb
@@ -0,0 +1,185 @@
+require "uri"
+
+module Distillator
+ class WringerEndpoint
+ CANONICAL_COMPATIBILITY_ENV = "DISTILLATOR_COMPAT_BASE_URL".freeze
+ COMPATIBILITY_ALIAS_ENV = "DISTILLATOR_COMPATIBILITY_BASE_URL".freeze
+ LEGACY_WRINGER_FALLBACK_ENV = "DISTILLATOR_LEGACY_WRINGER_FALLBACK".freeze
+ LOCAL_COMPATIBILITY_BASE_URL = "http://localhost:3000".freeze
+ LOCAL_LEGACY_BASE_URL = "http://localhost:3009".freeze
+
+ Result = Struct.new(
+ :compatibility_base_url,
+ :legacy_lookup_base_url,
+ :compatibility_source,
+ :state,
+ :status_label,
+ :status_detail,
+ keyword_init: true
+ ) do
+ def missing_config?
+ state == :missing_config
+ end
+
+ def local_development?
+ state == :local_development
+ end
+
+ def remote_configured?
+ state == :remote_configured
+ end
+
+ def unreachable?
+ state == :unreachable
+ end
+ end
+
+ def self.current(last_error: nil)
+ new(last_error: last_error).call
+ end
+
+ def initialize(config: Rails.application.config.x.distillator, env: Rails.env, env_vars: ENV, last_error: nil)
+ @config = config
+ @env = env.to_s
+ @env_vars = env_vars
+ @last_error = last_error
+ end
+
+ def call
+ compatibility, compatibility_source = configured_compatibility_base_url
+ legacy_lookup = configured_legacy_lookup_base_url(compatibility)
+
+ if compatibility.present?
+ return build_remote_result(compatibility, legacy_lookup, compatibility_source)
+ end
+
+ return build_local_result if allow_localhost_default?
+
+ build_missing_config_result
+ end
+
+ private
+
+ attr_reader :config, :env, :env_vars, :last_error
+
+ def allow_localhost_default?
+ configured = config_value(:allow_localhost_compatibility)
+ return configured if [true, false].include?(configured)
+
+ env == "development"
+ end
+
+ def configured_compatibility_base_url
+ return [normalize_base_url(config_value(:compatibility_base_url)), "config.x.distillator.compatibility_base_url"] if config_value(:compatibility_base_url).present?
+ return [normalize_base_url(env_vars[CANONICAL_COMPATIBILITY_ENV]), CANONICAL_COMPATIBILITY_ENV] if env_vars[CANONICAL_COMPATIBILITY_ENV].present?
+ return [normalize_base_url(env_vars[COMPATIBILITY_ALIAS_ENV]), COMPATIBILITY_ALIAS_ENV] if env_vars[COMPATIBILITY_ALIAS_ENV].present?
+
+ [nil, nil]
+ end
+
+ def configured_legacy_lookup_base_url(compatibility)
+ normalize_base_url(
+ config_value(:legacy_wringer_base_url) ||
+ env_vars["LEGACY_WRINGER_BASE_URL"]
+ ) || compatibility
+ end
+
+ def build_remote_result(compatibility, legacy_lookup, compatibility_source)
+ state = last_error.present? ? :unreachable : :remote_configured
+ label =
+ if state == :unreachable
+ "Current Wringer: Unreachable"
+ else
+ "Current Wringer: Remote configured"
+ end
+
+ detail =
+ if state == :unreachable
+ "last lookup failed"
+ else
+ [sanitize_display_url(compatibility), compatibility_source_detail(compatibility_source)].compact.join(" via ")
+ end
+
+ Result.new(
+ compatibility_base_url: compatibility,
+ legacy_lookup_base_url: legacy_lookup,
+ compatibility_source: compatibility_source,
+ state: state,
+ status_label: label,
+ status_detail: detail
+ )
+ end
+
+ def build_local_result
+ Result.new(
+ compatibility_base_url: LOCAL_COMPATIBILITY_BASE_URL,
+ legacy_lookup_base_url: normalize_base_url(config_value(:legacy_wringer_base_url) || env_vars["LEGACY_WRINGER_BASE_URL"]) || LOCAL_LEGACY_BASE_URL,
+ compatibility_source: "development_default",
+ state: :local_development,
+ status_label: "Current Wringer: Local development",
+ status_detail: sanitize_display_url(LOCAL_COMPATIBILITY_BASE_URL)
+ )
+ end
+
+ def build_missing_config_result
+ detail =
+ if env == "staging"
+ "comparisons disabled"
+ else
+ "legacy lookup unavailable"
+ end
+
+ label =
+ if env == "staging"
+ "Current Wringer: Missing staging config"
+ else
+ "Current Wringer: Missing config"
+ end
+
+ Result.new(
+ compatibility_base_url: nil,
+ legacy_lookup_base_url: nil,
+ compatibility_source: nil,
+ state: :missing_config,
+ status_label: label,
+ status_detail: detail
+ )
+ end
+
+ def normalize_base_url(value)
+ url = value.to_s.strip
+ return nil if url.blank?
+
+ url.sub(%r{/\z}, "")
+ end
+
+ def config_value(key)
+ return nil unless config.present?
+ return config.public_send(key) if config.respond_to?(key)
+
+ nil
+ end
+
+ def sanitize_display_url(value)
+ uri = URI.parse(value.to_s)
+ return value.to_s if uri.scheme.blank? || uri.host.blank?
+
+ "#{uri.scheme}://#{uri.host}#{uri.port ? port_suffix(uri) : ''}"
+ rescue URI::InvalidURIError
+ value.to_s
+ end
+
+ def port_suffix(uri)
+ return "" if (uri.scheme == "http" && uri.port == 80) || (uri.scheme == "https" && uri.port == 443)
+
+ ":#{uri.port}"
+ end
+
+ def compatibility_source_detail(source)
+ return nil if source.blank?
+ return source if [CANONICAL_COMPATIBILITY_ENV, COMPATIBILITY_ALIAS_ENV].include?(source)
+
+ source
+ end
+ end
+end
diff --git a/app/services/distillator/wringer_issue_set.rb b/app/services/distillator/wringer_issue_set.rb
new file mode 100644
index 00000000..c1ec1897
--- /dev/null
+++ b/app/services/distillator/wringer_issue_set.rb
@@ -0,0 +1,52 @@
+module Distillator
+ class WringerIssueSet
+ SEVERITY_RANK = {
+ "blocked" => 0,
+ "failed" => 1,
+ "warning" => 2,
+ "info" => 3
+ }.freeze
+
+ CATEGORY_RANK = {
+ "anti_bot" => 0,
+ "security" => 1,
+ "network" => 2,
+ "redirect" => 3,
+ "http" => 4,
+ "content" => 5,
+ "fetch_mode" => 6,
+ "renderer" => 7
+ }.freeze
+
+ Result = Struct.new(:matches, :primary, keyword_init: true)
+
+ def self.call(body:, http_code:, final_url:, hints: [], signals: {}, rules: nil, logger: Rails.logger)
+ matches = Distillator::WringerSystemErrorMatcher.all_matches(
+ body: body,
+ http_code: http_code,
+ final_url: final_url,
+ hints: hints,
+ signals: signals,
+ rules: rules,
+ logger: logger
+ )
+ Result.new(matches: matches, primary: select_primary(matches))
+ end
+
+ def self.select_primary(matches)
+ indexed = Array(matches).each_with_index.to_a
+
+ indexed.min_by do |(match, index)|
+ rule = match[:rule] || {}
+ category = rule["category"] || rule[:category]
+ severity = rule["severity"] || rule[:severity]
+
+ [
+ CATEGORY_RANK.fetch(category.to_s, 99),
+ SEVERITY_RANK.fetch(severity.to_s, 99),
+ index
+ ]
+ end&.first
+ end
+ end
+end
diff --git a/app/services/distillator/wringer_rules.rb b/app/services/distillator/wringer_rules.rb
new file mode 100644
index 00000000..49bf871b
--- /dev/null
+++ b/app/services/distillator/wringer_rules.rb
@@ -0,0 +1,34 @@
+module Distillator
+ class WringerRules
+ def self.all
+ @all ||= load_rules
+ end
+
+ def self.reset!
+ @all = nil
+ end
+
+ def self.find(key)
+ all.find { |rule_key, _rule| rule_key.to_s == key.to_s }
+ end
+
+ def self.load_rules
+ config_path = Rails.root.join("config/wringer.yml")
+ unless File.exist?(config_path)
+ Rails.logger.warn "[Wringer] Missing config/wringer.yml"
+ return []
+ end
+
+ raw_yaml = ERB.new(File.read(config_path)).result
+ parsed = YAML.safe_load(raw_yaml, aliases: true) || {}
+ exceptions = parsed["system_exceptions"] || parsed[:system_exceptions]
+ unless exceptions
+ Rails.logger.warn "[Wringer] No system_exceptions configured"
+ return []
+ end
+
+ exceptions.to_a
+ end
+ private_class_method :load_rules
+ end
+end
diff --git a/app/services/distillator/wringer_system_error_matcher.rb b/app/services/distillator/wringer_system_error_matcher.rb
new file mode 100644
index 00000000..41ba1b25
--- /dev/null
+++ b/app/services/distillator/wringer_system_error_matcher.rb
@@ -0,0 +1,227 @@
+module Distillator
+ class WringerSystemErrorMatcher
+ def self.call(body:, http_code:, final_url:, hints: [], signals: {}, rules: nil, logger: Rails.logger)
+ matches = all_matches(
+ body: body,
+ http_code: http_code,
+ final_url: final_url,
+ hints: hints,
+ signals: signals,
+ rules: rules,
+ logger: logger
+ )
+
+ Distillator::WringerIssueSet.select_primary(matches)
+ end
+
+ def self.all_matches(body:, http_code:, final_url:, hints: [], signals: {}, rules: nil, logger: Rails.logger)
+ new(
+ body: body,
+ http_code: http_code,
+ final_url: final_url,
+ hints: hints,
+ signals: signals,
+ rules: rules || Distillator::WringerRules.all,
+ logger: logger
+ ).all_matches
+ end
+
+ def initialize(body:, http_code:, final_url:, hints:, signals:, rules:, logger:)
+ @body_value = body
+ @body = body.to_s
+ @http_code = http_code.to_i
+ @final_url = final_url.to_s
+ @hints = Array(hints).map(&:to_s)
+ @signals = (signals || {}).to_h.stringify_keys
+ @rules = Array(rules)
+ @logger = logger || Rails.logger
+ end
+
+ def call
+ all_matches.first
+ end
+
+ def all_matches
+ rules.filter_map do |name, rule|
+ match = value(rule, :match) || {}
+ policy = value(rule, :policy) || {}
+ match_details = matched_details(match)
+ next unless match_details
+
+ logger.warn "[Wringer] #{name} matched (code=#{http_code}, url=#{final_url})"
+
+ {
+ key: name.to_s,
+ error: "#{name} detected",
+ error_type: value(policy, :error_code) || name.to_s,
+ policy: policy,
+ rule: rule,
+ match_details: match_details,
+ action: value(policy, :action),
+ retry: value(policy, :retry),
+ cache: value(policy, :cache),
+ delete: value(policy, :delete)
+ }
+ end
+ end
+
+ private
+
+ attr_reader :body, :body_value, :http_code, :final_url, :hints, :signals, :rules, :logger
+
+ def matched_details(match)
+ details = nil
+ http_code_match = value(match, :http_code)
+ if http_code_match
+ codes = Array(http_code_match).map(&:to_i)
+ return nil unless codes.include?(http_code)
+
+ details ||= {
+ source: "http_code",
+ pattern: codes.join(", "),
+ snippet: "HTTP #{http_code}"
+ }
+ end
+
+ body_contains = value(match, :body_contains)
+ if body_contains
+ matched_text = Array(body_contains).find { |text| body.include?(text) }
+ return nil unless matched_text
+
+ details ||= {
+ source: "html",
+ pattern: matched_text,
+ snippet: matched_snippet(body, matched_text)
+ }
+ end
+
+ body_not_contains = value(match, :body_not_contains)
+ if body_not_contains
+ blocked_text = Array(body_not_contains).find { |text| body.include?(text) }
+ return nil if blocked_text
+ end
+
+ body_text_contains = value(match, :body_text_contains)
+ if body_text_contains
+ matched_text = Array(body_text_contains).find { |text| body_text.include?(text) }
+ return nil unless matched_text
+
+ details ||= {
+ source: "body_text",
+ pattern: matched_text,
+ snippet: matched_snippet(body_text, matched_text)
+ }
+ end
+
+ body_text_not_contains = value(match, :body_text_not_contains)
+ if body_text_not_contains
+ blocked_text = Array(body_text_not_contains).find { |text| body_text.include?(text) }
+ return nil if blocked_text
+ end
+
+ body_blank = value(match, :body_blank)
+ if body_blank
+ return nil unless body_value.nil? || (body_value.respond_to?(:strip) && body_value.strip.empty?)
+
+ details ||= {
+ source: "body",
+ pattern: "body_blank",
+ snippet: "Body was blank"
+ }
+ end
+
+ hint_match = value(match, :hints)
+ if hint_match
+ matched_hint = Array(hint_match).find { |hint| hints.include?(hint.to_s) }
+ return nil unless matched_hint
+
+ details ||= {
+ source: "hint",
+ pattern: matched_hint.to_s,
+ snippet: matched_hint.to_s
+ }
+ end
+
+ network_status_match = value(match, :network_status)
+ if network_status_match.present?
+ return nil unless signals["network_status"].to_s == network_status_match.to_s
+
+ details ||= {
+ source: "signal",
+ pattern: "network_status=#{network_status_match}",
+ snippet: "network_status=#{signals['network_status']}"
+ }
+ end
+
+ content_type_match = value(match, :content_type)
+ if content_type_match.present?
+ return nil unless signals["content_type"].to_s == content_type_match.to_s
+
+ details ||= {
+ source: "signal",
+ pattern: "content_type=#{content_type_match}",
+ snippet: "content_type=#{signals['content_type']}"
+ }
+ end
+
+ signal_match = value(match, :signals)
+ if signal_match
+ matched_pair = signal_match.to_h.find do |key, expected|
+ signals[key.to_s].to_s == expected.to_s
+ end
+ return nil unless signal_match.to_h.all? { |key, expected| signals[key.to_s].to_s == expected.to_s }
+
+ details ||= {
+ source: "signal",
+ pattern: "#{matched_pair.first}=#{matched_pair.last}",
+ snippet: "#{matched_pair.first}=#{signals[matched_pair.first.to_s]}"
+ }
+ end
+
+ final_url_patterns = value(match, :final_url_patterns)
+ if final_url_patterns
+ matched_pattern = Array(final_url_patterns).find do |pattern|
+ Regexp.new(pattern).match?(final_url)
+ rescue RegexpError
+ false
+ end
+ return nil unless matched_pattern
+
+ details ||= {
+ source: "final_url",
+ pattern: matched_pattern,
+ snippet: matched_snippet(final_url, matched_pattern)
+ }
+ end
+
+ details || { source: "rule", pattern: "matched", snippet: "Rule matched" }
+ end
+
+ def value(hash, key)
+ return nil unless hash.respond_to?(:[])
+
+ return hash[key.to_s] if hash.respond_to?(:key?) && hash.key?(key.to_s)
+ return hash[key.to_sym] if hash.respond_to?(:key?) && hash.key?(key.to_sym)
+
+ hash[key.to_s] || hash[key.to_sym]
+ end
+
+ def body_text
+ @body_text ||= Nokogiri::HTML(body.to_s).text.to_s.squish
+ rescue StandardError
+ body.to_s
+ end
+
+ def matched_snippet(text, pattern, max_length: 140)
+ raw_text = text.to_s.squish
+ return raw_text.first(max_length) if pattern.blank?
+
+ index = raw_text.index(pattern.to_s)
+ return raw_text.first(max_length) unless index
+
+ start = [index - 40, 0].max
+ snippet = raw_text[start, max_length]
+ snippet.to_s
+ end
+ end
+end
diff --git a/app/services/distillator/wringer_url_key.rb b/app/services/distillator/wringer_url_key.rb
new file mode 100644
index 00000000..f3c8d68e
--- /dev/null
+++ b/app/services/distillator/wringer_url_key.rb
@@ -0,0 +1,61 @@
+require "addressable/uri"
+require "cgi"
+require "uri"
+
+module Distillator
+ class WringerUrlKey
+ Result = Struct.new(:uri_key, :normalized_url, keyword_init: true)
+
+ def self.call(url, include_fragment: false)
+ new(url, include_fragment: include_fragment).call
+ end
+
+ def initialize(url, include_fragment: false)
+ @url = url
+ @include_fragment = include_fragment
+ end
+
+ def call
+ uri = Addressable::URI.parse(validated_url)
+ key =
+ if %w[http https].include?(uri.scheme)
+ build_key(uri)
+ else
+ "Error: not a URI"
+ end
+ validate_key!(key)
+
+ Result.new(uri_key: CGI.escape(key), normalized_url: key)
+ end
+
+ private
+
+ attr_reader :url, :include_fragment
+
+ def validated_url
+ value = url.to_s.strip
+ return value if value.start_with?("http")
+
+ "http://#{value}"
+ end
+
+ def build_key(uri)
+ key = "#{uri.scheme}://#{uri.host}"
+ key += uri.path.to_s
+ key += "?#{uri.query}" if uri.query
+ key += "##{uri.fragment}" if include_fragment? && uri.fragment
+ key
+ end
+
+ def validate_key!(key)
+ return if key == "Error: not a URI"
+
+ uri = URI.parse(key)
+ raise URI::InvalidURIError, "Missing URL host: #{key}" if uri.host.blank?
+ end
+
+ def include_fragment?
+ Distillator::BooleanParam.parse(include_fragment)
+ end
+ end
+end
diff --git a/app/services/dsl/core/algorithm_runner.rb b/app/services/dsl/core/algorithm_runner.rb
new file mode 100644
index 00000000..5b8b2330
--- /dev/null
+++ b/app/services/dsl/core/algorithm_runner.rb
@@ -0,0 +1,895 @@
+require "stringio"
+module Dsl
+ module Core
+ class AlgorithmRunner
+ HALT = Object.new
+
+ StepTrace = Struct.new(
+ :step,
+ :type,
+ :code,
+ :input,
+ :output,
+ :error,
+ :url_before,
+ :url_after,
+ :duration_ms,
+ keyword_init: true
+ )
+
+ def initialize(ctx)
+ @url = ctx[:url]
+ @render_js = ctx[:render_js]
+ @scrape_opts = ctx[:scrape_options] || {}
+ @tracer = ctx[:tracer]
+ @mode = ctx[:mode] || @scrape_opts[:mode] || @scrape_opts["mode"]
+ @agent = Mechanize.new
+ @agent.user_agent_alias = 'Mac Safari'
+ @html = nil
+ @page = nil
+ @json = nil
+ @graph = nil
+ end
+
+ def abort_structure?(obj)
+ obj.is_a?(Array) &&
+ obj.length == 2 &&
+ obj.first == "abort_update" &&
+ obj.last.is_a?(Hash)
+ end
+
+ def run(algorithm)
+ results = []
+ previous_locals = snapshot_thread_locals
+
+ # reset thread-local DSL state for this run
+ Thread.current[:dsl_array] = []
+ Thread.current[:dsl_url] = @url
+ Thread.current[:dsl_json] = nil
+
+ @dsl_binding = binding
+
+ steps = algorithm.split(';').map(&:strip).reject(&:empty?)
+ if export_debug?
+ Rails.logger.warn(
+ "[EXPORT_DEBUG] DSL.run entry url=#{@url} render_js=#{@render_js.inspect} steps=#{steps.size} " \
+ "algorithm=#{algorithm.to_s[0, 240].inspect}"
+ )
+ end
+ previous_prefix = nil
+ previous_step_index = nil
+ @probed_url_step_indices = []
+
+ steps.each_with_index do |raw, idx|
+ prefix, code = raw.partition('=').values_at(0, 2)
+ step_index = idx + 1
+
+ @current_wringer_status = nil
+ input_copy = Marshal.load(Marshal.dump(results))
+ url_before = @url
+ start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+ # === UPDATE THREAD-LOCALS BEFORE EVERY STEP ===
+ Thread.current[:dsl_array] = results
+ Thread.current[:dsl_url] = @url
+ Thread.current[:dsl_json] = @json
+
+ out = execute(prefix, code, results)
+ input_preview = trace_preview(input_copy)
+
+ # handle abort payload
+ if abort_structure?(out)
+ end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ duration_ms = ((end_time - start_time) * 1000).round(1)
+
+ @tracer.step(
+ step: step_index,
+ type: prefix,
+ code: code,
+ input: input_preview,
+ output: [],
+ input_full: input_copy,
+ output_full: [],
+ probe: trace_probe_payload(nil),
+ error: out.last, # error message details
+ wringer: trace_wringer_payload,
+ url_before: url_before,
+ url_after: @url,
+ duration_ms: duration_ms
+ )
+
+ if export_debug?
+ Rails.logger.warn(
+ "[EXPORT_DEBUG] DSL.run abort step=#{step_index} type=#{prefix} url=#{@url} " \
+ "error_type=#{out.last[:error_type] || out.last['error_type']}"
+ )
+ end
+ return out
+ end
+
+ if halt_structure?(out)
+ end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ duration_ms = ((end_time - start_time) * 1000).round(1)
+ results = out.last
+
+ @tracer.step(
+ step: step_index,
+ type: prefix,
+ code: code,
+ input: input_preview,
+ output: trace_preview(results),
+ input_full: input_copy,
+ output_full: results,
+ probe: trace_probe_payload(nil),
+ error: nil,
+ wringer: trace_wringer_payload,
+ url_before: url_before,
+ url_after: @url,
+ duration_ms: duration_ms
+ )
+
+ if export_debug?
+ Rails.logger.warn(
+ "[EXPORT_DEBUG] DSL.run halt step=#{step_index} type=#{prefix} url=#{@url} results_count=#{Array(results).size}"
+ )
+ end
+ break
+ end
+
+ end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ duration_ms = ((end_time - start_time) * 1000).round(1)
+
+ url_after = @url
+ output = trace_preview(out)
+ probe = build_xpath_probe(previous_prefix, prefix, out, previous_step_index)
+ if abort_structure?(probe)
+ @tracer.step(
+ step: step_index,
+ type: prefix,
+ code: code,
+ input: input_preview,
+ output: [],
+ input_full: input_copy,
+ output_full: [],
+ probe: trace_probe_payload(nil),
+ error: probe.last,
+ wringer: trace_wringer_payload,
+ url_before: url_before,
+ url_after: url_after,
+ duration_ms: duration_ms
+ )
+
+ return probe
+ end
+
+ @tracer.step(
+ step: step_index,
+ type: prefix,
+ code: code,
+ input: input_preview,
+ output: output,
+ input_full: input_copy,
+ output_full: out,
+ probe: trace_probe_payload(probe),
+ error: nil,
+ wringer: trace_wringer_payload,
+ url_before: url_before,
+ url_after: url_after,
+ duration_ms: duration_ms
+ )
+
+ results = out
+ previous_prefix = prefix
+ previous_step_index = step_index
+ rescue StandardError, SyntaxError => e
+ end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ duration_ms = ((end_time - start_time) * 1000).round(1)
+
+ @tracer.step(
+ step: step_index,
+ type: prefix,
+ code: code,
+ input: input_preview,
+ output: [],
+ input_full: input_copy,
+ output_full: [],
+ probe: trace_probe_payload(nil),
+ error: e,
+ wringer: trace_wringer_payload,
+ url_before: url_before,
+ url_after: @url,
+ duration_ms: duration_ms
+ )
+ return abort_update(
+ error: e.message,
+ error_type: e.class.to_s,
+ step: prefix
+ )
+ end
+
+ if export_debug?
+ Rails.logger.warn("[EXPORT_DEBUG] DSL.run exit url=#{@url} results_count=#{Array(results).size}")
+ end
+ results
+ ensure
+ restore_thread_locals(previous_locals)
+ end
+
+ private
+
+ def export_debug?
+ ENV["EXPORT_DEBUG"].present?
+ end
+
+ def execute(prefix, code, arr)
+ case prefix
+
+ # accumulate = map + flatMap
+ # accumulate context:
+ # - $e = current item
+ # - $array = [current item]
+ # - $url = current item
+ when 'accumulate'
+ execute_accumulate(code, arr)
+
+ when 'sparql'
+ begin
+ graph_status = ensure_graph!(step: 'sparql')
+ return graph_status if abort_structure?(graph_status)
+
+ sparql = "PREFIX schema: select * where #{code}"
+ rows = SPARQL.execute(sparql, @graph)
+ [*(rows.count == 1 ? rows.first.answer.value : rows.map { |r| r.answer.value })]
+ rescue StandardError => e
+ abort_update(error: e.message, error_type: e.class.to_s, step: 'sparql')
+ end
+
+ when 'url' then handle_url_step(code, arr, step: 'url')
+
+ when 'renderjs_url' then handle_url_step(code, arr, render_js: true, step: 'renderjs_url')
+
+ when 'post_url'
+ temp_opts = @scrape_opts.merge(json_post: true).merge(force_scrape_every_hrs: 1)
+ handle_url_step(code, arr, opts: temp_opts, step: 'post_url')
+
+ when 'json_url'
+ result = resolve_and_fetch_url(code, arr, step: 'json_url')
+ return result if abort_structure?(result)
+
+ apply_json_text_result(result)
+
+ arr
+
+ when 'api'
+ new_url = resolve_url_only(code, arr, step: 'api')
+ return new_url if abort_structure?(new_url)
+
+ data = HTTParty.get(new_url)
+ raise "API error #{data.code}" unless data.code.to_s.start_with?('2')
+
+ JSON.parse(data.body)
+
+ when 'xpath'
+ execute_xpath(code)
+
+ when 'xpath_sanitize'
+ page_status = ensure_page!(step: 'xpath_sanitize')
+ return page_status if abort_structure?(page_status)
+
+ @page.xpath(code).map do |node|
+ sanitize(node.to_s,
+ tags: %w[h1 h2 h3 h4 h5 h6 p li ul ol strong em a i br],
+ attributes: %w[href])
+ end
+
+ when 'if_xpath'
+ page_status = ensure_page!(step: 'if_xpath')
+ return page_status if abort_structure?(page_status)
+
+ nodes = @page.xpath(code)
+ return [HALT, arr] if nodes.blank?
+
+ nodes.map { |n| n.text.to_s.squish }.reject(&:blank?)
+
+ when 'unless_xpath'
+ page_status = ensure_page!(step: 'unless_xpath')
+ return page_status if abort_structure?(page_status)
+
+ nodes = @page.xpath(code)
+ return [HALT, arr] if nodes.present?
+
+ arr
+
+ when 'css'
+ page_status = ensure_page!(step: 'css')
+ return page_status if abort_structure?(page_status)
+
+ @page.css(code).map { |n| n.text.to_s.squish }.reject(&:blank?)
+
+ when 'json'
+ page_status = ensure_page!(step: 'json')
+ return page_status if abort_structure?(page_status)
+
+ text = @page.respond_to?(:text) ? @page.text : @html.to_s
+ @json ||= JSON.parse(text)
+ Thread.current[:dsl_json] = @json
+ @dsl_binding.eval(sub(code, arr))
+
+ when 'time_zone'
+ ["time_zone: #{code}"]
+
+ when 'ruby'
+ # update thread-locals before eval
+ Thread.current[:dsl_array] = arr
+ Thread.current[:dsl_url] = @url
+ Thread.current[:dsl_json] = @json
+
+ result = @dsl_binding.eval(sub(code, arr))
+
+ # sync back DSL state
+ current_array = Thread.current[:dsl_array]
+ @url = Thread.current[:dsl_url]
+ @json = Thread.current[:dsl_json]
+
+ if result.nil? && current_array.equal?(arr)
+ result = current_array
+ end
+
+ result
+
+ when 'manual'
+ [code]
+
+ else
+ raise "Missing DSL prefix: #{prefix}=#{code}"
+ end
+ end
+
+ def build_xpath_probe(previous_prefix, current_prefix, output, previous_step_index = nil)
+ return nil unless previous_prefix == 'url'
+ return nil unless current_prefix == 'xpath'
+ return nil if output.present?
+ return nil if previous_step_index.nil?
+
+ @probed_url_step_indices ||= []
+ return nil if @probed_url_step_indices.include?(previous_step_index)
+
+ probe_result = execute_xpath("//title")
+ if abort_structure?(probe_result)
+ return probe_result
+ end
+
+ probe_output = Array(probe_result).compact.map(&:to_s).first(3)
+ @probed_url_step_indices << previous_step_index
+
+ {
+ status: "ok",
+ xpath: "//title",
+ output: probe_output
+ }
+ rescue StandardError
+ @probed_url_step_indices << previous_step_index if previous_step_index
+
+ {
+ status: "error",
+ exception: true,
+ xpath: "//title",
+ output: []
+ }
+ end
+
+ def resolve_and_fetch_url(code, arr, render_js: @render_js, opts: @scrape_opts, step: 'url')
+ raw = @dsl_binding.eval(sub(code, arr))
+ new_url = Dsl::Support::UrlResolver.extract(raw)
+ if new_url.blank?
+ Rails.logger.warn { "[DSL] abort invalid URL from #{raw.inspect}" }
+ return abort_update(
+ error: "Invalid URL resolved from #{raw.inspect}",
+ error_type: "InvalidURL",
+ step: step
+ )
+ end
+
+ Rails.logger.debug { "[DSL] #{code} → #{new_url}" }
+
+ @url = new_url
+
+ fetch_result = fetch_result_for(url: @url, render_js: render_js, scrape_options: opts)
+ @current_wringer_status =
+ (fetch_result[:wringer] || {}).merge(
+ duration_ms: fetch_result[:duration_ms]
+ )
+ raw = fetch_result[:body]
+
+ if fetch_result[:status] == :abort
+ Rails.logger.warn("[DSL] abort on #{new_url} (#{@current_wringer_status&.dig(:error_type)})")
+ return normalize_abort_result(raw, step: step)
+ end
+
+ raw
+ end
+
+ def resolve_url_only(code, arr, step: 'url')
+ raw = @dsl_binding.eval(sub(code, arr))
+ new_url = Dsl::Support::UrlResolver.extract(raw)
+ return new_url if new_url.present?
+
+ abort_update(
+ error: "Invalid URL resolved from #{raw.inspect}",
+ error_type: "InvalidURL",
+ step: step
+ )
+ end
+
+ def apply_html_result(html)
+ @html = html
+ @page = Nokogiri::HTML(@html, nil, Encoding::UTF_8.to_s)
+ @json = nil
+ Thread.current[:dsl_json] = nil
+ end
+
+ def apply_json_text_result(text)
+ @html = text
+ @page = Struct.new(:text).new(@html)
+ @json = nil
+ Thread.current[:dsl_json] = nil
+ end
+
+ def handle_url_step(code, arr, render_js: false, opts: @scrape_opts, step: 'url')
+ result = resolve_and_fetch_url(code, arr, render_js: render_js, opts: opts, step: step)
+ return result if abort_structure?(result)
+
+ apply_html_result(result)
+ arr
+ end
+
+ def execute_xpath(code)
+ page_status = ensure_page!(step: 'xpath')
+ return page_status if abort_structure?(page_status)
+
+ #@page.xpath(code).map(&:text)
+ # new default behavior
+ @page.xpath(code).map { |n| node_value(n).to_s.squish }.reject(&:blank?)
+ end
+
+ def halt_structure?(obj)
+ obj.is_a?(Array) &&
+ obj.length == 2 &&
+ obj.first.equal?(HALT)
+ end
+
+ # Rewrite DSL references into thread-locals
+ def sub(code, _)
+ code.to_s
+ .gsub('$array', 'Thread.current[:dsl_array]')
+ .gsub('$url', 'Thread.current[:dsl_url]')
+ .gsub('$json', 'Thread.current[:dsl_json]')
+ .gsub(/\$e\b/, 'Thread.current[:dsl_item]')
+ end
+
+ def ensure_page!(step: nil)
+ return :ok if @page
+
+ fetch_result = fetch_result_for(url: @url, render_js: @render_js, scrape_options: @scrape_opts)
+ @current_wringer_status =
+ (fetch_result[:wringer] || {}).merge(
+ duration_ms: fetch_result[:duration_ms]
+ )
+ raw = fetch_result[:body]
+
+ if fetch_result[:status] == :abort
+ return normalize_abort_result(raw, step: step)
+ end
+
+ @html = raw
+ @page = Nokogiri::HTML(@html, nil, Encoding::UTF_8.to_s)
+ :ok
+ end
+
+ def ensure_graph!(step: nil)
+ return :ok if @graph
+
+ fetch_result = fetch_result_for(url: @url, render_js: @render_js, scrape_options: @scrape_opts)
+ @current_wringer_status =
+ (fetch_result[:wringer] || {}).merge(
+ duration_ms: fetch_result[:duration_ms]
+ )
+ raw = fetch_result[:body]
+
+ if fetch_result[:status] == :abort
+ return normalize_abort_result(raw, step: step)
+ end
+
+ @graph = load_rdf_graph(raw, fetch_result)
+ :ok
+ end
+
+ def load_rdf_graph(body, fetch_result)
+ source_url = fetch_result[:final_url].presence || @url
+ headers = fetch_result[:headers] || {}
+ content_type = headers[:content_type] || headers["content_type"] || headers[:"content-type"] || headers["Content-Type"]
+ content_type = Array(content_type).first.to_s.presence
+ text = body.to_s
+ reader = RDF::Reader.for(content_type: content_type, file_name: source_url) { text[0, 1000] }
+ raise RDF::FormatError, "unknown RDF format for #{source_url}" unless reader
+
+ RDF::Graph.new do |graph|
+ reader.new(StringIO.new(text), base_uri: source_url) do |rdf|
+ graph << rdf
+ end
+ end
+ end
+
+ def abort_update(error:, error_type:, step: nil, source: "dsl_runner")
+ payload = {
+ error: error.to_s,
+ error_type: error_type.to_s,
+ source: source
+ }
+ payload[:step] = step if step.present?
+ ["abort_update", payload]
+ end
+
+ def normalize_abort_result(result, step: nil)
+ payload =
+ if abort_structure?(result)
+ result.last.respond_to?(:to_h) ? result.last.to_h.dup : {}
+ else
+ {}
+ end
+
+ error = payload[:error] || payload["error"] || "DSL runner abort"
+ error_type = payload[:error_type] || payload["error_type"] || "DslAbort"
+ normalized = payload.transform_keys { |k| k.respond_to?(:to_sym) ? k.to_sym : k }
+ effective_step = normalized[:step].presence || step
+ effective_source = normalized[:source].presence || "dsl_runner"
+
+ built = abort_update(
+ error: error,
+ error_type: error_type,
+ step: effective_step,
+ source: effective_source
+ ).last
+
+ # Preserve upstream metadata (retry/cache/signals/etc.) while enforcing the shared abort shape.
+ ["abort_update", built.merge(normalized.except(:error, :error_type, :step, :source))]
+ end
+
+ def fetch_result_for(url:, render_js:, scrape_options:)
+ options, log_context = sanitized_scrape_options(scrape_options)
+ if use_wringer_compatibility_client?(options)
+ wringer_client.fetch(url: url, render_js: render_js, scrape_options: options)
+ else
+ fetch_result_from_cache(url: url, render_js: render_js, scrape_options: options, log_context: log_context)
+ end
+ end
+
+ def fetch_result_from_cache(url:, render_js:, scrape_options:, log_context:)
+ website = scrape_options[:website] || scrape_options["website"]
+ website_id = scrape_options[:website_id] || scrape_options["website_id"] || log_context[:website_id]
+ fetch = Distillator::FetchCacheStore.fetch(
+ uri: url,
+ include_fragment: scrape_options.fetch(:include_fragment, true),
+ force_scrape: scrape_options.fetch(:force_scrape, false),
+ force_scrape_every_hrs: scrape_options[:force_scrape_every_hrs],
+ render_js: render_js,
+ mode: effective_distillator_mode(scrape_options),
+ use_phantomjs: scrape_options.fetch(:use_phantomjs, false),
+ absolute_src: scrape_options.fetch(:absolute_src, false),
+ json_post: scrape_options.fetch(:json_post, false),
+ website: website,
+ website_id: website_id,
+ agent: @agent,
+ use_wringer: method(:use_wringer),
+ safe_wringer_call: method(:safe_wringer_call),
+ logger: Rails.logger,
+ log_context: log_context
+ )
+ signals = (fetch.signals || {}).to_h.stringify_keys
+ hints = Array(fetch.hints)
+ body = fetch.body.presence || fetch.html
+ if fetch.respond_to?(:content_success?) && !fetch.content_success?
+ error_type = signals["blocking_issue_key"].presence || signals["primary_issue_key"].presence || "DistillatorContentFailure"
+ error_message = "Fetch content blocked by #{error_type}"
+ Rails.logger.warn(
+ {
+ event: "dsl.fetch.content_failure",
+ url: url,
+ error_type: error_type,
+ final_url: fetch.final_url.presence || url,
+ http_response_code: fetch.http_response_code,
+ cache_hit: fetch.cache_hit,
+ cache_write: fetch.cache_write,
+ cache_reason: fetch.cache_reason,
+ statement_id: log_context[:statement_id],
+ source_id: log_context[:source_id],
+ webpage_id: log_context[:webpage_id],
+ website_id: log_context[:website_id]
+ }
+ )
+ return {
+ status: :abort,
+ body: [
+ "abort_update",
+ {
+ error: error_message,
+ error_type: error_type,
+ source: "distillator_fetch_cache",
+ cache: fetch.respond_to?(:cache_policy) ? fetch.cache_policy : signals["cache"],
+ retry: fetch.respond_to?(:retry_policy) ? fetch.retry_policy : signals["retry"],
+ step: "url",
+ signals: signals,
+ hints: hints
+ }
+ ],
+ headers: fetch.headers || {},
+ final_url: fetch.final_url.presence || url,
+ redirect_chain: fetch.redirect_chain || [],
+ wringer: {
+ error_type: error_type,
+ source: "distillator_fetch_cache",
+ cache: fetch.respond_to?(:cache_policy) ? fetch.cache_policy : signals["cache"],
+ retry: fetch.respond_to?(:retry_policy) ? fetch.retry_policy : signals["retry"],
+ signals: signals,
+ hints: hints,
+ final_url: fetch.final_url.presence || url,
+ redirect_chain: fetch.redirect_chain || [],
+ fetch_path: fetch.fetch_path,
+ cache_hit: fetch.cache_hit,
+ cache_write: fetch.cache_write,
+ cache_reason: fetch.cache_reason,
+ uri_key: fetch.uri_key,
+ normalized_url: fetch.normalized_url
+ },
+ duration_ms: fetch.duration_ms,
+ http_code: fetch.http_response_code,
+ raw_body: body
+ }
+ end
+ abort_payload =
+ if body.is_a?(Array) && body.first == "abort_update" && body.second.is_a?(Hash)
+ body.second.with_indifferent_access
+ end
+
+ if abort_payload.present? || (body.blank? && signals["error_type"].present?)
+ error_type = abort_payload&.[](:error_type).presence || signals["error_type"]
+ error_message = abort_payload&.[](:error).presence || hints.first.presence || "Fetch cache returned no content"
+ return {
+ status: fetch.status,
+ body: if abort_payload.present?
+ body
+ else
+ [
+ "abort_update",
+ {
+ error: error_message,
+ error_type: error_type,
+ source: "distillator_fetch_cache",
+ cache: true,
+ step: "url"
+ }
+ ]
+ end,
+ headers: fetch.headers || {},
+ final_url: fetch.final_url.presence || url,
+ redirect_chain: fetch.redirect_chain || [],
+ wringer: {
+ error_type: error_type,
+ source: abort_payload&.[](:source).presence || "distillator_fetch_cache",
+ cache: abort_payload&.key?(:cache) ? abort_payload[:cache] : true,
+ retry: abort_payload&.[](:retry),
+ signals: signals,
+ hints: hints,
+ final_url: fetch.final_url.presence || url,
+ redirect_chain: fetch.redirect_chain || [],
+ fetch_path: fetch.fetch_path,
+ cache_hit: fetch.cache_hit,
+ cache_write: fetch.cache_write,
+ cache_reason: fetch.cache_reason,
+ uri_key: fetch.uri_key,
+ normalized_url: fetch.normalized_url
+ },
+ duration_ms: fetch.duration_ms,
+ http_code: fetch.http_response_code,
+ raw_body: body
+ }
+ end
+
+ {
+ status: fetch.status,
+ body: body,
+ headers: fetch.headers || {},
+ final_url: fetch.final_url.presence || url,
+ redirect_chain: fetch.redirect_chain || [],
+ wringer: {
+ cache: true,
+ signals: signals,
+ hints: hints,
+ final_url: fetch.final_url.presence || url,
+ redirect_chain: fetch.redirect_chain || [],
+ http_response_code: fetch.http_response_code,
+ fetch_path: fetch.fetch_path,
+ cache_hit: fetch.cache_hit,
+ cache_write: fetch.cache_write,
+ cache_reason: fetch.cache_reason,
+ uri_key: fetch.uri_key,
+ normalized_url: fetch.normalized_url
+ },
+ duration_ms: fetch.duration_ms,
+ http_code: fetch.http_response_code,
+ raw_body: body
+ }
+ end
+
+ def sanitized_scrape_options(scrape_options)
+ return [{}, {}] unless scrape_options.respond_to?(:deep_dup)
+
+ website = scrape_options[:website] || scrape_options["website"]
+ options = scrape_options.deep_dup.with_indifferent_access
+ log_context = options.delete(:log_context) || options.delete("log_context") || {}
+ normalized_options = options.to_h.symbolize_keys
+ normalized_options[:website] = website if website
+ [normalized_options, log_context.to_h.symbolize_keys]
+ end
+
+ def use_wringer_compatibility_client?(scrape_options)
+ options = scrape_options.respond_to?(:symbolize_keys) ? scrape_options.symbolize_keys : {}
+ Distillator::BooleanParam.parse(options[:wringer_compatibility] || options["wringer_compatibility"]) == true
+ end
+
+ def truthy?(value)
+ value == true || value.to_s == "true"
+ end
+
+ def effective_distillator_mode(scrape_options)
+ explicit =
+ scrape_options[:mode] ||
+ scrape_options["mode"] ||
+ @mode
+
+ return nil if explicit.blank?
+ normalized = explicit.to_s.strip.downcase
+ normalized = Distillator::FetchMode::ALIASES.fetch(normalized, normalized)
+ return nil unless Distillator::FetchMode::EXECUTION_MODES.include?(normalized)
+
+ normalized.to_sym
+ end
+
+ def use_wringer(u, rj, opt)
+ ApplicationController.helpers.use_wringer(u, rj, opt)
+ end
+
+ def safe_wringer_call(&blk)
+ ApplicationController.helpers.safe_wringer_call(&blk)
+ end
+
+ def wringer_client
+ @wringer_client ||= Dsl::Support::WringerClient.new(
+ agent: @agent,
+ render_js: @render_js,
+ scrape_options: @scrape_opts,
+ use_wringer: method(:use_wringer),
+ safe_wringer_call: method(:safe_wringer_call),
+ logger: Rails.logger
+ )
+ end
+
+ def node_value(n)
+ n.respond_to?(:value) ? n.value : n.text
+ end
+
+ def sanitize(*args)
+ ApplicationController.helpers.sanitize(*args)
+ end
+
+ def trace_preview(value)
+ value.is_a?(Array) ? value.flatten(1) : [value]
+ end
+
+ def trace_wringer_payload
+ wringer = @current_wringer_status.is_a?(Hash) ? @current_wringer_status.dup : {}
+ wringer = wringer.transform_keys { |k| k.respond_to?(:to_sym) ? k.to_sym : k }
+ wringer[:signals] = {} unless wringer[:signals].is_a?(Hash)
+ wringer[:hints] = [] unless wringer[:hints].is_a?(Array)
+ wringer[:inherited] = true if wringer.blank?
+ wringer[:inherited] = true if @current_wringer_status.blank?
+ wringer
+ end
+
+ def trace_probe_payload(probe_result)
+ return { skipped: true } unless probe_result.present?
+
+ {
+ result: probe_result,
+ ok: probe_result[:status].to_s == "ok"
+ }
+ end
+
+ def maybe_expand(url)
+ return url unless needs_expansion?(url)
+
+ Dsl::Network::UrlExpander.call(url, agent: @agent)
+ end
+
+ def extract_id(url)
+ Dsl::Identity::UrlIdentifier.call(url, @params)
+ end
+
+ def fallback_id(url)
+ Dsl::Identity::UrlFallback.id(url)
+ end
+
+ def execute_accumulate(code, arr)
+ return [] if arr.blank?
+
+ arr.flat_map do |item|
+ prev_item = Thread.current[:dsl_item]
+ prev_array = Thread.current[:dsl_array]
+ prev_url = Thread.current[:dsl_url]
+ prev_json = Thread.current[:dsl_json]
+
+ begin
+ Thread.current[:dsl_item] = item
+ Thread.current[:dsl_array] = [item]
+ Thread.current[:dsl_url] = item
+ Thread.current[:dsl_json] = nil
+
+ result = @dsl_binding.eval(sub(code, [item]))
+
+ result.is_a?(Array) ? result : [result]
+ ensure
+ Thread.current[:dsl_item] = prev_item
+ Thread.current[:dsl_array] = prev_array
+ Thread.current[:dsl_url] = prev_url
+ Thread.current[:dsl_json] = prev_json
+ end
+ end
+ end
+
+ def execute_make_uri(code, arr)
+ return [] if arr.blank?
+
+ # parse params (simple version)
+ params = parse_params(code)
+ prefix = params['prefix'] || 'default'
+
+ arr.map do |url|
+ next if url.blank?
+
+ id = extract_id_from_url(url)
+
+ next if id.blank?
+
+ "footlight:#{prefix}_#{id}"
+ end.compact.uniq
+ end
+
+ def snapshot_thread_locals
+ {
+ dsl_array: fetch_thread_local(:dsl_array),
+ dsl_url: fetch_thread_local(:dsl_url),
+ dsl_json: fetch_thread_local(:dsl_json)
+ }
+ end
+
+ def fetch_thread_local(key)
+ Thread.current.key?(key) ? Thread.current[key] : :__dsl_missing
+ end
+
+ def restore_thread_locals(previous)
+ restore_thread_local(:dsl_array, previous[:dsl_array])
+ restore_thread_local(:dsl_url, previous[:dsl_url])
+ restore_thread_local(:dsl_json, previous[:dsl_json])
+ end
+
+ def restore_thread_local(key, value)
+ if value == :__dsl_missing
+ # Thread#[] storage is cleared by assigning nil (Thread has no #delete).
+ Thread.current[key] = nil if Thread.current.key?(key)
+ else
+ Thread.current[key] = value
+ end
+ end
+ end
+ end
+end
diff --git a/app/services/dsl/dsl_algorithm_runner.rb b/app/services/dsl/dsl_algorithm_runner.rb
deleted file mode 100644
index 7feb5525..00000000
--- a/app/services/dsl/dsl_algorithm_runner.rb
+++ /dev/null
@@ -1,308 +0,0 @@
-# app/services/dsl/dsl_algorithm_runner.rb
-module Dsl
- class DslAlgorithmRunner
- StepTrace = Struct.new(
- :step,
- :type,
- :code,
- :input,
- :output,
- :error,
- :url_before,
- :url_after,
- :duration_ms,
- keyword_init: true
- )
-
- def initialize(ctx)
- @url = ctx[:url]
- @render_js = ctx[:render_js]
- @scrape_opts = ctx[:scrape_options] || {}
- @tracer = ctx[:tracer]
- @agent = Mechanize.new
- @agent.user_agent_alias = 'Mac Safari'
- @html = nil
- @page = nil
- @json = nil
- @graph = nil
- end
-
- def abort_structure?(obj)
- obj.is_a?(Array) &&
- obj.length == 2 &&
- obj.first == "abort_update" &&
- obj.last.is_a?(Hash)
- end
-
- def run(algorithm)
- results = []
-
- # reset thread-local DSL state for this run
- Thread.current[:dsl_array] = []
- Thread.current[:dsl_url] = @url
- Thread.current[:dsl_json] = nil
-
- @dsl_binding = binding
-
- steps = algorithm.split(';').map(&:strip).reject(&:empty?)
-
- steps.each_with_index do |raw, idx|
- prefix, code = raw.partition('=').values_at(0, 2)
- step_index = idx + 1
-
- input_copy = Marshal.load(Marshal.dump(results))
- url_before = @url
- start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-
- # === UPDATE THREAD-LOCALS BEFORE EVERY STEP ===
- Thread.current[:dsl_array] = results
- Thread.current[:dsl_url] = @url
- Thread.current[:dsl_json] = @json
-
- out = execute(prefix, code, results)
-
- # handle abort payload
- if abort_structure?(out)
- end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
- duration_ms = ((end_time - start_time) * 1000).round(1)
-
- @tracer.step(
- step: step_index,
- type: prefix,
- code: code,
- input: input_copy,
- output: [],
- error: out.last, # error message details
- url_before: url_before,
- url_after: @url,
- duration_ms: duration_ms
- )
-
- return out
- end
-
- end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
- duration_ms = ((end_time - start_time) * 1000).round(1)
-
- url_after = @url
- output = Array(out)
-
- @tracer.step(
- step: step_index,
- type: prefix,
- code: code,
- input: input_copy,
- output: output,
- error: nil,
- url_before: url_before,
- url_after: url_after,
- duration_ms: duration_ms
- )
-
- results = output
- rescue StandardError => e
- end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
- duration_ms = ((end_time - start_time) * 1000).round(1)
-
- @tracer.step(
- step: step_index,
- type: prefix,
- code: code,
- input: input_copy,
- output: [],
- error: e,
- url_before: url_before,
- url_after: @url,
- duration_ms: duration_ms
- )
- return ["abort_update", { error: e.message, error_type: e.class.to_s }]
- end
-
- results
- end
-
- private
-
- def execute(prefix, code, arr)
- case prefix
-
- when 'sparql'
- begin
- @graph ||= RDF::Graph.load(use_wringer(@url, @render_js, @scrape_opts))
- sparql = "PREFIX schema: select * where " + code
- rows = SPARQL.execute(sparql, @graph)
-
- if rows.count == 1
- [rows.first.answer.value]
- else
- rows.map { |r| r.answer.value }
- end
- rescue StandardError => e
- ["abort_update", { error: e.message, error_type: e.class.to_s }]
- end
-
- when 'url'
- new_url = @dsl_binding.eval(sub(code, arr))
- if new_url.nil? || !new_url.is_a?(String) || new_url.strip.empty?
- return ["abort_update", { error: "Invalid URL in DSL - url step: #{@url.inspect}", error_type: "StandardError" }]
- end
-
- @url = new_url
-
- raw = safe_wringer_call { @agent.get_file(use_wringer(@url, @render_js, @scrape_opts)) }
- return raw if abort_structure?(raw)
-
- @html = raw
- @page = Nokogiri::HTML(@html, nil, Encoding::UTF_8.to_s)
- arr
-
- when 'renderjs_url'
- new_url = @dsl_binding.eval(sub(code, arr))
- if new_url.nil? || !new_url.is_a?(String) || new_url.strip.empty?
- return ["abort_update", { error: "Invalid URL in DSL - renderjs_url step: #{@url.inspect}", error_type: "StandardError" }]
- end
-
- @url = new_url
-
- raw = safe_wringer_call { @agent.get_file(use_wringer(@url, true, @scrape_opts)) }
- return raw if abort_structure?(raw)
-
- @html = raw
- @page = Nokogiri::HTML(@html, nil, Encoding::UTF_8.to_s)
- arr
-
- when 'json_url'
- new_url = @dsl_binding.eval(sub(code, arr))
- if new_url.nil? || !new_url.is_a?(String) || new_url.strip.empty?
- return ["abort_update", { error: "Invalid URL in DSL - json_url step: #{@url.inspect}", error_type: "StandardError" }]
- end
-
- @url = new_url
-
- raw = safe_wringer_call { @agent.get_file(use_wringer(@url, @render_js, @scrape_opts)) }
- return raw if abort_structure?(raw)
-
- @html = raw
- Struct.new(:text).new(@html)
-
- when 'post_url'
- new_url = @dsl_binding.eval(sub(code, arr))
- if new_url.nil? || !new_url.is_a?(String) || new_url.strip.empty?
- return ["abort_update", { error: "Invalid URL in DSL - post_url step: #{@url.inspect}", error_type: "StandardError" }]
- end
-
- @url = new_url
-
- temp_opts = @scrape_opts.merge(json_post: true).merge(force_scrape_every_hrs: 1)
- data = @agent.get_file(use_wringer(@url, @render_js, temp_opts))
- @page = Nokogiri::HTML(data, nil, Encoding::UTF_8.to_s)
- arr
-
- when 'api'
- new_url = @dsl_binding.eval(sub(code, arr))
- if new_url.nil? || !new_url.is_a?(String) || new_url.strip.empty?
- return ["abort_update", { error: "Invalid URL in DSL - api step: #{@url.inspect}", error_type: "StandardError" }]
- end
-
- data = HTTParty.get(new_url)
- raise "API error #{data.code}" unless data.code.to_s.start_with?('2')
-
- JSON.parse(data.body)
-
- when 'xpath'
- ensure_page!
- @page.xpath(code).map(&:text)
-
- when 'xpath_sanitize'
- ensure_page!
- @page.xpath(code).map do |node|
- sanitize(node.to_s,
- tags: %w[h1 h2 h3 h4 h5 h6 p li ul ol strong em a i br],
- attributes: %w[href])
- end
-
- when 'if_xpath'
- ensure_page!
- nodes = @page.xpath(code)
- if nodes.blank?
- return []
- end
-
- nodes.map(&:text)
-
- when 'unless_xpath'
- ensure_page!
- nodes = @page.xpath(code)
- if nodes.present?
- return []
- end
-
- arr
-
- when 'css'
- ensure_page!
- @page.css(code).map(&:text)
-
- when 'json'
- ensure_page!
- text = @page.respond_to?(:text) ? @page.text : @html.to_s
- @json ||= JSON.parse(text)
- Thread.current[:dsl_json] = @json
- @dsl_binding.eval(sub(code, arr))
-
- when 'time_zone'
- ["time_zone: #{code}"]
-
- when 'ruby'
- # update thread-locals before eval
- Thread.current[:dsl_array] = arr
- Thread.current[:dsl_url] = @url
- Thread.current[:dsl_json] = @json
-
- result = @dsl_binding.eval(sub(code, arr))
-
- # sync back DSL state
- updated_arr = Thread.current[:dsl_array]
- @url = Thread.current[:dsl_url]
- @json = Thread.current[:dsl_json]
-
- updated_arr || result
-
- else
- raise "Missing DSL prefix: #{prefix}=#{code}"
- end
- end
-
- # Rewrite DSL references into thread-locals
- def sub(code, _)
- code.to_s
- .gsub('$array', 'Thread.current[:dsl_array]')
- .gsub('$url', 'Thread.current[:dsl_url]')
- .gsub('$json', 'Thread.current[:dsl_json]')
- end
-
- def ensure_page!
- return if @page
-
- raw = safe_wringer_call { @agent.get_file(use_wringer(@url, @render_js, @scrape_opts)) }
- if abort_structure?(raw)
- raise StandardError, raw.last[:error]
- end
-
- @html = raw
- @page = Nokogiri::HTML(@html, nil, Encoding::UTF_8.to_s)
- end
-
- def use_wringer(u, rj, opt)
- ApplicationController.helpers.use_wringer(u, rj, opt)
- end
-
- def safe_wringer_call(&blk)
- ApplicationController.helpers.safe_wringer_call(&blk)
- end
-
- def sanitize(*args)
- ApplicationController.helpers.sanitize(*args)
- end
- end
-end
\ No newline at end of file
diff --git a/app/services/dsl/dsl_null_tracer.rb b/app/services/dsl/dsl_null_tracer.rb
deleted file mode 100644
index e02a0567..00000000
--- a/app/services/dsl/dsl_null_tracer.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-# app/services/dsl/dsl_null_tracer.rb
-module Dsl
- class DslNullTracer
- def step(**); end
- def to_h = {}
- end
-end
diff --git a/app/services/dsl/dsl_runner.rb b/app/services/dsl/dsl_runner.rb
deleted file mode 100644
index 82016a6a..00000000
--- a/app/services/dsl/dsl_runner.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# app/services/dsl/dsl_runner.rb
-module Dsl
- class DslRunner
- def self.process_algorithm(algorithm:, url:, trace: false, trace_opts: {})
-
- # 🎯 Decide tracer type:
- collector = trace ? Dsl::DslTraceCollector.new(**trace_opts) : Dsl::DslNullTracer.new
-
- # 📌 Build base context
- ctx = Dsl::DslContext.new(
- url: url,
- array: [],
- tracer: collector
- )
-
- # 🛠 Run the internal runner
- result = new(ctx: ctx).run(algorithm)
-
- if trace
- [result, collector.to_h]
- else
- result
- end
- end
-
- def initialize(ctx:)
- @ctx = ctx
- end
-
- def run(algorithm)
- # Delegate to your existing core runner
- Dsl::DslAlgorithmRunner.new(@ctx).run(algorithm)
- end
- end
-end
\ No newline at end of file
diff --git a/app/services/dsl/dsl_trace_collector.rb b/app/services/dsl/dsl_trace_collector.rb
deleted file mode 100644
index 751bd4ed..00000000
--- a/app/services/dsl/dsl_trace_collector.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-# app/services/dsl/dsl_trace_collector.rb
-module Dsl
- class DslTraceCollector
- attr_reader :events
-
- def initialize
- @events = []
- end
-
- def step(
- step:,
- type:,
- code:,
- input:,
- output:,
- error: nil,
- url_before: nil,
- url_after: nil,
- duration_ms: nil
- )
- @events << {
- step: step,
- type: type,
- code: code,
- input_preview: input,
- output_preview: output,
- error_class: error.nil? ? nil : error.class.to_s,
- error_message: error.nil? ? nil : error.to_s,
- url_before: url_before,
- url_after: url_after,
- duration_ms: duration_ms
- }
- end
-
- def to_h
- @events
- end
- end
-end
\ No newline at end of file
diff --git a/app/services/dsl/identity/url_fallback.rb b/app/services/dsl/identity/url_fallback.rb
new file mode 100644
index 00000000..dc487d1d
--- /dev/null
+++ b/app/services/dsl/identity/url_fallback.rb
@@ -0,0 +1,10 @@
+# app/services/dsl/identity/url_fallback.rb
+module Dsl
+ module Identity
+ class UrlFallback
+ def self.id(url)
+ "u#{Digest::MD5.hexdigest(url.to_s)[0..10]}"
+ end
+ end
+ end
+end
diff --git a/app/services/dsl/identity/url_identifier.rb b/app/services/dsl/identity/url_identifier.rb
new file mode 100644
index 00000000..7081dba3
--- /dev/null
+++ b/app/services/dsl/identity/url_identifier.rb
@@ -0,0 +1,91 @@
+# app/services/dsl/identity/url_identifier.rb
+module Dsl
+ module Identity
+ class UrlIdentifier
+ def self.call(url, params = {})
+ new(url, params).call
+ end
+
+ def initialize(url, params)
+ @url = url
+ @params = params
+ end
+
+ def call
+ uri = begin
+ URI.parse(@url)
+ rescue StandardError
+ nil
+ end
+ return clean(raw_last_segment(@url)) unless uri
+
+ # query first
+ q = Rack::Utils.parse_query(uri.query)
+ return q["id"] || q["eventId"] || q["eid"] if q["id"] || q["eventId"] || q["eid"]
+ domain_id = extract_from_domain(uri)
+ return domain_id if domain_id.present?
+
+ segments = uri.path.split('/').reject(&:blank?)
+ return nil if segments.empty?
+
+ extract_from_path(segments)
+ end
+
+ private
+
+ def clean(id)
+ id.gsub(/\?.*/, "")
+ .gsub(/\.html?$/, "")
+ end
+
+ def extract_from_path(segments)
+ candidate = segments.reverse.find do |seg|
+ valid_identifier_segment?(seg)
+ end
+ return clean(candidate) if candidate.present?
+
+ last = segments.last.to_s
+ return nil if blacklist.include?(last.downcase)
+
+ clean(last)
+ end
+
+ def extract_from_domain(uri)
+ host = uri.host.to_s
+ path = uri.path.to_s
+
+ if match_domain?(host, "lepointdevente")
+ match = path[%r{/billets/([^/]+)}, 1].to_s
+ return clean(match) if match.present?
+ elsif match_domain?(host, "eventbrite")
+ match = path[/-(\d+)(?:\D|$)/, 1].to_s
+ return clean(match) if match.present?
+ elsif match_domain?(host, "ticketweb")
+ segments = path.split("/").reject(&:blank?)
+ numeric = segments.reverse.find { |seg| seg.match?(/\A\d+\z/) }
+ return clean(numeric.to_s) if numeric.present?
+ end
+
+ nil
+ end
+
+ def match_domain?(host, name)
+ host == name || host.start_with?("#{name}.") || host.include?(".#{name}.")
+ end
+
+ def blacklist
+ %w[show details index event events ticket tickets]
+ end
+
+ def valid_identifier_segment?(seg)
+ normalized = seg.to_s.downcase
+ (seg.to_s.match?(/\d/) || seg.to_s.length > 5) &&
+ !blacklist.include?(normalized)
+ end
+
+ def raw_last_segment(url)
+ url.to_s.split("?").first.to_s.split("/").reject(&:blank?).last.to_s
+ end
+ end
+ end
+end
diff --git a/app/services/dsl/instructions/make_uri.rb b/app/services/dsl/instructions/make_uri.rb
new file mode 100644
index 00000000..0bdfe048
--- /dev/null
+++ b/app/services/dsl/instructions/make_uri.rb
@@ -0,0 +1,96 @@
+# app/services/dsl/instructions/make_uri.rb
+module Dsl
+ module Instructions
+ class MakeUri
+ def self.call(arr, code, context:)
+ new(arr, code, context).call
+ end
+
+ def initialize(arr, code, context)
+ @arr = arr || []
+ @params = parse_params(code)
+ @context = context
+ @agent = context&.instance_variable_get(:@agent)
+ end
+
+ def call
+ prefix = @params["prefix"] || default_prefix
+
+ @arr.map do |url|
+ build_uri(url, prefix) || fallback_uri(prefix, url)
+ end
+ end
+
+ private
+
+ def build_uri(url, prefix)
+ return nil if url.blank?
+
+ final_url = expand_url_if_needed(url)
+ id = extract_best_id(final_url)
+ id = fallback_id(final_url) if id.blank?
+
+ "footlight:#{prefix}_#{normalize(id)}"
+ end
+
+ def expand_url_if_needed(url)
+ return url unless needs_expansion?(url)
+
+ Dsl::Network::UrlExpander.call(url, context: @context)
+ end
+
+ def extract_best_id(url)
+ Dsl::Identity::UrlIdentifier.call(url, @params)
+ end
+
+ def fallback_id(url)
+ Dsl::Identity::UrlFallback.id(url)
+ end
+
+ def fallback_uri(prefix, url)
+ "footlight:#{prefix}_#{normalize(fallback_id(url))}"
+ end
+
+ def default_prefix
+ seed =
+ if @context.respond_to?(:[])
+ @context[:seedurl] || @context["seedurl"]
+ end
+
+ normalize(seed.presence || "default")
+ end
+
+ def parse_params(code)
+ return {} if code.blank?
+
+ code.to_s.split(",").each_with_object({}) do |fragment, memo|
+ key, value = fragment.split("=", 2).map { |part| part.to_s.strip }
+ next if key.blank?
+
+ memo[key] =
+ case value&.downcase
+ when "true" then true
+ when "false" then false
+ else value
+ end
+ end
+ end
+
+ def needs_expansion?(url)
+ return true if @params["expand"] == true
+ return false if @params["expand"] == false
+
+ uri = URI.parse(url) rescue nil
+ return false unless uri
+
+ uri.host =~ /bit\.ly|t\.co|tinyurl|lpdv\.co/
+ end
+
+ def normalize(id)
+ id.to_s.downcase
+ .gsub(/[^a-z0-9]+/, "-").squeeze("-")
+ .gsub(/^-|-$/, "")
+ end
+ end
+ end
+end
diff --git a/app/services/dsl/network/url_expander.rb b/app/services/dsl/network/url_expander.rb
new file mode 100644
index 00000000..bde0647d
--- /dev/null
+++ b/app/services/dsl/network/url_expander.rb
@@ -0,0 +1,23 @@
+# app/services/dsl/network/url_expander.rb
+module Dsl
+ module Network
+ class UrlExpander
+ def self.call(url, context: nil, agent: nil)
+ resolved_agent = agent || context&.instance_variable_get(:@agent)
+ new(url, resolved_agent).call
+ end
+
+ def initialize(url, agent)
+ @url = url
+ @agent = agent
+ end
+
+ def call
+ response = @agent.get(@url)
+ response.uri.to_s
+ rescue StandardError
+ @url
+ end
+ end
+ end
+end
diff --git a/app/services/dsl/dsl_content_fetcher.rb b/app/services/dsl/parsing/content_fetcher.rb
similarity index 68%
rename from app/services/dsl/dsl_content_fetcher.rb
rename to app/services/dsl/parsing/content_fetcher.rb
index a9660a63..5564ea63 100644
--- a/app/services/dsl/dsl_content_fetcher.rb
+++ b/app/services/dsl/parsing/content_fetcher.rb
@@ -1,6 +1,7 @@
-# app/services/dsl/dsl_content_fetcher.rb
+# app/services/dsl/parsing/content_fetcher.rb
module Dsl
- class DslContentFetcher
+ module Parsing
+ class ContentFetcher
def initialize(url:, render_js: false)
@url = url
@render_js = render_js
@@ -10,5 +11,6 @@ def fetch
# Mechanize + Wringer logic goes here
# returns the raw HTML/JSON string
end
+ end
end
-end
\ No newline at end of file
+end
diff --git a/app/services/dsl/dsl_content_parser.rb b/app/services/dsl/parsing/content_parser.rb
similarity index 94%
rename from app/services/dsl/dsl_content_parser.rb
rename to app/services/dsl/parsing/content_parser.rb
index 8b2f35ac..3ff7d2ae 100644
--- a/app/services/dsl/dsl_content_parser.rb
+++ b/app/services/dsl/parsing/content_parser.rb
@@ -1,6 +1,7 @@
-# app/services/dsl/dsl_content_parser.rb
+# app/services/dsl/parsing/content_parser.rb
module Dsl
- class DslContentParser
+ module Parsing
+ class ContentParser
def initialize(html: nil)
@html = html
@page = Nokogiri::HTML(html) if html
@@ -70,5 +71,6 @@ def evaluate_ruby_expr(code, arr)
binding.eval(code.to_s.sub('$array', 'array'))
end
+ end
end
end
diff --git a/app/services/dsl/pipeline_diagnosis.rb b/app/services/dsl/pipeline_diagnosis.rb
new file mode 100644
index 00000000..17b4de65
--- /dev/null
+++ b/app/services/dsl/pipeline_diagnosis.rb
@@ -0,0 +1,357 @@
+module Dsl
+ class PipelineDiagnosis
+ def self.call(steps:, metrics:, wringer: {}, statement_status: nil)
+ normalized_metrics = metrics.respond_to?(:to_h) ? metrics.to_h : {}
+
+ if normalized_metrics.key?(:extraction) || normalized_metrics.key?("extraction")
+ extraction = normalized_metrics[:extraction] || normalized_metrics["extraction"]
+ normalized_metrics[:extraction_attempted] = true
+ normalized_metrics[:extraction_empty] = extraction.to_i == 0
+ end
+
+ if normalized_metrics.key?(:navigation) || normalized_metrics.key?("navigation")
+ navigation = normalized_metrics[:navigation] || normalized_metrics["navigation"]
+ if navigation.to_i > 0
+ normalized_metrics[:has_navigation] = true
+ normalized_metrics[:suspicious_navigation] = true if normalized_metrics[:extraction_empty]
+ end
+ end
+
+ new(
+ steps: steps,
+ metrics: normalized_metrics,
+ wringer: wringer,
+ statement_status: statement_status
+ ).result
+ end
+
+ def initialize(metrics:, wringer: {}, steps: nil, statement_status: nil)
+ @metrics = normalize_hash(metrics)
+ @wringer = normalize_hash(wringer)
+ @statement_status = statement_status.to_s
+ wringer_signals = normalize_hash(@wringer[:signals])
+ wringer_hints = Array(@wringer[:hints]).map(&:to_s)
+ @wringer_signals = wringer_signals
+ @wringer_hints = wringer_hints
+ @steps = normalize_steps(steps)
+ end
+
+ def result
+ # Rule priority (highest to lowest):
+ # 1) pipeline execution error
+ # 2) Wringer/fetch-layer failure
+ # 3) data loss
+ # 4) suspicious navigation
+ # 5) extraction failure
+ # 6) healthy fallback
+ #
+ # Wringer failures intentionally override navigation/extraction symptoms because
+ # upstream fetch instability can manifest as downstream extraction emptiness.
+ return error_diagnosis if @metrics[:error_present]
+ return wringer_diagnosis if wringer_failure?
+ return data_loss_diagnosis if @metrics[:data_loss]
+ return navigation_diagnosis if @metrics[:suspicious_navigation]
+ if @statement_status == "missing" && extraction_failure?
+ return {
+ status: :warning,
+ category: :empty_extraction,
+ message: "No data extracted despite successful pipeline execution."
+ }
+ end
+ return extraction_diagnosis if extraction_failure?
+
+ healthy_diagnosis
+ end
+
+ private
+
+ def normalize_hash(value)
+ hash_value = value.respond_to?(:to_h) ? value.to_h : {}
+ hash_value.is_a?(Hash) ? hash_value.with_indifferent_access : {}.with_indifferent_access
+ end
+
+ def wringer_failure?
+ return true if @wringer[:unreachable] || @wringer[:received_404] || @wringer[:system_error]
+ return true if @wringer_signals[:network_status] == "failed"
+ return true if wringer_error_type.present?
+
+ false
+ end
+
+ def extraction_failure?
+ extraction_attempted? && @metrics[:extraction_empty]
+ end
+
+ def error_diagnosis
+ if wringer_specific_message(wringer_error_type).present?
+ return wringer_diagnosis
+ end
+
+ {
+ status: :error,
+ category: :error,
+ message: "Pipeline reported an execution error.",
+ suggested_action: :investigate,
+ source: "dsl",
+ error_type: nil,
+ step: @metrics[:failure_step],
+ pipeline_category: "unknown",
+ details: {
+ failure_step: @metrics[:failure_step]
+ }
+ }
+ end
+
+ def wringer_diagnosis
+ if wringer_specific_message(wringer_error_type).present?
+ message = wringer_specific_message(wringer_error_type)
+ message += wringer_context_suffix(@wringer)
+
+ return {
+ status: :error,
+ category: :wringer_failure,
+ message: message,
+ suggested_action: :retry,
+ source: "wringer",
+ error_type: wringer_error_type,
+ step: @metrics[:failure_step],
+ pipeline_category: "fetch",
+ details: { wringer: @wringer }
+ }
+ end
+
+ if @wringer_signals[:network_status] == "failed"
+ message =
+ if @wringer_hints.include?("timeout")
+ "Network timeout occurred during fetch."
+ elsif @wringer_hints.include?("empty_body")
+ "Fetched page returned empty content."
+ else
+ "Network request failed before extraction."
+ end
+ message += wringer_context_suffix(@wringer)
+
+ return {
+ status: :error,
+ category: :wringer_failure,
+ message: message,
+ suggested_action: :retry,
+ source: "wringer",
+ error_type: wringer_error_type,
+ step: @metrics[:failure_step],
+ pipeline_category: "fetch",
+ details: { wringer: @wringer }
+ }
+ end
+
+ if @wringer[:received_404]
+ message = "Wringer returned 404 for the requested resource."
+ message += wringer_context_suffix(@wringer)
+
+ {
+ status: :error,
+ category: :wringer_failure,
+ message: message,
+ suggested_action: :check_url,
+ source: "wringer",
+ error_type: wringer_error_type,
+ step: @metrics[:failure_step],
+ pipeline_category: "fetch",
+ details: { wringer: @wringer }
+ }
+ elsif @wringer[:unreachable]
+ message = "Wringer appears unreachable during pipeline execution."
+ message += wringer_context_suffix(@wringer)
+
+ {
+ status: :error,
+ category: :wringer_failure,
+ message: message,
+ suggested_action: :retry,
+ source: "wringer",
+ error_type: wringer_error_type,
+ step: @metrics[:failure_step],
+ pipeline_category: "fetch",
+ details: { wringer: @wringer }
+ }
+ else
+ message = "Wringer reported a system-level failure."
+ message += wringer_context_suffix(@wringer)
+
+ {
+ status: :error,
+ category: :wringer_failure,
+ message: message,
+ suggested_action: :check_wringer,
+ source: "wringer",
+ error_type: wringer_error_type,
+ step: @metrics[:failure_step],
+ pipeline_category: "fetch",
+ details: { wringer: @wringer }
+ }
+ end
+ end
+
+ def data_loss_diagnosis
+ {
+ status: :warning,
+ category: :data_loss,
+ message: "Pipeline output lost data between steps.",
+ suggested_action: :investigate,
+ source: "dsl",
+ error_type: nil,
+ step: @metrics[:failure_step],
+ pipeline_category: "unknown",
+ details: {
+ failure_step: @metrics[:failure_step],
+ recovered_after_loss: @metrics[:recovered_after_loss]
+ }
+ }
+ end
+
+ def navigation_diagnosis
+ message = "Navigation succeeded, but extraction after navigation produced no data."
+ message += wringer_context_suffix(@wringer)
+
+ {
+ status: :warning,
+ category: :navigation_failure,
+ message: message,
+ suggested_action: :check_url,
+ source: "dsl",
+ error_type: nil,
+ step: @metrics[:failure_step],
+ pipeline_category: "navigation",
+ details: {
+ has_navigation: has_navigation?,
+ extraction_attempted: extraction_attempted?
+ }
+ }
+ end
+
+ def extraction_diagnosis
+ message = extraction_message
+ message += wringer_context_suffix(@wringer)
+
+ {
+ status: :warning,
+ category: :extraction_failure,
+ message: message,
+ suggested_action: :check_xpath,
+ source: "dsl",
+ error_type: nil,
+ step: @metrics[:failure_step],
+ pipeline_category: "extraction",
+ details: {
+ extraction_attempted: extraction_attempted?,
+ extraction_empty: @metrics[:extraction_empty]
+ }
+ }
+ end
+
+ def healthy_diagnosis
+ {
+ status: :ok,
+ category: :healthy,
+ message: "Pipeline metrics look healthy.",
+ suggested_action: :none,
+ source: "dsl",
+ error_type: nil,
+ step: @metrics[:failure_step],
+ pipeline_category: "unknown",
+ details: { failure_step: @metrics[:failure_step] }
+ }
+ end
+
+ def wringer_error_type
+ @wringer[:error_type].to_s.presence
+ end
+
+ def wringer_specific_message(error_type)
+ case error_type.to_s
+ when "WringerFetchError"
+ "Failed to fetch page (network or upstream service error). No data could be retrieved."
+ when "WringerSkip"
+ "Request was skipped by upstream policy."
+ when "WringerUnsupportedAction"
+ "Upstream service returned an unsupported control action."
+ end
+ end
+
+ def normalize_steps(steps)
+ Array(steps).map do |step|
+ source = step.respond_to?(:to_h) ? step.to_h : step
+ source.is_a?(Hash) ? source.with_indifferent_access : {}.with_indifferent_access
+ end
+ end
+
+ def extract_steps
+ @steps.select { |step| step_primitive(step) == :extract }
+ end
+
+ def navigate_steps
+ @steps.select { |step| step_primitive(step) == :navigate }
+ end
+
+ def extraction_attempted?
+ return @metrics[:extraction_attempted] if @steps.empty?
+
+ extract_steps.any?
+ end
+
+ def has_navigation?
+ return @metrics[:has_navigation] if @steps.empty?
+
+ navigate_steps.any?
+ end
+
+ def step_primitive(step)
+ primitive = step[:primitive]
+ return primitive.to_sym if primitive.present?
+
+ infer_primitive_from_type(step[:type])
+ end
+
+ def infer_primitive_from_type(type)
+ value = type.to_s
+
+ return :branch if value.start_with?("if_xpath")
+ return :extract if value.include?("xpath")
+ return :navigate if value.include?("url")
+ return :transform if value.include?("ruby")
+ return :transform if value.include?("sparql")
+
+ :unknown
+ end
+
+ def extraction_message
+ if has_navigation? && extract_steps.any?
+ "Extraction failed after navigation; no data was produced."
+ else
+ "Extraction failed; no data was produced."
+ end
+ end
+
+ def wringer_context_suffix(wringer)
+ return "" unless wringer.is_a?(Hash)
+
+ parts = []
+
+ if wringer[:error_type].present?
+ parts << "error: #{wringer[:error_type]}"
+ end
+
+ if wringer[:duration_ms].to_i > 1500
+ parts << "slow response (#{wringer[:duration_ms]}ms)"
+ end
+
+ if wringer[:redirect_chain].is_a?(Array) && wringer[:redirect_chain].size > 1
+ parts << "after redirect"
+ end
+
+ return "" if parts.empty?
+
+ " (#{parts.join(', ')})"
+ end
+ end
+end
diff --git a/app/services/dsl/pipeline_evaluator.rb b/app/services/dsl/pipeline_evaluator.rb
new file mode 100644
index 00000000..d33c4a15
--- /dev/null
+++ b/app/services/dsl/pipeline_evaluator.rb
@@ -0,0 +1,148 @@
+module Dsl
+ class PipelineEvaluator
+ DEFAULT_WRINGER = {
+ unreachable: false,
+ received_404: false,
+ system_error: false,
+ policy_action: nil
+ }.freeze
+
+ def self.evaluate(event:, wringer: nil)
+ new(event: event, wringer: wringer).evaluate
+ end
+
+ def initialize(event:, wringer: nil)
+ @event = event
+ @wringer = normalize_wringer(wringer)
+ end
+
+ def evaluate
+ steps = pipeline_steps
+ metrics = Dsl::PipelineInterpreter.new(steps).metrics
+ diagnosis = Dsl::PipelineDiagnosis.new(metrics: metrics, wringer: effective_wringer, steps: steps).result
+
+ {
+ metrics: metrics,
+ diagnosis: diagnosis
+ }
+ end
+
+ private
+
+ def pipeline_steps
+ step_counter = 0
+
+ event_statements.flat_map do |statement|
+ step_types = extract_step_types(statement.source&.algorithm_value)
+ next [] if step_types.empty?
+
+ output = normalize_output(statement.cache)
+ error = statement_error(statement)
+
+ step_types.map do |step_type|
+ step_counter += 1
+ {
+ step: step_counter,
+ type: step_type,
+ primitive: infer_primitive(step_type),
+ output: output,
+ error: error
+ }.compact
+ end
+ end
+ end
+
+ def event_statements
+ uri = event_identifier
+ return Statement.none if uri.blank?
+
+ Statement
+ .includes(:source, :webpage)
+ .where(webpages: { rdf_uri: uri })
+ .where(selected_individual: true)
+ .order(:id)
+ end
+
+ def event_identifier
+ if @event.respond_to?(:rdf_uri)
+ @event.rdf_uri
+ else
+ @event.to_s
+ end
+ end
+
+ def infer_step_type(algorithm_value)
+ expression = algorithm_value.to_s.strip
+ return "unknown" if expression.blank?
+
+ left_side = expression.split("=").first.to_s.strip.downcase
+ left_side.presence || "unknown"
+ end
+
+ def extract_step_types(algorithm_value)
+ expression = algorithm_value.to_s
+ return [] if expression.blank?
+
+ expression.split(";").map(&:strip).reject(&:blank?).filter_map do |segment|
+ segment.split("=", 2).first.to_s.strip.downcase.presence
+ end
+ end
+
+ def infer_primitive(type)
+ t = type.to_s
+
+ return :branch if t.start_with?("if_xpath")
+ return :extract if t.include?("xpath")
+ return :navigate if t.include?("url")
+ return :transform if t.include?("ruby")
+ return :transform if t.include?("sparql")
+
+ :unknown
+ end
+
+ def normalize_output(value)
+ string_value = value.to_s.strip
+ return nil if string_value.empty?
+
+ if string_value.start_with?("[") && string_value.end_with?("]")
+ begin
+ parsed = JSON.parse(string_value)
+ return parsed if parsed.is_a?(Array)
+ rescue JSON::ParserError
+ # Keep the original value when cache is not valid JSON.
+ end
+ end
+
+ value
+ end
+
+ def statement_error(statement)
+ return nil unless statement.is_a?(Statement)
+ return "statement_problem" if statement.status.to_s == "problem"
+ return "statement_error_cache" if statement.cache.to_s.downcase.include?("error:")
+
+ nil
+ end
+
+ def normalize_wringer(value)
+ return {} unless value.respond_to?(:to_h)
+
+ hash = value.to_h
+ hash.is_a?(Hash) ? hash.symbolize_keys : {}
+ end
+
+ def effective_wringer
+ DEFAULT_WRINGER.merge(@wringer.slice(
+ :unreachable,
+ :received_404,
+ :system_error,
+ :policy_action,
+ :error_type,
+ :signals,
+ :hints,
+ :final_url,
+ :redirect_chain
+ ))
+ end
+ end
+end
diff --git a/app/services/dsl/pipeline_interpreter.rb b/app/services/dsl/pipeline_interpreter.rb
new file mode 100644
index 00000000..8e882389
--- /dev/null
+++ b/app/services/dsl/pipeline_interpreter.rb
@@ -0,0 +1,200 @@
+module Dsl
+ class PipelineInterpreter
+ def initialize(steps)
+ @steps = Array(steps).map do |step|
+ source = step.respond_to?(:to_h) ? step.to_h : step
+ source.is_a?(Hash) ? source.with_indifferent_access : {}
+ end
+ end
+
+ def metrics
+ metric_values = {
+ # Truth signals: these report what was observed in the pipeline.
+ error_present: error_present?,
+ data_loss: data_loss?,
+ extraction_empty: extraction_empty?,
+ extraction_attempted: extraction_attempted?,
+ # Pre-diagnosis fetch-layer suspicion signal (Wringer wiring comes later).
+ suspicious_navigation: suspicious_navigation?,
+ # Best-effort location metadata; may be nil when no step number is available.
+ failure_step: failure_step,
+ recovered_after_loss: recovered_after_loss?,
+ steps_count: @steps.size,
+ final_empty: final_empty?,
+ has_navigation: has_navigation?
+ }
+
+ enforce_consistency(metric_values)
+ end
+
+ private
+
+ def outputs
+ Array(@steps).map do |step|
+ step.is_a?(Hash) ? step[:output] : nil
+ end
+ end
+
+ def error_present?
+ first_error_step.present?
+ end
+
+ def data_loss?
+ transition_loss_kinds.any?
+ end
+
+ def extraction_empty?
+ return false unless extraction_attempted?
+
+ final_empty?
+ end
+
+ def extraction_attempted?
+ @steps.any? { |step| step_primitive(step) == :extract }
+ end
+
+ def suspicious_navigation?
+ return false unless has_navigation?
+
+ navigation_index = @steps.find_index { |step| step_primitive(step) == :navigate }
+ return false if navigation_index.nil?
+
+ post_navigation_steps = @steps[(navigation_index + 1)..] || []
+ extraction_steps = post_navigation_steps.take_while do |step|
+ step_primitive(step) == :extract
+ end
+
+ return false if extraction_steps.empty?
+
+ extraction_steps.all? { |step| blank?(step[:output]) }
+ end
+
+ def failure_step
+ first_error = first_error_step
+ return extract_step_number(first_error) if first_error
+
+ return nil unless extraction_attempted?
+ return nil unless final_empty?
+
+ extract_step_number(@steps.last)
+ end
+
+ def recovered_after_loss?
+ lost = false
+ outputs.each_cons(2).any? do |prev, curr|
+ lost ||= transition_loss_kind(prev, curr).present?
+ lost && transition_recovered?(prev, curr)
+ end
+ end
+
+ def final_empty?
+ blank?(outputs.last)
+ end
+
+ def has_navigation?
+ @steps.any? { |step| step_primitive(step) == :navigate }
+ end
+
+ def step_type(step)
+ step.is_a?(Hash) ? step[:type].to_s : ""
+ end
+
+ def step_primitive(step)
+ return :unknown unless step.is_a?(Hash)
+
+ (step[:primitive]&.to_sym) || infer_primitive_from_type(step[:type])
+ end
+
+ def infer_primitive_from_type(type)
+ t = type.to_s
+
+ return :branch if t.start_with?("if_xpath")
+ return :extract if t.include?("xpath")
+ return :navigate if t.include?("url")
+ return :transform if t.include?("ruby")
+ return :transform if t.include?("sparql")
+
+ :unknown
+ end
+
+ def blank?(value)
+ normalized = normalize_output(value)
+ normalized.nil? || (normalized.respond_to?(:empty?) && normalized.empty?)
+ end
+
+ def extract_step_number(step)
+ return nil unless step.is_a?(Hash)
+
+ step[:step].presence
+ end
+
+ def first_error_step
+ @steps.find { |s| s[:error].present? }
+ end
+
+ def first_loss_step
+ @steps.each_cons(2) do |prev, curr|
+ prev_output = prev.is_a?(Hash) ? prev[:output] : nil
+ curr_output = curr.is_a?(Hash) ? curr[:output] : nil
+ return curr if transition_loss_kind(prev_output, curr_output).present?
+ end
+
+ nil
+ end
+
+ def transition_loss_kind(previous, current)
+ previous_normalized = normalize_output(previous)
+ current_normalized = normalize_output(current)
+
+ return :full_loss if present_normalized?(previous_normalized) && blank_normalized?(current_normalized)
+ return :partial_loss if previous_normalized.is_a?(Array) &&
+ current_normalized.is_a?(Array) &&
+ current_normalized.size < previous_normalized.size
+
+ nil
+ end
+
+ def transition_loss_kinds
+ outputs.each_cons(2).map do |prev, curr|
+ transition_loss_kind(prev, curr)
+ end.compact
+ end
+
+ def transition_recovered?(previous, current)
+ previous_normalized = normalize_output(previous)
+ current_normalized = normalize_output(current)
+
+ return false if blank_normalized?(current_normalized)
+ return true if blank_normalized?(previous_normalized) && present_normalized?(current_normalized)
+
+ previous_normalized.is_a?(Array) &&
+ current_normalized.is_a?(Array) &&
+ current_normalized.size > previous_normalized.size
+ end
+
+ def normalize_output(value)
+ return nil if value.nil?
+ return nil if value.is_a?(String) && value.strip.empty?
+ return value.compact if value.is_a?(Array)
+
+ value
+ end
+
+ def blank_normalized?(value)
+ value.nil? || (value.respond_to?(:empty?) && value.empty?)
+ end
+
+ def present_normalized?(value)
+ !blank_normalized?(value)
+ end
+
+ def extraction_xpath_steps
+ @steps.select { |step| step[:type].to_s.include?("xpath") }
+ end
+
+ def enforce_consistency(metric_values)
+ # Keep truth signals untouched; failure_step is best-effort location metadata.
+ metric_values
+ end
+ end
+end
diff --git a/app/services/dsl/semantic_interpreter.rb b/app/services/dsl/semantic_interpreter.rb
new file mode 100644
index 00000000..842d0092
--- /dev/null
+++ b/app/services/dsl/semantic_interpreter.rb
@@ -0,0 +1,133 @@
+module Dsl
+ class SemanticInterpreter
+ def annotate(steps)
+ previous_output = nil
+
+ Array(steps).map do |step|
+ source = step.respond_to?(:to_h) ? step.to_h : step
+ normalized_step = source.is_a?(Hash) ? source.with_indifferent_access : {}
+ current_output = normalized_step[:output]
+
+ interpreted_semantic = semantic(previous_output, current_output)
+ interpreted_delta = delta(previous_output, current_output, interpreted_semantic)
+ interpreted_intent = intent(normalized_step)
+
+ previous_output = current_output
+
+ normalized_step.merge(
+ semantic: interpreted_semantic,
+ delta: interpreted_delta,
+ intent: interpreted_intent
+ )
+ end
+ end
+
+ # State-only semantic classification.
+ def semantic(previous, current)
+ if previous.nil?
+ blank?(current) ? "No result" : "Δ added"
+ elsif blank?(current) && blank?(previous)
+ "No result"
+ elsif current == previous
+ "No change"
+ elsif present?(current) && blank?(previous)
+ "Δ added"
+ elsif blank?(current) && present?(previous)
+ "Δ removed"
+ else
+ "Δ changed"
+ end
+ end
+
+ # Secondary diff details for changed states only.
+ def delta(previous, current, semantic_value)
+ return nil unless semantic_value == "Δ changed"
+
+ compute_delta(previous, current)
+ end
+
+ def intent(step)
+ source = step.respond_to?(:to_h) ? step.to_h : step
+ normalized_step = source.is_a?(Hash) ? source.with_indifferent_access : {}
+
+ type = normalized_step[:type].to_s
+ code = normalized_step[:code].to_s
+
+ return "navigation" if type == "url"
+ return "extraction" if type == "xpath"
+ return "conditional extraction" if type == "if_xpath"
+
+ if type == "ruby"
+ return "filter" if code.include?("reject") || code.include?("select")
+ return "transform" if code.include?("map") || code.include?("gsub")
+
+ return "transform"
+ end
+
+ "unknown"
+ end
+
+ private
+
+ def blank?(value)
+ return true if value.nil? || value == [] || value == ""
+ return true if value.is_a?(String) && value.strip == "[]"
+
+ false
+ end
+
+ def present?(value)
+ !blank?(value)
+ end
+
+ def compute_delta(previous, current)
+ if previous.is_a?(Array) && current.is_a?(Array)
+ return compute_array_delta(previous, current)
+ end
+
+ if previous.is_a?(String) && current.is_a?(String)
+ previous_array = parse_summary_array_state(previous)
+ current_array = parse_summary_array_state(current)
+ return compute_array_delta(previous_array, current_array) if previous_array && current_array
+ return nil if previous == current
+
+ return "changed"
+ end
+
+ previous_array = Array(previous).map(&:to_s)
+ current_array = Array(current).map(&:to_s)
+ compute_array_delta(previous_array, current_array)
+ end
+
+ def compute_array_delta(previous_array, current_array)
+ added = current_array - previous_array
+ removed = previous_array - current_array
+
+ parts = []
+ parts << "+#{added.first(2).map { |v| truncate_delta(v) }.join(', ')}" if added.any?
+ parts << "-#{removed.first(2).map { |v| truncate_delta(v) }.join(', ')}" if removed.any?
+
+ return nil if parts.empty?
+
+ parts.join(" ")
+ end
+
+ def parse_summary_array_state(value)
+ return [] if value == "[]"
+ return nil unless value.is_a?(String)
+
+ match = value.match(/\A\[\d+ items:\s*(.*)\]\z/)
+ return nil unless match
+
+ body = match[1].to_s.strip
+ return [] if body.empty?
+
+ body.split(/\s*,\s*/)
+ end
+
+ def truncate_delta(value)
+ string_value = value.to_s
+ string_value.length > 60 ? "#{string_value[0, 60]}..." : string_value
+ end
+ end
+end
diff --git a/app/services/dsl/dsl_context.rb b/app/services/dsl/support/context.rb
similarity index 67%
rename from app/services/dsl/dsl_context.rb
rename to app/services/dsl/support/context.rb
index a2a8d4d2..97016499 100644
--- a/app/services/dsl/dsl_context.rb
+++ b/app/services/dsl/support/context.rb
@@ -1,6 +1,7 @@
-# app/services/dsl/dsl_context.rb
+# app/services/dsl/support/context.rb
module Dsl
- class DslContext
+ module Support
+ class Context
attr_reader :url, :array, :tracer
def initialize(url:, array:, tracer:)
@@ -8,5 +9,6 @@ def initialize(url:, array:, tracer:)
@array = array
@tracer = tracer
end
+ end
end
-end
\ No newline at end of file
+end
diff --git a/app/services/dsl/support/url_resolver.rb b/app/services/dsl/support/url_resolver.rb
new file mode 100644
index 00000000..076dccf4
--- /dev/null
+++ b/app/services/dsl/support/url_resolver.rb
@@ -0,0 +1,71 @@
+# app/services/dsl/support/url_resolver.rb
+module Dsl
+ module Support
+ class UrlResolver
+ class << self
+ def extract(value)
+ return nil if value.nil?
+
+ normalized = candidates(value).filter_map { |candidate| normalize_resolved_url(candidate) }
+ valid = normalized.find { |url| valid_url?(url) }
+
+ unless valid
+ Rails.logger.debug { "[DSL] skipped invalid URL from #{value.inspect}" }
+ end
+
+ valid
+ end
+
+ private
+
+ def candidates(value)
+ case value
+ when String
+ [value] + json_string_candidates(value)
+ when Array
+ value
+ else
+ [value]
+ end
+ end
+
+ def json_string_candidates(value)
+ parsed = JSON.parse(value)
+
+ case parsed
+ when String
+ [parsed]
+ when Hash
+ parsed.values.grep(String)
+ when Array
+ parsed.grep(String)
+ else
+ []
+ end
+ rescue JSON::ParserError
+ []
+ end
+
+ def normalize_resolved_url(value)
+ return nil if value.nil?
+
+ url = normalize_url(value.to_s)
+ return nil if url.blank?
+
+ url
+ end
+
+ def normalize_url(url)
+ ApplicationController.helpers.normalize_url(url)
+ end
+
+ def valid_url?(url)
+ uri = URI.parse(url)
+ uri.is_a?(URI::HTTP) && uri.host.present?
+ rescue URI::InvalidURIError
+ false
+ end
+ end
+ end
+ end
+end
diff --git a/app/services/dsl/support/wringer_client.rb b/app/services/dsl/support/wringer_client.rb
new file mode 100644
index 00000000..e31c101a
--- /dev/null
+++ b/app/services/dsl/support/wringer_client.rb
@@ -0,0 +1,27 @@
+# app/services/dsl/support/wringer_client.rb
+module Dsl
+ module Support
+ class WringerClient
+ def initialize(agent:, render_js:, scrape_options:, use_wringer:, safe_wringer_call:, logger:)
+ @agent = agent
+ @render_js = render_js
+ @scrape_options = scrape_options
+ @use_wringer = use_wringer
+ @safe_wringer_call = safe_wringer_call
+ @logger = logger
+ end
+
+ def fetch(url:, render_js: @render_js, scrape_options: @scrape_options)
+ Distillator::FetchService.fetch_wringer_backed(
+ url: url,
+ render_js: render_js,
+ scrape_options: scrape_options,
+ agent: @agent,
+ use_wringer: @use_wringer,
+ safe_wringer_call: @safe_wringer_call,
+ logger: @logger
+ )
+ end
+ end
+ end
+end
diff --git a/app/services/dsl/tracing/null_tracer.rb b/app/services/dsl/tracing/null_tracer.rb
new file mode 100644
index 00000000..68cc6a2d
--- /dev/null
+++ b/app/services/dsl/tracing/null_tracer.rb
@@ -0,0 +1,9 @@
+# app/services/dsl/tracing/null_tracer.rb
+module Dsl
+ module Tracing
+ class NullTracer
+ def step(**); end
+ def to_h = []
+ end
+ end
+end
diff --git a/app/services/dsl/tracing/trace_collector.rb b/app/services/dsl/tracing/trace_collector.rb
new file mode 100644
index 00000000..d72f29f7
--- /dev/null
+++ b/app/services/dsl/tracing/trace_collector.rb
@@ -0,0 +1,72 @@
+# app/services/dsl/tracing/trace_collector.rb
+module Dsl
+ module Tracing
+ class TraceCollector
+ attr_reader :events
+
+ def initialize
+ @events = []
+ end
+
+ def step(
+ step:,
+ type:,
+ code:,
+ input:,
+ output:,
+ input_full: nil,
+ output_full: nil,
+ probe: nil,
+ error: nil,
+ wringer: nil,
+ url_before: nil,
+ url_after: nil,
+ duration_ms: nil
+ )
+ compact_error_message =
+ case error
+ when Hash
+ compact_hash_error(error)
+ else
+ error.nil? ? nil : error.to_s
+ end
+
+ @events << {
+ step: step,
+ type: type,
+ code: code,
+ input_preview: input,
+ output_preview: output,
+ input_full: input_full.nil? ? input : input_full,
+ output_full: output_full.nil? ? output : output_full,
+ probe: probe.present? ? probe : { skipped: true },
+ error_class: error.nil? ? nil : error.class.to_s,
+ error_message: compact_error_message,
+ wringer: wringer.present? ? wringer : { inherited: true },
+ url_before: url_before,
+ url_after: url_after,
+ duration_ms: duration_ms
+ }
+ end
+
+ def to_h
+ @events
+ end
+
+ private
+
+ def compact_hash_error(error)
+ payload = error.respond_to?(:to_h) ? error.to_h.with_indifferent_access : {}
+ error_type = payload[:error_type].presence || "DslAbort"
+ step = payload[:step].presence
+ message = payload[:error].to_s.tr("\n", " ").squish
+ message = "#{message[0, 180]}..." if message.length > 180
+
+ parts = ["Scrape aborted (#{error_type})"]
+ parts << "step=#{step}" if step.present?
+ parts << message if message.present?
+ parts.join(": ")
+ end
+ end
+ end
+end
diff --git a/app/services/dsl/tracing/trace_formatter.rb b/app/services/dsl/tracing/trace_formatter.rb
new file mode 100644
index 00000000..7595f6b4
--- /dev/null
+++ b/app/services/dsl/tracing/trace_formatter.rb
@@ -0,0 +1,614 @@
+# frozen_string_literal: true
+
+module Dsl
+ module Tracing
+ class TraceFormatter
+ MAX_STRING_LENGTH = 200
+ ARRAY_SAMPLE_SIZE = 5
+ MAX_UI_EVENTS = 20
+ MAX_SESSION_BYTES = 2000
+
+ class << self
+ def normalize(raw_events)
+ return [] unless raw_events.is_a?(Array)
+
+ raw_events.map do |evt|
+ output_full = evt[:output_full] || evt["output_full"]
+ input_full = evt[:input_full] || evt["input_full"]
+
+ output_full ||= evt[:output_preview] || evt["output_preview"]
+ input_full ||= evt[:input_preview] || evt["input_preview"]
+
+ {
+ step: evt[:step] || evt["step"],
+ type: evt[:type] || evt["type"],
+ code: evt[:code] || evt["code"],
+ input_preview: evt[:input_preview] || evt["input_preview"] || [],
+ output_preview: evt[:output_preview] || evt["output_preview"] || [],
+ input_full: input_full,
+ output_full: output_full,
+ probe: evt[:probe] || evt["probe"],
+ url_before: (evt[:url_before] || evt["url_before"] || "").to_s,
+ url_after: (evt[:url_after] || evt["url_after"] || "").to_s,
+ duration_ms: evt[:duration_ms] || evt["duration_ms"] || 0,
+ error_class: evt[:error_class] || evt["error_class"],
+ error_message: evt[:error_message] || evt["error_message"],
+ wringer: evt[:wringer] || evt["wringer"]
+ }
+ end
+ end
+
+ def for_session_v2(trace)
+ return { version: 2, initial: { state: nil, url: nil }, urls: [], steps: [] } unless trace.is_a?(Array)
+
+ normalized = trace.each_with_index.map do |event, index|
+ normalize_event_for_session(event, index)
+ end
+
+ attempts = [
+ {
+ code_limit: 40,
+ error_code_limit: 120,
+ output_limit: 80,
+ initial_limit: 80,
+ include_duration: true
+ },
+ {
+ code_limit: 40,
+ error_code_limit: 120,
+ output_limit: 40,
+ initial_limit: 40,
+ include_duration: false
+ },
+ {
+ code_limit: 20,
+ error_code_limit: 80,
+ output_limit: 20,
+ initial_limit: 20,
+ include_duration: false
+ }
+ ]
+
+ attempts.each_with_index do |options, index|
+ compact = build_session_v2(normalized, **options)
+ return compact if serialized_size(compact) <= MAX_SESSION_BYTES
+
+ next unless index.zero?
+
+ Rails.logger.warn("[DSL TRACE] v2 trace exceeded session budget; trimming non-critical fields")
+ end
+
+ minimal = minimal_session_v2(normalized)
+ return minimal if serialized_size(minimal) <= MAX_SESSION_BYTES
+
+ Rails.logger.warn("[DSL TRACE] v2 trace still exceeded session budget after minimal fallback; omitting trace details")
+ omitted_session_v2
+ end
+
+ def for_session(trace)
+ return { version: 1, urls: [], events: [] } unless trace.is_a?(Array)
+
+ urls = []
+ events = trace.each_with_index.map do |event, index|
+ compact_event_for_session(event, urls, index)
+ end
+
+ compact = { version: 1, urls: urls, events: events }
+ return compact if serialized_size(compact) <= MAX_SESSION_BYTES
+
+ Rails.logger.warn("[DSL TRACE] Compact trace exceeded session budget; trimming non-critical fields")
+
+ reduced_events = events.map do |event|
+ reduced = event.dup
+ unless reduced.key?(:e)
+ reduced.delete(:i)
+ reduced.delete(:o)
+ reduced.delete(:d)
+ end
+ reduced[:c] = truncate_str(reduced[:c], 40) if reduced.key?(:c)
+ reduced
+ end
+
+ compact = { version: 1, urls: urls, events: reduced_events }
+ return compact if serialized_size(compact) <= MAX_SESSION_BYTES
+
+ final_events = reduced_events.map do |event|
+ final = event.dup
+ final[:c] = truncate_str(final[:c], 20) if final.key?(:c) && !final.key?(:e)
+ final
+ end
+
+ { version: 1, urls: urls, events: final_events }
+ end
+
+ def for_ui(trace)
+ return [] unless trace.is_a?(Array)
+
+ formatted = trace.map do |event|
+ formatted_event = format_event(event)
+
+ if formatted_event.nil?
+ Rails.logger.warn("[DSL TRACE] Dropped malformed event: #{event.inspect}")
+ end
+
+ formatted_event
+ end.compact
+ limit_for_session(formatted)
+ end
+
+ def format_event(event)
+ raw = event.respond_to?(:to_h) ? event.to_h : event
+ return nil unless raw.is_a?(Hash)
+
+ payload = raw.with_indifferent_access
+ error_text = [format_value(payload[:error_class]), format_value(payload[:error_message])].compact.join(": ").presence
+
+ {
+ step: payload[:step],
+ type: format_value(payload[:type]),
+ code: format_value(payload[:code]),
+ input: format_value(first_present(payload, :input, :input_preview)),
+ output: format_value(first_present(payload, :output, :output_preview)),
+ error: error_text,
+ url_before: format_value(payload[:url_before]),
+ url_after: format_value(payload[:url_after]),
+ duration_ms: payload[:duration_ms]
+ }
+ rescue StandardError => e
+ Rails.logger.debug { "[DSL] skipped malformed trace event: #{e.class}: #{e.message}" }
+ nil
+ end
+
+ def format_value(value)
+ return nil if value.nil?
+
+ case value
+ when Exception
+ truncate_str(value.message.to_s)
+ when Array
+ sample = value.first(ARRAY_SAMPLE_SIZE).map { |entry| format_value(entry) }
+ truncate_str("[Array size=#{value.size}, sample=#{sample.inspect}]")
+ when String
+ truncate_str(value)
+ when Hash
+ truncate_str(value.inspect)
+ else
+ truncate_str(value.inspect)
+ end
+ rescue StandardError
+ truncate_str(value.to_s)
+ end
+
+ def truncate_str(str, max = MAX_STRING_LENGTH)
+ string = str.to_s
+ return string if string.length <= max
+
+ "#{string[0, max]}..."
+ end
+
+ private
+
+ def normalize_event_for_session(event, fallback_step_index)
+ raw = event.respond_to?(:to_h) ? event.to_h : event
+ payload = raw.is_a?(Hash) ? raw.with_indifferent_access : {}
+
+ {
+ step: payload[:step].nil? ? fallback_step_index + 1 : payload[:step],
+ type: payload[:type].nil? ? nil : payload[:type].to_s,
+ code: payload[:code].nil? ? nil : payload[:code].to_s,
+ input: summarize_for_session(first_present(payload, :input, :input_preview), max: 120),
+ output: summarize_for_session(first_present(payload, :output, :output_preview), max: 120),
+ probe: normalize_probe_for_session(payload[:probe]),
+ wringer: normalize_wringer_for_session(payload[:wringer]),
+ url_before: normalize_session_url(payload[:url_before]),
+ url_after: normalize_session_url(payload[:url_after]),
+ duration_ms: payload[:duration_ms],
+ error: session_error_text(payload)
+ }
+ rescue StandardError => e
+ Rails.logger.warn("[DSL TRACE] Failed to normalize trace event for session: #{e.class}: #{e.message}")
+ {
+ step: fallback_step_index + 1,
+ type: nil,
+ code: nil,
+ input: nil,
+ output: nil,
+ url_before: nil,
+ url_after: nil,
+ duration_ms: nil,
+ error: "Trace normalization error: #{e.class}"
+ }
+ end
+
+ def build_session_v2(normalized, code_limit:, error_code_limit:, output_limit:, initial_limit:, include_duration:)
+ first = normalized.first || {}
+ initial_state = first[:input].nil? ? nil : truncate_str(first[:input], initial_limit)
+ initial_url = first[:url_before]
+
+ urls = []
+ current_url = initial_url
+
+ steps = normalized.map do |event|
+ step_payload = {}
+ severity = event_severity(event)
+ step_payload[:s] = event[:step]
+ step_payload[:t] = truncate_str(event[:type], 30) if event[:type].present?
+
+ if event[:code].present?
+ code_max = severity == :error ? error_code_limit : code_limit
+ step_payload[:c] = truncate_str(event[:code], code_max)
+
+ if severity != :ok
+ step_payload[:cf] = truncate_str(event[:code], error_code_limit)
+ end
+ end
+
+ if event[:output].present?
+ step_payload[:o] = truncate_str(event[:output], output_limit)
+
+ if severity == :error
+ step_payload[:of] = truncate_str(event[:output], output_limit * 2)
+ elsif severity == :warning
+ step_payload[:of] = truncate_str(event[:output], output_limit)
+ end
+ elsif event[:output] == "[]"
+ step_payload[:o] = "[]"
+ step_payload[:of] = "[]" if severity != :ok
+ end
+
+ if event[:probe].is_a?(Hash)
+ if event[:probe][:skipped]
+ step_payload[:p] = { sk: true }
+ else
+ probe_result = event[:probe][:result] || {}
+ step_payload[:p] = {
+ st: probe_result[:status],
+ x: probe_result[:xpath],
+ o: probe_result[:output],
+ ok: event[:probe][:ok]
+ }.compact
+ end
+ end
+
+ if event[:wringer].is_a?(Hash)
+ step_payload[:w] = {
+ i: event[:wringer][:inherited],
+ et: event[:wringer][:error_type],
+ r: event[:wringer][:retry],
+ c: event[:wringer][:cache],
+ u: event[:wringer][:unreachable],
+ r404: event[:wringer][:received_404],
+ se: event[:wringer][:system_error],
+ pa: event[:wringer][:policy_action],
+ s: event[:wringer][:signals],
+ h: event[:wringer][:hints],
+ fu: event[:wringer][:final_url],
+ rc: event[:wringer][:redirect_chain],
+ ct: event[:wringer][:content_type]
+ }.compact
+ end
+
+ next_url = event[:url_after] || current_url
+ if next_url.present? && next_url != current_url
+ step_payload[:ua] = url_index_for_session(urls, next_url)
+ current_url = next_url
+ end
+
+ step_payload[:d] = event[:duration_ms] if include_duration && !event[:duration_ms].nil?
+ step_payload[:e] = event[:error] if event[:error].present?
+
+ step_payload
+ end
+
+ {
+ version: 2,
+ initial: { state: initial_state, url: initial_url },
+ urls: urls,
+ steps: steps
+ }
+ end
+
+ def minimal_session_v2(normalized)
+ last_event = normalized.last || {}
+ message = last_event[:error].presence || "Trace omitted: exceeded session budget"
+
+ {
+ version: 2,
+ initial: { state: nil, url: nil },
+ urls: [],
+ steps: [
+ {
+ s: last_event[:step] || normalized.length,
+ t: "trace",
+ e: truncate_str(message, 160)
+ }.compact
+ ]
+ }
+ end
+
+ def omitted_session_v2
+ {
+ version: 2,
+ initial: { state: nil, url: nil },
+ urls: [],
+ steps: [
+ {
+ s: 1,
+ t: "trace",
+ e: "Trace omitted: exceeded session budget"
+ }
+ ]
+ }
+ end
+
+ def event_severity(event)
+ return :error if event[:error].present?
+
+ output_empty = event[:output].to_s == "[]"
+
+ probe_used =
+ event[:probe].is_a?(Hash) && !event[:probe][:skipped]
+
+ wringer_issue =
+ event[:wringer].is_a?(Hash) && (
+ event[:wringer][:error_type].present? ||
+ event[:wringer][:unreachable] ||
+ event[:wringer][:system_error]
+ )
+
+ return :warning if output_empty || probe_used || wringer_issue
+
+ :ok
+ end
+
+ def normalize_session_url(value)
+ return nil if value.nil?
+
+ url = value.to_s
+ return nil if url.empty?
+
+ url
+ end
+
+ def compact_event_for_session(event, urls, fallback_step_index)
+ raw = event.respond_to?(:to_h) ? event.to_h : event
+ payload = raw.is_a?(Hash) ? raw.with_indifferent_access : {}
+
+ error_text = session_error_text(payload)
+ error_step = error_text.present?
+ code_limit = error_step ? 200 : 80
+
+ compact = {}
+ compact[:s] = payload[:step].nil? ? fallback_step_index + 1 : payload[:step]
+
+ type = payload[:type]
+ compact[:t] = truncate_str(type.to_s, 30) unless type.nil?
+
+ code = payload[:code]
+ compact[:c] = truncate_str(code.to_s, code_limit) unless code.nil?
+
+ input_summary = summarize_for_session(first_present(payload, :input, :input_preview))
+ compact[:i] = input_summary unless input_summary.nil?
+
+ output_summary = summarize_for_session(first_present(payload, :output, :output_preview))
+ compact[:o] = output_summary unless output_summary.nil?
+
+ before_index = url_index_for_session(urls, payload[:url_before])
+ compact[:ub] = before_index unless before_index.nil?
+
+ after_index = url_index_for_session(urls, payload[:url_after])
+ compact[:ua] = after_index unless after_index.nil?
+
+ duration = payload[:duration_ms]
+ compact[:d] = duration unless duration.nil?
+
+ compact[:e] = error_text if error_text.present?
+
+ compact
+ rescue StandardError => e
+ Rails.logger.warn("[DSL TRACE] Failed to compact trace event: #{e.class}: #{e.message}")
+ { s: fallback_step_index + 1, e: "Trace compaction error: #{e.class}" }
+ end
+
+ def summarize_for_session(value, max: 80)
+ return nil if value.nil?
+
+ if value.is_a?(Array)
+ return "[]" if value.empty?
+
+ sample = value.first(2).map { |entry| truncate_str(entry.to_s, 60) }
+ "[#{value.size} items: #{sample.join(', ')}]"
+ else
+ truncate_str(value.to_s, max)
+ end
+ end
+
+ def normalize_probe_for_session(value)
+ raw = value.respond_to?(:to_h) ? value.to_h : value
+ return { skipped: true } unless raw.is_a?(Hash)
+
+ payload = raw.with_indifferent_access
+ return { skipped: true } if payload[:skipped]
+
+ probe_result =
+ if payload[:result].is_a?(Hash)
+ payload[:result].with_indifferent_access
+ else
+ payload
+ end
+
+ xpath = probe_result[:xpath].to_s
+ return { skipped: true } if xpath.blank?
+
+ status = probe_result[:status].to_s
+ status = "ok" if status.blank?
+ output = Array(probe_result[:output]).compact.map { |entry| truncate_str(entry.to_s, 80) }.first(3)
+
+ {
+ result: {
+ status: status,
+ xpath: truncate_str(xpath, 40),
+ output: output
+ }.compact,
+ ok: payload.key?(:ok) ? payload[:ok] : (status == "ok")
+ }.compact
+ rescue StandardError
+ { skipped: true }
+ end
+
+ def normalize_wringer_for_session(value)
+ raw = value.respond_to?(:to_h) ? value.to_h : value
+ return { inherited: true } unless raw.is_a?(Hash)
+
+ payload = raw.with_indifferent_access
+ return { inherited: true } if payload[:inherited]
+
+ signals = compact_wringer_signals(payload)
+ hints = compact_wringer_hints(payload)
+ final_url = normalize_session_url(payload[:final_url] || signals[:final_url])
+ content_type = signals[:content_type].presence
+
+ {
+ error_type: truncate_str(payload[:error_type].to_s, 60).presence,
+ retry: payload[:retry],
+ cache: payload[:cache],
+ unreachable: payload[:unreachable],
+ received_404: payload[:received_404],
+ system_error: payload[:system_error],
+ policy_action: truncate_str(payload[:policy_action].to_s, 40).presence,
+ content_type: truncate_str(content_type.to_s, 40).presence,
+ final_url: final_url,
+ redirect_chain: compact_redirect_chain(payload[:redirect_chain]),
+ signals: signals.presence,
+ hints: hints.presence
+ }.compact
+ end
+
+ def compact_wringer_signals(payload)
+ raw = payload[:signals]
+ return {} unless raw.respond_to?(:to_h)
+
+ signals = raw.to_h.with_indifferent_access
+
+ summary = {
+ network_status: truncate_str(signals[:network_status].to_s, 30).presence,
+ content_type: truncate_str(signals[:content_type].to_s, 40).presence,
+ blocking_issue_key: truncate_str(signals[:blocking_issue_key].to_s, 60).presence,
+ primary_issue_key: truncate_str(signals[:primary_issue_key].to_s, 60).presence,
+ primary_issue_label: truncate_str(signals[:primary_issue_label].to_s, 80).presence,
+ final_url: normalize_session_url(signals[:final_url]),
+ fetch_backend: truncate_str(signals[:fetch_backend].to_s, 40).presence,
+ fetched_body_state: truncate_str(signals[:fetched_body_state].to_s, 30).presence,
+ stored_body_state: truncate_str(signals[:stored_body_state].to_s, 30).presence
+ }
+
+ if summary[:fetched_body_state].blank? && summary[:stored_body_state].blank?
+ derived = compact_body_state(signals)
+ summary[:body_state] = derived if derived.present?
+ end
+
+ summary.compact
+ rescue StandardError
+ {}
+ end
+
+ def compact_body_state(signals)
+ fetched =
+ if signals.key?(:fetched_body_present)
+ signals[:fetched_body_present] ? "present" : "missing"
+ elsif signals.key?(:empty_body)
+ signals[:empty_body] ? "empty" : nil
+ end
+
+ stored =
+ if signals.key?(:stored_body_present)
+ signals[:stored_body_present] ? "present" : "missing"
+ end
+
+ [fetched, stored].compact.join("/")
+ end
+
+ def compact_wringer_hints(payload)
+ Array(payload[:hints]).first(2).map do |hint|
+ truncate_nested_session_value(hint, max: 60)
+ end.compact
+ rescue StandardError
+ []
+ end
+
+ def compact_redirect_chain(value)
+ chain = Array(value).compact.map { |entry| normalize_session_url(entry) }.compact
+ return nil if chain.empty?
+
+ chain.first(2).map { |entry| truncate_str(entry, 100) }
+ rescue StandardError
+ nil
+ end
+
+ def truncate_nested_session_value(value, max:)
+ case value
+ when Hash
+ value.to_h.each_with_object({}) do |(key, nested), compact|
+ compact[truncate_str(key.to_s, 30)] = truncate_nested_session_value(nested, max: max)
+ end
+ when Array
+ value.first(2).map { |entry| truncate_nested_session_value(entry, max: max) }
+ else
+ truncate_str(value.to_s, max)
+ end
+ rescue StandardError
+ truncate_str(value.to_s, max)
+ end
+
+ def session_error_text(payload)
+ explicit_error = payload[:error]
+ return explicit_error.to_s if explicit_error.present?
+
+ [payload[:error_class], payload[:error_message]].compact.join(": ").presence
+ end
+
+ def url_index_for_session(urls, value)
+ return nil if value.nil?
+
+ url = value.to_s
+ return nil if url.empty?
+
+ existing_index = urls.index(url)
+ return existing_index unless existing_index.nil?
+
+ urls << url
+ urls.length - 1
+ end
+
+ def first_present(hash, *keys)
+ keys.each do |key|
+ value = hash[key]
+ return value unless value.nil?
+ end
+ nil
+ end
+
+ def limit_for_session(events)
+ return events if serialized_size(events) <= MAX_SESSION_BYTES
+
+ Rails.logger.warn("[DSL TRACE] Trace too large, truncating fields (not steps)")
+
+ events.map do |e|
+ e.merge(
+ code: truncate_str(e[:code], 80),
+ input: truncate_str(e[:input], 80),
+ output: truncate_str(e[:output], 80),
+ error: truncate_str(e[:error], 120)
+ )
+ end
+ end
+
+ def serialized_size(payload)
+ JSON.generate(payload).bytesize
+ rescue StandardError
+ payload.to_s.bytesize
+ end
+ end
+ end
+ end
+end
diff --git a/app/services/export_artsdata_service.rb b/app/services/export_artsdata_service.rb
new file mode 100644
index 00000000..c6807d3d
--- /dev/null
+++ b/app/services/export_artsdata_service.rb
@@ -0,0 +1,119 @@
+class ExportArtsdataService
+ PUBLISHABLE_STATES = %w[ok updated].freeze
+
+ def self.call(seedurl:)
+ new(seedurl).call
+ end
+
+ def self.production_equivalent(seedurl:)
+ new(seedurl).production_equivalent
+ end
+
+ def initialize(seedurl)
+ @seedurl = seedurl
+ end
+
+ def call
+ grouped_started_at = Time.current if timing_enabled?
+ grouped_events = website_statements_by_event
+ if grouped_started_at
+ puts "Grouping: #{(Time.current - grouped_started_at).round(3)}s"
+ end
+
+ publishable_events = publishable_events_for(grouped_events)
+
+ dump_started_at = Time.current if timing_enabled?
+ dump = JsonldGenerator.dump_events(publishable_events)
+ if dump_started_at
+ puts "Dump: #{(Time.current - dump_started_at).round(3)}s"
+ end
+
+ dump
+ end
+
+ def production_equivalent
+ JsonldGenerator.dump_events_old(production_equivalent_publishable_events)
+ end
+
+ private
+
+ def website_statements_by_event
+ events_by_uri = Hash.new { |hash, key| hash[key] = {} }
+
+ statements.each do |statement|
+ property_label = make_key(statement.source.property.label, statement.source.language)
+ event_uri = statement.webpage.rdf_uri
+
+ if events_by_uri[event_uri][property_label].present?
+ events_by_uri[event_uri][property_label] = {
+ cache: statement.cache,
+ status: "problem",
+ selected_individual: statement.selected_individual
+ }
+ else
+ events_by_uri[event_uri][property_label] = {
+ cache: statement.cache,
+ status: statement.status,
+ selected_individual: statement.selected_individual
+ }
+ events_by_uri[event_uri][:archive_date] ||= { cache: statement.webpage.archive_date }
+ end
+ end
+
+ events_by_uri
+ end
+
+ def statements
+ return Statement.none if event_class_id.nil?
+
+ @statements ||=
+ Statement
+ .joins(:webpage, source: :website)
+ .includes({ source: [:property, :website] }, :webpage)
+ .where(
+ websites: { seedurl: @seedurl },
+ webpages: {
+ archive_date: [(Time.zone.now - 3000.years)..(Time.zone.now + 3000.years)],
+ rdfs_class_id: event_class_id
+ }
+ )
+ .where(selected_individual: true)
+ end
+
+ def event_class_id
+ @event_class_id ||= RdfsClass.find_by(name: "Event")&.id
+ end
+
+ def make_key(property, language)
+ key = property.sub(" ", "_").downcase
+ key += "_#{language.downcase}" if language.present?
+ key
+ rescue StandardError
+ "failed to make key"
+ end
+
+ def event_publishable?(data)
+ return false unless PUBLISHABLE_STATES.include?(data.dig("dates", :status))
+ return false unless PUBLISHABLE_STATES.include?(data.dig("location", :status)) ||
+ PUBLISHABLE_STATES.include?(data.dig("virtuallocation", :status))
+ return false unless PUBLISHABLE_STATES.include?(data.dig("title_en", :status)) ||
+ PUBLISHABLE_STATES.include?(data.dig("title_fr", :status)) ||
+ PUBLISHABLE_STATES.include?(data.dig("title", :status))
+
+ true
+ end
+
+ def timing_enabled?
+ ENV["EXPORT_ARTSDATA_TIMING"].present?
+ end
+
+ def production_equivalent_publishable_events
+ publishable_events_for(website_statements_by_event)
+ end
+
+ def publishable_events_for(grouped_events)
+ grouped_events.filter_map do |rdf_uri, event_data|
+ rdf_uri if event_publishable?(event_data)
+ end
+ end
+end
diff --git a/app/services/jsonld_generator.rb b/app/services/jsonld_generator.rb
index 05ea2e65..fccfeef8 100644
--- a/app/services/jsonld_generator.rb
+++ b/app/services/jsonld_generator.rb
@@ -3,7 +3,7 @@ class JsonldGenerator
extend ResourcesHelper # for method adjust_labels_for_api
# main method to dump all event statements into a graph to push to artsdata
- def self.dump_events(events) # list of event uris
+ def self.dump_events_old(events) # list of event uris
graphs = RDF::Graph.new
events.each do |uri|
statements = load_uri_statements(uri)
@@ -19,6 +19,46 @@ def self.dump_events(events) # list of event uris
graphs.dump(:jsonld)
end
+ def self.dump_events(events)
+ graphs = RDF::Graph.new
+
+ all_statements = load_all_statements(events)
+
+ events.each do |uri|
+ statements = all_statements[uri] || []
+
+ statements_hash = statements.map do |stat|
+ adjust_labels_for_api(
+ stat,
+ subject: stat.webpage.rdf_uri,
+ webpage_class_name: stat.webpage.rdfs_class.name
+ )
+ end
+
+ graph = timed("build_graph #{uri}") { build_graph(statements_hash, { for_artsdata: true }) }
+ graph = timed("contact #{uri}") { make_contact_series(graph, uri) }
+ graph = timed("offer #{uri}") { make_offer_series(graph, uri) }
+ graph = timed("event_series #{uri}") { make_event_series(graph, uri) }
+
+ graphs << graph
+ end
+
+ # ✅ global operations ONLY
+ graphs = timed("add_triples_from_footlight") { add_triples_from_footlight(graphs) }
+ graphs = timed("remove_annotations") { remove_annotations(graphs) }
+
+ graphs.dump(:jsonld)
+ end
+
+ def self.load_all_statements(uris)
+ webpages = Webpage.where(rdf_uri: uris)
+
+ Statement
+ .where(webpage_id: webpages.select(:id), selected_individual: true)
+ .includes(:webpage)
+ .group_by { |s| s.webpage.rdf_uri }
+ end
+
# main method to dump all statements bedsides events into a graph to push to artsdata
def self.dump_resources_besides_events(uri_list)
graphs = RDF::Graph.new
@@ -91,7 +131,9 @@ def self.frame_json(graph_json, main_class = 'Event', frame = nil)
# Add triples from artsdata.ca AND Footlight database using URIs of people, places and organizations
def self.add_triples_from_artsdata(local_graph)
- uris = extract_object_uris(local_graph)
+ uris = extract_object_uris(local_graph).map(&:to_s)
+ .uniq
+ .map { |u| RDF::URI(u) }
uris.each do |uri|
additional_graph = describe_uri(uri)
# TODO: fetch remote data from Wikidata if additional_graph.count == 0
@@ -102,17 +144,17 @@ def self.add_triples_from_artsdata(local_graph)
# Add triples ONLY from Footlight database (not Artsdata) using URIs of people, places and organizations
def self.add_triples_from_footlight(local_graph)
-
- ## Refresh local entities (People, Places, Organizations) entered manually into Footlight
- # ArtsdataGraph.graph << LocalGraphGenerator.graph_all
-
uris = extract_object_uris(local_graph)
+ .map(&:to_s)
+ .reject { |u| u.include?("http://kg.artsdata.ca/resource/K") }
+ .uniq
+ .map { |u| RDF::URI(u) }
+
uris.each do |uri|
- if !uri.value.include?("http://kg.artsdata.ca/resource/K")
- additional_graph = describe_uri(uri)
- local_graph << additional_graph
- end
+ additional_graph = describe_uri(uri)
+ local_graph << additional_graph
end
+
local_graph
end
@@ -423,7 +465,7 @@ def self.extract_object_uris(graph)
end
# Get triples about a URI from Artsdata.ca AND Footlight database
- def self.describe_uri(uri)
+ def self.describe_uri_old(uri)
query = RDF::Query.new do
pattern [uri, :p, :o]
end
@@ -474,6 +516,86 @@ def self.describe_uri(uri)
graph
end
+ @describe_cache ||= {}
+ @subject_index_cache ||= {}
+
+ def self.describe_uri(uri)
+ @describe_cache[uri] ||= begin
+ if describe_candidate_uri?(uri)
+ if footlight_uri?(uri)
+ describe_footlight_uri(uri)
+ else
+ graph = RDF::Graph.new
+
+ subject_triples(uri).each do |subject, predicate, object|
+ graph << [subject, predicate, object]
+
+ if object.uri? && (!object.value.start_with?("http://kg.artsdata.ca/resource/K") || object.value.include?("#PostalAddress"))
+ subject_triples(object).each do |nested_subject, nested_predicate, nested_object|
+ graph << [nested_subject, nested_predicate, nested_object]
+ end
+ end
+
+ if object.node?
+ subject_triples(object).each do |nested_subject, nested_predicate, nested_object|
+ graph << [nested_subject, nested_predicate, nested_object]
+ end
+ end
+ end
+
+ graph
+ end
+ else
+ RDF::Graph.new
+ end
+ end
+ end
+
+ def self.subject_triples(subject)
+ key = subject.to_s
+
+ @subject_index_cache[key] ||= ArtsdataGraph.graph
+ .query([subject, nil, nil])
+ .map { |statement| [statement.subject, statement.predicate, statement.object] }
+ end
+
+ def self.describe_candidate_uri?(uri)
+ value = uri.to_s
+ return false if value.start_with?("http://schema.org/", "https://schema.org/", "http://www.w3.org/")
+ return false unless value.start_with?("http://kg.artsdata.ca/resource/", "http://kg.footlight.io/resource/")
+ return false if value.start_with?("http://kg.footlight.io/resource/") &&
+ value.include?("#") &&
+ !value.end_with?("#PostalAddress")
+
+ true
+ end
+
+ def self.footlight_uri?(subject)
+ value = subject.to_s
+ value.start_with?("http://kg.footlight.io/resource/")
+ end
+
+ def self.describe_footlight_uri(uri)
+ webpages = Webpage.where(rdf_uri: uri.to_s)
+ return RDF::Graph.new if webpages.empty?
+
+ statements =
+ Statement.joins(source: :property)
+ .where(webpage_id: webpages, sources: { selected: true })
+
+ return RDF::Graph.new if statements.empty?
+
+ statements_hash = statements.map do |statement|
+ adjust_labels_for_api(
+ statement,
+ subject: statement.webpage.rdf_uri,
+ webpage_class_name: statement.webpage.rdfs_class.name
+ )
+ end
+
+ build_graph(statements_hash)
+ end
+
# Derefence URI and return a graph object
def self.dereference_uri(uri)
return RDF::Graph.new unless uri.value.include?('kg.artsdata.ca/resource/K')
@@ -495,6 +617,15 @@ def self.dereference_uri(uri)
end
end
+ def self.timed(label)
+ return yield unless ENV["EXPORT_ARTSDATA_TIMING"].present?
+
+ start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ result = yield
+ puts "#{label}: #{(Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).round(3)}s"
+ result
+ end
+
def self.artsdata_rank_api_url
if Rails.env.development? || Rails.env.test?
"http://localhost:#{ARTSDATA_API_PORT}/query"
diff --git a/app/services/pipeline/pipeline_status_evaluator.rb b/app/services/pipeline/pipeline_status_evaluator.rb
new file mode 100644
index 00000000..ca9a1b87
--- /dev/null
+++ b/app/services/pipeline/pipeline_status_evaluator.rb
@@ -0,0 +1,184 @@
+module Pipeline
+ class PipelineStatusEvaluator
+ # IMPORTANT:
+ # - Evaluator is pure and deterministic
+ # - Does NOT rely on trace ordering
+ # - Does NOT raise on invalid input (fault-tolerant)
+ # - Wringer rules are NOT reimplemented here
+ STATUS_PRIORITY = {
+ ok: 1,
+ partial: 2,
+ failed: 3
+ }.freeze
+
+ FATAL_ERROR_TYPES = %w[
+ redirect_to_listing
+ invalid_event_page
+ blocked_page
+ ].freeze
+
+ def self.call(result:, trace:)
+ new(result: result, trace: trace).call
+ end
+
+ def initialize(result:, trace:)
+ @result = result
+ @trace = trace
+ end
+
+ def call
+ invalid_input = invalid_input?
+ trace = @trace.is_a?(Array) ? @trace.compact : []
+ result = @result.is_a?(Hash) || @result.nil? ? @result : nil
+
+ return failure_response("dsl_error") if invalid_input
+ return failure_response("dsl_error") if result.nil?
+ return abort_response(result) if abort?(result)
+
+ events = extract_wringer_events(trace).map { |event| normalize_event(event) }.compact
+ return aggregated_ok(events) if events.empty?
+
+ event = aggregate_event(events)
+ error_type = event[:error_type]
+
+ {
+ status: status_for_event(event),
+ error_type: error_type,
+ retryable: aggregate_retryable(events),
+ cacheable: aggregate_cacheable(events),
+ reason: reason_for(error_type)
+ }
+ end
+
+ private
+
+ def invalid_input?
+ trace_invalid = !@trace.is_a?(Array)
+ result_invalid = !@result.is_a?(Hash) && !@result.nil?
+ trace_invalid || result_invalid
+ end
+
+ def extract_wringer_events(trace)
+ trace.select { |event| wringer_event?(event) }
+ end
+
+ def wringer_event?(event)
+ return false unless event.is_a?(Hash)
+ return true if value_for(event, :service) == "wringer" && value_for(event, :error).present?
+ return true if value_for(event, :wringer).is_a?(Hash)
+
+ false
+ end
+
+ def normalize_event(event)
+ if value_for(event, :service) == "wringer"
+ normalize_new_wringer_event(event)
+ elsif value_for(event, :wringer).is_a?(Hash)
+ normalize_legacy_wringer_event(value_for(event, :wringer))
+ end
+ end
+
+ def normalize_new_wringer_event(event)
+ payload = value_for(event, :error)
+ payload = event unless payload.is_a?(Hash)
+
+ {
+ error_type: normalize_error(value_for(payload, :error_type)),
+ retryable: !!value_for(payload, :retry),
+ cacheable: !!value_for(payload, :cache)
+ }
+ end
+
+ def normalize_legacy_wringer_event(payload)
+ {
+ error_type: normalize_error(value_for(payload, :error_type)),
+ retryable: !!value_for(payload, :retry),
+ cacheable: !!value_for(payload, :cache)
+ }
+ end
+
+ def normalize_error(error_type)
+ return nil if error_type.nil?
+
+ normalized = error_type.to_s.underscore.downcase
+ normalized.empty? ? nil : normalized
+ end
+
+ def abort?(result)
+ return false unless result.is_a?(Hash)
+
+ value_for(result, :abort) == true || !value_for(result, :error).nil?
+ end
+
+ def extract_abort_error(result)
+ normalize_error(value_for(result, :error_type)) || "dsl_error"
+ end
+
+ def status_for_event(event)
+ fatal_error?(event) ? :failed : :partial
+ end
+
+ def fatal_error?(event)
+ FATAL_ERROR_TYPES.include?(event[:error_type].to_s)
+ end
+
+ def aggregate_event(events)
+ events.max_by do |event|
+ [STATUS_PRIORITY.fetch(status_for_event(event), 0), event[:error_type].to_s]
+ end
+ end
+
+ def value_for(hash, key)
+ return hash[key] if hash.key?(key)
+ return hash[key.to_s] if hash.key?(key.to_s)
+
+ nil
+ end
+
+ def reason_for(error_type)
+ return :invalid_event_page if error_type == "redirect_to_listing"
+ return :invalid_event_page if error_type == "invalid_event_page"
+ return :blocked_page if error_type == "blocked_page"
+
+ nil
+ end
+
+ def aggregate_retryable(events)
+ events.any? { |event| event[:retryable] }
+ end
+
+ def aggregate_cacheable(events)
+ events.empty? || events.all? { |event| event[:cacheable] }
+ end
+
+ def aggregated_ok(_events)
+ {
+ status: :ok,
+ error_type: nil,
+ retryable: false,
+ cacheable: true,
+ reason: nil
+ }
+ end
+
+ def abort_response(result)
+ {
+ status: :failed,
+ error_type: extract_abort_error(result),
+ retryable: false,
+ cacheable: false,
+ reason: :abort
+ }
+ end
+
+ def failure_response(error_type)
+ {
+ status: :failed,
+ error_type: error_type,
+ retryable: false,
+ cacheable: false,
+ reason: reason_for(error_type)
+ }
+ end
+ end
+end
diff --git a/app/services/statements/extracted_parity_comparison_service.rb b/app/services/statements/extracted_parity_comparison_service.rb
new file mode 100644
index 00000000..fd9c9545
--- /dev/null
+++ b/app/services/statements/extracted_parity_comparison_service.rb
@@ -0,0 +1,207 @@
+module Statements
+ class ExtractedParityComparisonService
+ DEFAULT_PROPERTY_IDS = [1, 3, 5, 13].freeze
+
+ Result = Struct.new(
+ :url,
+ :webpage,
+ :property_ids,
+ :legacy_html_present,
+ :condenser_html_present,
+ :sources_count,
+ :groups,
+ :counts,
+ keyword_init: true
+ )
+
+ def self.call(**kwargs)
+ init_kwargs = kwargs.slice(:refresh_helper, :source_resolver)
+ call_kwargs = kwargs.except(:refresh_helper, :source_resolver)
+ new(**init_kwargs).call(**call_kwargs)
+ end
+
+ def initialize(refresh_helper: nil, source_resolver: Statements::RefreshWebpageStatementsService)
+ @refresh_helper = refresh_helper || StatementsHelper.build_refresh_proxy(cookies: {})
+ @source_resolver = source_resolver
+ end
+
+ def call(webpage:, default_language: "en", legacy_html:, condenser_html:, sources: nil, property_ids: DEFAULT_PROPERTY_IDS)
+ scoped_property_ids = normalized_property_ids(property_ids)
+ source_list = Array(sources).presence || source_resolver.sources_for_webpage(
+ webpage,
+ default_language: default_language,
+ property_ids: scoped_property_ids
+ )
+ rows = source_list.sort_by { |source| [source.property_id.to_i, source.id.to_i] }.map do |source|
+ compare_source(source, webpage, legacy_html: legacy_html, condenser_html: condenser_html)
+ end
+ groups = rows.group_by { |row| row[:group] }
+
+ Result.new(
+ url: webpage.url,
+ webpage: webpage,
+ property_ids: scoped_property_ids,
+ legacy_html_present: legacy_html.present?,
+ condenser_html_present: condenser_html.present?,
+ sources_count: source_list.count,
+ groups: {
+ same: groups.fetch(:same, []),
+ added: groups.fetch(:added, []),
+ removed: groups.fetch(:removed, []),
+ changed: groups.fetch(:changed, []),
+ extraction_errors: groups.fetch(:extraction_errors, [])
+ },
+ counts: {
+ same: groups.fetch(:same, []).count,
+ added: groups.fetch(:added, []).count,
+ removed: groups.fetch(:removed, []).count,
+ changed: groups.fetch(:changed, []).count,
+ extraction_errors: groups.fetch(:extraction_errors, []).count
+ }
+ )
+ end
+
+ private
+
+ attr_reader :refresh_helper, :source_resolver
+
+ def compare_source(source, webpage, legacy_html:, condenser_html:)
+ legacy = extract_value(source, webpage, html: legacy_html, side: :legacy)
+ condenser = extract_value(source, webpage, html: condenser_html, side: :condenser)
+
+ {
+ source_id: source.id,
+ statement_id: Statement.find_by(webpage_id: webpage.id, source_id: source.id)&.id,
+ property_label: source.property.label,
+ source_label: source_label(source),
+ legacy: legacy[:value],
+ condenser: condenser[:value],
+ legacy_error: legacy[:error],
+ condenser_error: condenser[:error],
+ group: classify_group(legacy, condenser)
+ }
+ end
+
+ def extract_value(source, webpage, html:, side:)
+ if html.blank?
+ return {
+ value: nil,
+ error: {
+ error: "#{side.to_s.humanize} HTML is not available for comparison",
+ error_type: "MissingComparisonHtml",
+ source: "statement_parity_compare"
+ }
+ }
+ end
+
+ runner = Dsl::Core::AlgorithmRunner.new(
+ url: webpage.url,
+ render_js: source.render_js,
+ scrape_options: refresh_helper.statement_scrape_options(
+ source: source,
+ webpage: webpage,
+ scrape_options: { force_scrape_every_hrs: 0 }
+ ),
+ tracer: Dsl::Tracing::NullTracer.new
+ )
+ runner.instance_variable_set(:@html, html)
+ runner.instance_variable_set(:@page, Nokogiri::HTML(html, nil, Encoding::UTF_8.to_s))
+ runner.define_singleton_method(:fetch_result_for) do |url:, render_js:, scrape_options:|
+ {
+ status: :abort,
+ body: [
+ "abort_update",
+ {
+ error: "Read-only statement parity comparison cannot fetch additional URLs",
+ error_type: "StatementParityFetchUnsupported",
+ source: "statement_parity_compare",
+ step: "url"
+ }
+ ],
+ headers: {},
+ final_url: url,
+ redirect_chain: [url],
+ wringer: {
+ error_type: "StatementParityFetchUnsupported",
+ source: "statement_parity_compare",
+ signals: {},
+ hints: ["read_only_compare"],
+ final_url: url,
+ redirect_chain: [url]
+ },
+ duration_ms: 0
+ }
+ end
+
+ raw = runner.run(source.algorithm_value)
+ return { value: nil, error: raw.second.to_h } if abort_structure?(raw)
+
+ formatted = refresh_helper.format_datatype(raw, source.property, webpage, statement_status: "initial")
+ return { value: nil, error: formatted.second.to_h } if abort_structure?(formatted)
+
+ value = blank_value?(formatted) ? nil : formatted
+ { value: value, error: nil }
+ rescue StandardError => error
+ {
+ value: nil,
+ error: {
+ error: error.message,
+ error_type: error.class.to_s,
+ source: "statement_parity_compare"
+ }
+ }
+ end
+
+ def classify_group(legacy, condenser)
+ return :extraction_errors if legacy[:error].present? || condenser[:error].present?
+ return :same if values_equal?(legacy[:value], condenser[:value])
+ return :added if blank_value?(legacy[:value]) && present_value?(condenser[:value])
+ return :removed if present_value?(legacy[:value]) && blank_value?(condenser[:value])
+
+ :changed
+ end
+
+ def values_equal?(left, right)
+ normalize_value(left) == normalize_value(right)
+ end
+
+ def normalize_value(value)
+ case value
+ when Array
+ value.map { |entry| normalize_value(entry) }
+ when Hash
+ value.transform_values { |entry| normalize_value(entry) }
+ else
+ value
+ end
+ end
+
+ def blank_value?(value)
+ return true if value.blank?
+ return value.all? { |entry| blank_value?(entry) } if value.is_a?(Array)
+ return value.values.all? { |entry| blank_value?(entry) } if value.is_a?(Hash)
+
+ false
+ end
+
+ def present_value?(value)
+ !blank_value?(value)
+ end
+
+ def abort_structure?(value)
+ value.is_a?(Array) && value.first == "abort_update" && value.second.respond_to?(:to_h)
+ end
+
+ def source_label(source)
+ "#{source.property.label} (source ##{source.id})"
+ end
+
+ def normalized_property_ids(property_ids)
+ ids = Array(property_ids).filter_map do |value|
+ integer = value.to_i
+ integer if integer.positive?
+ end
+ ids.presence || DEFAULT_PROPERTY_IDS
+ end
+ end
+end
diff --git a/app/services/statements/refresh_webpage_statements_service.rb b/app/services/statements/refresh_webpage_statements_service.rb
new file mode 100644
index 00000000..1def4128
--- /dev/null
+++ b/app/services/statements/refresh_webpage_statements_service.rb
@@ -0,0 +1,70 @@
+module Statements
+ class RefreshWebpageStatementsService
+ def self.sources_for_webpage(webpage, default_language: "en", property_ids: nil)
+ languages = [webpage.language]
+ languages << "" if webpage.language == default_language
+
+ scoped_property_ids = property_ids_for_class_name(webpage.rdfs_class.name, [])
+ if property_ids.present?
+ allowed_ids = Array(property_ids).map(&:to_i)
+ scoped_property_ids.select! { |property_id| allowed_ids.include?(property_id.to_i) }
+ end
+
+ scoped_property_ids.flat_map do |property_id|
+ Source.where(website_id: webpage.website_id, language: languages, property_id: property_id)
+ end
+ end
+
+ def self.property_ids_for_class_name(rdfs_class_name, property_ids)
+ class_list = rdfs_class_name.to_s.split(",")
+ class_list.each do |class_name|
+ rdfs_class = RdfsClass.where(name: class_name).first
+ next unless rdfs_class
+
+ rdfs_class.properties.each do |property|
+ property_ids << property.id
+ if property.value_datatype == "bnode" && property.expected_class != rdfs_class_name
+ property_ids_for_class_name(property.expected_class, property_ids)
+ end
+ end
+ end
+ property_ids
+ end
+
+ def initialize(refresh_helper: ApplicationController.helpers)
+ @refresh_helper = refresh_helper
+ end
+
+ def call(webpage:, default_language: "en", scrape_options: {})
+ error_list = []
+ refresh_options = normalized_scrape_options(scrape_options)
+
+ self.class.sources_for_webpage(webpage, default_language: default_language).each do |src|
+ stat = Statement.find_or_initialize_by(webpage_id: webpage.id, source_id: src.id)
+ if stat.new_record?
+ source_is_manual = src.algorithm_value.start_with?("manual=")
+ stat.manual = source_is_manual
+ stat.selected_individual = src.selected
+ stat.status = "initial"
+ stat.status_origin = "condenser_create"
+ end
+
+ next if stat.manual && %w[ok updated].include?(stat.status)
+
+ @refresh_helper.refresh_statement_helper(stat, refresh_options.dup)
+ stat.update(status: "updated") if src.auto_review && stat.status == "initial"
+ error_list << { "Property id #{src.property_id}" => stat.errors.messages } if stat.errors.any?
+ end
+
+ error_list
+ end
+
+ private
+
+ def normalized_scrape_options(scrape_options)
+ return {} unless scrape_options.respond_to?(:to_h)
+
+ scrape_options.to_h.symbolize_keys
+ end
+ end
+end
diff --git a/app/services/wringer_client.rb b/app/services/wringer_client.rb
new file mode 100644
index 00000000..d4e5087f
--- /dev/null
+++ b/app/services/wringer_client.rb
@@ -0,0 +1,107 @@
+# WringerClient is a normalization layer for Wringer responses.
+# It does NOT change how Wringer is called.
+# It is NOT yet integrated in main flow.
+# It prepares for future observability and DSL trace integration.
+class WringerClient
+ def self.fetch(url)
+ new(url).fetch
+ end
+
+ def initialize(url = nil)
+ @url = url
+ end
+
+ def fetch
+ start_time = Time.now
+
+ response = call_wringer
+
+ {
+ url: @url,
+ status: extract_status(response),
+ html: extract_html(response),
+ final_url: extract_final_url(response),
+ redirect_chain: extract_redirect_chain(response),
+ signals: extract_signals(response),
+ error: extract_error(response),
+ duration_ms: ((Time.now - start_time) * 1000).to_i
+ }
+ rescue StandardError => e
+ {
+ url: @url,
+ status: :error,
+ html: nil,
+ final_url: nil,
+ redirect_chain: [],
+ signals: {},
+ error: {
+ type: e.class.name,
+ message: e.message
+ },
+ duration_ms: ((Time.now - start_time) * 1000).to_i
+ }
+ end
+
+ private
+
+ def call_wringer
+ dsl_client = Dsl::Support::WringerClient.new(
+ agent: Mechanize.new,
+ render_js: false,
+ scrape_options: {},
+ use_wringer: ApplicationController.helpers.method(:use_wringer),
+ safe_wringer_call: ApplicationController.helpers.method(:safe_wringer_call),
+ logger: Rails.logger
+ )
+
+ result = dsl_client.fetch(url: @url)
+
+ result
+ end
+
+ def extract_status(response)
+ return :error if response.nil?
+ body = response[:body]
+
+ if body.is_a?(Array) && body.first == "abort_update"
+ :error
+ else
+ :ok
+ end
+ end
+
+ def extract_html(response)
+ body = response[:body]
+ return nil if error_response?(body)
+
+ body
+ end
+
+ def extract_final_url(response)
+ response[:final_url]
+ end
+
+ def extract_redirect_chain(response)
+ response[:redirect_chain] || []
+ end
+
+ def extract_signals(response)
+ response[:wringer] || {}
+ end
+
+ def extract_error(response)
+ body = response[:body]
+ return nil unless error_response?(body)
+
+ payload = body[1] rescue nil
+
+ {
+ type: payload && payload[:error_type] ? payload[:error_type].to_s : "unknown",
+ message: payload && payload[:error] ? payload[:error].to_s : "unknown error"
+ }
+ end
+
+ def error_response?(response)
+ response.is_a?(Array) && response.first == "abort_update"
+ end
+end
diff --git a/app/views/distillator/cache/compare.html.erb b/app/views/distillator/cache/compare.html.erb
new file mode 100644
index 00000000..295e2f9b
--- /dev/null
+++ b/app/views/distillator/cache/compare.html.erb
@@ -0,0 +1,119 @@
+Cache Comparison
+
+Read-only comparison only. No cache refresh was performed. Legacy data may be read from the configured Wringer cache endpoint.
+
+URI: <%= external_website_link(@comparison[:uri]) %>
+URI key: <%= @comparison[:uri_key] || "Unavailable" %>
+Comparison policy: <%= @comparison[:comparison_policy].to_s.humanize %> mode
+
+ <%= link_to "Operator mode", condenser_cache_compare_path(uri: @comparison[:uri], comparison_policy: "operator") %> |
+ <%= link_to "Strict mode", condenser_cache_compare_path(uri: @comparison[:uri], comparison_policy: "strict") %>
+
+
+
+ Summary
+ Outcome: <%= cache_compare_outcome_label(@comparison[:summary]) %>
+ Main reason: <%= @comparison.dig(:summary, :primary_reason) %>
+ Recommended action:
+ <% case @comparison.dig(:summary, :outcome).to_s %>
+ <% when "blocked" %>
+ Review the safety blockers before activating.
+ <% when "review" %>
+ Review the grouped differences, then decide whether the site is safe to activate.
+ <% when "ready_with_metadata_notes" %>
+ Metadata notes only. Activation can proceed when the rest of the evidence is ready.
+ <% else %>
+ No comparison action is needed right now.
+ <% end %>
+
+
+
+
+ Key evidence
+ Legacy source: <%= @comparison[:legacy_source] %>
+ Legacy lookup error: <%= @comparison[:legacy_lookup_error] || "None" %>
+ Condenser source: <%= @comparison[:condenser_source] %>
+ Legacy cache missing: <%= @comparison.dig(:missing, :legacy) %>
+ Condenser cache missing: <%= @comparison.dig(:missing, :condenser) %>
+ HTTP codes: legacy <%= @comparison.dig(:legacy_cache, :http_code).presence || "unknown" %> / condenser <%= @comparison.dig(:condenser_cache, :http_code).presence || "unknown" %>
+ Title match: <%= @comparison.dig(:diffs, :title, :same) ? "yes" : "no" %>
+ Content success: legacy <%= @comparison.dig(:legacy_cache, :content_success).inspect %> / condenser <%= @comparison.dig(:condenser_cache, :content_success).inspect %>
+ Transport success: legacy <%= @comparison.dig(:legacy_cache, :transport_success).inspect %> / condenser <%= @comparison.dig(:condenser_cache, :transport_success).inspect %>
+ Latest refresh: legacy <%= @comparison.dig(:legacy_cache, :successful_refresh).presence || "unknown" %> / condenser <%= @comparison.dig(:condenser_cache, :successful_refresh).presence || "unknown" %>
+ Links:
+ <%= safe_join(cache_inspection_links(@comparison[:uri]).map { |link| link_to(link[:label], link[:url]) }, " | ") %>
+
+
+
+
+ Differences grouped by importance
+ <% grouped_diffs = {
+ "Safety blockers" => :blocking_regression,
+ "Review-needed differences" => :review_needed,
+ "Metadata-only differences" => :metadata_only,
+ "Improvements" => :distillator_improvement
+ } %>
+ <% grouped_diffs.each do |label, classification| %>
+ <% rows = @comparison[:diffs].select { |_field, diff| diff[:classification] == classification } %>
+ <% next if rows.empty? && label == "Metadata-only differences" %>
+ <%= label %>
+ <% if rows.empty? %>
+ None.
+ <% else %>
+
+
+
+ | Field |
+ Legacy |
+ Condenser |
+
+
+
+ <% rows.each do |field, diff| %>
+
+ | <%= field %> |
+ <%= JSON.pretty_generate(diff[:legacy]) rescue diff[:legacy].inspect %> |
+ <%= JSON.pretty_generate(diff[:condenser]) rescue diff[:condenser].inspect %> |
+
+ <% end %>
+
+
+ <% end %>
+ <% end %>
+
+
+
+ Detailed comparison
+
+
+
+ | Field |
+ Same? |
+ Status |
+ Legacy |
+ Condenser |
+
+
+
+ <% @comparison[:diffs].each do |field, diff| %>
+
+ | <%= field %> |
+ <%= diff[:same] %> |
+ <%= diff[:classification] %> |
+ <%= JSON.pretty_generate(diff[:legacy]) rescue diff[:legacy].inspect %> |
+ <%= JSON.pretty_generate(diff[:condenser]) rescue diff[:condenser].inspect %> |
+
+ <% end %>
+
+
+
+
+
+ Raw payloads
+
+
Legacy payload
+
<%= JSON.pretty_generate(@comparison[:legacy_cache]) rescue @comparison[:legacy_cache].inspect %>
+
Condenser payload
+
<%= JSON.pretty_generate(@comparison[:condenser_cache]) rescue @comparison[:condenser_cache].inspect %>
+
diff --git a/app/views/distillator/cache/index.html.erb b/app/views/distillator/cache/index.html.erb
new file mode 100644
index 00000000..deccecb1
--- /dev/null
+++ b/app/views/distillator/cache/index.html.erb
@@ -0,0 +1,62 @@
+Condenser Cache
+
+<% if refresh_ui_enabled? %>
+ Fetch and refresh cache entries from this page.
+<% else %>
+ Cache inspection mode. Refresh actions are not available in this environment.
+<% end %>
+<% if @last_action_result.present? %>
+
+
Last Action Result
+
Fetch mode: <%= @last_action_result["fetch_kind_label"] %>
+
Normalized URL: <%= @last_action_result["normalized_url"] %>
+
HTTP response code: <%= @last_action_result["http_response_code"] || "Not recorded" %>
+
Fetch path: <%= @last_action_result["fetch_path"].presence || "unknown" %>
+
Cache reason: <%= @last_action_result["cache_reason"].presence || "unknown" %>
+
HTML result: <%= @last_action_result["html_state"].presence || "unknown" %>
+
Scrape date means the last attempted scrape. Successful refresh means the last successful 2xx content refresh.
+Cache rows are operational fetch records, not manually edited website records. Refresh instead of edit or destroy.
+
+<%= render "shared/cards/card_grid",
+ cards: cache_operator_summary_cards(@summary_cards),
+ variant: :summary,
+ css_class: "cache-summary-cards" %>
+
+<%= render "shared/tables/toolbar", preserved_index_params: @preserved_index_params, view_mode: @view_mode, refresh_ui_enabled: refresh_ui_enabled? %>
+
+<% sortable_filters = @filters.merge(per_page: @pagination[:per_page]) %>
+<%= render "shared/tables/table_shell",
+ caches: @caches,
+ filters: @filters,
+ pagination: @pagination,
+ sort: @sort,
+ direction: @direction,
+ view_mode: @view_mode,
+ sortable_filters: sortable_filters,
+ preserved_index_params: @preserved_index_params,
+ refresh_ui_enabled: refresh_ui_enabled? %>
+
+
+
+
+
+
+
Compatibility API
+
+ - Legacy-compatible endpoint:
/websites/wring.json?uri=URL
+ - Cached Wringer-compatible JSON: available from each cache row’s
.json link
+ - Cache fetch actions: use
Direct fetch, Rendered fetch, and POST fetch from the cache actions.
+
+
Cache Refresh Diagnostic
+
+Diagnostic only. No fetch is performed from this page.
+
+<%= form_with url: preview_distillator_cache_index_path, method: :get, local: true do %>
+
+ <%= label_tag :uri, "URI" %>
+ <%= text_field_tag :uri, params[:uri], size: 100 %>
+
+
+ <%= label_tag :fetch_kind, "Fetch mode" %>
+ <%= select_tag :fetch_kind, options_for_select([["normal", "normal"], ["rendered", "rendered"], ["post", "post"]], params[:fetch_kind].presence || "normal") %>
+
+
+ <%= label_tag :include_fragment, "Include fragment" %>
+ <%= check_box_tag :include_fragment, "true", params[:include_fragment].to_s == "true" %>
+
+
+ <%= label_tag :force_scrape, "Force scrape" %>
+ <%= check_box_tag :force_scrape, "true", params[:force_scrape].to_s == "true" %>
+
+
+ <%= label_tag :force_scrape_every_hrs, "Force scrape every hrs" %>
+ <%= text_field_tag :force_scrape_every_hrs, params[:force_scrape_every_hrs] %>
+
+ <%= submit_tag "Run diagnostic" %>
+<% end %>
+
+<% if params[:uri].present? %>
+ Diagnostic Result
+ Chosen fetch mode: <%= cache_fetch_kind_label(@preview[:fetch_kind]) %> (<%= @preview[:fetch_kind] %>)
+ Would refresh? <%= @preview[:would_refresh] %>
+ Reason: <%= @preview[:reason] %>
+ Cache exists? <%= @preview[:cache_exists] %>
+ Scrape date: <%= cache_timestamp_label(@preview[:scrape_date]) %>
+ Successful refresh: <%= cache_timestamp_label(@preview[:successful_refresh]) %>
+ HTTP response code: <%= @preview[:http_response_code] %>
+ URI key: <%= @preview[:uri_key] %>
+ Normalized URL: <%= @preview[:normalized_url] %>
+ Diagnostic JSON: <%= link_to "Open diagnostic JSON", "#{preview_distillator_cache_index_path(format: :json)}?#{request.query_parameters.to_query}" %>
+ <% if refresh_ui_enabled? %>
+
+ <%= button_to cache_fetch_kind_label(@preview[:fetch_kind]), fetch_distillator_cache_index_path, method: :post, params: {
+ uri: @preview[:normalized_url] || params[:uri],
+ fetch_kind: cache_fetch_kind_param(@preview[:fetch_kind]),
+ include_fragment: params[:include_fragment],
+ force_scrape: params[:force_scrape].presence || "true",
+ force_scrape_every_hrs: params[:force_scrape_every_hrs]
+ }, form_class: "cache-refresh-button" %>
+
+ <% end %>
+<% end %>
diff --git a/app/views/distillator/cache/raw_view.html.erb b/app/views/distillator/cache/raw_view.html.erb
new file mode 100644
index 00000000..b0c56694
--- /dev/null
+++ b/app/views/distillator/cache/raw_view.html.erb
@@ -0,0 +1,9 @@
+Raw Cached HTML
+
+Warning: cached HTML only; no refresh performed.
+Safety: the raw endpoint is sandboxed to reduce app-origin script execution.
+Scrape date: <%= cache_timestamp_label(@cache_payload[:scrape_date]) %>
+Successful refresh: <%= cache_timestamp_label(@cache_payload[:successful_refresh]) %>
+<%= link_to "Open raw cached HTML endpoint", raw_distillator_cache_path(@cache) %>
+
+<%= @cache.html.to_s %>
diff --git a/app/views/distillator/cache/show.html.erb b/app/views/distillator/cache/show.html.erb
new file mode 100644
index 00000000..53c3df94
--- /dev/null
+++ b/app/views/distillator/cache/show.html.erb
@@ -0,0 +1,173 @@
+Condenser Cache Detail
+
+<% if refresh_ui_enabled? %>
+ Fetch and refresh cache entries from this page.
+<% else %>
+ Cache inspection mode. Refresh actions are not available in this environment.
+<% end %>
+Cache rows are operational fetch records, not manually edited website records. Refresh instead of edit or destroy.
+
+<%= render "shared/cards/card_grid",
+ variant: :record,
+ css_class: "cache-detail-cards",
+ cards: [
+ {
+ title: "Status card",
+ status: { label: "Operator health", value: cache_health_badge(@cache_payload) },
+ rows: [
+ { label: "HTTP status", value: cache_status_badge(@cache_payload), html: true },
+ { label: "Fetch diagnosis", value: @cache_payload[:health_label] },
+ { label: "Fetched response", value: cache_fetch_response_summary(@cache_payload) },
+ { label: "Fetched body", value: cache_fetched_body_summary(@cache_payload) },
+ { label: "Stored cache body", value: cache_stored_body_summary(@cache_payload) },
+ { label: "Storage decision", value: cache_storage_decision(@cache_payload) },
+ { label: "Transport", value: cache_transport_status(@cache_payload) },
+ { label: "Content", value: cache_content_status(@cache_payload) },
+ { label: "Blocking issue", value: cache_blocking_issue_key(@cache_payload).presence || "none" },
+ { label: "Last good preserved", value: cache_last_good_content_preserved?(@cache_payload) ? "yes" : "no" },
+ { label: "Health reasons", value: Array(@cache_payload[:health_reasons]).presence&.join(", ") || "None" },
+ { label: "Primary issue match", value: cache_primary_issue_match_summary(@cache_payload).presence || "Not recorded" }
+ ],
+ body: (
+ if cache_content_failure_warning(@cache_payload).present?
+ content_tag(:p, content_tag(:strong, cache_content_failure_warning(@cache_payload)), class: "cache-content-warning") +
+ content_tag(:p, raw("Failed attempt body: not stored separately."))
+ end
+ )
+ },
+ {
+ title: "Identity card",
+ rows: [
+ { label: "Normalized URL", value: cache_linked_source_url(@cache_payload[:normalized_url]), value_class: "cache-normalized-url", html: true },
+ { label: "URI key", value: @cache_payload[:uri_key], value_class: "cache-uri-key" },
+ { label: "URI alias", value: @cache_payload[:uri], value_class: "cache-uri-key" },
+ { label: "Name", value: @cache_payload[:name] },
+ { label: "Final URL", value: cache_linked_final_url(@cache_payload), value_class: "cache-normalized-url", html: true },
+ { label: "Redirect summary", value: cache_redirect_summary(@cache_payload) }
+ ]
+ },
+ {
+ title: "Last fetch card",
+ rows: [
+ { label: "Scrape date", value: cache_timestamp_label(@cache_payload[:scrape_date]) },
+ { label: "Successful refresh", value: cache_timestamp_label(@cache_payload[:successful_refresh]) },
+ { label: "Fetch path", value: @cache_payload[:fetch_path].presence || "unknown" },
+ { label: "Native ineligible reason", value: @cache_payload[:native_ineligible_reason].presence || "None" },
+ { label: "Network status", value: @cache_payload[:network_status].presence || "unknown" },
+ { label: "Content type", value: @cache_payload[:content_type].presence || "unknown" },
+ { label: "Redirect type", value: @cache_payload[:redirect_type].presence || "none" },
+ { label: "Updated at", value: cache_timestamp_label(@cache_payload[:updated_at]) },
+ { label: "Created at", value: cache_timestamp_label(@cache_payload[:created_at]) }
+ ] + (
+ if @last_action_result.present?
+ [
+ { label: "Last fetch mode", value: @last_action_result["fetch_kind_label"] },
+ { label: "Last fetch path", value: @last_action_result["fetch_path"] },
+ { label: "Last cache reason", value: @last_action_result["cache_reason"] },
+ { label: "Last HTML result", value: @last_action_result["html_state"] }
+ ]
+ else
+ []
+ end
+ )
+ },
+ {
+ title: "Content card",
+ rows: [
+ { label: "HTML bytes", value: number_with_delimiter(@cache_payload[:html_bytes]) },
+ { label: "Body bytes", value: number_with_delimiter(@cache_payload[:body_bytes]) },
+ { label: "Has HTML", value: @cache_payload[:has_html] },
+ { label: "Raw cached HTML", value: link_to("Open raw cached HTML", cache_raw_view_url(@cache), data: { copy_url: cache_raw_url(@cache) }), html: true }
+ ],
+ body: content_tag(:div, class: "cache-content-preview") do
+ content_tag(:strong, "Content sample:") + content_tag(:p, cache_html_preview_text(@cache_payload), class: "harmonized-card-value")
+ end
+ }
+ ] %>
+
+
+ Diagnostics card
+
+
Signals
+
<%= JSON.pretty_generate(@cache_payload[:signals] || {}) %>
+
+
Hints
+ <% if cache_hints(@cache_payload).any? %>
+
+ <% cache_hints(@cache_payload).each do |hint| %>
+ - <%= hint %>
+ <% end %>
+
+ <% else %>
+
None.
+ <% end %>
+
+
Redirect Chain
+ <% if cache_redirect_chain(@cache_payload).any? %>
+
+ <% cache_redirect_chain(@cache_payload).each do |hop| %>
+ - <%= hop %>
+ <% end %>
+
+ <% else %>
+
None.
+ <% end %>
+
+
+
+ Compatibility card
+
+
Compatibility API
+
+ - Legacy-compatible endpoint:
/websites/wring.json?uri=URL
+ - Cached Wringer-compatible JSON: <%= link_to "Open cached Wringer-compatible JSON", cache_wring_json_view_url(@cache), data: { copy_url: cache_wring_json_url(@cache) } %>
+ - Compare: <%= link_to "Open cache comparison", cache_compare_url(@cache), data: { copy_url: cache_compare_url(@cache) } %>
+ - Legacy JSON compatibility URL: <%= link_to "Open legacy compatibility JSON", legacy_wring_json_url(@cache), data: { copy_url: legacy_wring_json_url(@cache) } %>
+
+
+
Compatibility actions
+ <%= render "shared/cards/action_bar",
+ groups: [
+ {
+ title: "View",
+ links: [
+ { label: "Show", url: distillator_cache_path(@cache), options: { data: { copy_url: distillator_cache_path(@cache) } } },
+ { label: "Raw", url: cache_raw_view_url(@cache), options: { data: { copy_url: cache_raw_url(@cache) } } },
+ { label: ".json", url: cache_wring_json_url(@cache), options: { data: { copy_url: cache_wring_json_url(@cache) } } },
+ { label: "Pretty JSON", url: cache_wring_json_view_url(@cache), options: { data: { copy_url: cache_wring_json_url(@cache) } } }
+ ]
+ },
+ {
+ title: "Diagnose",
+ links: [
+ { label: "Compare", url: cache_compare_url(@cache), options: { data: { copy_url: cache_compare_url(@cache) } } }
+ ]
+ }
+ ] %>
+
+
+<% if refresh_ui_enabled? %>
+
+ Refresh
+ <%= button_to cache_fetch_kind_label("normal"), fetch_distillator_cache_index_path, method: :post, params: { uri: @cache.normalized_url, fetch_kind: "normal", force_scrape: true }, form_class: "cache-refresh-button" %>
+ <%= button_to cache_fetch_kind_label("rendered"), fetch_distillator_cache_index_path, method: :post, params: { uri: @cache.normalized_url, fetch_kind: "rendered", force_scrape: true, include_fragment: true }, form_class: "cache-refresh-button" %>
+ <%= button_to cache_fetch_kind_label("post"), fetch_distillator_cache_index_path, method: :post, params: { uri: @cache.normalized_url, fetch_kind: "post", force_scrape: true }, form_class: "cache-refresh-button" %>
+
+<% end %>
+
+<% if @preview.present? %>
+
+
Refresh Diagnostic
+
Chosen fetch mode: <%= @preview[:fetch_kind] %>
+
Would refresh? <%= @preview[:would_refresh] %>
+
Reason: <%= @preview[:reason] %>
+
Cache exists? <%= @preview[:cache_exists] %>
+
Scrape date: <%= cache_timestamp_label(@preview[:scrape_date]) %>
+
Successful refresh: <%= cache_timestamp_label(@preview[:successful_refresh]) %>
+
HTTP response code: <%= @preview[:http_response_code] %>
+
URI key: <%= @preview[:uri_key] %>
+
Normalized URL: <%= @preview[:normalized_url] %>
+
Cached Wringer-compatible JSON
+
+This is the exact legacy-compatible cached payload. No fetch was performed.
+Scrape date: <%= cache_timestamp_label(@cache_payload[:scrape_date]) %>
+Successful refresh: <%= cache_timestamp_label(@cache_payload[:successful_refresh]) %>
+<%= link_to "Open cached Wringer-compatible JSON endpoint", wring_json_distillator_cache_path(@cache) %>
+
+<%= JSON.pretty_generate(@wring_payload) %>
diff --git a/app/views/distillator/capabilities/index.html.erb b/app/views/distillator/capabilities/index.html.erb
new file mode 100644
index 00000000..ad2aa822
--- /dev/null
+++ b/app/views/distillator/capabilities/index.html.erb
@@ -0,0 +1,62 @@
+Condenser / Distillator System Map
+
+This page is read-only. It documents the current Condenser, Distillator, and legacy Wringer surfaces without fetching, refreshing, exporting, or changing rollout state.
+
+
+ Activation readiness ladder
+
+
+
+ | Step |
+ Purpose |
+ Operator surface |
+ Status badges |
+
+
+
+ <% @capability_map.activation_readiness_ladder.each do |step| %>
+
+ | <%= step.name %> |
+ <%= step.purpose %> |
+ <%= step.operator_surface %> |
+ <%= capability_badges(step.badges) %> |
+
+ <% end %>
+
+
+
+
+<% @capability_map.sections.each do |section| %>
+
+ <%= section.title %>
+
+
+
+ | Feature name |
+ Purpose |
+ Implementation status |
+ Wringer dependency / parity status |
+ Transition relevance |
+ Evidence source |
+ Operator link |
+
+
+
+ <% section.features.each do |feature| %>
+
+ |
+ <%= feature.name %>
+ <%= capability_badges(feature.badges) %>
+ |
+ <%= feature.purpose %> |
+ <%= feature.implementation_status %> |
+ <%= feature.wringer_dependency_status %> |
+ <%= feature.transition_relevance %> |
+ <%= feature.evidence_source %> |
+ <%= link_to feature.operator_label, feature.operator_path %> |
+
+ <% end %>
+
+
+
+<% end %>
diff --git a/app/views/distillator/shadow_reports/index.html.erb b/app/views/distillator/shadow_reports/index.html.erb
new file mode 100644
index 00000000..28d0b018
--- /dev/null
+++ b/app/views/distillator/shadow_reports/index.html.erb
@@ -0,0 +1,109 @@
+Transition Report
+
+This dashboard is read-only. Use it to review legacy, shadow, and active rollout state without triggering fetches.
+Transition checks are run by batch job. This page displays the latest result for each website.
+<% if shadow_report_staging_rollout_warning.present? %>
+ <%= shadow_report_staging_rollout_warning %>
+<% end %>
+
+<%= render "shared/cards/card_grid",
+ cards: transition_dashboard_cards(@dashboard_counts),
+ variant: :summary,
+ css_class: "cache-summary-cards shadow-report-summary-cards" %>
+
+Top blockers on this page
+<%= render "shared/cards/card_grid",
+ cards: transition_blocker_cards(@blocker_counts),
+ variant: :summary,
+ css_class: "cache-summary-cards shadow-report-summary-cards" %>
+
+Current page
+<%= render "shared/cards/card_grid",
+ cards: shadow_report_summary_cards(@summary_counts, title_prefix: "Visible"),
+ variant: :summary,
+ css_class: "cache-summary-cards shadow-report-summary-cards" %>
+
+<%= form_with url: distillator_shadow_report_path, method: :get, local: true, html: { autocomplete: "off", class: "harmonized-table-filters" } do %>
+ <%= hidden_field_tag :sort, @sort if @sort.present? %>
+ <%= hidden_field_tag :direction, @direction if @direction.present? %>
+ <%= hidden_field_tag :limit, @pagination[:per_page] if @pagination[:per_page].present? %>
+
+
+
+
+ <% sortable_filters = @filters.merge(limit: @pagination[:per_page]) %>
+ | <%= admin_sortable("website", "Website", filters: sortable_filters, current_sort: @sort, current_direction: @direction) %> |
+ Cohort |
+ Mode |
+ Production backend |
+ <%= admin_sortable("status", "Readiness", filters: sortable_filters, current_sort: @sort, current_direction: @direction) %> |
+ Safety |
+ Confidence |
+ Severity |
+ Primary blocker |
+ Last checked |
+ Next action |
+ Actions |
+
+
+ | <%= text_field_tag :term, @filters[:term], autocomplete: "off" %> |
+ <%= select_tag :cohort, options_for_select(shadow_report_cohort_options, @filters[:cohort]) %> |
+ <%= select_tag :mode, options_for_select(shadow_report_mode_options, @filters[:mode]) %> |
+ |
+ <%= select_tag :status, options_for_select(shadow_report_status_options, @filters[:status] || @filters[:recommendation]) %> |
+ <%= select_tag :safety, options_for_select(shadow_report_safety_options, @filters[:safety]) %> |
+ <%= select_tag :confidence, options_for_select(shadow_report_confidence_options, @filters[:confidence]) %> |
+ |
+ |
+ |
+ |
+ <%= select_tag :promotable, options_for_select(shadow_report_promotable_options, @filters[:promotable]) %> |
+
+ <%= submit_tag "Apply filters" %>
+ <%= link_to "Reset filters", distillator_shadow_report_path %>
+ |
+
+
+
+ <% if @rows.any? %>
+ <% @rows.each do |row| %>
+
+ |
+ <%= row.website.name %>
+ <%= row.website.seedurl %>
+ |
+ <%= shadow_report_cohort_label(row) %> |
+ <%= shadow_report_mode_label(row) %> |
+ <%= shadow_report_production_backend_label(row) %> |
+ <%= shadow_report_status_label(row) %> |
+ <%= shadow_report_safety_label(row) %> |
+ <%= shadow_report_confidence_label(row) %> |
+ <%= shadow_report_severity_label(row) %> |
+ <%= row.primary_blocker.presence || "None recorded" %> |
+ <%= shadow_report_timestamp(row.last_checked) %> |
+ <%= row.primary_action %> |
+ <%= shadow_report_action_links(row) %> |
+
+ <% end %>
+ <% else %>
+
+ | No sites found. |
+
+ <% end %>
+
+
+<% end %>
+
+
+
+
diff --git a/app/views/distillator/shadow_reports/show.html.erb b/app/views/distillator/shadow_reports/show.html.erb
new file mode 100644
index 00000000..25776c96
--- /dev/null
+++ b/app/views/distillator/shadow_reports/show.html.erb
@@ -0,0 +1,302 @@
+Transition Report Detail
+
+This page is read-only. Transition checks are run by batch job; this page displays the latest recorded result.
+
+
+ Summary
+ Is this site ready? <%= shadow_report_decision_label(@detail.decision) %>
+ Primary blocker: <%= @detail.primary_blocker&.headline.presence || "No blocking issue is currently recorded." %>
+ Latest transition evidence: <%= shadow_report_timestamp(@detail.latest_transition_evidence_checked_at) %>
+ <% if @detail.pending_transition_batch_check %>
+
+ Batch check requested <%= time_ago_in_words(@detail.transition_check_requested_at) %> ago.
+ Latest evidence is still from <%= shadow_report_timestamp(@detail.latest_transition_evidence_checked_at) %>.
+
+ Refresh this page to check again.
+ <% end %>
+ Checked scope: <%= shadow_report_checked_pages_summary(@detail.checked_scope) %>
+ Links:
+ <%= link_to "Website batch jobs", website_path(@website, anchor: "website-batch-jobs-transition") %> |
+ <%= link_to "Inspect publishable event pages", webpages_path(seedurl: @website.seedurl) %> |
+ <%= link_to "Open cache diagnostics", website_cache_diagnostics_path(@website) %>
+
+
+
+
+ Decision
+ Decision: <%= shadow_report_decision_label(@detail.decision) %>
+ Safety: <%= @detail.transition_status.safety.to_s.humanize %>
+ Confidence: <%= @detail.transition_status.confidence.to_s.humanize %>
+ Why: <%= @detail.decision[:why].first %>
+ Recommended action: <%= @detail.decision[:recommended_action] %>
+
+ <% @detail.transition_status.checks.each do |check| %>
+ -
+ <%= shadow_report_check_icon(check[:state]) %>
+ <%= check[:label] %>
+ <% if check[:role].present? %>
+ (<%= check[:role].to_s.humanize.downcase %>)
+ <% end %>
+ — <%= shadow_report_check_label(check[:state]) %>
+
+ <% end %>
+
+
+
+<% if @detail.primary_blocker.present? || shadow_report_statement_failure_groups(@detail, severity: :blocker).any? || shadow_report_statement_failure_groups(@detail, severity: :warning).any? %>
+
+ Main blocker
+ <%= shadow_report_primary_blocker_heading(@detail.primary_blocker) %>
+ <% shadow_report_main_blocker_lines(@detail).each do |line| %>
+ <%= line %>
+ <% end %>
+ <% blocking_groups = shadow_report_statement_failure_groups(@detail, severity: :blocker) %>
+ <% warning_groups = shadow_report_statement_failure_groups(@detail, severity: :warning) %>
+ <% if blocking_groups.any? || warning_groups.any? %>
+ Statement failure summary
+ <% if blocking_groups.any? %>
+ Blocking statement failures
+
+ <% blocking_groups.each do |group| %>
+ -
+ <%= [group[:reason], group[:source], "#{group[:count]} affected"].compact.join(" - ") %>
+ <% if group[:statement_ids].any? %>
+
+ Raw statement IDs
+ <%= group[:statement_ids].join(", ") %>
+
+ <% end %>
+
+ <% end %>
+
+ <% end %>
+ <% if warning_groups.any? %>
+ Optional statement warnings
+
+ <% warning_groups.each do |group| %>
+ -
+ <%= [group[:reason], group[:source], "#{group[:count]} affected"].compact.join(" - ") %>
+ <% if group[:statement_ids].any? %>
+
+ Raw statement IDs
+ <%= group[:statement_ids].join(", ") %>
+
+ <% end %>
+
+ <% end %>
+
+ <% end %>
+ <% end %>
+ <% blocker_links = shadow_report_primary_blocker_links(@detail) %>
+ <% if blocker_links.any? %>
+ Next diagnostic links: <%= safe_join(blocker_links, " | ") %>
+ <% end %>
+ <%= link_to "Open website batch jobs", website_path(@website, anchor: "website-batch-jobs-transition") %>
+
+<% end %>
+
+
+ Sampled URL diagnostics
+ <% if @detail.url_matrix.any? %>
+
+
+
+ | URL |
+ Condenser fetch result |
+ Legacy lookup result |
+ Cache comparison result |
+ Statement check result |
+ Export impact |
+ Actions |
+
+
+
+ <% @detail.url_matrix.each do |row| %>
+
+ | <%= external_website_link(row[:url]) %> |
+
+ <%= row[:condenser_fetch_result] %>
+ <% if shadow_report_matrix_reason(row, :fetch_reason).present? %>
+ <%= shadow_report_matrix_reason(row, :fetch_reason) %>
+ <% end %>
+ |
+
+ <%= row[:legacy_lookup_result] %>
+ <% if shadow_report_matrix_reason(row, :legacy_lookup_reason).present? %>
+ <%= shadow_report_matrix_reason(row, :legacy_lookup_reason) %>
+ <% end %>
+ |
+
+ <%= row[:cache_comparison_result] %>
+ <% if shadow_report_matrix_reason(row, :cache_compare_reason).present? %>
+ <%= shadow_report_matrix_reason(row, :cache_compare_reason) %>
+ <% end %>
+ |
+
+ <%= row[:statement_check_result] %>
+ <% if shadow_report_matrix_reason(row, :statement_reason).present? %>
+ <%= shadow_report_matrix_reason(row, :statement_reason) %>
+ <% end %>
+ |
+
+ <%= row[:export_impact] %>
+ <% if shadow_report_matrix_reason(row, :export_reason).present? %>
+ <%= shadow_report_matrix_reason(row, :export_reason) %>
+ <% end %>
+ |
+ <%= shadow_report_url_matrix_actions(row, @website) %> |
+
+ <% end %>
+
+
+ <% else %>
+ No sampled representative URLs were recorded.
+ <% end %>
+
+
+
+ Checked scope
+
+ <% shadow_report_scope_lines(@detail.checked_scope).each do |line| %>
+ - <%= line %>
+ <% end %>
+
+ <% if shadow_report_scope_warning(@detail.checked_scope).present? %>
+ <%= shadow_report_scope_warning(@detail.checked_scope) %>
+ <% end %>
+
+
+
+ Audit
+ Website: <%= @website.name %>
+ Seedurl: <%= @website.seedurl %>
+ Cohort: <%= @detail.summary.cohort_label.presence || "Other" %>
+ Current rollout mode: <%= Distillator::RolloutCopy.label(@website.distillator_mode) %>
+ Production backend: <%= shadow_report_production_backend_label(@detail.summary) %>
+ Comparison policy: <%= @detail.transition_status.comparison_policy.to_s.humanize %> mode
+ Latest attempt: <%= shadow_report_timestamp(@detail.summary.latest_refresh) %>
+ Latest successful refresh: <%= shadow_report_timestamp(@detail.summary.latest_successful_refresh) %>
+ Health: <%= @detail.summary.health_summary %>
+ Issue: <%= @detail.summary.issue_key.presence || "None" %>
+ Cache links: <%= shadow_report_cache_diagnostic_links(@detail.summary) %>
+
+ Check explanations
+ <% @detail.transition_evidence_explanations.each do |explanation| %>
+
+ <%= explanation.check %> details
+ State: <%= explanation.state %>
+ Headline: <%= explanation.headline %>
+ <% if explanation.details.any? %>
+ <% visible_details = explanation.details.first(10) %>
+
+ <% visible_details.each do |line| %>
+ - <%= line %>
+ <% end %>
+
+ <% if explanation.details.size > visible_details.size %>
+ <%= explanation.details.size - visible_details.size %> additional detail lines hidden from this summary.
+ <% end %>
+ <% end %>
+ Next action: <%= explanation.next_action %>
+ <% if explanation.links.any? %>
+ Links: <%= safe_join(shadow_report_explanation_links(explanation, @website, detail: @detail), " | ") %>
+ <% end %>
+
+ <% end %>
+
+ Recent rollout events
+ <% if @detail.recent_rollout_events.any? %>
+
+ <% @detail.recent_rollout_events.each do |event| %>
+ -
+ <%= [event.from_mode.presence || "unknown", "to", event.to_mode, event.created_at].join(" ") %>
+ <% if event.readiness_snapshot["warnings"].present? %>
+ | warnings: <%= Array(event.readiness_snapshot["warnings"]).join(", ") %>
+ <% end %>
+
+ <% end %>
+
+ <% else %>
+ No rollout events recorded yet.
+ <% end %>
+
+ <% if @detail.transition_status.review_activation_eligible %>
+ Review checklist
+ <% sampled_webpages = shadow_report_sampled_webpages(@detail) %>
+ <% first_sample = sampled_webpages.first %>
+
+ - <%= link_to "Open source website", first_sample[:links].find { |link| link[:label] == "Source website" }[:url], target: "_blank", rel: "noopener" if first_sample.present? && first_sample[:links].any? { |link| link[:label] == "Source website" } %>
+ - <%= link_to "Open compare page", first_sample[:links].find { |link| link[:label] == "Compare" }[:url] if first_sample.present? && first_sample[:links].any? { |link| link[:label] == "Compare" } %>
+ - <%= link_to "Open active cache", first_sample[:links].find { |link| link[:label] == "Open active cache" }[:url] if first_sample.present? && first_sample[:links].any? { |link| link[:label] == "Open active cache" } %>
+ - <%= link_to "Open Condenser cache", first_sample[:links].find { |link| link[:label] == "Open Condenser cache" }[:url] if first_sample.present? && first_sample[:links].any? { |link| link[:label] == "Open Condenser cache" } %>
+ - <%= link_to "Open webpage record", first_sample[:links].find { |link| link[:label] == "Webpage record" }[:url] if first_sample.present? && first_sample[:links].any? { |link| link[:label] == "Webpage record" } %>
+ - Compare title, date, location, and ticket link.
+ - <%= link_to "Check statements page", statements_path(seedurl: @website.seedurl) %>
+ - Check JSON-LD/export if relevant.
+ - Activate after review, with reason.
+
+ <% end %>
+
+
+ Raw legacy transition evidence
+
+
+
+ | Check |
+ Status |
+ Checked at |
+ URL used |
+ Headline |
+ Relevant fields |
+ Next action |
+
+
+
+ <% shadow_report_transition_evidence_rows(@detail).each do |row| %>
+
+ | <%= row[:label] %> |
+ <%= row[:status] %> |
+ <%= row[:checked_at] %> |
+ <%= external_website_link(row[:url], max: 120) %> |
+ <%= row[:headline] %> |
+
+ <% if row[:details].any? %>
+
+ Show details
+ <% visible_details = row[:details].first(10) %>
+
+ <% visible_details.each do |detail| %>
+ - <%= detail %>
+ <% end %>
+
+ <% if row[:details].size > visible_details.size %>
+ <%= row[:details].size - visible_details.size %> additional detail lines hidden from this summary.
+ <% end %>
+
+ <% else %>
+ Recorded
+ <% end %>
+ |
+
+ <%= row[:next_action] %>
+ <% if row[:links].any? %>
+ <%= safe_join(row[:links], " | ") %>
+ <% end %>
+ |
+
+ <% end %>
+
+
+
+
+
+<% if @website.distillator_mode == "active" %>
+
+ Rollback path
+ Set rollout mode to Legacy in website options.
+ Wringer becomes production backend.
+ Condenser cache remains available for inspection.
+
+<% end %>
+
+<%= link_to "Back to Transition Report", distillator_shadow_report_path %>
diff --git a/app/views/events/_harmonized_row.html.erb b/app/views/events/_harmonized_row.html.erb
new file mode 100644
index 00000000..189c1166
--- /dev/null
+++ b/app/views/events/_harmonized_row.html.erb
@@ -0,0 +1,19 @@
+
+ | <%= event_row[:title] %> |
+ <%= event_row[:date] %> |
+ <%= event_row[:archive_date] %> |
+
+ <% statuses = [] %>
+ <% statuses << "to review" if event_row[:statements_status][:to_review] %>
+ <% statuses << "updated" if event_row[:statements_status][:updated] %>
+ <% statuses << "problem" if event_row[:statements_status][:problem] %>
+ <% statuses << "publishable" if event_row[:statements_status][:publishable] %>
+ <%= statuses.presence&.join(", ") || "none" %>
+ |
+
+ <%= link_to "Statements", statements_path(rdf_uri: event_row[:rdf_uri]) %>
+ <% if event_row[:rdf_uri].present? %>
+ <%= link_to "Console", "#{get_console_url_per_environment}/events/#{event_row[:rdf_uri]}" %>
+ <% end %>
+ |
+
diff --git a/app/views/events/index.html.erb b/app/views/events/index.html.erb
index 4f550b0f..a63fddc1 100644
--- a/app/views/events/index.html.erb
+++ b/app/views/events/index.html.erb
@@ -1,10 +1,17 @@
- Events <%= @total_events %>
+Events <%= @total_events %>
-
-<% @events.each do |event| %>
-<%= event[:rdf_uri] %>
-<%= event[:statements_status] %>
-<%= event[:title] %>
-<%= event[:photo] %>
-
-<% end %>
+<%= render "shared/tables/table_shell",
+ table_key: :events,
+ records: @events,
+ row_partial: "events/harmonized_row",
+ row_name: :event_row,
+ headers: @event_table_headers,
+ filters: @filters,
+ pagination: @pagination,
+ sort: @sort,
+ direction: @direction,
+ sortable_filters: @sortable_filters.merge(seedurl: params[:seedurl]),
+ table_css_class: "cache-table events-table",
+ render_filters: true,
+ seedurl: params[:seedurl],
+ empty_message: "No events match the current filters." %>
diff --git a/app/views/layouts/_header.html.erb b/app/views/layouts/_header.html.erb
index f2de97fe..d46b3a5c 100644
--- a/app/views/layouts/_header.html.erb
+++ b/app/views/layouts/_header.html.erb
@@ -1,36 +1,10 @@
-<%
- # Pick a website just for navigation.
- # Priority:
- # 1. @website set by controller (show/edit/etc.)
- # 2. params[:seedurl]
- # 3. first website in the list
- nav_website =
- if defined?(@website) && @website.present?
- @website
- elsif params[:seedurl].present? && params[:seedurl] != "all"
- Website.find_by(seedurl: params[:seedurl])
- else
- list_of_websites.first
- end
-%>
-
-<%= link_to "all websites", websites_path %> >
-
-
-<% if !nav_website.id.nil? %>
- <%= link_to nav_website.seedurl, website_path(nav_website) %> |
- <%= link_to "webpages", webpages_path %> |
- <%= link_to "sources", sources_path %> |
- <%= link_to "events", events_websites_path(seedurl: nav_website.seedurl) %> |
- <%= link_to "places", places_path(seedurl: nav_website.seedurl) %> |
-<% else %>
- webpages |
- sources |
- events |
- places |
+<%= link_to "all websites", websites_path %> |
+<% if transition_context_website.present? %>
+<%= link_to transition_context_website.name, website_path(transition_context_website) %> |
<% end %>
-
-<%= link_to "statements", statements_path %> |
+<%= link_to "webpages", contextual_webpages_path %> |
+<%= link_to "sources", contextual_sources_path %> |
+<%= link_to "statements", contextual_statements_path %> |
<%= link_to "console", "#{get_console_url_per_environment}/events?seedurl=#{cookies[:seedurl]}" %> ||
<%= link_to "properties", properties_path %> |
<%= link_to "classes", rdfs_classes_path %> |
@@ -40,13 +14,15 @@
<%= link_to "databus", databus_index_path %> |
<%= link_to "messages", messages_path %> |
<%= link_to "job queue", sidekiq_web_path %> |
+<%= link_to "cache", "/distillator/cache" %> |
+<%= link_to "Transition report", distillator_shadow_report_path %> |
<%= link_to "options", options_path %> |
Current Wringer:
-<%= cookies[:wringer_url] || "default" %> |
+<%= current_wringer_status_text.sub("Current Wringer: ", "") %> |
Trace:
<%= cookies[:dsl_trace] == "true" ? "ON" : "OFF" %>
-
+
-
diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb
index f0f5d049..40718913 100644
--- a/app/views/layouts/application.html.erb
+++ b/app/views/layouts/application.html.erb
@@ -45,6 +45,15 @@
<%= render "layouts/header" %>
<%= render "shared/flash" %>
+ <% if show_transition_context? %>
+ <%= render "shared/transition_context",
+ website: transition_context_website,
+ contract: website_transition_contract(transition_context_website),
+ compact: true %>
+ <% end %>
+ <% if !suppress_operator_context_card? && (operator_context = operator_context_payload).present? %>
+ <%= render "shared/operator_context_card", context: operator_context %>
+ <% end %>
<%= yield %>
-