DSL tracing infrastructure, execution safety fixes, and monitoring support

.gitignore

@@ -61,3 +61,8 @@ test/fixtures/*.yml.old
 .idea/
 .vscode/
 *~
+
+# Misc
+*.gz
+*.txt
+env.sh

Gemfile

@@ -51,6 +51,11 @@ group :development do
   gem 'derailed_benchmarks'
 end
 
+group :staging do
+  gem "get_process_mem"
+  gem "memory_profiler"
+end
+
 group :test do
   gem 'rails-controller-testing'
   gem 'simplecov', require: false

Gemfile.lock

@@ -645,12 +645,14 @@ DEPENDENCIES
   capybara (~> 3.40)
   chronic_duration
   derailed_benchmarks
+  get_process_mem
   httparty
   jbuilder (~> 2.11)
   jsbundling-rails
   linkeddata (~> 3.2.0)
   listen (~> 3.7)
   mechanize
+  memory_profiler
   minitest
   mocha
   pg (~> 1.5)

app/assets/stylesheets/application.css

@@ -17,4 +17,86 @@
  /* Add this CSS to your stylesheet */
 table tr:hover {
   background-color: #f5f5f5; /* Change this color to your desired highlight color */
+}
+
+.trace-tooltip {
+  position: relative;
+  cursor: help;
+}
+
+.trace-tooltip:hover::after {
+  content: attr(data-tooltip);
+  position: absolute;
+  left: 0;
+  top: 100%;
+  white-space: pre;
+  background: #333;
+  color: #fff;
+  padding: 4px 8px;
+  border-radius: 4px;
+  z-index: 10;
+  max-width: 400px;
+  font-size: 0.85em;
+}
+
+/* Options page form styling */
+.options-form {
+  max-width: 600px;
+  margin: 1.5rem auto;
+  padding: 1.5rem;
+  background: #f9f9f9;
+  border: 1px solid #ddd;
+  border-radius: 8px;
+}
+
+.options-form h2 {
+  font-size: 1.75rem;
+  margin-bottom: 1rem;
+  text-align: center;
+}
+
+.options-form label {
+  display: block;
+  margin-bottom: 1rem;
+  font-weight: 500;
+  color: #333;
+}
+
+.options-form input[type="number"] {
+  width: 100%;
+  padding: 8px 10px;
+  font-size: 1rem;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+  margin-top: 4px;
+}
+
+.options-form .btn {
+  display: inline-block;
+  padding: 10px 16px;
+  margin-top: 0.5rem;
+  font-size: 1rem;
+  color: #fff;
+  background-color: #007bff;
+  border-radius: 4px;
+  text-decoration: none;
+  text-align: center;
+  cursor: pointer;
+}
+
+.options-form .btn-secondary {
+  background-color: #6c757d;
+}
+
+.options-form .btn:hover {
+  opacity: 0.95;
+}
+
+.options-form .flash.notice {
+  margin-top: 1rem;
+  padding: 10px;
+  background: #e6ffed;
+  border: 1px solid #8acc8a;
+  border-radius: 4px;
+  color: #27632a;
 }

app/controllers/dashboard_metrics_controller.rb

@@ -0,0 +1,177 @@
+# app/controllers/dashboard_metrics_controller.rb
+class DashboardMetricsController < ApplicationController
+
+  def index
+    results = Rails.cache.fetch("dashboard_metrics", expires_in: 10.minutes) do
+      compute_all_metrics
+    end
+
+    render json: results
+  end
+
+  def broken
+    results = Rails.cache.fetch("dashboard_metrics", expires_in: 10.minutes) do
+      compute_all_metrics
+    end
+
+    broken = results.select do |_seedurl, m|
+      m[:new_webpages_7d].zero? &&
+        m[:statements_updated_24hr].zero? &&
+        m[:event_horizon_days].negative?
+    end
+
+    render json: broken
+  end
+
+  private
+
+  def compute_all_metrics
+    websites = Website.where(monitorable: true).pluck(:id, :seedurl)
+
+    seedurls = websites.map { |(_, seedurl)| seedurl }
+
+    website_ids = websites.to_h { |id, seedurl| [seedurl, id] }
+
+    # ----------------------------
+    # total webpages
+    # ----------------------------
+
+    event_webpages =
+      Webpage.joins(:website)
+             .where(websites: { monitorable: true })
+             .where(rdfs_class_id: 1)
+             .group("websites.seedurl")
+             .count
+
+    # ----------------------------
+    # last webpage creation
+    # ----------------------------
+
+    last_webpages =
+      Webpage.joins(:website)
+             .where(websites: { monitorable: true })
+             .where(rdfs_class_id: 1)
+             .group("websites.seedurl")
+             .maximum(:created_at)
+
+    # ----------------------------
+    # new webpages in last 7 days
+    # ----------------------------
+
+    new_webpages =
+      Webpage.joins(:website)
+             .where(websites: { monitorable: true })
+             .where(rdfs_class_id: 1)
+             .where("webpages.created_at >= ?", 7.days.ago)
+             .group("websites.seedurl")
+             .count
+
+    # ----------------------------
+    # publishable webpages
+    # ----------------------------
+
+    publishable_webpages =
+      Statement.joins(webpage: :website)
+               .joins(source: :property)
+               .where(websites: { monitorable: true })
+               .where(properties: { label: %w[Title Location Dates] })
+               .where(status: %w[ok updated])
+               .where(webpages: { rdfs_class_id: 1 })
+               .group("websites.seedurl", "webpages.id")
+               .having("COUNT(DISTINCT properties.label) = 3")
+               .count
+               .keys
+               .map(&:first)
+               .tally
+
+    # ----------------------------
+    # archive horizon
+    # ----------------------------
+
+    max_archive_dates =
+      Webpage.joins(:website)
+             .where(websites: { monitorable: true })
+             .group("websites.seedurl")
+             .maximum(:archive_date)
+
+    # ----------------------------
+    # statements grouped
+    # ----------------------------
+
+    statements_grouped =
+      Statement.joins(webpage: :website)
+               .where(websites: { monitorable: true })
+               .group("websites.seedurl")
+               .count
+
+    # ----------------------------
+    # refreshed last 24h
+    # ----------------------------
+
+    refreshed_24h =
+      Statement.joins(webpage: :website)
+               .where(websites: { monitorable: true })
+               .where("statements.cache_refreshed >= ?", 24.hours.ago)
+               .group("websites.seedurl")
+               .count
+
+    # ----------------------------
+    # updated last 24h
+    # ----------------------------
+
+    updated_24h =
+      Statement.joins(webpage: :website)
+               .where(websites: { monitorable: true })
+               .where("statements.cache_changed >= ?", 24.hours.ago)
+               .group("websites.seedurl")
+               .count
+
+    # ----------------------------
+    # last statement change
+    # ----------------------------
+
+    last_statement_change =
+      Statement.joins(webpage: :website)
+               .where(websites: { monitorable: true })
+               .group("websites.seedurl")
+               .maximum("statements.cache_changed")
+
+    # ----------------------------
+    # build result
+    # ----------------------------
+
+    seedurls.index_with do |seedurl|
+      total_webpages = event_webpages[seedurl] || 0
+      publishable = publishable_webpages[seedurl] || 0
+
+      publishable_ratio =
+        if total_webpages.zero?
+          0
+        else
+          (publishable.to_f / total_webpages * 100).round(1)
+        end
+
+      archive = max_archive_dates[seedurl]
+
+      horizon_days =
+        if archive
+          (archive.to_date - Time.zone.today).to_i
+        else
+          0
+        end
+
+      {
+        website_id: website_ids[seedurl],
+        total_webpages: total_webpages,
+        publishable_ratio: publishable_ratio,
+        statements_grouped: statements_grouped[seedurl] || 0,
+        statements_refreshed_24hr: refreshed_24h[seedurl] || 0,
+        statements_updated_24hr: updated_24h[seedurl] || 0,
+        last_webpage_created_at: last_webpages[seedurl],
+        last_statement_change: last_statement_change[seedurl],
+        new_webpages_7d: new_webpages[seedurl] || 0,
+        event_horizon_days: horizon_days
+      }
+    end
+  end
+end

app/controllers/events_controller.rb

@@ -16,7 +16,9 @@ def index_by_property
     @seedurl = params[:seedurl]
     time_span = create_timespan(params[:startDate], params[:endDate])
     # Add title property for the first column in the table
-    @property_ids = [Property.where(label: "Title").first.id] << params[:property].to_i
+    property_id = params[:property].to_i
+    @property_ids = [Property.find_by(label: "Title")&.id].compact
+    @property_ids << property_id if property_id.positive?
     @property_labels =  @property_ids.map { |id| Property.find(id).label }
 
     # Get statements matching critria
@@ -90,7 +92,7 @@ def website_statements_by_event(seedurl, archive_date_range = [Time.zone.now - 3
         events_by_uri[s.webpage.rdf_uri]
           .merge!({ property_label => { cache: s.cache, status: s.status, selected_individual: s.selected_individual} })
           .merge!({ archive_date: { cache: s.webpage.archive_date } })
-       end
+      end
     end
 
     events_by_uri
@@ -157,5 +159,17 @@ def create_timespan(start_date_input, end_date_input)
     end 
     return [start_date..end_date]
   end
+
+  def require_seedurl!
+    return if params[:seedurl].present?
+
+    respond_to do |format|
+      format.json { render json: { error: "Missing seedurl" }, status: :bad_request }
+      format.html do
+        redirect_back fallback_location: websites_path,
+                      alert: "Missing seedurl in URL (expected /websites/:seedurl/events_by_property)"
+      end
+    end
+  end
 
 end