Fix job timeout

Registry.Web/Models/Configuration/AppSettings.cs

@@ -282,6 +282,15 @@ public class AppSettings
     /// </summary>
     public int HangfireJobRetentionDays { get; set; } = 2;
 
+    /// <summary>
+    /// MySQL storage invisibility timeout in hours. When using the MySQL Hangfire
+    /// provider, a job that runs longer than this timeout may be re-dequeued by
+    /// another worker because the storage backend treats it as abandoned. Raise
+    /// this value when builds can take longer than the default (e.g. large point
+    /// clouds or meshes). Default: 4 (hours).
+    /// </summary>
+    public int HangfireInvisibilityTimeoutHours { get; set; } = 4;
+
     /// <summary>
     /// Password complexity policy. When null, no password requirements are enforced.
     /// </summary>

Registry.Web/Services/Adapters/ConfigurationDataBuilder.cs

@@ -498,7 +498,12 @@ private ConfigurationSectionDataDto BuildCronJobsSection()
                 NumberField("HangfireJobRetentionDays", "Hangfire Job Retention (days)",
                     "Expiration timeout for succeeded Hangfire jobs. Failed jobs are kept for diagnostics. Minimum: 1 day.",
                     _settings.HangfireJobRetentionDays, _defaults.HangfireJobRetentionDays,
-                    minValue: 1, unit: "days")
+                    minValue: 1, unit: "days"),
+
+                NumberField("HangfireInvisibilityTimeoutHours", "Hangfire Invisibility Timeout (hours)",
+                    "MySQL storage: hours before a running job is considered abandoned and eligible for re-dequeue. Raise for long-running builds. Minimum: 1 hour.",
+                    _settings.HangfireInvisibilityTimeoutHours, _defaults.HangfireInvisibilityTimeoutHours,
+                    minValue: 1, unit: "hours")
             ]);
     }
 

Registry.Web/Utilities/HangfireUtils.cs

@@ -59,23 +59,20 @@ public static void BuildWrapper(IDDB ddb, string path, bool force,
         writeLine($"In BuildWrapper('{ddb.DatasetFolderPath}', '{path}', '{force}')");
 
         writeLine("Running build");
+        var skipped = false;
         try
         {
             ddb.Build(path, force: force);
         }
         catch (DdbBuildInProgressException ex)
         {
-            // Another DDB process holds the kernel-managed build lock. With the
-            // refactored cross-platform locking the lock is auto-released when
-            // the holder dies, so a short backoff + force retry is the safest
-            // recovery path. Inline retry avoids re-queueing through Hangfire,
-            // which would otherwise restart the whole job from scratch.
-            writeLine($"Build lock currently held by another process ({ex.Message}); waiting 10s and retrying with force=true");
-            Thread.Sleep(TimeSpan.FromSeconds(10));
-            ddb.Build(path, force: true);
+            // Benign: the lock holder is doing the work. Throwing here would trip
+            // AutomaticRetry -> Failed -> BuildJobFailureFilter and wrongly invalidate the cache.
+            writeLine($"Build lock currently held by another process ({ex.Message}); skipping");
+            skipped = true;
         }
 
-        writeLine("Done build");
+        writeLine(skipped ? "Done build (skipped: lock held elsewhere)" : "Done build");
     }
 
     [AutomaticRetry(Attempts = 1, OnAttemptsExceeded = AttemptsExceededAction.Fail)]
@@ -86,18 +83,19 @@ public static void BuildPendingWrapper(IDDB ddb, PerformContext context)
         writeLine($"In BuildPendingWrapper('{ddb.DatasetFolderPath}')");
 
         writeLine("Running build pending");
+        var skipped = false;
         try
         {
             ddb.BuildPending();
         }
         catch (DdbBuildInProgressException ex)
         {
-            writeLine($"Build lock currently held by another process ({ex.Message}); waiting 10s and retrying");
-            Thread.Sleep(TimeSpan.FromSeconds(10));
-            ddb.BuildPending();
+            // Benign: the lock holder will process the pending builds. See BuildWrapper.
+            writeLine($"Build lock currently held by another process ({ex.Message}); skipping");
+            skipped = true;
         }
 
-        writeLine("Done build pending");
+        writeLine(skipped ? "Done build pending (skipped: lock held elsewhere)" : "Done build pending");
     }
 
     [AutomaticRetry(Attempts = 1, OnAttemptsExceeded = AttemptsExceededAction.Fail)]

Registry.Web/Utilities/StartupExtenders.cs

@@ -105,6 +105,7 @@ public static void AddHangfireProvider(this IServiceCollection services, AppSett
                                 QueuePollInterval = TimeSpan.FromSeconds(3),
                                 JobExpirationCheckInterval = TimeSpan.FromMinutes(15),
                                 CountersAggregateInterval = TimeSpan.FromMinutes(5),
+                                InvisibilityTimeout = TimeSpan.FromHours(Math.Max(1, appSettings.HangfireInvisibilityTimeoutHours)),
                                 PrepareSchemaIfNecessary = true,
                                 DashboardJobListLimit = 50000,
                                 TransactionTimeout = TimeSpan.FromMinutes(30),

Registry.Web/appsettings-default.json

@@ -46,6 +46,7 @@
     "JobIndexCleanupCron": "0 4 * * *",
     "JobIndexRetentionDays": 60,
     "HangfireJobRetentionDays": 2,
+    "HangfireInvisibilityTimeoutHours": 4,
     "EnableOrganizationMemberManagement": false,
     "EnableDefaultUserOrganization": true,
     "AllowedOrigins": null,