diff --git a/Registry.Web/Models/Configuration/AppSettings.cs b/Registry.Web/Models/Configuration/AppSettings.cs
index 9e9bbc91..ddd0edd1 100644
--- a/Registry.Web/Models/Configuration/AppSettings.cs
+++ b/Registry.Web/Models/Configuration/AppSettings.cs
@@ -282,6 +282,15 @@ public class AppSettings
///
public int HangfireJobRetentionDays { get; set; } = 2;
+ ///
+ /// MySQL storage invisibility timeout in hours. When using the MySQL Hangfire
+ /// provider, a job that runs longer than this timeout may be re-dequeued by
+ /// another worker because the storage backend treats it as abandoned. Raise
+ /// this value when builds can take longer than the default (e.g. large point
+ /// clouds or meshes). Default: 4 (hours).
+ ///
+ public int HangfireInvisibilityTimeoutHours { get; set; } = 4;
+
///
/// Password complexity policy. When null, no password requirements are enforced.
///
diff --git a/Registry.Web/Services/Adapters/ConfigurationDataBuilder.cs b/Registry.Web/Services/Adapters/ConfigurationDataBuilder.cs
index 22ae09e1..e35fe8ae 100644
--- a/Registry.Web/Services/Adapters/ConfigurationDataBuilder.cs
+++ b/Registry.Web/Services/Adapters/ConfigurationDataBuilder.cs
@@ -498,7 +498,12 @@ private ConfigurationSectionDataDto BuildCronJobsSection()
NumberField("HangfireJobRetentionDays", "Hangfire Job Retention (days)",
"Expiration timeout for succeeded Hangfire jobs. Failed jobs are kept for diagnostics. Minimum: 1 day.",
_settings.HangfireJobRetentionDays, _defaults.HangfireJobRetentionDays,
- minValue: 1, unit: "days")
+ minValue: 1, unit: "days"),
+
+ NumberField("HangfireInvisibilityTimeoutHours", "Hangfire Invisibility Timeout (hours)",
+ "MySQL storage: hours before a running job is considered abandoned and eligible for re-dequeue. Raise for long-running builds. Minimum: 1 hour.",
+ _settings.HangfireInvisibilityTimeoutHours, _defaults.HangfireInvisibilityTimeoutHours,
+ minValue: 1, unit: "hours")
]);
}
diff --git a/Registry.Web/Utilities/HangfireUtils.cs b/Registry.Web/Utilities/HangfireUtils.cs
index 33e8a8fb..ed7ba0b1 100644
--- a/Registry.Web/Utilities/HangfireUtils.cs
+++ b/Registry.Web/Utilities/HangfireUtils.cs
@@ -59,23 +59,20 @@ public static void BuildWrapper(IDDB ddb, string path, bool force,
writeLine($"In BuildWrapper('{ddb.DatasetFolderPath}', '{path}', '{force}')");
writeLine("Running build");
+ var skipped = false;
try
{
ddb.Build(path, force: force);
}
catch (DdbBuildInProgressException ex)
{
- // Another DDB process holds the kernel-managed build lock. With the
- // refactored cross-platform locking the lock is auto-released when
- // the holder dies, so a short backoff + force retry is the safest
- // recovery path. Inline retry avoids re-queueing through Hangfire,
- // which would otherwise restart the whole job from scratch.
- writeLine($"Build lock currently held by another process ({ex.Message}); waiting 10s and retrying with force=true");
- Thread.Sleep(TimeSpan.FromSeconds(10));
- ddb.Build(path, force: true);
+ // Benign: the lock holder is doing the work. Throwing here would trip
+ // AutomaticRetry -> Failed -> BuildJobFailureFilter and wrongly invalidate the cache.
+ writeLine($"Build lock currently held by another process ({ex.Message}); skipping");
+ skipped = true;
}
- writeLine("Done build");
+ writeLine(skipped ? "Done build (skipped: lock held elsewhere)" : "Done build");
}
[AutomaticRetry(Attempts = 1, OnAttemptsExceeded = AttemptsExceededAction.Fail)]
@@ -86,18 +83,19 @@ public static void BuildPendingWrapper(IDDB ddb, PerformContext context)
writeLine($"In BuildPendingWrapper('{ddb.DatasetFolderPath}')");
writeLine("Running build pending");
+ var skipped = false;
try
{
ddb.BuildPending();
}
catch (DdbBuildInProgressException ex)
{
- writeLine($"Build lock currently held by another process ({ex.Message}); waiting 10s and retrying");
- Thread.Sleep(TimeSpan.FromSeconds(10));
- ddb.BuildPending();
+ // Benign: the lock holder will process the pending builds. See BuildWrapper.
+ writeLine($"Build lock currently held by another process ({ex.Message}); skipping");
+ skipped = true;
}
- writeLine("Done build pending");
+ writeLine(skipped ? "Done build pending (skipped: lock held elsewhere)" : "Done build pending");
}
[AutomaticRetry(Attempts = 1, OnAttemptsExceeded = AttemptsExceededAction.Fail)]
diff --git a/Registry.Web/Utilities/StartupExtenders.cs b/Registry.Web/Utilities/StartupExtenders.cs
index 4dd339bd..93648a0a 100644
--- a/Registry.Web/Utilities/StartupExtenders.cs
+++ b/Registry.Web/Utilities/StartupExtenders.cs
@@ -105,6 +105,7 @@ public static void AddHangfireProvider(this IServiceCollection services, AppSett
QueuePollInterval = TimeSpan.FromSeconds(3),
JobExpirationCheckInterval = TimeSpan.FromMinutes(15),
CountersAggregateInterval = TimeSpan.FromMinutes(5),
+ InvisibilityTimeout = TimeSpan.FromHours(Math.Max(1, appSettings.HangfireInvisibilityTimeoutHours)),
PrepareSchemaIfNecessary = true,
DashboardJobListLimit = 50000,
TransactionTimeout = TimeSpan.FromMinutes(30),
diff --git a/Registry.Web/appsettings-default.json b/Registry.Web/appsettings-default.json
index 265cb1d0..c97339b5 100644
--- a/Registry.Web/appsettings-default.json
+++ b/Registry.Web/appsettings-default.json
@@ -46,6 +46,7 @@
"JobIndexCleanupCron": "0 4 * * *",
"JobIndexRetentionDays": 60,
"HangfireJobRetentionDays": 2,
+ "HangfireInvisibilityTimeoutHours": 4,
"EnableOrganizationMemberManagement": false,
"EnableDefaultUserOrganization": true,
"AllowedOrigins": null,