diff --git a/Registry.Web/Models/Configuration/AppSettings.cs b/Registry.Web/Models/Configuration/AppSettings.cs index 9e9bbc91..ddd0edd1 100644 --- a/Registry.Web/Models/Configuration/AppSettings.cs +++ b/Registry.Web/Models/Configuration/AppSettings.cs @@ -282,6 +282,15 @@ public class AppSettings /// public int HangfireJobRetentionDays { get; set; } = 2; + /// + /// MySQL storage invisibility timeout in hours. When using the MySQL Hangfire + /// provider, a job that runs longer than this timeout may be re-dequeued by + /// another worker because the storage backend treats it as abandoned. Raise + /// this value when builds can take longer than the default (e.g. large point + /// clouds or meshes). Default: 4 (hours). + /// + public int HangfireInvisibilityTimeoutHours { get; set; } = 4; + /// /// Password complexity policy. When null, no password requirements are enforced. /// diff --git a/Registry.Web/Services/Adapters/ConfigurationDataBuilder.cs b/Registry.Web/Services/Adapters/ConfigurationDataBuilder.cs index 22ae09e1..e35fe8ae 100644 --- a/Registry.Web/Services/Adapters/ConfigurationDataBuilder.cs +++ b/Registry.Web/Services/Adapters/ConfigurationDataBuilder.cs @@ -498,7 +498,12 @@ private ConfigurationSectionDataDto BuildCronJobsSection() NumberField("HangfireJobRetentionDays", "Hangfire Job Retention (days)", "Expiration timeout for succeeded Hangfire jobs. Failed jobs are kept for diagnostics. Minimum: 1 day.", _settings.HangfireJobRetentionDays, _defaults.HangfireJobRetentionDays, - minValue: 1, unit: "days") + minValue: 1, unit: "days"), + + NumberField("HangfireInvisibilityTimeoutHours", "Hangfire Invisibility Timeout (hours)", + "MySQL storage: hours before a running job is considered abandoned and eligible for re-dequeue. Raise for long-running builds. Minimum: 1 hour.", + _settings.HangfireInvisibilityTimeoutHours, _defaults.HangfireInvisibilityTimeoutHours, + minValue: 1, unit: "hours") ]); } diff --git a/Registry.Web/Utilities/HangfireUtils.cs b/Registry.Web/Utilities/HangfireUtils.cs index 33e8a8fb..ed7ba0b1 100644 --- a/Registry.Web/Utilities/HangfireUtils.cs +++ b/Registry.Web/Utilities/HangfireUtils.cs @@ -59,23 +59,20 @@ public static void BuildWrapper(IDDB ddb, string path, bool force, writeLine($"In BuildWrapper('{ddb.DatasetFolderPath}', '{path}', '{force}')"); writeLine("Running build"); + var skipped = false; try { ddb.Build(path, force: force); } catch (DdbBuildInProgressException ex) { - // Another DDB process holds the kernel-managed build lock. With the - // refactored cross-platform locking the lock is auto-released when - // the holder dies, so a short backoff + force retry is the safest - // recovery path. Inline retry avoids re-queueing through Hangfire, - // which would otherwise restart the whole job from scratch. - writeLine($"Build lock currently held by another process ({ex.Message}); waiting 10s and retrying with force=true"); - Thread.Sleep(TimeSpan.FromSeconds(10)); - ddb.Build(path, force: true); + // Benign: the lock holder is doing the work. Throwing here would trip + // AutomaticRetry -> Failed -> BuildJobFailureFilter and wrongly invalidate the cache. + writeLine($"Build lock currently held by another process ({ex.Message}); skipping"); + skipped = true; } - writeLine("Done build"); + writeLine(skipped ? "Done build (skipped: lock held elsewhere)" : "Done build"); } [AutomaticRetry(Attempts = 1, OnAttemptsExceeded = AttemptsExceededAction.Fail)] @@ -86,18 +83,19 @@ public static void BuildPendingWrapper(IDDB ddb, PerformContext context) writeLine($"In BuildPendingWrapper('{ddb.DatasetFolderPath}')"); writeLine("Running build pending"); + var skipped = false; try { ddb.BuildPending(); } catch (DdbBuildInProgressException ex) { - writeLine($"Build lock currently held by another process ({ex.Message}); waiting 10s and retrying"); - Thread.Sleep(TimeSpan.FromSeconds(10)); - ddb.BuildPending(); + // Benign: the lock holder will process the pending builds. See BuildWrapper. + writeLine($"Build lock currently held by another process ({ex.Message}); skipping"); + skipped = true; } - writeLine("Done build pending"); + writeLine(skipped ? "Done build pending (skipped: lock held elsewhere)" : "Done build pending"); } [AutomaticRetry(Attempts = 1, OnAttemptsExceeded = AttemptsExceededAction.Fail)] diff --git a/Registry.Web/Utilities/StartupExtenders.cs b/Registry.Web/Utilities/StartupExtenders.cs index 4dd339bd..93648a0a 100644 --- a/Registry.Web/Utilities/StartupExtenders.cs +++ b/Registry.Web/Utilities/StartupExtenders.cs @@ -105,6 +105,7 @@ public static void AddHangfireProvider(this IServiceCollection services, AppSett QueuePollInterval = TimeSpan.FromSeconds(3), JobExpirationCheckInterval = TimeSpan.FromMinutes(15), CountersAggregateInterval = TimeSpan.FromMinutes(5), + InvisibilityTimeout = TimeSpan.FromHours(Math.Max(1, appSettings.HangfireInvisibilityTimeoutHours)), PrepareSchemaIfNecessary = true, DashboardJobListLimit = 50000, TransactionTimeout = TimeSpan.FromMinutes(30), diff --git a/Registry.Web/appsettings-default.json b/Registry.Web/appsettings-default.json index 265cb1d0..c97339b5 100644 --- a/Registry.Web/appsettings-default.json +++ b/Registry.Web/appsettings-default.json @@ -46,6 +46,7 @@ "JobIndexCleanupCron": "0 4 * * *", "JobIndexRetentionDays": 60, "HangfireJobRetentionDays": 2, + "HangfireInvisibilityTimeoutHours": 4, "EnableOrganizationMemberManagement": false, "EnableDefaultUserOrganization": true, "AllowedOrigins": null,