From 18941ad475c091c8114e6d325d27e56a44fdf9f0 Mon Sep 17 00:00:00 2001
From: Jay <shenjielx@gmail.com>
Date: Fri, 4 Mar 2022 16:16:46 +0800
Subject: [PATCH] add try-catch DistributedLockTimeoutException means another
 Hangfire server did this work.

---
 .../Server/RecurringJobScheduler.cs           | 50 ++++++++++++-------
 1 file changed, 31 insertions(+), 19 deletions(-)

diff --git a/src/Hangfire.Core/Server/RecurringJobScheduler.cs b/src/Hangfire.Core/Server/RecurringJobScheduler.cs
index 0fc6cc549..e63dec8bc 100644
--- a/src/Hangfire.Core/Server/RecurringJobScheduler.cs
+++ b/src/Hangfire.Core/Server/RecurringJobScheduler.cs
@@ -232,29 +232,41 @@ private void TryEnqueueBackgroundJob(
             string recurringJobId,
             DateTime now)
         {
-            using (connection.AcquireDistributedRecurringJobLock(recurringJobId, LockTimeout))
+            try
             {
-                var recurringJob = connection.GetRecurringJob(recurringJobId, _timeZoneResolver, now);
-
-                if (recurringJob == null)
+                using (connection.AcquireDistributedRecurringJobLock(recurringJobId, LockTimeout))
                 {
-                    RemoveRecurringJob(connection, recurringJobId);
-                    return;
-                }
+                    var recurringJob = connection.GetRecurringJob(recurringJobId, _timeZoneResolver, now);
 
-                Exception exception;
+                    if (recurringJob == null)
+                    {
+                        RemoveRecurringJob(connection, recurringJobId);
+                        return;
+                    }
 
-                try
-                {
-                    ScheduleRecurringJob(context, connection, recurringJobId, recurringJob, now);
-                    return;
-                }
-                catch (BackgroundJobClientException ex)
-                {
-                    exception = ex.InnerException;
-                }
+                    Exception exception;
+
+                    try
+                    {
+                        ScheduleRecurringJob(context, connection, recurringJobId, recurringJob, now);
+                        return;
+                    }
+                    catch (BackgroundJobClientException ex)
+                    {
+                        exception = ex.InnerException;
+                    }
 
-                RetryRecurringJob(connection, recurringJobId, recurringJob, exception);
+                    RetryRecurringJob(connection, recurringJobId, recurringJob, exception);
+                }
+            }
+            catch (DistributedLockTimeoutException e) when (e.Resource.EndsWith($"lock:recurring-job:{recurringJobId}"))
+            {
+                // DistributedLockTimeoutException here doesn't mean that others weren't scheduled.
+                // It just means another Hangfire server did this work.
+                _logger.Log(
+                    LogLevel.Debug,
+                    () => $@"An exception was thrown during acquiring distributed lock the {recurringJobId} resource within {LockTimeout.TotalSeconds} seconds. The recurring job has not been handled this time.",
+                    e);
             }
         }
 
@@ -419,4 +431,4 @@ private bool IsBatchingAvailable(IStorageConnection connection)
                 });
         }
     }
-}
\ No newline at end of file
+}