Bug 2106933 - processing BRE jobs can stop if any kill switch is thrown

Fix all kill switches stopping BRE job processing.
Fix serialization error on bug update throwing kill switch.
Add grabbed_until to job queue report.
Fix debug logging for job queue not going to normal logs.
Increase sleep time for no jobs found.

Change-Id: I66524e7cbcac6fd9f76748a274ef2d593a7ffc56

6 files changed, 21 insertions, 29 deletions

diff --git a/Bugzilla/JobQueue.pm b/Bugzilla/JobQueue.pm
index 187618449..c51b9343a 100644
--- a/Bugzilla/JobQueue.pm
+++ b/Bugzilla/JobQueue.pm
@@ -168,7 +168,7 @@ JOBS:
     }
     else {
       Bugzilla->logger->debug("No jobs found");
-      sleep(1);
+      sleep(5);
     }
   }
 
diff --git a/Bugzilla/JobQueue/Runner.pm b/Bugzilla/JobQueue/Runner.pm
index 624550e2e..ade0d6d1c 100644
--- a/Bugzilla/JobQueue/Runner.pm
+++ b/Bugzilla/JobQueue/Runner.pm
@@ -216,7 +216,14 @@ sub _do_work {
 
   my @can_do = @{$self->{gd_args}{cando} // []};
   my $jq     = Bugzilla->job_queue();
-  $jq->set_verbose($self->{debug});
+  if ($self->{debug}) {
+    $jq->set_verbose(sub {
+      my $msg = shift;
+      my $job = shift;
+      Bugzilla->logger->debug($msg);
+    });
+  }
+
   $jq->set_pidfile($self->{gd_pidfile});
   while (my ($key, $module) = each %{Bugzilla::JobQueue->job_map()}) {
     next if @can_do and !grep { $_ eq $key } @can_do;
diff --git a/extensions/ActivityReport/lib/Reports.pm b/extensions/ActivityReport/lib/Reports.pm
index d94e4fb97..25a6d2764 100644
--- a/extensions/ActivityReport/lib/Reports.pm
+++ b/extensions/ActivityReport/lib/Reports.pm
@@ -550,6 +550,7 @@ sub email_queue_report {
             j.jobid,
             j.insert_time,
             j.run_after,
+            j.grabbed_until,
             COUNT(e.jobid) AS error_count,
             MAX(e.error_time) AS error_time,
             (
diff --git a/extensions/ActivityReport/template/en/default/pages/email_queue.html.tmpl b/extensions/ActivityReport/template/en/default/pages/email_queue.html.tmpl
index 3243a70b6..4e7c1882a 100644
--- a/extensions/ActivityReport/template/en/default/pages/email_queue.html.tmpl
+++ b/extensions/ActivityReport/template/en/default/pages/email_queue.html.tmpl
@@ -39,6 +39,7 @@
     <th>ID</th>
     <th>Insert Time</th>
     <th>Run After</th>
+    <th>Grabbed Until</th>
     <th>Age</th>
     <th>Error Count</th>
     <th>Last Error</th>
@@ -53,6 +54,7 @@
       <td>[% job.jobid FILTER html %]</td>
       <td>[% job.insert_time FILTER unixtime("%Y-%m-%d %H:%M %Z") %]</td>
       <td class="run_after">[% job.run_after FILTER unixtime("%Y-%m-%d %H:%M %Z") %]</td>
+      <td class="grabbed_until">[% IF job.grabbed_until%][% job.grabbed_until FILTER unixtime("%Y-%m-%d %H:%M %Z") %][% END %]</td>
       <td>
         [% age = now - job.insert_time %]
         [% IF age < 60 %]
diff --git a/extensions/RuleEngine/Extension.pm b/extensions/RuleEngine/Extension.pm
index 8b43de5c2..e5359c541 100644
--- a/extensions/RuleEngine/Extension.pm
+++ b/extensions/RuleEngine/Extension.pm
@@ -1256,12 +1256,13 @@ sub _rh_rule_changedfrom_changedto {
 sub jobqueue_before_run {
   my ($self, $args) = @_;
 
+## RED HAT EXTENSION 2106933 - Disable this as it blocks all BRE processing
 # If there's active kill switches, stop TheSchwartz from selecting rule engine jobs.
-  if (scalar(@{Bugzilla::Extension::RuleEngine::RuleState->active_list})) {
-    $args->{job_queue}->set_strict_remove_ability(1);
-    $args->{job_queue}
-      ->temporarily_remove_ability($args->{job_queue}->job_map->{rule_engine});
-  }
+#  if (scalar(@{Bugzilla::Extension::RuleEngine::RuleState->active_list})) {
+#    $args->{job_queue}->set_strict_remove_ability(1);
+#    $args->{job_queue}
+#      ->temporarily_remove_ability($args->{job_queue}->job_map->{rule_engine});
+#  }
 
   return;
 }
diff --git a/extensions/RuleEngine/lib/Job.pm b/extensions/RuleEngine/lib/Job.pm
index e6b64dd06..183c355df 100644
--- a/extensions/RuleEngine/lib/Job.pm
+++ b/extensions/RuleEngine/lib/Job.pm
@@ -1609,29 +1609,10 @@ sub _rule_do_action {
     $changes = $bug->update();
   }
   catch {
-    # Check to see if a deadlock happened.
-    if ( $_ =~ /Deadlock found when trying to get lock/
-      || $_ =~ /deadlock detected/
-      || $_ =~ /database reported a query serialization error/
-      || $_ =~ /could not serialize access due to concurrent update/)
-    {
-      Bugzilla->logger->info('BRE: DB transient error when updating bug '
-          . $bug->id
-          . ' hit by rule \''
-          . $rule->name
-          . '\'');
-
-      my $dbh = Bugzilla->dbh;
-      $dbh->bz_rollback_transaction() if $dbh->bz_in_transaction();
-
-      # Abort the run.
-      exit 1;
-    }
-    else {
-      # Not a deadlock exception - rethrow the error.
-      kill_me($rule, "For Bug " . $bug->id . " error: " . $_);
-    }
+    # let 'sub work' decide if a killswitch should be thrown
+    die("For Bug " . $bug->id . " error: " . $_);
   };
+
   return ($changes, \%extra_changes);
 }