Merge pull request #2106 from cms-sw/test-manual-approve-2

Allow manual override to PR test results

githublabels.py

@@ -42,6 +42,8 @@ def get_dpg_pog():
         continue
     TYPE_COMMANDS[lab] = [LABEL_COLORS["doc"], lab, "mtype"]
 
+TEST_IGNORE_REASON = ["manual-override", "ib-failure", "external-failure"]
+
 COMMON_LABELS = {
     "tests-started": LABEL_COLORS["hold"],
     "fully-signed": LABEL_COLORS["approved"],
@@ -60,6 +62,9 @@ def get_dpg_pog():
 for lab in TYPE_COMMANDS:
     COMMON_LABELS[lab] = TYPE_COMMANDS[lab][0]
 
+for reason in TEST_IGNORE_REASON:
+    COMMON_LABELS["tests-" + reason] = LABEL_COLORS["info"]
+
 COMPARISON_LABELS = {
     "comparison-notrun": "bfe5bf",
     "comparison-available": LABEL_TYPES["approved"],

process_pr.py

@@ -22,7 +22,7 @@
     CREATE_REPO,
 )
 from cms_static import BACKPORT_STR, GH_CMSSW_ORGANIZATION, CMSBOT_NO_NOTIFY_MSG
-from githublabels import TYPE_COMMANDS
+from githublabels import TYPE_COMMANDS, TEST_IGNORE_REASON
 from repo_config import GH_REPO_ORGANIZATION
 import re, time
 from datetime import datetime
@@ -121,6 +121,10 @@ def format(s, **kwds):
 REGEX_TEST_ABORT = re.compile(
     "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)abort(\s+test|)$", re.I
 )
+REGEX_TEST_IGNORE = re.compile(
+    r"^\s*(?:(?:@|)cmsbuild\s*[,]*\s+|)(?:please\s*[,]*\s+|)ignore\s+tests-rejected\s+(?:with|)([a-z -]+)$",
+    re.I,
+)
 TEST_WAIT_GAP = 720
 ALL_CHECK_FUNCTIONS = None
 EXTRA_RELVALS_TESTS = ["threading", "gpu", "high-stats", "nano"]
@@ -966,6 +970,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
     test_params_msg = ""
     test_params_comment = None
     code_check_apply_patch = False
+    override_tests_failure = None
 
     # start of parsing comments section
     for c in issue.get_comments():
@@ -1237,6 +1242,18 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
                     abort_test = comment
                     test_comment = None
                     signatures["tests"] = "pending"
+                elif REGEX_TEST_IGNORE.match(first_line):
+                    reason = REGEX_TEST_IGNORE.match(first_line)[1].strip()
+                    if reason not in TEST_IGNORE_REASON:
+                        print("Invalid ignore reason:", reason)
+                        if not dryRun:
+                            set_comment_emoji(comment.id, repository, "-1")
+                        reason = ""
+
+                    if reason:
+                        override_tests_failure = reason
+                        if not dryRun:
+                            set_comment_emoji(comment.id, repository)
 
         # Check L2 signoff for users in this PR signing categories
         if [x for x in commenter_categories if x in signing_categories]:
@@ -1267,7 +1284,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
                     signatures[sign] = "rejected"
                     if sign == "orp":
                         mustClose = False
-            continue
 
     # end of parsing comments section
 
@@ -1492,9 +1508,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
                 print(
                     "DryRun: Setting status Waiting for authorized user to issue the test command."
                 )
-
     # Labels coming from signature.
     labels = []
+
+    # Process "ignore tests failure"
+    if override_tests_failure and signatures["tests"] == "rejected":
+        signatures["tests"] = "approved"
+        labels.append("tests-" + override_tests_failure)
+
     for cat in signing_categories:
         l = cat + "-pending"
         if cat in signatures:
@@ -1559,6 +1580,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
     xlabs = ["backport", "urgent", "backport-ok", "compilation-warnings"]
     for lab in TYPE_COMMANDS:
         xlabs.append(lab)
+
     missingApprovals = [
         x
         for x in labels
@@ -1728,7 +1750,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
 
     # Do not complain about tests
     requiresTestMessage = " after it passes the integration tests"
-    if "tests-approved" in labels:
+    if labels.intersection(set("tests-" + x for x in TEST_IGNORE_REASON)):
+        requiresTestMessage = " (test failures were overridden)"
+    elif "tests-approved" in labels:
         requiresTestMessage = " (tests are also fine)"
     elif "tests-rejected" in labels:
         requiresTestMessage = " (but tests are reportedly failing)"