Produce CDash URLs to all builds and nonpassing tests for same repo v…

…ersion (TriBITSPub#483) This makes it easy to click on links to these other CDash URLs to see what is happening with all of the builds for the same repo version. The main target is for the GitHub Actions drivers for TriBITS testing. This makes it easy to get at those results.
bartlettroscoe · Jun 5, 2022 · 86b58d3 · 86b58d3
1 parent 32cc1ad
commit 86b58d3
Show file tree

Hide file tree

Showing 5 changed files with 356 additions and 9 deletions.
diff --git a/test/ctest_driver/CTestDriverUnitTests.cmake b/test/ctest_driver/CTestDriverUnitTests.cmake
@@ -53,6 +53,42 @@ include(TribitsReadTagFile)
 include(TribitsGetCDashUrlsInsideCTestS)
 
 
+function(unittest_tribits_get_cdash_revision_builds_url)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_revision_builds_url()")
+  message("***\n")
+
+  tribits_get_cdash_revision_builds_url(
+     CDASH_SITE_URL "somesite.com/my-cdash"
+     PROJECT_NAME  goodProject
+     GIT_REPO_SHA1 "abc123"
+     CDASH_REVISION_BUILDS_URL_OUT  cdashRevesionBuildsUrlOut
+     )
+  unittest_compare_const(cdashRevesionBuildsUrlOut
+    "somesite.com/my-cdash/index.php?project=goodProject&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=abc123")
+
+endfunction()
+
+
+function(unittest_tribits_get_cdash_revision_nonpassing_tests_url)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_revision_nonpassing_tests_url()")
+  message("***\n")
+
+  tribits_get_cdash_revision_nonpassing_tests_url(
+     CDASH_SITE_URL "somesite.com/my-cdash"
+     PROJECT_NAME  goodProject
+     GIT_REPO_SHA1 "abc123"
+     CDASH_REVISION_NONPASSING_TESTS_URL_OUT  cdashRevisionNonpassingTestsUrlOut
+     )
+  unittest_compare_const(cdashRevisionNonpassingTestsUrlOut
+    "somesite.com/my-cdash/queryTests.php?project=goodProject&filtercount=2&showfilters=1&filtercombine=and&field1=revision&compare1=61&value1=abc123=status&compare2=62&value2=passed")
+
+endfunction()
+
+
 function(unittest_tribits_read_ctest_tag_file)
 
   message("\n***")
@@ -158,6 +194,8 @@ endfunction()
 unittest_initialize_vars()
 
 # Run the unit test functions
+unittest_tribits_get_cdash_revision_builds_url()
+unittest_tribits_get_cdash_revision_nonpassing_tests_url()
 unittest_tribits_read_ctest_tag_file()
 unittest_tribits_get_cdash_site_from_drop_site_and_location()
 unittest_tribits_get_cdash_index_php_from_drop_site_and_location()

diff --git a/test/ctest_driver/TribitsExampleMetaProject/CMakeLists.txt b/test/ctest_driver/TribitsExampleMetaProject/CMakeLists.txt
@@ -98,6 +98,19 @@ set(TribitsExampleMetaProject_COMMON_CONFIG_ARGS
 ##########################################################################################
 
 
+  set(cdash_build_url_expected_regex
+    "https://testing[.]sandia[.]gov/cdash/index[.]php[?]project=TribitsExampleMetaProject&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=CustomSite&field2=buildname&compare2=61&value2=CTestDriver_TribitsExMetaProj_clone_default_branch_remote&field3=buildstamp&compare3=61&value3=[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]-Continuous" )
+  # NOTE: Above, we have to repeat [0-9] 8 times and 4 times for the regex for
+  # the number of digits in the buildstarttime.  CMake regex does not support
+  # \d{8}-\d{4} :-(
+
+  set(cdash_revisions_builds_url_expected_regex
+     "https://testing[.]sandia[.]gov/cdash/index.php[?]project=TribitsExampleMetaProject&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=863461e3035d24c632e175c087761e83db28bdc3")
+
+  set(cdash_revisions_nonpassing_tests_expected_regex
+    "https://testing[.]sandia[.]gov/cdash/queryTests.php[?]project=TribitsExampleMetaProject&filtercount=2&showfilters=1&filtercombine=and&field1=revision&compare1=61&value1=863461e3035d24c632e175c087761e83db28bdc3=status&compare2=62&value2=passed" )
+
+
 tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_remote
   OVERALL_WORKING_DIRECTORY TEST_NAME
   OVERALL_NUM_MPI_PROCS 1
@@ -111,11 +124,19 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
       ${COMMON_ENV_ARGS}
       TribitsExMetaProj_ENABLE_SECONDARY_TESTED_CODE=TRUE
       CTEST_BUILD_NAME=CTestDriver_TribitsExMetaProj_clone_default_branch_remote
+      CTEST_SITE=CustomSite
       ${CTEST_S_SCRIPT_ARGS}
     PASS_REGULAR_EXPRESSION_ALL
       "CTEST_NOTES_FILES=''"
       "First perform the initial checkout: .*/git. clone -o origin .*github.com.tribits/TribitsExampleMetaProject.git"
       "Perform checkout in directory: .*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote"
+      "Results will be submitted on CDash at the following links:"
+      "Link to this build's results on CDash:"
+      "${cdash_build_url_expected_regex}"
+      "Link all builds for this repo version on CDash:"
+      "${cdash_revisions_builds_url_expected_regex}"
+      "Link to all nonpassing tests for all builds for this repo version on CDash:"
+      "${cdash_revisions_nonpassing_tests_expected_regex}"
       "Calling ctest_update[(][)] to update base source repo '.*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote/TribitsExampleMetaProject"
       "Old revision of repository is: [a-z0-9]+"
       "New revision of repository is: [a-z0-9]+"
@@ -315,6 +336,11 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
   # * Makes sure some git commit SHA1s are shown in 'Old/New revision of
   #   repository' printouts from ctest_update().
   #
+  # * Checks that CDash URLs that get produced, including for all builds with
+  # * the same repo version.  (This check has to be here since the base
+  # * project repo must have a .git/ directory and we must know the exact SHA1
+  # * of that repo for the test, which we don in this case.)
+  #
   # NOTE: The reason that so many tests are done in a single ctest test is the
   # cost of the initial configure which checks the compilers which is very
   # expensive.  The other runs use

diff --git a/test/ctest_driver/TribitsExampleProject/CMakeLists.txt b/test/ctest_driver/TribitsExampleProject/CMakeLists.txt
@@ -1264,7 +1264,8 @@ function(generate_aao_tests)
     # CONFIGURE_OPTIONS list is checked to make sure that it does not change
     # without being noticed.  Follow-on tests will not check all of that.
 
-  set(cdash_url_expected_regex
+
+  set(cdash_build_url_expected_regex
     "https://cdash[.]site[.]com/cdash/index[.]php[?]project=CustomTribitsExProj&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=My%20Site&field2=buildname&compare2=61&value2=TriBITS_CTestDriver_AAOP_CDASH_URL&field3=buildstamp&compare3=61&value3=[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]-Custom%20CDash%20Group" )
   # NOTE: Above, we have to repeat [0-9] 8 times and 4 times for the regex for
   # the number of digits in the buildstarttime.  CMake regex does not support
@@ -1285,7 +1286,7 @@ function(generate_aao_tests)
         CTEST_DASHBOARD_ROOT=PWD
         ${AAO_COMMON_ENV_ARGS}
         CTEST_PARALLEL_LEVEL=3
-        CTEST_DO_SUBMIT=OFF  # Never submit this
+        CTEST_DO_SUBMIT=OFF  # Never submit this (see below note)
         CTEST_SITE="My Site"
         CTEST_DROP_SITE="cdash.site.com"
         CTEST_PROJECT_NAME="CustomTribitsExProj"
@@ -1299,17 +1300,18 @@ function(generate_aao_tests)
         "NONE does not exist, skipping extra repositories"
         "Final set of enabled packages:  SimpleCxx 1"
         "Final set of enabled SE packages:  SimpleCxx 1"
-        "Results will be submitted to CDash at:"
+        "Results will be submitted on CDash at the following links:"
+        "Link to this build's results on CDash:"
+        "${cdash_build_url_expected_regex}"
         "File '' does NOT exist so all tests passed"
-        "See results submitted to CDash at:"
-        "${cdash_url_expected_regex}"
+        "See results submitted on CDash at the following links:"
         "TRIBITS_CTEST_DRIVER: OVERALL: ALL PASSED"
       ALWAYS_FAIL_ON_NONZERO_RETURN
     TEST_2
       MESSAGE "Check that CDashBuildUrl.txt exists and lists the right URL."
       CMND cat ARGS BUILD/CDashBuildUrl.txt
       PASS_REGULAR_EXPRESSION_ALL
-        "${cdash_url_expected_regex}"
+        "${cdash_build_url_expected_regex}"
     )
     # NOTE: The above test is never actually submitted to CDash, even when
     # ${PACKAGE_NAME}_CTEST_DRIVER_SUBMIT_TO is set.  This is so we can

diff --git a/tribits/ctest_driver/TribitsCTestDriverCore.cmake b/tribits/ctest_driver/TribitsCTestDriverCore.cmake
@@ -2093,11 +2093,23 @@ function(tribits_ctest_driver)
   ctest_start(${CTEST_START_ARGS})
 
   tribits_remember_if_configure_attempted()
+
+  # ToDo: Remove this!
   tribits_get_build_url_and_write_to_file(CDASH_BUILD_URL
     "${CTEST_BINARY_DIRECTORY}/CDashBuildUrl.txt")
   tribits_print_cdash_url("Results will be submitted to CDash at:"
     "${CDASH_BUILD_URL}")
 
+  # ToDo: Get this working!
+  tribits_get_cdash_results_string_and_write_to_file(
+    "${CTEST_BINARY_DIRECTORY}/CDashBuildUrl.txt"
+    CDASH_RESULTS_STRING_OUT  CDASH_RESULTS_STRING)
+  message("Results will be submitted on CDash at the following links:\n\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
+    "${CDASH_RESULTS_STRING}\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
+    )
+
   message(
     "\n***"
     "\n*** Update the source code repositories ..."
@@ -2370,8 +2382,11 @@ function(tribits_ctest_driver)
 
   report_queued_errors()
 
-  tribits_print_cdash_url("See results submitted to CDash at:"
-    "${CDASH_BUILD_URL}")
+  message("\nSee results submitted on CDash at the following links:\n\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
+    "${CDASH_RESULTS_STRING}\n"
+    "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
+    )
 
   if ((NOT UPDATE_FAILED) AND ("${${PROJECT_NAME}_FAILED_PACKAGES}" STREQUAL ""))
     message(