Add CDash URLs to builds and nonpassing tests with same base repo version

.github/workflows/tribits_testing.yml

@@ -92,6 +92,9 @@ jobs:
       - name: URL to results on CDash
         run: |
           cd ..
-          echo "Result on CDash are posted at:"
+          echo "See results posted on CDash:"
+          echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
           echo
-          cat tribits-build/BUILD/CDashBuildUrl.txt
+          cat tribits-build/BUILD/CDashResults.txt
+          echo
+          echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"

test/core/CMakeLists.txt

@@ -49,7 +49,7 @@ tribits_add_advanced_test( TestingFunctionMacro_UnitTests
       -D${PROJECT_NAME}_TRIBITS_DIR=${${PROJECT_NAME}_TRIBITS_DIR}
       -P "${CMAKE_CURRENT_SOURCE_DIR}/TestingFunctionMacro_UnitTests.cmake"
     PASS_REGULAR_EXPRESSION_ALL
-      "Final UnitTests Result: num_run = 695"
+      "Final UnitTests Result: num_run = 702"
       "Final UnitTests Result: PASSED"
   )
 

test/core/ExamplesUnitTests/UserErrorChecking_Tests.cmake

@@ -298,7 +298,7 @@ tribits_add_advanced_test( TribitsExampleProject_PkgWithUserErrors_UNPARSED_ARGU
       -DTribitsExProj_ENABLE_TESTS=ON
       TribitsExampleProject
     PASS_REGULAR_EXPRESSION_ALL
-      "CMake Error at.*TribitsGeneralMacros.cmake:"
+      "CMake Error at.*TribitsParseArgumentsHelpers.cmake:"
       "Arguments passed in unrecognized.  PARSE_UNPARSED_ARGUMENTS ="
       "nonsense_jdslkfhlskd"
       "tribits_read_toplevel_package_deps_files_add_to_graph"
@@ -320,7 +320,7 @@ tribits_add_advanced_test( TribitsExampleProject_PkgWithUserErrors_UNPARSED_ARGU
       -DTribitsExProj_ENABLE_TESTS=ON
       TribitsExampleProject
     PASS_REGULAR_EXPRESSION_ALL
-      "CMake Error at.*TribitsGeneralMacros.cmake:"
+      "CMake Error at.*TribitsParseArgumentsHelpers.cmake:"
       "Arguments passed in unrecognized.  PARSE_UNPARSED_ARGUMENTS ="
       "this_shouldnt_be_here"
       "PkgWithUserErrors/CMakeLists.txt.*tribits_add_library"
@@ -343,7 +343,7 @@ tribits_add_advanced_test( TribitsExampleProject_PkgWithUserErrors_UNPARSED_ARGU
       -DTribitsExProj_ENABLE_TESTS=ON
       TribitsExampleProject
     PASS_REGULAR_EXPRESSION_ALL
-      "CMake Error at.*TribitsGeneralMacros.cmake:"
+      "CMake Error at.*TribitsParseArgumentsHelpers.cmake:"
       "Arguments passed in unrecognized.  PARSE_UNPARSED_ARGUMENTS ="
       "misspelled_argument"
       "PkgWithUserErrors/CMakeLists.txt.*tribits_add_executable"

test/core/TestingFunctionMacro_UnitTests.cmake

@@ -60,34 +60,30 @@ include(TribitsFilepathHelpers)
 include(TribitsGetVersionDate)
 include(TribitsTplFindIncludeDirsAndLibraries)
 include(TribitsReportInvalidTribitsUsage)
+include(TribitsGitRepoVersionInfo)
 include(UnitTestHelpers)
 include(GlobalSet)
 include(GlobalNullSet)
 include(AppendStringVar)
 
 
-#####################################################################
+################################################################################
 #
-# Unit tests for tribits_add_xxx(...) CMake commands run as CMake code
+# Unit tests for a collection of TriBITS Core CMake code
 #
-# This file contains a set of unit tests for the package_arch macros
-# tribits_add_xxx(...) functions.  These unit tests are written in CMake
-# itself.  This is not a very advanced unit testing system and it not that
-# easy to work with.  However, it does perform some pretty strong testing and
-# is much better than doing nothing.
+# These unit tests are written in CMake itself.  This is not a very advanced
+# unit testing system and it not that easy to work with.  However, it does
+# perform some pretty strong testing and is much better than doing nothing.
+# Each set of tests is in a function scope so as not to impact other tests.
 #
-#####################################################################
+################################################################################
 
 
+################################################################################
 #
-# Set up unit test functions that will be called below to actually run the
-# unit tests.
-#
-# The reason that we use functions is so that we can change variables just
-# inside of the functions that have their own variable scoping.  In that way,
-# we can keep variables that are set in one unit test from affecting the
-# others.
+# Testing misc functions and macros
 #
+################################################################################
 
 
 function(unittest_append_string_var)
@@ -492,6 +488,55 @@ function(unittest_tribits_get_version_date_from_raw_git_commit_utc_time)
 endfunction()
 
 
+function(unittest_tribits_git_repo_sha1)
+
+  message("\n***")
+  message("*** Testing tribits_git_repo_sha1()")
+  message("***\n")
+
+  find_program(GIT_EXECUTABLE ${GIT_NAME})
+
+  set(tribitsProjDir "${${PROJECT_NAME}_TRIBITS_DIR}/..")
+
+  # Matches any 40-char SHA1
+  set(anySha1Regex "")
+  foreach(i RANGE 0 39)
+    string(APPEND anySha1Regex "[a-z0-9]")
+  endforeach()
+
+  if (IS_DIRECTORY "${tribitsProjDir}/.git")
+
+    message("Testing tribits_git_repo_sha1() with the base TriBITS project (without FAILURE_MESSAGE_OUT)\n")
+    tribits_git_repo_sha1("${tribitsProjDir}" gitRepoSha1)
+    unittest_string_var_regex(gitRepoSha1 REGEX_STRINGS "${anySha1Regex}")
+
+    message("Testing tribits_git_repo_sha1() with the base TriBITS project (with FAILURE_MESSAGE_OUT)\n")
+    tribits_git_repo_sha1("${tribitsProjDir}" gitRepoSha1
+      FAILURE_MESSAGE_OUT  failureMsg)
+    unittest_string_var_regex(gitRepoSha1 REGEX_STRINGS "${anySha1Regex}")
+    unittest_compare_const(failureMsg "")
+
+  endif()
+
+   message("Testing tribits_git_repo_sha1(): No GIT_EXECUTABLE (with FAILURE_MESSAGE_OUT)\n")
+  set(GIT_EXECUTABLE_SAVED "${GIT_EXECUTABLE}")
+  set(GIT_EXECUTABLE "")
+  tribits_git_repo_sha1("${tribitsProjDir}" gitRepoSha1
+    FAILURE_MESSAGE_OUT  failureMsg)
+  unittest_compare_const(gitRepoSha1 "")
+  unittest_compare_const(failureMsg "ERROR: The program 'git' could not be found!")
+  set(GIT_EXECUTABLE "${GIT_EXECUTABLE_SAVED}")
+  unset(GIT_EXECUTABLE_SAVED)
+
+   message("Testing tribits_git_repo_sha1(): Invalid repo dir (with FAILURE_MESSAGE_OUT)\n")
+  tribits_git_repo_sha1("/repo/does/not/exist" gitRepoSha1
+    FAILURE_MESSAGE_OUT  failureMsg)
+  unittest_compare_const(gitRepoSha1 "")
+  unittest_compare_const(failureMsg "ERROR: The directory /repo/does/not/exist/.git does not exist!")
+
+endfunction()
+
+
 function(unittest_tribits_tpl_allow_pre_find_package)
 
   message("\n***")
@@ -670,6 +715,13 @@ function(unittest_tribits_report_invalid_tribits_usage)
 endfunction()
 
 
+################################################################################
+#
+# Testing tribits_add_test()
+#
+################################################################################
+
+
 function(unittest_tribits_add_test_basic)
 
   message("\n***")
@@ -2314,6 +2366,13 @@ function(unittest_tribits_add_test_properties)
 endfunction()
 
 
+################################################################################
+#
+# Testing tribits_add_advanced_test()
+#
+################################################################################
+
+
 function(unittest_tribits_add_advanced_test_basic)
 
   message("\n***")
@@ -4118,6 +4177,13 @@ function(unittest_tribits_add_executable_and_test)
 endfunction()
 
 
+################################################################################
+#
+# Testing TriBITS ETI support code
+#
+################################################################################
+
+
 function(unittest_tribits_eti_type_expansion)
 
   message("*** Test passing invalid arguments to tribits_eti_type_expansion( ... )\n")
@@ -4265,6 +4331,7 @@ function(unittest_tribits_add_eti_instantiations_initial)
 
 endfunction()
 
+
 function(unittest_tribits_add_eti_instantiations_cumulative)
 
   set(package ${PROJECT_NAME}Framework)
@@ -4346,6 +4413,7 @@ function(unittest_tribits_eti_explode)
 
 endfunction()
 
+
 function(unittest_tribits_eti_mangle_symbol)
 
   message("*** Testing ETI Mangling ***")
@@ -4469,9 +4537,12 @@ function(unittest_tribits_eti_generate_macros)
 
 endfunction()
 
+
+################################################################################
 #
 # Execute the unit tests
 #
+################################################################################
 
 # Set up some global environment stuff
 set(${PROJECT_NAME}_HOSTNAME testhost.nowhere.com)
@@ -4487,7 +4558,7 @@ set( TRIBITS_ADD_TEST_ADD_TEST_UNITTEST TRUE )
 set( TRIBITS_ADD_TEST_ADD_TEST_CAPTURE TRUE )
 
 message("\n***")
-message("*** Running little bits of tests")
+message("*** Testing misc TriBITS functions and macros")
 message("***\n")
 
 unittest_append_string_var()
@@ -4499,6 +4570,7 @@ unittest_tribits_misc()
 unittest_tribits_strip_quotes_from_str()
 unittest_tribits_get_version_date_from_raw_git_commit_utc_time()
 unittest_tribits_get_raw_git_commit_utc_time()
+unittest_tribits_git_repo_sha1()
 unittest_tribits_tpl_allow_pre_find_package()
 unittest_tribits_report_invalid_tribits_usage()
 
@@ -4555,4 +4627,4 @@ message("*** Determine final result of all unit tests")
 message("***\n")
 
 # Pass in the number of expected tests that must pass!
-unittest_final_result(695)
+unittest_final_result(702)

test/ctest_driver/CMakeLists.txt

@@ -6,7 +6,7 @@ tribits_add_advanced_test( CTestDriverUnitTests
       -D${PROJECT_NAME}_TRIBITS_DIR=${${PROJECT_NAME}_TRIBITS_DIR}
       -P "${CMAKE_CURRENT_SOURCE_DIR}/CTestDriverUnitTests.cmake"
     PASS_REGULAR_EXPRESSION_ALL
-      "Final UnitTests Result: num_run = 6"
+      "Final UnitTests Result: num_run = 9"
       "Final UnitTests Result: PASSED"
   )
 

test/ctest_driver/CTestDriverUnitTests.cmake

@@ -50,7 +50,43 @@ include(GlobalSet)
 include(UnitTestHelpers)
 
 include(TribitsReadTagFile)
-include(TribitsGetCDashUrlFromTagFile)
+include(TribitsGetCDashUrlsInsideCTestS)
+
+
+function(unittest_tribits_get_cdash_revision_builds_url)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_revision_builds_url()")
+  message("***\n")
+
+  tribits_get_cdash_revision_builds_url(
+     CDASH_SITE_URL "somesite.com/my-cdash"
+     PROJECT_NAME  goodProject
+     GIT_REPO_SHA1 "abc123"
+     CDASH_REVISION_BUILDS_URL_OUT  cdashRevesionBuildsUrlOut
+     )
+  unittest_compare_const(cdashRevesionBuildsUrlOut
+    "somesite.com/my-cdash/index.php?project=goodProject&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=abc123")
+
+endfunction()
+
+
+function(unittest_tribits_get_cdash_revision_nonpassing_tests_url)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_revision_nonpassing_tests_url()")
+  message("***\n")
+
+  tribits_get_cdash_revision_nonpassing_tests_url(
+     CDASH_SITE_URL "somesite.com/my-cdash"
+     PROJECT_NAME  goodProject
+     GIT_REPO_SHA1 "abc123"
+     CDASH_REVISION_NONPASSING_TESTS_URL_OUT  cdashRevisionNonpassingTestsUrlOut
+     )
+  unittest_compare_const(cdashRevisionNonpassingTestsUrlOut
+    "somesite.com/my-cdash/queryTests.php?project=goodProject&filtercount=2&showfilters=1&filtercombine=and&field1=revision&compare1=61&value1=abc123&field2=status&compare2=62&value2=passed")
+
+endfunction()
 
 
 function(unittest_tribits_read_ctest_tag_file)
@@ -73,6 +109,23 @@ function(unittest_tribits_read_ctest_tag_file)
 endfunction()
 
 
+function(unittest_tribits_get_cdash_site_from_drop_site_and_location)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_site_from_drop_site_and_location()")
+  message("***\n")
+
+  tribits_get_cdash_site_from_drop_site_and_location(
+    CTEST_DROP_SITE  "some.site.com"
+    CTEST_DROP_LOCATION  "/cdash/submit.php?project=SomeProject"
+    CDASH_SITE_URL_OUT  cdashSiteUrl
+    )
+
+  unittest_compare_const(cdashSiteUrl "https://some.site.com/cdash")
+
+endfunction()
+
+
 function(unittest_tribits_get_cdash_index_php_from_drop_site_and_location)
 
   message("\n***")
@@ -85,7 +138,7 @@ function(unittest_tribits_get_cdash_index_php_from_drop_site_and_location)
     INDEX_PHP_URL_OUT indexPhpUrl
     )
 
-  unittest_compare_const(indexPhpUrl "some.site.com/cdash/index.php")
+  unittest_compare_const(indexPhpUrl "https://some.site.com/cdash/index.php")
 
 endfunction()
 
@@ -141,7 +194,10 @@ endfunction()
 unittest_initialize_vars()
 
 # Run the unit test functions
+unittest_tribits_get_cdash_revision_builds_url()
+unittest_tribits_get_cdash_revision_nonpassing_tests_url()
 unittest_tribits_read_ctest_tag_file()
+unittest_tribits_get_cdash_site_from_drop_site_and_location()
 unittest_tribits_get_cdash_index_php_from_drop_site_and_location()
 unittest_tribits_get_cdash_build_url_from_parts()
 unittest_tribits_get_cdash_build_url_from_tag_file()
@@ -151,4 +207,4 @@ message("*** Determine final result of all unit tests")
 message("***\n")
 
 # Pass in the number of expected tests that must pass!
-unittest_final_result(6)
+unittest_final_result(9)

test/ctest_driver/TribitsExampleMetaProject/CMakeLists.txt

@@ -98,6 +98,19 @@ set(TribitsExampleMetaProject_COMMON_CONFIG_ARGS
 ##########################################################################################
 
 
+  set(cdash_build_url_expected_regex
+    "https://testing[.]sandia[.]gov/cdash/index[.]php[?]project=TribitsExampleMetaProject&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=CustomSite&field2=buildname&compare2=61&value2=CTestDriver_TribitsExMetaProj_clone_default_branch_remote&field3=buildstamp&compare3=61&value3=[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]-Continuous" )
+  # NOTE: Above, we have to repeat [0-9] 8 times and 4 times for the regex for
+  # the number of digits in the buildstarttime.  CMake regex does not support
+  # \d{8}-\d{4} :-(
+
+  set(cdash_revisions_builds_url_expected_regex
+     "https://testing[.]sandia[.]gov/cdash/index.php[?]project=TribitsExampleMetaProject&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=863461e3035d24c632e175c087761e83db28bdc3")
+
+  set(cdash_revisions_nonpassing_tests_expected_regex
+    "https://testing[.]sandia[.]gov/cdash/queryTests.php[?]project=TribitsExampleMetaProject&filtercount=2&showfilters=1&filtercombine=and&field1=revision&compare1=61&value1=863461e3035d24c632e175c087761e83db28bdc3&field2=status&compare2=62&value2=passed" )
+
+
 tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_remote
   OVERALL_WORKING_DIRECTORY TEST_NAME
   OVERALL_NUM_MPI_PROCS 1
@@ -111,11 +124,19 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
       ${COMMON_ENV_ARGS}
       TribitsExMetaProj_ENABLE_SECONDARY_TESTED_CODE=TRUE
       CTEST_BUILD_NAME=CTestDriver_TribitsExMetaProj_clone_default_branch_remote
+      CTEST_SITE=CustomSite
       ${CTEST_S_SCRIPT_ARGS}
     PASS_REGULAR_EXPRESSION_ALL
       "CTEST_NOTES_FILES=''"
       "First perform the initial checkout: .*/git. clone -o origin .*github.com.tribits/TribitsExampleMetaProject.git"
       "Perform checkout in directory: .*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote"
+      "Results will be submitted on CDash at the following links:"
+      "Link to this build's results on CDash:"
+      "${cdash_build_url_expected_regex}"
+      "Link to all builds for this repo version on CDash:"
+      "${cdash_revisions_builds_url_expected_regex}"
+      "Link to all nonpassing tests for all builds for this repo version on CDash:"
+      "${cdash_revisions_nonpassing_tests_expected_regex}"
       "Calling ctest_update[(][)] to update base source repo '.*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote/TribitsExampleMetaProject"
       "Old revision of repository is: [a-z0-9]+"
       "New revision of repository is: [a-z0-9]+"
@@ -315,6 +336,11 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
   # * Makes sure some git commit SHA1s are shown in 'Old/New revision of
   #   repository' printouts from ctest_update().
   #
+  # * Checks that CDash URLs that get produced, including for all builds with
+  # * the same repo version.  (This check has to be here since the base
+  # * project repo must have a .git/ directory and we must know the exact SHA1
+  # * of that repo for the test, which we don in this case.)
+  #
   # NOTE: The reason that so many tests are done in a single ctest test is the
   # cost of the initial configure which checks the compilers which is very
   # expensive.  The other runs use