Skip to content

Commit

Permalink
Produce CDash URLs to all builds and nonpassing tests for same repo v…
Browse files Browse the repository at this point in the history
…ersion (TriBITSPub#483)

This makes it easy to click on links to these other CDash URLs to see what is
happening with all of the builds for the same repo version.

The main target is for the GitHub Actions drivers for TriBITS testing.  This
makes it easy to get at those results.

As part of this, I also renamed the function
tribits_get_build_url_and_write_to_file() to tribits_get_cdash_build_url() and
removed the ability to write to a file.  (We just don't need that anymore and
it is trivial to write a string to a file.)
  • Loading branch information
bartlettroscoe committed Jun 7, 2022
1 parent b6e986f commit decd86b
Show file tree
Hide file tree
Showing 6 changed files with 422 additions and 29 deletions.
7 changes: 5 additions & 2 deletions .github/workflows/tribits_testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ jobs:
- name: URL to results on CDash
run: |
cd ..
echo "Result on CDash are posted at:"
echo "See results posted on CDash:"
echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
echo
cat tribits-build/BUILD/CDashBuildUrl.txt
cat tribits-build/BUILD/CDashResults.txt
echo
echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
38 changes: 38 additions & 0 deletions test/ctest_driver/CTestDriverUnitTests.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,42 @@ include(TribitsReadTagFile)
include(TribitsGetCDashUrlsInsideCTestS)


function(unittest_tribits_get_cdash_revision_builds_url)

message("\n***")
message("*** Testing tribits_get_cdash_revision_builds_url()")
message("***\n")

tribits_get_cdash_revision_builds_url(
CDASH_SITE_URL "somesite.com/my-cdash"
PROJECT_NAME goodProject
GIT_REPO_SHA1 "abc123"
CDASH_REVISION_BUILDS_URL_OUT cdashRevesionBuildsUrlOut
)
unittest_compare_const(cdashRevesionBuildsUrlOut
"somesite.com/my-cdash/index.php?project=goodProject&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=abc123")

endfunction()


function(unittest_tribits_get_cdash_revision_nonpassing_tests_url)

message("\n***")
message("*** Testing tribits_get_cdash_revision_nonpassing_tests_url()")
message("***\n")

tribits_get_cdash_revision_nonpassing_tests_url(
CDASH_SITE_URL "somesite.com/my-cdash"
PROJECT_NAME goodProject
GIT_REPO_SHA1 "abc123"
CDASH_REVISION_NONPASSING_TESTS_URL_OUT cdashRevisionNonpassingTestsUrlOut
)
unittest_compare_const(cdashRevisionNonpassingTestsUrlOut
"somesite.com/my-cdash/queryTests.php?project=goodProject&filtercount=2&showfilters=1&filtercombine=and&field1=revision&compare1=61&value1=abc123&field2=status&compare2=62&value2=passed")

endfunction()


function(unittest_tribits_read_ctest_tag_file)

message("\n***")
Expand Down Expand Up @@ -158,6 +194,8 @@ endfunction()
unittest_initialize_vars()

# Run the unit test functions
unittest_tribits_get_cdash_revision_builds_url()
unittest_tribits_get_cdash_revision_nonpassing_tests_url()
unittest_tribits_read_ctest_tag_file()
unittest_tribits_get_cdash_site_from_drop_site_and_location()
unittest_tribits_get_cdash_index_php_from_drop_site_and_location()
Expand Down
26 changes: 26 additions & 0 deletions test/ctest_driver/TribitsExampleMetaProject/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,19 @@ set(TribitsExampleMetaProject_COMMON_CONFIG_ARGS
##########################################################################################


set(cdash_build_url_expected_regex
"https://testing[.]sandia[.]gov/cdash/index[.]php[?]project=TribitsExampleMetaProject&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=CustomSite&field2=buildname&compare2=61&value2=CTestDriver_TribitsExMetaProj_clone_default_branch_remote&field3=buildstamp&compare3=61&value3=[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]-Continuous" )
# NOTE: Above, we have to repeat [0-9] 8 times and 4 times for the regex for
# the number of digits in the buildstarttime. CMake regex does not support
# \d{8}-\d{4} :-(

set(cdash_revisions_builds_url_expected_regex
"https://testing[.]sandia[.]gov/cdash/index.php[?]project=TribitsExampleMetaProject&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=863461e3035d24c632e175c087761e83db28bdc3")

set(cdash_revisions_nonpassing_tests_expected_regex
"https://testing[.]sandia[.]gov/cdash/queryTests.php[?]project=TribitsExampleMetaProject&filtercount=2&showfilters=1&filtercombine=and&field1=revision&compare1=61&value1=863461e3035d24c632e175c087761e83db28bdc3&field2=status&compare2=62&value2=passed" )


tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_remote
OVERALL_WORKING_DIRECTORY TEST_NAME
OVERALL_NUM_MPI_PROCS 1
Expand All @@ -111,11 +124,19 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
${COMMON_ENV_ARGS}
TribitsExMetaProj_ENABLE_SECONDARY_TESTED_CODE=TRUE
CTEST_BUILD_NAME=CTestDriver_TribitsExMetaProj_clone_default_branch_remote
CTEST_SITE=CustomSite
${CTEST_S_SCRIPT_ARGS}
PASS_REGULAR_EXPRESSION_ALL
"CTEST_NOTES_FILES=''"
"First perform the initial checkout: .*/git. clone -o origin .*github.com.tribits/TribitsExampleMetaProject.git"
"Perform checkout in directory: .*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote"
"Results will be submitted on CDash at the following links:"
"Link to this build's results on CDash:"
"${cdash_build_url_expected_regex}"
"Link to all builds for this repo version on CDash:"
"${cdash_revisions_builds_url_expected_regex}"
"Link to all nonpassing tests for all builds for this repo version on CDash:"
"${cdash_revisions_nonpassing_tests_expected_regex}"
"Calling ctest_update[(][)] to update base source repo '.*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote/TribitsExampleMetaProject"
"Old revision of repository is: [a-z0-9]+"
"New revision of repository is: [a-z0-9]+"
Expand Down Expand Up @@ -315,6 +336,11 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
# * Makes sure some git commit SHA1s are shown in 'Old/New revision of
# repository' printouts from ctest_update().
#
# * Checks that CDash URLs that get produced, including for all builds with
# * the same repo version. (This check has to be here since the base
# * project repo must have a .git/ directory and we must know the exact SHA1
# * of that repo for the test, which we don in this case.)
#
# NOTE: The reason that so many tests are done in a single ctest test is the
# cost of the initial configure which checks the compilers which is very
# expensive. The other runs use
Expand Down
19 changes: 11 additions & 8 deletions test/ctest_driver/TribitsExampleProject/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1264,7 +1264,8 @@ function(generate_aao_tests)
# CONFIGURE_OPTIONS list is checked to make sure that it does not change
# without being noticed. Follow-on tests will not check all of that.

set(cdash_url_expected_regex

set(cdash_build_url_expected_regex
"https://cdash[.]site[.]com/cdash/index[.]php[?]project=CustomTribitsExProj&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=My%20Site&field2=buildname&compare2=61&value2=TriBITS_CTestDriver_AAOP_CDASH_URL&field3=buildstamp&compare3=61&value3=[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]-Custom%20CDash%20Group" )
# NOTE: Above, we have to repeat [0-9] 8 times and 4 times for the regex for
# the number of digits in the buildstarttime. CMake regex does not support
Expand All @@ -1285,7 +1286,7 @@ function(generate_aao_tests)
CTEST_DASHBOARD_ROOT=PWD
${AAO_COMMON_ENV_ARGS}
CTEST_PARALLEL_LEVEL=3
CTEST_DO_SUBMIT=OFF # Never submit this
CTEST_DO_SUBMIT=OFF # Never submit this (see below note)
CTEST_SITE="My Site"
CTEST_DROP_SITE="cdash.site.com"
CTEST_PROJECT_NAME="CustomTribitsExProj"
Expand All @@ -1299,17 +1300,19 @@ function(generate_aao_tests)
"NONE does not exist, skipping extra repositories"
"Final set of enabled packages: SimpleCxx 1"
"Final set of enabled SE packages: SimpleCxx 1"
"Results will be submitted to CDash at:"
"Results will be submitted on CDash at the following links:"
"Link to this build's results on CDash:"
"${cdash_build_url_expected_regex}"
"File '' does NOT exist so all tests passed"
"See results submitted to CDash at:"
"${cdash_url_expected_regex}"
"See results submitted on CDash at the following links:"
"TRIBITS_CTEST_DRIVER: OVERALL: ALL PASSED"
ALWAYS_FAIL_ON_NONZERO_RETURN
TEST_2
MESSAGE "Check that CDashBuildUrl.txt exists and lists the right URL."
CMND cat ARGS BUILD/CDashBuildUrl.txt
MESSAGE "Check that CDashResults.txt exists and lists the right URL."
CMND cat ARGS BUILD/CDashResults.txt
PASS_REGULAR_EXPRESSION_ALL
"${cdash_url_expected_regex}"
"Link to this build's results on CDash:"
"${cdash_build_url_expected_regex}"
)
# NOTE: The above test is never actually submitted to CDash, even when
# ${PACKAGE_NAME}_CTEST_DRIVER_SUBMIT_TO is set. This is so we can
Expand Down
78 changes: 72 additions & 6 deletions tribits/ctest_driver/TribitsCTestDriverCore.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,7 @@ include(TribitsCTestDriverCoreHelpers)
# * `Determining what testing-related actions are performed (tribits_ctest_driver())`_
# * `Determining how the results are displayed on CDash (tribits_ctest_driver())`_
# * `Specifying where the results go to CDash (tribits_ctest_driver())`_
# * `Links to results on CDash (tribits_ctest_driver())`_
# * `Determining what TriBITS repositories are included (tribits_ctest_driver())`_
# * `All-at-once versus package-by-package mode (tribits_ctest_driver())`_
# * `Multiple ctest -S invocations (tribits_ctest_driver())`_
Expand Down Expand Up @@ -1166,6 +1167,63 @@ include(TribitsCTestDriverCoreHelpers)
# not be performed. For more details, see `TRIBITS_2ND_CTEST_DROP_SITE`_ and
# `TRIBITS_2ND_CTEST_DROP_LOCATION`_.
#
# .. _Links to results on CDash (tribits_ctest_driver()):
#
# **Links to results on CDash (tribits_ctest_driver()):**
#
# Links to where the results will be posted on CDash are printed to STDOUT
# before it performs any actions and at end after all of the actions and
# submits have been completed.
#
# The results are printed to STDOUT in a section that looks like::
#
# Link to this build's results on CDash:
#
# <cdash-build-url>
#
# Link to all builds for this repo version on CDash:
#
# <cdash-revision-builds-url>
#
# Link to all nonpassing tests for all builds for this repo version on CDash:
#
# <cdash-revision-nonpassing-tests-url>
#
# The URL ``<cdash-build-url>`` is created from the buildname, site, and
# buildstartime fields which is known from the TAG file created by CTest.
# This allows access the results for this particular build on CDash by just
# clicking that link.
#
# The URL ``<cdash-revision-builds-url>`` provides a link to a CDash
# ``index.php`` query that includes all of the builds with the same base Git
# repo SHA1. This allows comparing the results of this build for other builds
# for this same version of the base Git repository.
#
# The URL ``<cdash-revision-nonpassing-tests-url>`` gives a link to a CDash
# ``queryTests.php`` query for all of the nonpassing tests for all of the
# builds with this same base project Git repo SHA1. This allows comparing
# test failures across all of the builds for the same base project Git repo
# version.
#
# NOTE: The links ``<cdash-revision-builds-url>`` and
# ``<cdash-revision-nonpassing-tests-url>`` are only provided if the base
# project Git repo has the ``.git/`` subdirectory and if ``git log``
# successfully returns the SHA1 for that base Git repo.
#
# NOTE: The links ``<cdash-revision-builds-url>`` and
# ``<cdash-revision-nonpassing-tests-url>`` only consider the Git SHA1 of the
# base project Git repo. For multi-repo projects (see `Multi-Repository
# Support`_), you may get results for builds with different subrepo versions
# and therefore may be comparing apples and oranges. (Projects that commit a
# ``<Project>SubRepoVersion.txt`` file to their base Git repo or use Git
# Submodules will have unique base project Git repo SHA1s for different
# versions of the project's repos.)
#
# In addition, a text file ``CDashResults.txt`` will be written in the build
# directory that contains this same CDash link information shown above. This
# allows a process to cat the file ``CDashResults.txt`` to get links to the
# results on CDash.
#
# .. _Determining what TriBITS repositories are included (tribits_ctest_driver()):
#
# **Determining what TriBITS repositories are included (tribits_ctest_driver()):**
Expand Down Expand Up @@ -2093,10 +2151,15 @@ function(tribits_ctest_driver)
ctest_start(${CTEST_START_ARGS})

tribits_remember_if_configure_attempted()
tribits_get_build_url_and_write_to_file(CDASH_BUILD_URL
"${CTEST_BINARY_DIRECTORY}/CDashBuildUrl.txt")
tribits_print_cdash_url("Results will be submitted to CDash at:"
"${CDASH_BUILD_URL}")

tribits_get_cdash_results_string_and_write_to_file(
CDASH_RESULTS_STRING_OUT CDASH_RESULTS_STRING
CDASH_RESULTS_FILE_OUT "${CTEST_BINARY_DIRECTORY}/CDashResults.txt" )
message("Results will be submitted on CDash at the following links:\n\n"
"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
"${CDASH_RESULTS_STRING}\n"
"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
)

message(
"\n***"
Expand Down Expand Up @@ -2370,8 +2433,11 @@ function(tribits_ctest_driver)

report_queued_errors()

tribits_print_cdash_url("See results submitted to CDash at:"
"${CDASH_BUILD_URL}")
message("\nSee results submitted on CDash at the following links:\n\n"
"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
"${CDASH_RESULTS_STRING}\n"
"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
)

if ((NOT UPDATE_FAILED) AND ("${${PROJECT_NAME}_FAILED_PACKAGES}" STREQUAL ""))
message(
Expand Down
Loading

0 comments on commit decd86b

Please sign in to comment.