From 5d16bea899d64abb1d948823dacd8f88a67dcfc1 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Tue, 31 Jan 2023 16:35:06 -0500 Subject: [PATCH] implement download fallback URLs pointing to download.lammps.org for CMake --- cmake/Modules/ExternalCMakeProject.cmake | 18 ++++++++++++++++-- cmake/Modules/LAMMPSUtils.cmake | 11 +++++++++++ cmake/Modules/Packages/GPU.cmake | 6 ++++-- cmake/Modules/Packages/KOKKOS.cmake | 4 +++- cmake/Modules/Packages/LATTE.cmake | 3 ++- cmake/Modules/Packages/MDI.cmake | 3 ++- cmake/Modules/Packages/ML-HDNNP.cmake | 3 ++- cmake/Modules/Packages/ML-PACE.cmake | 16 +++++++++++++++- cmake/Modules/Packages/PLUMED.cmake | 3 ++- cmake/Modules/Packages/SCAFACOS.cmake | 4 +++- 10 files changed, 60 insertions(+), 11 deletions(-) diff --git a/cmake/Modules/ExternalCMakeProject.cmake b/cmake/Modules/ExternalCMakeProject.cmake index 855ce254c9b..75c33ab99ec 100644 --- a/cmake/Modules/ExternalCMakeProject.cmake +++ b/cmake/Modules/ExternalCMakeProject.cmake @@ -9,8 +9,22 @@ function(ExternalCMakeProject target url hash basedir cmakedir cmakefile) get_filename_component(archive ${url} NAME) file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/_deps/src) - message(STATUS "Downloading ${url}") - file(DOWNLOAD ${url} ${CMAKE_BINARY_DIR}/_deps/${archive} EXPECTED_HASH MD5=${hash} SHOW_PROGRESS) + if(EXISTS ${CMAKE_BINARY_DIR}/_deps/${archive}) + file(MD5 ${CMAKE_BINARY_DIR}/_deps/${archive} DL_MD5) + endif() + if(NOT "${DL_MD5}" STREQUAL "${hash}") + message(STATUS "Downloading ${url}") + file(DOWNLOAD ${url} ${CMAKE_BINARY_DIR}/_deps/${archive} STATUS DL_STATUS SHOW_PROGRESS) + file(MD5 ${CMAKE_BINARY_DIR}/_deps/${archive} DL_MD5) + if((NOT DL_STATUS EQUAL 0) OR (NOT "${DL_MD5}" STREQUAL "${hash}")) + set(${target}_URL ${url}) + GetFallbackURL(${target}_URL fallback) + message(WARNING "Download from primary URL ${url} failed\nTrying fallback URL ${fallback}") + file(DOWNLOAD ${fallback} ${CMAKE_BINARY_DIR}/_deps/${archive} EXPECTED_HASH MD5=${hash} SHOW_PROGRESS) + endif() + else() + message(STATUS "Using already downloaded archive ${CMAKE_BINARY_DIR}/_deps/${archive}") + endif() message(STATUS "Unpacking and configuring ${archive}") execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${CMAKE_BINARY_DIR}/_deps/${archive} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/_deps/src) diff --git a/cmake/Modules/LAMMPSUtils.cmake b/cmake/Modules/LAMMPSUtils.cmake index 9b42dafc441..bb691043088 100644 --- a/cmake/Modules/LAMMPSUtils.cmake +++ b/cmake/Modules/LAMMPSUtils.cmake @@ -158,3 +158,14 @@ if((CMAKE_SYSTEM_NAME STREQUAL "Linux") AND (EXISTS /etc/os-release)) set(CMAKE_LINUX_DISTRO ${distro}) set(CMAKE_DISTRO_VERSION ${disversion}) endif() + +function(GetFallbackURL input output) + string(REPLACE "_URL" "" _tmp ${input}) + string(TOLOWER ${_tmp} libname) + string(REGEX REPLACE "^https://.*/([^/]+gz)" "${LAMMPS_THIRDPARTY_URL}/${libname}-\\1" newurl "${${input}}") + if ("${newurl}" STREQUAL "${${input}}") + set(${output} "" PARENT_SCOPE) + else() + set(${output} ${newurl} PARENT_SCOPE) + endif() +endfunction(GetFallbackURL) diff --git a/cmake/Modules/Packages/GPU.cmake b/cmake/Modules/Packages/GPU.cmake index 24d95382061..cf321eee9a0 100644 --- a/cmake/Modules/Packages/GPU.cmake +++ b/cmake/Modules/Packages/GPU.cmake @@ -428,15 +428,17 @@ elseif(GPU_API STREQUAL "HIP") if(DOWNLOAD_CUB) message(STATUS "CUB download requested") - set(CUB_URL "https://github.com/NVlabs/cub/archive/1.12.0.tar.gz" CACHE STRING "URL for CUB tarball") + # TODO: test update to current version 1.17.2 + set(CUB_URL "https://github.com/nvidia/cub/archive/1.12.0.tar.gz" CACHE STRING "URL for CUB tarball") set(CUB_MD5 "1cf595beacafff104700921bac8519f3" CACHE STRING "MD5 checksum of CUB tarball") mark_as_advanced(CUB_URL) mark_as_advanced(CUB_MD5) + GetFallbackURL(CUB_URL CUB_FALLBACK) include(ExternalProject) ExternalProject_Add(CUB - URL ${CUB_URL} + URL ${CUB_URL} ${CUB_FALLBACK} URL_MD5 ${CUB_MD5} PREFIX "${CMAKE_CURRENT_BINARY_DIR}" CONFIGURE_COMMAND "" diff --git a/cmake/Modules/Packages/KOKKOS.cmake b/cmake/Modules/Packages/KOKKOS.cmake index de64df7268b..b4eb227d61b 100644 --- a/cmake/Modules/Packages/KOKKOS.cmake +++ b/cmake/Modules/Packages/KOKKOS.cmake @@ -53,8 +53,10 @@ if(DOWNLOAD_KOKKOS) set(KOKKOS_MD5 "f140e02b826223b1045207d9bc10d404" CACHE STRING "MD5 checksum of KOKKOS tarball") mark_as_advanced(KOKKOS_URL) mark_as_advanced(KOKKOS_MD5) + GetFallbackURL(KOKKOS_URL KOKKOS_FALLBACK) + ExternalProject_Add(kokkos_build - URL ${KOKKOS_URL} + URL ${KOKKOS_URL} ${KOKKOS_FALLBACK} URL_MD5 ${KOKKOS_MD5} CMAKE_ARGS ${KOKKOS_LIB_BUILD_ARGS} BUILD_BYPRODUCTS /lib/libkokkoscore.a /lib/libkokkoscontainers.a diff --git a/cmake/Modules/Packages/LATTE.cmake b/cmake/Modules/Packages/LATTE.cmake index d7793fa257c..bedd7a64fca 100644 --- a/cmake/Modules/Packages/LATTE.cmake +++ b/cmake/Modules/Packages/LATTE.cmake @@ -19,6 +19,7 @@ if(DOWNLOAD_LATTE) set(LATTE_MD5 "820e73a457ced178c08c71389a385de7" CACHE STRING "MD5 checksum of LATTE tarball") mark_as_advanced(LATTE_URL) mark_as_advanced(LATTE_MD5) + GetFallbackURL(LATTE_URL LATTE_FALLBACK) # CMake cannot pass BLAS or LAPACK library variable to external project if they are a list list(LENGTH BLAS_LIBRARIES} NUM_BLAS) @@ -30,7 +31,7 @@ if(DOWNLOAD_LATTE) include(ExternalProject) ExternalProject_Add(latte_build - URL ${LATTE_URL} + URL ${LATTE_URL} ${LATTE_FALLBACK} URL_MD5 ${LATTE_MD5} SOURCE_SUBDIR cmake CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= ${CMAKE_REQUEST_PIC} -DCMAKE_INSTALL_LIBDIR=lib diff --git a/cmake/Modules/Packages/MDI.cmake b/cmake/Modules/Packages/MDI.cmake index c1e25347af5..67c1b82dabe 100644 --- a/cmake/Modules/Packages/MDI.cmake +++ b/cmake/Modules/Packages/MDI.cmake @@ -12,6 +12,7 @@ if(DOWNLOAD_MDI) set(MDI_MD5 "7a222353ae8e03961d5365e6cd48baee" CACHE STRING "MD5 checksum for MDI tarball") mark_as_advanced(MDI_URL) mark_as_advanced(MDI_MD5) + GetFallbackURL(MDI_URL MDI_FALLBACK) enable_language(C) # only ON/OFF are allowed for "mpi" flag when building MDI library @@ -63,7 +64,7 @@ if(DOWNLOAD_MDI) # support cross-compilation and ninja-build include(ExternalProject) ExternalProject_Add(mdi_build - URL ${MDI_URL} + URL ${MDI_URL} ${MDI_FALLBACK} URL_MD5 ${MDI_MD5} PREFIX ${CMAKE_CURRENT_BINARY_DIR}/mdi_build_ext CMAKE_ARGS diff --git a/cmake/Modules/Packages/ML-HDNNP.cmake b/cmake/Modules/Packages/ML-HDNNP.cmake index f71d08f3ab1..adffcfa77e8 100644 --- a/cmake/Modules/Packages/ML-HDNNP.cmake +++ b/cmake/Modules/Packages/ML-HDNNP.cmake @@ -10,6 +10,7 @@ if(DOWNLOAD_N2P2) set(N2P2_MD5 "9595b066636cd6b90b0fef93398297a5" CACHE STRING "MD5 checksum of N2P2 tarball") mark_as_advanced(N2P2_URL) mark_as_advanced(N2P2_MD5) + GetFallbackURL(N2P2_URL N2P2_FALLBACK) # adjust settings from detected compiler to compiler platform in n2p2 library # set compiler specific flag to turn on C++11 syntax (required on macOS and CentOS 7) @@ -72,7 +73,7 @@ if(DOWNLOAD_N2P2) # download compile n2p2 library. much patch MPI calls in LAMMPS interface to accommodate MPI-2 (e.g. for cross-compiling) include(ExternalProject) ExternalProject_Add(n2p2_build - URL ${N2P2_URL} + URL ${N2P2_URL} ${N2P2_FALLBACK} URL_MD5 ${N2P2_MD5} UPDATE_COMMAND "" CONFIGURE_COMMAND "" diff --git a/cmake/Modules/Packages/ML-PACE.cmake b/cmake/Modules/Packages/ML-PACE.cmake index b64584e5a70..f0e4b49e499 100644 --- a/cmake/Modules/Packages/ML-PACE.cmake +++ b/cmake/Modules/Packages/ML-PACE.cmake @@ -3,9 +3,23 @@ set(PACELIB_URL "https://github.com/ICAMS/lammps-user-pace/archive/refs/tags/v.2 set(PACELIB_MD5 "f418d32b60e531063ac4285bf702b468" CACHE STRING "MD5 checksum of PACE evaluator library tarball") mark_as_advanced(PACELIB_URL) mark_as_advanced(PACELIB_MD5) +GetFallbackURL(PACELIB_URL PACELIB_FALLBACK) # download library sources to build folder -file(DOWNLOAD ${PACELIB_URL} ${CMAKE_BINARY_DIR}/libpace.tar.gz EXPECTED_HASH MD5=${PACELIB_MD5}) #SHOW_PROGRESS +if(EXISTS ${CMAKE_BINARY_DIR}/libpace.tar.gz) + file(MD5 ${CMAKE_BINARY_DIR}/libpace.tar.gz DL_MD5) +endif() +if(NOT "${DL_MD5}" STREQUAL "${PACELIB_MD5}") + message(STATUS "Downloading ${PACELIB_URL}") + file(DOWNLOAD ${PACELIB_URL} ${CMAKE_BINARY_DIR}/libpace.tar.gz STATUS DL_STATUS SHOW_PROGRESS) + file(MD5 ${CMAKE_BINARY_DIR}/libpace.tar.gz DL_MD5) + if((NOT DL_STATUS EQUAL 0) OR (NOT "${DL_MD5}" STREQUAL "${PACELIB_MD5}")) + message(WARNING "Download from primary URL ${PACELIB_URL} failed\nTrying fallback URL ${PACELIB_FALLBACK}") + file(DOWNLOAD ${PACELIB_FALLBACK} ${CMAKE_BINARY_DIR}/libpace.tar.gz EXPECTED_HASH MD5=${PACELIB_MD5} SHOW_PROGRESS) + else() + message(STATUS "Using already downloaded archive ${CMAKE_BINARY_DIR}/libpace.tar.gz") + endif() +endif() # uncompress downloaded sources execute_process( diff --git a/cmake/Modules/Packages/PLUMED.cmake b/cmake/Modules/Packages/PLUMED.cmake index e0e4ff0dde6..f231d148bde 100644 --- a/cmake/Modules/Packages/PLUMED.cmake +++ b/cmake/Modules/Packages/PLUMED.cmake @@ -59,10 +59,11 @@ if(DOWNLOAD_PLUMED) mark_as_advanced(PLUMED_URL) mark_as_advanced(PLUMED_MD5) + GetFallbackURL(PLUMED_URL PLUMED_FALLBACK) include(ExternalProject) ExternalProject_Add(plumed_build - URL ${PLUMED_URL} + URL ${PLUMED_URL} ${PLUMED_FALLBACK} URL_MD5 ${PLUMED_MD5} BUILD_IN_SOURCE 1 CONFIGURE_COMMAND /configure --prefix= diff --git a/cmake/Modules/Packages/SCAFACOS.cmake b/cmake/Modules/Packages/SCAFACOS.cmake index de611a1edbf..9a5580163f3 100644 --- a/cmake/Modules/Packages/SCAFACOS.cmake +++ b/cmake/Modules/Packages/SCAFACOS.cmake @@ -18,6 +18,8 @@ if(DOWNLOAD_SCAFACOS) set(SCAFACOS_MD5 "bd46d74e3296bd8a444d731bb10c1738" CACHE STRING "MD5 checksum of SCAFACOS tarball") mark_as_advanced(SCAFACOS_URL) mark_as_advanced(SCAFACOS_MD5) + GetFallbackURL(SCAFACOS_URL SCAFACOS_FALLBACK) + # version 1.0.1 needs a patch to compile and linke cleanly with GCC 10 and later. file(DOWNLOAD ${LAMMPS_THIRDPARTY_URL}/scafacos-1.0.1-fix.diff ${CMAKE_CURRENT_BINARY_DIR}/scafacos-1.0.1.fix.diff @@ -30,7 +32,7 @@ if(DOWNLOAD_SCAFACOS) include(ExternalProject) ExternalProject_Add(scafacos_build - URL ${SCAFACOS_URL} + URL ${SCAFACOS_URL} ${SCAFACOS_FALLBACK} URL_MD5 ${SCAFACOS_MD5} PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_BINARY_DIR}/scafacos-1.0.1.fix.diff CONFIGURE_COMMAND /configure --prefix= --disable-doc