Merge branch 'develop' into collected-small-fixes

cmake/Modules/ExternalCMakeProject.cmake

@@ -9,8 +9,22 @@ function(ExternalCMakeProject target url hash basedir cmakedir cmakefile)
 
   get_filename_component(archive ${url} NAME)
   file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/_deps/src)
-  message(STATUS "Downloading ${url}")
-  file(DOWNLOAD ${url} ${CMAKE_BINARY_DIR}/_deps/${archive} EXPECTED_HASH MD5=${hash} SHOW_PROGRESS)
+  if(EXISTS ${CMAKE_BINARY_DIR}/_deps/${archive})
+    file(MD5 ${CMAKE_BINARY_DIR}/_deps/${archive} DL_MD5)
+  endif()
+  if(NOT "${DL_MD5}" STREQUAL "${hash}")
+    message(STATUS "Downloading ${url}")
+    file(DOWNLOAD ${url} ${CMAKE_BINARY_DIR}/_deps/${archive} STATUS DL_STATUS SHOW_PROGRESS)
+    file(MD5 ${CMAKE_BINARY_DIR}/_deps/${archive} DL_MD5)
+    if((NOT DL_STATUS EQUAL 0) OR (NOT "${DL_MD5}" STREQUAL "${hash}"))
+      set(${target}_URL ${url})
+      GetFallbackURL(${target}_URL fallback)
+      message(WARNING "Download from primary URL ${url} failed\nTrying fallback URL ${fallback}")
+      file(DOWNLOAD ${fallback} ${CMAKE_BINARY_DIR}/_deps/${archive} EXPECTED_HASH MD5=${hash} SHOW_PROGRESS)
+    endif()
+  else()
+    message(STATUS "Using already downloaded archive ${CMAKE_BINARY_DIR}/_deps/${archive}")
+  endif()
   message(STATUS "Unpacking and configuring ${archive}")
   execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${CMAKE_BINARY_DIR}/_deps/${archive}
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/_deps/src)

cmake/Modules/LAMMPSUtils.cmake

@@ -158,3 +158,14 @@ if((CMAKE_SYSTEM_NAME STREQUAL "Linux") AND (EXISTS /etc/os-release))
   set(CMAKE_LINUX_DISTRO ${distro})
   set(CMAKE_DISTRO_VERSION ${disversion})
 endif()
+
+function(GetFallbackURL input output)
+  string(REPLACE "_URL" "" _tmp ${input})
+  string(TOLOWER ${_tmp} libname)
+  string(REGEX REPLACE "^https://.*/([^/]+gz)" "${LAMMPS_THIRDPARTY_URL}/${libname}-\\1" newurl "${${input}}")
+  if ("${newurl}" STREQUAL "${${input}}")
+    set(${output} "" PARENT_SCOPE)
+  else()
+    set(${output} ${newurl} PARENT_SCOPE)
+  endif()
+endfunction(GetFallbackURL)

cmake/Modules/Packages/GPU.cmake

@@ -428,15 +428,17 @@ elseif(GPU_API STREQUAL "HIP")
 
       if(DOWNLOAD_CUB)
         message(STATUS "CUB download requested")
-        set(CUB_URL "https://github.com/NVlabs/cub/archive/1.12.0.tar.gz" CACHE STRING "URL for CUB tarball")
+        # TODO: test update to current version 1.17.2
+        set(CUB_URL "https://github.com/nvidia/cub/archive/1.12.0.tar.gz" CACHE STRING "URL for CUB tarball")
         set(CUB_MD5 "1cf595beacafff104700921bac8519f3" CACHE STRING "MD5 checksum of CUB tarball")
         mark_as_advanced(CUB_URL)
         mark_as_advanced(CUB_MD5)
+        GetFallbackURL(CUB_URL CUB_FALLBACK)
 
         include(ExternalProject)
 
         ExternalProject_Add(CUB
-          URL     ${CUB_URL}
+          URL     ${CUB_URL} ${CUB_FALLBACK}
           URL_MD5 ${CUB_MD5}
           PREFIX "${CMAKE_CURRENT_BINARY_DIR}"
           CONFIGURE_COMMAND ""

cmake/Modules/Packages/KOKKOS.cmake

@@ -53,8 +53,10 @@ if(DOWNLOAD_KOKKOS)
   set(KOKKOS_MD5 "f140e02b826223b1045207d9bc10d404" CACHE STRING "MD5 checksum of KOKKOS tarball")
   mark_as_advanced(KOKKOS_URL)
   mark_as_advanced(KOKKOS_MD5)
+  GetFallbackURL(KOKKOS_URL KOKKOS_FALLBACK)
+
   ExternalProject_Add(kokkos_build
-    URL     ${KOKKOS_URL}
+    URL     ${KOKKOS_URL} ${KOKKOS_FALLBACK}
     URL_MD5 ${KOKKOS_MD5}
     CMAKE_ARGS ${KOKKOS_LIB_BUILD_ARGS}
     BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libkokkoscore.a <INSTALL_DIR>/lib/libkokkoscontainers.a

cmake/Modules/Packages/LATTE.cmake

@@ -19,6 +19,7 @@ if(DOWNLOAD_LATTE)
   set(LATTE_MD5 "820e73a457ced178c08c71389a385de7" CACHE STRING "MD5 checksum of LATTE tarball")
   mark_as_advanced(LATTE_URL)
   mark_as_advanced(LATTE_MD5)
+  GetFallbackURL(LATTE_URL LATTE_FALLBACK)
 
   # CMake cannot pass BLAS or LAPACK library variable to external project if they are a list
   list(LENGTH BLAS_LIBRARIES} NUM_BLAS)
@@ -30,7 +31,7 @@ if(DOWNLOAD_LATTE)
 
   include(ExternalProject)
   ExternalProject_Add(latte_build
-    URL     ${LATTE_URL}
+    URL     ${LATTE_URL} ${LATTE_FALLBACK}
     URL_MD5 ${LATTE_MD5}
     SOURCE_SUBDIR cmake
     CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> ${CMAKE_REQUEST_PIC} -DCMAKE_INSTALL_LIBDIR=lib

cmake/Modules/Packages/MDI.cmake

@@ -12,6 +12,7 @@ if(DOWNLOAD_MDI)
   set(MDI_MD5 "7a222353ae8e03961d5365e6cd48baee" CACHE STRING "MD5 checksum for MDI tarball")
   mark_as_advanced(MDI_URL)
   mark_as_advanced(MDI_MD5)
+  GetFallbackURL(MDI_URL MDI_FALLBACK)
   enable_language(C)
 
   # only ON/OFF are allowed for "mpi" flag when building MDI library
@@ -63,7 +64,7 @@ if(DOWNLOAD_MDI)
   # support cross-compilation and ninja-build
   include(ExternalProject)
   ExternalProject_Add(mdi_build
-    URL     ${MDI_URL}
+    URL     ${MDI_URL} ${MDI_FALLBACK}
     URL_MD5 ${MDI_MD5}
     PREFIX ${CMAKE_CURRENT_BINARY_DIR}/mdi_build_ext
     CMAKE_ARGS

cmake/Modules/Packages/ML-HDNNP.cmake

@@ -10,6 +10,7 @@ if(DOWNLOAD_N2P2)
   set(N2P2_MD5 "a2d9ab7f676b3a74a324fc1eda0a911d" CACHE STRING "MD5 checksum of N2P2 tarball")
   mark_as_advanced(N2P2_URL)
   mark_as_advanced(N2P2_MD5)
+  GetFallbackURL(N2P2_URL N2P2_FALLBACK)
 
   # adjust settings from detected compiler to compiler platform in n2p2 library
   # set compiler specific flag to turn on C++11 syntax (required on macOS and CentOS 7)
@@ -72,7 +73,7 @@ if(DOWNLOAD_N2P2)
   # download compile n2p2 library. much patch MPI calls in LAMMPS interface to accommodate MPI-2 (e.g. for cross-compiling)
   include(ExternalProject)
   ExternalProject_Add(n2p2_build
-    URL     ${N2P2_URL}
+    URL     ${N2P2_URL} ${N2P2_FALLBACK}
     URL_MD5 ${N2P2_MD5}
     UPDATE_COMMAND ""
     CONFIGURE_COMMAND ""

cmake/Modules/Packages/ML-PACE.cmake

@@ -3,9 +3,23 @@ set(PACELIB_URL "https://github.com/ICAMS/lammps-user-pace/archive/refs/tags/v.2
 set(PACELIB_MD5 "4f0b3b5b14456fe9a73b447de3765caa" CACHE STRING "MD5 checksum of PACE evaluator library tarball")
 mark_as_advanced(PACELIB_URL)
 mark_as_advanced(PACELIB_MD5)
+GetFallbackURL(PACELIB_URL PACELIB_FALLBACK)
 
 # download library sources to build folder
-file(DOWNLOAD ${PACELIB_URL} ${CMAKE_BINARY_DIR}/libpace.tar.gz EXPECTED_HASH MD5=${PACELIB_MD5}) #SHOW_PROGRESS
+if(EXISTS ${CMAKE_BINARY_DIR}/libpace.tar.gz)
+  file(MD5 ${CMAKE_BINARY_DIR}/libpace.tar.gz DL_MD5)
+endif()
+if(NOT "${DL_MD5}" STREQUAL "${PACELIB_MD5}")
+  message(STATUS "Downloading ${PACELIB_URL}")
+  file(DOWNLOAD ${PACELIB_URL} ${CMAKE_BINARY_DIR}/libpace.tar.gz STATUS DL_STATUS SHOW_PROGRESS)
+  file(MD5 ${CMAKE_BINARY_DIR}/libpace.tar.gz DL_MD5)
+  if((NOT DL_STATUS EQUAL 0) OR (NOT "${DL_MD5}" STREQUAL "${PACELIB_MD5}"))
+    message(WARNING "Download from primary URL ${PACELIB_URL} failed\nTrying fallback URL ${PACELIB_FALLBACK}")
+    file(DOWNLOAD ${PACELIB_FALLBACK} ${CMAKE_BINARY_DIR}/libpace.tar.gz EXPECTED_HASH MD5=${PACELIB_MD5} SHOW_PROGRESS)
+  else()
+    message(STATUS "Using already downloaded archive ${CMAKE_BINARY_DIR}/libpace.tar.gz")
+  endif()
+endif()
 
 # uncompress downloaded sources
 execute_process(

cmake/Modules/Packages/PLUMED.cmake

@@ -59,10 +59,11 @@ if(DOWNLOAD_PLUMED)
 
   mark_as_advanced(PLUMED_URL)
   mark_as_advanced(PLUMED_MD5)
+  GetFallbackURL(PLUMED_URL PLUMED_FALLBACK)
 
   include(ExternalProject)
   ExternalProject_Add(plumed_build
-    URL     ${PLUMED_URL}
+    URL     ${PLUMED_URL} ${PLUMED_FALLBACK}
     URL_MD5 ${PLUMED_MD5}
     BUILD_IN_SOURCE 1
     CONFIGURE_COMMAND <SOURCE_DIR>/configure --prefix=<INSTALL_DIR>

cmake/Modules/Packages/SCAFACOS.cmake

@@ -18,6 +18,8 @@ if(DOWNLOAD_SCAFACOS)
   set(SCAFACOS_MD5 "bd46d74e3296bd8a444d731bb10c1738" CACHE STRING "MD5 checksum of SCAFACOS tarball")
   mark_as_advanced(SCAFACOS_URL)
   mark_as_advanced(SCAFACOS_MD5)
+  GetFallbackURL(SCAFACOS_URL SCAFACOS_FALLBACK)
+
 
   # version 1.0.1 needs a patch to compile and linke cleanly with GCC 10 and later.
   file(DOWNLOAD ${LAMMPS_THIRDPARTY_URL}/scafacos-1.0.1-fix.diff ${CMAKE_CURRENT_BINARY_DIR}/scafacos-1.0.1.fix.diff
@@ -30,7 +32,7 @@ if(DOWNLOAD_SCAFACOS)
 
   include(ExternalProject)
   ExternalProject_Add(scafacos_build
-    URL     ${SCAFACOS_URL}
+    URL     ${SCAFACOS_URL} ${SCAFACOS_FALLBACK}
     URL_MD5 ${SCAFACOS_MD5}
     PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_BINARY_DIR}/scafacos-1.0.1.fix.diff
     CONFIGURE_COMMAND <SOURCE_DIR>/configure --prefix=<INSTALL_DIR> --disable-doc

lib/hdnnp/.gitignore

@@ -0,0 +1,3 @@
+/includelink
+/liblink
+/n2p2-*

lib/hdnnp/Install.py

@@ -10,7 +10,7 @@
 from argparse import ArgumentParser
 
 sys.path.append('..')
-from install_helpers import get_cpus, fullpath, geturl, checkmd5sum
+from install_helpers import get_cpus, fullpath, geturl, checkmd5sum, getfallback
 
 parser = ArgumentParser(prog='Install.py',
                         description="LAMMPS library build wrapper script")
@@ -77,14 +77,21 @@
 
 if buildflag:
   url = "https://github.com/CompPhysVienna/n2p2/archive/v%s.tar.gz" % (version)
-  filename = "n2p2-%s.tar.gz" %version
-  print("Downloading n2p2 ...")
-  geturl(url, filename)
+  filename = "n2p2-%s.tar.gz" % version
+  fallback = getfallback('n2p2', url)
+  print("Downloading n2p2 from", url)
+  try:
+    geturl(url, filename)
+  except:
+    geturl(fallback, filename)
 
   # verify downloaded archive integrity via md5 checksum, if known.
   if version in checksums:
     if not checkmd5sum(checksums[version], filename):
-      sys.exit("Checksum for n2p2 library does not match")
+      print("Checksum did not match. Trying fallback URL", fallback)
+      geturl(fallback, filename)
+      if not checkmd5sum(checksums[version], filename):
+        sys.exit("Checksum for n2p2 library does not match for fallback, too.")
 
   print("Unpacking n2p2 source tarball ...")
   if os.path.exists("%s/n2p2-%s" % (homepath, version)):

lib/install_helpers.py

@@ -1,4 +1,7 @@
-import hashlib,os,subprocess
+import hashlib
+import os
+import re
+import subprocess
 
 # try to auto-detect the maximum number of available CPUs
 def get_cpus():
@@ -32,31 +35,31 @@ def is_exe(fpath):
 
   return None
 
-def geturl(url,fname):
+def geturl(url, fname):
   success = False
 
   if which('curl') != None:
-    cmd = 'curl -L -o "%s" %s' % (fname,url)
+    cmd = 'curl -L -o "%s" %s' % (fname, url)
     try:
-      subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
+      subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
       success = True
     except subprocess.CalledProcessError as e:
       print("Calling curl failed with: %s" % e.output.decode('UTF-8'))
 
   if not success and which('wget') != None:
-    cmd = 'wget -O "%s" %s' % (fname,url)
+    cmd = 'wget -O "%s" %s' % (fname, url)
     try:
-      subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
+      subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
       success = True
     except subprocess.CalledProcessError as e:
       print("Calling wget failed with: %s" % e.output.decode('UTF-8'))
 
   if not success:
-    error("Failed to download source code with 'curl' or 'wget'")
+    raise Exception("Failed to download source code with 'curl' or 'wget' from " + url)
   return
 
-def checkmd5sum(md5sum,fname):
-    with open(fname,'rb') as fh:
+def checkmd5sum(md5sum, fname):
+    with open(fname, 'rb') as fh:
         m = hashlib.md5()
         while True:
             data = fh.read(81920)
@@ -66,3 +69,6 @@ def checkmd5sum(md5sum,fname):
     fh.close()
     return m.hexdigest() == md5sum
 
+def getfallback(lib, url):
+  archive = re.sub(r'^https://.*/([^/]+gz)', r'-\1', url)
+  return 'https://download.lammps.org/thirdparty/' + lib + archive

lib/latte/Install.py

@@ -10,7 +10,7 @@
 from argparse import ArgumentParser
 
 sys.path.append('..')
-from install_helpers import fullpath, geturl, checkmd5sum
+from install_helpers import fullpath, geturl, checkmd5sum, getfallback
 
 parser = ArgumentParser(prog='Install.py',
                         description="LAMMPS library build wrapper script")
@@ -86,17 +86,25 @@
   url = "https://github.com/lanl/LATTE/archive/v%s.tar.gz" % version
   lattepath = fullpath(homepath)
   lattedir = os.path.join(lattepath, homedir)
+  fallback = getfallback('latte', url)
+  filename = 'LATTE.tar.gz'
 
 # download and unpack LATTE tarball
 
 if buildflag:
   print("Downloading LATTE ...")
-  geturl(url, "LATTE.tar.gz")
+  try:
+    geturl(url, filename)
+  except:
+    geturl(fallback, filename)
 
   # verify downloaded archive integrity via md5 checksum, if known.
   if version in checksums:
-    if not checkmd5sum(checksums[version], 'LATTE.tar.gz'):
-      sys.exit("Checksum for LATTE library does not match")
+    if not checkmd5sum(checksums[version], filename):
+      print("Checksum did not match. Trying fallback URL", fallback)
+      geturl(fallback, filename)
+      if not checkmd5sum(checksums[version], filename):
+        sys.exit("Checksum for LATTE library does not match for fallback, too.")
 
   print("Unpacking LATTE ...")
   if os.path.exists(lattedir):