From 5d16bea899d64abb1d948823dacd8f88a67dcfc1 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Tue, 31 Jan 2023 16:35:06 -0500 Subject: [PATCH 1/2] implement download fallback URLs pointing to download.lammps.org for CMake --- cmake/Modules/ExternalCMakeProject.cmake | 18 ++++++++++++++++-- cmake/Modules/LAMMPSUtils.cmake | 11 +++++++++++ cmake/Modules/Packages/GPU.cmake | 6 ++++-- cmake/Modules/Packages/KOKKOS.cmake | 4 +++- cmake/Modules/Packages/LATTE.cmake | 3 ++- cmake/Modules/Packages/MDI.cmake | 3 ++- cmake/Modules/Packages/ML-HDNNP.cmake | 3 ++- cmake/Modules/Packages/ML-PACE.cmake | 16 +++++++++++++++- cmake/Modules/Packages/PLUMED.cmake | 3 ++- cmake/Modules/Packages/SCAFACOS.cmake | 4 +++- 10 files changed, 60 insertions(+), 11 deletions(-) diff --git a/cmake/Modules/ExternalCMakeProject.cmake b/cmake/Modules/ExternalCMakeProject.cmake index 855ce254c9b..75c33ab99ec 100644 --- a/cmake/Modules/ExternalCMakeProject.cmake +++ b/cmake/Modules/ExternalCMakeProject.cmake @@ -9,8 +9,22 @@ function(ExternalCMakeProject target url hash basedir cmakedir cmakefile) get_filename_component(archive ${url} NAME) file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/_deps/src) - message(STATUS "Downloading ${url}") - file(DOWNLOAD ${url} ${CMAKE_BINARY_DIR}/_deps/${archive} EXPECTED_HASH MD5=${hash} SHOW_PROGRESS) + if(EXISTS ${CMAKE_BINARY_DIR}/_deps/${archive}) + file(MD5 ${CMAKE_BINARY_DIR}/_deps/${archive} DL_MD5) + endif() + if(NOT "${DL_MD5}" STREQUAL "${hash}") + message(STATUS "Downloading ${url}") + file(DOWNLOAD ${url} ${CMAKE_BINARY_DIR}/_deps/${archive} STATUS DL_STATUS SHOW_PROGRESS) + file(MD5 ${CMAKE_BINARY_DIR}/_deps/${archive} DL_MD5) + if((NOT DL_STATUS EQUAL 0) OR (NOT "${DL_MD5}" STREQUAL "${hash}")) + set(${target}_URL ${url}) + GetFallbackURL(${target}_URL fallback) + message(WARNING "Download from primary URL ${url} failed\nTrying fallback URL ${fallback}") + file(DOWNLOAD ${fallback} ${CMAKE_BINARY_DIR}/_deps/${archive} EXPECTED_HASH MD5=${hash} SHOW_PROGRESS) + endif() + else() + message(STATUS "Using already downloaded archive ${CMAKE_BINARY_DIR}/_deps/${archive}") + endif() message(STATUS "Unpacking and configuring ${archive}") execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${CMAKE_BINARY_DIR}/_deps/${archive} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/_deps/src) diff --git a/cmake/Modules/LAMMPSUtils.cmake b/cmake/Modules/LAMMPSUtils.cmake index 9b42dafc441..bb691043088 100644 --- a/cmake/Modules/LAMMPSUtils.cmake +++ b/cmake/Modules/LAMMPSUtils.cmake @@ -158,3 +158,14 @@ if((CMAKE_SYSTEM_NAME STREQUAL "Linux") AND (EXISTS /etc/os-release)) set(CMAKE_LINUX_DISTRO ${distro}) set(CMAKE_DISTRO_VERSION ${disversion}) endif() + +function(GetFallbackURL input output) + string(REPLACE "_URL" "" _tmp ${input}) + string(TOLOWER ${_tmp} libname) + string(REGEX REPLACE "^https://.*/([^/]+gz)" "${LAMMPS_THIRDPARTY_URL}/${libname}-\\1" newurl "${${input}}") + if ("${newurl}" STREQUAL "${${input}}") + set(${output} "" PARENT_SCOPE) + else() + set(${output} ${newurl} PARENT_SCOPE) + endif() +endfunction(GetFallbackURL) diff --git a/cmake/Modules/Packages/GPU.cmake b/cmake/Modules/Packages/GPU.cmake index 24d95382061..cf321eee9a0 100644 --- a/cmake/Modules/Packages/GPU.cmake +++ b/cmake/Modules/Packages/GPU.cmake @@ -428,15 +428,17 @@ elseif(GPU_API STREQUAL "HIP") if(DOWNLOAD_CUB) message(STATUS "CUB download requested") - set(CUB_URL "https://github.com/NVlabs/cub/archive/1.12.0.tar.gz" CACHE STRING "URL for CUB tarball") + # TODO: test update to current version 1.17.2 + set(CUB_URL "https://github.com/nvidia/cub/archive/1.12.0.tar.gz" CACHE STRING "URL for CUB tarball") set(CUB_MD5 "1cf595beacafff104700921bac8519f3" CACHE STRING "MD5 checksum of CUB tarball") mark_as_advanced(CUB_URL) mark_as_advanced(CUB_MD5) + GetFallbackURL(CUB_URL CUB_FALLBACK) include(ExternalProject) ExternalProject_Add(CUB - URL ${CUB_URL} + URL ${CUB_URL} ${CUB_FALLBACK} URL_MD5 ${CUB_MD5} PREFIX "${CMAKE_CURRENT_BINARY_DIR}" CONFIGURE_COMMAND "" diff --git a/cmake/Modules/Packages/KOKKOS.cmake b/cmake/Modules/Packages/KOKKOS.cmake index de64df7268b..b4eb227d61b 100644 --- a/cmake/Modules/Packages/KOKKOS.cmake +++ b/cmake/Modules/Packages/KOKKOS.cmake @@ -53,8 +53,10 @@ if(DOWNLOAD_KOKKOS) set(KOKKOS_MD5 "f140e02b826223b1045207d9bc10d404" CACHE STRING "MD5 checksum of KOKKOS tarball") mark_as_advanced(KOKKOS_URL) mark_as_advanced(KOKKOS_MD5) + GetFallbackURL(KOKKOS_URL KOKKOS_FALLBACK) + ExternalProject_Add(kokkos_build - URL ${KOKKOS_URL} + URL ${KOKKOS_URL} ${KOKKOS_FALLBACK} URL_MD5 ${KOKKOS_MD5} CMAKE_ARGS ${KOKKOS_LIB_BUILD_ARGS} BUILD_BYPRODUCTS /lib/libkokkoscore.a /lib/libkokkoscontainers.a diff --git a/cmake/Modules/Packages/LATTE.cmake b/cmake/Modules/Packages/LATTE.cmake index d7793fa257c..bedd7a64fca 100644 --- a/cmake/Modules/Packages/LATTE.cmake +++ b/cmake/Modules/Packages/LATTE.cmake @@ -19,6 +19,7 @@ if(DOWNLOAD_LATTE) set(LATTE_MD5 "820e73a457ced178c08c71389a385de7" CACHE STRING "MD5 checksum of LATTE tarball") mark_as_advanced(LATTE_URL) mark_as_advanced(LATTE_MD5) + GetFallbackURL(LATTE_URL LATTE_FALLBACK) # CMake cannot pass BLAS or LAPACK library variable to external project if they are a list list(LENGTH BLAS_LIBRARIES} NUM_BLAS) @@ -30,7 +31,7 @@ if(DOWNLOAD_LATTE) include(ExternalProject) ExternalProject_Add(latte_build - URL ${LATTE_URL} + URL ${LATTE_URL} ${LATTE_FALLBACK} URL_MD5 ${LATTE_MD5} SOURCE_SUBDIR cmake CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= ${CMAKE_REQUEST_PIC} -DCMAKE_INSTALL_LIBDIR=lib diff --git a/cmake/Modules/Packages/MDI.cmake b/cmake/Modules/Packages/MDI.cmake index c1e25347af5..67c1b82dabe 100644 --- a/cmake/Modules/Packages/MDI.cmake +++ b/cmake/Modules/Packages/MDI.cmake @@ -12,6 +12,7 @@ if(DOWNLOAD_MDI) set(MDI_MD5 "7a222353ae8e03961d5365e6cd48baee" CACHE STRING "MD5 checksum for MDI tarball") mark_as_advanced(MDI_URL) mark_as_advanced(MDI_MD5) + GetFallbackURL(MDI_URL MDI_FALLBACK) enable_language(C) # only ON/OFF are allowed for "mpi" flag when building MDI library @@ -63,7 +64,7 @@ if(DOWNLOAD_MDI) # support cross-compilation and ninja-build include(ExternalProject) ExternalProject_Add(mdi_build - URL ${MDI_URL} + URL ${MDI_URL} ${MDI_FALLBACK} URL_MD5 ${MDI_MD5} PREFIX ${CMAKE_CURRENT_BINARY_DIR}/mdi_build_ext CMAKE_ARGS diff --git a/cmake/Modules/Packages/ML-HDNNP.cmake b/cmake/Modules/Packages/ML-HDNNP.cmake index f71d08f3ab1..adffcfa77e8 100644 --- a/cmake/Modules/Packages/ML-HDNNP.cmake +++ b/cmake/Modules/Packages/ML-HDNNP.cmake @@ -10,6 +10,7 @@ if(DOWNLOAD_N2P2) set(N2P2_MD5 "9595b066636cd6b90b0fef93398297a5" CACHE STRING "MD5 checksum of N2P2 tarball") mark_as_advanced(N2P2_URL) mark_as_advanced(N2P2_MD5) + GetFallbackURL(N2P2_URL N2P2_FALLBACK) # adjust settings from detected compiler to compiler platform in n2p2 library # set compiler specific flag to turn on C++11 syntax (required on macOS and CentOS 7) @@ -72,7 +73,7 @@ if(DOWNLOAD_N2P2) # download compile n2p2 library. much patch MPI calls in LAMMPS interface to accommodate MPI-2 (e.g. for cross-compiling) include(ExternalProject) ExternalProject_Add(n2p2_build - URL ${N2P2_URL} + URL ${N2P2_URL} ${N2P2_FALLBACK} URL_MD5 ${N2P2_MD5} UPDATE_COMMAND "" CONFIGURE_COMMAND "" diff --git a/cmake/Modules/Packages/ML-PACE.cmake b/cmake/Modules/Packages/ML-PACE.cmake index b64584e5a70..f0e4b49e499 100644 --- a/cmake/Modules/Packages/ML-PACE.cmake +++ b/cmake/Modules/Packages/ML-PACE.cmake @@ -3,9 +3,23 @@ set(PACELIB_URL "https://github.com/ICAMS/lammps-user-pace/archive/refs/tags/v.2 set(PACELIB_MD5 "f418d32b60e531063ac4285bf702b468" CACHE STRING "MD5 checksum of PACE evaluator library tarball") mark_as_advanced(PACELIB_URL) mark_as_advanced(PACELIB_MD5) +GetFallbackURL(PACELIB_URL PACELIB_FALLBACK) # download library sources to build folder -file(DOWNLOAD ${PACELIB_URL} ${CMAKE_BINARY_DIR}/libpace.tar.gz EXPECTED_HASH MD5=${PACELIB_MD5}) #SHOW_PROGRESS +if(EXISTS ${CMAKE_BINARY_DIR}/libpace.tar.gz) + file(MD5 ${CMAKE_BINARY_DIR}/libpace.tar.gz DL_MD5) +endif() +if(NOT "${DL_MD5}" STREQUAL "${PACELIB_MD5}") + message(STATUS "Downloading ${PACELIB_URL}") + file(DOWNLOAD ${PACELIB_URL} ${CMAKE_BINARY_DIR}/libpace.tar.gz STATUS DL_STATUS SHOW_PROGRESS) + file(MD5 ${CMAKE_BINARY_DIR}/libpace.tar.gz DL_MD5) + if((NOT DL_STATUS EQUAL 0) OR (NOT "${DL_MD5}" STREQUAL "${PACELIB_MD5}")) + message(WARNING "Download from primary URL ${PACELIB_URL} failed\nTrying fallback URL ${PACELIB_FALLBACK}") + file(DOWNLOAD ${PACELIB_FALLBACK} ${CMAKE_BINARY_DIR}/libpace.tar.gz EXPECTED_HASH MD5=${PACELIB_MD5} SHOW_PROGRESS) + else() + message(STATUS "Using already downloaded archive ${CMAKE_BINARY_DIR}/libpace.tar.gz") + endif() +endif() # uncompress downloaded sources execute_process( diff --git a/cmake/Modules/Packages/PLUMED.cmake b/cmake/Modules/Packages/PLUMED.cmake index e0e4ff0dde6..f231d148bde 100644 --- a/cmake/Modules/Packages/PLUMED.cmake +++ b/cmake/Modules/Packages/PLUMED.cmake @@ -59,10 +59,11 @@ if(DOWNLOAD_PLUMED) mark_as_advanced(PLUMED_URL) mark_as_advanced(PLUMED_MD5) + GetFallbackURL(PLUMED_URL PLUMED_FALLBACK) include(ExternalProject) ExternalProject_Add(plumed_build - URL ${PLUMED_URL} + URL ${PLUMED_URL} ${PLUMED_FALLBACK} URL_MD5 ${PLUMED_MD5} BUILD_IN_SOURCE 1 CONFIGURE_COMMAND /configure --prefix= diff --git a/cmake/Modules/Packages/SCAFACOS.cmake b/cmake/Modules/Packages/SCAFACOS.cmake index de611a1edbf..9a5580163f3 100644 --- a/cmake/Modules/Packages/SCAFACOS.cmake +++ b/cmake/Modules/Packages/SCAFACOS.cmake @@ -18,6 +18,8 @@ if(DOWNLOAD_SCAFACOS) set(SCAFACOS_MD5 "bd46d74e3296bd8a444d731bb10c1738" CACHE STRING "MD5 checksum of SCAFACOS tarball") mark_as_advanced(SCAFACOS_URL) mark_as_advanced(SCAFACOS_MD5) + GetFallbackURL(SCAFACOS_URL SCAFACOS_FALLBACK) + # version 1.0.1 needs a patch to compile and linke cleanly with GCC 10 and later. file(DOWNLOAD ${LAMMPS_THIRDPARTY_URL}/scafacos-1.0.1-fix.diff ${CMAKE_CURRENT_BINARY_DIR}/scafacos-1.0.1.fix.diff @@ -30,7 +32,7 @@ if(DOWNLOAD_SCAFACOS) include(ExternalProject) ExternalProject_Add(scafacos_build - URL ${SCAFACOS_URL} + URL ${SCAFACOS_URL} ${SCAFACOS_FALLBACK} URL_MD5 ${SCAFACOS_MD5} PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_BINARY_DIR}/scafacos-1.0.1.fix.diff CONFIGURE_COMMAND /configure --prefix= --disable-doc From 89b37c51dffb7dadffcf2f4d16ac6e58e27cd673 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Wed, 1 Feb 2023 06:47:25 -0500 Subject: [PATCH 2/2] implement download fallback for traditional make build --- lib/hdnnp/.gitignore | 3 ++ lib/hdnnp/Install.py | 17 ++++++++---- lib/install_helpers.py | 24 ++++++++++------ lib/latte/Install.py | 16 ++++++++--- lib/mdi/Install.py | 61 ++++++++--------------------------------- lib/mscg/Install.py | 16 +++++++++-- lib/pace/Install.py | 15 +++++++--- lib/plumed/Install.py | 16 ++++++++--- lib/scafacos/Install.py | 21 +++++++++----- 9 files changed, 105 insertions(+), 84 deletions(-) create mode 100644 lib/hdnnp/.gitignore diff --git a/lib/hdnnp/.gitignore b/lib/hdnnp/.gitignore new file mode 100644 index 00000000000..2a6742ed7ad --- /dev/null +++ b/lib/hdnnp/.gitignore @@ -0,0 +1,3 @@ +/includelink +/liblink +/n2p2-* diff --git a/lib/hdnnp/Install.py b/lib/hdnnp/Install.py index 3b232f400b0..45c14a5affb 100644 --- a/lib/hdnnp/Install.py +++ b/lib/hdnnp/Install.py @@ -10,7 +10,7 @@ from argparse import ArgumentParser sys.path.append('..') -from install_helpers import get_cpus, fullpath, geturl, checkmd5sum +from install_helpers import get_cpus, fullpath, geturl, checkmd5sum, getfallback parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script") @@ -76,14 +76,21 @@ if buildflag: url = "https://github.com/CompPhysVienna/n2p2/archive/v%s.tar.gz" % (version) - filename = "n2p2-%s.tar.gz" %version - print("Downloading n2p2 ...") - geturl(url, filename) + filename = "n2p2-%s.tar.gz" % version + fallback = getfallback('n2p2', url) + print("Downloading n2p2 from", url) + try: + geturl(url, filename) + except: + geturl(fallback, filename) # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: if not checkmd5sum(checksums[version], filename): - sys.exit("Checksum for n2p2 library does not match") + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, filename) + if not checkmd5sum(checksums[version], filename): + sys.exit("Checksum for n2p2 library does not match for fallback, too.") print("Unpacking n2p2 source tarball ...") if os.path.exists("%s/n2p2-%s" % (homepath, version)): diff --git a/lib/install_helpers.py b/lib/install_helpers.py index e1ebd1cde75..59171ee4e07 100644 --- a/lib/install_helpers.py +++ b/lib/install_helpers.py @@ -1,4 +1,7 @@ -import hashlib,os,subprocess +import hashlib +import os +import re +import subprocess # try to auto-detect the maximum number of available CPUs def get_cpus(): @@ -32,31 +35,31 @@ def is_exe(fpath): return None -def geturl(url,fname): +def geturl(url, fname): success = False if which('curl') != None: - cmd = 'curl -L -o "%s" %s' % (fname,url) + cmd = 'curl -L -o "%s" %s' % (fname, url) try: - subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) + subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) success = True except subprocess.CalledProcessError as e: print("Calling curl failed with: %s" % e.output.decode('UTF-8')) if not success and which('wget') != None: - cmd = 'wget -O "%s" %s' % (fname,url) + cmd = 'wget -O "%s" %s' % (fname, url) try: - subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) + subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) success = True except subprocess.CalledProcessError as e: print("Calling wget failed with: %s" % e.output.decode('UTF-8')) if not success: - error("Failed to download source code with 'curl' or 'wget'") + raise Exception("Failed to download source code with 'curl' or 'wget' from " + url) return -def checkmd5sum(md5sum,fname): - with open(fname,'rb') as fh: +def checkmd5sum(md5sum, fname): + with open(fname, 'rb') as fh: m = hashlib.md5() while True: data = fh.read(81920) @@ -66,3 +69,6 @@ def checkmd5sum(md5sum,fname): fh.close() return m.hexdigest() == md5sum +def getfallback(lib, url): + archive = re.sub(r'^https://.*/([^/]+gz)', r'-\1', url) + return 'https://download.lammps.org/thirdparty/' + lib + archive diff --git a/lib/latte/Install.py b/lib/latte/Install.py index 2e8f9bee8dd..4e3b27859cb 100644 --- a/lib/latte/Install.py +++ b/lib/latte/Install.py @@ -10,7 +10,7 @@ from argparse import ArgumentParser sys.path.append('..') -from install_helpers import fullpath, geturl, checkmd5sum +from install_helpers import fullpath, geturl, checkmd5sum, getfallback parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script") @@ -86,17 +86,25 @@ url = "https://github.com/lanl/LATTE/archive/v%s.tar.gz" % version lattepath = fullpath(homepath) lattedir = os.path.join(lattepath, homedir) + fallback = getfallback('latte', url) + filename = 'LATTE.tar.gz' # download and unpack LATTE tarball if buildflag: print("Downloading LATTE ...") - geturl(url, "LATTE.tar.gz") + try: + geturl(url, filename) + except: + geturl(fallback, filename) # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: - if not checkmd5sum(checksums[version], 'LATTE.tar.gz'): - sys.exit("Checksum for LATTE library does not match") + if not checkmd5sum(checksums[version], filename): + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, filename) + if not checkmd5sum(checksums[version], filename): + sys.exit("Checksum for LATTE library does not match for fallback, too.") print("Unpacking LATTE ...") if os.path.exists(lattedir): diff --git a/lib/mdi/Install.py b/lib/mdi/Install.py index 59d218b1f9b..77d13d3e8b5 100644 --- a/lib/mdi/Install.py +++ b/lib/mdi/Install.py @@ -9,7 +9,7 @@ import glob sys.path.append('..') -from install_helpers import checkmd5sum +from install_helpers import fullpath, geturl, checkmd5sum, getfallback # help message @@ -51,49 +51,6 @@ def error(str=None): # expand to full path name # process leading '~' or relative path -def fullpath(path): - return os.path.abspath(os.path.expanduser(path)) - -def which(program): - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - for path in os.environ["PATH"].split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - - return None - -def geturl(url,fname): - success = False - - if which('curl') != None: - cmd = 'curl -L -o "%s" %s' % (fname,url) - try: - subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) - success = True - except subprocess.CalledProcessError as e: - print("Calling curl failed with: %s" % e.output.decode('UTF-8')) - - if not success and which('wget') != None: - cmd = 'wget -O "%s" %s' % (fname,url) - try: - subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) - success = True - except subprocess.CalledProcessError as e: - print("Calling wget failed with: %s" % e.output.decode('UTF-8')) - - if not success: - error("Failed to download source code with 'curl' or 'wget'") - return - # parse args args = sys.argv[1:] @@ -123,17 +80,24 @@ def geturl(url,fname): # download and unpack MDI_Library tarball -homepath = "." +homepath = fullpath('.') homedir = "%s/MDI_Library" % homepath print("Downloading MDI_Library ...") -mditar = "%s/v%s.tar.gz" % (homepath,version) -geturl(url, mditar) +mditar = "%s/v%s.tar.gz" % (homepath, version) +fallback = getfallback('mdi', url) +try: + geturl(url, mditar) +except: + geturl(fallback, mditar) # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: if not checkmd5sum(checksums[version], mditar): - sys.exit("Checksum for MDI library does not match") + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, mditar) + if not checkmd5sum(checksums[version], mditar): + sys.exit("Checksum for MDI library does not match") print("Unpacking MDI_Library tarball ...") if os.path.exists("%s/v%s" % (homepath,version)): @@ -199,7 +163,6 @@ def geturl(url,fname): makefile_lammps.write(str(rpath_option) + "\n") makefile_lammps.close() - shared_files = glob.glob( os.path.join( homepath, "liblink", "lib%s.a" % lib) ) if len(shared_files) > 0: print("Build was successful") diff --git a/lib/mscg/Install.py b/lib/mscg/Install.py index 9d18b7c083d..df17005dcab 100644 --- a/lib/mscg/Install.py +++ b/lib/mscg/Install.py @@ -10,7 +10,7 @@ from argparse import ArgumentParser sys.path.append('..') -from install_helpers import fullpath, geturl +from install_helpers import fullpath, geturl, checkmd5sum, getfallback parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script") @@ -84,7 +84,19 @@ if buildflag: print("Downloading MS-CG ...") tarname = os.path.join(homepath, tarname) - geturl(url, tarname) + fallback = getfallback('mscg', url) + try: + geturl(url, tarname) + except: + geturl(fallback, tarname) + + # verify downloaded archive integrity via md5 checksum, if known. + if mscgver in checksums: + if not checkmd5sum(checksums[mscgver], tarname): + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, tarname) + if not checkmd5sum(checksums[mscgver], tarname): + sys.exit("Checksum for LATTE library does not match for fallback, too.") print("Unpacking MS-CG tarfile ...") diff --git a/lib/pace/Install.py b/lib/pace/Install.py index f7f50d4f6d0..21cadbb20a2 100644 --- a/lib/pace/Install.py +++ b/lib/pace/Install.py @@ -13,7 +13,7 @@ from argparse import ArgumentParser sys.path.append('..') -from install_helpers import fullpath, geturl, checkmd5sum +from install_helpers import fullpath, geturl, checkmd5sum, getfallback # settings @@ -77,14 +77,21 @@ print("Downloading pace tarball ...") archive_filename = "%s.%s" % (version, archive_extension) download_filename = "%s/%s" % (thisdir, archive_filename) + fallback = getfallback('pacelib', url) print("Downloading from ", url, " to ", download_filename, end=" ") - geturl(url, download_filename) + try: + geturl(url, download_filename) + except: + geturl(fallback, download_filename) print(" done") # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: - if not checkmd5sum(checksums[version], archive_filename): - sys.exit("Checksum for pace library does not match") + if not checkmd5sum(checksums[version], download_filename): + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, download_filename) + if not checkmd5sum(checksums[version], download_filename): + sys.exit("Checksum for pace library does not match for fallback, too.") print("Unpacking pace tarball ...") src_folder = thisdir + "/src" diff --git a/lib/plumed/Install.py b/lib/plumed/Install.py index bb3a158bd40..bb6f27fddbf 100644 --- a/lib/plumed/Install.py +++ b/lib/plumed/Install.py @@ -10,7 +10,7 @@ from argparse import ArgumentParser sys.path.append('..') -from install_helpers import get_cpus, fullpath, geturl, checkmd5sum +from install_helpers import get_cpus, fullpath, geturl, checkmd5sum, getfallback parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script") @@ -86,6 +86,7 @@ pathflag = args.path is not None plumedpath = args.path mode = args.mode +version = args.version homepath = fullpath('.') homedir = "%s/plumed2" % (homepath) @@ -101,14 +102,21 @@ if buildflag: url = "https://github.com/plumed/plumed2/releases/download/v%s/plumed-src-%s.tgz" % (version, version) - filename = "plumed-src-%s.tar.gz" %version + filename = "plumed-src-%s.tar.gz" % version + fallback = getfallback('plumed', url) print("Downloading plumed ...") - geturl(url, filename) + try: + geturl(url, filename) + except: + geturl(fallback, filename) # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: if not checkmd5sum(checksums[version], filename): - sys.exit("Checksum for plumed2 library does not match") + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, filename) + if not checkmd5sum(checksums[version], filename): + sys.exit("Checksum for plumed2 library does not match for fallback, too.") print("Unpacking plumed2 source tarball ...") if os.path.exists("%s/plumed-%s" % (homepath, version)): diff --git a/lib/scafacos/Install.py b/lib/scafacos/Install.py index 8044215d478..28bcf430863 100644 --- a/lib/scafacos/Install.py +++ b/lib/scafacos/Install.py @@ -10,15 +10,13 @@ from argparse import ArgumentParser sys.path.append('..') -from install_helpers import fullpath, geturl, get_cpus, checkmd5sum +from install_helpers import fullpath, geturl, get_cpus, checkmd5sum, getfallback -parser = ArgumentParser(prog='Install.py', - description="LAMMPS library build wrapper script") +parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script") # settings version = "1.0.1" -url = "https://github.com/scafacos/scafacos/releases/download/v%s/scafacos-%s.tar.gz" % (version, version) # known checksums for different ScaFaCoS versions. used to validate the download. checksums = { \ @@ -59,6 +57,7 @@ buildflag = args.build pathflag = args.path is not None version = args.version +url = "https://github.com/scafacos/scafacos/releases/download/v%s/scafacos-%s.tar.gz" % (version, version) homepath = fullpath(".") scafacospath = os.path.join(homepath, "scafacos-%s" % version) @@ -76,12 +75,20 @@ if buildflag: print("Downloading ScaFaCoS ...") - geturl(url, "%s/scafacos-%s.tar.gz" % (homepath, version)) + filename = "%s/scafacos-%s.tar.gz" % (homepath, version) + fallback = getfallback('scafacos', url) + try: + geturl(url, filename) + except: + geturl(fallback, filename) # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: - if not checkmd5sum(checksums[version], '%s/scafacos-%s.tar.gz' % (homepath, version)): - sys.exit("Checksum for ScaFaCoS library does not match") + if not checkmd5sum(checksums[version], filename): + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, filename) + if not checkmd5sum(checksums[version], filename): + sys.exit("Checksum for ScaFaCoS library does not match for fallback, too.") print("Unpacking ScaFaCoS tarball ...") if os.path.exists(scafacospath):