From 446a155a4d235eca1ed9c5785188b14072e1ccd0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 10 Jun 2019 16:42:55 +0200 Subject: [PATCH] Dockerfile: manylinux1 Prepare the Dockerfile for manylinux1 builds. --- .dockerignore | 2 + Dockerfile | 222 ++++++++++++++++++---------- setup.py | 15 +- share/openPMD/cmake/FindADIOS.cmake | 9 ++ 4 files changed, 166 insertions(+), 82 deletions(-) diff --git a/.dockerignore b/.dockerignore index 0be892ca51..039cd3f750 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ +Dockerfile + # Created by .ignore support plugin (hsz.mobi) ### Vim template # Swap diff --git a/Dockerfile b/Dockerfile index f16a4a2962..46a66fe304 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,86 +1,150 @@ -FROM ubuntu:16.04 -MAINTAINER Fabian Koller +# manylinux1 wheels +# https://github.com/pypa/manylinux +# +# FROM quay.io/pypa/manylinux2010_x86_64 as build-env +FROM quay.io/pypa/manylinux1_x86_64 as build-env +MAINTAINER Axel Huebl +ENV DEBIAN_FRONTEND noninteractive + +# Python 3.5-3.7 via "35 36 37" +ARG PY_VERSIONS="35 36 37" + +# static libs need relocatable symbols for linking to shared python lib +ENV CFLAGS="-fPIC ${CFLAGS}" +ENV CXXFLAGS="-fPIC ${CXXFLAGS}" + +# install dependencies +# CMake, zlib?, HDF5, c-blosc, ADIOS1, ADIOS2? +RUN yum check-update -y \ + && yum -y install \ + glibc-static \ + tar +#RUN curl -sOL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.tar.gz \ +# && file cmake*.tar.gz \ +# && tar -xzf cmake*.tar.gz \ +# && rm cmake*.tar.gz \ +# && mv cmake* /opt/cmake +#ENV PATH=/opt/cmake/bin:${PATH} +RUN for PY_TARGET in ${PY_VERSIONS}; do \ + PY_BIN=/opt/python/cp${PY_TARGET}-cp${PY_TARGET}m/bin/python \ + && ${PY_BIN} -m pip install setuptools cmake; \ + done; + +RUN curl -sLo hdf5-1.10.5.tar.gz https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.5/src/hdf5-1.10.5.tar.gz \ + && file hdf5*.tar.gz \ + && tar -xzf hdf5*.tar.gz \ + && rm hdf5*.tar.gz \ + && cd hdf5-* \ + && ./configure --disable-parallel --disable-shared --enable-static --prefix /usr \ + && make \ + && make install + +# avoid picking up a static libpthread in adios (also: those libs lack -fPIC) +RUN rm /usr/lib64/libpthread.a /usr/lib64/libm.a /usr/lib64/librt.a + +RUN curl -sLo c-blosc-1.15.0.tar.gz https://github.com/Blosc/c-blosc/archive/v1.15.0.tar.gz \ + && file c-blosc*.tar.gz \ + && tar -xzf c-blosc*.tar.gz \ + && rm c-blosc*.tar.gz \ + && mkdir c-blosc-build \ + && cd c-blosc-build \ + && PY_TARGET=${PY_VERSIONS:0:2} \ + && PY_BIN=/opt/python/cp${PY_TARGET}-cp${PY_TARGET}m/bin/python \ + && CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" \ + && PATH=${CMAKE_BIN}:${PATH} cmake -DBUILD_SHARED=OFF -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DCMAKE_INSTALL_PREFIX=/usr ../c-blosc-1.15.0 \ + && make \ + && make install + +# TODO: avoid shared wrapper libs for static-only ADIOS1 linking +# FIXME: libpthread.a is not compiled with -fPIC on this system, also SHOULD better be linked shared!! +# snappy linker flag forwarding bug in c-blosc +ENV CFLAGS_OLD=${CFLAGS} +ENV CFLAGS="-lstdc++ ${CFLAGS}" +RUN curl -sLo adios-1.13.1.tar.gz http://users.nccs.gov/~pnorbert/adios-1.13.1.tar.gz \ + && file adios*.tar.gz \ + && tar -xzf adios*.tar.gz \ + && rm adios*.tar.gz \ + && cd adios-* \ + && ./configure --enable-static --disable-shared --disable-fortran --without-mpi --prefix=/usr --with-blosc=/usr \ + && make \ + && make install +ENV CFLAGS=${CFLAGS_OLD} + +ADD . /opt/src -RUN useradd test +RUN ls /opt/python/ -ENV HOME /home/test -ADD . $HOME/src/openPMD-api -ENV CACHE $HOME/src/openPMD-api/.cache +ENV HDF5_USE_STATIC_LIBRARIES=ON \ + ADIOS_USE_STATIC_LIBS=ON \ + BUILD_SHARED_LIBS=OFF \ + BUILD_TESTING=OFF \ + BUILD_EXAMPLES=OFF + +# build matrix +RUN cd /opt/src; \ + for PY_TARGET in ${PY_VERSIONS}; do \ + PY_BIN=/opt/python/cp${PY_TARGET}-cp${PY_TARGET}m/bin/python \ + && CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" \ + && PATH=${CMAKE_BIN}:${PATH} ${PY_BIN} setup.py bdist_wheel \ + && ${PY_BIN} setup.py clean --all \ + && rm -rf openPMD_api.egg-info/; \ + done; \ + ls dist/ + +#RUN mkdir build \ +# && cd build \ +# && /opt/cmake/bin/cmake \ +# -DPYTHON_EXECUTABLE=$(which /opt/python/cp${PY_TARGET}-cp${PY_TARGET}m/bin/python) \ +# -DHDF5_USE_STATIC_LIBRARIES=ON \ +# -DBUILD_SHARED_LIBS=OFF \ +# -DBUILD_TESTING=OFF \ +# -DBUILD_EXAMPLES=OFF \ +# /opt/src \ +# && make + + +# verify wheel +# https://github.com/pypa/auditwheel +#RUN pip install auditwheel # already installed +RUN for whl in /opt/src/dist/*.whl; do \ + auditwheel show ${whl}; \ + done \ + && du -hs /opt/src/dist/* + +# test in fresh env: Debian:Buster + Python 3.7 +FROM debian:buster ENV DEBIAN_FRONTEND noninteractive +COPY --from=build-env /opt/src/dist/openPMD_api-*-cp37-cp37m-linux_x86_64.whl . +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-pip \ + && rm -rf /var/lib/apt/lists/* +RUN python3 --version \ + && python3 -m pip install openPMD_api-*-cp37-cp37m-linux_x86_64.whl +RUN python3 -c "import openpmd_api as api; print(api.__version__); print(api.variants)" +#RUN echo "* soft core 100000" >> /etc/security/limits.conf && \ +# python3 -c "import openpmd_api as api;"; \ +# gdb -ex bt -c core +# test in fresh env: Ubuntu:18.04 + Python 3.6 +FROM ubuntu:18.04 +ENV DEBIAN_FRONTEND noninteractive +COPY --from=build-env /opt/src/dist/openPMD_api-*-cp36-cp36m-linux_x86_64.whl . +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-pip \ + && rm -rf /var/lib/apt/lists/* +RUN python3 --version \ + && python3 -m pip install openPMD_api-*-cp36-cp36m-linux_x86_64.whl +RUN python3 -c "import openpmd_api as api; print(api.__version__); print(api.variants)" -# install dependencies provided by packages -RUN apt-get -qq update \ - && apt-get install -y -qq --no-install-recommends \ - autoconf \ - build-essential \ - ca-certificates \ - clang-4.0 \ - gcc-5 \ - git \ - libopenmpi-dev \ - make \ - openmpi-bin \ - pkg-config \ - python \ - ssh \ - wget \ +# test in fresh env: Debian:Stretch + Python 3.5 +FROM debian:stretch +ENV DEBIAN_FRONTEND noninteractive +COPY --from=build-env /opt/src/dist/openPMD_api-*-cp35-cp35m-linux_x86_64.whl . +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-pip \ && rm -rf /var/lib/apt/lists/* +RUN python3 --version \ + && python3 -m pip install openPMD_api-*-cp35-cp35m-linux_x86_64.whl +RUN python3 -c "import openpmd_api as api; print(api.__version__); print(api.variants)" -RUN cd $HOME/src/openPMD-api/ \ - && chmod +x scripts/travis/cache_ci_dependencies.sh \ - && scripts/travis/cache_ci_dependencies.sh - -# obtain sample data -RUN if [ ! -d $HOME/src/openPMD-api/samples/git-sample/ ]; then \ - mkdir -p $HOME/src/openPMD-api/samples/git-sample/; \ - cd $HOME/src/openPMD-api/samples/git-sample/; \ - wget -nv https://github.com/openPMD/openPMD-example-datasets/raw/draft/example-3d.tar.gz; \ - tar -xf example-3d.tar.gz; \ - cp example-3d/hdf5/* $HOME/src/openPMD-api/samples/git-sample/; \ - chmod 777 $HOME/src/openPMD-api/samples/; \ - rm -rf example-3d.* example-3d; \ - fi - -ARG CMAKE_VERSION -ARG CMAKE_PATCH -# install CMake (can be deprecated when CMake >=3.11 is default) -RUN cd $CACHE \ - && sh cmake-${CMAKE_VERSION}.${CMAKE_PATCH}-Linux-x86_64.sh --prefix=/usr --exclude-subdir - -ARG BOOST_MINOR -ARG BOOST_PATCH -# install Boost -RUN cd $CACHE/boost_1_${BOOST_MINOR}_${BOOST_PATCH} \ - && ./b2 install -d0 -j4 - -ARG HDF5_MINOR -ARG HDF5_PATCH -# install HDF5 -RUN cd $CACHE/hdf5-1.${HDF5_MINOR}.${HDF5_PATCH} \ - && make install -j4 - -ARG ADIOS1_MINOR -ARG ADIOS1_PATCH -# install ADIOS -RUN cd $CACHE/adios-1.${ADIOS1_MINOR}.${ADIOS1_PATCH} \ - && make install -j4 - -ARG ADIOS2_MINOR -ARG ADIOS2_PATCH -# install ADIOS2 -RUN cd $CACHE/ADIOS2/v2.${ADIOS2_MINOR}.${ADIOS2_PATCH} \ - && make install -j4 - -# build -RUN cd $HOME/src/openPMD-api \ - && rm -rf CMakeCache.txt CMakeFiles/ cmake_install.cmake Makefile \ - && mkdir -p build \ - && cd build \ - && rm -rf ../build/* \ - && cmake $HOME/src/openPMD-api \ - && make -j4 - -# run tests -RUN chown test -R $HOME/src/openPMD-api/build \ - && runuser -l test -c 'cd $HOME/src/openPMD-api/build && CTEST_OUTPUT_ON_FAILURE=1 make test' +# copy wheels to mounted directory diff --git a/setup.py b/setup.py index 3433e3f244..3f97dee4e9 100644 --- a/setup.py +++ b/setup.py @@ -50,8 +50,12 @@ def build_extension(self, ext): # variants '-DopenPMD_USE_MPI:BOOL=' + openPMD_USE_MPI, # skip building tests & examples - '-DBUILD_TESTING:BOOL=OFF', - '-DBUILD_EXAMPLES:BOOL=OFF', + '-DBUILD_TESTING:BOOL=' + BUILD_TESTING, + '-DBUILD_EXAMPLES:BOOL=' + BUILD_EXAMPLES, + # static/shared libs + '-DBUILD_SHARED_LIBS:BOOL=' + BUILD_SHARED_LIBS, + '-DHDF5_USE_STATIC_LIBRARIES:BOOL=' + HDF5_USE_STATIC_LIBRARIES, + '-DADIOS_USE_STATIC_LIBS:BOOL=' + ADIOS_USE_STATIC_LIBS, # Unix: rpath to current dir when packaged '-DCMAKE_INSTALL_RPATH=$ORIGIN', '-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON', @@ -99,8 +103,13 @@ def build_extension(self, ext): # Allow to control options via environment vars. # Work-around for https://github.com/pypa/setuptools/issues/1712 -# note: changed default for MPI +# note: changed default for MPI, TESTING and EXAMPLES openPMD_USE_MPI = os.environ.get('openPMD_USE_MPI', 'OFF') +HDF5_USE_STATIC_LIBRARIES = os.environ.get('HDF5_USE_STATIC_LIBRARIES', 'OFF') +ADIOS_USE_STATIC_LIBS = os.environ.get('ADIOS_USE_STATIC_LIBS', 'OFF') +BUILD_SHARED_LIBS = os.environ.get('BUILD_SHARED_LIBS', 'ON') +BUILD_TESTING = os.environ.get('BUILD_TESTING', 'OFF') +BUILD_EXAMPLES = os.environ.get('BUILD_EXAMPLES', 'OFF') # https://cmake.org/cmake/help/v3.0/command/if.html if openPMD_USE_MPI.upper() in ['1', 'ON', 'YES', 'TRUE', 'YES']: diff --git a/share/openPMD/cmake/FindADIOS.cmake b/share/openPMD/cmake/FindADIOS.cmake index ea553ef3f1..2020c115a8 100644 --- a/share/openPMD/cmake/FindADIOS.cmake +++ b/share/openPMD/cmake/FindADIOS.cmake @@ -229,6 +229,15 @@ if(ADIOS_FOUND) find_library(_LIB_DIR NAMES ${_LIB} PATHS ${ADIOS_LIBRARY_DIRS}) endif() + # pthread should not be linked statically, allow fallback to shared + if(NOT _LIB_DIR AND _LIB MATCHES "pthread|m|rt") + if(ADIOS_USE_STATIC_LIBS) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) + find_library(_LIB_DIR NAMES ${_LIB} PATHS ${ADIOS_LIBRARY_DIRS}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_STATIC_LIBRARY_SUFFIX}) + endif() + endif() + if(_LIB MATCHES "^.*nompi.*$") set(ADIOS_HAVE_SEQUENTIAL TRUE) endif()