Dockerfile: manylinux1

Prepare the Dockerfile for manylinux1 builds.

.dockerignore

@@ -1,3 +1,5 @@
+Dockerfile
+
 # Created by .ignore support plugin (hsz.mobi)
 ### Vim template
 # Swap

Dockerfile

@@ -1,86 +1,150 @@
-FROM       ubuntu:16.04
-MAINTAINER Fabian Koller <[email protected]>
+# manylinux1 wheels
+# https://github.com/pypa/manylinux
+#
+# FROM       quay.io/pypa/manylinux2010_x86_64 as build-env
+FROM       quay.io/pypa/manylinux1_x86_64 as build-env
+MAINTAINER Axel Huebl <[email protected]>
+ENV        DEBIAN_FRONTEND noninteractive
+
+# Python 3.5-3.7 via "35 36 37"
+ARG        PY_VERSIONS="35 36 37"
+
+# static libs need relocatable symbols for linking to shared python lib
+ENV        CFLAGS="-fPIC ${CFLAGS}"
+ENV        CXXFLAGS="-fPIC ${CXXFLAGS}"
+
+# install dependencies
+#   CMake, zlib?, HDF5, c-blosc, ADIOS1, ADIOS2?
+RUN        yum check-update -y \
+           && yum -y install \
+               glibc-static \
+               tar
+#RUN        curl -sOL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.tar.gz \
+#           && file cmake*.tar.gz \
+#           && tar -xzf cmake*.tar.gz \
+#           && rm cmake*.tar.gz \
+#           && mv cmake* /opt/cmake
+#ENV        PATH=/opt/cmake/bin:${PATH}
+RUN        for PY_TARGET in ${PY_VERSIONS}; do \
+               PY_BIN=/opt/python/cp${PY_TARGET}-cp${PY_TARGET}m/bin/python \
+               && ${PY_BIN} -m pip install setuptools cmake; \
+           done;
+
+RUN        curl -sLo hdf5-1.10.5.tar.gz https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.5/src/hdf5-1.10.5.tar.gz \
+           && file hdf5*.tar.gz \
+           && tar -xzf hdf5*.tar.gz \
+           && rm hdf5*.tar.gz \
+           && cd hdf5-* \
+           && ./configure --disable-parallel --disable-shared --enable-static --prefix /usr \
+           && make \
+           && make install
+
+# avoid picking up a static libpthread in adios (also: those libs lack -fPIC)
+RUN        rm /usr/lib64/libpthread.a /usr/lib64/libm.a /usr/lib64/librt.a
+
+RUN        curl -sLo c-blosc-1.15.0.tar.gz https://github.com/Blosc/c-blosc/archive/v1.15.0.tar.gz \
+           && file c-blosc*.tar.gz \
+           && tar -xzf c-blosc*.tar.gz \
+           && rm c-blosc*.tar.gz \
+           && mkdir c-blosc-build \
+           && cd c-blosc-build \
+           && PY_TARGET=${PY_VERSIONS:0:2} \
+           && PY_BIN=/opt/python/cp${PY_TARGET}-cp${PY_TARGET}m/bin/python \
+           && CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" \
+           && PATH=${CMAKE_BIN}:${PATH} cmake -DBUILD_SHARED=OFF -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DCMAKE_INSTALL_PREFIX=/usr ../c-blosc-1.15.0 \
+           && make \
+           && make install
+
+# TODO: avoid shared wrapper libs for static-only ADIOS1 linking
+# FIXME: libpthread.a is not compiled with -fPIC on this system, also SHOULD better be linked shared!!
+#          snappy linker flag forwarding bug in c-blosc
+ENV        CFLAGS_OLD=${CFLAGS}
+ENV        CFLAGS="-lstdc++ ${CFLAGS}"
+RUN        curl -sLo adios-1.13.1.tar.gz http://users.nccs.gov/~pnorbert/adios-1.13.1.tar.gz \
+           && file adios*.tar.gz \
+           && tar -xzf adios*.tar.gz \
+           && rm adios*.tar.gz \
+           && cd adios-* \
+           && ./configure --enable-static --disable-shared --disable-fortran --without-mpi --prefix=/usr --with-blosc=/usr \
+           && make \
+           && make install
+ENV        CFLAGS=${CFLAGS_OLD}
+
+ADD        . /opt/src
 
-RUN        useradd test
+RUN        ls /opt/python/
 
-ENV        HOME /home/test
-ADD        . $HOME/src/openPMD-api
-ENV        CACHE $HOME/src/openPMD-api/.cache
+ENV        HDF5_USE_STATIC_LIBRARIES=ON \
+           ADIOS_USE_STATIC_LIBS=ON \
+           BUILD_SHARED_LIBS=OFF \
+           BUILD_TESTING=OFF \
+           BUILD_EXAMPLES=OFF
+
+# build matrix
+RUN        cd /opt/src; \
+           for PY_TARGET in ${PY_VERSIONS}; do \
+               PY_BIN=/opt/python/cp${PY_TARGET}-cp${PY_TARGET}m/bin/python \
+               && CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" \
+               && PATH=${CMAKE_BIN}:${PATH} ${PY_BIN} setup.py bdist_wheel \
+               && ${PY_BIN} setup.py clean --all \
+               && rm -rf openPMD_api.egg-info/; \
+           done; \
+           ls dist/
+
+#RUN        mkdir build \
+#           && cd build \
+#           && /opt/cmake/bin/cmake \
+#               -DPYTHON_EXECUTABLE=$(which /opt/python/cp${PY_TARGET}-cp${PY_TARGET}m/bin/python) \
+#               -DHDF5_USE_STATIC_LIBRARIES=ON \
+#               -DBUILD_SHARED_LIBS=OFF \
+#               -DBUILD_TESTING=OFF \
+#               -DBUILD_EXAMPLES=OFF \
+#               /opt/src \
+#           && make
+
+
+# verify wheel
+# https://github.com/pypa/auditwheel
+#RUN        pip install auditwheel  # already installed
+RUN         for whl in /opt/src/dist/*.whl; do \
+                auditwheel show ${whl}; \
+            done \
+            && du -hs /opt/src/dist/*
+
+# test in fresh env: Debian:Buster + Python 3.7
+FROM       debian:buster
 ENV        DEBIAN_FRONTEND noninteractive
+COPY --from=build-env /opt/src/dist/openPMD_api-*-cp37-cp37m-linux_x86_64.whl .
+RUN        apt-get update \
+           && apt-get install -y --no-install-recommends python3 python3-pip \
+           && rm -rf /var/lib/apt/lists/*
+RUN        python3 --version \
+           && python3 -m pip install openPMD_api-*-cp37-cp37m-linux_x86_64.whl
+RUN        python3 -c "import openpmd_api as api; print(api.__version__); print(api.variants)"
+#RUN        echo "* soft core 100000" >> /etc/security/limits.conf && \
+#           python3 -c "import openpmd_api as api;"; \
+#           gdb -ex bt -c core
 
+# test in fresh env: Ubuntu:18.04 + Python 3.6
+FROM       ubuntu:18.04
+ENV        DEBIAN_FRONTEND noninteractive
+COPY --from=build-env /opt/src/dist/openPMD_api-*-cp36-cp36m-linux_x86_64.whl .
+RUN        apt-get update \
+           && apt-get install -y --no-install-recommends python3 python3-pip \
+           && rm -rf /var/lib/apt/lists/*
+RUN        python3 --version \
+           && python3 -m pip install openPMD_api-*-cp36-cp36m-linux_x86_64.whl
+RUN        python3 -c "import openpmd_api as api; print(api.__version__); print(api.variants)"
 
-# install dependencies provided by packages
-RUN        apt-get -qq update \
-           && apt-get install -y -qq --no-install-recommends \
-              autoconf \
-              build-essential \
-              ca-certificates \
-              clang-4.0 \
-              gcc-5 \
-              git \
-              libopenmpi-dev \
-              make \
-              openmpi-bin \
-              pkg-config \
-              python \
-              ssh \
-              wget \
+# test in fresh env: Debian:Stretch + Python 3.5
+FROM       debian:stretch
+ENV        DEBIAN_FRONTEND noninteractive
+COPY --from=build-env /opt/src/dist/openPMD_api-*-cp35-cp35m-linux_x86_64.whl .
+RUN        apt-get update \
+           && apt-get install -y --no-install-recommends python3 python3-pip \
            && rm -rf /var/lib/apt/lists/*
+RUN        python3 --version \
+           && python3 -m pip install openPMD_api-*-cp35-cp35m-linux_x86_64.whl
+RUN        python3 -c "import openpmd_api as api; print(api.__version__); print(api.variants)"
 
-RUN         cd $HOME/src/openPMD-api/ \
-            && chmod +x scripts/travis/cache_ci_dependencies.sh \
-            && scripts/travis/cache_ci_dependencies.sh
-
-# obtain sample data
-RUN        if [ ! -d $HOME/src/openPMD-api/samples/git-sample/ ]; then \
-               mkdir -p $HOME/src/openPMD-api/samples/git-sample/; \
-               cd $HOME/src/openPMD-api/samples/git-sample/; \
-               wget -nv https://github.com/openPMD/openPMD-example-datasets/raw/draft/example-3d.tar.gz; \
-               tar -xf example-3d.tar.gz; \
-               cp example-3d/hdf5/* $HOME/src/openPMD-api/samples/git-sample/; \
-               chmod 777 $HOME/src/openPMD-api/samples/; \
-               rm -rf example-3d.* example-3d; \
-           fi
-
-ARG CMAKE_VERSION
-ARG CMAKE_PATCH
-# install CMake (can be deprecated when CMake >=3.11 is default)
-RUN         cd $CACHE \
-            && sh cmake-${CMAKE_VERSION}.${CMAKE_PATCH}-Linux-x86_64.sh --prefix=/usr --exclude-subdir
-
-ARG BOOST_MINOR
-ARG BOOST_PATCH
-# install Boost
-RUN         cd $CACHE/boost_1_${BOOST_MINOR}_${BOOST_PATCH} \
-            && ./b2 install -d0 -j4
-
-ARG HDF5_MINOR
-ARG HDF5_PATCH
-# install HDF5
-RUN        cd $CACHE/hdf5-1.${HDF5_MINOR}.${HDF5_PATCH} \
-           && make install -j4
-
-ARG ADIOS1_MINOR
-ARG ADIOS1_PATCH
-# install ADIOS
-RUN        cd $CACHE/adios-1.${ADIOS1_MINOR}.${ADIOS1_PATCH} \
-           && make install -j4
-
-ARG ADIOS2_MINOR
-ARG ADIOS2_PATCH
-# install ADIOS2
-RUN        cd $CACHE/ADIOS2/v2.${ADIOS2_MINOR}.${ADIOS2_PATCH} \
-           && make install -j4
-
-# build
-RUN        cd $HOME/src/openPMD-api \
-           && rm -rf CMakeCache.txt CMakeFiles/ cmake_install.cmake Makefile \
-           && mkdir -p build \
-           && cd build \
-           && rm -rf ../build/* \
-           && cmake $HOME/src/openPMD-api \
-           && make -j4
-
-# run tests
-RUN        chown test -R $HOME/src/openPMD-api/build \
-           && runuser -l test -c 'cd $HOME/src/openPMD-api/build && CTEST_OUTPUT_ON_FAILURE=1 make test'
+# copy wheels to mounted directory

setup.py

@@ -50,8 +50,12 @@ def build_extension(self, ext):
             # variants
             '-DopenPMD_USE_MPI:BOOL=' + openPMD_USE_MPI,
             # skip building tests & examples
-            '-DBUILD_TESTING:BOOL=OFF',
-            '-DBUILD_EXAMPLES:BOOL=OFF',
+            '-DBUILD_TESTING:BOOL=' + BUILD_TESTING,
+            '-DBUILD_EXAMPLES:BOOL=' + BUILD_EXAMPLES,
+            # static/shared libs
+            '-DBUILD_SHARED_LIBS:BOOL=' + BUILD_SHARED_LIBS,
+            '-DHDF5_USE_STATIC_LIBRARIES:BOOL=' + HDF5_USE_STATIC_LIBRARIES,
+            '-DADIOS_USE_STATIC_LIBS:BOOL=' + ADIOS_USE_STATIC_LIBS,
             # Unix: rpath to current dir when packaged
             '-DCMAKE_INSTALL_RPATH=$ORIGIN',
             '-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON',
@@ -99,8 +103,13 @@ def build_extension(self, ext):
 
 # Allow to control options via environment vars.
 # Work-around for https://github.com/pypa/setuptools/issues/1712
-# note: changed default for MPI
+# note: changed default for MPI, TESTING and EXAMPLES
 openPMD_USE_MPI = os.environ.get('openPMD_USE_MPI', 'OFF')
+HDF5_USE_STATIC_LIBRARIES = os.environ.get('HDF5_USE_STATIC_LIBRARIES', 'OFF')
+ADIOS_USE_STATIC_LIBS = os.environ.get('ADIOS_USE_STATIC_LIBS', 'OFF')
+BUILD_SHARED_LIBS = os.environ.get('BUILD_SHARED_LIBS', 'ON')
+BUILD_TESTING = os.environ.get('BUILD_TESTING', 'OFF')
+BUILD_EXAMPLES = os.environ.get('BUILD_EXAMPLES', 'OFF')
 
 # https://cmake.org/cmake/help/v3.0/command/if.html
 if openPMD_USE_MPI.upper() in ['1', 'ON', 'YES', 'TRUE', 'YES']:

share/openPMD/cmake/FindADIOS.cmake

@@ -229,6 +229,15 @@ if(ADIOS_FOUND)
             find_library(_LIB_DIR NAMES ${_LIB} PATHS ${ADIOS_LIBRARY_DIRS})
         endif()
 
+        # pthread should not be linked statically, allow fallback to shared
+        if(NOT _LIB_DIR AND _LIB MATCHES "pthread|m|rt")
+            if(ADIOS_USE_STATIC_LIBS)
+                set(CMAKE_FIND_LIBRARY_SUFFIXES ${_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
+                find_library(_LIB_DIR NAMES ${_LIB} PATHS ${ADIOS_LIBRARY_DIRS})
+                set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_STATIC_LIBRARY_SUFFIX})
+            endif()
+        endif()
+
         if(_LIB MATCHES "^.*nompi.*$")
             set(ADIOS_HAVE_SEQUENTIAL TRUE)
         endif()