diff --git a/.github/workflows/ubuntu-sycl.yml b/.github/workflows/ubuntu-sycl.yml index bce4e90f4d7..622c8005ef6 100644 --- a/.github/workflows/ubuntu-sycl.yml +++ b/.github/workflows/ubuntu-sycl.yml @@ -3,6 +3,11 @@ permissions: {} on: workflow_dispatch: + inputs: + developer_build: + description: 'Set to OFF for Release wheels' + required: false + default: 'ON' push: branches: - main @@ -16,11 +21,12 @@ concurrency: env: NPROC: 2 GCE_CLI_GHA_VERSION: "416.0.0" + DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} jobs: ubuntu-sycl: permissions: - contents: read + contents: write # Release upload runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index 411fe6a37bb..684a13a462a 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -109,7 +109,7 @@ jobs: - name: Upload ccache to GCS if: ${{ github.ref == 'refs/heads/main' }} run: | - gsutil cp ${GITHUB_WORKSPACE}/${{ env.CCACHE_TAR_NAME }}.tar.gz gs://open3d-ci-cache/ + gsutil cp ${GITHUB_WORKSPACE}/${{ env.CCACHE_TAR_NAME }}.tar.xz gs://open3d-ci-cache/ - name: Update devel release if: ${{ github.ref == 'refs/heads/main' }} env: diff --git a/3rdparty/find_dependencies.cmake b/3rdparty/find_dependencies.cmake index a782265a553..13f6501c8de 100644 --- a/3rdparty/find_dependencies.cmake +++ b/3rdparty/find_dependencies.cmake @@ -1383,8 +1383,8 @@ if(BUILD_GUI) # Ensure that libstdc++ gets linked first. target_link_libraries(3rdparty_filament INTERFACE -lstdc++ - ${CPP_LIBRARY} ${CPPABI_LIBRARY}) - message(STATUS "Filament C++ libraries: ${CPP_LIBRARY} ${CPPABI_LIBRARY}") + ${CPP_LIBRARY}.1 ${CPPABI_LIBRARY}) + message(STATUS "Filament C++ libraries: ${CPP_LIBRARY}.1 ${CPPABI_LIBRARY}") if (LIBCPP_VERSION GREATER 11) message(WARNING "libc++ (LLVM) version ${LIBCPP_VERSION} > 11 includes libunwind that " "interferes with the system libunwind.so.8 and may crash Python code when exceptions " diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index c15dabae54a..d43f1676806 100755 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -95,7 +95,7 @@ RUN apt-get update && apt-get install -y \ && c++ -v; \ fi \ && rm -rf /var/lib/apt/lists/* -# OneDPL TBB backend requires libstdc++ >= v11. This makes the created wheel UBuntu 22.04+ only. +# OneDPL TBB backend requires libstdc++ >= v11. This makes the created wheel Ubuntu 22.04+ only. # pyenv # The pyenv python paths are used during docker run, in this way docker run @@ -166,7 +166,12 @@ COPY . /root/Open3D WORKDIR /root/Open3D # Open3D C++ dependencies -RUN ./util/install_deps_ubuntu.sh assume-yes +# If BUILD_SYCL_MODULE=ON, do not install clang - this installs libomp5-dev which gets linked into the shared library. +RUN if [[ "$(uname -m)" = "x86_64" ]] ; then \ + ./util/install_deps_ubuntu.sh assume-yes no-filament-deps ; \ + else \ + ./util/install_deps_ubuntu.sh assume-yes ; \ + fi # Open3D Python dependencies RUN source util/ci_utils.sh \ diff --git a/docker/Dockerfile.docs b/docker/Dockerfile.docs new file mode 100755 index 00000000000..5dd9dae51af --- /dev/null +++ b/docker/Dockerfile.docs @@ -0,0 +1,98 @@ +# FROM must be called before other ARGS except for ARG BASE_IMAGE +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +# For bash-specific commands +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Required build args, should be specified in docker_build.sh +ARG DEVELOPER_BUILD +ARG PYTHON_VERSION +ARG CI + +RUN if [[ -z "${DEVELOPER_BUILD}" ]]; then echo "Error: ARG DEVELOPER_BUILD not specified."; exit 1; fi \ + && if [[ -z "${PYTHON_VERSION}" ]]; then echo "Error: ARG PYTHON_VERSION not specified."; exit 1; fi + +# Forward all ARG to ENV +# ci_utils.sh may require these environment variables +ENV DEVELOPER_BUILD=${DEVELOPER_BUILD} +ENV PYTHON_VERSION=${PYTHON_VERSION} + +# Prevent interactive inputs when installing packages +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=America/Los_Angeles +ENV SUDO=command +ENV OPEN3D_ML_ROOT=/root/Open3D-ML + +# Always keep /root/Open3D as the WORKDIR +WORKDIR /root/Open3D +COPY util/ci_utils.sh util/install_deps_ubuntu.sh util/ +COPY python/requirements*.txt python/ +COPY docs/requirements.txt docs/ + +# Dependencies: basic and python-build +RUN apt-get update && apt-get install -y \ + git \ + curl \ + build-essential \ + pkg-config \ + zlib1g \ + zlib1g-dev \ + libssl-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libxml2-dev \ + libxmlsec1-dev \ + libffi-dev \ + liblzma-dev \ + && rm -rf /var/lib/apt/lists/* + +# Checkout Open3D-ML main branch +RUN git clone --depth 1 https://github.com/isl-org/Open3D-ML.git ${OPEN3D_ML_ROOT} +# Open3D docs dependencies +RUN source util/ci_utils.sh && install_docs_dependencies "${OPEN3D_ML_ROOT}" + +# Download ccache from GCS bucket +# If it doesn't exist on the cloud, an empty ${CCACHE_DIR} will be created. +# Example directory structure: +# - CCACHE_DIR = ~/.cache/ccache +# - CCACHE_DIR_NAME = ccache +# - CCACHE_DIR_PARENT = ~/.cache +# We need to set ccache size explicitly with -M, otherwise the defualt size is +# *not* determined by ccache's default, but the downloaded ccache file's config. +RUN CCACHE_DIR=$(ccache -p | grep cache_dir | grep -oE "[^ ]+$") \ + && CCACHE_DIR_NAME=$(basename ${CCACHE_DIR}) \ + && CCACHE_DIR_PARENT=$(dirname ${CCACHE_DIR}) \ + && CCACHE_TAR_NAME="open3d-ci-cpu" \ + && mkdir -p ${CCACHE_DIR_PARENT} \ + && cd ${CCACHE_DIR_PARENT} \ + && (curl -s -O https://storage.googleapis.com/open3d-ci-cache/${CCACHE_TAR_NAME}.tar.xz || true) \ + && if [ -f ${CCACHE_TAR_NAME}.tar.xz ]; then tar -xf ${CCACHE_TAR_NAME}.tar.xz || true; fi \ + && mkdir -p ${CCACHE_DIR} \ + && ccache -M 4G \ + && ccache -s + +# Open3D repo +COPY . /root/Open3D + +# PWD: Open3D/docs after build_docs +# Docs in docs/_out/html +RUN source util/ci_utils.sh && build_docs "$DEVELOPER_BUILD" \ +&& ccache -s \ +tar -C _out -cvzf "/root/Open3D/open3d-$(git rev-parse HEAD)-docs.tar.gz" html + +# Compress ccache folder, move to / directory +RUN ccache -s \ + && CCACHE_DIR=$(ccache -p | grep cache_dir | grep -oE "[^ ]+$") \ + && CCACHE_DIR_NAME=$(basename ${CCACHE_DIR}) \ + && CCACHE_DIR_PARENT=$(dirname ${CCACHE_DIR}) \ + && cd ${CCACHE_DIR_PARENT} \ + && tar -caf /${CCACHE_TAR_NAME}.tar.xz ${CCACHE_DIR_NAME} \ + && mv /root/Open3D/build/package/open3d-devel*.tar.xz /; fi \ + && ls -alh / + +RUN echo "Docker documentation build done." diff --git a/docker/docker_build.sh b/docker/docker_build.sh index 176657c795b..5ff0a1a0942 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -449,7 +449,7 @@ sycl-shared_export_env() { # https://hub.docker.com/r/intel/oneapi-basekit # https://github.com/intel/oneapi-containers/blob/master/images/docker/basekit/Dockerfile.ubuntu-20.04 export BASE_IMAGE=intel/oneapi-basekit:2024.1.0-devel-ubuntu20.04 - export DEVELOPER_BUILD=ON + export DEVELOPER_BUILD=${DEVELOPER_BUILD:-ON} export CCACHE_TAR_NAME=open3d-ci-sycl export PYTHON_VERSION=3.10 export BUILD_SHARED_LIBS=ON diff --git a/docs/sycl.rst b/docs/sycl.rst index e17d085f79f..24342397ec1 100644 --- a/docs/sycl.rst +++ b/docs/sycl.rst @@ -30,8 +30,7 @@ and (optionally) SYCL runtime for your `Nvidia `_ GPU. For Python, the wheels will automatically install the DPC++ runtime package -(`dpcpp-cpp-rt`). You will also need `libomp5` installed: `apt-get install -libomp5-11`. Make sure to have the `correct drivers installed +(`dpcpp-cpp-rt`). Make sure to have the `correct drivers installed `_ for your GPU. For raycasting on Intel GPUs, you will also need the `intel-level-zero-gpu-raytracing` package. diff --git a/util/ci_utils.sh b/util/ci_utils.sh index b6110110d7b..0d798ca9139 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -340,27 +340,29 @@ test_cpp_example() { # Usage: install_docs_dependencies "${OPEN3D_ML_ROOT}" install_docs_dependencies() { echo - echo Install ubuntu dependencies - ./util/install_deps_ubuntu.sh assume-yes - sudo apt-get install --yes cmake - sudo apt-get install --yes libxml2-dev libxslt-dev python3-dev - sudo apt-get install --yes doxygen - sudo apt-get install --yes texlive - sudo apt-get install --yes texlive-latex-extra - sudo apt-get install --yes ghostscript - sudo apt-get install --yes pandoc - sudo apt-get install --yes ccache + echo Install ubuntu dependencies from $(pwd) + util/install_deps_ubuntu.sh assume-yes + $SUDO apt-get install --yes \ + libxml2-dev libxslt-dev \ + python3-dev python-is-python3 python3-pip \ + doxygen \ + texlive \ + texlive-latex-extra \ + ghostscript \ + pandoc \ + ccache echo echo Install Python dependencies for building docs command -v python python -V python -m pip install -U -q "pip==$PIP_VER" - python -m pip install -U -q -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" wheel yapf + which cmake || python -m pip install -U -q cmake + python -m pip install -U -q -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" if [[ -d "$1" ]]; then OPEN3D_ML_ROOT="$1" echo Installing Open3D-ML dependencies from "${OPEN3D_ML_ROOT}" python -m pip install -r "${OPEN3D_ML_ROOT}/requirements.txt" - python -m pip install -r "${OPEN3D_ML_ROOT}/requirements-torch.txt" + python -m pip install -r "${OPEN3D_ML_ROOT}/requirements-torch-cxx11-abi.txt" python -m pip install -r "${OPEN3D_ML_ROOT}/requirements-tensorflow.txt" else echo OPEN3D_ML_ROOT="$OPEN3D_ML_ROOT" not specified or invalid. Skipping ML dependencies. @@ -396,9 +398,8 @@ build_docs() { "-DWITH_OPENMP=ON" "-DBUILD_AZURE_KINECT=ON" "-DBUILD_LIBREALSENSE=ON" - "-DGLIBCXX_USE_CXX11_ABI=OFF" - # TODO: PyTorch still use old CXX ABI, re-enable Tensorflow when PyTorch is updated to use new ABI - "-DBUILD_TENSORFLOW_OPS=OFF" + "-DGLIBCXX_USE_CXX11_ABI=ON" + "-DBUILD_TENSORFLOW_OPS=ON" "-DBUILD_PYTORCH_OPS=ON" "-DBUILD_EXAMPLES=OFF" ) diff --git a/util/install_deps_ubuntu.sh b/util/install_deps_ubuntu.sh index 2786b19c23b..4b3200ca257 100755 --- a/util/install_deps_ubuntu.sh +++ b/util/install_deps_ubuntu.sh @@ -1,13 +1,17 @@ #!/usr/bin/env bash -# Use: install_deps_ubuntu.sh [ assume-yes ] +# Use: install_deps_ubuntu.sh [ assume-yes ] [ no-filament-deps ] set -ev SUDO=${SUDO:=sudo} # SUDO=command in docker (running as root, sudo not available) -if [ "$1" == "assume-yes" ]; then +options="$(echo "$@" | tr ' ' '|')" +APT_CONFIRM="" +if [[ "assume-yes" =~ ^($options)$ ]]; then APT_CONFIRM="--assume-yes" -else - APT_CONFIRM="" +fi +FILAMENT_DEPS="yes" +if [[ "no-filament-deps" =~ ^($options)$ ]]; then + FILAMENT_DEPS="" fi deps=( @@ -17,12 +21,10 @@ deps=( libxcb-shm0 libglu1-mesa-dev python3-dev - # Filament build-from-source - clang + # filament linking libc++-dev libc++abi-dev libsdl2-dev - ninja-build libxi-dev # ML libtbb-dev @@ -34,6 +36,12 @@ deps=( libtool ) +if [[ "$FILAMENT_DEPS" ]]; then # Filament build-from-source + deps+=(clang + ninja-build + ) +fi + eval $( source /etc/lsb-release; echo DISTRIB_ID="$DISTRIB_ID";