Skip to content

Commit

Permalink
Merge pull request #189 from FloopCZ/v1.15.0
Browse files Browse the repository at this point in the history
Update TensorFlow to v1.15.0
  • Loading branch information
FloopCZ authored Oct 30, 2019
2 parents 88cf2ab + 9075a3b commit 5fefb85
Show file tree
Hide file tree
Showing 12 changed files with 136 additions and 46 deletions.
2 changes: 1 addition & 1 deletion Dockerfiles/BAZEL_VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.21.0
0.26.1
2 changes: 1 addition & 1 deletion Dockerfiles/archlinux-shared
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM archlinux/base
FROM archlinux:latest

# copy the contents of this repository to the container
COPY . tensorflow_cc
Expand Down
4 changes: 2 additions & 2 deletions Dockerfiles/archlinux-shared-cuda
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM archlinux/base
FROM archlinux:latest

# copy the contents of this repository to the container
COPY . tensorflow_cc
Expand All @@ -10,4 +10,4 @@ RUN ./tensorflow_cc/Dockerfiles/install-archlinux.sh --cuda --shared

ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ENV NVIDIA_REQUIRE_CUDA "cuda>=9.2"
ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1"
1 change: 0 additions & 1 deletion Dockerfiles/install-archlinux.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ pacman -Syu --noconfirm --needed \

if $shared; then
pacman -S --noconfirm --needed \
gcc \
java-environment=8 \
libarchive \
protobuf \
Expand Down
2 changes: 1 addition & 1 deletion Dockerfiles/install-ubuntu.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ if $shared; then
fi
if $cuda; then
# install libcupti
apt-get -y install cuda-command-line-tools-10-0
apt-get -y install cuda-command-line-tools-10-1
fi

apt-get -y clean
Expand Down
2 changes: 1 addition & 1 deletion Dockerfiles/ubuntu-shared-cuda
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04
FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04

# copy the contents of this repository to the container
COPY . tensorflow_cc
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# tensorflow_cc
[![Build Status](http://ash.floop.cz:8080/buildStatus/icon?job=tensorflow_cc)](http://ash.floop.cz:8080/job/tensorflow_cc/)
[![TF version](https://img.shields.io/badge/TF%20version-1.13.1-brightgreen.svg)]()
[![TF version](https://img.shields.io/badge/TF%20version-1.15.0-brightgreen.svg)]()

This repository makes possible the usage of the [TensorFlow C++](https://www.tensorflow.org/api_docs/cc/) API from the outside of the TensorFlow source code folders and without the use of the [Bazel](https://bazel.build/) build system.

Expand Down Expand Up @@ -60,7 +60,7 @@ sudo pacman -S gcc7 bazel cuda cudnn nvidia
```

**Warning:** Newer versions of TensorFlow sometimes fail to build with the latest version of Bazel. You may wish
to install an older version of Bazel (e.g., 0.16.1).
to install an older version of Bazel (e.g., 0.26.1).

#### 2) Clone this repository
```
Expand Down
25 changes: 20 additions & 5 deletions tensorflow_cc/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.3 FATAL_ERROR)
project(
"tensorflow_cc"
VERSION 1.13.1
VERSION 1.15.0
)

# Static library with no GPU support is built by default.
Expand All @@ -10,7 +10,7 @@ project(
option(TENSORFLOW_SHARED "Build shared library (required for GPU support)." OFF)
option(ALLOW_CUDA "When building the shared library, try to find and use CUDA." ON)
option(TENSORFLOW_STATIC "Build static library." ON)
set(TENSORFLOW_TAG "v1.13.1" CACHE STRING "The tensorflow release tag to be checked out (default v1.13.1).")
set(TENSORFLOW_TAG "v1.15.0" CACHE STRING "The tensorflow release tag to be checked out (default v1.15.0).")
option(SYSTEM_PROTOBUF "Use system protobuf instead of static protobuf from contrib/makefile." OFF)

# -------------
Expand Down Expand Up @@ -79,7 +79,7 @@ if(TENSORFLOW_SHARED)
)
target_link_libraries(
tensorflow_cc_shared INTERFACE
"${CMAKE_INSTALL_PREFIX}/lib/tensorflow_cc/libtensorflow_cc.so"
"${CMAKE_INSTALL_PREFIX}/lib/libtensorflow_cc.so.${PROJECT_VERSION_MAJOR}"
dl pthread
)
if(SYSTEM_PROTOBUF)
Expand Down Expand Up @@ -204,6 +204,12 @@ install(
PATTERN "*.tar.*" EXCLUDE
PATTERN "*.zip" EXCLUDE
)
# install all the *.inc files
install(
DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tensorflow/tensorflow/contrib/makefile/gen/"
DESTINATION include/tensorflow/tensorflow/contrib/makefile/gen
FILES_MATCHING PATTERN "*.inc"
)
# install all files from third_party folder (e.g., Eigen/Tensor)
install(
DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tensorflow/third_party/"
Expand All @@ -220,14 +226,23 @@ endif()
if(TENSORFLOW_SHARED)
install(
FILES "${CMAKE_CURRENT_BINARY_DIR}/tensorflow/bazel-bin/tensorflow/libtensorflow_cc.so"
DESTINATION lib/tensorflow_cc
DESTINATION lib
RENAME libtensorflow_cc.so.${PROJECT_VERSION_MAJOR}
OPTIONAL
)
install(
FILES "${CMAKE_CURRENT_BINARY_DIR}/tensorflow/bazel-bin/bin/tensorflow/libtensorflow_cc.so"
DESTINATION lib/tensorflow_cc
DESTINATION lib
RENAME libtensorflow_cc.so.${PROJECT_VERSION_MAJOR}
OPTIONAL
)
# Make versioned symlinks to the library
install(CODE "execute_process( \
COMMAND ${CMAKE_COMMAND} -E create_symlink \
libtensorflow_cc.so.${PROJECT_VERSION_MAJOR} \
${CMAKE_INSTALL_PREFIX}/lib/libtensorflow_cc.so \
)"
)
endif()
# static library specific
if(TENSORFLOW_STATIC)
Expand Down
7 changes: 6 additions & 1 deletion tensorflow_cc/cmake/TensorflowShared.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,12 @@ ExternalProject_Add(
BUILD_IN_SOURCE 1
DOWNLOAD_COMMAND ""
UPDATE_COMMAND ""
CONFIGURE_COMMAND tensorflow/contrib/makefile/compile_linux_protobuf.sh
# The grpc integrated in v1.15.0 needs patching, see https://github.com/clearlinux/distribution/issues/1151.
CONFIGURE_COMMAND cp "${CMAKE_CURRENT_SOURCE_DIR}/patches/v1.15.0-Rename-gettid-functions.patch" .
COMMAND git apply v1.15.0-Rename-gettid-functions.patch
COMMAND cp "${CMAKE_CURRENT_SOURCE_DIR}/patches/v1.15.0-workspace.bzl.patch" .
COMMAND git apply v1.15.0-workspace.bzl.patch
COMMAND tensorflow/contrib/makefile/compile_linux_protobuf.sh
# Do not fail on warnings when building nsync
COMMAND sed -i "s/ -Werror//g" tensorflow/contrib/makefile/compile_nsync.sh
COMMAND tensorflow/contrib/makefile/compile_nsync.sh
Expand Down
39 changes: 8 additions & 31 deletions tensorflow_cc/cmake/build_tensorflow.sh.in
Original file line number Diff line number Diff line change
Expand Up @@ -28,48 +28,25 @@ export TF_NEED_KAFKA=${TF_NEED_KAFKA:-0}
export TF_DOWNLOAD_CLANG=${TF_DOWNLOAD_CLANG:-0}
export TF_NEED_IGNITE=${TF_NEED_IGNITE:-0}
export TF_NEED_ROCM=${TF_NEED_ROCM:-0}
export TF_NCCL_VERSION=${TF_NCCL_VERSION:-2.3} # _DEFAULT_NCCL_VERSION from configure.py
export NCCL_INSTALL_PATH=${NCCL_INSTALL_PATH:-/usr}
export PYTHON_BIN_PATH=${PYTHON_BIN_PATH:-"$(which python3)"}
export PYTHON_LIB_PATH="$($PYTHON_BIN_PATH -c 'import site; print(site.getsitepackages()[0])')"

# configure cuda environmental variables

if [ -e /opt/cuda ]; then
echo "Using CUDA from /opt/cuda"
export CUDA_TOOLKIT_PATH=/opt/cuda
elif [ -e /usr/local/cuda ]; then
echo "Using CUDA from /usr/local/cuda"
export CUDA_TOOLKIT_PATH=/usr/local/cuda
fi

if [ -e /opt/cuda/include/cudnn.h ]; then
echo "Using CUDNN from /opt/cuda"
export CUDNN_INSTALL_PATH=/opt/cuda
elif [ -e /usr/local/cuda/include/cudnn.h ]; then
echo "Using CUDNN from /usr/local/cuda"
export CUDNN_INSTALL_PATH=/usr/local/cuda
elif [ -e /usr/include/cudnn.h ]; then
echo "Using CUDNN from /usr"
export CUDNN_INSTALL_PATH=/usr
fi

if [ "@ALLOW_CUDA@" = "ON" ] && [ -n "${CUDA_TOOLKIT_PATH}" ]; then
if [[ -z "${CUDNN_INSTALL_PATH}" ]]; then
echo "CUDA found but no cudnn.h found. Please install cuDNN."
exit 1
fi
# check if cuda support requested and supported
if [ "@ALLOW_CUDA@" = "ON" ] && hash nvcc 2>/dev/null; then
echo "CUDA support enabled"
cuda_config_opts="--config=cuda"
export TF_NEED_CUDA=1
export TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES:-"3.5,7.0"} # default from configure.py
export TF_CUDA_VERSION="$($CUDA_TOOLKIT_PATH/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p')"
export TF_CUDNN_VERSION="$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' $CUDNN_INSTALL_PATH/include/cudnn.h)"
export TF_CUDA_PATHS=${TF_CUDA_PATHS:-"/opt/cuda,/usr/local/cuda,/usr/local,/usr/cuda,/usr"}
export TF_CUDA_VERSION="$(nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p')"
export TF_NCCL_VERSION="$(find / -name 'libnccl.so.*' | tail -n1 | sed -r 's/^.*\.so\.//')"
export TF_CUDNN_VERSION="$(find / -name 'libcudnn.so.*' | tail -n1 | sed -r 's/^.*\.so\.//')"

# choose the right version of CUDA compiler
if [ -z "$GCC_HOST_COMPILER_PATH" ]; then
if hash gcc 2>/dev/null && version_gt 8.5 `gcc -dumpversion`; then
export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc"}
if hash gcc-8 2>/dev/null && version_gt 8.5 `gcc-8 -dumpversion`; then
export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-8"}
elif hash gcc-7 2>/dev/null && version_gt 7.5 `gcc-7 -dumpversion`; then
export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-7"}
elif hash gcc-6 2>/dev/null && version_gt 6.4 `gcc-6 -dumpversion`; then
Expand Down
83 changes: 83 additions & 0 deletions tensorflow_cc/patches/v1.15.0-Rename-gettid-functions.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
new file mode 100644
index 00000000..90bd9115
--- /dev/null
+++ b/third_party/Rename-gettid-functions.patch
@@ -0,0 +1,78 @@
+From d1d017390b799c59d6fdf7b8afa6136d218bdd61 Mon Sep 17 00:00:00 2001
+From: Benjamin Peterson <[email protected]>
+Date: Fri, 3 May 2019 08:11:00 -0700
+Subject: [PATCH] Rename gettid() functions.
+
+glibc 2.30 will declare its own gettid; see https://sourceware.org/git/?p=glibc.git;a=commit;h=1d0fc213824eaa2a8f8c4385daaa698ee8fb7c92. Rename the grpc versions to avoid naming conflicts.
+---
+ src/core/lib/gpr/log_linux.cc | 4 ++--
+ src/core/lib/gpr/log_posix.cc | 4 ++--
+ src/core/lib/iomgr/ev_epollex_linux.cc | 4 ++--
+ 3 files changed, 6 insertions(+), 6 deletions(-)
+
+diff --git a/src/core/lib/gpr/log_linux.cc b/src/core/lib/gpr/log_linux.cc
+index 561276f0c20..8b597b4cf2f 100644
+--- a/src/core/lib/gpr/log_linux.cc
++++ b/src/core/lib/gpr/log_linux.cc
+@@ -40,7 +40,7 @@
+ #include <time.h>
+ #include <unistd.h>
+
+-static long gettid(void) { return syscall(__NR_gettid); }
++static long sys_gettid(void) { return syscall(__NR_gettid); }
+
+ void gpr_log(const char* file, int line, gpr_log_severity severity,
+ const char* format, ...) {
+@@ -70,7 +70,7 @@ void gpr_default_log(gpr_log_func_args* args) {
+ gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME);
+ struct tm tm;
+ static __thread long tid = 0;
+- if (tid == 0) tid = gettid();
++ if (tid == 0) tid = sys_gettid();
+
+ timer = static_cast<time_t>(now.tv_sec);
+ final_slash = strrchr(args->file, '/');
+diff --git a/src/core/lib/gpr/log_posix.cc b/src/core/lib/gpr/log_posix.cc
+index b6edc14ab6b..2f7c6ce3760 100644
+--- a/src/core/lib/gpr/log_posix.cc
++++ b/src/core/lib/gpr/log_posix.cc
+@@ -31,7 +31,7 @@
+ #include <string.h>
+ #include <time.h>
+
+-static intptr_t gettid(void) { return (intptr_t)pthread_self(); }
++static intptr_t sys_gettid(void) { return (intptr_t)pthread_self(); }
+
+ void gpr_log(const char* file, int line, gpr_log_severity severity,
+ const char* format, ...) {
+@@ -86,7 +86,7 @@ void gpr_default_log(gpr_log_func_args* args) {
+ char* prefix;
+ gpr_asprintf(&prefix, "%s%s.%09d %7" PRIdPTR " %s:%d]",
+ gpr_log_severity_string(args->severity), time_buffer,
+- (int)(now.tv_nsec), gettid(), display_file, args->line);
++ (int)(now.tv_nsec), sys_gettid(), display_file, args->line);
+
+ fprintf(stderr, "%-70s %s\n", prefix, args->message);
+ gpr_free(prefix);
+diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc
+index 08116b3ab53..76f59844312 100644
+--- a/src/core/lib/iomgr/ev_epollex_linux.cc
++++ b/src/core/lib/iomgr/ev_epollex_linux.cc
+@@ -1102,7 +1102,7 @@ static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker,
+ }
+
+ #ifndef NDEBUG
+-static long gettid(void) { return syscall(__NR_gettid); }
++static long sys_gettid(void) { return syscall(__NR_gettid); }
+ #endif
+
+ /* pollset->mu lock must be held by the caller before calling this.
+@@ -1122,7 +1122,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset,
+ #define WORKER_PTR (&worker)
+ #endif
+ #ifndef NDEBUG
+- WORKER_PTR->originator = gettid();
++ WORKER_PTR->originator = sys_gettid();
+ #endif
+ if (grpc_polling_trace.enabled()) {
+ gpr_log(GPR_INFO,
11 changes: 11 additions & 0 deletions tensorflow_cc/patches/v1.15.0-workspace.bzl.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
index 55d7eb93..33e86087 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -486,6 +486,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
# WARNING: make sure ncteisen@ and vpai@ are cc-ed on any CL to change the below rule
tf_http_archive(
name = "grpc",
+ patch_file = clean_dep("//third_party:Rename-gettid-functions.patch"),
sha256 = "67a6c26db56f345f7cee846e681db2c23f919eba46dd639b09462d1b6203d28c",
strip_prefix = "grpc-4566c2a29ebec0835643b972eb99f4306c4234a3",
system_build_file = clean_dep("//third_party/systemlibs:grpc.BUILD"),

0 comments on commit 5fefb85

Please sign in to comment.