Skip to content

Commit

Permalink
[Split Build] Add option to create libtorch wheel and use it to build…
Browse files Browse the repository at this point in the history 
… pytorch as a separate wheel (pytorch#126328)

Creates an option to just build the libtorch portion of pytorch such that we have the necessary .so files.  Then it builds a torch package using the libtorch wheel. These options are enabled using ` BUILD_LIBTORCH_WHL` and `BUILD_PYTHON_ONLY`.

We run

```
 BUILD_LIBTORCH_WHL=1 python setup.py install
python setup.py clean
BUILD_PYTHON_ONLY=1 python setup.py install
```

to produce

```
sahanp@devgpu086 ~/pytorch (detached HEAD|REBASE-i 3/5)> ls /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/torch/lib/                                                                                                                (pytorch-3.10)
libshm.so*  libtorch_global_deps.so*  libtorch_python.so*
sahanp@devgpu086 ~/pytorch (detached HEAD|REBASE-i 3/5)> ldd build/lib/libtorch_python.so                                                                                                                                                                (pytorch-3.10)
        linux-vdso.so.1 (0x00007ffdc2d37000)
        libtorch.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/libtorch/lib/libtorch.so (0x00007f539fe99000)
        libshm.so => /home/sahanp/pytorch/build/lib/libshm.so (0x00007f539fe90000)
        libcudnn.so.8 => /usr/local/cuda-12.1/targets/x86_64-linux/lib/libcudnn.so.8 (0x00007f539e800000)
        libnvToolsExt.so.1 => /usr/local/cuda/lib64/libnvToolsExt.so.1 (0x00007f539e400000)
        libstdc++.so.6 => /lib64/libstdc++.so.6 (0x00007f539e000000)
        libm.so.6 => /lib64/libm.so.6 (0x00007f539fda5000)
        libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00007f539ebe5000)
        libc.so.6 => /lib64/libc.so.6 (0x00007f539dc00000)
        /lib64/ld-linux-x86-64.so.2 (0x00007f539fea0000)
        libtorch_cpu.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/libtorch/lib/libtorch_cpu.so (0x00007f5392400000)
        libtorch_cuda.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/libtorch/lib/libtorch_cuda.so (0x00007f5380000000)
        librt.so.1 => /lib64/librt.so.1 (0x00007f539fd9e000)
        libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f539fd99000)
        libdl.so.2 => /lib64/libdl.so.2 (0x00007f539fd94000)
        libc10.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/libtorch/lib/libc10.so (0x00007f539eb07000)
        libmkl_intel_lp64.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_intel_lp64.so.2 (0x00007f537ec00000)
        libmkl_gnu_thread.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_gnu_thread.so.2 (0x00007f537ce00000)
        libmkl_core.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_core.so.2 (0x00007f5378800000)
        libomp.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/libomp.so (0x00007f539e707000)
        libcupti.so.12 => /usr/local/cuda/lib64/libcupti.so.12 (0x00007f5377e00000)
        libcudart.so.12 => /usr/local/cuda/lib64/libcudart.so.12 (0x00007f5377a00000)
        libc10_cuda.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/libtorch/lib/libc10_cuda.so (0x00007f539ea6a000)
        libcusparse.so.12 => /usr/local/cuda/lib64/libcusparse.so.12 (0x00007f5368400000)
        libcufft.so.11 => /usr/local/cuda/lib64/libcufft.so.11 (0x00007f535ee00000)
        libcusolver.so.11 => /usr/local/cuda/lib64/libcusolver.so.11 (0x00007f534c800000)
        libcurand.so.10 => /usr/local/cuda/lib64/libcurand.so.10 (0x00007f5346200000)
        libcublas.so.12 => /usr/local/cuda/lib64/libcublas.so.12 (0x00007f533f800000)
        libcublasLt.so.12 => /usr/local/cuda/lib64/libcublasLt.so.12 (0x00007f531e800000)
        libutil.so.1 => /lib64/libutil.so.1 (0x00007f539ea63000)
        libnvJitLink.so.12 => /usr/local/cuda/lib64/libnvJitLink.so.12 (0x00007f531b800000)
sahanp@devgpu086 ~/pytorch (detached HEAD|REBASE-i 3/5)> ldd build/lib/libtorch_global_deps.so                                                                                                                                                           (pytorch-3.10)
        linux-vdso.so.1 (0x00007ffc265df000)
        libmkl_intel_lp64.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_intel_lp64.so.2 (0x00007fa93fc00000)
        libmkl_gnu_thread.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_gnu_thread.so.2 (0x00007fa93de00000)
        libmkl_core.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_core.so.2 (0x00007fa939800000)
        libm.so.6 => /lib64/libm.so.6 (0x00007fa940f05000)
        libcudart.so.12 => /usr/local/cuda/lib64/libcudart.so.12 (0x00007fa939400000)
        libnvToolsExt.so.1 => /usr/local/cuda/lib64/libnvToolsExt.so.1 (0x00007fa939000000)
        libgomp.so.1 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libgomp.so.1 (0x00007fa93fb07000)
        libc.so.6 => /lib64/libc.so.6 (0x00007fa938c00000)
        libdl.so.2 => /lib64/libdl.so.2 (0x00007fa940efe000)
        libpthread.so.0 => /lib64/libpthread.so.0 (0x00007fa940ef9000)
        /lib64/ld-linux-x86-64.so.2 (0x00007fa940ff5000)
        librt.so.1 => /lib64/librt.so.1 (0x00007fa940ef2000)
        libstdc++.so.6 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libstdc++.so.6 (0x00007fa93921d000)
        libgcc_s.so.1 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libgcc_s.so.1 (0x00007fa93faec000)
        ```
Pull Request resolved: pytorch#126328
Approved by: https://github.com/atalman
  • Loading branch information
@PaliC @pytorchmergebot
PaliC authored and pytorchmergebot committed May 29, 2024
1 parent 8090145 commit a25b28a
Show file tree
Hide file tree
Showing 14 changed files with 923 additions and 605 deletions.
1,007 changes: 580 additions & 427 deletions CMakeLists.txt
View file

Large diffs are not rendered by default.

227 changes: 119 additions & 108 deletions c10/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,120 +12,128 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
# protobuf header files, because protobuf header files will transitively force
# one to link against a specific protobuf version.

# ---[ Configure macro file.
set(C10_USE_GFLAGS ${USE_GFLAGS}) # used in cmake_macros.h.in
set(C10_USE_GLOG ${USE_GLOG}) # used in cmake_macros.h.in
set(C10_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) # used in cmake_macros.h.in
set(C10_USE_NUMA ${USE_NUMA})
set(C10_USE_MSVC_STATIC_RUNTIME ${CAFFE2_USE_MSVC_STATIC_RUNTIME})
set(C10_USE_ROCM_KERNEL_ASSERT ${USE_ROCM_KERNEL_ASSERT})
configure_file(
${CMAKE_CURRENT_LIST_DIR}/macros/cmake_macros.h.in
${CMAKE_BINARY_DIR}/c10/macros/cmake_macros.h)

# Note: if you want to add ANY dependency to the c10 library, make sure you
# check with the core PyTorch developers as the dependency will be
# transitively passed on to all libraries dependent on PyTorch.
file(GLOB C10_SRCS
*.cpp
core/*.cpp
core/impl/*.cpp
mobile/*.cpp
macros/*.cpp
util/*.cpp
)
file(GLOB C10_HEADERS
*.h
core/*.h
core/impl/*.h
mobile/*.h
macros/*.h
util/*.h
)
add_library(c10 ${C10_SRCS} ${C10_HEADERS})
target_compile_options_if_supported(c10 "-Wdeprecated")
if(HAVE_SOVERSION)
set_target_properties(c10 PROPERTIES
VERSION ${TORCH_VERSION} SOVERSION ${TORCH_SOVERSION})
endif()
# If building shared library, set dllimport/dllexport proper.
target_compile_options(c10 PRIVATE "-DC10_BUILD_MAIN_LIB")
# Enable hidden visibility if compiler supports it.
if(${COMPILER_SUPPORTS_HIDDEN_VISIBILITY})
target_compile_options(c10 PRIVATE "-fvisibility=hidden")
if(BUILD_LIBTORCHLESS)
find_library(C10_LIB c10 PATHS $ENV{LIBTORCH_LIB_PATH} NO_DEFAULT_PATH)
else()
set(C10_LIB c10)
endif()

option(C10_USE_IWYU "Use include-what-you-use to clean up header inclusion" OFF)
if(C10_USE_IWYU)
find_program(iwyu NAMES include-what-you-use)
if(iwyu)
set(iwyu_cmd
"include-what-you-use"
"-Xiwyu"
"--transitive_includes_only"
"-Xiwyu"
"--no_fwd_decls"
"-Xiwyu"
"--prefix_header_includes=keep"
"-Xiwyu"
"--mapping_file=${CMAKE_CURRENT_LIST_DIR}/../tools/iwyu/all.imp"
)
set_property(TARGET c10 PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${iwyu_cmd})
# ---[ Configure macro file.
set(C10_USE_GFLAGS ${USE_GFLAGS}) # used in cmake_macros.h.in
set(C10_USE_GLOG ${USE_GLOG}) # used in cmake_macros.h.in
set(C10_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) # used in cmake_macros.h.in
set(C10_USE_NUMA ${USE_NUMA})
set(C10_USE_MSVC_STATIC_RUNTIME ${CAFFE2_USE_MSVC_STATIC_RUNTIME})
set(C10_USE_ROCM_KERNEL_ASSERT ${USE_ROCM_KERNEL_ASSERT})
configure_file(
${CMAKE_CURRENT_LIST_DIR}/macros/cmake_macros.h.in
${CMAKE_BINARY_DIR}/c10/macros/cmake_macros.h)

# Note: if you want to add ANY dependency to the c10 library, make sure you
# check with the core PyTorch developers as the dependency will be
# transitively passed on to all libraries dependent on PyTorch.
file(GLOB C10_SRCS
*.cpp
core/*.cpp
core/impl/*.cpp
mobile/*.cpp
macros/*.cpp
util/*.cpp
)
file(GLOB C10_HEADERS
*.h
core/*.h
core/impl/*.h
mobile/*.h
macros/*.h
util/*.h
)
if(NOT BUILD_LIBTORCHLESS)
add_library(c10 ${C10_SRCS} ${C10_HEADERS})
target_compile_options_if_supported(c10 "-Wdeprecated")
if(HAVE_SOVERSION)
set_target_properties(c10 PROPERTIES
VERSION ${TORCH_VERSION} SOVERSION ${TORCH_SOVERSION})
endif()
# If building shared library, set dllimport/dllexport proper.
target_compile_options(c10 PRIVATE "-DC10_BUILD_MAIN_LIB")
# Enable hidden visibility if compiler supports it.
if(${COMPILER_SUPPORTS_HIDDEN_VISIBILITY})
target_compile_options(c10 PRIVATE "-fvisibility=hidden")
endif()
endif()

if(WERROR)
target_compile_options_if_supported(c10 PRIVATE "-Werror=sign-compare")
target_compile_options_if_supported(c10 PRIVATE "-Werror=shadow")
endif()
option(C10_USE_IWYU "Use include-what-you-use to clean up header inclusion" OFF)
if(C10_USE_IWYU)
find_program(iwyu NAMES include-what-you-use)
if(iwyu)
set(iwyu_cmd
"include-what-you-use"
"-Xiwyu"
"--transitive_includes_only"
"-Xiwyu"
"--no_fwd_decls"
"-Xiwyu"
"--prefix_header_includes=keep"
"-Xiwyu"
"--mapping_file=${CMAKE_CURRENT_LIST_DIR}/../tools/iwyu/all.imp"
)
set_property(TARGET c10 PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${iwyu_cmd})
endif()
endif()

# ---[ Dependency of c10
if(C10_USE_GFLAGS)
target_link_libraries(c10 PUBLIC gflags)
endif()
if(WERROR)
target_compile_options_if_supported(c10 PRIVATE "-Werror=sign-compare")
target_compile_options_if_supported(c10 PRIVATE "-Werror=shadow")
endif()

if(C10_USE_GLOG)
target_link_libraries(c10 PUBLIC glog::glog)
endif()
target_link_libraries(c10 PRIVATE fmt::fmt-header-only)

if(C10_USE_NUMA)
message(STATUS "NUMA paths:")
message(STATUS ${Numa_INCLUDE_DIR})
message(STATUS ${Numa_LIBRARIES})
target_include_directories(c10 PRIVATE ${Numa_INCLUDE_DIR})
target_link_libraries(c10 PRIVATE ${Numa_LIBRARIES})
else()
message(STATUS "don't use NUMA")
endif()
# ---[ Dependency of c10
if(C10_USE_GFLAGS)
target_link_libraries(c10 PUBLIC gflags)
endif()

if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "s390x" AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
target_link_libraries(c10 PRIVATE cpuinfo)
endif()
if(C10_USE_GLOG)
target_link_libraries(c10 PUBLIC glog::glog)
endif()
target_link_libraries(c10 PRIVATE fmt::fmt-header-only)

if(C10_USE_NUMA)
message(STATUS "NUMA paths:")
message(STATUS ${Numa_INCLUDE_DIR})
message(STATUS ${Numa_LIBRARIES})
target_include_directories(c10 PRIVATE ${Numa_INCLUDE_DIR})
target_link_libraries(c10 PRIVATE ${Numa_LIBRARIES})
else()
message(STATUS "don't use NUMA")
endif()

find_package(Backtrace)
if(Backtrace_FOUND)
target_include_directories(c10 PRIVATE ${Backtrace_INCLUDE_DIRS})
target_link_libraries(c10 PRIVATE ${Backtrace_LIBRARIES})
target_compile_definitions(c10 PRIVATE SUPPORTS_BACKTRACE=1)
else()
target_compile_definitions(c10 PRIVATE SUPPORTS_BACKTRACE=0)
endif()
if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "s390x" AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
target_link_libraries(c10 PRIVATE cpuinfo)
endif()

if(USE_MIMALLOC)
target_link_libraries(c10 PRIVATE "mimalloc-static")
add_dependencies(c10 mimalloc-static)
endif()
find_package(Backtrace)
if(Backtrace_FOUND)
target_include_directories(c10 PRIVATE ${Backtrace_INCLUDE_DIRS})
target_link_libraries(c10 PRIVATE ${Backtrace_LIBRARIES})
target_compile_definitions(c10 PRIVATE SUPPORTS_BACKTRACE=1)
else()
target_compile_definitions(c10 PRIVATE SUPPORTS_BACKTRACE=0)
endif()

if(ANDROID)
target_link_libraries(c10 PRIVATE log)
endif()
if(USE_MIMALLOC)
target_link_libraries(c10 PRIVATE "mimalloc-static")
add_dependencies(c10 mimalloc-static)
endif()

if(ANDROID)
target_link_libraries(c10 PRIVATE log)
endif()

target_include_directories(
c10 PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../>
$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}>
$<INSTALL_INTERFACE:include>)
target_include_directories(
c10 PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../>
$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}>
$<INSTALL_INTERFACE:include>)
endif()

add_subdirectory(test)
add_subdirectory(benchmark)
Expand All @@ -144,11 +152,14 @@ if(USE_XPU)
add_subdirectory(xpu)
endif()

# ---[ Installation
# Note: for now, we will put all export path into one single Caffe2Targets group
# to deal with the cmake deployment need. Inside the Caffe2Targets set, the
# individual libraries like libc10.so and libcaffe2.so are still self-contained.
install(TARGETS c10 EXPORT Caffe2Targets DESTINATION lib)
if(NOT BUILD_LIBTORCHLESS)
# ---[ Installation
# Note: for now, we will put all export path into one single Caffe2Targets group
# to deal with the cmake deployment need. Inside the Caffe2Targets set, the
# individual libraries like libc10.so and libcaffe2.so are still self-contained.
install(TARGETS c10 EXPORT Caffe2Targets DESTINATION lib)
endif()

install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
DESTINATION include
FILES_MATCHING PATTERN "*.h")
Expand Down
2 changes: 1 addition & 1 deletion c10/benchmark/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ if(BUILD_TEST)
get_filename_component(bench_file_name ${bench_src} NAME_WE)
set(bench_name "c10_${bench_file_name}")
add_executable(${bench_name} "${bench_src}")
target_link_libraries(${bench_name} c10 benchmark)
target_link_libraries(${bench_name} ${C10_LIB} benchmark)
if(INSTALL_TEST)
install(TARGETS ${bench_name} DESTINATION test)
endif()
Expand Down
50 changes: 30 additions & 20 deletions c10/cuda/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ configure_file(
${CMAKE_CURRENT_LIST_DIR}/impl/cuda_cmake_macros.h.in
${CMAKE_BINARY_DIR}/c10/cuda/impl/cuda_cmake_macros.h)

if(BUILD_LIBTORCHLESS)
find_library(C10_CUDA_LIB c10_cuda PATHS $ENV{LIBTORCH_LIB_PATH} NO_DEFAULT_PATH)
endif()

# Note: if you want to add ANY dependency to the c10 library, make sure you
# check with the core PyTorch developers as the dependency will be
# transitively passed on to all libraries dependent on PyTorch.
Expand Down Expand Up @@ -47,36 +51,42 @@ set(C10_CUDA_HEADERS
impl/CUDATest.h
)
set(CUDA_LINK_LIBRARIES_KEYWORD PRIVATE)
torch_cuda_based_add_library(c10_cuda ${C10_CUDA_SRCS} ${C10_CUDA_HEADERS})
set(CUDA_LINK_LIBRARIES_KEYWORD)
# If building shared library, set dllimport/dllexport proper.
target_compile_options(c10_cuda PRIVATE "-DC10_CUDA_BUILD_MAIN_LIB")
# Enable hidden visibility if compiler supports it.
if(${COMPILER_SUPPORTS_HIDDEN_VISIBILITY})
target_compile_options(c10_cuda PRIVATE "-fvisibility=hidden")
endif()

# ---[ Dependency of c10_cuda
target_link_libraries(c10_cuda PUBLIC c10 torch::cudart)
if(NOT BUILD_LIBTORCHLESS)
torch_cuda_based_add_library(c10_cuda ${C10_CUDA_SRCS} ${C10_CUDA_HEADERS})
set(CUDA_LINK_LIBRARIES_KEYWORD)
# If building shared library, set dllimport/dllexport proper.
target_compile_options(c10_cuda PRIVATE "-DC10_CUDA_BUILD_MAIN_LIB")
# Enable hidden visibility if compiler supports it.
if(${COMPILER_SUPPORTS_HIDDEN_VISIBILITY})
target_compile_options(c10_cuda PRIVATE "-fvisibility=hidden")
endif()

if(NOT WIN32)
target_link_libraries(c10_cuda PRIVATE dl)
target_compile_options(c10_cuda PRIVATE "-DPYTORCH_C10_DRIVER_API_SUPPORTED")
endif()
# ---[ Dependency of c10_cuda
target_link_libraries(c10_cuda PUBLIC ${C10_LIB} torch::cudart)

target_include_directories(
c10_cuda PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../..>
$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}>
$<INSTALL_INTERFACE:include>)
if(NOT WIN32)
target_link_libraries(c10_cuda PRIVATE dl)
target_compile_options(c10_cuda PRIVATE "-DPYTORCH_C10_DRIVER_API_SUPPORTED")
endif()

add_subdirectory(test)
target_include_directories(
c10_cuda PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../..>
$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}>
$<INSTALL_INTERFACE:include>)
set(C10_CUDA_LIB c10_cuda)

# ---[ Installation
# Note: for now, we will put all export path into one single Caffe2Targets group
# to deal with the cmake deployment need. Inside the Caffe2Targets set, the
# individual libraries like libc10.so and libcaffe2.so are still self-contained.
install(TARGETS c10_cuda EXPORT Caffe2Targets DESTINATION lib)

endif()

add_subdirectory(test)

foreach(file ${C10_CUDA_HEADERS})
get_filename_component( dir ${file} DIRECTORY )
install( FILES ${file} DESTINATION include/c10/cuda/${dir} )
Expand Down
2 changes: 1 addition & 1 deletion c10/cuda/test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ if(BUILD_TEST)
get_filename_component(test_file_name ${test_src} NAME_WE)
set(test_name "c10_cuda_${test_file_name}")
add_executable(${test_name} "${test_src}")
target_link_libraries(${test_name} c10_cuda gtest_main)
target_link_libraries(${test_name} ${C10_CUDA_LIB} ${C10_LIB} gtest_main)
add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
if(INSTALL_TEST)
install(TARGETS ${test_name} DESTINATION test)
Expand Down
Loading

0 comments on commit a25b28a

Please sign in to comment.