[1/2] Remove caffe2 db and distributed from build system (pytorch#125092

)

This PR tries to decompose pytorch#122527 into a smaller one. Caffe2 db, distributed and some binaries have been removed.
To be noted, this was inspired and is co-dev with @r-barnes.

Pull Request resolved: pytorch#125092
Approved by: https://github.com/malfet

.ci/docker/common/install_base.sh

@@ -113,7 +113,6 @@ install_centos() {
     glibc-devel \
     glibc-headers \
     glog-devel \
-    hiredis-devel \
     libstdc++-devel \
     libsndfile-devel \
     make \

.ci/docker/common/install_db.sh

@@ -4,11 +4,6 @@ set -ex
 
 install_ubuntu() {
   apt-get update
-  apt-get install -y --no-install-recommends \
-          libhiredis-dev \
-          libleveldb-dev \
-          liblmdb-dev \
-          libsnappy-dev
 
   # Cleanup
   apt-get autoclean && apt-get clean
@@ -20,12 +15,6 @@ install_centos() {
   # See http://fedoraproject.org/wiki/EPEL
   yum --enablerepo=extras install -y epel-release
 
-  yum install -y \
-      hiredis-devel \
-      leveldb-devel \
-      lmdb-devel \
-      snappy-devel
-
   # Cleanup
   yum clean all
   rm -rf /var/cache/yum

BUILD.bazel

@@ -503,16 +503,6 @@ filegroup(
     ],
 )
 
-filegroup(
-    name = "caffe2_distributed_srcs",
-    srcs = [
-        "caffe2/distributed/file_store_handler.cc",
-        "caffe2/distributed/file_store_handler_op.cc",
-        "caffe2/distributed/store_handler.cc",
-        "caffe2/distributed/store_ops.cc",
-    ],
-)
-
 filegroup(
     name = "caffe2_ideep_srcs",
     srcs = [
@@ -1011,8 +1001,6 @@ filegroup(
         "caffe2/core/common_cudnn.cc",
         "caffe2/core/common_gpu.cc",
         "caffe2/core/event_gpu.cc",
-        "caffe2/db/create_db_op_gpu.cc",
-        "caffe2/distributed/file_store_handler_op_gpu.cc",
         "caffe2/operators/communicator_op_gpu.cc",
         "caffe2/operators/concat_split_op_gpu.cc",
         "caffe2/operators/conv_op_cache_cudnn.cc",
@@ -1262,8 +1250,6 @@ cc_library(
             "caffe2/core/nomnigraph/include/nomnigraph/Support/*.h",
             "caffe2/core/nomnigraph/include/nomnigraph/Transformations/*.h",
             "caffe2/core/nomnigraph/tests/*.h",
-            "caffe2/db/*.h",
-            "caffe2/distributed/*.h",
             "caffe2/ideep/*.h",
             "caffe2/ideep/operators/*.h",
             "caffe2/ideep/operators/quantization/*.h",
@@ -1335,11 +1321,8 @@ cc_library(
 cc_library(
     name = "caffe2",
     srcs = [
-        "caffe2/db/create_db_op.cc",
-        "caffe2/db/protodb.cc",
         "caffe2/share/contrib/depthwise/depthwise3x3_conv_op.cc",
         ":caffe2_core_srcs",
-        ":caffe2_distributed_srcs",
         ":caffe2_ideep_srcs",
         ":caffe2_onnx_srcs",
         ":caffe2_operators_srcs",

CMakeLists.txt

@@ -230,9 +230,7 @@ option(USE_CUPTI_SO "Use CUPTI as a shared library" ON)
 option(USE_FAKELOWP "Use FakeLowp operators" OFF)
 option(USE_GFLAGS "Use GFLAGS" OFF)
 option(USE_GLOG "Use GLOG" OFF)
-option(USE_LEVELDB "Use LEVELDB" OFF)
 option(USE_LITE_PROTO "Use lite protobuf instead of full." OFF)
-option(USE_LMDB "Use LMDB" OFF)
 option(USE_MAGMA "Use MAGMA" ON)
 option(USE_METAL "Use Metal for Caffe2 iOS build" ON)
 option(USE_PYTORCH_METAL "Use Metal for PyTorch iOS build" OFF)
@@ -269,8 +267,6 @@ option(USE_PRECOMPILED_HEADERS "Use pre-compiled headers to accelerate build." O
 option(USE_PROF "Use profiling" OFF)
 option(USE_QNNPACK "Use QNNPACK (quantized 8-bit operators)" ON)
 option(USE_PYTORCH_QNNPACK "Use ATen/QNNPACK (quantized 8-bit operators)" ON)
-option(USE_REDIS "Use Redis" OFF)
-option(USE_ROCKSDB "Use RocksDB" OFF)
 option(USE_SNPE "Use Qualcomm's SNPE library" OFF)
 option(USE_SYSTEM_EIGEN_INSTALL
     "Use system Eigen instead of the one under third_party" OFF)
@@ -292,7 +288,6 @@ option(USE_VULKAN_FP16_INFERENCE "Vulkan - Use fp16 inference" OFF)
 option(USE_VULKAN_RELAXED_PRECISION "Vulkan - Use relaxed precision math in the kernels (mediump)" OFF)
 # option USE_XNNPACK: try to enable xnnpack by default.
 option(USE_XNNPACK "Use XNNPACK" ON)
-option(USE_ZMQ "Use ZMQ" OFF)
 option(USE_ZSTD "Use ZSTD" OFF)
 option(USE_ROCM_KERNEL_ASSERT "Use Kernel Assert for ROCm" OFF)
 # Ensure that an ITT build is the default for x86 CPUs

binaries/CMakeLists.txt

@@ -8,33 +8,13 @@ if(INTERN_BUILD_MOBILE)
 endif()
 
 if(BUILD_CAFFE2)
-  caffe2_binary_target("convert_caffe_image_db.cc")
-  caffe2_binary_target("convert_db.cc")
-  caffe2_binary_target("make_cifar_db.cc")
-  caffe2_binary_target("make_mnist_db.cc")
-  caffe2_binary_target("predictor_verifier.cc")
-  caffe2_binary_target("speed_benchmark.cc")
-  caffe2_binary_target("split_db.cc")
-  caffe2_binary_target("print_registered_core_operators.cc")
-
-  if(USE_OBSERVERS)
-    caffe2_binary_target(caffe2_benchmark "caffe2_benchmark.cc" "benchmark_helper.cc")
-    target_include_directories(caffe2_benchmark PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../modules)
-  endif()
-
   caffe2_binary_target("at_launch_benchmark.cc")
   target_include_directories(at_launch_benchmark PUBLIC
     ${CMAKE_BINARY_DIR}/aten/src)
 
   caffe2_binary_target("intra_inter_benchmark.cc")
   target_include_directories(intra_inter_benchmark PUBLIC
     ${CMAKE_BINARY_DIR}/aten/src)
-
-  caffe2_binary_target("run_plan.cc")
-  caffe2_binary_target("db_throughput.cc")
-
-  # ---[ tutorials
-  caffe2_binary_target("tutorial_blob.cc")
 endif()
 
 caffe2_binary_target("parallel_info.cc")
@@ -77,34 +57,11 @@ if(USE_ROCM)
   endif()
 endif()
 
-if(USE_ZMQ)
-  caffe2_binary_target("zmq_feeder.cc")
-  target_link_libraries(zmq_feeder ${ZMQ_LIBRARIES})
-endif()
-
 if(USE_MPI)
   caffe2_binary_target("run_plan_mpi.cc")
   target_link_libraries(run_plan_mpi ${MPI_CXX_LIBRARIES})
 endif()
 
-if(USE_OPENCV AND USE_LEVELDB)
-  caffe2_binary_target("convert_encoded_to_raw_leveldb.cc")
-  target_link_libraries(
-      convert_encoded_to_raw_leveldb
-      ${OpenCV_LIBS} ${LevelDB_LIBRARIES} ${Snappy_LIBRARIES})
-endif()
-
-if(USE_OPENCV)
-  caffe2_binary_target("make_image_db.cc")
-  target_link_libraries(make_image_db ${OpenCV_LIBS})
-  caffe2_binary_target("convert_image_to_tensor.cc")
-  target_link_libraries(convert_image_to_tensor ${OpenCV_LIBS})
-endif()
-
-if(USE_OBSERVERS AND USE_OPENCV)
-  caffe2_binary_target("convert_and_benchmark.cc")
-  target_link_libraries(convert_and_benchmark ${OpenCV_LIBS})
-endif()
 
 caffe2_binary_target("dump_operator_names.cc")
 caffe2_binary_target("optimize_for_mobile.cc")