diff --git a/.bazelrc b/.bazelrc index d60866a..0b78c5e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -2,9 +2,8 @@ build -c opt build --cxxopt=-std=c++17 build --host_cxxopt=-std=c++17 build --experimental_repo_remote_exec - -# TODO(fchern): Use non-hardcode path. -build --action_env=PYTHON_BIN_PATH="/usr/bin/python3" -build --action_env=PYTHON_LIB_PATH="/usr/lib/python3" -build --repo_env=PYTHON_BIN_PATH="/usr/bin/python3" -build --python_path="/usr/bin/python3" +build --linkopt="-lpthread -lm -framework CoreFoundation" +build --action_env=PYTHON_BIN_PATH="/Users/pierremarcenac/.pyenv/shims/python" +build --python_path="/Users/pierremarcenac/.pyenv/shims/python" +build --action_env=PYTHON_LIB_PATH="/Users/pierremarcenac/.pyenv/versions/python3.10/lib/python3.10/site-packages" +build --action_env=CXXFLAGS=-stdlib=libc++ --action_env=LDFLAGS=-stdlib=libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++ --action_env=BAZEL_LINKOPTS=-lc++:-lm diff --git a/WORKSPACE b/WORKSPACE index e1c5c7f..bddd4af 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -55,9 +55,9 @@ cc_library( # `pybind11_bazel` (https://github.com/pybind/pybind11_bazel): 20230130 http_archive( name = "pybind11_bazel", - strip_prefix = "pybind11_bazel-5f458fa53870223a0de7eeb60480dd278b442698", - sha256 = "b35f3abc3d52ee5c753fdeeb2b5129b99e796558754ca5d245e28e51c1072a21", - urls = ["https://github.com/pybind/pybind11_bazel/archive/5f458fa53870223a0de7eeb60480dd278b442698.tar.gz"], + strip_prefix = "pybind11_bazel-master", +# sha256 = "b35f3abc3d52ee5c753fdeeb2b5129b99e796558754ca5d245e28e51c1072a21", + urls = ["https://github.com/pybind/pybind11_bazel/archive/master.tar.gz"], ) # V2.10.3, 20230130 http_archive( @@ -131,9 +131,9 @@ http_archive( http_archive( name = "highwayhash", build_file = "@com_google_riegeli//third_party:highwayhash.BUILD", - sha256 = "cf891e024699c82aabce528a024adbe16e529f2b4e57f954455e0bf53efae585", - strip_prefix = "highwayhash-276dd7b4b6d330e4734b756e97ccfb1b69cc2e12", - urls = ["https://github.com/google/highwayhash/archive/276dd7b4b6d330e4734b756e97ccfb1b69cc2e12.zip"], # 2019-02-22 + # sha256 = "cf891e024699c82aabce528a024adbe16e529f2b4e57f954455e0bf53efae585", + strip_prefix = "highwayhash-master", + urls = ["https://github.com/google/highwayhash/archive/master.zip"], # 2019-02-22 ) # Tensorflow, 20230130 diff --git a/cpp/BUILD b/cpp/BUILD index 1c77e47..2352372 100644 --- a/cpp/BUILD +++ b/cpp/BUILD @@ -54,6 +54,7 @@ cc_library( deps = [ "@com_google_absl//absl/flags:flag", "@eigen3//:eigen3", + "@highwayhash//:hh_neon", ], ) @@ -123,6 +124,7 @@ cc_library( "@com_google_riegeli//riegeli/chunk_encoding:simple_encoder", "@com_google_riegeli//riegeli/chunk_encoding:transpose_encoder", "@com_google_riegeli//riegeli/records:records_metadata_cc_proto", + "@highwayhash//:hh_neon", ], ) @@ -171,6 +173,7 @@ cc_library( "@com_google_riegeli//riegeli/chunk_encoding:chunk_decoder", "@com_google_riegeli//riegeli/records:chunk_reader", "@com_google_riegeli//riegeli/records:record_position", + "@highwayhash//:hh_neon", ], ) diff --git a/cpp/array_record_reader.cc b/cpp/array_record_reader.cc index 28b88d6..1da62f8 100644 --- a/cpp/array_record_reader.cc +++ b/cpp/array_record_reader.cc @@ -317,7 +317,7 @@ absl::Status ArrayRecordReaderBase::ParallelReadRecords( return absl::OkStatus(); } uint64_t num_chunk_groups = - CeilOfRatio(state_->footer.size(), state_->chunk_group_size); + CeilOfRatio(state_->footer.size(), state_->chunk_group_size); const auto reader = get_backing_reader(); Reader* mutable_reader = const_cast( reinterpret_cast(reader.get())); @@ -326,7 +326,7 @@ absl::Status ArrayRecordReaderBase::ParallelReadRecords( uint64_t chunk_idx_start = buf_idx * state_->chunk_group_size; // inclusive index, not the conventional exclusive index. uint64_t last_chunk_idx = - std::min((buf_idx + 1) * state_->chunk_group_size - 1, + std::min((buf_idx + 1) * state_->chunk_group_size - 1, state_->footer.size() - 1); uint64_t buf_len = state_->ChunkEndOffset(last_chunk_idx) - state_->footer[chunk_idx_start].chunk_offset(); @@ -654,7 +654,7 @@ bool ArrayRecordReaderBase::ReadAheadFromBuffer(uint64_t buffer_idx) { std::vector decoders; decoders.reserve(state_->chunk_group_size); uint64_t chunk_start = buffer_idx * state_->chunk_group_size; - uint64_t chunk_end = std::min(state_->footer.size(), + uint64_t chunk_end = std::min(state_->footer.size(), (buffer_idx + 1) * state_->chunk_group_size); const auto reader = get_backing_reader(); for (uint64_t chunk_idx = chunk_start; chunk_idx < chunk_end; ++chunk_idx) { @@ -693,7 +693,7 @@ bool ArrayRecordReaderBase::ReadAheadFromBuffer(uint64_t buffer_idx) { std::vector chunk_offsets; chunk_offsets.reserve(state_->chunk_group_size); uint64_t chunk_start = buffer_to_add * state_->chunk_group_size; - uint64_t chunk_end = std::min( + uint64_t chunk_end = std::min( state_->footer.size(), (buffer_to_add + 1) * state_->chunk_group_size); for (uint64_t chunk_idx = chunk_start; chunk_idx < chunk_end; ++chunk_idx) { chunk_offsets.push_back(state_->footer[chunk_idx].chunk_offset()); diff --git a/oss/build_whl.sh b/oss/build_whl.sh index 6f3a5be..993c4ca 100755 --- a/oss/build_whl.sh +++ b/oss/build_whl.sh @@ -4,7 +4,7 @@ set -e -x export PYTHON_VERSION="${PYTHON_VERSION}" -PYTHON="python${PYTHON_VERSION}" +PYTHON="python" PYTHON_MAJOR_VERSION=$(${PYTHON} -c 'import sys; print(sys.version_info.major)') PYTHON_MINOR_VERSION=$(${PYTHON} -c 'import sys; print(sys.version_info.minor)') BAZEL_FLAGS="--crosstool_top=" @@ -21,18 +21,20 @@ function main() { write_to_bazelrc "build -c opt" write_to_bazelrc "build --cxxopt=-std=c++17" write_to_bazelrc "build --host_cxxopt=-std=c++17" - write_to_bazelrc "build --linkopt=\"-lrt -lm\"" + # write_to_bazelrc "build --linkopt=\"-lrt -lm\"" write_to_bazelrc "build --experimental_repo_remote_exec" - write_to_bazelrc "build --action_env=PYTHON_BIN_PATH=\"/usr/bin/${PYTHON}\"" - write_to_bazelrc "build --action_env=PYTHON_LIB_PATH=\"/usr/lib/${PYTHON}\"" - write_to_bazelrc "build --python_path=\"/usr/bin/${PYTHON}\"" + write_to_bazelrc "build --action_env=PYTHON_BIN_PATH=\"/Users/pierremarcenac/.pyenv/shims/python\"" + write_to_bazelrc "build --action_env=PYTHON_LIB_PATH=\"/Users/pierremarcenac/.pyenv/versions/python3.10/lib/python3.10/site-packages\"" + write_to_bazelrc "build --python_path=\"/Users/pierremarcenac/.pyenv/shims/python\"" + + ${PYTHON} -m pip install absl-py etils wheel # Using a previous version of Blaze to avoid: # https://github.com/bazelbuild/bazel/issues/8622 export USE_BAZEL_VERSION=5.4.0 - bazel clean - bazel build ${BAZEL_FLAGS} ... - bazel test ${BAZEL_FLAGS} --verbose_failures --test_output=errors ... + bazel clean --expunge + bazel build //... + bazel test --verbose_failures --test_output=errors //... DEST="/tmp/array_record/all_dist" # Create the directory, then do dirname on a non-existent file inside it to diff --git a/python/BUILD b/python/BUILD index 6494b2e..c8038a4 100644 --- a/python/BUILD +++ b/python/BUILD @@ -17,6 +17,7 @@ pybind_extension( "@com_google_absl//absl/strings", "@com_google_riegeli//riegeli/bytes:fd_reader", "@com_google_riegeli//riegeli/bytes:fd_writer", + "@highwayhash//:hh_neon", ], )