From e4a25cf0b2e3a8056866a1820584b96bd986b593 Mon Sep 17 00:00:00 2001 From: pichuan Date: Tue, 10 Sep 2024 11:24:58 -0700 Subject: [PATCH] Migrate to Pybind11. PiperOrigin-RevId: 673038055 --- README.md | 2 +- WORKSPACE | 9 + build-prereq.sh | 65 ++- deeptrio/protos/BUILD | 6 - deepvariant/BUILD | 1 + deepvariant/protos/BUILD | 11 - deepvariant/python/BUILD | 202 +++++--- deepvariant/python/allelecounter.clif | 55 -- deepvariant/python/allelecounter_pybind.cc | 7 +- deepvariant/python/direct_phasing.clif | 46 -- deepvariant/python/direct_phasing_pybind.cc | 4 +- deepvariant/python/make_examples_native.clif | 68 --- .../python/make_examples_native_pybind.cc | 2 +- deepvariant/python/pileup_image_native.clif | 63 --- .../python/pileup_image_native_pybind.cc | 2 +- deepvariant/python/postprocess_variants.clif | 38 -- .../python/postprocess_variants_pybind.cc | 2 +- deepvariant/python/variant_calling.clif | 43 -- .../python/variant_calling_multisample.clif | 42 -- .../variant_calling_multisample_pybind.cc | 3 +- deepvariant/python/variant_calling_pybind.cc | 3 +- deepvariant/realigner/BUILD | 1 + deepvariant/realigner/python/BUILD | 108 +++- .../realigner/python/debruijn_graph.clif | 45 -- .../realigner/python/debruijn_graph_pybind.cc | 2 +- .../realigner/python/fast_pass_aligner.clif | 46 -- .../python/fast_pass_aligner_pybind.cc | 2 +- deepvariant/realigner/python/ssw.clif | 90 ---- deepvariant/realigner/python/ssw_pybind.cc | 2 +- .../realigner/python/window_selector.clif | 39 -- .../python/window_selector_pybind.cc | 2 +- third_party/clif.bzl | 454 ---------------- third_party/nucleus/core/BUILD | 12 - third_party/nucleus/core/python/BUILD | 37 +- .../core/python/statusor_examples.clif | 52 -- .../nucleus/core/statusor_clif_converters.cc | 57 -- .../nucleus/core/statusor_clif_converters.h | 82 --- third_party/nucleus/io/BUILD | 9 + third_party/nucleus/io/python/BUILD | 487 +++++++++++++----- third_party/nucleus/io/python/bed_reader.clif | 60 --- .../nucleus/io/python/bed_reader_pybind.cc | 2 +- third_party/nucleus/io/python/bed_writer.clif | 47 -- .../nucleus/io/python/bed_writer_pybind.cc | 4 +- .../nucleus/io/python/bedgraph_reader.clif | 59 --- .../io/python/bedgraph_reader_pybind.cc | 6 +- .../nucleus/io/python/bedgraph_writer.clif | 44 -- .../io/python/bedgraph_writer_pybind.cc | 2 +- .../nucleus/io/python/fastq_reader.clif | 58 --- .../nucleus/io/python/fastq_reader_pybind.cc | 4 +- .../io/python/fastq_reader_wrap_test.py | 7 +- .../nucleus/io/python/fastq_writer.clif | 45 -- .../nucleus/io/python/fastq_writer_pybind.cc | 4 +- third_party/nucleus/io/python/gff_reader.clif | 60 --- .../nucleus/io/python/gff_reader_pybind.cc | 2 +- .../nucleus/io/python/gff_reader_wrap_test.py | 2 +- third_party/nucleus/io/python/gff_writer.clif | 47 -- .../nucleus/io/python/gff_writer_pybind.cc | 2 +- third_party/nucleus/io/python/gfile.clif | 64 --- third_party/nucleus/io/python/gfile_pybind.cc | 2 +- .../nucleus/io/python/hts_verbose.clif | 34 -- .../nucleus/io/python/hts_verbose_pybind.cc | 2 +- .../nucleus/io/python/merge_variants.clif | 45 -- .../io/python/merge_variants_pybind.cc | 4 +- third_party/nucleus/io/python/reference.clif | 85 --- .../nucleus/io/python/reference_pybind.cc | 2 +- third_party/nucleus/io/python/sam_reader.clif | 64 --- .../nucleus/io/python/sam_reader_pybind.cc | 4 +- .../nucleus/io/python/sam_reader_wrap_test.py | 8 +- third_party/nucleus/io/python/sam_writer.clif | 47 -- .../nucleus/io/python/sam_writer_pybind.cc | 4 +- .../nucleus/io/python/tabix_indexer.clif | 35 -- .../nucleus/io/python/tabix_indexer_pybind.cc | 2 +- .../nucleus/io/python/tfrecord_reader.clif | 43 -- .../io/python/tfrecord_reader_pybind.cc | 2 +- .../nucleus/io/python/tfrecord_writer.clif | 41 -- third_party/nucleus/io/python/vcf_concat.clif | 32 -- .../nucleus/io/python/vcf_concat_pybind.cc | 2 +- third_party/nucleus/io/python/vcf_reader.clif | 76 --- .../nucleus/io/python/vcf_reader_pybind.cc | 4 +- third_party/nucleus/io/python/vcf_writer.clif | 46 -- .../nucleus/io/python/vcf_writer_pybind.cc | 4 +- third_party/nucleus/protos/BUILD | 60 --- third_party/nucleus/util/BUILD | 15 +- .../nucleus/util/proto_clif_converter.cc | 50 -- .../nucleus/util/proto_clif_converter.h | 119 ----- third_party/nucleus/util/python/BUILD | 52 +- third_party/nucleus/util/python/math.clif | 42 -- .../nucleus/util/python/math_pybind.cc | 2 +- .../python/type_caster_nucleus_proto_ptr.h | 4 +- third_party/nucleus/util/python/utils.clif | 40 -- .../nucleus/util/python/utils_wrap_test.py | 9 + third_party/tensorflow.bzl.patch | 141 +++++ tools/{build_clif.sh => build_absl.sh} | 44 +- 93 files changed, 917 insertions(+), 2969 deletions(-) delete mode 100644 deepvariant/python/allelecounter.clif delete mode 100644 deepvariant/python/direct_phasing.clif delete mode 100644 deepvariant/python/make_examples_native.clif delete mode 100644 deepvariant/python/pileup_image_native.clif delete mode 100644 deepvariant/python/postprocess_variants.clif delete mode 100644 deepvariant/python/variant_calling.clif delete mode 100644 deepvariant/python/variant_calling_multisample.clif delete mode 100644 deepvariant/realigner/python/debruijn_graph.clif delete mode 100644 deepvariant/realigner/python/fast_pass_aligner.clif delete mode 100644 deepvariant/realigner/python/ssw.clif delete mode 100644 deepvariant/realigner/python/window_selector.clif delete mode 100644 third_party/clif.bzl delete mode 100644 third_party/nucleus/core/python/statusor_examples.clif delete mode 100644 third_party/nucleus/core/statusor_clif_converters.cc delete mode 100644 third_party/nucleus/core/statusor_clif_converters.h delete mode 100644 third_party/nucleus/io/python/bed_reader.clif delete mode 100644 third_party/nucleus/io/python/bed_writer.clif delete mode 100644 third_party/nucleus/io/python/bedgraph_reader.clif delete mode 100644 third_party/nucleus/io/python/bedgraph_writer.clif delete mode 100644 third_party/nucleus/io/python/fastq_reader.clif delete mode 100644 third_party/nucleus/io/python/fastq_writer.clif delete mode 100644 third_party/nucleus/io/python/gff_reader.clif delete mode 100644 third_party/nucleus/io/python/gff_writer.clif delete mode 100644 third_party/nucleus/io/python/gfile.clif delete mode 100644 third_party/nucleus/io/python/hts_verbose.clif delete mode 100644 third_party/nucleus/io/python/merge_variants.clif delete mode 100644 third_party/nucleus/io/python/reference.clif delete mode 100644 third_party/nucleus/io/python/sam_reader.clif delete mode 100644 third_party/nucleus/io/python/sam_writer.clif delete mode 100644 third_party/nucleus/io/python/tabix_indexer.clif delete mode 100644 third_party/nucleus/io/python/tfrecord_reader.clif delete mode 100644 third_party/nucleus/io/python/tfrecord_writer.clif delete mode 100644 third_party/nucleus/io/python/vcf_concat.clif delete mode 100644 third_party/nucleus/io/python/vcf_reader.clif delete mode 100644 third_party/nucleus/io/python/vcf_writer.clif delete mode 100644 third_party/nucleus/util/proto_clif_converter.cc delete mode 100644 third_party/nucleus/util/proto_clif_converter.h delete mode 100644 third_party/nucleus/util/python/math.clif delete mode 100644 third_party/nucleus/util/python/utils.clif create mode 100644 third_party/tensorflow.bzl.patch rename tools/{build_clif.sh => build_absl.sh} (67%) diff --git a/README.md b/README.md index c6812edd..fd7c98e6 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ specifically call out a few key ones: * [Boost Graph Library](http://www.boost.org/doc/libs/1_65_1/libs/graph/doc/index.html) * [abseil-cpp](https://github.com/abseil/abseil-cpp) and [abseil-py](https://github.com/abseil/abseil-py) -* [CLIF](https://github.com/google/clif) +* [pybind11](https://github.com/pybind/pybind11) * [GNU Parallel](https://www.gnu.org/software/parallel/) * [htslib & samtools](http://www.htslib.org/) * [Nucleus](https://github.com/google/nucleus) diff --git a/WORKSPACE b/WORKSPACE index 6abf2553..5d38e2c0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,6 +98,15 @@ http_archive( ], ) +http_archive( + name = "pybind11_protobuf", + sha256 = "21e0c32d81ece8039a3a8e6daafbd7f64cb0c2744492f3b00f11baa0e276d1a5", + strip_prefix = "pybind11_protobuf-de94308491982c32ddfe305a5dfc3c38bc9ff2bc", + urls = [ + "https://github.com/pichuan/pybind11_protobuf/archive/de94308491982c32ddfe305a5dfc3c38bc9ff2bc.zip", + ], +) + # Import all of the tensorflow dependencies. # Copied from tensorflow/WORKSPACE. Updated in v2.5.0: load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3") diff --git a/build-prereq.sh b/build-prereq.sh index d4097ca9..1d6a1e7c 100755 --- a/build-prereq.sh +++ b/build-prereq.sh @@ -82,31 +82,8 @@ function ensure_wanted_bazel_version { ensure_wanted_bazel_version "${DV_BAZEL_VERSION}" -################################################################################ -# CLIF -################################################################################ - -note_build_stage "Install CLIF binary" - -if [[ -e /usr/local/bin/pyclif ]]; -then - echo "CLIF already installed." -else - # Build clif binary from scratch. Might not be ideal because it installs a - # bunch of dependencies, but this works fine when we used this in a Dockerfile - # because we don't do build-prereq.sh in the final image. - note_build_stage "Build CLIF." - time sudo ./tools/build_clif.sh - # TODO: - # Figure out why these symbolic links are needed and see if - # we can do this better. - sudo mkdir -p /usr/clang/bin/ - sudo ln -sf /usr/local/bin/clif-matcher /usr/clang/bin/clif-matcher - sudo mkdir -p /usr/local/clif/bin - sudo ln -sf /usr/local/bin/pyclif* /usr/local/clif/bin/ - DIST_PACKAGES_DIR=$(python3 -c "import site; print(site.getsitepackages()[0])") - sudo ln -sf "${DIST_PACKAGES_DIR}"/clif/python /usr/local/clif/ -fi +# This is used for building examples_from_stream.so later. +time sudo ./tools/build_absl.sh ################################################################################ # TensorFlow @@ -114,6 +91,9 @@ fi note_build_stage "Download and configure TensorFlow sources" +# Getting the directory before switching out. +DV_DIR=$(pwd) + if [[ ! -d ../tensorflow ]]; then note_build_stage "Cloning TensorFlow from github as ../tensorflow doesn't exist" (cd .. && git clone https://github.com/tensorflow/tensorflow) @@ -131,8 +111,38 @@ fi # r2.13. Eventually we'll want to update to TF 2.13. But for now this works. # TODO: After updating to v2.13, we can remove this. wget https://raw.githubusercontent.com/tensorflow/tensorflow/r2.13/third_party/absl/workspace.bzl -O ../tensorflow/third_party/absl/workspace.bzl -wget https://raw.githubusercontent.com/tensorflow/tensorflow/r2.13/third_party/absl/absl_designated_initializers.patch -O ../tensorflow/third_party/absl/absl_design\ -ated_initializers.patch +rm -f ../tensorflow/third_party/absl/absl_designated_initializers.patch +# To get the @com_google_absl//absl/strings:string_view target: +sed -i -e 's|b971ac5250ea8de900eae9f95e06548d14cd95fe|29bf8085f3bf17b84d30e34b3d7ff8248fda404e|g' ../tensorflow/third_party/absl/workspace.bzl +sed -i -e 's|8eeec9382fc0338ef5c60053f3a4b0e0708361375fe51c9e65d0ce46ccfe55a7|affb64f374b16877e47009df966d0a9403dbf7fe613fe1f18e49802c84f6421e|g' ../tensorflow/third_party/absl/workspace.bzl +sed -i -e 's|patch_file = \["//third_party/absl:absl_designated_initializers.patch"\],||g' ../tensorflow/third_party/absl/workspace.bzl + +# Update tensorflow.bzl. This updates the `pybind_extension` rule to use the +# _message.so file. +patch ../tensorflow/tensorflow/tensorflow.bzl "${DV_DIR}"/third_party/tensorflow.bzl.patch + +# I want to replace this part in ../tensorflow/tensorflow/workspace2.bzl +# From: +# tf_http_archive( +# name = "pybind11", +# urls = tf_mirror_urls("https://github.com/pybind/pybind11/archive/v2.10.0.tar.gz"), +# sha256 = "eacf582fa8f696227988d08cfc46121770823839fe9e301a20fbce67e7cd70ec", +# strip_prefix = "pybind11-2.10.0", +# build_file = "//third_party:pybind11.BUILD", +# system_build_file = "//third_party/systemlibs:pybind11.BUILD", +# ) +# To: +# tf_http_archive( +# name = "pybind11", +# urls = tf_mirror_urls("https://github.com/pybind/pybind11/archive/a7b91e33269ab6f3f90167291af2c4179fc878f5.zip"), +# sha256 = "09d2ab67e91457c966eb335b361bdc4d27ece2d4dea681d22e5d8307e0e0c023", +# strip_prefix = "pybind11-a7b91e33269ab6f3f90167291af2c4179fc878f5", +# build_file = "//third_party:pybind11.BUILD", +# system_build_file = "//third_party/systemlibs:pybind11.BUILD", +# ) +sed -i -e 's|v2.10.0.tar.gz|a7b91e33269ab6f3f90167291af2c4179fc878f5.zip|g' ../tensorflow/tensorflow/workspace2.bzl +sed -i -e 's|eacf582fa8f696227988d08cfc46121770823839fe9e301a20fbce67e7cd70ec|09d2ab67e91457c966eb335b361bdc4d27ece2d4dea681d22e5d8307e0e0c023|g' ../tensorflow/tensorflow/workspace2.bzl +sed -i -e 's|pybind11-2.10.0|pybind11-a7b91e33269ab6f3f90167291af2c4179fc878f5|g' ../tensorflow/tensorflow/workspace2.bzl # Inspired by part of https://raw.githubusercontent.com/tensorflow/tensorflow/r2.11/third_party/protobuf/protobuf.patch. # This is necessary for Python 3.10. @@ -157,6 +167,7 @@ index 3530a9b37..c31fa8fcc 100644 } EOM +# TODO: Test removing this version pinning. note_build_stage "Set pyparsing to 2.2.2 for CLIF." export PATH="$HOME/.local/bin":$PATH pip3 uninstall -y pyparsing && pip3 install -Iv 'pyparsing==2.2.2' diff --git a/deeptrio/protos/BUILD b/deeptrio/protos/BUILD index dd1e3011..8b58a094 100644 --- a/deeptrio/protos/BUILD +++ b/deeptrio/protos/BUILD @@ -1,5 +1,4 @@ load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library", "py_proto_library") -load("//third_party:clif.bzl", "pyclif_proto_library") package( default_visibility = [ @@ -51,8 +50,3 @@ py_proto_library( "//third_party/nucleus/protos:reads_py_pb2", ], ) - -pyclif_proto_library( - name = "deeptrio_pyclif", - proto_lib = ":deeptrio_proto", # NO COPYBARA -) diff --git a/deepvariant/BUILD b/deepvariant/BUILD index 439c0ce7..83d6f3ac 100644 --- a/deepvariant/BUILD +++ b/deepvariant/BUILD @@ -772,6 +772,7 @@ py_test( ":allele_frequency", ":py_testdata", "//deepvariant/protos:deepvariant_py_pb2", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/io:fasta", "//third_party/nucleus/io:vcf", "//third_party/nucleus/protos:variants_py_pb2", diff --git a/deepvariant/protos/BUILD b/deepvariant/protos/BUILD index 95424213..82bfed3c 100644 --- a/deepvariant/protos/BUILD +++ b/deepvariant/protos/BUILD @@ -1,5 +1,4 @@ load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library", "py_proto_library") -load("//third_party:clif.bzl", "pyclif_proto_library") package( default_visibility = [ @@ -52,11 +51,6 @@ py_proto_library( ], ) -pyclif_proto_library( - name = "deepvariant_pyclif", - proto_lib = ":deepvariant_proto", # NO COPYBARA -) - # ------------------------------------------------------------------------------ # realigner proto # ------------------------------------------------------------------------------ @@ -84,11 +78,6 @@ proto_library( deps = ["//third_party/nucleus/protos:range_proto"], # NO COPYBARA ) -pyclif_proto_library( - name = "realigner_pyclif", - proto_lib = ":realigner_proto", # NO COPYBARA -) - # ------------------------------------------------------------------------------ # resources proto # ------------------------------------------------------------------------------ diff --git a/deepvariant/python/BUILD b/deepvariant/python/BUILD index 13598cc5..2763e2cf 100644 --- a/deepvariant/python/BUILD +++ b/deepvariant/python/BUILD @@ -1,5 +1,5 @@ -load("//third_party:clif.bzl", "py_clif_cc") # Placeholder: load py_test +load("@org_tensorflow//tensorflow:tensorflow.default.bzl", "pybind_extension", "pybind_library") package( default_visibility = [ @@ -7,21 +7,31 @@ package( ], ) -py_clif_cc( +pybind_extension( name = "allelecounter", - srcs = ["allelecounter.clif"], - clif_deps = [ - "//third_party/nucleus/io/python:reference", - ], - pyclif_deps = [ - "//third_party/nucleus/protos:reads_pyclif", - "//third_party/nucleus/protos:range_pyclif", - "//third_party/nucleus/protos:cigar_pyclif", - "//deepvariant/protos:deepvariant_pyclif", + srcs = ["allelecounter_pybind.cc"], + deps = [ + "//deepvariant:allelecounter", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:reference", + "//third_party/nucleus/protos:cigar_cc_pb2", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "allelecounter_cclib", + srcs = ["allelecounter_pybind.cc"], deps = [ "//deepvariant:allelecounter", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:reference", + "//third_party/nucleus/protos:cigar_cc_pb2", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -48,29 +58,58 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "postprocess_variants", - srcs = ["postprocess_variants.clif"], - pyclif_deps = [ - "//third_party/nucleus/protos:range_pyclif", - "//third_party/nucleus/protos:reference_pyclif", - "//deepvariant/protos:deepvariant_pyclif", + srcs = ["postprocess_variants_pybind.cc"], + deps = [ + "//deepvariant:postprocess_variants_lib", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/protos:range_cc_pb2", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", + ], +) + +pybind_library( + name = "postprocess_variants_cclib", + srcs = ["postprocess_variants_pybind.cc"], + deps = [ + "//deepvariant:postprocess_variants_lib", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/protos:range_cc_pb2", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], - deps = ["//deepvariant:postprocess_variants_lib"], ) -py_clif_cc( +pybind_extension( name = "variant_calling", - srcs = ["variant_calling.clif"], - clif_deps = [ - ":allelecounter", - "//third_party/nucleus/io/python:reference", # other py_clif_cc rules - "//third_party/nucleus/io/python:vcf_reader", + srcs = ["variant_calling_pybind.cc"], + deps = [ + "//deepvariant:variant_calling", + "//deepvariant/protos:deepvariant_cc_pb2", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:vcf_reader", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], - pyclif_deps = [ - "//deepvariant/protos:deepvariant_pyclif", +) + +pybind_library( + name = "variant_calling_cclib", + srcs = ["variant_calling_pybind.cc"], + deps = [ + "//deepvariant:variant_calling", + "//deepvariant/protos:deepvariant_cc_pb2", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:vcf_reader", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], - deps = ["//deepvariant:variant_calling"], ) py_test( @@ -94,57 +133,79 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "direct_phasing", - srcs = ["direct_phasing.clif"], + srcs = ["direct_phasing_pybind.cc"], + deps = [ + "//deepvariant:direct_phasing", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", + ], +) + +pybind_library( + name = "direct_phasing_cclib", + srcs = ["direct_phasing_pybind.cc"], deps = [ "//deepvariant:direct_phasing", - "//deepvariant/protos:deepvariant_pyclif", - "//third_party/nucleus/protos:reads_pyclif", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) -py_clif_cc( +pybind_extension( name = "pileup_image_native", - srcs = ["pileup_image_native.clif"], + srcs = ["pileup_image_native_pybind.cc"], deps = [ - ":clif_converters", "//deepvariant:pileup_image_native", - "//deepvariant/protos:deepvariant_pyclif", - "//third_party/nucleus/protos:reads_pyclif", - "//third_party/nucleus/util:proto_clif_converter", + "//deepvariant/protos:deepvariant_cc_pb2", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) -cc_library( - name = "clif_converters", - srcs = ["clif_converters.cc"], - hdrs = ["clif_converters.h"], +pybind_library( + name = "pileup_image_native_cclib", + srcs = ["pileup_image_native_pybind.cc"], deps = [ - "//deepvariant:pileup_channel_lib", "//deepvariant:pileup_image_native", "//deepvariant/protos:deepvariant_cc_pb2", - "@clif//:cpp_runtime", - "@com_google_absl//absl/log:check", - "@local_config_python//:numpy_headers", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) -py_clif_cc( +pybind_extension( name = "variant_calling_multisample", - srcs = ["variant_calling_multisample.clif"], - clif_deps = [ - ":allelecounter", - "//third_party/nucleus/io/python:reference", # other py_clif_cc rules - "//third_party/nucleus/io/python:vcf_reader", - ], - pyclif_deps = [ - "//deepvariant/protos:deepvariant_pyclif", + srcs = ["variant_calling_multisample_pybind.cc"], + deps = [ + "//deepvariant:variant_calling_multisample", + "//deepvariant/protos:deepvariant_cc_pb2", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "variant_calling_multisample_cclib", + srcs = ["variant_calling_multisample_pybind.cc"], deps = [ "//deepvariant:variant_calling_multisample", - "//third_party/nucleus/util:proto_clif_converter", + "//deepvariant/protos:deepvariant_cc_pb2", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -169,19 +230,26 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "make_examples_native", - srcs = ["make_examples_native.clif"], - clif_deps = [ - ], - pyclif_deps = [ - "//third_party/nucleus/protos:range_pyclif", - "//third_party/nucleus/protos:variants_pyclif", - "//deepvariant/protos:deepvariant_pyclif", + srcs = ["make_examples_native_pybind.cc"], + deps = [ + "//deepvariant:make_examples_native", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "make_examples_native_cclib", + srcs = ["make_examples_native_pybind.cc"], deps = [ "//deepvariant:make_examples_native", - "//third_party/nucleus/protos:reads_pyclif", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) diff --git a/deepvariant/python/allelecounter.clif b/deepvariant/python/allelecounter.clif deleted file mode 100644 index 865758c4..00000000 --- a/deepvariant/python/allelecounter.clif +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2017 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "deepvariant/protos/deepvariant_pyclif.h" import * -from "third_party/nucleus/io/python/reference.h" import * -from "third_party/nucleus/protos/range_pyclif.h" import * -from "third_party/nucleus/protos/reads_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/protos/cigar_pyclif.h" import * - -from "deepvariant/allelecounter.h": - namespace `learning::genomics::deepvariant`: - class AlleleCounter: - def __init__(self, - ref: GenomeReference, - interval: Range, - candidate_positions: list, - options: AlleleCounterOptions) - @add__init__ - def Default(self, - ref: GenomeReference, - interval: Range, - reads_interval: Range, - candidate_positions: list, - options: AlleleCounterOptions) - def `AddPython` as add(self, read: ConstProtoPtr, sample: str) - def `NormalizeAndAddPython` as normalize_and_add(self, read: ConstProtoPtr, sample: str) -> (cigar: list, shift: int) - def `Counts` as counts(self) -> list - def `SummaryCounts` as summary_counts(self, left_padding: int = default, right_padding: int = default) -> list diff --git a/deepvariant/python/allelecounter_pybind.cc b/deepvariant/python/allelecounter_pybind.cc index 4421c26d..aad759ea 100644 --- a/deepvariant/python/allelecounter_pybind.cc +++ b/deepvariant/python/allelecounter_pybind.cc @@ -29,6 +29,9 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include + +#include "third_party/nucleus/io/reference.h" #if true // Trick to stop tooling from moving the #include around. // MUST appear before any standard headers are included. #include @@ -41,7 +44,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/protos/cigar.pb.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; @@ -65,7 +68,7 @@ PYBIND11_MODULE(allelecounter, m) { const nucleus::genomics::v1::Range&, const std::vector&, const AlleleCounterOptions&>()) .def("add", &AlleleCounter::AddPython, py::arg("read"), py::arg("sample")) - .def("NormalizeAndAddPython", + .def("normalize_and_add", [](AlleleCounter* self, ::nucleus::ConstProtoPtr wrapped, diff --git a/deepvariant/python/direct_phasing.clif b/deepvariant/python/direct_phasing.clif deleted file mode 100644 index 050e6edf..00000000 --- a/deepvariant/python/direct_phasing.clif +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2021 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "deepvariant/protos/deepvariant_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/protos/reads_pyclif.h" import * - -from "deepvariant/direct_phasing.h": - namespace `learning::genomics::deepvariant`: - class PhasedVariant: - position: int - phase_1_bases: str - phase_2_bases: str - - class DirectPhasing: - def __init__(self) - def `GraphViz` as graphviz(self) -> str - def `GetPhasedVariants` as get_phased_variants(self) -> list - def `PhaseReadsPython` as phase(self, candidates: list, - reads: list>) -> list diff --git a/deepvariant/python/direct_phasing_pybind.cc b/deepvariant/python/direct_phasing_pybind.cc index 7abbae87..c5c5d261 100644 --- a/deepvariant/python/direct_phasing_pybind.cc +++ b/deepvariant/python/direct_phasing_pybind.cc @@ -29,7 +29,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#include "third_party/pybind11/include/pybind11/cast.h" +#include "pybind11/cast.h" #if true // Trick to stop tooling from moving the #include around. // MUST appear before any standard headers are included. #include @@ -41,7 +41,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_status.h" #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/deepvariant/python/make_examples_native.clif b/deepvariant/python/make_examples_native.clif deleted file mode 100644 index d8b93f11..00000000 --- a/deepvariant/python/make_examples_native.clif +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2024 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/protos/reads_pyclif.h" import * -from "deepvariant/protos/deepvariant_pyclif.h" import * -from "third_party/nucleus/protos/range_pyclif.h" import * -from "third_party/nucleus/protos/variants_pyclif.h" import * - -from "deepvariant/make_examples_native.h": - namespace `learning::genomics::deepvariant`: - class VariantLabel: - def __init__(self, is_confident: bool, variant: Variant, genotype: list, is_denovo: bool) - is_confident: bool - variant: Variant - genotype: list - is_denovo : bool - - class CustomizedClassesLabel(VariantLabel): - def __init__(self, is_confident: bool, variant: Variant, truth_variant: Variant, classes_dict: dict, info_field_name: str) - is_confident: bool - variant: Variant - truth_variant: Variant - classes_dict: dict - info_field_name: str - - class ExamplesGenerator: - def __init__(self, - options: MakeExamplesOptions, - example_filenames: dict, - test_mode: bool = default) - def `WriteExamplesInRegion` as write_examples_in_region( - self, - candidates: list>, - reads: list>>, - sample_order: list, - role: str, - mean_coverage_per_sample: list) -> (n_stats: dict, image_shape: list) - - def `AppendLabel` as append_label(self, label: VariantLabel) - - def `SignalShardFinished` as signal_shard_finished(self) diff --git a/deepvariant/python/make_examples_native_pybind.cc b/deepvariant/python/make_examples_native_pybind.cc index 9ff8eeea..a40a2ec7 100644 --- a/deepvariant/python/make_examples_native_pybind.cc +++ b/deepvariant/python/make_examples_native_pybind.cc @@ -40,7 +40,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_status.h" #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/deepvariant/python/pileup_image_native.clif b/deepvariant/python/pileup_image_native.clif deleted file mode 100644 index 6ab248a8..00000000 --- a/deepvariant/python/pileup_image_native.clif +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2017 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/reads_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "deepvariant/protos/deepvariant_pyclif.h" import * -from "deepvariant/python/clif_converters.h" import * - -from "deepvariant/pileup_image_native.h": - namespace `learning::genomics::deepvariant`: - class PileupImageEncoderNative: - - def __init__(self, options: PileupImageOptions) - - def `AllChannelsEnum` as all_channels_enum( - self, alt_aligned_pileup: str) -> list - - def `BuildPileupForOneSamplePython` as build_pileup_for_one_sample( - self, - dv_call: ConstProtoPtr, - ref_bases: str, - reads: list>, - image_start_pos: int, - alt_alleles: list, - sample_options: SampleOptions) -> list - - def `EncodeReadPython` as encode_read( - self, - dv_call: ConstProtoPtr, - ref_bases: str, - read: ConstProtoPtr, - image_start_pos: int, - alt_alleles: list) -> ImageRow - - def `EncodeReference` as encode_reference( - self, ref_bases: str) -> ImageRow - diff --git a/deepvariant/python/pileup_image_native_pybind.cc b/deepvariant/python/pileup_image_native_pybind.cc index 25a6ffc4..6f52ac3b 100644 --- a/deepvariant/python/pileup_image_native_pybind.cc +++ b/deepvariant/python/pileup_image_native_pybind.cc @@ -42,7 +42,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_status.h" #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/deepvariant/python/postprocess_variants.clif b/deepvariant/python/postprocess_variants.clif deleted file mode 100644 index 59788cf0..00000000 --- a/deepvariant/python/postprocess_variants.clif +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2017 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/reference_pyclif.h" import * -from "third_party/nucleus/protos/range_pyclif.h" import * - -from "deepvariant/postprocess_variants.h": - namespace `learning::genomics::deepvariant`: - def `ProcessSingleSiteCallTfRecords` as process_single_sites_tfrecords( - contigs: list, tfrecord_paths: list, - output_tfrecord_path: str, - partitions: list) -> int diff --git a/deepvariant/python/postprocess_variants_pybind.cc b/deepvariant/python/postprocess_variants_pybind.cc index 09b1442b..91542fa0 100644 --- a/deepvariant/python/postprocess_variants_pybind.cc +++ b/deepvariant/python/postprocess_variants_pybind.cc @@ -41,7 +41,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/protos/range.pb.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/deepvariant/python/variant_calling.clif b/deepvariant/python/variant_calling.clif deleted file mode 100644 index b3975989..00000000 --- a/deepvariant/python/variant_calling.clif +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2017 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "deepvariant/protos/deepvariant_pyclif.h" import * -from "deepvariant/python/allelecounter.h" import * -from "third_party/nucleus/io/python/vcf_reader.h" import * - -from "deepvariant/variant_calling.h": - namespace `learning::genomics::deepvariant::vcf_candidate_importer`: - class VariantCaller: - def __init__(self, options: VariantCallerOptions) - def `CallsFromAlleleCounter` as calls_from_allele_counter( - self, allele_counter: AlleleCounter) -> list - def `CallsFromVcf` as calls_from_vcf( - self, allele_counter: AlleleCounter, vcf_reader: VcfReader) -> list - def `CallPositionsFromVcf` as call_positions_from_vcf( - self, allele_counter: AlleleCounter, vcf_reader: VcfReader) -> list diff --git a/deepvariant/python/variant_calling_multisample.clif b/deepvariant/python/variant_calling_multisample.clif deleted file mode 100644 index fe82b6f8..00000000 --- a/deepvariant/python/variant_calling_multisample.clif +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2017 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "deepvariant/protos/deepvariant_pyclif.h" import * -from "deepvariant/python/allelecounter.h" import * -from "third_party/nucleus/io/python/vcf_reader.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * - -from "deepvariant/variant_calling_multisample.h": - namespace `learning::genomics::deepvariant::multi_sample`: - class VariantCaller: - def __init__(self, options: VariantCallerOptions) - def `CallsFromAlleleCounts` as calls_from_allele_counts( - self, allele_counters: dict, target_sample: str) -> list - def `CallPositionsFromAlleleCounts` as call_positions_from_allele_counts( - self, allele_counters: dict, target_sample: str) -> list diff --git a/deepvariant/python/variant_calling_multisample_pybind.cc b/deepvariant/python/variant_calling_multisample_pybind.cc index 0e2dcc89..fc8e60da 100644 --- a/deepvariant/python/variant_calling_multisample_pybind.cc +++ b/deepvariant/python/variant_calling_multisample_pybind.cc @@ -36,11 +36,12 @@ #include +#include "deepvariant/protos/deepvariant.pb.h" #include "deepvariant/variant_calling_multisample.h" #include "third_party/nucleus/core/python/type_caster_nucleus_status.h" #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/deepvariant/python/variant_calling_pybind.cc b/deepvariant/python/variant_calling_pybind.cc index 648ad635..b05cc58a 100644 --- a/deepvariant/python/variant_calling_pybind.cc +++ b/deepvariant/python/variant_calling_pybind.cc @@ -36,12 +36,13 @@ #include +#include "deepvariant/protos/deepvariant.pb.h" #include "deepvariant/variant_calling.h" #include "third_party/nucleus/core/python/type_caster_nucleus_status.h" #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/vcf_reader.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/deepvariant/realigner/BUILD b/deepvariant/realigner/BUILD index 8f3faff2..8dd95a9c 100644 --- a/deepvariant/realigner/BUILD +++ b/deepvariant/realigner/BUILD @@ -65,6 +65,7 @@ py_library( name = "window_selector", srcs = ["window_selector.py"], deps = [ + "//deepvariant/protos:deepvariant_py_pb2", "//deepvariant/protos:realigner_py_pb2", "//deepvariant/python:allelecounter", "//deepvariant/realigner/python:window_selector", diff --git a/deepvariant/realigner/python/BUILD b/deepvariant/realigner/python/BUILD index 4d4ad7a2..2016988e 100644 --- a/deepvariant/realigner/python/BUILD +++ b/deepvariant/realigner/python/BUILD @@ -1,5 +1,5 @@ # Placeholder: load py_test -load("//third_party:clif.bzl", "py_clif_cc") +load("@org_tensorflow//tensorflow:tensorflow.default.bzl", "pybind_extension", "pybind_library") package( default_visibility = [ @@ -7,28 +7,51 @@ package( ], ) -py_clif_cc( +pybind_extension( name = "fast_pass_aligner", - srcs = ["fast_pass_aligner.clif"], - py_deps = [], - pyclif_deps = [ - "//third_party/nucleus/protos:reads_pyclif", - "//deepvariant/protos:realigner_pyclif", + srcs = ["fast_pass_aligner_pybind.cc"], + deps = [ + "//deepvariant/realigner:fast_pass_aligner", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", + ], +) + +pybind_library( + name = "fast_pass_aligner_cclib", + srcs = ["fast_pass_aligner_pybind.cc"], + deps = [ + "//deepvariant/realigner:fast_pass_aligner", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], - deps = ["//deepvariant/realigner:fast_pass_aligner"], ) -py_clif_cc( +pybind_extension( name = "debruijn_graph", - srcs = ["debruijn_graph.clif"], - py_deps = [], - pyclif_deps = [ - "//deepvariant/protos:realigner_pyclif", - "//third_party/nucleus/protos:reads_pyclif", + srcs = ["debruijn_graph_pybind.cc"], + deps = [ + "//deepvariant/realigner:debruijn_graph", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "debruijn_graph_cclib", + srcs = ["debruijn_graph_pybind.cc"], deps = [ "//deepvariant/realigner:debruijn_graph", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -52,11 +75,28 @@ py_test( ) # CLIF wrap for the SSW C++ interface. -py_clif_cc( +pybind_extension( name = "ssw", - srcs = ["ssw.clif"], - py_deps = ["//third_party/nucleus/io:clif_postproc"], - deps = ["//deepvariant/realigner:ssw"], + srcs = ["ssw_pybind.cc"], + deps = [ + "//deepvariant/realigner:ssw", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", + ], +) + +pybind_library( + name = "ssw_cclib", + srcs = ["ssw_pybind.cc"], + deps = [ + "//deepvariant/realigner:ssw", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", + ], ) py_test( @@ -66,6 +106,7 @@ py_test( python_version = "PY3", deps = [ ":ssw", + "//third_party/nucleus/io:clif_postproc", "@absl_py//absl/testing:absltest", ], ) @@ -77,20 +118,31 @@ py_test( python_version = "PY3", deps = [ ":ssw", + "//third_party/nucleus/io:clif_postproc", "@absl_py//absl/testing:absltest", ], ) -py_clif_cc( +pybind_extension( name = "window_selector", - srcs = ["window_selector.clif"], - clif_deps = [ - "//deepvariant/python:allelecounter", + srcs = ["window_selector_pybind.cc"], + deps = [ + "//deepvariant/realigner:cpp_window_selector", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], - py_deps = [], - pyclif_deps = [ - "//deepvariant/protos:deepvariant_pyclif", - "//deepvariant/protos:realigner_pyclif", +) + +pybind_library( + name = "window_selector_cclib", + srcs = ["window_selector_pybind.cc"], + deps = [ + "//deepvariant/realigner:cpp_window_selector", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], - deps = ["//deepvariant/realigner:cpp_window_selector"], ) diff --git a/deepvariant/realigner/python/debruijn_graph.clif b/deepvariant/realigner/python/debruijn_graph.clif deleted file mode 100644 index 0dab99dd..00000000 --- a/deepvariant/realigner/python/debruijn_graph.clif +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2017 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "deepvariant/protos/realigner_pyclif.h" import * -from "third_party/nucleus/protos/reads_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * - -from "deepvariant/realigner/debruijn_graph.h": - namespace `learning::genomics::deepvariant`: - class DeBruijnGraph: - def `GraphViz` as graphviz(self) -> str - def `CandidateHaplotypes` as candidate_haplotypes(self) -> list - kmer_size: int = property(`KmerSize`) - - staticmethods from `DeBruijnGraph`: - def `Build` as build(ref: str, - reads: list>, - options: DeBruijnGraphOptions) - -> DeBruijnGraph diff --git a/deepvariant/realigner/python/debruijn_graph_pybind.cc b/deepvariant/realigner/python/debruijn_graph_pybind.cc index 72b9bb76..654a17ee 100644 --- a/deepvariant/realigner/python/debruijn_graph_pybind.cc +++ b/deepvariant/realigner/python/debruijn_graph_pybind.cc @@ -40,7 +40,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_status.h" #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/deepvariant/realigner/python/fast_pass_aligner.clif b/deepvariant/realigner/python/fast_pass_aligner.clif deleted file mode 100644 index a8308982..00000000 --- a/deepvariant/realigner/python/fast_pass_aligner.clif +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "deepvariant/protos/realigner_pyclif.h" import * -from "third_party/nucleus/protos/reads_pyclif.h" import * - -from "deepvariant/realigner/fast_pass_aligner.h": - namespace `learning::genomics::deepvariant`: - class FastPassAligner: - def `set_reference` as set_reference(self, ref: bytes) - def `set_ref_start` as set_ref_start(self, chr: bytes, pos: int) - def `set_haplotypes` as set_haplotypes(self, hyplotypes:list) - def `set_options` as set_options(self, options:AlignerOptions) - def `set_is_debug` as set_is_debug(self, isDebug: bool) - def `set_normalize_reads` as set_normalize_reads(self, - normalizeReads: bool) - def `set_debug_read_id` as set_debug_read_id(self, readId: int) - def `set_ref_prefix_len` as set_ref_prefix_len(self, ref_prefix_len: int) - def `set_ref_suffix_len` as set_ref_suffix_len(self, set_ref_suffix_len: int) - def `AlignReads` as realign_reads(self, reads:list) -> list diff --git a/deepvariant/realigner/python/fast_pass_aligner_pybind.cc b/deepvariant/realigner/python/fast_pass_aligner_pybind.cc index 2163683e..10eb7ecd 100644 --- a/deepvariant/realigner/python/fast_pass_aligner_pybind.cc +++ b/deepvariant/realigner/python/fast_pass_aligner_pybind.cc @@ -40,7 +40,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_status.h" #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; PYBIND11_MODULE(fast_pass_aligner, m) { diff --git a/deepvariant/realigner/python/ssw.clif b/deepvariant/realigner/python/ssw.clif deleted file mode 100644 index 7221d8c1..00000000 --- a/deepvariant/realigner/python/ssw.clif +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2017 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from third_party.nucleus.io.clif_postproc import ValueErrorOnInaccurate - -from "deepvariant/realigner/ssw.h": - - namespace `learning::genomics::deepvariant`: - - class Alignment: - # The best alignment score - sw_score: int - sw_score_next_best: int # The next best alignment score - ref_begin: int # Reference begin position of the best alignment - ref_end: int # Reference end position of the best alignment - query_begin: int # Query begin position of the best alignment - query_end: int # Query end position of the best alignment - ref_end_next_best: int # Reference end position of next best alignment - mismatches: int # Number of mismatches of the alignment - cigar_string: bytes # Cigar string of the best alignment - - - # TODO: make use of score_filter. - class Filter: - report_begin_position: bool # Give ref_begin and query_begin. - # If it is not set, ref_begin and - # query_begin are -1. - report_cigar: bool # Give cigar_string and cigar. - # report_begin_position is automatically - # TRUE. - - # When *report_cigar* is true and alignment passes these two filters, - # cigar_string and cigar will be given. - score_filter: int # score >= score_filter - distance_filter: int # ((ref_end - ref_begin) < distance_filter) && - # ((query_end - read_begin) < distance_filter) - - # Wrapper for the constructor with arguments. - @add__init__ - def `ConstructFromValues` as construct(self, pos: bool, cigar: bool, - score: int, dis: int) - - - # Wrap for a *subset* of the functionality in the Aligner class. In - # particular, we haven't wrapped support for custom score matrices or - # non-ATGCN alphabets, nor the "rebuild" functions. - class Aligner: - - # Wrap for the constructor with arguments. - @add__init__ - def `ConstructWithParameters` as construct(self, - match_score: int, - mismatch_penalty: int, - gap_opening_penalty: int, - gap_extending_penalty: int) - - def `SetReferenceSequence` as set_reference_sequence(self, - seq: bytes) -> int - - # Align query against previously set reference sequence. - # Prefer this method if aligning many queries to a single reference. - def `Align` as align(self, query: bytes, filter: Filter, mask_len: int) - -> (accuracy: int, alignment: Alignment): - return ValueErrorOnInaccurate(...) diff --git a/deepvariant/realigner/python/ssw_pybind.cc b/deepvariant/realigner/python/ssw_pybind.cc index 1ffdb41c..b13fa418 100644 --- a/deepvariant/realigner/python/ssw_pybind.cc +++ b/deepvariant/realigner/python/ssw_pybind.cc @@ -40,7 +40,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_status.h" #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/deepvariant/realigner/python/window_selector.clif b/deepvariant/realigner/python/window_selector.clif deleted file mode 100644 index 57038f05..00000000 --- a/deepvariant/realigner/python/window_selector.clif +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2017 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "deepvariant/python/allelecounter.h" import * -from "deepvariant/protos/realigner_pyclif.h" import * - -from "deepvariant/realigner/window_selector.h": - namespace `learning::genomics::deepvariant`: - - def `VariantReadsWindowSelectorCandidates` as variant_reads_candidates_from_allele_counter(counter: AlleleCounter) - -> list - def `AlleleCountLinearWindowSelectorCandidates` as allele_count_linear_candidates_from_allele_counter(counter: AlleleCounter, config: WindowSelectorModel.AlleleCountLinearModel) - -> list diff --git a/deepvariant/realigner/python/window_selector_pybind.cc b/deepvariant/realigner/python/window_selector_pybind.cc index 09899e2f..14ced7ee 100644 --- a/deepvariant/realigner/python/window_selector_pybind.cc +++ b/deepvariant/realigner/python/window_selector_pybind.cc @@ -40,7 +40,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_status.h" #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/clif.bzl b/third_party/clif.bzl deleted file mode 100644 index 0380dcba..00000000 --- a/third_party/clif.bzl +++ /dev/null @@ -1,454 +0,0 @@ -"""Provide bazel rules for OSS CLIF.""" - -# Label for our OSS CLIF binary pyclif. -CLIF_PYCLIF = "@clif//:pyclif" - -# Label for our OSS CLIF protobuf compiler. -CLIF_PROTO = "@clif//:proto" - -# Label for our OSS CLIF C++ runtime headers and sources. -CLIF_CPP_RUNTIME = "@clif//:cpp_runtime" - -# The CLIF generated code only compiles with C++17. -EXTRA_CC_FLAGS = ["-std=c++17"] - -_PROTO_LIBRARY_SUFFIX = "_pyclif" - -PYCLIF_PYEXT_SUFFIX = ".so" - -PYCLIF_CC_LIB_SUFFIX = "_cclib" - -PYCLIF_WRAP_SUFFIX = "_clif_wrap" - -def _get_transitive_headers(hdrs, deps): - """Obtain the header files for a target and its transitive dependencies. - - Args: - hdrs: a list of header files - deps: a list of targets that are direct dependencies - - Returns: - a collection of the transitive headers - """ - return depset( - hdrs, - transitive = [dep[CcInfo].compilation_context.headers for dep in deps], - ) - -def _clif_wrap_cc_impl(ctx): - """Executes CLIF cmdline tool to produce C++ python model from a CLIF spec.""" - if len(ctx.files.srcs) != 1: - fail("Exactly one CLIF source file label must be specified.", "srcs") - - clif_spec_file = ctx.files.srcs[0] - - # Inputs is a set of all of the things we depend on, not inputs to the CLIF - # program itself. - initial_inputs = [clif_spec_file] - initial_inputs += ctx.files._cliflib - initial_inputs += ctx.files.clif_deps - initial_inputs += ctx.files.toolchain_deps - inputs = _get_transitive_headers(initial_inputs, ctx.attr.deps) - - # Compute the set of include directories for CLIF so it can find header files - # used in the CLIF specification. These are the repo roots for all of our - # inputs (aka deps) plus all of the quote and system includes for our C++ - # deps. - include_dirs = depset( - _get_repository_roots(ctx, inputs), - transitive = [ - dep[CcInfo].compilation_context.quote_includes - for dep in ctx.attr.deps - ] + [ - dep[CcInfo].compilation_context.system_includes - for dep in ctx.attr.deps - ], - ) - - # Construct our arguments for CLIF. - args = [ - "--modname", - ctx.attr.package_name + "." + ctx.attr.module_name, - "-c", - ctx.outputs.cc_out.path, - "-g", - ctx.outputs.h_out.path, - "-i", - ctx.outputs.ccinit_out.path, - "--prepend", - "clif/python/types.h", - ] - include_args = ["-I" + i for i in include_dirs.to_list()] - - # Add these includes to CLIF itself. - args += include_args - - # Add these includes to those passed through by CLIF to its C++ matcher. - args += ["-f" + " ".join(include_args + EXTRA_CC_FLAGS)] - - # The last argument is the actual CLIF specification file. - args += [clif_spec_file.path] - - outputs = [ctx.outputs.cc_out, ctx.outputs.h_out, ctx.outputs.ccinit_out] - ctx.actions.run( - executable = ctx.executable._clif, - arguments = args, - inputs = inputs.to_list(), - outputs = outputs, - mnemonic = "CLIF", - progress_message = "CLIF wrapping " + clif_spec_file.path, - ) - -_clif_wrap_cc = rule( - attrs = { - "srcs": attr.label_list( - mandatory = True, - allow_files = True, - ), - "deps": attr.label_list( - allow_files = True, - providers = [CcInfo], - ), - "toolchain_deps": attr.label_list( - allow_files = True, - ), - # For rule "//foo/python:bar_clif" this should be "bar". - "module_name": attr.string(mandatory = True), - # For rule "//foo/python:bar_clif" this should be "foo/python". - "package_name": attr.string(mandatory = True), - "clif_deps": attr.label_list(allow_files = True), - # Hidden attribute: the Label for our PYCLIF binary itself. - "_clif": attr.label( - default = Label(CLIF_PYCLIF), - executable = True, - cfg = "exec", - ), - # Hidden attribute: The label to the C++ CLIF header files. - "_cliflib": attr.label( - default = Label(CLIF_CPP_RUNTIME), - allow_files = True, - ), - }, - outputs = { - "cc_out": "%{module_name}.cc", - "h_out": "%{module_name}.h", - "ccinit_out": "%{module_name}_init.cc", - }, - implementation = _clif_wrap_cc_impl, -) - -def _get_repository_roots(ctx, files): - """Returns abnormal root directories under which files reside. - - When running a ctx.action, source files within the main repository are all - relative to the current directory; however, files that are generated or exist - in remote repositories will have their root directory be a subdirectory, - e.g. bazel-out/local-fastbuild/genfiles/external/jpeg_archive. This function - returns the set of these devious directories, ranked and sorted by popularity - in order to hopefully minimize the number of I/O system calls within the - compiler, because includes have quadratic complexity. - - Args: - ctx: context - files: list of paths - Returns: - list of directories - """ - ctx = ctx # unused - result = {} - for f in files.to_list(): - root = f.root.path - if root: - if root not in result: - result[root] = 0 - result[root] -= 1 - work = f.owner.workspace_root - if work: - if root: - root += "/" - root += work - if root: - if root not in result: - result[root] = 0 - result[root] -= 1 - return [k for v, k in sorted([(v, k) for k, v in result.items()])] - -def _clif_to_lib(label, extension): - """Gets a C++/python/etc library corresponding to a CLIF library rule. - - Args: - label: string. The name of a clif_rule. If the name is of the - form _pyclif we will stripe off the `_pyclif` ending. - extension: string. The expected extension of our name library. - - Returns: - _extension. - """ - if label.endswith(_PROTO_LIBRARY_SUFFIX): - basename = label[:-len(_PROTO_LIBRARY_SUFFIX)] - else: - basename = label - return basename + extension - -def pyclifs_to_pyproto_libs(labels): - """Gets the py protobuf label for each of pyclif label as a list.""" - return [_clif_to_lib(name, "_py_pb2") for name in labels] - -def pyclifs_to_ccproto_libs(labels): - """Gets the cc protobuf label for each of pyclif label as a list.""" - return [_clif_to_lib(name, "_cc_pb2") for name in labels] - -def clif_deps_to_cclibs(labels): - """Gets the cc_library name for each of label as a list.""" - return [_clif_to_lib(name, PYCLIF_CC_LIB_SUFFIX) for name in labels] - -def _symlink_impl(ctx): - """Creates a symbolic link between src and out.""" - out = ctx.outputs.out - src = ctx.attr.src.files.to_list()[0] - cmd = "ln -f -r -s %s %s" % (src.path, out.path) - ctx.actions.run_shell( - inputs = [src], - outputs = [out], - command = cmd, - ) - -symlink = rule( - implementation = _symlink_impl, - attrs = { - "src": attr.label( - mandatory = True, - allow_single_file = True, - ), - "out": attr.output(mandatory = True), - }, -) - -def py_clif_cc( - name, - srcs, - clif_deps = [], - pyclif_deps = [], - deps = [], - copts = [], - py_deps = [], - **kwargs): - """Defines a CLIF wrapper rule making C++ libraries accessible to Python. - - Here are two example working py_clif_cc rules: - - py_clif_cc( - name = "proto_cpp", - srcs = ["proto_cpp.clif"], - pyclif_deps = ["//oss_clif:oss_pyclif"], - deps = ["//oss_clif:proto_cpp_lib"], - ) - - py_clif_cc( - name = "pyclif_dep", - srcs = ["pyclif_dep.clif"], - deps = ["//oss_clif:pyclif_dep_lib"], - ) - - Args: - name: The name of the rule. This name becomes a suitable target for Python - libraries to access the C++ code. - srcs: A list that must contain a single file named .clif containing - our CLIF specification. - clif_deps: A list of other CLIF rules included by this one. - pyclif_deps: A potentially empty list of pyclif_proto_library rules - deps: A list of C++ dependencies. - copts: List of copts to provide to our native.cc_library when building our - python extension module. - py_deps: List of dependencies to provide to our the native.py_library - created by this rule. - **kwargs: kwargs passed to py_library rule created by this rule. - """ - pyext_so = name + PYCLIF_PYEXT_SUFFIX - cc_library_name = name + PYCLIF_CC_LIB_SUFFIX - extended_cc_deps = deps + [CLIF_CPP_RUNTIME] + pyclif_deps - - # Here's a rough outline of how we build our pyclif library: - # - # Suppose we have a module named 'foo'. - # - # _clif_wrap_cc runs pyclif to produce foo.cc, foo.h, and foo_init.cc which - # C++ python extension module. - # - # native.cc_library is a normal C++ library with those sources, effectively - # our "python module" as a bazel C++ library allowing other rules to depend - # on that C++ code. - # - # native.cc_binary depends on foo's cc_library to create a shared python - # extension module (.so) which python will load via its dlopen mechanism. - # This .so library is also used by the _clif_wrap_cc rule to get include paths - # when building CLIF specs depending on other clif specs. - # - # native.py_library named `name` which provides a python bazel target that - # loads the cc_binary, as data, producing a py extension module. This also - # allows client python code to depend on this module. - - # This _clif_wrap_cc handles clif_deps differently than most clif.bzl's - # do. That is because we are doing something quite different than the - # standard clif.bzl does -- we are replacing the generated extension module - # with a symbolic link to protobuf's _message.so. That file, in turn, is - # a dependency of every single Python protocol buffer target. And it needs - # to depend on all of the _cclibs generated by the cc_library rule below. - # - # So because of all this, the normal strategy of having this rule depend - # on clif_deps or clif_deps_to_pyexts(clif_deps) would create circular - # dependencies. Our strategy to avoid those circular dependencies is to - # have anything the cc_library rule depends on be only C++ and not Python. - # The raw clif_deps are a mixture of C++ and Python, so we have to very - # carefully depend on only their C++ part (given by the - # clif_deps_to_cclibs). - _clif_wrap_cc( - name = name + PYCLIF_WRAP_SUFFIX, - srcs = srcs, - deps = extended_cc_deps + clif_deps_to_cclibs(clif_deps), - clif_deps = clif_deps_to_cclibs(clif_deps), - toolchain_deps = ["@bazel_tools//tools/cpp:current_cc_toolchain"], - module_name = name, - # Turns //foo/bar:baz_pyclif into foo.bar to create our fully-qualified - # python package name. - package_name = native.package_name().replace("/", "."), - ) - - native.cc_library( - name = cc_library_name, - hdrs = [ - name + ".h", - ], - srcs = [ - name + ".cc", - name + "_init.cc", - ], - copts = copts + EXTRA_CC_FLAGS, - deps = extended_cc_deps + clif_deps_to_cclibs(clif_deps), - ) - - # To prevent ODR violations, all of the extensions must live in one - # extension module. And to be compatible with existing protobuf - # generated code, that module must be _message.so. - symlink( - name = name + "_symlink", - out = pyext_so, - src = "@com_google_protobuf//:python/google/protobuf/pyext/_message.so", - ) - - # We create our python module which is just a thin wrapper around our real - # python module pyext_so (producing name.so for python to load). This - # rule allows python code to depend on this module, even through its written - # in C++. - native.py_library( - name = name, - srcs = [], - srcs_version = "PY3", - deps = pyclifs_to_pyproto_libs(pyclif_deps) + clif_deps + py_deps, - data = [pyext_so], - **kwargs - ) - -# Copied from: devtools/clif/python/clif_build_rule.bzl with heavy -# modifications. - -def _clif_proto_parser_rule_impl(ctx): - """Implementation of _run_clif_proto_parser_rule.""" - proto_file = ctx.files.src[0] - args = [ - "-c", - ctx.outputs.cc.path, - "-h", - ctx.outputs.hdr.path, - "--strip_dir=%s" % ctx.configuration.genfiles_dir.path, - "--source_dir='.'", - "%s" % proto_file.path, - ] - inputs = [] - for d in ctx.attr.deps: - if "proto" in dir(d): - inputs += list(d[ProtoInfo].transitive_sources) - ctx.actions.run( - mnemonic = "ClifProtoLibraryGeneration", - arguments = args, - executable = ctx.executable.parser, - inputs = inputs, - outputs = [ctx.outputs.hdr, ctx.outputs.cc], - ) - -_run_clif_proto_parser_rule = rule( - attrs = { - "src": attr.label(allow_files = [".proto"]), - "hdr": attr.output(), - "cc": attr.output(), - "deps": attr.label_list(), - "parser": attr.label( - executable = True, - default = Label(CLIF_PROTO), - cfg = "exec", - ), - }, - implementation = _clif_proto_parser_rule_impl, -) - -def pyclif_proto_library( - name, - proto_lib, - proto_srcfile = "", - deps = [], - visibility = None, - compatible_with = None, - testonly = None): - """Generate C++ CLIF extension for using a proto and dependent py_proto_lib. - - Args: - name: generated cc_library (name.h) to use in py_clif_cc clif_deps - proto_lib: name of a proto_library rule - proto_srcfile: the proto name if it does not match proto_lib rule name - deps: passed to cc_library - visibility: passed to all generated "files": name.h name.a name_pb2.py - compatible_with: compatibility list - testonly: available for test rules only flag (default from package) - """ - if not name.endswith(_PROTO_LIBRARY_SUFFIX): - fail("The name of the 'pyclif_proto_library' target should be of the " + - "form '%s' where the proto " % _PROTO_LIBRARY_SUFFIX + - "file being wrapped has the name '.proto'.") - if proto_srcfile: - required_name = proto_srcfile[:-len(".proto")] + _PROTO_LIBRARY_SUFFIX - if name != required_name: - fail("The name of the 'pyclif_proto_library' target should be " + - "'%s' as it is wrapping %s." % (required_name, proto_srcfile)) - - hdr_file = name + ".h" - cpp_file = name + ".cc" - clifrule = name + "_clif_rule" - src = name[:-len(_PROTO_LIBRARY_SUFFIX)] + ".proto" - - _run_clif_proto_parser_rule( - name = clifrule, - src = src, - hdr = hdr_file, - cc = cpp_file, - deps = deps + [proto_lib], - testonly = testonly, - ) - - # In OSS world, we cannot provide proto_lib as a direct dependency to our - # cc_library as it doesn't provide a cc file: - # in deps attribute of cc_library rule //oss_clif:oss_pyclif: proto_library - # rule '//oss_clif:oss_proto' is misplaced here (expected cc_inc_library, - # cc_library, objc_library, experimental_objc_library or cc_proto_library). - # So we need to synthesize our protobuf cc library name from our name as - # pyclif_name = proto_pyclif - # cc_proto_lib = proto_cc_pb2 - native.cc_library( - name = name, - srcs = [cpp_file], - hdrs = [hdr_file], - deps = deps + [CLIF_CPP_RUNTIME] + pyclifs_to_ccproto_libs([name]), - visibility = visibility, - compatible_with = compatible_with, - testonly = testonly, - copts = EXTRA_CC_FLAGS, - ) diff --git a/third_party/nucleus/core/BUILD b/third_party/nucleus/core/BUILD index a5a719ff..fdff52c3 100644 --- a/third_party/nucleus/core/BUILD +++ b/third_party/nucleus/core/BUILD @@ -27,7 +27,6 @@ cc_library( name = "vendor_cpp", deps = [ ":statusor", - ":statusor_clif_converters", ":statusor_examples", ], ) @@ -69,17 +68,6 @@ cc_test( ], ) -cc_library( - name = "statusor_clif_converters", - srcs = ["statusor_clif_converters.cc"], - hdrs = ["statusor_clif_converters.h"], - deps = [ - ":statusor", - "//third_party/nucleus/core:status", - "@clif//:cpp_runtime", - ], -) - cc_library( name = "statusor_examples", hdrs = ["statusor_examples.h"], diff --git a/third_party/nucleus/core/python/BUILD b/third_party/nucleus/core/python/BUILD index b39ae0aa..be02bfa7 100644 --- a/third_party/nucleus/core/python/BUILD +++ b/third_party/nucleus/core/python/BUILD @@ -1,5 +1,5 @@ -load("//third_party:clif.bzl", "py_clif_cc") # Placeholder: load py_test +load("@org_tensorflow//tensorflow:tensorflow.default.bzl", "pybind_extension", "pybind_library") package( #default_applicable_licenses = ["//third_party/nucleus:license"], @@ -10,11 +10,40 @@ package( licenses(["notice"]) -py_clif_cc( +pybind_library( + name = "type_caster_nucleus_status", + hdrs = ["type_caster_nucleus_status.h"], + deps = [ + "//third_party/nucleus/core:status", + ], +) + +pybind_library( + name = "type_caster_nucleus_statusor", + hdrs = ["type_caster_nucleus_statusor.h"], + deps = [ + ":type_caster_nucleus_status", + "//third_party/nucleus/core:status", + "//third_party/nucleus/core:statusor", + ], +) + +pybind_extension( name = "statusor_examples", - srcs = ["statusor_examples.clif"], + srcs = ["statusor_examples_pybind.cc"], + deps = [ + ":type_caster_nucleus_status", + ":type_caster_nucleus_statusor", + "//third_party/nucleus/core:statusor_examples", + ], +) + +pybind_library( + name = "statusor_examples_cclib", + srcs = ["statusor_examples_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + ":type_caster_nucleus_status", + ":type_caster_nucleus_statusor", "//third_party/nucleus/core:statusor_examples", ], ) diff --git a/third_party/nucleus/core/python/statusor_examples.clif b/third_party/nucleus/core/python/statusor_examples.clif deleted file mode 100644 index d992359f..00000000 --- a/third_party/nucleus/core/python/statusor_examples.clif +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from "third_party/nucleus/core/statusor_examples.h": - namespace `nucleus`: - def MakeIntOK() -> StatusOr - def MakeIntFail() -> StatusOr - def MakeStrOK() -> StatusOr - # Note: this "stripped" type doesn't work for strings. The test will fail if - # enabled. See internal for more information. - # def MakeStrOKStrippedType() -> str - def MakeStrFail() -> StatusOr - def MakeIntUniquePtrOK() -> StatusOr - def MakeIntUniquePtrFail() -> StatusOr - def MakeIntVectorOK() -> StatusOr> - def MakeIntVectorFail() -> StatusOr> - def FuncReturningStatusOK() -> Status - def FuncReturningStatusFail() -> Status - - class StringOwner: - @classmethod - def Factory(cls) -> StatusOr - - def GetText(self) -> str diff --git a/third_party/nucleus/core/statusor_clif_converters.cc b/third_party/nucleus/core/statusor_clif_converters.cc deleted file mode 100644 index c4420acb..00000000 --- a/third_party/nucleus/core/statusor_clif_converters.cc +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include "third_party/nucleus/core/statusor_clif_converters.h" - -#include - -namespace nucleus { - -PyObject* Clif_PyObjFrom(const nucleus::Status& c, const clif::py::PostConv&) { - if (!c.ok()) { - ::nucleus::internal::ErrorFromStatus(c); - return nullptr; - } else { - Py_RETURN_NONE; - } -} - -} // namespace nucleus - -namespace nucleus { -namespace internal { - -void ErrorFromStatus(const nucleus::Status& status) { - PyErr_SetString(PyExc_ValueError, status.ToString().c_str()); -} - -} // namespace internal -} // namespace nucleus diff --git a/third_party/nucleus/core/statusor_clif_converters.h b/third_party/nucleus/core/statusor_clif_converters.h deleted file mode 100644 index 15c40cad..00000000 --- a/third_party/nucleus/core/statusor_clif_converters.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef THIRD_PARTY_NUCLEUS_VENDOR_STATUSOR_CLIF_CONVERTERS_H_ -#define THIRD_PARTY_NUCLEUS_VENDOR_STATUSOR_CLIF_CONVERTERS_H_ - -#include "clif/python/postconv.h" -#include "clif/python/types.h" -#include "third_party/nucleus/core/status.h" -#include "third_party/nucleus/core/statusor.h" - -// Note: comment below is an instruction to CLIF. -// NOLINTNEXTLINE -// CLIF use `::nucleus::StatusOr` as StatusOr, NumTemplateParameter:1, HasPyObjFromOnly -// CLIF use `::nucleus::Status` as Status, HasPyObjFromOnly - -namespace nucleus { - -PyObject* Clif_PyObjFrom(const nucleus::Status& c, const ::clif::py::PostConv&); - -} // namespace nucleus - -namespace nucleus { -namespace internal { - -void ErrorFromStatus(const ::nucleus::Status& status); - -} // namespace internal - -template -PyObject* Clif_PyObjFrom(const StatusOr& c, const ::clif::py::PostConv& pc) { - if (!c.ok()) { - internal::ErrorFromStatus(c.status()); - return nullptr; - } else { - using ::clif::Clif_PyObjFrom; - return Clif_PyObjFrom(c.ValueOrDie(), pc.Get(0)); - } -} - -template -PyObject* Clif_PyObjFrom(StatusOr&& c, const ::clif::py::PostConv& pc) { - if (!c.ok()) { - internal::ErrorFromStatus(c.status()); - return nullptr; - } else { - using ::clif::Clif_PyObjFrom; - return Clif_PyObjFrom(c.ConsumeValueOrDie(), pc.Get(0)); - } -} - -} // namespace nucleus - -#endif // THIRD_PARTY_NUCLEUS_VENDOR_STATUSOR_CLIF_CONVERTERS_H_ diff --git a/third_party/nucleus/io/BUILD b/third_party/nucleus/io/BUILD index 070e7ad6..00eedd3a 100644 --- a/third_party/nucleus/io/BUILD +++ b/third_party/nucleus/io/BUILD @@ -84,6 +84,7 @@ py_library( deps = [ ":genomics_reader", ":genomics_writer", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/io/python:vcf_reader", "//third_party/nucleus/io/python:vcf_writer", "//third_party/nucleus/protos:variants_py_pb2", @@ -118,6 +119,7 @@ py_library( deps = [ ":genomics_reader", ":genomics_writer", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/io/python:sam_reader", "//third_party/nucleus/io/python:sam_writer", "//third_party/nucleus/protos:reads_py_pb2", @@ -154,6 +156,7 @@ py_library( deps = [ ":genomics_reader", ":genomics_writer", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/io/python:bed_reader", "//third_party/nucleus/io/python:bed_writer", "//third_party/nucleus/protos:bed_py_pb2", @@ -169,6 +172,7 @@ py_test( srcs_version = "PY3", deps = [ ":bed", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/protos:bed_py_pb2", "//third_party/nucleus/testing:py_test_utils", "@absl_py//absl/testing:absltest", @@ -182,6 +186,7 @@ py_library( deps = [ ":genomics_reader", ":genomics_writer", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/io/python:bedgraph_reader", "//third_party/nucleus/io/python:bedgraph_writer", "//third_party/nucleus/protos:bedgraph_py_pb2", @@ -209,6 +214,7 @@ py_library( srcs = ["fasta.py"], deps = [ ":genomics_reader", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/io/python:reference", "//third_party/nucleus/protos:fasta_py_pb2", "//third_party/nucleus/util:ranges", @@ -238,6 +244,7 @@ py_library( deps = [ ":genomics_reader", ":genomics_writer", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/io/python:fastq_reader", "//third_party/nucleus/io/python:fastq_writer", "//third_party/nucleus/protos:fastq_py_pb2", @@ -266,6 +273,7 @@ py_library( deps = [ ":genomics_reader", ":genomics_writer", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/io/python:gff_reader", "//third_party/nucleus/io/python:gff_writer", "//third_party/nucleus/protos:gff_py_pb2", @@ -336,6 +344,7 @@ py_library( name = "tabix", srcs = ["tabix.py"], deps = [ + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/io/python:tabix_indexer", ], ) diff --git a/third_party/nucleus/io/python/BUILD b/third_party/nucleus/io/python/BUILD index 299636c9..2eb9f5e5 100644 --- a/third_party/nucleus/io/python/BUILD +++ b/third_party/nucleus/io/python/BUILD @@ -1,5 +1,5 @@ -load("//third_party:clif.bzl", "py_clif_cc") # Placeholder: load py_test +load("@org_tensorflow//tensorflow:tensorflow.default.bzl", "pybind_extension", "pybind_library") package( #default_applicable_licenses = ["//third_party/nucleus:license"], @@ -17,21 +17,27 @@ cc_library( ], ) -py_clif_cc( +pybind_extension( name = "vcf_reader", - srcs = ["vcf_reader.clif"], - py_deps = [ - "//third_party/nucleus/io:clif_postproc", - ], - pyclif_deps = [ - "//third_party/nucleus/protos:range_pyclif", - "//third_party/nucleus/protos:reference_pyclif", - "//third_party/nucleus/protos:variants_pyclif", + srcs = ["vcf_reader_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:vcf_reader", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "vcf_reader_cclib", + srcs = ["vcf_reader_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:vcf_reader", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -44,6 +50,7 @@ py_test( srcs_version = "PY3", deps = [ ":vcf_reader", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/protos:reference_py_pb2", "//third_party/nucleus/protos:variants_py_pb2", "//third_party/nucleus/testing:py_test_utils", @@ -52,24 +59,51 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "vcf_writer", - srcs = ["vcf_writer.clif"], - pyclif_deps = [ - "//third_party/nucleus/protos:variants_pyclif", + srcs = ["vcf_writer_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:vcf_writer", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "vcf_writer_cclib", + srcs = ["vcf_writer_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:vcf_writer", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) -py_clif_cc( +pybind_extension( name = "vcf_concat", - srcs = ["vcf_concat.clif"], + srcs = ["vcf_concat_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:vcf_concat", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", + ], +) + +pybind_library( + name = "vcf_concat_cclib", + srcs = ["vcf_concat_pybind.cc"], deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:vcf_concat", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -92,21 +126,27 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "sam_reader", - srcs = ["sam_reader.clif"], - py_deps = [ - "//third_party/nucleus/io:clif_postproc", - ], - pyclif_deps = [ - "//third_party/nucleus/protos:range_pyclif", - "//third_party/nucleus/protos:reads_pyclif", - "//third_party/nucleus/protos:reference_pyclif", + srcs = ["sam_reader_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:sam_reader", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "sam_reader_cclib", + srcs = ["sam_reader_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:sam_reader", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -129,33 +169,53 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "sam_writer", - srcs = ["sam_writer.clif"], - py_deps = [ - "//third_party/nucleus/io:clif_postproc", - ], - pyclif_deps = [ - "//third_party/nucleus/protos:reads_pyclif", + srcs = ["sam_writer_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:sam_writer", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "sam_writer_cclib", + srcs = ["sam_writer_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:sam_writer", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) -py_clif_cc( +pybind_extension( name = "reference", - srcs = ["reference.clif"], - pyclif_deps = [ - "//third_party/nucleus/protos:fasta_pyclif", - "//third_party/nucleus/protos:range_pyclif", - "//third_party/nucleus/protos:reference_pyclif", + srcs = ["reference_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:reader_base", + "//third_party/nucleus/io:reference", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "reference_cclib", + srcs = ["reference_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:reader_base", "//third_party/nucleus/io:reference", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -168,6 +228,7 @@ py_test( srcs_version = "PY3", deps = [ ":reference", + "//third_party/nucleus/protos:fasta_py_pb2", "//third_party/nucleus/testing:py_test_utils", "//third_party/nucleus/util:ranges", "@absl_py//absl/testing:absltest", @@ -175,19 +236,27 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "bed_reader", - srcs = ["bed_reader.clif"], - py_deps = [ - "//third_party/nucleus/io:clif_postproc", - ], - pyclif_deps = [ - "//third_party/nucleus/protos:bed_pyclif", + srcs = ["bed_reader_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:bed_reader", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "bed_reader_cclib", + srcs = ["bed_reader_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:bed_reader", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -200,6 +269,7 @@ py_test( srcs_version = "PY3", deps = [ ":bed_reader", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/protos:bed_py_pb2", "//third_party/nucleus/testing:py_test_utils", "@absl_py//absl/testing:absltest", @@ -207,16 +277,27 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "bed_writer", - srcs = ["bed_writer.clif"], - pyclif_deps = [ - "//third_party/nucleus/protos:bed_pyclif", + srcs = ["bed_writer_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:bed_writer", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "bed_writer_cclib", + srcs = ["bed_writer_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:bed_writer", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -237,48 +318,75 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "bedgraph_reader", - srcs = ["bedgraph_reader.clif"], - py_deps = [ - "//third_party/nucleus/io:clif_postproc", - ], - pyclif_deps = [ - "//third_party/nucleus/protos:bedgraph_pyclif", + srcs = ["bedgraph_reader_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:bedgraph_reader", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "bedgraph_reader_cclib", + srcs = ["bedgraph_reader_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:bedgraph_reader", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) -py_clif_cc( +pybind_extension( name = "bedgraph_writer", - srcs = ["bedgraph_writer.clif"], - pyclif_deps = [ - "//third_party/nucleus/protos:bedgraph_pyclif", + srcs = ["bedgraph_writer_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:bedgraph_writer", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "bedgraph_writer_cclib", + srcs = ["bedgraph_writer_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:bedgraph_writer", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) -py_clif_cc( +pybind_extension( name = "fastq_reader", - srcs = ["fastq_reader.clif"], - py_deps = [ - "//third_party/nucleus/io:clif_postproc", - ], - pyclif_deps = [ - "//third_party/nucleus/protos:fastq_pyclif", + srcs = ["fastq_reader_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:fastq_reader", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "fastq_reader_cclib", + srcs = ["fastq_reader_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:fastq_reader", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -291,6 +399,7 @@ py_test( srcs_version = "PY3", deps = [ ":fastq_reader", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/protos:fastq_py_pb2", "//third_party/nucleus/testing:py_test_utils", "@absl_py//absl/testing:absltest", @@ -298,16 +407,27 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "fastq_writer", - srcs = ["fastq_writer.clif"], - pyclif_deps = [ - "//third_party/nucleus/protos:fastq_pyclif", + srcs = ["fastq_writer_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:fastq_writer", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "fastq_writer_cclib", + srcs = ["fastq_writer_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:fastq_writer", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -320,6 +440,7 @@ py_test( srcs_version = "PY3", deps = [ ":fastq_writer", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/io:fastq", "//third_party/nucleus/io:tfrecord", "//third_party/nucleus/protos:fastq_py_pb2", @@ -329,11 +450,27 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "hts_verbose", - srcs = ["hts_verbose.clif"], + srcs = ["hts_verbose_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:hts_verbose", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", + ], +) + +pybind_library( + name = "hts_verbose_cclib", + srcs = ["hts_verbose_pybind.cc"], deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:hts_verbose", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -349,19 +486,27 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "gff_reader", - srcs = ["gff_reader.clif"], - py_deps = [ - "//third_party/nucleus/io:clif_postproc", - ], - pyclif_deps = [ - "//third_party/nucleus/protos:gff_pyclif", + srcs = ["gff_reader_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:gff_reader", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "gff_reader_cclib", + srcs = ["gff_reader_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:gff_reader", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -374,6 +519,7 @@ py_test( srcs_version = "PY3", deps = [ ":gff_reader", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/protos:gff_py_pb2", "//third_party/nucleus/testing:py_test_utils", "@absl_py//absl/testing:absltest", @@ -381,19 +527,27 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "gff_writer", - srcs = ["gff_writer.clif"], - py_deps = [ - "//third_party/nucleus/io:clif_postproc", - ], - pyclif_deps = [ - "//third_party/nucleus/protos:gff_pyclif", + srcs = ["gff_writer_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:gff_writer", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "gff_writer_cclib", + srcs = ["gff_writer_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:gff_writer", - "//third_party/nucleus/util:proto_clif_converter", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) @@ -406,6 +560,7 @@ py_test( srcs_version = "PY3", deps = [ ":gff_writer", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/io:tfrecord", "//third_party/nucleus/protos:gff_py_pb2", "//third_party/nucleus/testing:py_test_utils", @@ -415,48 +570,122 @@ py_test( ], ) -py_clif_cc( +pybind_extension( name = "tabix_indexer", - srcs = ["tabix_indexer.clif"], + srcs = ["tabix_indexer_pybind.cc"], deps = [ - "//third_party/nucleus/core:statusor_clif_converters", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:tabix_indexer", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) -py_clif_cc( - name = "tfrecord_reader", - srcs = ["tfrecord_reader.clif"], +pybind_library( + name = "tabix_indexer_cclib", + srcs = ["tabix_indexer_pybind.cc"], deps = [ - ":tstring_clif_conversions", - "//third_party/nucleus/io:tfrecord_reader", + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:tabix_indexer", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) -py_clif_cc( +pybind_extension( name = "tfrecord_writer", - srcs = ["tfrecord_writer.clif"], + srcs = ["tfrecord_writer_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:tfrecord_writer", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", + ], +) + +pybind_library( + name = "tfrecord_writer_cclib", + srcs = ["tfrecord_writer_pybind.cc"], deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:tfrecord_writer", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", + ], +) + +pybind_extension( + name = "tfrecord_reader", + srcs = ["tfrecord_reader_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:tfrecord_reader", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) -py_clif_cc( +pybind_library( + name = "tfrecord_reader_cclib", + srcs = ["tfrecord_reader_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:tfrecord_reader", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", + ], +) + +pybind_extension( name = "gfile", - srcs = ["gfile.clif"], + srcs = ["gfile_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:gfile_cc", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", + ], +) + +pybind_library( + name = "gfile_cclib", + srcs = ["gfile_pybind.cc"], deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:gfile_cc", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) -py_clif_cc( +pybind_extension( name = "merge_variants", - srcs = ["merge_variants.clif"], - pyclif_deps = [ - "//third_party/nucleus/protos:range_pyclif", - "//third_party/nucleus/protos:variants_pyclif", + srcs = ["merge_variants_pybind.cc"], + deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", + "//third_party/nucleus/io:merge_variants", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], +) + +pybind_library( + name = "merge_variants_cclib", + srcs = ["merge_variants_pybind.cc"], deps = [ + "//third_party/nucleus/core/python:type_caster_nucleus_status", + "//third_party/nucleus/core/python:type_caster_nucleus_statusor", "//third_party/nucleus/io:merge_variants", + "//third_party/nucleus/util/python:type_caster_nucleus_proto_ptr", + "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], ) diff --git a/third_party/nucleus/io/python/bed_reader.clif b/third_party/nucleus/io/python/bed_reader.clif deleted file mode 100644 index 66ee5b16..00000000 --- a/third_party/nucleus/io/python/bed_reader.clif +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/bed_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from third_party.nucleus.io.clif_postproc import WrappedBedIterable - - -from "third_party/nucleus/io/bed_reader.h": - namespace `nucleus`: - - class BedIterable: - def PythonNext(self, bed: EmptyProtoPtr) -> StatusOr - def Release(self) -> Status - @__enter__ - def PythonEnter(self) -> Status - @__exit__ - def PythonExit(self) -> Status - - class BedReader: - @classmethod - def `FromFile` as from_file(cls, bedPath: str, options: BedReaderOptions) - -> StatusOr - - def `Iterate` as iterate(self) -> StatusOr: - return WrappedBedIterable(...) - - @__enter__ - def PythonEnter(self) - @__exit__ - def Close(self) -> Status - header: BedHeader = property(`Header`) diff --git a/third_party/nucleus/io/python/bed_reader_pybind.cc b/third_party/nucleus/io/python/bed_reader_pybind.cc index f6943695..d45a0e72 100755 --- a/third_party/nucleus/io/python/bed_reader_pybind.cc +++ b/third_party/nucleus/io/python/bed_reader_pybind.cc @@ -41,7 +41,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/bed_reader.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/bed_writer.clif b/third_party/nucleus/io/python/bed_writer.clif deleted file mode 100644 index 09e6494e..00000000 --- a/third_party/nucleus/io/python/bed_writer.clif +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/bed_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from "third_party/nucleus/io/bed_writer.h": - namespace `nucleus`: - class BedWriter: - @classmethod - def `ToFile` as to_file(cls, bedPath: str, - header: BedHeader, - options: BedWriterOptions) - -> StatusOr - def `WritePython` as write(self, bedMessage: ConstProtoPtr) -> Status - @__enter__ - def PythonEnter(self) - @__exit__ - def Close(self) -> Status - header: BedHeader = property(`Header`) diff --git a/third_party/nucleus/io/python/bed_writer_pybind.cc b/third_party/nucleus/io/python/bed_writer_pybind.cc index 6313a163..b15304c6 100755 --- a/third_party/nucleus/io/python/bed_writer_pybind.cc +++ b/third_party/nucleus/io/python/bed_writer_pybind.cc @@ -30,7 +30,7 @@ * */ -#include "third_party/pybind11/include/pybind11/cast.h" +#include "pybind11/cast.h" #if true // Trick to stop tooling from moving the #include around. // MUST appear before any standard headers are included. #include @@ -42,7 +42,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/bed_writer.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/bedgraph_reader.clif b/third_party/nucleus/io/python/bedgraph_reader.clif deleted file mode 100644 index 4d84a1d4..00000000 --- a/third_party/nucleus/io/python/bedgraph_reader.clif +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/bedgraph_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from third_party.nucleus.io.clif_postproc import WrappedBedGraphIterable - - -from "third_party/nucleus/io/bedgraph_reader.h": - namespace `nucleus`: - - class BedGraphIterable: - def PythonNext(self, bedgraph: EmptyProtoPtr) -> StatusOr - def Release(self) -> Status - @__enter__ - def PythonEnter(self) -> Status - @__exit__ - def PythonExit(self) -> Status - - class BedGraphReader: - @classmethod - def `FromFile` as from_file(cls, bedGraphPath: str) - -> StatusOr - - def `Iterate` as iterate(self) -> StatusOr: - return WrappedBedGraphIterable(...) - - @__enter__ - def PythonEnter(self) - @__exit__ - def Close(self) -> Status diff --git a/third_party/nucleus/io/python/bedgraph_reader_pybind.cc b/third_party/nucleus/io/python/bedgraph_reader_pybind.cc index da527a76..a497eb5f 100755 --- a/third_party/nucleus/io/python/bedgraph_reader_pybind.cc +++ b/third_party/nucleus/io/python/bedgraph_reader_pybind.cc @@ -30,7 +30,7 @@ * */ -#include "third_party/pybind11/include/pybind11/cast.h" +#include "pybind11/cast.h" #if true // Trick to stop tooling from moving the #include around. // MUST appear before any standard headers are included. #include @@ -40,8 +40,8 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/bedgraph_reader.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11/include/pybind11/stl.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11/stl.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/bedgraph_writer.clif b/third_party/nucleus/io/python/bedgraph_writer.clif deleted file mode 100644 index f5daf45e..00000000 --- a/third_party/nucleus/io/python/bedgraph_writer.clif +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/bedgraph_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from "third_party/nucleus/io/bedgraph_writer.h": - namespace `nucleus`: - class BedGraphWriter: - @classmethod - def `ToFile` as to_file(cls, bedPath: str) - -> StatusOr - def `WritePython` as write(self, bedGraphMessage: ConstProtoPtr) -> Status - @__enter__ - def PythonEnter(self) - @__exit__ - def Close(self) -> Status diff --git a/third_party/nucleus/io/python/bedgraph_writer_pybind.cc b/third_party/nucleus/io/python/bedgraph_writer_pybind.cc index c5f304df..5a35f6a7 100755 --- a/third_party/nucleus/io/python/bedgraph_writer_pybind.cc +++ b/third_party/nucleus/io/python/bedgraph_writer_pybind.cc @@ -39,7 +39,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/bedgraph_writer.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/fastq_reader.clif b/third_party/nucleus/io/python/fastq_reader.clif deleted file mode 100644 index d2411a8b..00000000 --- a/third_party/nucleus/io/python/fastq_reader.clif +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/fastq_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from third_party.nucleus.io.clif_postproc import WrappedFastqIterable - - -from "third_party/nucleus/io/fastq_reader.h": - namespace `nucleus`: - - class FastqIterable: - def PythonNext(self, fastq: EmptyProtoPtr) -> StatusOr - def Release(self) -> Status - @__enter__ - def PythonEnter(self) -> Status - @__exit__ - def PythonExit(self) -> Status - - class FastqReader: - @classmethod - def `FromFile` as from_file(cls, fastqPath: str, options: FastqReaderOptions) - -> StatusOr - - def `Iterate` as iterate(self) -> StatusOr: - return WrappedFastqIterable(...) - @__enter__ - def PythonEnter(self) -> Status - @__exit__ - def Close(self) -> Status diff --git a/third_party/nucleus/io/python/fastq_reader_pybind.cc b/third_party/nucleus/io/python/fastq_reader_pybind.cc index decd74d8..268ffd96 100755 --- a/third_party/nucleus/io/python/fastq_reader_pybind.cc +++ b/third_party/nucleus/io/python/fastq_reader_pybind.cc @@ -30,7 +30,7 @@ * */ -#include "third_party/pybind11/include/pybind11/cast.h" +#include "pybind11/cast.h" #if true // Trick to stop tooling from moving the #include around. // MUST appear before any standard headers are included. #include @@ -40,7 +40,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/fastq_reader.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/fastq_reader_wrap_test.py b/third_party/nucleus/io/python/fastq_reader_wrap_test.py index 0669a2e8..6a42e432 100644 --- a/third_party/nucleus/io/python/fastq_reader_wrap_test.py +++ b/third_party/nucleus/io/python/fastq_reader_wrap_test.py @@ -47,13 +47,14 @@ def setUp(self): self.fastq = test_utils.genomics_core_testdata('test_reads.fastq') self.options = fastq_pb2.FastqReaderOptions() - @parameterized.parameters('test_reads.fastq', 'test_reads.fastq.gz', - 'test_reads.bgzip.fastq.gz') + @parameterized.parameters( + 'test_reads.fastq', 'test_reads.fastq.gz', 'test_reads.bgzip.fastq.gz' + ) def test_fastq_iterate(self, filename): path = test_utils.genomics_core_testdata(filename) with fastq_reader.FastqReader.from_file(path, self.options) as reader: iterable = reader.iterate() - self.assertIsInstance(iterable, clif_postproc.WrappedCppIterable) + self.assertIsInstance(iterable, clif_postproc.WrappedFastqIterable) self.assertEqual(test_utils.iterable_len(iterable), 4) def test_from_file_raises_with_missing_fastq(self): diff --git a/third_party/nucleus/io/python/fastq_writer.clif b/third_party/nucleus/io/python/fastq_writer.clif deleted file mode 100644 index cd5e1259..00000000 --- a/third_party/nucleus/io/python/fastq_writer.clif +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/fastq_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from "third_party/nucleus/io/fastq_writer.h": - namespace `nucleus`: - class FastqWriter: - @classmethod - def `ToFile` as to_file(cls, fastqPath: str, - options: FastqWriterOptions) - -> StatusOr - def `WritePython` as write(self, fastqMessage: ConstProtoPtr) -> Status - @__enter__ - def PythonEnter(self) - @__exit__ - def Close(self) -> Status diff --git a/third_party/nucleus/io/python/fastq_writer_pybind.cc b/third_party/nucleus/io/python/fastq_writer_pybind.cc index 4751db19..cc006b8c 100755 --- a/third_party/nucleus/io/python/fastq_writer_pybind.cc +++ b/third_party/nucleus/io/python/fastq_writer_pybind.cc @@ -30,7 +30,7 @@ * */ -#include "third_party/pybind11/include/pybind11/cast.h" +#include "pybind11/cast.h" #if true // Trick to stop tooling from moving the #include around. // MUST appear before any standard headers are included. #include @@ -40,7 +40,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/fastq_writer.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/gff_reader.clif b/third_party/nucleus/io/python/gff_reader.clif deleted file mode 100644 index f8d4ff4d..00000000 --- a/third_party/nucleus/io/python/gff_reader.clif +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/gff_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from third_party.nucleus.io.clif_postproc import WrappedGffIterable - - -from "third_party/nucleus/io/gff_reader.h": - namespace `nucleus`: - - class GffIterable: - def PythonNext(self, gff: EmptyProtoPtr) -> StatusOr - def Release(self) -> Status - @__enter__ - def PythonEnter(self) -> Status - @__exit__ - def PythonExit(self) -> Status - - class GffReader: - @classmethod - def `FromFile` as from_file(cls, gffPath: str, options: GffReaderOptions) - -> StatusOr - - def `Iterate` as iterate(self) -> StatusOr: - return WrappedGffIterable(...) - - @__enter__ - def PythonEnter(self) - @__exit__ - def Close(self) -> Status - header: GffHeader = property(`Header`) diff --git a/third_party/nucleus/io/python/gff_reader_pybind.cc b/third_party/nucleus/io/python/gff_reader_pybind.cc index 5419ed05..126b48f9 100755 --- a/third_party/nucleus/io/python/gff_reader_pybind.cc +++ b/third_party/nucleus/io/python/gff_reader_pybind.cc @@ -41,7 +41,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/gff_reader.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/gff_reader_wrap_test.py b/third_party/nucleus/io/python/gff_reader_wrap_test.py index 047a56b0..fc6af5b0 100644 --- a/third_party/nucleus/io/python/gff_reader_wrap_test.py +++ b/third_party/nucleus/io/python/gff_reader_wrap_test.py @@ -69,7 +69,7 @@ def test_gff_iterate(self, test_features_gff_filename): file_path = test_utils.genomics_core_testdata(test_features_gff_filename) with gff_reader.GffReader.from_file(file_path, self.options) as reader: iterable = reader.iterate() - self.assertIsInstance(iterable, clif_postproc.WrappedCppIterable) + self.assertIsInstance(iterable, clif_postproc.WrappedGffIterable) actual = list(iterable) self.assertLen(actual, 2) self.assertEqual(actual[0], self.first) diff --git a/third_party/nucleus/io/python/gff_writer.clif b/third_party/nucleus/io/python/gff_writer.clif deleted file mode 100644 index a477306c..00000000 --- a/third_party/nucleus/io/python/gff_writer.clif +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/gff_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from "third_party/nucleus/io/gff_writer.h": - namespace `nucleus`: - class GffWriter: - @classmethod - def `ToFile` as to_file(cls, gffPath: str, - header: GffHeader, - options: GffWriterOptions) - -> StatusOr - def `WritePython` as write(self, gffMessage: ConstProtoPtr) -> Status - @__enter__ - def PythonEnter(self) - @__exit__ - def Close(self) -> Status - header: GffHeader = property(`Header`) diff --git a/third_party/nucleus/io/python/gff_writer_pybind.cc b/third_party/nucleus/io/python/gff_writer_pybind.cc index 8af55248..0e6519d4 100755 --- a/third_party/nucleus/io/python/gff_writer_pybind.cc +++ b/third_party/nucleus/io/python/gff_writer_pybind.cc @@ -39,7 +39,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/gff_writer.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/gfile.clif b/third_party/nucleus/io/python/gfile.clif deleted file mode 100644 index 27cec464..00000000 --- a/third_party/nucleus/io/python/gfile.clif +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2019 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/io/gfile.h": - namespace `nucleus`: - - def `Exists` as Exists(filename: str) -> bool - - def `Glob` as Glob(pattern: str) -> list - - class ReadableFile: - @classmethod - def `New` as New(cls, filename: str) -> ReadableFile - - def `Readline` as Readline(self) -> (ok: bool, s: str) - - def `Close` as close(self) - - @__enter__ - def PythonEnter(self) - - @__exit__ - def Close(self) - - - class WritableFile: - @classmethod - def `New` as New(cls, filename: str) -> WritableFile - - def `Write` as write(self, s: str) -> bool - - def `Close` as close(self) - - @__enter__ - def PythonEnter(self) - - @__exit__ - def Close(self) diff --git a/third_party/nucleus/io/python/gfile_pybind.cc b/third_party/nucleus/io/python/gfile_pybind.cc index b97d6bdf..738ea968 100755 --- a/third_party/nucleus/io/python/gfile_pybind.cc +++ b/third_party/nucleus/io/python/gfile_pybind.cc @@ -41,7 +41,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/gfile.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/hts_verbose.clif b/third_party/nucleus/io/python/hts_verbose.clif deleted file mode 100644 index ff21313e..00000000 --- a/third_party/nucleus/io/python/hts_verbose.clif +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/io/hts_verbose.h": - enum htsLogLevel - namespace `nucleus`: - def `HtsGetLogLevel` as get() -> htsLogLevel - def `HtsSetLogLevel` as set(level: htsLogLevel) diff --git a/third_party/nucleus/io/python/hts_verbose_pybind.cc b/third_party/nucleus/io/python/hts_verbose_pybind.cc index 857a47a3..ba7a7bde 100755 --- a/third_party/nucleus/io/python/hts_verbose_pybind.cc +++ b/third_party/nucleus/io/python/hts_verbose_pybind.cc @@ -41,7 +41,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/hts_verbose.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/merge_variants.clif b/third_party/nucleus/io/python/merge_variants.clif deleted file mode 100644 index a2d6a555..00000000 --- a/third_party/nucleus/io/python/merge_variants.clif +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/variants_pyclif.h" import * -from "third_party/nucleus/protos/range_pyclif.h" import * - -from "third_party/nucleus/io/merge_variants.h": - namespace `nucleus`: - def `MergeAndWriteVariantsAndNonVariants` as merge_and_write_variants_and_nonvariants( - only_keep_pass: bool, - variant_file_path: str, - non_variant_file_paths: list, - fasta_path: str, - vcf_out_file_path: str, - gvcf_out_file_path: str, - header: VcfHeader, - ranges: list, - process_somatic: bool = default) - diff --git a/third_party/nucleus/io/python/merge_variants_pybind.cc b/third_party/nucleus/io/python/merge_variants_pybind.cc index 0e34013d..964127fc 100755 --- a/third_party/nucleus/io/python/merge_variants_pybind.cc +++ b/third_party/nucleus/io/python/merge_variants_pybind.cc @@ -30,7 +30,7 @@ * */ -#include "third_party/pybind11/include/pybind11/cast.h" +#include "pybind11/cast.h" #if true // Trick to stop tooling from moving the #include around. // MUST appear before any standard headers are included. #include @@ -42,7 +42,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/merge_variants.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/reference.clif b/third_party/nucleus/io/python/reference.clif deleted file mode 100644 index a57c9f37..00000000 --- a/third_party/nucleus/io/python/reference.clif +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/fasta_pyclif.h" import * -from "third_party/nucleus/protos/range_pyclif.h" import * -from "third_party/nucleus/protos/reference_pyclif.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from third_party.nucleus.io.clif_postproc import WrappedReferenceIterable - -from "third_party/nucleus/io/reference.h": - namespace `nucleus`: - class GenomeReferenceRecordIterable: - def Next(self) -> (not_done: StatusOr, fasta: tuple) - def Release(self) -> Status - @__enter__ - def PythonEnter(self) -> Status - @__exit__ - def PythonExit(self) -> Status - - class GenomeReference: - contig_names: list = property(`ContigNames`) - def `Contig` as contig(self, chrom: str) -> StatusOr - def `GetBases` as bases(self, region: Range) -> StatusOr - def `Iterate` as iterate(self) -> StatusOr: - return WrappedReferenceIterable(...) - def `HasContig` as has_contig(self, contig_name: str) -> bool - def `IsValidInterval` as is_valid_interval(self, region: Range) -> bool - - # These are pure virtual superclass methods but including these above - # results in a linker error. - contigs: list = property(`Contigs`) - - @__enter__ - def PythonEnter(self) -> Status - @__exit__ - def Close(self) -> Status - - class IndexedFastaReader(GenomeReference): - @classmethod - def `FromFile` as from_file(cls, - fasta_path: str, - fai_path: str, - options: FastaReaderOptions, - cache_size_bases: int = default) - -> StatusOr - - class UnindexedFastaReader(GenomeReference): - @classmethod - def `FromFile` as from_file(cls, - fasta_path: str) - -> StatusOr - - class InMemoryFastaReader(GenomeReference): - @classmethod - def `Create` as create(cls, contigs: list, seqs: list) - -> StatusOr - - reference_sequences: dict = property(`ReferenceSequences`) diff --git a/third_party/nucleus/io/python/reference_pybind.cc b/third_party/nucleus/io/python/reference_pybind.cc index f27d4dbd..3c1ad2e4 100755 --- a/third_party/nucleus/io/python/reference_pybind.cc +++ b/third_party/nucleus/io/python/reference_pybind.cc @@ -42,7 +42,7 @@ #include "third_party/nucleus/io/reader_base.h" #include "third_party/nucleus/io/reference.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/sam_reader.clif b/third_party/nucleus/io/python/sam_reader.clif deleted file mode 100644 index ce2424f3..00000000 --- a/third_party/nucleus/io/python/sam_reader.clif +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/range_pyclif.h" import * -from "third_party/nucleus/protos/reads_pyclif.h" import * -from "third_party/nucleus/protos/reference_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from third_party.nucleus.io.clif_postproc import WrappedSamIterable - - -from "third_party/nucleus/io/sam_reader.h": - namespace `nucleus`: - - class SamIterable: - def PythonNext(self, read: EmptyProtoPtr) -> StatusOr - def Release(self) -> Status - @__enter__ - def PythonEnter(self) -> Status - @__exit__ - def PythonExit(self) -> Status - - class SamReader: - @classmethod - def `FromFile` as from_file( - cls, reads_path: str, ref_path: str, options: SamReaderOptions) - -> StatusOr - - def `Iterate` as iterate(self) -> StatusOr: - return WrappedSamIterable(...) - def `Query` as query(self, region: Range) -> StatusOr: - return WrappedSamIterable(...) - header: SamHeader = property(`Header`) - @__enter__ - def PythonEnter(self) -> Status - @__exit__ - def Close(self) -> Status diff --git a/third_party/nucleus/io/python/sam_reader_pybind.cc b/third_party/nucleus/io/python/sam_reader_pybind.cc index 1591ed4e..b64a1a0a 100755 --- a/third_party/nucleus/io/python/sam_reader_pybind.cc +++ b/third_party/nucleus/io/python/sam_reader_pybind.cc @@ -30,7 +30,7 @@ * */ -#include "third_party/pybind11/include/pybind11/cast.h" +#include "pybind11/cast.h" #if true // Trick to stop tooling from moving the #include around. // MUST appear before any standard headers are included. #include @@ -42,7 +42,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/sam_reader.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/sam_reader_wrap_test.py b/third_party/nucleus/io/python/sam_reader_wrap_test.py index eecd0c38..6e952579 100644 --- a/third_party/nucleus/io/python/sam_reader_wrap_test.py +++ b/third_party/nucleus/io/python/sam_reader_wrap_test.py @@ -54,7 +54,7 @@ def test_bam_iterate(self): reads_path=self.bam, ref_path='', options=self.options) with reader: iterable = reader.iterate() - self.assertIsInstance(iterable, clif_postproc.WrappedCppIterable) + self.assertIsInstance(iterable, clif_postproc.WrappedSamIterable) self.assertEqual(test_utils.iterable_len(iterable), 106) def test_bam_query(self): @@ -65,7 +65,7 @@ def test_bam_query(self): with reader: for interval, n_expected in expected: with reader.query(interval) as iterable: - self.assertIsInstance(iterable, clif_postproc.WrappedCppIterable) + self.assertIsInstance(iterable, clif_postproc.WrappedSamIterable) self.assertEqual(test_utils.iterable_len(iterable), n_expected) def test_bam_samples(self): @@ -139,10 +139,10 @@ def test_context_manager(self): with reader: with reader.query(region) as query_iterable1: self.assertIsNotNone(query_iterable1) - self.assertIsInstance(query_iterable1, clif_postproc.WrappedCppIterable) + self.assertIsInstance(query_iterable1, clif_postproc.WrappedSamIterable) with reader.query(region) as query_iterable2: self.assertIsNotNone(query_iterable2) - self.assertIsInstance(query_iterable2, clif_postproc.WrappedCppIterable) + self.assertIsInstance(query_iterable2, clif_postproc.WrappedSamIterable) def test_from_file_raises_with_missing_bam(self): # TODO: OpError exception not propagated. diff --git a/third_party/nucleus/io/python/sam_writer.clif b/third_party/nucleus/io/python/sam_writer.clif deleted file mode 100644 index 17c77544..00000000 --- a/third_party/nucleus/io/python/sam_writer.clif +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/reads_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from "third_party/nucleus/io/sam_writer.h": - namespace `nucleus`: - class SamWriter: - @classmethod - def `ToFile` as to_file(cls, samPath: str, - refPath: str, - embedRef: bool, - header: SamHeader) - -> StatusOr - def `WritePython` as write(self, samMessage: ConstProtoPtr) -> Status - @__enter__ - def PythonEnter(self) - @__exit__ - def Close(self) -> Status diff --git a/third_party/nucleus/io/python/sam_writer_pybind.cc b/third_party/nucleus/io/python/sam_writer_pybind.cc index df4f81aa..dd52e0b9 100755 --- a/third_party/nucleus/io/python/sam_writer_pybind.cc +++ b/third_party/nucleus/io/python/sam_writer_pybind.cc @@ -30,7 +30,7 @@ * */ -#include "third_party/pybind11/include/pybind11/cast.h" +#include "pybind11/cast.h" #if true // Trick to stop tooling from moving the #include around. // MUST appear before any standard headers are included. #include @@ -40,7 +40,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/sam_writer.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/tabix_indexer.clif b/third_party/nucleus/io/python/tabix_indexer.clif deleted file mode 100644 index e4bc3d10..00000000 --- a/third_party/nucleus/io/python/tabix_indexer.clif +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2019 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from "third_party/nucleus/io/tabix_indexer.h": - namespace `nucleus`: - def `TbxIndexBuild` as tbx_index_build(path: str) -> Status - def `CSIIndexBuild` as csi_index_build(path: str, min_shift:int) -> Status diff --git a/third_party/nucleus/io/python/tabix_indexer_pybind.cc b/third_party/nucleus/io/python/tabix_indexer_pybind.cc index 83f2ee5c..4684df78 100755 --- a/third_party/nucleus/io/python/tabix_indexer_pybind.cc +++ b/third_party/nucleus/io/python/tabix_indexer_pybind.cc @@ -41,7 +41,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/tabix_indexer.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/tfrecord_reader.clif b/third_party/nucleus/io/python/tfrecord_reader.clif deleted file mode 100644 index 7c5bd750..00000000 --- a/third_party/nucleus/io/python/tfrecord_reader.clif +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2019 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/io/python/tstring_clif_conversions.h" import * - -from "third_party/nucleus/io/tfrecord_reader.h": - namespace `nucleus`: - - class TFRecordReader: - @classmethod - def `New` as from_file(cls, filename: str, compression_type: str) -> TFRecordReader - - def `GetNext` as get_next(self) -> bool - - def `record` as get_record(self) -> bytes - - def `Close` as close(self) diff --git a/third_party/nucleus/io/python/tfrecord_reader_pybind.cc b/third_party/nucleus/io/python/tfrecord_reader_pybind.cc index f1bf0ae8..1103e7ed 100755 --- a/third_party/nucleus/io/python/tfrecord_reader_pybind.cc +++ b/third_party/nucleus/io/python/tfrecord_reader_pybind.cc @@ -41,7 +41,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/tfrecord_reader.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/tfrecord_writer.clif b/third_party/nucleus/io/python/tfrecord_writer.clif deleted file mode 100644 index e8dae5da..00000000 --- a/third_party/nucleus/io/python/tfrecord_writer.clif +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2019 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/io/tfrecord_writer.h": - namespace `nucleus`: - - class TFRecordWriter: - @classmethod - def `New` as from_file(cls, filename: str, compression_type: str) -> TFRecordWriter - - def `WriteRecord` as write(self, record: str) -> bool - - def `Flush` as flush(self) -> bool - def `Close` as close(self) -> bool - diff --git a/third_party/nucleus/io/python/vcf_concat.clif b/third_party/nucleus/io/python/vcf_concat.clif deleted file mode 100644 index 1da0d9b0..00000000 --- a/third_party/nucleus/io/python/vcf_concat.clif +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2024 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/io/vcf_concat.h": - namespace `nucleus`: - def `VcfConcat` as concat(output_fname: str, fnames: list) diff --git a/third_party/nucleus/io/python/vcf_concat_pybind.cc b/third_party/nucleus/io/python/vcf_concat_pybind.cc index d1e2468e..0db2cbad 100755 --- a/third_party/nucleus/io/python/vcf_concat_pybind.cc +++ b/third_party/nucleus/io/python/vcf_concat_pybind.cc @@ -41,7 +41,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/vcf_concat.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/vcf_reader.clif b/third_party/nucleus/io/python/vcf_reader.clif deleted file mode 100644 index 1f7210e2..00000000 --- a/third_party/nucleus/io/python/vcf_reader.clif +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/range_pyclif.h" import * -from "third_party/nucleus/protos/reference_pyclif.h" import * -from "third_party/nucleus/protos/variants_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from third_party.nucleus.io.clif_postproc import ValueErrorOnFalse -from third_party.nucleus.io.clif_postproc import WrappedVariantIterable - - -from "third_party/nucleus/io/vcf_reader.h": - namespace `nucleus`: - - class VariantIterable: - def PythonNext(self, variant: EmptyProtoPtr) -> StatusOr - def Release(self) -> Status - @__enter__ - def PythonEnter(self) -> Status - @__exit__ - def PythonExit(self) -> Status - - class VcfReader: - @classmethod - def `FromFile` as from_file(cls, variantsPath: str, options: VcfReaderOptions) - -> StatusOr - - @classmethod - def `FromFileWithHeader` as from_file_with_header(cls, variantsPath: str, options: VcfReaderOptions, header: VcfHeader) - -> StatusOr - - def `Iterate` as iterate(self) -> StatusOr: - return WrappedVariantIterable(...) - def `Query` as query(self, region: Range) -> StatusOr: - return WrappedVariantIterable(...) - - def `FromStringPython` as from_string(self, vcf_line: str) -> (status: StatusOr, variant: Variant): - # If status is an error object, the statusor_clif_converters - # will have already converted it into a Python error message. - # We call ValueErrorOnFalse here so that users only see a single - # return object (the Variant). - return ValueErrorOnFalse(...) - - @__enter__ - def PythonEnter(self) -> Status - @__exit__ - def Close(self) -> Status - header: VcfHeader = property(`Header`) diff --git a/third_party/nucleus/io/python/vcf_reader_pybind.cc b/third_party/nucleus/io/python/vcf_reader_pybind.cc index 6adbfe68..7364914c 100755 --- a/third_party/nucleus/io/python/vcf_reader_pybind.cc +++ b/third_party/nucleus/io/python/vcf_reader_pybind.cc @@ -30,7 +30,7 @@ * */ -#include "third_party/pybind11/include/pybind11/cast.h" +#include "pybind11/cast.h" #if true // Trick to stop tooling from moving the #include around. // MUST appear before any standard headers are included. #include @@ -40,7 +40,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/vcf_reader.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/io/python/vcf_writer.clif b/third_party/nucleus/io/python/vcf_writer.clif deleted file mode 100644 index 6188e97c..00000000 --- a/third_party/nucleus/io/python/vcf_writer.clif +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/variants_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * -from "third_party/nucleus/core/statusor_clif_converters.h" import * - -from "third_party/nucleus/io/vcf_writer.h": - namespace `nucleus`: - class VcfWriter: - @classmethod - def `ToFile` as to_file(cls, variantsPath: str, vcfHeader: VcfHeader, - options: VcfWriterOptions) - -> StatusOr - def `WritePython` as write(self, variantMessage: ConstProtoPtr) -> Status - def `WriteSomaticPython` as write_somatic(self, variantMessage: ConstProtoPtr) -> Status - @__enter__ - def PythonEnter(self) - @__exit__ - def Close(self) -> Status diff --git a/third_party/nucleus/io/python/vcf_writer_pybind.cc b/third_party/nucleus/io/python/vcf_writer_pybind.cc index 5a336718..704203f3 100755 --- a/third_party/nucleus/io/python/vcf_writer_pybind.cc +++ b/third_party/nucleus/io/python/vcf_writer_pybind.cc @@ -30,7 +30,7 @@ * */ -#include "third_party/pybind11/include/pybind11/cast.h" +#include "pybind11/cast.h" #if true // Trick to stop tooling from moving the #include around. // MUST appear before any standard headers are included. #include @@ -40,7 +40,7 @@ #include "third_party/nucleus/core/python/type_caster_nucleus_statusor.h" #include "third_party/nucleus/io/vcf_writer.h" #include "third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h" -#include "third_party/pybind11_protobuf/native_proto_caster.h" +#include "pybind11_protobuf/native_proto_caster.h" namespace py = pybind11; diff --git a/third_party/nucleus/protos/BUILD b/third_party/nucleus/protos/BUILD index bb5115c4..a9dd37db 100644 --- a/third_party/nucleus/protos/BUILD +++ b/third_party/nucleus/protos/BUILD @@ -1,5 +1,4 @@ load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library", "py_proto_library") -load("//third_party:clif.bzl", "pyclif_proto_library") package(default_visibility = [ "//visibility:public", @@ -400,62 +399,3 @@ py_proto_library( ":feature_py_pb2", ], ) - -# ---------------------------------------------------------------------- -# pyclif targets -# ---------------------------------------------------------------------- - -pyclif_proto_library( - name = "bed_pyclif", - proto_lib = ":bed_proto", # NO COPYBARA -) - -pyclif_proto_library( - name = "bedgraph_pyclif", - proto_lib = ":bedgraph_proto", # NO COPYBARA -) - -pyclif_proto_library( - name = "gff_pyclif", - proto_lib = ":gff_proto", # NO COPYBARA -) - -pyclif_proto_library( - name = "fasta_pyclif", - proto_lib = ":fasta_proto", # NO COPYBARA -) - -pyclif_proto_library( - name = "fastq_pyclif", - proto_lib = ":fastq_proto", # NO COPYBARA -) - -pyclif_proto_library( - name = "position_pyclif", - proto_lib = ":position_proto", # NO COPYBARA -) - -pyclif_proto_library( - name = "range_pyclif", - proto_lib = ":range_proto", # NO COPYBARA -) - -pyclif_proto_library( - name = "reads_pyclif", - proto_lib = ":reads_proto", # NO COPYBARA -) - -pyclif_proto_library( - name = "reference_pyclif", - proto_lib = ":reference_proto", # NO COPYBARA -) - -pyclif_proto_library( - name = "variants_pyclif", - proto_lib = ":variants_proto", # NO COPYBARA -) - -pyclif_proto_library( - name = "cigar_pyclif", # NO COPYBARA - proto_lib = ":cigar_proto", -) diff --git a/third_party/nucleus/util/BUILD b/third_party/nucleus/util/BUILD index d27fe626..6f3bc927 100644 --- a/third_party/nucleus/util/BUILD +++ b/third_party/nucleus/util/BUILD @@ -109,7 +109,7 @@ py_test( py_test( name = "py_utils_test", - size = "small", + size = "medium", srcs = ["utils_test.py"], main = "utils_test.py", python_version = "PY3", @@ -365,19 +365,6 @@ cc_library( ], ) -cc_library( - name = "proto_clif_converter", - srcs = ["proto_clif_converter.cc"], - hdrs = ["proto_clif_converter.h"], - deps = [ - ":proto_ptr", - "@clif//:cpp_runtime", - "@com_google_protobuf//:proto_api", - "@com_google_protobuf//:protobuf", - "@org_tensorflow//tensorflow/core:lib", - ], -) - py_library( name = "vis", srcs = ["vis.py"], diff --git a/third_party/nucleus/util/proto_clif_converter.cc b/third_party/nucleus/util/proto_clif_converter.cc deleted file mode 100644 index d13cc43e..00000000 --- a/third_party/nucleus/util/proto_clif_converter.cc +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ -#include "third_party/nucleus/util/proto_clif_converter.h" - -#include "python/google/protobuf/proto_api.h" -#include "clif/python/types.h" -#include "google/protobuf/message.h" - -namespace nucleus { - -static const google::protobuf::python::PyProto_API* py_proto_api = nullptr; - -const google::protobuf::python::PyProto_API* GetPyProtoApi(PyObject* py) { - if (py_proto_api == nullptr) { - py_proto_api = static_cast( - PyCapsule_Import(google::protobuf::python::PyProtoAPICapsuleName(), 0)); - } - return py_proto_api; -} - -} // namespace nucleus diff --git a/third_party/nucleus/util/proto_clif_converter.h b/third_party/nucleus/util/proto_clif_converter.h deleted file mode 100644 index 4b3b1ee7..00000000 --- a/third_party/nucleus/util/proto_clif_converter.h +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ -#ifndef THIRD_PARTY_NUCLEUS_UTIL_PROTO_CLIF_CONVERTER_H_ -#define THIRD_PARTY_NUCLEUS_UTIL_PROTO_CLIF_CONVERTER_H_ - -#include "python/google/protobuf/proto_api.h" -#include "clif/python/types.h" -#include "third_party/nucleus/util/proto_ptr.h" -#include "google/protobuf/message.h" -#include "tensorflow/core/platform/logging.h" - -namespace nucleus { - -// Note: the comments below are instructions to CLIF. -// CLIF use `::nucleus::EmptyProtoPtr` as EmptyProtoPtr, NumTemplateParameter:1 -// CLIF use `::nucleus::ConstProtoPtr` as ConstProtoPtr, NumTemplateParameter:1 - -const ::google::protobuf::python::PyProto_API* GetPyProtoApi(PyObject* py); - -// Convert from Python protocol buffer object py to a C++ pointer. -// Unlike the conversions that CLIF automatically generates for protocol -// buffers, this one does no copying if the Python protocol buffer uses -// the C++ memory layout. -template -bool Clif_PyObjAs(PyObject* py, EmptyProtoPtr* c) { - CHECK(c != nullptr); - - auto* py_proto_api = GetPyProtoApi(py); - if (py_proto_api == nullptr) { - PyErr_SetString(PyExc_RuntimeError, "Could not load PyProto API"); - return false; - } - - ::google::protobuf::Message* cpb = py_proto_api->GetMutableMessagePointer(py); - if (cpb == nullptr) { - // Clients might depend on our non-copying semantics, so we can't fall - // back on CLIF here but instead must fail loudly. - PyErr_SetString(PyExc_RuntimeError, - "Python protobuf did not contain a mutable C++ protobuf"); - return false; - } else { - c->p_ = dynamic_cast(cpb); - if (c->p_ == nullptr) { - // DO NOT DELETE THIS WARNING! Without it, the above dynamic_cast - // will fail when running from a Python 3 pip package. - LOG(WARNING) << "Failed to cast type " << typeid(*cpb).name(); - PyErr_SetString(PyExc_RuntimeError, "Dynamic cast failed"); - return false; - } - return true; - } -} - -// Convert from Python protocol buffer object py to a C++ pointer. -// Unlike the conversions that CLIF automatically generates for protocol -// buffers, this one does no copying if the Python protocol buffer uses -// the C++ memory layout. -template -bool Clif_PyObjAs(PyObject* py, ConstProtoPtr* c) { - CHECK(c != nullptr); - - auto* py_proto_api = GetPyProtoApi(py); - if (py_proto_api == nullptr) { - PyErr_SetString(PyExc_RuntimeError, "Could not load PyProto API"); - return false; - } - - const ::google::protobuf::Message* cpb = py_proto_api->GetMessagePointer(py); - if (cpb == nullptr) { - // Clients might depend on our non-copying semantics, so we can't fall - // back on CLIF here but instead must fail loudly. - PyErr_SetString(PyExc_RuntimeError, - "Python protobuf did not contain a C++ protobuf"); - return false; - } else { - c->p_ = dynamic_cast(cpb); - if (c->p_ == nullptr) { - // DO NOT DELETE THIS WARNING! Without it, the above dynamic_cast - // will fail when running from a Python 3 pip package. - LOG(WARNING) << "Failed to cast type " << typeid(*cpb).name(); - PyErr_SetString(PyExc_RuntimeError, "Dynamic cast failed"); - return false; - } - return true; - } -} - -} // namespace nucleus - -#endif // THIRD_PARTY_NUCLEUS_UTIL_PROTO_CLIF_CONVERTER_H_ diff --git a/third_party/nucleus/util/python/BUILD b/third_party/nucleus/util/python/BUILD index 3ef676e4..5a3f313e 100644 --- a/third_party/nucleus/util/python/BUILD +++ b/third_party/nucleus/util/python/BUILD @@ -1,5 +1,5 @@ -load("//third_party:clif.bzl", "py_clif_cc") # Placeholder: load py_test +load("@org_tensorflow//tensorflow:tensorflow.default.bzl", "pybind_extension", "pybind_library") package( #default_applicable_licenses = ["//third_party/nucleus:license"], @@ -10,12 +10,21 @@ package( licenses(["notice"]) -py_clif_cc( +pybind_extension( name = "math", - srcs = ["math.clif"], - py_deps = [], - pyclif_deps = [], - deps = ["//third_party/nucleus/util:cpp_math"], + srcs = ["math_pybind.cc"], + deps = [ + "//third_party/nucleus/util:cpp_math", + "@pybind11", + ], +) + +pybind_library( + name = "math_cclib", + srcs = ["math_pybind.cc"], + deps = [ + "//third_party/nucleus/util:cpp_math", + ], ) py_test( @@ -31,17 +40,33 @@ py_test( ], ) -py_clif_cc( +pybind_library( + name = "type_caster_nucleus_proto_ptr", + hdrs = ["type_caster_nucleus_proto_ptr.h"], + deps = [ + "//third_party/nucleus/util:proto_ptr", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_protobuf//:proto_api", + "@com_google_protobuf//:protobuf", + ], +) + +pybind_extension( name = "utils", - srcs = ["utils.clif"], - py_deps = [], - pyclif_deps = [ - "//third_party/nucleus/protos:range_pyclif", - "//third_party/nucleus/protos:reads_pyclif", + srcs = ["utils_pybind.cc"], + deps = [ + ":type_caster_nucleus_proto_ptr", + "//third_party/nucleus/util:cpp_utils", ], +) + +pybind_library( + name = "utils_cclib", + srcs = ["utils_pybind.cc"], deps = [ + ":type_caster_nucleus_proto_ptr", "//third_party/nucleus/util:cpp_utils", - "//third_party/nucleus/util:proto_clif_converter", ], ) @@ -53,6 +78,7 @@ py_test( srcs_version = "PY3", deps = [ ":utils", + "//third_party/nucleus/io:clif_postproc", "//third_party/nucleus/protos:range_py_pb2", "//third_party/nucleus/protos:reads_py_pb2", "@absl_py//absl/testing:absltest", diff --git a/third_party/nucleus/util/python/math.clif b/third_party/nucleus/util/python/math.clif deleted file mode 100644 index 55d09330..00000000 --- a/third_party/nucleus/util/python/math.clif +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/util/math.h": - namespace `nucleus`: - def `Log10PTrueToPhred` as log10_ptrue_to_phred( - log10_ptrue: float, value_if_not_finite: float) -> float - def `PhredToPError` as phred_to_perror(phred: int) -> float - def `PhredToLog10PError` as phred_to_log10_perror(phred: int) -> float - def `PErrorToLog10PError` as perror_to_log10_perror(perror: float) -> float - def `PErrorToPhred` as perror_to_phred(perror: float) -> float - def `Log10PErrorToPhred` as log10_perror_to_phred(log10_perror: float) -> float - def `PErrorToRoundedPhred` as perror_to_rounded_phred(perror: float) -> int - def `Log10PErrorToRoundedPhred` as log10_perror_to_rounded_phred(log10_perror: float) -> int - def `Log10ToReal` as log10_perror_to_perror(log10_perror: float) -> float - def `ZeroShiftLikelihoods` as zero_shift_log10_probs(log10_probs: list) -> list diff --git a/third_party/nucleus/util/python/math_pybind.cc b/third_party/nucleus/util/python/math_pybind.cc index a3306091..ab5e1c13 100755 --- a/third_party/nucleus/util/python/math_pybind.cc +++ b/third_party/nucleus/util/python/math_pybind.cc @@ -36,7 +36,7 @@ #endif #include "third_party/nucleus/util/math.h" -#include "third_party/pybind11/include/pybind11/stl.h" +#include "pybind11/stl.h" namespace py = pybind11; diff --git a/third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h b/third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h index 96dce1ef..eb73aad0 100644 --- a/third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h +++ b/third_party/nucleus/util/python/type_caster_nucleus_proto_ptr.h @@ -41,8 +41,8 @@ #include "absl/log/absl_log.h" #include "third_party/nucleus/util/proto_ptr.h" #include "google/protobuf/message.h" -#include "third_party/py/google/protobuf/proto_api.h" -#include "third_party/pybind11/include/pybind11/gil_safe_call_once.h" +#include "python/google/protobuf/proto_api.h" +#include "pybind11/gil_safe_call_once.h" namespace nucleus { namespace internal { diff --git a/third_party/nucleus/util/python/utils.clif b/third_party/nucleus/util/python/utils.clif deleted file mode 100644 index a5e0b923..00000000 --- a/third_party/nucleus/util/python/utils.clif +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2018 Google LLC. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from "third_party/nucleus/protos/range_pyclif.h" import * -from "third_party/nucleus/protos/reads_pyclif.h" import * -from "third_party/nucleus/util/proto_clif_converter.h" import * - -from "third_party/nucleus/util/utils.h": - namespace `nucleus`: - def `ReadEndPython` as read_end(read: ConstProtoPtr) -> int - def `ReadRangePython` as read_range( - read: ConstProtoPtr, range_pb: EmptyProtoPtr) - def `ReadOverlapsRegionPython` as read_overlaps_region( - read: ConstProtoPtr, range_pb: ConstProtoPtr) -> bool diff --git a/third_party/nucleus/util/python/utils_wrap_test.py b/third_party/nucleus/util/python/utils_wrap_test.py index 85440d83..ba9a1761 100644 --- a/third_party/nucleus/util/python/utils_wrap_test.py +++ b/third_party/nucleus/util/python/utils_wrap_test.py @@ -81,6 +81,15 @@ def test_read_overlaps_region_with_content(self): range_pb.end = 15 self.assertFalse(cpp_utils.read_overlaps_region(read_pb, range_pb)) + def test_EmptyProtoPtr_TypeError(self): + with self.assertRaises(TypeError): + cpp_utils.TestOnlyPassEmptyProtoPtrRead('plain Python str') + + def test_ConstProtoPtr_TypeError(self): + with self.assertRaises(TypeError): + cpp_utils.read_end('plain Python str') + + if __name__ == '__main__': absltest.main() diff --git a/third_party/tensorflow.bzl.patch b/third_party/tensorflow.bzl.patch new file mode 100644 index 00000000..fe71e76b --- /dev/null +++ b/third_party/tensorflow.bzl.patch @@ -0,0 +1,141 @@ +--- a/tensorflow/tensorflow.bzl ++++ b/tensorflow/tensorflow.bzl +@@ -2877,6 +2877,28 @@ + **kwargs + ) + ++def _symlink_impl(ctx): ++ """Creates a symbolic link between src and out.""" ++ out = ctx.outputs.out ++ src = ctx.attr.src.files.to_list()[0] ++ cmd = "ln -f -r -s %s %s" % (src.path, out.path) ++ ctx.actions.run_shell( ++ inputs = [src], ++ outputs = [out], ++ command = cmd, ++ ) ++ ++symlink = rule( ++ implementation = _symlink_impl, ++ attrs = { ++ "src": attr.label( ++ mandatory = True, ++ allow_single_file = True, ++ ), ++ "out": attr.output(mandatory = True), ++ }, ++) ++ + # buildozer: disable=function-docstring-args + def pybind_extension_opensource( + name, +@@ -3012,46 +3034,46 @@ + if link_in_framework: + srcs += tf_binary_additional_srcs() + +- cc_binary( +- name = so_file, +- srcs = srcs + hdrs, +- data = data, +- copts = copts + [ +- "-fno-strict-aliasing", +- "-fexceptions", +- ] + select({ +- clean_dep("//tensorflow:windows"): [], +- "//conditions:default": [ +- "-fvisibility=hidden", +- ], +- }), +- linkopts = linkopts + _rpath_linkopts(name) + select({ +- clean_dep("//tensorflow:macos"): [ +- # TODO: the -w suppresses a wall of harmless warnings about hidden typeinfo symbols +- # not being exported. There should be a better way to deal with this. +- "-Wl,-w", +- "-Wl,-exported_symbols_list,$(location %s)" % exported_symbols_file, +- ], +- clean_dep("//tensorflow:windows"): [], +- "//conditions:default": [ +- "-Wl,--version-script", +- "$(location %s)" % version_script_file, +- ], +- }), +- deps = deps + [ +- exported_symbols_file, +- version_script_file, +- ], +- defines = defines, +- features = features + ["-use_header_modules"], +- linkshared = 1, +- testonly = testonly, +- licenses = licenses, +- visibility = visibility, +- deprecation = deprecation, +- restricted_to = restricted_to, +- compatible_with = compatible_with, +- ) ++ # cc_binary( ++ # name = so_file, ++ # srcs = srcs + hdrs, ++ # data = data, ++ # copts = copts + [ ++ # "-fno-strict-aliasing", ++ # "-fexceptions", ++ # ] + select({ ++ # clean_dep("//tensorflow:windows"): [], ++ # "//conditions:default": [ ++ # "-fvisibility=hidden", ++ # ], ++ # }), ++ # linkopts = linkopts + _rpath_linkopts(name) + select({ ++ # clean_dep("//tensorflow:macos"): [ ++ # # TODO: the -w suppresses a wall of harmless warnings about hidden typeinfo symbols ++ # # not being exported. There should be a better way to deal with this. ++ # "-Wl,-w", ++ # "-Wl,-exported_symbols_list,$(location %s)" % exported_symbols_file, ++ # ], ++ # clean_dep("//tensorflow:windows"): [], ++ # "//conditions:default": [ ++ # "-Wl,--version-script", ++ # "$(location %s)" % version_script_file, ++ # ], ++ # }), ++ # deps = deps + [ ++ # exported_symbols_file, ++ # version_script_file, ++ # ], ++ # defines = defines, ++ # features = features + ["-use_header_modules"], ++ # linkshared = 1, ++ # testonly = testonly, ++ # licenses = licenses, ++ # visibility = visibility, ++ # deprecation = deprecation, ++ # restricted_to = restricted_to, ++ # compatible_with = compatible_with, ++ # ) + + # For Windows, emulate the above filegroup with the shared object. + native.alias( +@@ -3073,11 +3095,21 @@ + testonly = testonly, + ) + ++ # To prevent ODR violations, all of the extensions must live in one ++ # extension module. And to be compatible with existing protobuf ++ # generated code, that module must be _message.so. ++ pyext_so = name + ".so" ++ symlink( ++ name = name + "_symlink", ++ out = pyext_so, ++ src = "@com_google_protobuf//:python/google/protobuf/pyext/_message.so", ++ ) ++ + native.py_library( + name = name, + data = select({ + "@org_tensorflow//tensorflow:windows": [pyd_file], +- "//conditions:default": [so_file], ++ "//conditions:default": [pyext_so], + }) + pytype_srcs, + deps = pytype_deps, + srcs_version = srcs_version, diff --git a/tools/build_clif.sh b/tools/build_absl.sh similarity index 67% rename from tools/build_clif.sh rename to tools/build_absl.sh index d77650f8..02071818 100755 --- a/tools/build_clif.sh +++ b/tools/build_absl.sh @@ -32,17 +32,10 @@ set -eux -o pipefail -echo ========== This script has been tested on Ubuntu18.04 and Ubuntu20.04. -echo ========== See https://github.com/google/clif for how to build on different Unix distributions. +echo ========== This script has been tested on Ubuntu22.04. echo ========== Run this script in root mode. ABSL_PIN="${ABSL_PIN-29bf8085f3bf17b84d30e34b3d7ff8248fda404e}" -PROTOBUF_VERSION=3.13.0 -CLIF_PYTHON_VERSION="${CLIF_PYTHON_VERSION-3.10}" -# CLIF_PIN can be set to a specific commit hash on -# https://github.com/google/clif/commits/main. -# If not set, the default is to checkout the latest commit. -CLIF_PIN="${CLIF_PIN-9ec44bde4f7f40de342a1286f84f5b608633a2d7}" APT_ARGS=( "-y" @@ -64,7 +57,7 @@ NEEDRESTART_MODE=a apt-get install "${APT_ARGS[@]}" --no-install-recommends \ wget \ unzip -# Install CLIF dependencies +# Install dependencies apt-get update "${APT_ARGS[@]}" NEEDRESTART_MODE=a apt-get install "${APT_ARGS[@]}" \ clang-11 \ @@ -90,24 +83,6 @@ make install cd ../.. rm -rf abseil-cpp -# Compile and install protobuf from source -wget "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-cpp-$PROTOBUF_VERSION.tar.gz" && \ - tar -xf "protobuf-cpp-$PROTOBUF_VERSION.tar.gz" && \ - cd "protobuf-$PROTOBUF_VERSION" && \ - # Configure and install C++ libraries - ./autogen.sh && \ - ./configure && \ - make -j"$(nproc)" && \ - make install && \ - ldconfig && \ - cd .. && \ - rm -rf "protobuf-$PROTOBUF_VERSION" "protobuf-cpp-$PROTOBUF_VERSION.tar.gz" - -# Install googletest -cd /usr/src/googletest && \ - cmake . && \ - make install - curl -o get-pip.py https://bootstrap.pypa.io/get-pip.py python3 get-pip.py --force-reinstall --user rm -f get-pip.py @@ -118,21 +93,8 @@ echo "$(pip3 --version)" # Install python runtime and test dependencies pip3 install \ absl-py \ - parameterized \ - protobuf=="$PROTOBUF_VERSION" \ - pyparsing==2.2.2 - -ln -sf /usr/bin/python$CLIF_PYTHON_VERSION /usr/local/bin/python3 - -cd && rm -rf clif && git clone https://github.com/google/clif.git && cd clif - -if [[ ! -z ${CLIF_PIN} ]]; then - git checkout "${CLIF_PIN}" -fi - + parameterized # On GPU machines, this might be necessary because of the reason mentioned in: # https://stackoverflow.com/a/74605488 NEEDRESTART_MODE=a apt-get install "${APT_ARGS[@]}" libstdc++-12-dev - -./INSTALL.sh