From f57032c3e019c049f628fa28c5fa67f0d622169d Mon Sep 17 00:00:00 2001 From: kuaashish <98159216+kuaashish@users.noreply.github.com> Date: Fri, 17 May 2024 16:12:16 +0530 Subject: [PATCH 001/126] Update 00-task-issue-template.yaml --- .github/ISSUE_TEMPLATE/00-task-issue-template.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/00-task-issue-template.yaml b/.github/ISSUE_TEMPLATE/00-task-issue-template.yaml index d6130edb6d..9576d40ad7 100644 --- a/.github/ISSUE_TEMPLATE/00-task-issue-template.yaml +++ b/.github/ISSUE_TEMPLATE/00-task-issue-template.yaml @@ -52,12 +52,11 @@ body: label: Describe the expected behaviour validations: required: true - - type: textarea + - type: input id: what-happened_model attributes: label: Standalone code/steps you may have used to try to get what you need description: If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem - render: shell validations: required: true - type: textarea From 22709acc7a9f65afda7a1027f71946eb76e5c3cc Mon Sep 17 00:00:00 2001 From: kuaashish <98159216+kuaashish@users.noreply.github.com> Date: Fri, 17 May 2024 16:12:38 +0530 Subject: [PATCH 002/126] Update 11-model-maker-issue-template.yaml --- .github/ISSUE_TEMPLATE/11-model-maker-issue-template.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/11-model-maker-issue-template.yaml b/.github/ISSUE_TEMPLATE/11-model-maker-issue-template.yaml index 7a6d92152f..dcc4d8aaa5 100644 --- a/.github/ISSUE_TEMPLATE/11-model-maker-issue-template.yaml +++ b/.github/ISSUE_TEMPLATE/11-model-maker-issue-template.yaml @@ -53,12 +53,11 @@ body: label: Describe the expected behaviour validations: required: true - - type: textarea + - type: input id: what-happened_model attributes: label: Standalone code/steps you may have used to try to get what you need description: If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem - render: shell validations: required: true - type: textarea From 171131e9ea19aef1153564129727039ef653ee81 Mon Sep 17 00:00:00 2001 From: kuaashish <98159216+kuaashish@users.noreply.github.com> Date: Fri, 17 May 2024 16:13:09 +0530 Subject: [PATCH 003/126] Update 12-studio-issue-template.yaml --- .github/ISSUE_TEMPLATE/12-studio-issue-template.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/12-studio-issue-template.yaml b/.github/ISSUE_TEMPLATE/12-studio-issue-template.yaml index ffaa315f94..135e7b7810 100644 --- a/.github/ISSUE_TEMPLATE/12-studio-issue-template.yaml +++ b/.github/ISSUE_TEMPLATE/12-studio-issue-template.yaml @@ -43,12 +43,11 @@ body: label: Describe the expected behaviour validations: required: false - - type: textarea + - type: input id: what-happened_model attributes: label: Standalone code/steps you may have used to try to get what you need description: If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem - render: shell validations: required: false - type: textarea From 2f1e21ee82a60bed41561f0cd501f15f194a3942 Mon Sep 17 00:00:00 2001 From: kuaashish <98159216+kuaashish@users.noreply.github.com> Date: Fri, 17 May 2024 16:14:14 +0530 Subject: [PATCH 004/126] Update 16-bug-issue-template.yaml --- .github/ISSUE_TEMPLATE/16-bug-issue-template.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/16-bug-issue-template.yaml b/.github/ISSUE_TEMPLATE/16-bug-issue-template.yaml index efa925b44c..206e3c6abf 100644 --- a/.github/ISSUE_TEMPLATE/16-bug-issue-template.yaml +++ b/.github/ISSUE_TEMPLATE/16-bug-issue-template.yaml @@ -92,12 +92,11 @@ body: label: Describe the expected behaviour validations: required: true - - type: textarea + - type: input id: what-happened_model attributes: label: Standalone code/steps you may have used to try to get what you need description: If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem - render: shell validations: required: true - type: textarea From 1a4ad34a608c128884c722b8b71539d6be5c8b21 Mon Sep 17 00:00:00 2001 From: kuaashish <98159216+kuaashish@users.noreply.github.com> Date: Fri, 17 May 2024 16:14:45 +0530 Subject: [PATCH 005/126] Update 18-solution-legacy-issue-template.yaml --- .github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml b/.github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml index acb0f5b891..dc477b3270 100644 --- a/.github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml +++ b/.github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml @@ -60,12 +60,11 @@ body: label: Describe the expected behaviour validations: required: false - - type: textarea + - type: input id: what-happened_model attributes: label: Standalone code/steps you may have used to try to get what you need description: If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem - render: shell validations: required: false - type: textarea From 214f44113e46505bb0bffbc29f01a76bb107a146 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 25 Oct 2024 03:27:42 -0700 Subject: [PATCH 006/126] Improve message when input tensor mismatch is detected. PiperOrigin-RevId: 689723445 --- mediapipe/calculators/tensor/inference_calculator_utils.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mediapipe/calculators/tensor/inference_calculator_utils.cc b/mediapipe/calculators/tensor/inference_calculator_utils.cc index 829bc676e5..8d7761f6ab 100644 --- a/mediapipe/calculators/tensor/inference_calculator_utils.cc +++ b/mediapipe/calculators/tensor/inference_calculator_utils.cc @@ -240,7 +240,11 @@ absl::Status CopyCpuInputIntoTfLiteTensor(const Tensor& input_tensor, const Tensor::ElementType input_tensor_type = input_tensor.element_type(); RET_CHECK(input_tensor_type == interpreter_tensor_type) .SetCode(absl::StatusCode::kInvalidArgument) - << "Input and interpreter tensor type do not match."; + << absl::StrFormat( + "Input and interpreter tensor type do not match: Input tensor " + "type %s vs interpreter tensor type %s.", + GetTensorTypeString(input_tensor_type), + TfLiteTypeGetName(interpreter_tensor_type)); switch (interpreter_tensor_type) { case TfLiteType::kTfLiteFloat16: case TfLiteType::kTfLiteFloat32: { From 7a05846bd3930e4f13e4fe11220da68a9aa04aae Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 25 Oct 2024 13:26:35 -0700 Subject: [PATCH 007/126] Fix ParseFromString() compilation issue in OSS PiperOrigin-RevId: 689892908 --- .../modules/face_geometry/geometry_pipeline_calculator.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc b/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc index 35efb005a7..cad22dd21f 100644 --- a/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc +++ b/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc @@ -169,7 +169,8 @@ class GeometryPipelineCalculator : public CalculatorBase { _ << "Failed to read a metadata blob from file!"); face_geometry::GeometryPipelineMetadata metadata; - RET_CHECK(metadata.ParseFromString(metadata_blob->ToStringView())) + absl::string_view metadata_str = metadata_blob->ToStringView(); + RET_CHECK(metadata.ParseFromArray(metadata_str.data(), metadata_str.size())) << "Failed to parse a metadata proto from a binary blob!"; return metadata; From a616f81eb85996f438015d136868511cba3c37d2 Mon Sep 17 00:00:00 2001 From: Pulkit Midha Date: Sat, 26 Oct 2024 20:21:55 +0530 Subject: [PATCH 008/126] All the dead links fixed --- .../15-build-install-issue-template.yaml | 4 +-- .../ISSUE_TEMPLATE/16-bug-issue-template.yaml | 2 +- .../18-solution-legacy-issue-template.yaml | 4 +-- docs/solutions/models.md | 22 ++++++------ docs/solutions/solutions.md | 36 +++++++++---------- mediapipe/docs/autoflip.md | 2 +- mediapipe/docs/face_detection_desktop.md | 2 +- mediapipe/docs/face_detection_mobile_gpu.md | 2 +- .../docs/hair_segmentation_mobile_gpu.md | 2 +- mediapipe/docs/hand_tracking_desktop.md | 2 +- mediapipe/docs/hand_tracking_mobile_gpu.md | 2 +- .../docs/multi_hand_tracking_mobile_gpu.md | 2 +- mediapipe/docs/object_detection_desktop.md | 2 +- mediapipe/docs/object_detection_mobile_gpu.md | 2 +- mediapipe/docs/object_tracking_mobile_gpu.md | 2 +- mediapipe/docs/objectron_mobile_gpu.md | 2 +- .../docs/template_matching_mobile_cpu.md | 2 +- .../vision/gesture_recognizer/dataset.py | 2 +- .../modules/objectron/calculators/epnp.cc | 2 +- 19 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/15-build-install-issue-template.yaml b/.github/ISSUE_TEMPLATE/15-build-install-issue-template.yaml index ded9d09a63..040e859a84 100644 --- a/.github/ISSUE_TEMPLATE/15-build-install-issue-template.yaml +++ b/.github/ISSUE_TEMPLATE/15-build-install-issue-template.yaml @@ -5,7 +5,7 @@ body: - type: markdown id: link attributes: - value: Please make sure that this is a build/installation issue and also refer to the [troubleshooting](https://google.github.io/mediapipe/getting_started/troubleshooting.html) documentation before raising any issues. + value: Please make sure that this is a build/installation issue and also refer to the [troubleshooting](https://google-ai-edge.github.io/mediapipe/getting_started/troubleshooting.html) documentation before raising any issues. - type: input id: os attributes: @@ -91,7 +91,7 @@ body: id: what-happened attributes: label: Describe the problem - description: Provide the exact sequence of commands / steps that you executed before running into the [problem](https://google.github.io/mediapipe/getting_started/getting_started.html) + description: Provide the exact sequence of commands / steps that you executed before running into the [problem](https://google-ai-edge.github.io/mediapipe/getting_started/getting_started.html) placeholder: Tell us what you see! value: "A bug happened!" validations: diff --git a/.github/ISSUE_TEMPLATE/16-bug-issue-template.yaml b/.github/ISSUE_TEMPLATE/16-bug-issue-template.yaml index efa925b44c..b7ae46485a 100644 --- a/.github/ISSUE_TEMPLATE/16-bug-issue-template.yaml +++ b/.github/ISSUE_TEMPLATE/16-bug-issue-template.yaml @@ -5,7 +5,7 @@ body: - type: markdown id: link attributes: - value: Please make sure that this is a bug and also refer to the [troubleshooting](https://google.github.io/mediapipe/getting_started/troubleshooting.html), FAQ documentation before raising any issues. + value: Please make sure that this is a bug and also refer to the [troubleshooting](https://google-ai-edge.github.io/mediapipe/getting_started/troubleshooting.html), FAQ documentation before raising any issues. - type: dropdown id: customcode_model attributes: diff --git a/.github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml b/.github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml index acb0f5b891..b755336129 100644 --- a/.github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml +++ b/.github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml @@ -1,11 +1,11 @@ name: Solution(Legacy) Issue -description: Use this template for assistance with a specific Mediapipe solution (google.github.io/mediapipe/solutions) such as "Pose", including inference model usage/training, solution-specific calculators etc. +description: Use this template for assistance with a specific Mediapipe solution (google-ai-edge.github.io/mediapipe/solutions) such as "Pose", including inference model usage/training, solution-specific calculators etc. labels: 'type:support' body: - type: markdown id: linkmodel attributes: - value: Please make sure that this is a [solution](https://google.github.io/mediapipe/solutions/solutions.html) issue. + value: Please make sure that this is a [solution](https://google-ai-edge.github.io/mediapipe/solutions/solutions.html) issue. - type: dropdown id: customcode_model attributes: diff --git a/docs/solutions/models.md b/docs/solutions/models.md index 0af91eb489..5c52296cd5 100644 --- a/docs/solutions/models.md +++ b/docs/solutions/models.md @@ -25,7 +25,7 @@ We encourage you to check out the new MediaPipe Solutions at: ---- -### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection) +### [Face Detection](https://google-ai-edge.github.io/mediapipe/solutions/face_detection) * Short-range model (best for faces within 2 meters from the camera): [TFLite model](https://storage.googleapis.com/mediapipe-assets/face_detection_short_range.tflite), @@ -49,7 +49,7 @@ sparse model is ~30% faster when executing on CPU via demonstrate comparable latencies. Depending on your application, you may prefer one over the other. -### [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) +### [Face Mesh](https://google-ai-edge.github.io/mediapipe/solutions/face_mesh) * Face landmark model: [TFLite model](https://storage.googleapis.com/mediapipe-assets/face_landmark.tflite), @@ -59,13 +59,13 @@ one over the other. * [Model card](https://mediapipe.page.link/facemesh-mc), [Model card (w/ attention)](https://mediapipe.page.link/attentionmesh-mc) -### [Iris](https://google.github.io/mediapipe/solutions/iris) +### [Iris](https://google-ai-edge.github.io/mediapipe/solutions/iris) * Iris landmark model: [TFLite model](https://storage.googleapis.com/mediapipe-assets/iris_landmark.tflite) * [Model card](https://mediapipe.page.link/iris-mc) -### [Hands](https://google.github.io/mediapipe/solutions/hands) +### [Hands](https://google-ai-edge.github.io/mediapipe/solutions/hands) * Palm detection model: [TFLite model (lite)](https://storage.googleapis.com/mediapipe-assets/palm_detection_lite.tflite), @@ -77,7 +77,7 @@ one over the other. [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1) * [Model card](https://mediapipe.page.link/handmc) -### [Pose](https://google.github.io/mediapipe/solutions/pose) +### [Pose](https://google-ai-edge.github.io/mediapipe/solutions/pose) * Pose detection model: [TFLite model](https://storage.googleapis.com/mediapipe-assets/pose_detection.tflite) @@ -87,30 +87,30 @@ one over the other. [TFLite model (heavy)](https://storage.googleapis.com/mediapipe-assets/pose_landmark_heavy.tflite) * [Model card](https://mediapipe.page.link/blazepose-mc) -### [Holistic](https://google.github.io/mediapipe/solutions/holistic) +### [Holistic](https://google-ai-edge.github.io/mediapipe/solutions/holistic) * Hand recrop model: [TFLite model](https://storage.googleapis.com/mediapipe-assets/hand_recrop.tflite) -### [Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) +### [Selfie Segmentation](https://google-ai-edge.github.io/mediapipe/solutions/selfie_segmentation) * [TFLite model (general)](https://storage.googleapis.com/mediapipe-assets/selfie_segmentation.tflite) * [TFLite model (landscape)](https://storage.googleapis.com/mediapipe-assets/selfie_segmentation_landscape.tflite) * [Model card](https://mediapipe.page.link/selfiesegmentation-mc) -### [Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) +### [Hair Segmentation](https://google-ai-edge.github.io/mediapipe/solutions/hair_segmentation) * [TFLite model](https://storage.googleapis.com/mediapipe-assets/hair_segmentation.tflite) * [Model card](https://mediapipe.page.link/hairsegmentation-mc) -### [Object Detection](https://google.github.io/mediapipe/solutions/object_detection) +### [Object Detection](https://google-ai-edge.github.io/mediapipe/solutions/object_detection) * [TFLite model](https://storage.googleapis.com/mediapipe-assets/ssdlite_object_detection.tflite) * [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite) * [TensorFlow model](https://storage.googleapis.com/mediapipe-assets/object_detection_saved_model/archive.zip) * [Model information](https://storage.googleapis.com/mediapipe-assets/object_detection_saved_model/README.md) -### [Objectron](https://google.github.io/mediapipe/solutions/objectron) +### [Objectron](https://google-ai-edge.github.io/mediapipe/solutions/objectron) * [TFLite model for shoes](https://storage.googleapis.com/mediapipe-assets/object_detection_3d_sneakers.tflite) * [TFLite model for chairs](https://storage.googleapis.com/mediapipe-assets/object_detection_3d_chair.tflite) @@ -120,7 +120,7 @@ one over the other. * [Single-stage TFLite model for chairs](https://storage.googleapis.com/mediapipe-assets/object_detection_3d_chair_1stage.tflite) * [Model card](https://mediapipe.page.link/objectron-mc) -### [KNIFT](https://google.github.io/mediapipe/solutions/knift) +### [KNIFT](https://google-ai-edge.github.io/mediapipe/solutions/knift) * [TFLite model for up to 200 keypoints](https://storage.googleapis.com/mediapipe-assets/knift_float.tflite) * [TFLite model for up to 400 keypoints](https://storage.googleapis.com/mediapipe-assets/knift_float_400.tflite) diff --git a/docs/solutions/solutions.md b/docs/solutions/solutions.md index 10551b7c90..b1aebe277e 100644 --- a/docs/solutions/solutions.md +++ b/docs/solutions/solutions.md @@ -33,25 +33,25 @@ and streaming media. -[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md) +[]() | [Android](https://google-ai-edge.github.io/mediapipe/getting_started/android) | [iOS](https://google-ai-edge.github.io/mediapipe/getting_started/ios) | [C++](https://google-ai-edge.github.io/mediapipe/getting_started/cpp) | [Python](https://google-ai-edge.github.io/mediapipe/getting_started/python) | [JS](https://google-ai-edge.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md) :---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------: -[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ -[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ | -[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | | -[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ | -[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ | -[Holistic](https://google.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ | -[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ | -[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | | -[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅ -[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | | -[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | | -[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ | -[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | | -[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | | -[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | | -[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | | +[Face Detection](https://google-ai-edge.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ +[Face Mesh](https://google-ai-edge.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ | +[Iris](https://google-ai-edge.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | | +[Hands](https://google-ai-edge.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ | +[Pose](https://google-ai-edge.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ | +[Holistic](https://google-ai-edge.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ | +[Selfie Segmentation](https://google-ai-edge.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ | +[Hair Segmentation](https://google-ai-edge.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | | +[Object Detection](https://google-ai-edge.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅ +[Box Tracking](https://google-ai-edge.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | | +[Instant Motion Tracking](https://google-ai-edge.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | | +[Objectron](https://google-ai-edge.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ | +[KNIFT](https://google-ai-edge.github.io/mediapipe/solutions/knift) | ✅ | | | | | +[AutoFlip](https://google-ai-edge.github.io/mediapipe/solutions/autoflip) | | | ✅ | | | +[MediaSequence](https://google-ai-edge.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | | +[YouTube 8M](https://google-ai-edge.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | | See also -[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models) +[MediaPipe Models and Model Cards](https://google-ai-edge.github.io/mediapipe/solutions/models) for ML models released in MediaPipe. diff --git a/mediapipe/docs/autoflip.md b/mediapipe/docs/autoflip.md index 4d7367810d..1765595f3b 100644 --- a/mediapipe/docs/autoflip.md +++ b/mediapipe/docs/autoflip.md @@ -1,2 +1,2 @@ Content moved to -[AutoFlip: Saliency-aware Video Cropping](https://google.github.io/mediapipe/solutions/autoflip) +[AutoFlip: Saliency-aware Video Cropping](https://google-ai-edge.github.io/mediapipe/solutions/autoflip) diff --git a/mediapipe/docs/face_detection_desktop.md b/mediapipe/docs/face_detection_desktop.md index 8377e8df1d..42a8aab7ca 100644 --- a/mediapipe/docs/face_detection_desktop.md +++ b/mediapipe/docs/face_detection_desktop.md @@ -1,2 +1,2 @@ Content moved to -[MediaPipe Face Detection](https://google.github.io/mediapipe/solutions/face_detection) +[MediaPipe Face Detection](https://google-ai-edge.github.io/mediapipe/solutions/face_detection) diff --git a/mediapipe/docs/face_detection_mobile_gpu.md b/mediapipe/docs/face_detection_mobile_gpu.md index 8377e8df1d..42a8aab7ca 100644 --- a/mediapipe/docs/face_detection_mobile_gpu.md +++ b/mediapipe/docs/face_detection_mobile_gpu.md @@ -1,2 +1,2 @@ Content moved to -[MediaPipe Face Detection](https://google.github.io/mediapipe/solutions/face_detection) +[MediaPipe Face Detection](https://google-ai-edge.github.io/mediapipe/solutions/face_detection) diff --git a/mediapipe/docs/hair_segmentation_mobile_gpu.md b/mediapipe/docs/hair_segmentation_mobile_gpu.md index 43116a4f65..08236d7f80 100644 --- a/mediapipe/docs/hair_segmentation_mobile_gpu.md +++ b/mediapipe/docs/hair_segmentation_mobile_gpu.md @@ -1,2 +1,2 @@ Content moved to -[MediaPipe Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) +[MediaPipe Hair Segmentation](https://google-ai-edge.github.io/mediapipe/solutions/hair_segmentation) diff --git a/mediapipe/docs/hand_tracking_desktop.md b/mediapipe/docs/hand_tracking_desktop.md index 02bb1312cc..f9dc45399c 100644 --- a/mediapipe/docs/hand_tracking_desktop.md +++ b/mediapipe/docs/hand_tracking_desktop.md @@ -1 +1 @@ -Content moved to [MediaPipe Hands](https://google.github.io/mediapipe/solutions/hands) +Content moved to [MediaPipe Hands](https://google-ai-edge.github.io/mediapipe/solutions/hands) diff --git a/mediapipe/docs/hand_tracking_mobile_gpu.md b/mediapipe/docs/hand_tracking_mobile_gpu.md index 02bb1312cc..f9dc45399c 100644 --- a/mediapipe/docs/hand_tracking_mobile_gpu.md +++ b/mediapipe/docs/hand_tracking_mobile_gpu.md @@ -1 +1 @@ -Content moved to [MediaPipe Hands](https://google.github.io/mediapipe/solutions/hands) +Content moved to [MediaPipe Hands](https://google-ai-edge.github.io/mediapipe/solutions/hands) diff --git a/mediapipe/docs/multi_hand_tracking_mobile_gpu.md b/mediapipe/docs/multi_hand_tracking_mobile_gpu.md index 02bb1312cc..f9dc45399c 100644 --- a/mediapipe/docs/multi_hand_tracking_mobile_gpu.md +++ b/mediapipe/docs/multi_hand_tracking_mobile_gpu.md @@ -1 +1 @@ -Content moved to [MediaPipe Hands](https://google.github.io/mediapipe/solutions/hands) +Content moved to [MediaPipe Hands](https://google-ai-edge.github.io/mediapipe/solutions/hands) diff --git a/mediapipe/docs/object_detection_desktop.md b/mediapipe/docs/object_detection_desktop.md index 2e565cefda..3313fd0781 100644 --- a/mediapipe/docs/object_detection_desktop.md +++ b/mediapipe/docs/object_detection_desktop.md @@ -1,2 +1,2 @@ Content moved to -[MediaPipe Object Detection](https://google.github.io/mediapipe/solutions/object_detection) +[MediaPipe Object Detection](https://google-ai-edge.github.io/mediapipe/solutions/object_detection) diff --git a/mediapipe/docs/object_detection_mobile_gpu.md b/mediapipe/docs/object_detection_mobile_gpu.md index 2e565cefda..3313fd0781 100644 --- a/mediapipe/docs/object_detection_mobile_gpu.md +++ b/mediapipe/docs/object_detection_mobile_gpu.md @@ -1,2 +1,2 @@ Content moved to -[MediaPipe Object Detection](https://google.github.io/mediapipe/solutions/object_detection) +[MediaPipe Object Detection](https://google-ai-edge.github.io/mediapipe/solutions/object_detection) diff --git a/mediapipe/docs/object_tracking_mobile_gpu.md b/mediapipe/docs/object_tracking_mobile_gpu.md index c74d942f6a..89a8ff7840 100644 --- a/mediapipe/docs/object_tracking_mobile_gpu.md +++ b/mediapipe/docs/object_tracking_mobile_gpu.md @@ -1,2 +1,2 @@ Content moved to -[MediaPipe Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) +[MediaPipe Box Tracking](https://google-ai-edge.github.io/mediapipe/solutions/box_tracking) diff --git a/mediapipe/docs/objectron_mobile_gpu.md b/mediapipe/docs/objectron_mobile_gpu.md index 231fc512ca..19e02b9c57 100644 --- a/mediapipe/docs/objectron_mobile_gpu.md +++ b/mediapipe/docs/objectron_mobile_gpu.md @@ -1,2 +1,2 @@ Content moved to -[MediaPipe Objectron](https://google.github.io/mediapipe/solutions/objectron) +[MediaPipe Objectron](https://google-ai-edge.github.io/mediapipe/solutions/objectron) diff --git a/mediapipe/docs/template_matching_mobile_cpu.md b/mediapipe/docs/template_matching_mobile_cpu.md index 02150175cc..a47f5ea716 100644 --- a/mediapipe/docs/template_matching_mobile_cpu.md +++ b/mediapipe/docs/template_matching_mobile_cpu.md @@ -1,2 +1,2 @@ Content moved to -[MediaPipe KNIFT](https://google.github.io/mediapipe/solutions/knift) +[MediaPipe KNIFT](https://google-ai-edge.github.io/mediapipe/solutions/knift) diff --git a/mediapipe/model_maker/python/vision/gesture_recognizer/dataset.py b/mediapipe/model_maker/python/vision/gesture_recognizer/dataset.py index 8e2095a33d..31436c653c 100644 --- a/mediapipe/model_maker/python/vision/gesture_recognizer/dataset.py +++ b/mediapipe/model_maker/python/vision/gesture_recognizer/dataset.py @@ -50,7 +50,7 @@ class HandDataPreprocessingParams: class HandData: """A dataclass represents hand data for training gesture recognizer model. - See https://google.github.io/mediapipe/solutions/hands#mediapipe-hands for + See https://google-ai-edge.github.io/mediapipe/solutions/hands#mediapipe-hands for more details of the hand gesture data API. Attributes: diff --git a/mediapipe/modules/objectron/calculators/epnp.cc b/mediapipe/modules/objectron/calculators/epnp.cc index 03b78c7284..eed3883c23 100644 --- a/mediapipe/modules/objectron/calculators/epnp.cc +++ b/mediapipe/modules/objectron/calculators/epnp.cc @@ -99,7 +99,7 @@ absl::Status SolveEpnp(const float focal_x, const float focal_y, // Convert 2d point from `pixel coordinates` to `NDC coordinates`([-1, 1]) // following to the definitions in: - // https://google.github.io/mediapipe/solutions/objectron#ndc-space + // https://google-ai-edge.github.io/mediapipe/solutions/objectron#ndc-space // If portrait mode is been used, it's the caller's responsibility to // convert the input 2d points' coordinates. float x_ndc, y_ndc; From 7571b1b77683a9d64ea9972d7c9d5f09666b9ba7 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 28 Oct 2024 12:44:03 +0530 Subject: [PATCH 009/126] Fixed empty pose world landmarks in iOS holistic landmarker --- .../holistic_landmarker/MPPHolisticLandmarkerTests.mm | 9 +++++++++ .../utils/sources/MPPHolisticLandmarkerResult+Helpers.mm | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/test/vision/holistic_landmarker/MPPHolisticLandmarkerTests.mm b/mediapipe/tasks/ios/test/vision/holistic_landmarker/MPPHolisticLandmarkerTests.mm index d36a7d7ed5..943e76eae9 100644 --- a/mediapipe/tasks/ios/test/vision/holistic_landmarker/MPPHolisticLandmarkerTests.mm +++ b/mediapipe/tasks/ios/test/vision/holistic_landmarker/MPPHolisticLandmarkerTests.mm @@ -533,17 +533,26 @@ - (void)assertHolisticLandmarkerResult:(MPPHolisticLandmarkerResult *)holisticLa withLandmarkTypeName:@"pose_landmarks" areApproximatelyEqualToExpectedNormalizedLandmarks:expectedHolisticLandmarkerResult .poseLandmarks]; + // Comparing world landmark counts of the actual result to pose landmark counts of expected result + // to ensure world landmarks are present since expected result does not contain world landmarks. + XCTAssertEqual(holisticLandmarkerResult.poseWorldLandmarks.count, + expectedHolisticLandmarkerResult.poseLandmarks.count); [self assertNormalizedLandmarks:holisticLandmarkerResult.leftHandLandmarks withLandmarkTypeName:@"left_hand_landmarks" areApproximatelyEqualToExpectedNormalizedLandmarks:expectedHolisticLandmarkerResult .leftHandLandmarks]; + XCTAssertEqual(holisticLandmarkerResult.leftHandWorldLandmarks.count, + expectedHolisticLandmarkerResult.leftHandLandmarks.count); [self assertNormalizedLandmarks:holisticLandmarkerResult.rightHandLandmarks withLandmarkTypeName:@"right_hand_landmarks" areApproximatelyEqualToExpectedNormalizedLandmarks:expectedHolisticLandmarkerResult .rightHandLandmarks]; + XCTAssertEqual(holisticLandmarkerResult.rightHandWorldLandmarks.count, + expectedHolisticLandmarkerResult.rightHandLandmarks.count); + [self assertFaceBlendshapes:holisticLandmarkerResult.faceBlendshapes areApproximatelyEqualToExpectedFaceBlendshapes:expectedHolisticLandmarkerResult .faceBlendshapes]; diff --git a/mediapipe/tasks/ios/vision/holistic_landmarker/utils/sources/MPPHolisticLandmarkerResult+Helpers.mm b/mediapipe/tasks/ios/vision/holistic_landmarker/utils/sources/MPPHolisticLandmarkerResult+Helpers.mm index 3cf088dd65..0f202fe96b 100644 --- a/mediapipe/tasks/ios/vision/holistic_landmarker/utils/sources/MPPHolisticLandmarkerResult+Helpers.mm +++ b/mediapipe/tasks/ios/vision/holistic_landmarker/utils/sources/MPPHolisticLandmarkerResult+Helpers.mm @@ -73,7 +73,8 @@ @implementation MPPHolisticLandmarkerResult (Helpers) faceBlendshapesProto:faceBlendshapesProto poseLandmarksProto:NormalizedLandmarkListFromPacket( poseLandmarksPacket) - poseWorldLandmarksProto:LandmarkListFromPacket(poseLandmarksPacket) + poseWorldLandmarksProto:LandmarkListFromPacket( + poseWorldLandmarksPacket) poseSegmentationMaskProto:poseSegmentationMaskProto leftHandLandmarksProto:NormalizedLandmarkListFromPacket( leftHandLandmarksPacket) From 51d1f3bdd76daae9b44e8be0ff6b3251e6ce83a9 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 28 Oct 2024 08:18:30 -0700 Subject: [PATCH 010/126] Adds the canonical `toBuilder` method to the `LlmInferenceOptions` object. PiperOrigin-RevId: 690617797 --- .../mediapipe/tasks/genai/llminference/LlmInference.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInference.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInference.java index 9f09ec9128..dba1e5b955 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInference.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInference.java @@ -232,6 +232,9 @@ public final LlmInferenceOptions build() { /** The error listener to use for the {@link LlmInference#generateAsync} API. */ public abstract Optional errorListener(); + /** Returns a new builder with the same values as this instance. */ + public abstract Builder toBuilder(); + /** Instantiates a new LlmInferenceOptions builder. */ public static Builder builder() { return new AutoValue_LlmInference_LlmInferenceOptions.Builder() From fbc49991ab6b7616f984735f488320251ddc4402 Mon Sep 17 00:00:00 2001 From: Laura Pak Date: Mon, 28 Oct 2024 10:43:16 -0700 Subject: [PATCH 011/126] Move tensorflow lite python calls to ai-edge-litert. PiperOrigin-RevId: 690670455 --- mediapipe/model_maker/python/core/utils/BUILD | 2 -- mediapipe/model_maker/python/core/utils/model_util.py | 3 +-- .../model_maker/python/core/utils/quantization_test.py | 5 ++--- mediapipe/model_maker/requirements.txt | 1 - mediapipe/model_maker/requirements_bazel.txt | 1 - mediapipe/model_maker/requirements_lock.txt | 7 +------ 6 files changed, 4 insertions(+), 15 deletions(-) diff --git a/mediapipe/model_maker/python/core/utils/BUILD b/mediapipe/model_maker/python/core/utils/BUILD index ef6e652845..4b8ff1bf97 100644 --- a/mediapipe/model_maker/python/core/utils/BUILD +++ b/mediapipe/model_maker/python/core/utils/BUILD @@ -44,7 +44,6 @@ py_library( deps = [ ":quantization", "//mediapipe/model_maker/python/core/data:dataset", - "@model_maker_pip_deps_ai_edge_litert_nightly//:pkg", "@model_maker_pip_deps_numpy//:pkg", "@model_maker_pip_deps_tensorflow//:pkg", ], @@ -173,7 +172,6 @@ py_test( ":quantization", ":test_util", "@model_maker_pip_deps_absl_py//:pkg", - "@model_maker_pip_deps_ai_edge_litert_nightly//:pkg", "@model_maker_pip_deps_tensorflow//:pkg", ], ) diff --git a/mediapipe/model_maker/python/core/utils/model_util.py b/mediapipe/model_maker/python/core/utils/model_util.py index 0a13095e6a..32b509797f 100644 --- a/mediapipe/model_maker/python/core/utils/model_util.py +++ b/mediapipe/model_maker/python/core/utils/model_util.py @@ -28,7 +28,6 @@ from mediapipe.model_maker.python.core.data import dataset from mediapipe.model_maker.python.core.utils import quantization -from ai_edge_litert import interpreter as tfl_interpreter DEFAULT_SCALE, DEFAULT_ZERO_POINT = 0, 0 ESTIMITED_STEPS_PER_EPOCH = 1000 @@ -274,7 +273,7 @@ def __init__(self, tflite_model: bytearray): Args: tflite_model: A valid flatbuffer representing the TFLite model. """ - self.interpreter = tfl_interpreter.Interpreter(model_content=tflite_model) + self.interpreter = tf.lite.Interpreter(model_content=tflite_model) self.interpreter.allocate_tensors() self.input_details = self.interpreter.get_input_details() self.output_details = self.interpreter.get_output_details() diff --git a/mediapipe/model_maker/python/core/utils/quantization_test.py b/mediapipe/model_maker/python/core/utils/quantization_test.py index 0164d39bf8..57523d4056 100644 --- a/mediapipe/model_maker/python/core/utils/quantization_test.py +++ b/mediapipe/model_maker/python/core/utils/quantization_test.py @@ -17,7 +17,6 @@ from mediapipe.model_maker.python.core.utils import quantization from mediapipe.model_maker.python.core.utils import test_util -from ai_edge_litert import interpreter as tfl_interpreter class QuantizationTest(tf.test.TestCase, parameterized.TestCase): @@ -60,7 +59,7 @@ def test_set_converter_with_quantization_from_int8_config(self): self.assertEqual(config.supported_ops, [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]) tflite_model = converter.convert() - interpreter = tfl_interpreter.Interpreter(model_content=tflite_model) + interpreter = tf.lite.Interpreter(model_content=tflite_model) self.assertEqual(interpreter.get_input_details()[0]['dtype'], tf.uint8) self.assertEqual(interpreter.get_output_details()[0]['dtype'], tf.uint8) @@ -83,7 +82,7 @@ def test_set_converter_with_quantization_from_float16_config(self): converter = config.set_converter_with_quantization(converter=converter) self.assertEqual(config.supported_types, [tf.float16]) tflite_model = converter.convert() - interpreter = tfl_interpreter.Interpreter(model_content=tflite_model) + interpreter = tf.lite.Interpreter(model_content=tflite_model) # The input and output are expected to be set to float32 by default. self.assertEqual(interpreter.get_input_details()[0]['dtype'], tf.float32) self.assertEqual(interpreter.get_output_details()[0]['dtype'], tf.float32) diff --git a/mediapipe/model_maker/requirements.txt b/mediapipe/model_maker/requirements.txt index 419d0166ca..3ce977b671 100644 --- a/mediapipe/model_maker/requirements.txt +++ b/mediapipe/model_maker/requirements.txt @@ -1,5 +1,4 @@ absl-py -ai-edge-litert-nightly mediapipe>=0.10.0 numpy<2 opencv-python diff --git a/mediapipe/model_maker/requirements_bazel.txt b/mediapipe/model_maker/requirements_bazel.txt index 003085ed07..fd6c421cf8 100644 --- a/mediapipe/model_maker/requirements_bazel.txt +++ b/mediapipe/model_maker/requirements_bazel.txt @@ -1,5 +1,4 @@ absl-py -ai-edge-litert-nightly numpy<2 opencv-python setuptools==70.3.0 # needed due to https://github.com/pypa/setuptools/issues/4487 diff --git a/mediapipe/model_maker/requirements_lock.txt b/mediapipe/model_maker/requirements_lock.txt index 62972c57da..83eb07a121 100644 --- a/mediapipe/model_maker/requirements_lock.txt +++ b/mediapipe/model_maker/requirements_lock.txt @@ -15,8 +15,6 @@ absl-py==1.4.0 # tensorflow-metadata # tensorflow-model-optimization # tf-slim -ai-edge-litert-nightly==1.0.1.dev20241022 - # via -r mediapipe/opensource_only/model_maker_requirements_bazel.txt array-record==0.5.1 # via tensorflow-datasets astunparse==1.6.3 @@ -50,9 +48,7 @@ etils[array-types,enp,epath,epy,etqdm,etree]==1.5.2 # array-record # tensorflow-datasets flatbuffers==24.3.25 - # via - # ai-edge-litert-nightly - # tensorflow + # via tensorflow fonttools==4.54.1 # via matplotlib fsspec==2024.9.0 @@ -124,7 +120,6 @@ ml-dtypes==0.3.2 numpy==1.26.4 # via # -r mediapipe/opensource_only/model_maker_requirements_bazel.txt - # ai-edge-litert-nightly # contourpy # etils # h5py From b5cbef59a43577b2f467bfe8c3445240e929c285 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 28 Oct 2024 13:23:28 -0700 Subject: [PATCH 012/126] No public description PiperOrigin-RevId: 690728138 --- mediapipe/tasks/web/audio/audio_classifier/BUILD | 2 +- mediapipe/tasks/web/audio/audio_embedder/BUILD | 2 +- mediapipe/tasks/web/components/processors/BUILD | 10 +++++----- mediapipe/tasks/web/components/utils/BUILD | 2 +- mediapipe/tasks/web/core/BUILD | 2 +- mediapipe/tasks/web/text/language_detector/BUILD | 2 +- mediapipe/tasks/web/text/text_classifier/BUILD | 2 +- mediapipe/tasks/web/text/text_embedder/BUILD | 2 +- mediapipe/tasks/web/vision/core/BUILD | 8 ++++---- mediapipe/tasks/web/vision/face_detector/BUILD | 2 +- mediapipe/tasks/web/vision/face_landmarker/BUILD | 2 +- mediapipe/tasks/web/vision/face_stylizer/BUILD | 2 +- mediapipe/tasks/web/vision/gesture_recognizer/BUILD | 2 +- mediapipe/tasks/web/vision/hand_landmarker/BUILD | 2 +- mediapipe/tasks/web/vision/holistic_landmarker/BUILD | 2 +- mediapipe/tasks/web/vision/image_classifier/BUILD | 2 +- mediapipe/tasks/web/vision/image_embedder/BUILD | 2 +- mediapipe/tasks/web/vision/image_segmenter/BUILD | 2 +- mediapipe/tasks/web/vision/interactive_segmenter/BUILD | 2 +- mediapipe/tasks/web/vision/object_detector/BUILD | 2 +- mediapipe/tasks/web/vision/pose_landmarker/BUILD | 2 +- mediapipe/web/graph_runner/BUILD | 2 +- 22 files changed, 29 insertions(+), 29 deletions(-) diff --git a/mediapipe/tasks/web/audio/audio_classifier/BUILD b/mediapipe/tasks/web/audio/audio_classifier/BUILD index a94b4931d9..14c2d453cf 100644 --- a/mediapipe/tasks/web/audio/audio_classifier/BUILD +++ b/mediapipe/tasks/web/audio/audio_classifier/BUILD @@ -64,5 +64,5 @@ mediapipe_ts_library( jasmine_node_test( name = "audio_classifier_test", - deps = [":audio_classifier_test_lib"], + srcs = [":audio_classifier_test_lib"], ) diff --git a/mediapipe/tasks/web/audio/audio_embedder/BUILD b/mediapipe/tasks/web/audio/audio_embedder/BUILD index 61afacfdb8..12929029ff 100644 --- a/mediapipe/tasks/web/audio/audio_embedder/BUILD +++ b/mediapipe/tasks/web/audio/audio_embedder/BUILD @@ -61,5 +61,5 @@ mediapipe_ts_library( jasmine_node_test( name = "audio_embedder_test", - deps = [":audio_embedder_test_lib"], + srcs = [":audio_embedder_test_lib"], ) diff --git a/mediapipe/tasks/web/components/processors/BUILD b/mediapipe/tasks/web/components/processors/BUILD index d81fbc79a4..e44e933beb 100644 --- a/mediapipe/tasks/web/components/processors/BUILD +++ b/mediapipe/tasks/web/components/processors/BUILD @@ -27,7 +27,7 @@ mediapipe_ts_library( jasmine_node_test( name = "classifier_options_test", - deps = [":classifier_options_test_lib"], + srcs = [":classifier_options_test_lib"], ) mediapipe_ts_library( @@ -53,7 +53,7 @@ mediapipe_ts_library( jasmine_node_test( name = "classifier_result_test", - deps = [":classifier_result_test_lib"], + srcs = [":classifier_result_test_lib"], ) mediapipe_ts_library( @@ -98,7 +98,7 @@ mediapipe_ts_library( jasmine_node_test( name = "embedder_result_test", - deps = [":embedder_result_test_lib"], + srcs = [":embedder_result_test_lib"], ) mediapipe_ts_library( @@ -123,7 +123,7 @@ mediapipe_ts_library( jasmine_node_test( name = "embedder_options_test", - deps = [":embedder_options_test_lib"], + srcs = [":embedder_options_test_lib"], ) mediapipe_ts_library( @@ -147,5 +147,5 @@ mediapipe_ts_library( jasmine_node_test( name = "landmark_result_test", - deps = [":landmark_result_test_lib"], + srcs = [":landmark_result_test_lib"], ) diff --git a/mediapipe/tasks/web/components/utils/BUILD b/mediapipe/tasks/web/components/utils/BUILD index 4844294cd8..af49cc730b 100644 --- a/mediapipe/tasks/web/components/utils/BUILD +++ b/mediapipe/tasks/web/components/utils/BUILD @@ -23,5 +23,5 @@ mediapipe_ts_library( jasmine_node_test( name = "cosine_similarity_test", - deps = [":cosine_similarity_test_lib"], + srcs = [":cosine_similarity_test_lib"], ) diff --git a/mediapipe/tasks/web/core/BUILD b/mediapipe/tasks/web/core/BUILD index 51c8b3c378..156946abd1 100644 --- a/mediapipe/tasks/web/core/BUILD +++ b/mediapipe/tasks/web/core/BUILD @@ -74,7 +74,7 @@ mediapipe_ts_library( jasmine_node_test( name = "task_runner_test", - deps = [":task_runner_test_lib"], + srcs = [":task_runner_test_lib"], ) mediapipe_ts_declaration( diff --git a/mediapipe/tasks/web/text/language_detector/BUILD b/mediapipe/tasks/web/text/language_detector/BUILD index da47a8d6c0..802873e988 100644 --- a/mediapipe/tasks/web/text/language_detector/BUILD +++ b/mediapipe/tasks/web/text/language_detector/BUILD @@ -62,5 +62,5 @@ mediapipe_ts_library( jasmine_node_test( name = "language_detector_test", - deps = [":language_detector_test_lib"], + srcs = [":language_detector_test_lib"], ) diff --git a/mediapipe/tasks/web/text/text_classifier/BUILD b/mediapipe/tasks/web/text/text_classifier/BUILD index 61af214060..74a4a6fecb 100644 --- a/mediapipe/tasks/web/text/text_classifier/BUILD +++ b/mediapipe/tasks/web/text/text_classifier/BUILD @@ -65,5 +65,5 @@ mediapipe_ts_library( jasmine_node_test( name = "text_classifier_test", - deps = [":text_classifier_test_lib"], + srcs = [":text_classifier_test_lib"], ) diff --git a/mediapipe/tasks/web/text/text_embedder/BUILD b/mediapipe/tasks/web/text/text_embedder/BUILD index eb3efaddae..8b120d51fb 100644 --- a/mediapipe/tasks/web/text/text_embedder/BUILD +++ b/mediapipe/tasks/web/text/text_embedder/BUILD @@ -63,5 +63,5 @@ mediapipe_ts_library( jasmine_node_test( name = "text_embedder_test", - deps = [":text_embedder_test_lib"], + srcs = [":text_embedder_test_lib"], ) diff --git a/mediapipe/tasks/web/vision/core/BUILD b/mediapipe/tasks/web/vision/core/BUILD index 2c0b61e417..46f9a7c959 100644 --- a/mediapipe/tasks/web/vision/core/BUILD +++ b/mediapipe/tasks/web/vision/core/BUILD @@ -61,7 +61,7 @@ mediapipe_ts_library( jasmine_node_test( name = "drawing_utils_test", - deps = [":drawing_utils_test_lib"], + srcs = [":drawing_utils_test_lib"], ) mediapipe_ts_library( @@ -87,7 +87,7 @@ mediapipe_ts_library( jasmine_node_test( name = "image_test", - deps = [":image_test_lib"], + srcs = [":image_test_lib"], ) mediapipe_ts_library( @@ -113,7 +113,7 @@ mediapipe_ts_library( jasmine_node_test( name = "mask_test", - deps = [":mask_test_lib"], + srcs = [":mask_test_lib"], ) mediapipe_ts_library( @@ -152,5 +152,5 @@ mediapipe_ts_library( jasmine_node_test( name = "vision_task_runner_test", - deps = [":vision_task_runner_test_lib"], + srcs = [":vision_task_runner_test_lib"], ) diff --git a/mediapipe/tasks/web/vision/face_detector/BUILD b/mediapipe/tasks/web/vision/face_detector/BUILD index b5d6d04910..cb76a21ea4 100644 --- a/mediapipe/tasks/web/vision/face_detector/BUILD +++ b/mediapipe/tasks/web/vision/face_detector/BUILD @@ -66,6 +66,6 @@ mediapipe_ts_library( jasmine_node_test( name = "face_detector_test", + srcs = [":face_detector_test_lib"], tags = ["nomsan"], - deps = [":face_detector_test_lib"], ) diff --git a/mediapipe/tasks/web/vision/face_landmarker/BUILD b/mediapipe/tasks/web/vision/face_landmarker/BUILD index 4a5e36a1bd..f9daeda67b 100644 --- a/mediapipe/tasks/web/vision/face_landmarker/BUILD +++ b/mediapipe/tasks/web/vision/face_landmarker/BUILD @@ -85,6 +85,6 @@ mediapipe_ts_library( jasmine_node_test( name = "face_landmarker_test", + srcs = [":face_landmarker_test_lib"], tags = ["nomsan"], - deps = [":face_landmarker_test_lib"], ) diff --git a/mediapipe/tasks/web/vision/face_stylizer/BUILD b/mediapipe/tasks/web/vision/face_stylizer/BUILD index 253fdee387..da3bafbeeb 100644 --- a/mediapipe/tasks/web/vision/face_stylizer/BUILD +++ b/mediapipe/tasks/web/vision/face_stylizer/BUILD @@ -53,6 +53,6 @@ mediapipe_ts_library( jasmine_node_test( name = "face_stylizer_test", + srcs = [":face_stylizer_test_lib"], tags = ["nomsan"], - deps = [":face_stylizer_test_lib"], ) diff --git a/mediapipe/tasks/web/vision/gesture_recognizer/BUILD b/mediapipe/tasks/web/vision/gesture_recognizer/BUILD index c74b2a01eb..a9df1493a0 100644 --- a/mediapipe/tasks/web/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/web/vision/gesture_recognizer/BUILD @@ -75,6 +75,6 @@ mediapipe_ts_library( jasmine_node_test( name = "gesture_recognizer_test", + srcs = [":gesture_recognizer_test_lib"], tags = ["nomsan"], - deps = [":gesture_recognizer_test_lib"], ) diff --git a/mediapipe/tasks/web/vision/hand_landmarker/BUILD b/mediapipe/tasks/web/vision/hand_landmarker/BUILD index 8d2111de65..3137a712ba 100644 --- a/mediapipe/tasks/web/vision/hand_landmarker/BUILD +++ b/mediapipe/tasks/web/vision/hand_landmarker/BUILD @@ -71,8 +71,8 @@ mediapipe_ts_library( jasmine_node_test( name = "hand_landmarker_test", + srcs = [":hand_landmarker_test_lib"], tags = ["nomsan"], - deps = [":hand_landmarker_test_lib"], ) mediapipe_ts_library( diff --git a/mediapipe/tasks/web/vision/holistic_landmarker/BUILD b/mediapipe/tasks/web/vision/holistic_landmarker/BUILD index 4f613ffdbf..93d5b85421 100644 --- a/mediapipe/tasks/web/vision/holistic_landmarker/BUILD +++ b/mediapipe/tasks/web/vision/holistic_landmarker/BUILD @@ -80,8 +80,8 @@ mediapipe_ts_library( jasmine_node_test( name = "holistic_landmarker_test", + srcs = [":holistic_landmarker_test_lib"], tags = ["nomsan"], - deps = [":holistic_landmarker_test_lib"], ) mediapipe_ts_library( diff --git a/mediapipe/tasks/web/vision/image_classifier/BUILD b/mediapipe/tasks/web/vision/image_classifier/BUILD index a163bb3ab3..1591865a1d 100644 --- a/mediapipe/tasks/web/vision/image_classifier/BUILD +++ b/mediapipe/tasks/web/vision/image_classifier/BUILD @@ -66,6 +66,6 @@ mediapipe_ts_library( jasmine_node_test( name = "image_classifier_test", + srcs = [":image_classifier_test_lib"], tags = ["nomsan"], - deps = [":image_classifier_test_lib"], ) diff --git a/mediapipe/tasks/web/vision/image_embedder/BUILD b/mediapipe/tasks/web/vision/image_embedder/BUILD index e45221e732..be0c8208bb 100644 --- a/mediapipe/tasks/web/vision/image_embedder/BUILD +++ b/mediapipe/tasks/web/vision/image_embedder/BUILD @@ -65,5 +65,5 @@ mediapipe_ts_library( jasmine_node_test( name = "image_embedder_test", - deps = [":image_embedder_test_lib"], + srcs = [":image_embedder_test_lib"], ) diff --git a/mediapipe/tasks/web/vision/image_segmenter/BUILD b/mediapipe/tasks/web/vision/image_segmenter/BUILD index 14688f36b8..c8bfee7c33 100644 --- a/mediapipe/tasks/web/vision/image_segmenter/BUILD +++ b/mediapipe/tasks/web/vision/image_segmenter/BUILD @@ -56,6 +56,6 @@ mediapipe_ts_library( jasmine_node_test( name = "image_segmenter_test", + srcs = [":image_segmenter_test_lib"], tags = ["nomsan"], - deps = [":image_segmenter_test_lib"], ) diff --git a/mediapipe/tasks/web/vision/interactive_segmenter/BUILD b/mediapipe/tasks/web/vision/interactive_segmenter/BUILD index a78f088b56..3fc3382678 100644 --- a/mediapipe/tasks/web/vision/interactive_segmenter/BUILD +++ b/mediapipe/tasks/web/vision/interactive_segmenter/BUILD @@ -61,6 +61,6 @@ mediapipe_ts_library( jasmine_node_test( name = "interactive_segmenter_test", + srcs = [":interactive_segmenter_test_lib"], tags = ["nomsan"], - deps = [":interactive_segmenter_test_lib"], ) diff --git a/mediapipe/tasks/web/vision/object_detector/BUILD b/mediapipe/tasks/web/vision/object_detector/BUILD index 0e8387e27e..74fef73527 100644 --- a/mediapipe/tasks/web/vision/object_detector/BUILD +++ b/mediapipe/tasks/web/vision/object_detector/BUILD @@ -66,6 +66,6 @@ mediapipe_ts_library( jasmine_node_test( name = "object_detector_test", + srcs = [":object_detector_test_lib"], tags = ["nomsan"], - deps = [":object_detector_test_lib"], ) diff --git a/mediapipe/tasks/web/vision/pose_landmarker/BUILD b/mediapipe/tasks/web/vision/pose_landmarker/BUILD index a0ad1da8bc..5b9b7bf10b 100644 --- a/mediapipe/tasks/web/vision/pose_landmarker/BUILD +++ b/mediapipe/tasks/web/vision/pose_landmarker/BUILD @@ -64,8 +64,8 @@ mediapipe_ts_library( jasmine_node_test( name = "pose_landmarker_test", + srcs = [":pose_landmarker_test_lib"], tags = ["nomsan"], - deps = [":pose_landmarker_test_lib"], ) mediapipe_ts_library( diff --git a/mediapipe/web/graph_runner/BUILD b/mediapipe/web/graph_runner/BUILD index ad63311415..64ca2ee02a 100644 --- a/mediapipe/web/graph_runner/BUILD +++ b/mediapipe/web/graph_runner/BUILD @@ -113,7 +113,7 @@ mediapipe_ts_library( jasmine_node_test( name = "platform_utils_test", - deps = [ + srcs = [ ":platform_utils_test_lib", ], ) From 30397f08556f0b727e933d1f44e89c24c8a6cab0 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 29 Oct 2024 00:58:30 -0700 Subject: [PATCH 013/126] No public description PiperOrigin-RevId: 690911485 --- mediapipe/util/sequence/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/mediapipe/util/sequence/BUILD b/mediapipe/util/sequence/BUILD index 2385830433..59b0888c78 100644 --- a/mediapipe/util/sequence/BUILD +++ b/mediapipe/util/sequence/BUILD @@ -25,6 +25,7 @@ cc_library( hdrs = ["media_sequence_util.h"], visibility = [ "//home/interaction:__subpackages__", + "//learning/eval/canon/util/conversion/tensorflow/mediasequence:__pkg__", "//mediapipe:__subpackages__", ], deps = [ From 9f068f139409a293ad0038226fd99a572687a14a Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 29 Oct 2024 09:04:43 -0700 Subject: [PATCH 014/126] No public description PiperOrigin-RevId: 691041405 --- WORKSPACE | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 67866ed18c..01a8c2f1a0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -611,8 +611,6 @@ http_archive( ], ) -# More iOS deps. - http_archive( name = "google_toolbox_for_mac", url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip", @@ -621,6 +619,7 @@ http_archive( build_file = "@//third_party:google_toolbox_for_mac.BUILD", ) +# Hermetic CUDA load( "@org_tensorflow//third_party/gpus/cuda/hermetic:cuda_json_init_repository.bzl", "cuda_json_init_repository", From 6f742b632fe75da0bcc570a91ab6a3d72273dce5 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 29 Oct 2024 10:23:42 -0700 Subject: [PATCH 015/126] Add troubleshooting tip for unsupported XNNPACK flags during build Fixes https://github.com/google-ai-edge/mediapipe/issues/5518 PiperOrigin-RevId: 691070380 --- docs/getting_started/troubleshooting.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/getting_started/troubleshooting.md b/docs/getting_started/troubleshooting.md index e7dff332cd..f27a2c88f5 100644 --- a/docs/getting_started/troubleshooting.md +++ b/docs/getting_started/troubleshooting.md @@ -282,3 +282,15 @@ calculators designed specifically for this purpose such as [`CalculatorBase::Close`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator_base.h [`FlowLimiterCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/flow_limiter_calculator.cc [`How to process realtime input streams`]: faq.md#how-to-process-realtime-input-streams + +## Unsupported flags during build + +If you are using Clang 18 or older, you may have to disable some compiler +optimizations in our CPU backend. + +To disable support for `avxvnniint8`, add the following to you `.bazelrc`: + +``` +build --define=xnn_enable_avxvnniint8=false +``` + From 3a79fffc363ebe4e19f73d49eb255e0ddee1d103 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 29 Oct 2024 14:27:58 -0700 Subject: [PATCH 016/126] No public description PiperOrigin-RevId: 691163384 --- mediapipe/modules/face_geometry/BUILD | 1 + mediapipe/modules/face_geometry/effect_renderer_calculator.cc | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mediapipe/modules/face_geometry/BUILD b/mediapipe/modules/face_geometry/BUILD index 710747e361..6cb45b2851 100644 --- a/mediapipe/modules/face_geometry/BUILD +++ b/mediapipe/modules/face_geometry/BUILD @@ -82,6 +82,7 @@ cc_library( "//mediapipe/util:resource_util", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@com_google_absl//absl/types:optional", ], alwayslink = 1, diff --git a/mediapipe/modules/face_geometry/effect_renderer_calculator.cc b/mediapipe/modules/face_geometry/effect_renderer_calculator.cc index f48e130498..3a7eebdbcb 100644 --- a/mediapipe/modules/face_geometry/effect_renderer_calculator.cc +++ b/mediapipe/modules/face_geometry/effect_renderer_calculator.cc @@ -20,6 +20,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" @@ -256,9 +257,10 @@ class EffectRendererCalculator : public CalculatorBase { MP_ASSIGN_OR_RETURN(std::unique_ptr mesh_3d_blob, ReadContentBlobFromFile(cc, mesh_3d_path), _ << "Failed to read mesh 3D blob from file!"); + absl::string_view mesh_str = mesh_3d_blob->ToStringView(); face_geometry::Mesh3d mesh_3d; - RET_CHECK(mesh_3d.ParseFromString(mesh_3d_blob->ToStringView())) + RET_CHECK(mesh_3d.ParseFromArray(mesh_str.data(), mesh_str.size())) << "Failed to parse a mesh 3D proto from a binary blob!"; return mesh_3d; From 593fa5efbd94b589a83e6a077884b03a38c3ad81 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 29 Oct 2024 16:37:48 -0700 Subject: [PATCH 017/126] Add comments to explain how to configure OpenCV in the opencv_macos.BUILD file. PiperOrigin-RevId: 691204643 --- third_party/opencv_macos.BUILD | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/third_party/opencv_macos.BUILD b/third_party/opencv_macos.BUILD index d05c83ccc1..7ed8ab8f28 100644 --- a/third_party/opencv_macos.BUILD +++ b/third_party/opencv_macos.BUILD @@ -7,8 +7,29 @@ licenses(["notice"]) # BSD license exports_files(["LICENSE"]) -# The path to OpenCV is a combination of the path set for "macos_opencv" -# in the WORKSPACE file and the prefix here. +# Example configurations: +# +# To configure OpenCV 3, obtain the path of OpenCV 3 from Homebrew: +# +# $ brew ls opencv@3 | grep version.hpp +# $ /opt/homebrew/Cellar/opencv@3/3.4.16_10/include/opencv2/core/version.hpp +# +# Then set path in "macos_opencv" rule in the WORKSPACE file to +# "/opt/homebrew/Cellar" and the PREFIX below to "opencv@3/3.4.16_10". +# +# +# To configure OpenCV 4, obtain the path of OpenCV 4 from Homebrew: +# +# $ brew ls opencv | grep version.hpp +# $ /opt/homebrew/Cellar/opencv/4.10.0_12/include/opencv4/opencv2/core/version.hpp +# $ /opt/homebrew/Cellar/opencv/4.10.0_12/include/opencv4/opencv2/dnn/version.hpp +# +# Then set path in "macos_opencv" rule in the WORKSPACE file to +# "/opt/homebrew/Cellar" and the PREFIX below to "opencv/4.10.0_12". For OpenCV +# 4, you will also need to adjust the include paths. The header search path +# should be "include/opencv4/opencv2/**/*.h*" and the include prefix needs to +# be set to "include/opencv4". + PREFIX = "opt/opencv@3" cc_library( From fbcfdc4c7d9dc9b5957e20361fa15a7ce52abc95 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 30 Oct 2024 14:28:27 -0700 Subject: [PATCH 018/126] Add libc++_shared.so to MediaPipe Android examples. PiperOrigin-RevId: 691554917 --- .../com/google/mediapipe/apps/basic/BUILD | 1 + .../mediapipe/apps/facedetectioncpu/BUILD | 1 + .../apps/facedetectionfullrangegpu/BUILD | 1 + .../mediapipe/apps/facedetectiongpu/BUILD | 1 + .../google/mediapipe/apps/faceeffect/BUILD | 1 + .../google/mediapipe/apps/facemeshgpu/BUILD | 1 + .../mediapipe/apps/hairsegmentationgpu/BUILD | 1 + .../mediapipe/apps/handdetectiongpu/BUILD | 1 + .../mediapipe/apps/handtrackinggpu/BUILD | 1 + .../mediapipe/apps/holistictrackinggpu/BUILD | 1 + .../apps/instantmotiontracking/BUILD | 1 + .../mediapipe/apps/iristrackinggpu/BUILD | 1 + .../mediapipe/apps/objectdetection3d/BUILD | 1 + .../mediapipe/apps/objectdetectioncpu/BUILD | 1 + .../mediapipe/apps/objectdetectiongpu/BUILD | 1 + .../mediapipe/apps/objecttrackinggpu/BUILD | 1 + .../mediapipe/apps/posetrackinggpu/BUILD | 1 + .../apps/selfiesegmentationgpu/BUILD | 1 + .../mediapipe/apps/templatematchingcpu/BUILD | 1 + mediapipe/framework/tool/mediapipe_files.bzl | 2 +- third_party/BUILD | 21 +++++++++++++++++++ third_party/external_files.bzl | 10 +++++++-- 22 files changed, 49 insertions(+), 3 deletions(-) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/BUILD index ae4652dba9..9c75207c0a 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/BUILD @@ -80,5 +80,6 @@ android_binary( deps = [ ":basic_lib", ":mediapipe_jni_lib", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD index cf71048b5e..e26e228b3b 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD @@ -56,5 +56,6 @@ android_binary( deps = [ ":mediapipe_jni_lib", "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectionfullrangegpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectionfullrangegpu/BUILD index 03891439da..6d1188bd93 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectionfullrangegpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectionfullrangegpu/BUILD @@ -56,5 +56,6 @@ android_binary( deps = [ ":mediapipe_jni_lib", "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD index 9c60d5a573..1c31f71f6c 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD @@ -56,5 +56,6 @@ android_binary( deps = [ ":mediapipe_jni_lib", "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD index becef5246a..72800a0609 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD @@ -67,5 +67,6 @@ android_binary( "//mediapipe/framework/formats:matrix_data_java_proto_lite", "//mediapipe/java/com/google/mediapipe/framework:android_framework", "//mediapipe/modules/face_geometry/protos:face_geometry_java_proto_lite", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD index edef0b8604..78a0a0f341 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD @@ -59,5 +59,6 @@ android_binary( "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", "//mediapipe/framework/formats:landmark_java_proto_lite", "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD index df58f27131..f8753045f3 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD @@ -56,5 +56,6 @@ android_binary( deps = [ ":mediapipe_jni_lib", "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD index 550d61ec05..f1c405f0c7 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD @@ -56,5 +56,6 @@ android_binary( deps = [ ":mediapipe_jni_lib", "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD index b79fc5ba1e..4f3c10681b 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD @@ -63,5 +63,6 @@ android_binary( "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", "//mediapipe/framework/formats:landmark_java_proto_lite", "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD index e7e6cfb975..3263afe889 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD @@ -65,5 +65,6 @@ android_binary( "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", "//mediapipe/framework/formats:landmark_java_proto_lite", "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD index 3dea64053e..7b9d61d977 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD @@ -96,5 +96,6 @@ android_binary( ":instantmotiontracking_lib", ":mediapipe_jni_lib", "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD index 482932b193..3812673b33 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD @@ -60,6 +60,7 @@ android_binary( "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", "//mediapipe/framework/formats:landmark_java_proto_lite", "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//third_party:libc++_shared_lib", "@com_google_protobuf//:protobuf_javalite", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD index 783ae200e2..a45a6a8c31 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD @@ -186,5 +186,6 @@ android_binary( "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", "//mediapipe/framework/formats:landmark_java_proto_lite", "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD index 9bb0549364..3656d2617b 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD @@ -57,5 +57,6 @@ android_binary( deps = [ ":mediapipe_jni_lib", "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD index 81f2ed3e69..5abb518bce 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD @@ -57,5 +57,6 @@ android_binary( deps = [ ":mediapipe_jni_lib", "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD index 50ea70f898..095f4b27bb 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD @@ -57,5 +57,6 @@ android_binary( deps = [ ":mediapipe_jni_lib", "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackinggpu/BUILD index d1c45345fd..b21488eb60 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackinggpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackinggpu/BUILD @@ -59,6 +59,7 @@ android_binary( "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", "//mediapipe/framework/formats:landmark_java_proto_lite", "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//third_party:libc++_shared_lib", "@com_google_protobuf//:protobuf_javalite", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu/BUILD index 6bfcf34c17..68efaae0f4 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu/BUILD @@ -56,5 +56,6 @@ android_binary( deps = [ ":mediapipe_jni_lib", "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/BUILD index ed3a63a70e..51152c567d 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/BUILD @@ -58,5 +58,6 @@ android_binary( deps = [ ":mediapipe_jni_lib", "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//third_party:libc++_shared_lib", ], ) diff --git a/mediapipe/framework/tool/mediapipe_files.bzl b/mediapipe/framework/tool/mediapipe_files.bzl index 2e0ec9b42b..53c3065787 100644 --- a/mediapipe/framework/tool/mediapipe_files.bzl +++ b/mediapipe/framework/tool/mediapipe_files.bzl @@ -13,7 +13,7 @@ def mediapipe_files(srcs): """ for src in srcs: - archive_name = "com_google_mediapipe_%s" % src.replace("/", "_").replace(".", "_") + archive_name = "com_google_mediapipe_%s" % src.replace("/", "_").replace(".", "_").replace("+", "_") native.genrule( name = "%s_ln" % archive_name, srcs = ["@%s//file" % archive_name], diff --git a/third_party/BUILD b/third_party/BUILD index 799b6cd231..6b00be7e9b 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -15,6 +15,10 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("@rules_foreign_cc//foreign_cc:cmake.bzl", "cmake") +load( + "//mediapipe/framework/tool:mediapipe_files.bzl", + "mediapipe_files", +) licenses(["notice"]) # Apache License 2.0 @@ -28,6 +32,10 @@ exports_files([ "requirements_lock_3_12.txt", ]) +mediapipe_files(srcs = [ + "libc++_shared.so", +]) + cc_library( name = "glog", visibility = ["//visibility:public"], @@ -427,3 +435,16 @@ java_import( "@com_google_protobuf//java/core:libcore.jar", ], ) + +cc_binary( + name = "libc++_shared", + srcs = ["libc++_shared.so"], + linkshared = 1, + linkstatic = 1, +) + +cc_library( + name = "libc++_shared_lib", + srcs = [":libc++_shared.so"], + alwayslink = 1, +) diff --git a/third_party/external_files.bzl b/third_party/external_files.bzl index cb07233293..7a04fd8c22 100644 --- a/third_party/external_files.bzl +++ b/third_party/external_files.bzl @@ -288,8 +288,8 @@ def external_files(): http_file( name = "com_google_mediapipe_efficientdet_lite0_fp16_no_nms_tflite", - sha256 = "bcda125c96d3767bca894c8cbe7bc458379c9974c9fd8bdc6204e7124a74082a", - urls = ["https://storage.googleapis.com/mediapipe-assets/efficientdet_lite0_fp16_no_nms.tflite?generation=1728573738871723"], + sha256 = "237a58389081333e5cf4154e42b593ce7dd357445536fcaf4ca5bc51c2c50f1c", + urls = ["https://storage.googleapis.com/mediapipe-assets/efficientdet_lite0_fp16_no_nms.tflite?generation=1730305296514873"], ) http_file( @@ -712,6 +712,12 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/leopard.jpg?generation=1685997280368627"], ) + http_file( + name = "com_google_mediapipe_libc___shared_so", + sha256 = "816d497229b6678db485b5dc16ae7d2ac63dc015691b1828bc35c4aa2ed6eed4", + urls = ["https://storage.googleapis.com/mediapipe-assets/libc++_shared.so?generation=1730305298946708"], + ) + http_file( name = "com_google_mediapipe_libimagegenerator_gpu_so", sha256 = "e4407c7c0a2559b168a0f76cda6eb23ce2d167fa757a0d4887ccf57af70c0179", From 24460b0e7c2013bfe8dde43cae49fa387a8074a6 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 30 Oct 2024 15:04:43 -0700 Subject: [PATCH 019/126] Add linkstatic to OpenCV prebuilts PiperOrigin-RevId: 691566846 --- third_party/prebuilts/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/third_party/prebuilts/BUILD b/third_party/prebuilts/BUILD index 5597763bce..8b60a11811 100644 --- a/third_party/prebuilts/BUILD +++ b/third_party/prebuilts/BUILD @@ -38,6 +38,7 @@ cc_import( cc_library( name = "opencv_darwin", + linkstatic = 1, deps = [ ":opencv_core_darwin", ":opencv_imgproc_darwin", @@ -47,6 +48,7 @@ cc_library( cc_library( name = "opencv_darwin_arm64", + linkstatic = 1, deps = [ ":opencv_core_darwin_arm64", ":opencv_imgproc_darwin_arm64", From 009037748306870949ab3d09f71ff63c99abc301 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 30 Oct 2024 20:42:22 -0700 Subject: [PATCH 020/126] No public description PiperOrigin-RevId: 691654690 --- .../tensorflow/lapped_tensor_buffer_calculator.cc | 2 +- .../tensorflow/tensorflow_inference_calculator.cc | 6 +++--- .../tensorflow_session_from_frozen_graph_calculator.cc | 4 ++-- .../tensorflow_session_from_frozen_graph_generator.cc | 4 ++-- .../tensorflow_session_from_saved_model_calculator.cc | 2 +- .../tensorflow_session_from_saved_model_generator.cc | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc b/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc index a07b95ccc6..58bc60aeb3 100644 --- a/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc +++ b/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc @@ -236,7 +236,7 @@ absl::Status LappedTensorBufferCalculator::AddBatchDimension( absl::Status LappedTensorBufferCalculator::ProcessBuffer( CalculatorContext* cc) { auto concatenated = ::absl::make_unique(); - const tf::Status concat_status = tf::tensor::Concat( + const absl::Status concat_status = tf::tensor::Concat( std::vector(buffer_->begin(), buffer_->end()), concatenated.get()); RET_CHECK(concat_status.ok()) << concat_status.ToString(); diff --git a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc index c750b61b8d..d673895aae 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc @@ -515,7 +515,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase { keyed_tensors.second.end(), keyed_tensors.second[0]); } tf::Tensor concated; - const tf::Status concat_status = + const absl::Status concat_status = tf::tensor::Concat(keyed_tensors.second, &concated); ABSL_CHECK(concat_status.ok()) << concat_status.ToString(); input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first], @@ -547,7 +547,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase { session_run_throttle->Acquire(1); } const int64_t run_start_time = absl::ToUnixMicros(clock_->TimeNow()); - tf::Status tf_status; + absl::Status tf_status; { #if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__) tsl::profiler::TraceMe trace(absl::string_view(cc->NodeName())); @@ -597,7 +597,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase { } } else { std::vector split_tensors; - const tf::Status split_status = + const absl::Status split_status = tf::tensor::Split(outputs[i], split_vector, &split_tensors); ABSL_CHECK(split_status.ok()) << split_status.ToString(); // Loop over timestamps so that we don't copy the padding. diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc index 358b50cd31..5b3be901ff 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc @@ -140,14 +140,14 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase { SetPreferredDevice(&graph_def, options.preferred_device_id()); } - const tf::Status tf_status = session->session->Create(graph_def); + const absl::Status tf_status = session->session->Create(graph_def); RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.ToString(); for (const auto& key_value : options.tag_to_tensor_names()) { session->tag_to_tensor_map[key_value.first] = key_value.second; } if (!initialization_op_names.empty()) { - const tf::Status tf_status = + const absl::Status tf_status = session->session->Run({}, {}, initialization_op_names, {}); // RET_CHECK on the tf::Status object itself in order to print an // informative error message. diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc index e340a098b5..b83dd632c8 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc @@ -139,14 +139,14 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator { SetPreferredDevice(&graph_def, options.preferred_device_id()); } - const tf::Status tf_status = session->session->Create(graph_def); + const absl::Status tf_status = session->session->Create(graph_def); RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.ToString(); for (const auto& key_value : options.tag_to_tensor_names()) { session->tag_to_tensor_map[key_value.first] = key_value.second; } if (!initialization_op_names.empty()) { - const tf::Status tf_status = + const absl::Status tf_status = session->session->Run({}, {}, initialization_op_names, {}); // RET_CHECK on the tf::Status object itself in order to print an // informative error message. diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc index 4ca4cb8d6e..18523b32af 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc @@ -143,7 +143,7 @@ class TensorFlowSessionFromSavedModelCalculator : public CalculatorBase { tensorflow::SessionOptions session_options; session_options.config = options.session_config(); auto saved_model = absl::make_unique(); - ::tensorflow::Status status = tensorflow::LoadSavedModel( + absl::Status status = tensorflow::LoadSavedModel( session_options, run_options, path, tags_set, saved_model.get()); if (!status.ok()) { return absl::Status(static_cast(status.code()), diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc index 9596224470..208f4b066b 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc @@ -145,7 +145,7 @@ class TensorFlowSessionFromSavedModelGenerator : public PacketGenerator { tensorflow::SessionOptions session_options; session_options.config = options.session_config(); auto saved_model = absl::make_unique(); - ::tensorflow::Status status = tensorflow::LoadSavedModel( + absl::Status status = tensorflow::LoadSavedModel( session_options, run_options, path, tags_set, saved_model.get()); if (!status.ok()) { return absl::Status(static_cast(status.code()), From 81292bd69706825a3a5597a0165f9baee7779c4e Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 31 Oct 2024 12:14:35 -0700 Subject: [PATCH 021/126] No public description PiperOrigin-RevId: 691892416 --- .../tensorflow/pack_media_sequence_calculator.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc index 6e27542775..66456345d7 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -361,11 +362,12 @@ class PackMediaSequenceCalculator : public CalculatorBase { } absl::Status VerifySize() { - const int64_t MAX_PROTO_BYTES = 1073741823; + constexpr int kMaxProtoBytes = INT_MAX; + std::string id = mpms::HasExampleId(*sequence_) ? mpms::GetExampleId(*sequence_) : "example"; - RET_CHECK_LT(sequence_->ByteSizeLong(), MAX_PROTO_BYTES) + RET_CHECK_LT(sequence_->ByteSizeLong(), kMaxProtoBytes) << "sequence '" << id << "' would be too many bytes to serialize after adding features."; return absl::OkStatus(); From ba4b77058947def6c20efed3eb789f51775f1ee4 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 31 Oct 2024 14:17:43 -0700 Subject: [PATCH 022/126] No public description PiperOrigin-RevId: 691933222 --- .../web/graph_runner/graph_runner_webgpu.ts | 47 +++++++++++++------ package.json | 2 +- yarn.lock | 8 ++-- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/mediapipe/web/graph_runner/graph_runner_webgpu.ts b/mediapipe/web/graph_runner/graph_runner_webgpu.ts index 21556be9f8..83ce45ff67 100644 --- a/mediapipe/web/graph_runner/graph_runner_webgpu.ts +++ b/mediapipe/web/graph_runner/graph_runner_webgpu.ts @@ -31,8 +31,13 @@ export declare interface GPUDeviceWithAdapterInfo extends GPUDevice { * manner. */ export declare interface WasmAsyncCloseModule { - ccall: (name: string, type: string, inParams: unknown, outParams: unknown, - options: unknown) => Promise; + ccall: ( + name: string, + type: string, + inParams: unknown, + outParams: unknown, + options: unknown, + ) => Promise; } /** @@ -49,8 +54,9 @@ export function SupportWebGpu(Base: TBase) { * @param adapter The adapter to request GPUDevice. */ static async requestWebGpuDevice( - deviceDescriptor?: GPUDeviceDescriptor, - adapter?: GPUAdapter): Promise { + deviceDescriptor?: GPUDeviceDescriptor, + adapter?: GPUAdapter, + ): Promise { if (!adapter) { adapter = await WebGpuSupportedGraphRunner.requestWebGpuAdapter(); } @@ -65,13 +71,14 @@ export function SupportWebGpu(Base: TBase) { } const updatedDescriptor: GPUDeviceDescriptor = { ...deviceDescriptor, - requiredFeatures: supportedFeatures + requiredFeatures: supportedFeatures, }; try { device = await adapter.requestDevice(updatedDescriptor); } catch (e: unknown) { console.error( - 'Unable to initialize WebGPU with the requested features.'); + 'Unable to initialize WebGPU with the requested features.', + ); // Rethrow original error. throw e; } @@ -81,8 +88,8 @@ export function SupportWebGpu(Base: TBase) { // Our inference engines can utilize the adapter info to optimize WebGPU // shader performance. Therefore, we attempt to attach that information to // our internal GPUDevice reference. - const adapterInfo = await adapter.requestAdapterInfo(); - (device as unknown as GPUDeviceWithAdapterInfo).adapterInfo = adapterInfo; + (device as unknown as GPUDeviceWithAdapterInfo).adapterInfo = + adapter.info; return device; } @@ -92,11 +99,13 @@ export function SupportWebGpu(Base: TBase) { * @param adapterDescriptor The adapterDescriptor to request GPUAdapter. */ static async requestWebGpuAdapter( - adapterDescriptor?: GPURequestAdapterOptions): Promise { + adapterDescriptor?: GPURequestAdapterOptions, + ): Promise { const adapter = await navigator.gpu.requestAdapter(adapterDescriptor); if (!adapter) { throw new Error( - 'Unable to request adapter from navigator.gpu; Ensure WebGPU is enabled.'); + 'Unable to request adapter from navigator.gpu; Ensure WebGPU is enabled.', + ); } return adapter; } @@ -111,14 +120,17 @@ export function SupportWebGpu(Base: TBase) { * canvas will be created. */ initializeForWebGpu( - device: GPUDevice, canvas?: HTMLCanvasElement|OffscreenCanvas) { + device: GPUDevice, + canvas?: HTMLCanvasElement | OffscreenCanvas, + ) { if (!canvas) { canvas = new OffscreenCanvas(1, 1); } else if ( - typeof HTMLCanvasElement !== 'undefined' && - canvas instanceof HTMLCanvasElement) { + typeof HTMLCanvasElement !== 'undefined' && + canvas instanceof HTMLCanvasElement + ) { // TODO b/327324051 - Stop using a hard-coded `canvas_webgpu` selector. - canvas.id = 'canvas_webgpu'; // id used as default for WebGPU code + canvas.id = 'canvas_webgpu'; // id used as default for WebGPU code } const context = canvas.getContext('webgpu') as GPUCanvasContext; context.configure({ @@ -136,7 +148,12 @@ export function SupportWebGpu(Base: TBase) { */ closeGraphAsync(): Promise { return (this.wasmModule as unknown as WasmAsyncCloseModule).ccall( - "closeGraph", "void", [], [], {async: true}); + 'closeGraph', + 'void', + [], + [], + {async: true}, + ); } }; } diff --git a/package.json b/package.json index 99494df7a5..2b799c335a 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "@types/jasmine": "^4.3.1", "@types/node": "^18.11.11", "@types/offscreencanvas": "^2019.7.0", - "@webgpu/types": "^0.1.40", + "@webgpu/types": "^0.1.49", "google-protobuf": "^3.21.2", "jasmine": "^4.5.0", "jasmine-core": "^4.5.0", diff --git a/yarn.lock b/yarn.lock index 12cde6067e..c0268f5310 100644 --- a/yarn.lock +++ b/yarn.lock @@ -249,10 +249,10 @@ resolved "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz" integrity sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q== -"@webgpu/types@^0.1.40": - version "0.1.40" - resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.40.tgz#cf72d1df6f9f8adc5d39556041f20ff2e8a58885" - integrity sha512-/BBkHLS6/eQjyWhY2H7Dx5DHcVrS2ICj9owvSRdgtQT6KcafLZA86tPze0xAOsd4FbsYKCUBUQyNi87q7gV7kw== +"@webgpu/types@^0.1.49": + version "0.1.49" + resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.49.tgz#eb9f6535e321214e5c6e9dcc6c7d17e0f7584707" + integrity sha512-NMmS8/DofhH/IFeW+876XrHVWel+J/vdcFCHLDqeJgkH9x0DeiwjVd8LcBdaxdG/T7Rf8VUAYsA8X1efMzLjRQ== "@xmldom/xmldom@^0.8.5": version "0.8.10" From a32bf68504931153a869f4d416c91e6a8dde49a3 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 31 Oct 2024 15:23:08 -0700 Subject: [PATCH 023/126] Add UniqueId::Dup. PiperOrigin-RevId: 691953163 --- mediapipe/framework/formats/BUILD | 4 ++++ mediapipe/framework/formats/unique_fd.h | 10 +++++++++- mediapipe/framework/formats/unique_fd_test.cc | 10 ++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index 3ee0472434..3e74db39cc 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -302,8 +302,11 @@ cc_library( name = "unique_fd", hdrs = ["unique_fd.h"], deps = [ + "//mediapipe/framework/port:ret_check", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", ], ) @@ -313,6 +316,7 @@ cc_test( deps = [ ":unique_fd", "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:status_matchers", ], ) diff --git a/mediapipe/framework/formats/unique_fd.h b/mediapipe/framework/formats/unique_fd.h index 666584919e..015628d9c4 100644 --- a/mediapipe/framework/formats/unique_fd.h +++ b/mediapipe/framework/formats/unique_fd.h @@ -7,6 +7,8 @@ #include "absl/base/attributes.h" #include "absl/log/absl_log.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/port/ret_check.h" #if (__ANDROID_API__ >= 29) && defined(__BIONIC__) && !defined(NDEBUG) #define MEDIAPIPE_USE_FDSAN 1 @@ -56,11 +58,17 @@ class UniqueFd { } // Returns a non-owned file descriptor. - int Get() { return fd_; } + int Get() const { return fd_; } // Checks if a valid file descriptor is wrapped. bool IsValid() const { return fd_ >= 0; } + absl::StatusOr Dup() const { + RET_CHECK(IsValid()); + int dup_fd = dup(Get()); + return UniqueFd(dup_fd); + } + // Releases ownership of the file descriptor and returns it. ABSL_MUST_USE_RESULT int Release() { if (!IsValid()) { diff --git a/mediapipe/framework/formats/unique_fd_test.cc b/mediapipe/framework/formats/unique_fd_test.cc index 885b5199eb..7a6f4556d4 100644 --- a/mediapipe/framework/formats/unique_fd_test.cc +++ b/mediapipe/framework/formats/unique_fd_test.cc @@ -6,6 +6,7 @@ #include #include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/status_matchers.h" namespace mediapipe { @@ -60,6 +61,15 @@ TEST(UniqueFdTest, ShouldCreateValidFd) { EXPECT_FALSE(unique_fd.IsValid()); } +TEST(UniqueFdTest, ShouldDupValidFd) { + UniqueFd unique_fd(GetValidFd()); + + MP_ASSERT_OK_AND_ASSIGN(UniqueFd dup_unique_fd, unique_fd.Dup()); + + EXPECT_TRUE(dup_unique_fd.IsValid()); + EXPECT_NE(dup_unique_fd.Get(), unique_fd.Get()); +} + TEST(UniqueFdTest, ShouldReleaseValidFd) { UniqueFd unique_fd(GetValidFd()); EXPECT_TRUE(unique_fd.IsValid()); From cf5a2ed51be68d6ba8f35a9fe5038ad84a2f0465 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 1 Nov 2024 07:55:16 -0700 Subject: [PATCH 024/126] Updating the XNNPACK latest commit hash PiperOrigin-RevId: 692176126 --- WORKSPACE | 46 +++++++++---------- .../text/text_embedder/text_embedder_test.cc | 2 +- .../text_classifier/text_classifier_test.cc | 6 +-- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 01a8c2f1a0..3c4fa53379 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -225,13 +225,13 @@ http_archive( ], ) -# XNNPACK on 2024-09-24 +# XNNPACK on 2024-10-30 http_archive( name = "XNNPACK", # `curl -L | shasum -a 256` - sha256 = "feecde71526d955a0125f7ddd28b9f2d282cd6fca6c1c6bde48f29f86365dd0b", - strip_prefix = "XNNPACK-9007aa93227010168e615f9c6552035040c94a15", - url = "https://github.com/google/XNNPACK/archive/9007aa93227010168e615f9c6552035040c94a15.zip", + sha256 = "7587e5b9272c9d37ea6c9feac46568858dd153c99aa2fbd9c7744fa6415ca297", + strip_prefix = "XNNPACK-92b6d8722c34198be870ee956ffe504a11f44093", + url = "https://github.com/google/XNNPACK/archive/92b6d8722c34198be870ee956ffe504a11f44093.zip", ) # 2020-07-09 @@ -262,6 +262,25 @@ http_archive( ], ) +# KleidiAI is needed to get the best possible performance out of XNNPack +http_archive( + name = "KleidiAI", + sha256 = "ccdb61c0c5df9174128f433f82430b1afa8a809cf17f43ecae9a1eec9c2dfabf", + strip_prefix = "kleidiai-0dadcdb307f4e5ac98a42e4d4888aad7c23edaf3", + urls = [ + "https://gitlab.arm.com/kleidi/kleidiai/-/archive/0dadcdb307f4e5ac98a42e4d4888aad7c23edaf3/kleidiai-0dadcdb307f4e5ac98a42e4d4888aad7c23edaf3.zip", + ], +) + +http_archive( + name = "cpuinfo", + sha256 = "e2bd8049d29dfbed675a0bc7c01947f8b8bd3f17f706b827d3f6c1e5c64dd8c3", + strip_prefix = "cpuinfo-8df44962d437a0477f07ba6b8843d0b6a48646a4", + urls = [ + "https://github.com/pytorch/cpuinfo/archive/8df44962d437a0477f07ba6b8843d0b6a48646a4.zip", + ], +) + # TF on 2024-09-24 _TENSORFLOW_GIT_COMMIT = "5329ec8dd396487982ef3e743f98c0195af39a6b" @@ -349,25 +368,6 @@ rules_foreign_cc_dependencies() load("@bazel_features//:deps.bzl", "bazel_features_deps") bazel_features_deps() -http_archive( - name = "cpuinfo", - sha256 = "2bf2b62eb86e2d2eaf862d0b9683a6c467a4d69fb2f7f1dc47c799809148608f", - strip_prefix = "cpuinfo-fa1c679da8d19e1d87f20175ae1ec10995cd3dd3", - urls = [ - "https://github.com/pytorch/cpuinfo/archive/fa1c679da8d19e1d87f20175ae1ec10995cd3dd3.zip", - ], -) - -# KleidiAI is needed to get the best possible performance out of XNNPack -http_archive( - name = "KleidiAI", - sha256 = "88233e427be6579560073267575f00f3b5fc370a31a43bbdd87a1810bd4bf1b6", - strip_prefix = "kleidiai-cddf991af5de49fd34949fa39690e4e906e04074", - urls = [ - "https://gitlab.arm.com/kleidi/kleidiai/-/archive/cddf991af5de49fd34949fa39690e4e906e04074/kleidiai-cddf991af5de49fd34949fa39690e4e906e04074.zip", - ], -) - # TODO: This is an are indirect dependency. We should factor it out. http_archive( name = "pthreadpool", diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc index 951237896f..9701aa6c9d 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc @@ -91,7 +91,7 @@ TEST(TextEmbedderTest, SucceedsWithCosineSimilarity) { double similarity; text_embedder_cosine_similarity(&result0.embeddings[0], &result1.embeddings[0], &similarity, nullptr); - double expected_similarity = 0.98513 + 0.00512937; + double expected_similarity = 0.98103; EXPECT_LE(abs(similarity - expected_similarity), kPrecision); text_embedder_close_result(&result0); diff --git a/mediapipe/tasks/cc/text/text_classifier/text_classifier_test.cc b/mediapipe/tasks/cc/text/text_classifier/text_classifier_test.cc index c4f63aab63..18b8e8ae31 100644 --- a/mediapipe/tasks/cc/text/text_classifier/text_classifier_test.cc +++ b/mediapipe/tasks/cc/text/text_classifier/text_classifier_test.cc @@ -155,7 +155,7 @@ TEST_F(TextClassifierTest, TextClassifierWithBert) { /*head_index=*/0, /*head_name=*/"probability"}); positive_expected.classifications.emplace_back(Classifications{ - /*categories=*/{{1, 0.9999413, "positive"}, {0, 0.000058, "negative"}}, + /*categories=*/{{1, 0.9999370, "positive"}, {0, 0.0000629, "negative"}}, /*head_index=*/0, /*head_name=*/"probability"}); #endif // _WIN32 @@ -251,8 +251,8 @@ TEST_F(TextClassifierTest, BertLongPositive) { categories.push_back( {/*index=*/0, /*score=*/0.023313, /*category_name=*/"negative"}); #else - categories.push_back({1, 0.983276, "positive"}); - categories.push_back({0, 0.016723, "negative"}); + categories.push_back({1, 0.981097, "positive"}); + categories.push_back({0, 0.018902, "negative"}); #endif // _WIN32 expected.classifications.emplace_back( From cff4666d73b1e1efe2dbf8f63a2492f96b9fcaff Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 1 Nov 2024 10:20:17 -0700 Subject: [PATCH 025/126] Format Workspace file PiperOrigin-RevId: 692218948 --- WORKSPACE | 260 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 150 insertions(+), 110 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 3c4fa53379..21edd7667f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -16,23 +16,27 @@ http_archive( "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", ], ) + load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") + bazel_skylib_workspace() + load("@bazel_skylib//lib:versions.bzl", "versions") + versions.check(minimum_bazel_version = "3.7.2") # ABSL on 2023-10-18 http_archive( name = "com_google_absl", - urls = ["https://github.com/abseil/abseil-cpp/archive//9687a8ea750bfcddf790372093245a1d041b21a3.tar.gz"], - patches = [ - "@//third_party:com_google_absl_windows_patch.diff" - ], patch_args = [ "-p1", ], - strip_prefix = "abseil-cpp-9687a8ea750bfcddf790372093245a1d041b21a3", + patches = [ + "@//third_party:com_google_absl_windows_patch.diff", + ], sha256 = "f841f78243f179326f2a80b719f2887c38fe226d288ecdc46e2aa091e6aa43bc", + strip_prefix = "abseil-cpp-9687a8ea750bfcddf790372093245a1d041b21a3", + urls = ["https://github.com/abseil/abseil-cpp/archive//9687a8ea750bfcddf790372093245a1d041b21a3.tar.gz"], ) http_archive( @@ -41,7 +45,6 @@ http_archive( url = "https://github.com/bazelbuild/rules_java/releases/download/5.3.5/rules_java-5.3.5.tar.gz", ) - http_archive( name = "rules_android_ndk", sha256 = "d230a980e0d3a42b85d5fce2cb17ec3ac52b88d2cff5aaf86bae0f05b48adc55", @@ -53,40 +56,39 @@ load("@rules_android_ndk//:rules.bzl", "android_ndk_repository") http_archive( name = "build_bazel_rules_apple", - sha256 = "3e2c7ae0ddd181c4053b6491dad1d01ae29011bc322ca87eea45957c76d3a0c3", - url = "https://github.com/bazelbuild/rules_apple/releases/download/2.1.0/rules_apple.2.1.0.tar.gz", + patch_args = [ + "-p1", + ], patches = [ # Bypass checking ios unit test runner when building MP ios applications. "@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff", # https://github.com/bazelbuild/rules_apple/commit/95b1305255dc29874cacc3dc7fdc017f16d8dbe8 - "@//third_party:build_bazel_rules_apple_multi_arch_split_with_new_transition.diff" - ], - patch_args = [ - "-p1", + "@//third_party:build_bazel_rules_apple_multi_arch_split_with_new_transition.diff", ], + sha256 = "3e2c7ae0ddd181c4053b6491dad1d01ae29011bc322ca87eea45957c76d3a0c3", + url = "https://github.com/bazelbuild/rules_apple/releases/download/2.1.0/rules_apple.2.1.0.tar.gz", ) http_archive( name = "com_google_protobuf", - sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422", - strip_prefix = "protobuf-3.19.1", - urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz"], - patches = [ - "@//third_party:com_google_protobuf_fixes.diff" - ], patch_args = [ "-p1", ], + patches = [ + "@//third_party:com_google_protobuf_fixes.diff", + ], + sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422", + strip_prefix = "protobuf-3.19.1", + urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz"], ) - # GoogleTest/GoogleMock framework. Used by most unit-tests. # Last updated 2021-07-02. http_archive( name = "com_google_googletest", - urls = ["https://github.com/google/googletest/archive/4ec4cd23f486bf70efcc5d2caa40f24368f752e3.zip"], - strip_prefix = "googletest-4ec4cd23f486bf70efcc5d2caa40f24368f752e3", sha256 = "de682ea824bfffba05b4e33b67431c247397d6175962534305136aa06f92e049", + strip_prefix = "googletest-4ec4cd23f486bf70efcc5d2caa40f24368f752e3", + urls = ["https://github.com/google/googletest/archive/4ec4cd23f486bf70efcc5d2caa40f24368f752e3.zip"], ) # Load Zlib before initializing TensorFlow and the iOS build rules to guarantee @@ -94,48 +96,48 @@ http_archive( http_archive( name = "zlib", build_file = "@//third_party:zlib.BUILD", - sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", - strip_prefix = "zlib-1.2.13", - url = "http://zlib.net/fossils/zlib-1.2.13.tar.gz", - patches = [ - "@//third_party:zlib.diff", - ], patch_args = [ "-p1", ], + patches = [ + "@//third_party:zlib.diff", + ], + sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", + strip_prefix = "zlib-1.2.13", + url = "http://zlib.net/fossils/zlib-1.2.13.tar.gz", ) - # gflags needed by glog http_archive( name = "com_github_gflags_gflags", - strip_prefix = "gflags-2.2.2", sha256 = "19713a36c9f32b33df59d1c79b4958434cb005b5b47dc5400a7a4b078111d9b5", + strip_prefix = "gflags-2.2.2", url = "https://github.com/gflags/gflags/archive/v2.2.2.zip", ) # 2020-08-21 http_archive( name = "com_github_glog_glog", - strip_prefix = "glog-0.6.0", sha256 = "8a83bf982f37bb70825df71a9709fa90ea9f4447fb3c099e1d720a439d88bad6", + strip_prefix = "glog-0.6.0", urls = [ "https://github.com/google/glog/archive/v0.6.0.tar.gz", ], ) + http_archive( name = "com_github_glog_glog_no_gflags", - strip_prefix = "glog-0.6.0", - sha256 = "8a83bf982f37bb70825df71a9709fa90ea9f4447fb3c099e1d720a439d88bad6", build_file = "@//third_party:glog_no_gflags.BUILD", - urls = [ - "https://github.com/google/glog/archive/v0.6.0.tar.gz", + patch_args = [ + "-p1", ], patches = [ "@//third_party:com_github_glog_glog.diff", ], - patch_args = [ - "-p1", + sha256 = "8a83bf982f37bb70825df71a9709fa90ea9f4447fb3c099e1d720a439d88bad6", + strip_prefix = "glog-0.6.0", + urls = [ + "https://github.com/google/glog/archive/v0.6.0.tar.gz", ], ) @@ -144,28 +146,29 @@ http_archive( # crashes on some Android devices. http_archive( name = "com_github_glog_glog_windows", - strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372", - sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb", - urls = [ - "https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip", + patch_args = [ + "-p1", ], patches = [ "@//third_party:com_github_glog_glog.diff", "@//third_party:com_github_glog_glog_windows_patch.diff", ], - patch_args = [ - "-p1", + sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb", + strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372", + urls = [ + "https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip", ], ) # Maven dependencies. RULES_JVM_EXTERNAL_TAG = "4.0" + RULES_JVM_EXTERNAL_SHA = "31701ad93dbfe544d597dbe62c9a1fdd76d81d8a9150c2bf1ecf928ecdf97169" http_archive( name = "rules_jvm_external", - strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, sha256 = RULES_JVM_EXTERNAL_SHA, + strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, ) @@ -204,13 +207,13 @@ maven_install( "junit:junit:4.12", "org.hamcrest:hamcrest-library:1.3", ], + fetch_sources = True, repositories = [ "https://maven.google.com", "https://dl.google.com/dl/android/maven2", "https://repo1.maven.org/maven2", "https://jcenter.bintray.com", ], - fetch_sources = True, version_conflict_policy = "pinned", ) @@ -237,20 +240,20 @@ http_archive( # 2020-07-09 http_archive( name = "pybind11_bazel", + sha256 = "75922da3a1bdb417d820398eb03d4e9bd067c4905a4246d35a44c01d62154d91", strip_prefix = "pybind11_bazel-203508e14aab7309892a1c5f7dd05debda22d9a5", urls = ["https://github.com/pybind/pybind11_bazel/archive/203508e14aab7309892a1c5f7dd05debda22d9a5.zip"], - sha256 = "75922da3a1bdb417d820398eb03d4e9bd067c4905a4246d35a44c01d62154d91", ) # 2022-10-20 http_archive( name = "pybind11", + build_file = "@pybind11_bazel//:pybind11.BUILD", + sha256 = "fcf94065efcfd0a7a828bacf118fa11c43f6390d0c805e3e6342ac119f2e9976", + strip_prefix = "pybind11-2.10.1", urls = [ "https://github.com/pybind/pybind11/archive/v2.10.1.zip", ], - sha256 = "fcf94065efcfd0a7a828bacf118fa11c43f6390d0c805e3e6342ac119f2e9976", - strip_prefix = "pybind11-2.10.1", - build_file = "@pybind11_bazel//:pybind11.BUILD", ) http_archive( @@ -289,8 +292,8 @@ _TENSORFLOW_SHA256 = "eb1f8d740d59ea3dee91108ab1fc19d91c4e9ac2fd17d9ab86d865c3c4 http_archive( name = "org_tensorflow", - urls = [ - "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT, + patch_args = [ + "-p1", ], patches = [ "@//third_party:org_tensorflow_c_api_experimental.diff", @@ -300,59 +303,72 @@ http_archive( # See https://github.com/bazelbuild/bazel/issues/19912 "@//third_party:org_tensorflow_objc_build_fixes.diff", ], - patch_args = [ - "-p1", - ], - strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT, sha256 = _TENSORFLOW_SHA256, + strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT, + urls = [ + "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT, + ], ) load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3") + tf_workspace3() # Initialize hermetic Python load("@org_tensorflow//third_party/xla/third_party/py:python_init_rules.bzl", "python_init_rules") + python_init_rules() load("@org_tensorflow//third_party/xla/third_party/py:python_init_repositories.bzl", "python_init_repositories") + python_init_repositories( default_python_version = "system", local_wheel_dist_folder = "dist", + local_wheel_inclusion_list = ["mediapipe*"], + local_wheel_workspaces = ["//:WORKSPACE"], requirements = { "3.9": "//:requirements_lock.txt", "3.10": "//:requirements_lock_3_10.txt", "3.11": "//:requirements_lock_3_11.txt", "3.12": "//:requirements_lock_3_12.txt", }, - local_wheel_inclusion_list = ["mediapipe*"], - local_wheel_workspaces = ["//:WORKSPACE"], ) load("@org_tensorflow//third_party/xla/third_party/py:python_init_toolchains.bzl", "python_init_toolchains") + python_init_toolchains() load("@org_tensorflow//third_party/xla/third_party/py:python_init_pip.bzl", "python_init_pip") + python_init_pip() load("@pypi//:requirements.bzl", "install_deps") + install_deps() # End hermetic Python initialization load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2") + tf_workspace2() load("@rules_python//python:pip.bzl", "pip_parse") + pip_parse( name = "mediapipe_pip_deps", requirements_lock = "@//:requirements_lock.txt", ) + load("@mediapipe_pip_deps//:requirements.bzl", mp_install_deps = "install_deps") + mp_install_deps() + pip_parse( name = "model_maker_pip_deps", requirements_lock = "@//mediapipe/model_maker:requirements_lock.txt", ) + load("@model_maker_pip_deps//:requirements.bzl", mm_install_deps = "install_deps") + mm_install_deps() http_archive( @@ -363,9 +379,11 @@ http_archive( ) load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies") + rules_foreign_cc_dependencies() load("@bazel_features//:deps.bzl", "bazel_features_deps") + bazel_features_deps() # TODO: This is an are indirect dependency. We should factor it out. @@ -380,53 +398,56 @@ load( "@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies", ) + apple_rules_dependencies() load( "@build_bazel_rules_swift//swift:repositories.bzl", "swift_rules_dependencies", ) + swift_rules_dependencies() load( "@build_bazel_rules_swift//swift:extras.bzl", "swift_rules_extra_dependencies", ) + swift_rules_extra_dependencies() load( "@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies", ) + apple_support_dependencies() # This is used to select all contents of the archives for CMake-based packages to give CMake access to them. all_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])""" - # Google Benchmark library v1.6.1 released on 2022-01-10. http_archive( name = "com_google_benchmark", - urls = ["https://github.com/google/benchmark/archive/refs/tags/v1.6.1.tar.gz"], - strip_prefix = "benchmark-1.6.1", - sha256 = "6132883bc8c9b0df5375b16ab520fac1a85dc9e4cf5be59480448ece74b278d4", build_file = "@//third_party:benchmark.BUILD", + sha256 = "6132883bc8c9b0df5375b16ab520fac1a85dc9e4cf5be59480448ece74b278d4", + strip_prefix = "benchmark-1.6.1", + urls = ["https://github.com/google/benchmark/archive/refs/tags/v1.6.1.tar.gz"], ) # easyexif http_archive( name = "easyexif", - url = "https://github.com/mayanklahiri/easyexif/archive/master.zip", - strip_prefix = "easyexif-master", build_file = "@//third_party:easyexif.BUILD", + strip_prefix = "easyexif-master", + url = "https://github.com/mayanklahiri/easyexif/archive/master.zip", ) # libyuv http_archive( name = "libyuv", + build_file = "@//third_party:libyuv.BUILD", # Error: operand type mismatch for `vbroadcastss' caused by commit 8a13626e42f7fdcf3a6acbb0316760ee54cda7d8. urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/2525698acba9bf9b701ba6b4d9584291a1f62257.tar.gz"], - build_file = "@//third_party:libyuv.BUILD", ) # Note: protobuf-javalite is no longer released as a separate download, it's included in the main Java download. @@ -439,38 +460,43 @@ http_archive( ) load("@//third_party/flatbuffers:workspace.bzl", flatbuffers = "repo") + flatbuffers() http_archive( name = "com_google_audio_tools", - strip_prefix = "multichannel-audio-tools-1f6b1319f13282eda6ff1317be13de67f4723860", - urls = ["https://github.com/google/multichannel-audio-tools/archive/1f6b1319f13282eda6ff1317be13de67f4723860.zip"], - sha256 = "fe346e1aee4f5069c4cbccb88706a9a2b2b4cf98aeb91ec1319be77e07dd7435", - repo_mapping = {"@com_github_glog_glog" : "@com_github_glog_glog_no_gflags"}, + patch_args = ["-p1"], # TODO: Fix this in AudioTools directly patches = ["@//third_party:com_google_audio_tools_fixes.diff"], - patch_args = ["-p1"] + repo_mapping = {"@com_github_glog_glog": "@com_github_glog_glog_no_gflags"}, + sha256 = "fe346e1aee4f5069c4cbccb88706a9a2b2b4cf98aeb91ec1319be77e07dd7435", + strip_prefix = "multichannel-audio-tools-1f6b1319f13282eda6ff1317be13de67f4723860", + urls = ["https://github.com/google/multichannel-audio-tools/archive/1f6b1319f13282eda6ff1317be13de67f4723860.zip"], ) http_archive( name = "pffft", + build_file = "@//third_party:pffft.BUILD", strip_prefix = "jpommier-pffft-7c3b5a7dc510", urls = ["https://bitbucket.org/jpommier/pffft/get/7c3b5a7dc510.zip"], - build_file = "@//third_party:pffft.BUILD", ) # Sentencepiece http_archive( name = "com_google_sentencepiece", - strip_prefix = "sentencepiece-0.1.96", add_prefix = "sentencepiece", + build_file = "@//third_party:sentencepiece.BUILD", + patch_args = [ + "-d", + "sentencepiece", + "-p1", + ], + patches = ["@//third_party:com_google_sentencepiece.diff"], sha256 = "8409b0126ebd62b256c685d5757150cf7fcb2b92a2f2b98efb3f38fc36719754", + strip_prefix = "sentencepiece-0.1.96", urls = [ - "https://github.com/google/sentencepiece/archive/refs/tags/v0.1.96.zip" + "https://github.com/google/sentencepiece/archive/refs/tags/v0.1.96.zip", ], - build_file = "@//third_party:sentencepiece.BUILD", - patches = ["@//third_party:com_google_sentencepiece.diff"], - patch_args = ["-d", "sentencepiece", "-p1"], ) http_archive( @@ -485,17 +511,17 @@ http_archive( http_archive( name = "org_tensorflow_text", - sha256 = "f64647276f7288d1b1fe4c89581d51404d0ce4ae97f2bcc4c19bd667549adca8", - strip_prefix = "text-2.2.0", - urls = [ - "https://github.com/tensorflow/text/archive/v2.2.0.zip", - ], + patch_args = ["-p1"], patches = [ "@//third_party:tensorflow_text_remove_tf_deps.diff", "@//third_party:tensorflow_text_a0f49e63.diff", ], - patch_args = ["-p1"], repo_mapping = {"@com_google_re2": "@com_googlesource_code_re2"}, + sha256 = "f64647276f7288d1b1fe4c89581d51404d0ce4ae97f2bcc4c19bd667549adca8", + strip_prefix = "text-2.2.0", + urls = [ + "https://github.com/tensorflow/text/archive/v2.2.0.zip", + ], ) http_archive( @@ -510,15 +536,15 @@ http_archive( # Point to the commit that deprecates the usage of Eigen::MappedSparseMatrix. http_archive( name = "ceres_solver", - url = "https://github.com/ceres-solver/ceres-solver/archive/123fba61cf2611a3c8bddc9d91416db26b10b558.zip", - patches = [ - "@//third_party:ceres_solver_compatibility_fixes.diff" - ], patch_args = [ "-p1", ], + patches = [ + "@//third_party:ceres_solver_compatibility_fixes.diff", + ], + sha256 = "8b7b16ceb363420e0fd499576daf73fa338adb0b1449f58bea7862766baa1ac7", strip_prefix = "ceres-solver-123fba61cf2611a3c8bddc9d91416db26b10b558", - sha256 = "8b7b16ceb363420e0fd499576daf73fa338adb0b1449f58bea7862766baa1ac7" + url = "https://github.com/ceres-solver/ceres-solver/archive/123fba61cf2611a3c8bddc9d91416db26b10b558.zip", ) http_archive( @@ -537,7 +563,7 @@ new_local_repository( new_local_repository( name = "linux_ffmpeg", build_file = "@//third_party:ffmpeg_linux.BUILD", - path = "/usr" + path = "/usr", ) new_local_repository( @@ -575,8 +601,8 @@ http_archive( # '-DBUILD_PROTOBUF=OFF -DBUILD_opencv_dnn=OFF'. http_archive( name = "ios_opencv", - sha256 = "7dd536d06f59e6e1156b546bd581523d8df92ce83440002885ec5abc06558de2", build_file = "@//third_party:opencv_ios.BUILD", + sha256 = "7dd536d06f59e6e1156b546bd581523d8df92ce83440002885ec5abc06558de2", type = "zip", url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip", ) @@ -591,32 +617,32 @@ http_archive( # Task libraries are built. http_archive( name = "ios_opencv_source", - sha256 = "a61e7a4618d353140c857f25843f39b2abe5f451b018aab1604ef0bc34cd23d5", build_file = "@//third_party:opencv_ios_source.BUILD", + sha256 = "a61e7a4618d353140c857f25843f39b2abe5f451b018aab1604ef0bc34cd23d5", type = "zip", url = "https://github.com/opencv/opencv/archive/refs/tags/4.5.3.zip", ) http_archive( name = "stblib", - strip_prefix = "stb-b42009b3b9d4ca35bc703f5310eedc74f584be58", - sha256 = "13a99ad430e930907f5611325ec384168a958bf7610e63e60e2fd8e7b7379610", - urls = ["https://github.com/nothings/stb/archive/b42009b3b9d4ca35bc703f5310eedc74f584be58.tar.gz"], build_file = "@//third_party:stblib.BUILD", - patches = [ - "@//third_party:stb_image_impl.diff" - ], patch_args = [ "-p1", ], + patches = [ + "@//third_party:stb_image_impl.diff", + ], + sha256 = "13a99ad430e930907f5611325ec384168a958bf7610e63e60e2fd8e7b7379610", + strip_prefix = "stb-b42009b3b9d4ca35bc703f5310eedc74f584be58", + urls = ["https://github.com/nothings/stb/archive/b42009b3b9d4ca35bc703f5310eedc74f584be58.tar.gz"], ) http_archive( name = "google_toolbox_for_mac", - url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip", + build_file = "@//third_party:google_toolbox_for_mac.BUILD", sha256 = "e3ac053813c989a88703556df4dc4466e424e30d32108433ed6beaec76ba4fdc", strip_prefix = "google-toolbox-for-mac-2.2.1", - build_file = "@//third_party:google_toolbox_for_mac.BUILD", + url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip", ) # Hermetic CUDA @@ -637,9 +663,11 @@ load( "cuda_redist_init_repositories", "cudnn_redist_init_repository", ) + cuda_redist_init_repositories( cuda_redistributions = CUDA_REDISTRIBUTIONS, ) + cudnn_redist_init_repository( cudnn_redistributions = CUDNN_REDISTRIBUTIONS, ) @@ -648,23 +676,26 @@ load( "@org_tensorflow//third_party/gpus/cuda/hermetic:cuda_configure.bzl", "cuda_configure", ) + cuda_configure(name = "local_config_cuda") # Edge TPU http_archive( - name = "libedgetpu", - sha256 = "14d5527a943a25bc648c28a9961f954f70ba4d79c0a9ca5ae226e1831d72fe80", - strip_prefix = "libedgetpu-3164995622300286ef2bb14d7fdc2792dae045b7", - urls = [ - "https://github.com/google-coral/libedgetpu/archive/3164995622300286ef2bb14d7fdc2792dae045b7.tar.gz" - ], + name = "libedgetpu", + sha256 = "14d5527a943a25bc648c28a9961f954f70ba4d79c0a9ca5ae226e1831d72fe80", + strip_prefix = "libedgetpu-3164995622300286ef2bb14d7fdc2792dae045b7", + urls = [ + "https://github.com/google-coral/libedgetpu/archive/3164995622300286ef2bb14d7fdc2792dae045b7.tar.gz", + ], ) + load("@libedgetpu//:workspace.bzl", "libedgetpu_dependencies") + libedgetpu_dependencies() load("@coral_crosstool//:configure.bzl", "cc_crosstool") -cc_crosstool(name = "crosstool") +cc_crosstool(name = "crosstool") # Node dependencies http_archive( @@ -674,11 +705,14 @@ http_archive( ) load("@build_bazel_rules_nodejs//:repositories.bzl", "build_bazel_rules_nodejs_dependencies") + build_bazel_rules_nodejs_dependencies() # fetches nodejs, npm, and yarn load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "yarn_install") + node_repositories() + yarn_install( name = "npm", package_json = "@//:package.json", @@ -700,18 +734,24 @@ http_archive( urls = ["https://github.com/protocolbuffers/protobuf-javascript/archive/refs/tags/v3.21.2.tar.gz"], ) -load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains", "rules_proto_grpc_repos") +load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains") + rules_proto_grpc_toolchains() + rules_proto_grpc_repos() load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") + rules_proto_dependencies() + rules_proto_toolchains() load("@//third_party:external_files.bzl", "external_files") + external_files() load("@//third_party:wasm_files.bzl", "wasm_files") + wasm_files() # Halide @@ -719,39 +759,39 @@ wasm_files() new_local_repository( name = "halide", build_file = "@//third_party/halide:BUILD.bazel", - path = "third_party/halide" + path = "third_party/halide", ) http_archive( name = "linux_halide", + build_file = "@//third_party:halide.BUILD", sha256 = "d290fadf3f358c94aacf43c883de6468bb98883e26116920afd491ec0e440cd2", strip_prefix = "Halide-15.0.1-x86-64-linux", urls = ["https://github.com/halide/Halide/releases/download/v15.0.1/Halide-15.0.1-x86-64-linux-4c63f1befa1063184c5982b11b6a2cc17d4e5815.tar.gz"], - build_file = "@//third_party:halide.BUILD", ) http_archive( name = "macos_x86_64_halide", + build_file = "@//third_party:halide.BUILD", sha256 = "48ff073ac1aee5c4aca941a4f043cac64b38ba236cdca12567e09d803594a61c", strip_prefix = "Halide-15.0.1-x86-64-osx", urls = ["https://github.com/halide/Halide/releases/download/v15.0.1/Halide-15.0.1-x86-64-osx-4c63f1befa1063184c5982b11b6a2cc17d4e5815.tar.gz"], - build_file = "@//third_party:halide.BUILD", ) http_archive( name = "macos_arm_64_halide", + build_file = "@//third_party:halide.BUILD", sha256 = "db5d20d75fa7463490fcbc79c89f0abec9c23991f787c8e3e831fff411d5395c", strip_prefix = "Halide-15.0.1-arm-64-osx", urls = ["https://github.com/halide/Halide/releases/download/v15.0.1/Halide-15.0.1-arm-64-osx-4c63f1befa1063184c5982b11b6a2cc17d4e5815.tar.gz"], - build_file = "@//third_party:halide.BUILD", ) http_archive( name = "windows_halide", + build_file = "@//third_party:halide.BUILD", sha256 = "61fd049bd75ee918ac6c30d0693aac6048f63f8d1fc4db31001573e58eae8dae", strip_prefix = "Halide-15.0.1-x86-64-windows", urls = ["https://github.com/halide/Halide/releases/download/v15.0.1/Halide-15.0.1-x86-64-windows-4c63f1befa1063184c5982b11b6a2cc17d4e5815.zip"], - build_file = "@//third_party:halide.BUILD", ) http_archive( @@ -763,9 +803,9 @@ http_archive( http_archive( name = "com_github_nlohmann_json", + build_file = "@//third_party:nlohmann.BUILD", sha256 = "6bea5877b1541d353bd77bdfbdb2696333ae5ed8f9e8cc22df657192218cad91", urls = ["https://github.com/nlohmann/json/releases/download/v3.9.1/include.zip"], - build_file = "@//third_party:nlohmann.BUILD", ) http_archive( From b303d4fc3e70c7567675f586976e3123b63f0c1f Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 1 Nov 2024 12:06:40 -0700 Subject: [PATCH 026/126] Add EglSync wrapper. PiperOrigin-RevId: 692252113 --- mediapipe/gpu/BUILD | 32 ++++++ mediapipe/gpu/egl_errors.cc | 72 ++++++++++++ mediapipe/gpu/egl_errors.h | 13 +++ mediapipe/gpu/egl_sync.cc | 214 ++++++++++++++++++++++++++++++++++++ mediapipe/gpu/egl_sync.h | 66 +++++++++++ mediapipe/gpu/gl_base.h | 1 + 6 files changed, 398 insertions(+) create mode 100644 mediapipe/gpu/egl_errors.cc create mode 100644 mediapipe/gpu/egl_errors.h create mode 100644 mediapipe/gpu/egl_sync.cc create mode 100644 mediapipe/gpu/egl_sync.h diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index ba6e7f7cef..73cc464b1d 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -841,6 +841,38 @@ cc_library( }), ) +cc_library( + name = "egl_errors", + srcs = ["egl_errors.cc"], + hdrs = ["egl_errors.h"], + visibility = ["//visibility:public"], + deps = [ + ":gl_base", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "egl_sync", + srcs = ["egl_sync.cc"], + hdrs = ["egl_sync.h"], + visibility = ["//visibility:public"], + deps = [ + ":egl_errors", + ":gl_base", + "//mediapipe/framework/deps:no_destructor", + "//mediapipe/framework/formats:unique_fd", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + ], +) + cc_library( name = "gl_texture_util", srcs = ["gl_texture_util.cc"], diff --git a/mediapipe/gpu/egl_errors.cc b/mediapipe/gpu/egl_errors.cc new file mode 100644 index 0000000000..e3f9be3725 --- /dev/null +++ b/mediapipe/gpu/egl_errors.cc @@ -0,0 +1,72 @@ +#include "mediapipe/gpu/egl_errors.h" + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "mediapipe/gpu/gl_base.h" + +namespace mediapipe { + +absl::Status GetEglError() { + EGLint error = eglGetError(); + switch (error) { + case EGL_SUCCESS: + return absl::OkStatus(); + case EGL_NOT_INITIALIZED: + return absl::InternalError( + "EGL is not initialized, or could not be initialized, for the " + "specified EGL display connection."); + case EGL_BAD_ACCESS: + return absl::InternalError( + "EGL cannot access a requested resource (for example a context is " + "bound in another thread)."); + case EGL_BAD_ALLOC: + return absl::InternalError( + "EGL failed to allocate resources for the requested operation."); + case EGL_BAD_ATTRIBUTE: + return absl::InternalError( + "An unrecognized attribute or attribute value was passed in the " + "attribute list."); + case EGL_BAD_CONTEXT: + return absl::InternalError( + "An EGLContext argument does not name a valid EGL rendering " + "context."); + case EGL_BAD_CONFIG: + return absl::InternalError( + "An EGLConfig argument does not name a valid EGL frame buffer " + "configuration."); + case EGL_BAD_CURRENT_SURFACE: + return absl::InternalError( + "The current surface of the calling thread is a window, pixel buffer " + "or pixmap that is no longer valid."); + case EGL_BAD_DISPLAY: + return absl::InternalError( + "An EGLDisplay argument does not name a valid EGL display " + "connection."); + case EGL_BAD_SURFACE: + return absl::InternalError( + "An EGLSurface argument does not name a valid surface (window, pixel " + "buffer or pixmap) configured for GL rendering."); + case EGL_BAD_MATCH: + return absl::InternalError( + "Arguments are inconsistent (for example, a valid context requires " + "buffers not supplied by a valid surface)."); + case EGL_BAD_PARAMETER: + return absl::InternalError("One or more argument values are invalid."); + case EGL_BAD_NATIVE_PIXMAP: + return absl::InternalError( + "A NativePixmapType argument does not refer to a valid native " + "pixmap."); + case EGL_BAD_NATIVE_WINDOW: + return absl::InternalError( + "A NativeWindowType argument does not refer to a valid native " + "window."); + case EGL_CONTEXT_LOST: + return absl::InternalError( + "A power management event has occurred. The application must destroy " + "all contexts and reinitialize OpenGL ES state and objects to " + "continue rendering."); + } + return absl::UnknownError(absl::StrCat("EGL error: ", error)); +} + +} // namespace mediapipe diff --git a/mediapipe/gpu/egl_errors.h b/mediapipe/gpu/egl_errors.h new file mode 100644 index 0000000000..9a14f0c79b --- /dev/null +++ b/mediapipe/gpu/egl_errors.h @@ -0,0 +1,13 @@ +#ifndef MEDIAPIPE_GPU_EGL_ERRORS_H_ +#define MEDIAPIPE_GPU_EGL_ERRORS_H_ + +#include "absl/status/status.h" + +namespace mediapipe { + +// Returns the error of the last called EGL function in the current thread. +absl::Status GetEglError(); + +} // namespace mediapipe + +#endif // MEDIAPIPE_GPU_EGL_ERRORS_H_ diff --git a/mediapipe/gpu/egl_sync.cc b/mediapipe/gpu/egl_sync.cc new file mode 100644 index 0000000000..b6a6746e46 --- /dev/null +++ b/mediapipe/gpu/egl_sync.cc @@ -0,0 +1,214 @@ +#include "mediapipe/gpu/egl_sync.h" + +#include + +#include +#include + +#include "absl/cleanup/cleanup.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "mediapipe/framework/deps/no_destructor.h" +#include "mediapipe/framework/formats/unique_fd.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/gpu/egl_errors.h" +#include "mediapipe/gpu/gl_base.h" + +namespace mediapipe { + +namespace { + +PFNEGLCREATESYNCKHRPROC eglCreateSyncKHR; +PFNEGLWAITSYNCKHRPROC eglWaitSyncKHR; +PFNEGLCLIENTWAITSYNCKHRPROC eglClientWaitSyncKHR; +PFNEGLDESTROYSYNCKHRPROC eglDestroySyncKHR; +PFNEGLDUPNATIVEFENCEFDANDROIDPROC eglDupNativeFenceFDANDROID; +PFNEGLGETSYNCATTRIBKHRPROC eglGetSyncAttribKHR; + +bool HasExtension(EGLDisplay display, const char* extension) { + const char* extensions = eglQueryString(display, EGL_EXTENSIONS); + return extensions && std::strstr(extensions, extension); +} + +absl::Status CheckEglFenceSyncSupported(EGLDisplay display) { + static bool supported = HasExtension(display, "EGL_KHR_fence_sync"); + if (supported) { + return absl::OkStatus(); + } + return absl::UnavailableError("EGL_KHR_fence_sync unavailable."); +} + +absl::Status CheckEglWaitSyncSupported(EGLDisplay display) { + static bool supported = HasExtension(display, "EGL_KHR_wait_sync"); + if (supported) { + return absl::OkStatus(); + } + return absl::UnavailableError("EGL_KHR_wait_sync unavailable."); +} + +absl::Status CheckEglAndroidNativeSyncSupported(EGLDisplay display) { + static bool supported = + HasExtension(display, "EGL_ANDROID_native_fence_sync"); + if (supported) { + return absl::OkStatus(); + } + return absl::UnavailableError("EGL_ANDROID_native_fence_sync unavailable."); +} + +absl::Status CheckEglSyncSupported(EGLDisplay egl_display) { + static NoDestructor support_status([&]() -> absl::Status { + MP_RETURN_IF_ERROR(CheckEglFenceSyncSupported(egl_display)); + MP_RETURN_IF_ERROR(CheckEglWaitSyncSupported(egl_display)); + + RET_CHECK(eglCreateSyncKHR = reinterpret_cast( + eglGetProcAddress("eglCreateSyncKHR"))); + RET_CHECK(eglWaitSyncKHR = reinterpret_cast( + eglGetProcAddress("eglWaitSyncKHR"))); + RET_CHECK(eglClientWaitSyncKHR = + reinterpret_cast( + eglGetProcAddress("eglClientWaitSyncKHR"))); + RET_CHECK(eglDestroySyncKHR = reinterpret_cast( + eglGetProcAddress("eglDestroySyncKHR"))); + RET_CHECK(eglGetSyncAttribKHR = + reinterpret_cast( + eglGetProcAddress("eglGetSyncAttribKHR"))); + return absl::OkStatus(); + }()); + return *support_status; +} + +absl::Status CheckEglNativeSyncSupported(EGLDisplay egl_display) { + static NoDestructor support_status([&]() -> absl::Status { + MP_RETURN_IF_ERROR(CheckEglAndroidNativeSyncSupported(egl_display)); + RET_CHECK(eglDupNativeFenceFDANDROID = + reinterpret_cast( + eglGetProcAddress("eglDupNativeFenceFDANDROID"))); + return absl::OkStatus(); + }()); + return *support_status; +} + +} // namespace + +absl::StatusOr EglSync::Create(EGLDisplay display) { + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); + + const EGLSyncKHR egl_sync = + eglCreateSyncKHR(display, EGL_SYNC_FENCE_KHR, nullptr); + RET_CHECK_NE(egl_sync, EGL_NO_SYNC_KHR) + << "Create/eglCreateSyncKHR failed: " << GetEglError(); + return EglSync(display, egl_sync); +} + +absl::StatusOr EglSync::CreateNative(EGLDisplay display) { + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); + MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); + + const EGLSyncKHR egl_sync = + eglCreateSyncKHR(display, EGL_SYNC_NATIVE_FENCE_ANDROID, nullptr); + RET_CHECK_NE(egl_sync, EGL_NO_SYNC_KHR) + << "CreateNative/eglCreateSyncKHR failed: " << GetEglError(); + return EglSync(display, egl_sync); +} + +absl::StatusOr EglSync::CreateNative(EGLDisplay display, + const UniqueFd& native_fence_fd) { + RET_CHECK(native_fence_fd.IsValid()); + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); + MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); + + MP_ASSIGN_OR_RETURN(UniqueFd fd_for_egl, native_fence_fd.Dup()); + // NOTE: it looks like one could rely on `UniqueFd` for the cleanup, but + // there's some clashing on ownership of the FD when passing it to + // eglCreateSyncKHR and then using `UniqueFd::Release`, hence relying on + // absl::Cleanup. + const int fd = fd_for_egl.Release(); + absl::Cleanup fd_cleanup = [fd]() { close(fd); }; + const EGLint sync_attribs[] = {EGL_SYNC_NATIVE_FENCE_FD_ANDROID, + static_cast(fd), EGL_NONE}; + const EGLSyncKHR egl_sync = + eglCreateSyncKHR(display, EGL_SYNC_NATIVE_FENCE_ANDROID, sync_attribs); + RET_CHECK_NE(egl_sync, EGL_NO_SYNC_KHR) << absl::StrCat( + "CreateNative/eglCreateSyncKHR with original FD: ", native_fence_fd.Get(), + " and dup FD: ", fd, " - failed: ", GetEglError()); + // EGL took ownership of the passed FD as eglCreateSyncKHR succeeded, so + // cancelling the cleanup. + std::move(fd_cleanup).Cancel(); + + return EglSync(display, egl_sync); +} + +EglSync::EglSync(EglSync&& sync) { *this = std::move(sync); } + +EglSync& EglSync::operator=(EglSync&& sync) { + if (this != &sync) { + Invalidate(); + + using std::swap; + sync_ = std::exchange(sync.sync_, EGL_NO_SYNC_KHR); + display_ = std::exchange(sync.display_, EGL_NO_DISPLAY); + } + return *this; +} + +void EglSync::Invalidate() { + if (sync_ == EGL_NO_SYNC_KHR || display_ == EGL_NO_DISPLAY) { + return; + } + + const absl::Status egl_sync_support = CheckEglSyncSupported(display_); + if (!egl_sync_support.ok()) { + ABSL_LOG(DFATAL) << "Attempt to destroy an EGL sync: " << egl_sync_support; + return; + } + + // Needs extension: EGL_KHR_fence_sync (EGL) / GL_OES_EGL_sync (OpenGL ES). + // Note: we're doing nothing when the function pointer is nullptr, or the + // call returns EGL_FALSE. + const EGLBoolean result = eglDestroySyncKHR(display_, sync_); + if (result == EGL_FALSE) { + ABSL_LOG(DFATAL) << "eglDestroySyncKHR failed: " << GetEglError(); + } + sync_ = EGL_NO_SYNC_KHR; +} + +absl::Status EglSync::WaitOnGpu() { + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display_)); + + const EGLint result = eglWaitSyncKHR(display_, sync_, 0); + RET_CHECK_EQ(result, EGL_TRUE) << "eglWaitSyncKHR failed: " << GetEglError(); + return absl::OkStatus(); +} + +absl::Status EglSync::Wait() { + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display_)); + + const EGLint result = eglClientWaitSyncKHR( + display_, sync_, EGL_SYNC_FLUSH_COMMANDS_BIT_KHR, EGL_FOREVER_KHR); + RET_CHECK_EQ(result, EGL_CONDITION_SATISFIED_KHR) + << "eglClientWaitSyncKHR failed: " << GetEglError(); + return absl::OkStatus(); +} + +absl::StatusOr EglSync::DupNativeFd() { + MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display_)); + + const int fd = eglDupNativeFenceFDANDROID(display_, sync_); + RET_CHECK_NE(fd, EGL_NO_NATIVE_FENCE_FD_ANDROID) + << "eglDupNativeFenceFDANDROID failed: " << GetEglError(); + return UniqueFd(fd); +} + +absl::StatusOr EglSync::IsSignaled() { + EGLint status; + const EGLBoolean success = + eglGetSyncAttribKHR(display_, sync_, EGL_SYNC_STATUS_KHR, &status); + RET_CHECK_EQ(success, EGL_TRUE) + << "eglGetSyncAttribKHR failed: " << GetEglError(); + return status == EGL_SIGNALED_KHR; +} + +} // namespace mediapipe diff --git a/mediapipe/gpu/egl_sync.h b/mediapipe/gpu/egl_sync.h new file mode 100644 index 0000000000..a1f00d66c8 --- /dev/null +++ b/mediapipe/gpu/egl_sync.h @@ -0,0 +1,66 @@ +#ifndef MEDIAPIPE_GPU_EGL_SYNC_H_ +#define MEDIAPIPE_GPU_EGL_SYNC_H_ + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/unique_fd.h" +#include "mediapipe/gpu/gl_base.h" + +namespace mediapipe { + +// RAII wrapper for EGL sync object. +class EglSync { + public: + // Creates a fence in OpenGL command stream. This sync is enqueued and *not* + // flushed. + static absl::StatusOr Create(EGLDisplay display); + + // Creates a native fence in OpenGL command stream. This sync is enqueued and + // *not* flushed. + static absl::StatusOr CreateNative(EGLDisplay display); + + // Create a native fence in OpenGL command stream based on a native fence FD. + static absl::StatusOr CreateNative(EGLDisplay display, + const UniqueFd& native_fence_fd); + + // Move-only + EglSync(EglSync&& sync); + EglSync& operator=(EglSync&& sync); + + EglSync(const EglSync&) = delete; + EglSync& operator=(const EglSync&) = delete; + + ~EglSync() { Invalidate(); } + + // Causes GPU to block and wait until this sync has been signaled. + // This call does not block and returns immediately. + absl::Status WaitOnGpu(); + + // Causes CPU to block and wait until this sync has been signaled. + absl::Status Wait(); + + // Returns the EGLDisplay on which this instance was created. + EGLDisplay display() const { return display_; } + + // Returns the EGLSyncKHR wrapped by this instance. + EGLSyncKHR sync() const { return sync_; } + + // Returns true if this EGL sync is signaled. + absl::StatusOr IsSignaled(); + + // Duplicates the file descriptor stored in native EGL fence sync. + absl::StatusOr DupNativeFd(); + + private: + EglSync(EGLDisplay display, EGLSyncKHR sync) + : display_(display), sync_(sync) {} + + void Invalidate(); + + EGLDisplay display_; + EGLSyncKHR sync_ = EGL_NO_SYNC_KHR; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_GPU_EGL_SYNC_H_ diff --git a/mediapipe/gpu/gl_base.h b/mediapipe/gpu/gl_base.h index 12a04e0bbf..eb309e6857 100644 --- a/mediapipe/gpu/gl_base.h +++ b/mediapipe/gpu/gl_base.h @@ -55,6 +55,7 @@ #define HAS_EGL 1 #include +#include #include #include #if defined(__ANDROID__) From 9ce100adf9971a12bad0ad026d13974e301dd751 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 1 Nov 2024 14:28:24 -0700 Subject: [PATCH 027/126] Update the expected accuracy for text embedder test. PiperOrigin-RevId: 692291740 --- mediapipe/tasks/python/test/text/text_embedder_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/python/test/text/text_embedder_test.py b/mediapipe/tasks/python/test/text/text_embedder_test.py index 3890bf2bdd..ebc69ef809 100644 --- a/mediapipe/tasks/python/test/text/text_embedder_test.py +++ b/mediapipe/tasks/python/test/text/text_embedder_test.py @@ -287,7 +287,7 @@ def test_embed_in_context(self, l2_normalize, quantize, model_name, @parameterized.parameters( # TODO: The similarity should likely be lower - (_BERT_MODEL_FILE, 0.99025), + (_BERT_MODEL_FILE, 0.98103), (_USE_MODEL_FILE, 0.780334), ) def test_embed_with_different_themes(self, model_file, expected_similarity): From 371ff2c157a33741343dcc9b3ae3da513f3b8cbb Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 1 Nov 2024 14:38:18 -0700 Subject: [PATCH 028/126] No public description PiperOrigin-RevId: 692294588 --- mediapipe/gpu/BUILD | 32 ------ mediapipe/gpu/egl_errors.cc | 72 ------------ mediapipe/gpu/egl_errors.h | 13 --- mediapipe/gpu/egl_sync.cc | 214 ------------------------------------ mediapipe/gpu/egl_sync.h | 66 ----------- mediapipe/gpu/gl_base.h | 1 - 6 files changed, 398 deletions(-) delete mode 100644 mediapipe/gpu/egl_errors.cc delete mode 100644 mediapipe/gpu/egl_errors.h delete mode 100644 mediapipe/gpu/egl_sync.cc delete mode 100644 mediapipe/gpu/egl_sync.h diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 73cc464b1d..ba6e7f7cef 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -841,38 +841,6 @@ cc_library( }), ) -cc_library( - name = "egl_errors", - srcs = ["egl_errors.cc"], - hdrs = ["egl_errors.h"], - visibility = ["//visibility:public"], - deps = [ - ":gl_base", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "egl_sync", - srcs = ["egl_sync.cc"], - hdrs = ["egl_sync.h"], - visibility = ["//visibility:public"], - deps = [ - ":egl_errors", - ":gl_base", - "//mediapipe/framework/deps:no_destructor", - "//mediapipe/framework/formats:unique_fd", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/cleanup", - "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - ], -) - cc_library( name = "gl_texture_util", srcs = ["gl_texture_util.cc"], diff --git a/mediapipe/gpu/egl_errors.cc b/mediapipe/gpu/egl_errors.cc deleted file mode 100644 index e3f9be3725..0000000000 --- a/mediapipe/gpu/egl_errors.cc +++ /dev/null @@ -1,72 +0,0 @@ -#include "mediapipe/gpu/egl_errors.h" - -#include "absl/status/status.h" -#include "absl/strings/str_cat.h" -#include "mediapipe/gpu/gl_base.h" - -namespace mediapipe { - -absl::Status GetEglError() { - EGLint error = eglGetError(); - switch (error) { - case EGL_SUCCESS: - return absl::OkStatus(); - case EGL_NOT_INITIALIZED: - return absl::InternalError( - "EGL is not initialized, or could not be initialized, for the " - "specified EGL display connection."); - case EGL_BAD_ACCESS: - return absl::InternalError( - "EGL cannot access a requested resource (for example a context is " - "bound in another thread)."); - case EGL_BAD_ALLOC: - return absl::InternalError( - "EGL failed to allocate resources for the requested operation."); - case EGL_BAD_ATTRIBUTE: - return absl::InternalError( - "An unrecognized attribute or attribute value was passed in the " - "attribute list."); - case EGL_BAD_CONTEXT: - return absl::InternalError( - "An EGLContext argument does not name a valid EGL rendering " - "context."); - case EGL_BAD_CONFIG: - return absl::InternalError( - "An EGLConfig argument does not name a valid EGL frame buffer " - "configuration."); - case EGL_BAD_CURRENT_SURFACE: - return absl::InternalError( - "The current surface of the calling thread is a window, pixel buffer " - "or pixmap that is no longer valid."); - case EGL_BAD_DISPLAY: - return absl::InternalError( - "An EGLDisplay argument does not name a valid EGL display " - "connection."); - case EGL_BAD_SURFACE: - return absl::InternalError( - "An EGLSurface argument does not name a valid surface (window, pixel " - "buffer or pixmap) configured for GL rendering."); - case EGL_BAD_MATCH: - return absl::InternalError( - "Arguments are inconsistent (for example, a valid context requires " - "buffers not supplied by a valid surface)."); - case EGL_BAD_PARAMETER: - return absl::InternalError("One or more argument values are invalid."); - case EGL_BAD_NATIVE_PIXMAP: - return absl::InternalError( - "A NativePixmapType argument does not refer to a valid native " - "pixmap."); - case EGL_BAD_NATIVE_WINDOW: - return absl::InternalError( - "A NativeWindowType argument does not refer to a valid native " - "window."); - case EGL_CONTEXT_LOST: - return absl::InternalError( - "A power management event has occurred. The application must destroy " - "all contexts and reinitialize OpenGL ES state and objects to " - "continue rendering."); - } - return absl::UnknownError(absl::StrCat("EGL error: ", error)); -} - -} // namespace mediapipe diff --git a/mediapipe/gpu/egl_errors.h b/mediapipe/gpu/egl_errors.h deleted file mode 100644 index 9a14f0c79b..0000000000 --- a/mediapipe/gpu/egl_errors.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef MEDIAPIPE_GPU_EGL_ERRORS_H_ -#define MEDIAPIPE_GPU_EGL_ERRORS_H_ - -#include "absl/status/status.h" - -namespace mediapipe { - -// Returns the error of the last called EGL function in the current thread. -absl::Status GetEglError(); - -} // namespace mediapipe - -#endif // MEDIAPIPE_GPU_EGL_ERRORS_H_ diff --git a/mediapipe/gpu/egl_sync.cc b/mediapipe/gpu/egl_sync.cc deleted file mode 100644 index b6a6746e46..0000000000 --- a/mediapipe/gpu/egl_sync.cc +++ /dev/null @@ -1,214 +0,0 @@ -#include "mediapipe/gpu/egl_sync.h" - -#include - -#include -#include - -#include "absl/cleanup/cleanup.h" -#include "absl/log/absl_log.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" -#include "mediapipe/framework/deps/no_destructor.h" -#include "mediapipe/framework/formats/unique_fd.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status_macros.h" -#include "mediapipe/gpu/egl_errors.h" -#include "mediapipe/gpu/gl_base.h" - -namespace mediapipe { - -namespace { - -PFNEGLCREATESYNCKHRPROC eglCreateSyncKHR; -PFNEGLWAITSYNCKHRPROC eglWaitSyncKHR; -PFNEGLCLIENTWAITSYNCKHRPROC eglClientWaitSyncKHR; -PFNEGLDESTROYSYNCKHRPROC eglDestroySyncKHR; -PFNEGLDUPNATIVEFENCEFDANDROIDPROC eglDupNativeFenceFDANDROID; -PFNEGLGETSYNCATTRIBKHRPROC eglGetSyncAttribKHR; - -bool HasExtension(EGLDisplay display, const char* extension) { - const char* extensions = eglQueryString(display, EGL_EXTENSIONS); - return extensions && std::strstr(extensions, extension); -} - -absl::Status CheckEglFenceSyncSupported(EGLDisplay display) { - static bool supported = HasExtension(display, "EGL_KHR_fence_sync"); - if (supported) { - return absl::OkStatus(); - } - return absl::UnavailableError("EGL_KHR_fence_sync unavailable."); -} - -absl::Status CheckEglWaitSyncSupported(EGLDisplay display) { - static bool supported = HasExtension(display, "EGL_KHR_wait_sync"); - if (supported) { - return absl::OkStatus(); - } - return absl::UnavailableError("EGL_KHR_wait_sync unavailable."); -} - -absl::Status CheckEglAndroidNativeSyncSupported(EGLDisplay display) { - static bool supported = - HasExtension(display, "EGL_ANDROID_native_fence_sync"); - if (supported) { - return absl::OkStatus(); - } - return absl::UnavailableError("EGL_ANDROID_native_fence_sync unavailable."); -} - -absl::Status CheckEglSyncSupported(EGLDisplay egl_display) { - static NoDestructor support_status([&]() -> absl::Status { - MP_RETURN_IF_ERROR(CheckEglFenceSyncSupported(egl_display)); - MP_RETURN_IF_ERROR(CheckEglWaitSyncSupported(egl_display)); - - RET_CHECK(eglCreateSyncKHR = reinterpret_cast( - eglGetProcAddress("eglCreateSyncKHR"))); - RET_CHECK(eglWaitSyncKHR = reinterpret_cast( - eglGetProcAddress("eglWaitSyncKHR"))); - RET_CHECK(eglClientWaitSyncKHR = - reinterpret_cast( - eglGetProcAddress("eglClientWaitSyncKHR"))); - RET_CHECK(eglDestroySyncKHR = reinterpret_cast( - eglGetProcAddress("eglDestroySyncKHR"))); - RET_CHECK(eglGetSyncAttribKHR = - reinterpret_cast( - eglGetProcAddress("eglGetSyncAttribKHR"))); - return absl::OkStatus(); - }()); - return *support_status; -} - -absl::Status CheckEglNativeSyncSupported(EGLDisplay egl_display) { - static NoDestructor support_status([&]() -> absl::Status { - MP_RETURN_IF_ERROR(CheckEglAndroidNativeSyncSupported(egl_display)); - RET_CHECK(eglDupNativeFenceFDANDROID = - reinterpret_cast( - eglGetProcAddress("eglDupNativeFenceFDANDROID"))); - return absl::OkStatus(); - }()); - return *support_status; -} - -} // namespace - -absl::StatusOr EglSync::Create(EGLDisplay display) { - MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); - - const EGLSyncKHR egl_sync = - eglCreateSyncKHR(display, EGL_SYNC_FENCE_KHR, nullptr); - RET_CHECK_NE(egl_sync, EGL_NO_SYNC_KHR) - << "Create/eglCreateSyncKHR failed: " << GetEglError(); - return EglSync(display, egl_sync); -} - -absl::StatusOr EglSync::CreateNative(EGLDisplay display) { - MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); - MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); - - const EGLSyncKHR egl_sync = - eglCreateSyncKHR(display, EGL_SYNC_NATIVE_FENCE_ANDROID, nullptr); - RET_CHECK_NE(egl_sync, EGL_NO_SYNC_KHR) - << "CreateNative/eglCreateSyncKHR failed: " << GetEglError(); - return EglSync(display, egl_sync); -} - -absl::StatusOr EglSync::CreateNative(EGLDisplay display, - const UniqueFd& native_fence_fd) { - RET_CHECK(native_fence_fd.IsValid()); - MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); - MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); - - MP_ASSIGN_OR_RETURN(UniqueFd fd_for_egl, native_fence_fd.Dup()); - // NOTE: it looks like one could rely on `UniqueFd` for the cleanup, but - // there's some clashing on ownership of the FD when passing it to - // eglCreateSyncKHR and then using `UniqueFd::Release`, hence relying on - // absl::Cleanup. - const int fd = fd_for_egl.Release(); - absl::Cleanup fd_cleanup = [fd]() { close(fd); }; - const EGLint sync_attribs[] = {EGL_SYNC_NATIVE_FENCE_FD_ANDROID, - static_cast(fd), EGL_NONE}; - const EGLSyncKHR egl_sync = - eglCreateSyncKHR(display, EGL_SYNC_NATIVE_FENCE_ANDROID, sync_attribs); - RET_CHECK_NE(egl_sync, EGL_NO_SYNC_KHR) << absl::StrCat( - "CreateNative/eglCreateSyncKHR with original FD: ", native_fence_fd.Get(), - " and dup FD: ", fd, " - failed: ", GetEglError()); - // EGL took ownership of the passed FD as eglCreateSyncKHR succeeded, so - // cancelling the cleanup. - std::move(fd_cleanup).Cancel(); - - return EglSync(display, egl_sync); -} - -EglSync::EglSync(EglSync&& sync) { *this = std::move(sync); } - -EglSync& EglSync::operator=(EglSync&& sync) { - if (this != &sync) { - Invalidate(); - - using std::swap; - sync_ = std::exchange(sync.sync_, EGL_NO_SYNC_KHR); - display_ = std::exchange(sync.display_, EGL_NO_DISPLAY); - } - return *this; -} - -void EglSync::Invalidate() { - if (sync_ == EGL_NO_SYNC_KHR || display_ == EGL_NO_DISPLAY) { - return; - } - - const absl::Status egl_sync_support = CheckEglSyncSupported(display_); - if (!egl_sync_support.ok()) { - ABSL_LOG(DFATAL) << "Attempt to destroy an EGL sync: " << egl_sync_support; - return; - } - - // Needs extension: EGL_KHR_fence_sync (EGL) / GL_OES_EGL_sync (OpenGL ES). - // Note: we're doing nothing when the function pointer is nullptr, or the - // call returns EGL_FALSE. - const EGLBoolean result = eglDestroySyncKHR(display_, sync_); - if (result == EGL_FALSE) { - ABSL_LOG(DFATAL) << "eglDestroySyncKHR failed: " << GetEglError(); - } - sync_ = EGL_NO_SYNC_KHR; -} - -absl::Status EglSync::WaitOnGpu() { - MP_RETURN_IF_ERROR(CheckEglSyncSupported(display_)); - - const EGLint result = eglWaitSyncKHR(display_, sync_, 0); - RET_CHECK_EQ(result, EGL_TRUE) << "eglWaitSyncKHR failed: " << GetEglError(); - return absl::OkStatus(); -} - -absl::Status EglSync::Wait() { - MP_RETURN_IF_ERROR(CheckEglSyncSupported(display_)); - - const EGLint result = eglClientWaitSyncKHR( - display_, sync_, EGL_SYNC_FLUSH_COMMANDS_BIT_KHR, EGL_FOREVER_KHR); - RET_CHECK_EQ(result, EGL_CONDITION_SATISFIED_KHR) - << "eglClientWaitSyncKHR failed: " << GetEglError(); - return absl::OkStatus(); -} - -absl::StatusOr EglSync::DupNativeFd() { - MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display_)); - - const int fd = eglDupNativeFenceFDANDROID(display_, sync_); - RET_CHECK_NE(fd, EGL_NO_NATIVE_FENCE_FD_ANDROID) - << "eglDupNativeFenceFDANDROID failed: " << GetEglError(); - return UniqueFd(fd); -} - -absl::StatusOr EglSync::IsSignaled() { - EGLint status; - const EGLBoolean success = - eglGetSyncAttribKHR(display_, sync_, EGL_SYNC_STATUS_KHR, &status); - RET_CHECK_EQ(success, EGL_TRUE) - << "eglGetSyncAttribKHR failed: " << GetEglError(); - return status == EGL_SIGNALED_KHR; -} - -} // namespace mediapipe diff --git a/mediapipe/gpu/egl_sync.h b/mediapipe/gpu/egl_sync.h deleted file mode 100644 index a1f00d66c8..0000000000 --- a/mediapipe/gpu/egl_sync.h +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef MEDIAPIPE_GPU_EGL_SYNC_H_ -#define MEDIAPIPE_GPU_EGL_SYNC_H_ - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "mediapipe/framework/formats/unique_fd.h" -#include "mediapipe/gpu/gl_base.h" - -namespace mediapipe { - -// RAII wrapper for EGL sync object. -class EglSync { - public: - // Creates a fence in OpenGL command stream. This sync is enqueued and *not* - // flushed. - static absl::StatusOr Create(EGLDisplay display); - - // Creates a native fence in OpenGL command stream. This sync is enqueued and - // *not* flushed. - static absl::StatusOr CreateNative(EGLDisplay display); - - // Create a native fence in OpenGL command stream based on a native fence FD. - static absl::StatusOr CreateNative(EGLDisplay display, - const UniqueFd& native_fence_fd); - - // Move-only - EglSync(EglSync&& sync); - EglSync& operator=(EglSync&& sync); - - EglSync(const EglSync&) = delete; - EglSync& operator=(const EglSync&) = delete; - - ~EglSync() { Invalidate(); } - - // Causes GPU to block and wait until this sync has been signaled. - // This call does not block and returns immediately. - absl::Status WaitOnGpu(); - - // Causes CPU to block and wait until this sync has been signaled. - absl::Status Wait(); - - // Returns the EGLDisplay on which this instance was created. - EGLDisplay display() const { return display_; } - - // Returns the EGLSyncKHR wrapped by this instance. - EGLSyncKHR sync() const { return sync_; } - - // Returns true if this EGL sync is signaled. - absl::StatusOr IsSignaled(); - - // Duplicates the file descriptor stored in native EGL fence sync. - absl::StatusOr DupNativeFd(); - - private: - EglSync(EGLDisplay display, EGLSyncKHR sync) - : display_(display), sync_(sync) {} - - void Invalidate(); - - EGLDisplay display_; - EGLSyncKHR sync_ = EGL_NO_SYNC_KHR; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_GPU_EGL_SYNC_H_ diff --git a/mediapipe/gpu/gl_base.h b/mediapipe/gpu/gl_base.h index eb309e6857..12a04e0bbf 100644 --- a/mediapipe/gpu/gl_base.h +++ b/mediapipe/gpu/gl_base.h @@ -55,7 +55,6 @@ #define HAS_EGL 1 #include -#include #include #include #if defined(__ANDROID__) From 5efa314ec862dd13a98320fd8285478324f26699 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 4 Nov 2024 11:24:19 -0800 Subject: [PATCH 029/126] Update Bazel version to 6.5.0 PiperOrigin-RevId: 693034121 --- .bazelversion | 2 +- Dockerfile | 2 +- docs/getting_started/install.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bazelversion b/.bazelversion index f3b5af39e4..f22d756da3 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.1.1 +6.5.0 diff --git a/Dockerfile b/Dockerfile index 3ff6a5a33a..46b9cd4dcd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,7 +67,7 @@ RUN pip3 install tf_slim RUN ln -s /usr/bin/python3 /usr/bin/python # Install bazel -ARG BAZEL_VERSION=6.1.1 +ARG BAZEL_VERSION=6.5.0 RUN mkdir /bazel && \ wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\ azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \ diff --git a/docs/getting_started/install.md b/docs/getting_started/install.md index b30284779a..a00b29a573 100644 --- a/docs/getting_started/install.md +++ b/docs/getting_started/install.md @@ -583,7 +583,7 @@ next section. Option 1. Follow [the official Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html) - to install Bazel 6.1.1 or higher. + to install Bazel 6.5.0 or higher. Option 2. Follow the official [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html) From 484ce05898708e625b71d580efaba9e9bd2d6d68 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 4 Nov 2024 14:16:24 -0800 Subject: [PATCH 030/126] Update sync_wait to support UniqueFd. PiperOrigin-RevId: 693089474 --- mediapipe/util/BUILD | 2 ++ mediapipe/util/sync_wait.cc | 6 ++++++ mediapipe/util/sync_wait.h | 6 ++++++ mediapipe/util/sync_wait_test.cc | 28 +++++++++------------------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/mediapipe/util/BUILD b/mediapipe/util/BUILD index f783b9f5c0..92af0ee0a2 100644 --- a/mediapipe/util/BUILD +++ b/mediapipe/util/BUILD @@ -201,6 +201,7 @@ cc_library( hdrs = ["sync_wait.h"], visibility = ["//mediapipe:__subpackages__"], deps = [ + "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@com_google_absl//absl/cleanup", @@ -217,6 +218,7 @@ cc_test( deps = [ ":sync_wait", "//mediapipe/framework:port", + "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:benchmark", "//mediapipe/framework/port:gtest_main", "@com_google_absl//absl/log:absl_check", diff --git a/mediapipe/util/sync_wait.cc b/mediapipe/util/sync_wait.cc index 14ed0a320e..4a1df09f21 100644 --- a/mediapipe/util/sync_wait.cc +++ b/mediapipe/util/sync_wait.cc @@ -9,6 +9,7 @@ #include "absl/status/status.h" #include "absl/strings/str_format.h" #include "absl/time/time.h" +#include "mediapipe/framework/formats/unique_fd.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status_macros.h" @@ -46,4 +47,9 @@ absl::Status SyncWait(int fd, absl::Duration timeout) { absl::StrFormat("Failed to wait for fd: %d.", fd)); } +absl::Status SyncWait(const UniqueFd& fd, absl::Duration timeout) { + RET_CHECK(fd.IsValid()); + return SyncWait(fd.Get(), timeout); +} + } // namespace mediapipe diff --git a/mediapipe/util/sync_wait.h b/mediapipe/util/sync_wait.h index ed99f6ede2..19ce9a435d 100644 --- a/mediapipe/util/sync_wait.h +++ b/mediapipe/util/sync_wait.h @@ -3,6 +3,7 @@ #include "absl/status/status.h" #include "absl/time/time.h" +#include "mediapipe/framework/formats/unique_fd.h" namespace mediapipe { @@ -11,6 +12,11 @@ namespace mediapipe { // signaled. absl::Status SyncWait(int fd, absl::Duration timeout); +// `fd` - represents a sync +// `timeout` - wait timeout, pass `absl::InfiniteDuration()` to wait until +// signaled. +absl::Status SyncWait(const UniqueFd& fd, absl::Duration timeout); + } // namespace mediapipe #endif // MEDIAPIPE_UTIL_SYNC_WAIT_H_ diff --git a/mediapipe/util/sync_wait_test.cc b/mediapipe/util/sync_wait_test.cc index 803e9d1701..00aefbc2f6 100644 --- a/mediapipe/util/sync_wait_test.cc +++ b/mediapipe/util/sync_wait_test.cc @@ -5,6 +5,7 @@ #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/time/time.h" +#include "mediapipe/framework/formats/unique_fd.h" #include "mediapipe/framework/port.h" // IWYU pragma: keep (DRIHSTI_OSX) #include "mediapipe/framework/port/benchmark.h" #include "mediapipe/framework/port/gmock.h" @@ -21,16 +22,7 @@ namespace mediapipe { namespace { struct TestTimer { - TestTimer() = default; - ~TestTimer() { - if (fd != -1) { - ABSL_CHECK_EQ(close(fd), 0); - } - } - TestTimer(TestTimer&& timer) = default; - TestTimer& operator=(TestTimer&& timer) = default; - - int fd = -1; + UniqueFd fd; }; #ifdef MEDIAPIPE_OSX @@ -45,24 +37,22 @@ TestTimer CreateTestTimer(absl::Duration duration) { NOTE_CRITICAL, timeout, NULL); kevent(kq, &kev, 1, NULL, 0, NULL); - TestTimer timer; - timer.fd = kq; - return timer; + return TestTimer{UniqueFd(kq)}; } #else TestTimer CreateTestTimer(absl::Duration duration) { - TestTimer timer; - timer.fd = timerfd_create(CLOCK_MONOTONIC, /*flags*/ 0); - ABSL_CHECK_NE(timer.fd, -1); + const int fd = timerfd_create(CLOCK_MONOTONIC, /*flags*/ 0); + ABSL_CHECK_NE(fd, -1); + TestTimer timer = {UniqueFd(fd)}; struct itimerspec new_value; new_value.it_value = absl::ToTimespec(duration); new_value.it_interval.tv_sec = 0; new_value.it_interval.tv_nsec = 0; - ABSL_CHECK_NE( - timerfd_settime(timer.fd, /*flags=*/0, &new_value, /*oldtimer=*/nullptr), - -1); + ABSL_CHECK_NE(timerfd_settime(timer.fd.Get(), /*flags=*/0, &new_value, + /*oldtimer=*/nullptr), + -1); return timer; } From 983fcd0323847c7ce514a522bd863518ab98d4de Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 5 Nov 2024 09:52:43 -0800 Subject: [PATCH 031/126] No public description PiperOrigin-RevId: 693384483 --- mediapipe/gpu/BUILD | 33 ++++++ mediapipe/gpu/egl_errors.cc | 72 ++++++++++++ mediapipe/gpu/egl_errors.h | 13 +++ mediapipe/gpu/egl_sync.cc | 215 ++++++++++++++++++++++++++++++++++++ mediapipe/gpu/egl_sync.h | 69 ++++++++++++ mediapipe/gpu/gl_base.h | 1 + 6 files changed, 403 insertions(+) create mode 100644 mediapipe/gpu/egl_errors.cc create mode 100644 mediapipe/gpu/egl_errors.h create mode 100644 mediapipe/gpu/egl_sync.cc create mode 100644 mediapipe/gpu/egl_sync.h diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index ba6e7f7cef..567d893c76 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -841,6 +841,39 @@ cc_library( }), ) +cc_library( + name = "egl_errors", + srcs = ["egl_errors.cc"], + hdrs = ["egl_errors.h"], + visibility = ["//visibility:public"], + deps = [ + ":gl_base", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "egl_sync", + srcs = ["egl_sync.cc"], + hdrs = ["egl_sync.h"], + visibility = ["//visibility:public"], + deps = [ + ":egl_errors", + ":gl_base", + "//mediapipe/framework/deps:no_destructor", + "//mediapipe/framework/formats:unique_fd", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "//third_party/GL:EGL_headers", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + ], +) + cc_library( name = "gl_texture_util", srcs = ["gl_texture_util.cc"], diff --git a/mediapipe/gpu/egl_errors.cc b/mediapipe/gpu/egl_errors.cc new file mode 100644 index 0000000000..e3f9be3725 --- /dev/null +++ b/mediapipe/gpu/egl_errors.cc @@ -0,0 +1,72 @@ +#include "mediapipe/gpu/egl_errors.h" + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "mediapipe/gpu/gl_base.h" + +namespace mediapipe { + +absl::Status GetEglError() { + EGLint error = eglGetError(); + switch (error) { + case EGL_SUCCESS: + return absl::OkStatus(); + case EGL_NOT_INITIALIZED: + return absl::InternalError( + "EGL is not initialized, or could not be initialized, for the " + "specified EGL display connection."); + case EGL_BAD_ACCESS: + return absl::InternalError( + "EGL cannot access a requested resource (for example a context is " + "bound in another thread)."); + case EGL_BAD_ALLOC: + return absl::InternalError( + "EGL failed to allocate resources for the requested operation."); + case EGL_BAD_ATTRIBUTE: + return absl::InternalError( + "An unrecognized attribute or attribute value was passed in the " + "attribute list."); + case EGL_BAD_CONTEXT: + return absl::InternalError( + "An EGLContext argument does not name a valid EGL rendering " + "context."); + case EGL_BAD_CONFIG: + return absl::InternalError( + "An EGLConfig argument does not name a valid EGL frame buffer " + "configuration."); + case EGL_BAD_CURRENT_SURFACE: + return absl::InternalError( + "The current surface of the calling thread is a window, pixel buffer " + "or pixmap that is no longer valid."); + case EGL_BAD_DISPLAY: + return absl::InternalError( + "An EGLDisplay argument does not name a valid EGL display " + "connection."); + case EGL_BAD_SURFACE: + return absl::InternalError( + "An EGLSurface argument does not name a valid surface (window, pixel " + "buffer or pixmap) configured for GL rendering."); + case EGL_BAD_MATCH: + return absl::InternalError( + "Arguments are inconsistent (for example, a valid context requires " + "buffers not supplied by a valid surface)."); + case EGL_BAD_PARAMETER: + return absl::InternalError("One or more argument values are invalid."); + case EGL_BAD_NATIVE_PIXMAP: + return absl::InternalError( + "A NativePixmapType argument does not refer to a valid native " + "pixmap."); + case EGL_BAD_NATIVE_WINDOW: + return absl::InternalError( + "A NativeWindowType argument does not refer to a valid native " + "window."); + case EGL_CONTEXT_LOST: + return absl::InternalError( + "A power management event has occurred. The application must destroy " + "all contexts and reinitialize OpenGL ES state and objects to " + "continue rendering."); + } + return absl::UnknownError(absl::StrCat("EGL error: ", error)); +} + +} // namespace mediapipe diff --git a/mediapipe/gpu/egl_errors.h b/mediapipe/gpu/egl_errors.h new file mode 100644 index 0000000000..9a14f0c79b --- /dev/null +++ b/mediapipe/gpu/egl_errors.h @@ -0,0 +1,13 @@ +#ifndef MEDIAPIPE_GPU_EGL_ERRORS_H_ +#define MEDIAPIPE_GPU_EGL_ERRORS_H_ + +#include "absl/status/status.h" + +namespace mediapipe { + +// Returns the error of the last called EGL function in the current thread. +absl::Status GetEglError(); + +} // namespace mediapipe + +#endif // MEDIAPIPE_GPU_EGL_ERRORS_H_ diff --git a/mediapipe/gpu/egl_sync.cc b/mediapipe/gpu/egl_sync.cc new file mode 100644 index 0000000000..dbfdb7a353 --- /dev/null +++ b/mediapipe/gpu/egl_sync.cc @@ -0,0 +1,215 @@ +#include "mediapipe/gpu/egl_sync.h" + +#include +#include +#include + +#include +#include + +#include "absl/cleanup/cleanup.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "mediapipe/framework/deps/no_destructor.h" +#include "mediapipe/framework/formats/unique_fd.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/gpu/egl_errors.h" + +namespace mediapipe { + +namespace { + +PFNEGLCREATESYNCKHRPROC eglCreateSyncKHR; +PFNEGLWAITSYNCKHRPROC eglWaitSyncKHR; +PFNEGLCLIENTWAITSYNCKHRPROC eglClientWaitSyncKHR; +PFNEGLDESTROYSYNCKHRPROC eglDestroySyncKHR; +PFNEGLDUPNATIVEFENCEFDANDROIDPROC eglDupNativeFenceFDANDROID; +PFNEGLGETSYNCATTRIBKHRPROC eglGetSyncAttribKHR; + +bool HasExtension(EGLDisplay display, const char* extension) { + const char* extensions = eglQueryString(display, EGL_EXTENSIONS); + return extensions && std::strstr(extensions, extension); +} + +absl::Status CheckEglFenceSyncSupported(EGLDisplay display) { + static bool supported = HasExtension(display, "EGL_KHR_fence_sync"); + if (supported) { + return absl::OkStatus(); + } + return absl::UnavailableError("EGL_KHR_fence_sync unavailable."); +} + +absl::Status CheckEglWaitSyncSupported(EGLDisplay display) { + static bool supported = HasExtension(display, "EGL_KHR_wait_sync"); + if (supported) { + return absl::OkStatus(); + } + return absl::UnavailableError("EGL_KHR_wait_sync unavailable."); +} + +absl::Status CheckEglAndroidNativeSyncSupported(EGLDisplay display) { + static bool supported = + HasExtension(display, "EGL_ANDROID_native_fence_sync"); + if (supported) { + return absl::OkStatus(); + } + return absl::UnavailableError("EGL_ANDROID_native_fence_sync unavailable."); +} + +absl::Status CheckEglSyncSupported(EGLDisplay egl_display) { + static NoDestructor support_status([&]() -> absl::Status { + MP_RETURN_IF_ERROR(CheckEglFenceSyncSupported(egl_display)); + MP_RETURN_IF_ERROR(CheckEglWaitSyncSupported(egl_display)); + + RET_CHECK(eglCreateSyncKHR = reinterpret_cast( + eglGetProcAddress("eglCreateSyncKHR"))); + RET_CHECK(eglWaitSyncKHR = reinterpret_cast( + eglGetProcAddress("eglWaitSyncKHR"))); + RET_CHECK(eglClientWaitSyncKHR = + reinterpret_cast( + eglGetProcAddress("eglClientWaitSyncKHR"))); + RET_CHECK(eglDestroySyncKHR = reinterpret_cast( + eglGetProcAddress("eglDestroySyncKHR"))); + RET_CHECK(eglGetSyncAttribKHR = + reinterpret_cast( + eglGetProcAddress("eglGetSyncAttribKHR"))); + return absl::OkStatus(); + }()); + return *support_status; +} + +absl::Status CheckEglNativeSyncSupported(EGLDisplay egl_display) { + static NoDestructor support_status([&]() -> absl::Status { + MP_RETURN_IF_ERROR(CheckEglAndroidNativeSyncSupported(egl_display)); + RET_CHECK(eglDupNativeFenceFDANDROID = + reinterpret_cast( + eglGetProcAddress("eglDupNativeFenceFDANDROID"))); + return absl::OkStatus(); + }()); + return *support_status; +} + +} // namespace + +absl::StatusOr EglSync::Create(EGLDisplay display) { + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); + + const EGLSyncKHR egl_sync = + eglCreateSyncKHR(display, EGL_SYNC_FENCE_KHR, nullptr); + RET_CHECK_NE(egl_sync, EGL_NO_SYNC_KHR) + << "Create/eglCreateSyncKHR failed: " << GetEglError(); + return EglSync(display, egl_sync); +} + +absl::StatusOr EglSync::CreateNative(EGLDisplay display) { + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); + MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); + + const EGLSyncKHR egl_sync = + eglCreateSyncKHR(display, EGL_SYNC_NATIVE_FENCE_ANDROID, nullptr); + RET_CHECK_NE(egl_sync, EGL_NO_SYNC_KHR) + << "CreateNative/eglCreateSyncKHR failed: " << GetEglError(); + return EglSync(display, egl_sync); +} + +absl::StatusOr EglSync::CreateNative(EGLDisplay display, + const UniqueFd& native_fence_fd) { + RET_CHECK(native_fence_fd.IsValid()); + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); + MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); + + MP_ASSIGN_OR_RETURN(UniqueFd fd_for_egl, native_fence_fd.Dup()); + // NOTE: it looks like one could rely on `UniqueFd` for the cleanup, but + // there's some clashing on ownership of the FD when passing it to + // eglCreateSyncKHR and then using `UniqueFd::Release`, hence relying on + // absl::Cleanup. + const int fd = fd_for_egl.Release(); + absl::Cleanup fd_cleanup = [fd]() { close(fd); }; + const EGLint sync_attribs[] = {EGL_SYNC_NATIVE_FENCE_FD_ANDROID, + static_cast(fd), EGL_NONE}; + const EGLSyncKHR egl_sync = + eglCreateSyncKHR(display, EGL_SYNC_NATIVE_FENCE_ANDROID, sync_attribs); + RET_CHECK_NE(egl_sync, EGL_NO_SYNC_KHR) << absl::StrCat( + "CreateNative/eglCreateSyncKHR with original FD: ", native_fence_fd.Get(), + " and dup FD: ", fd, " - failed: ", GetEglError()); + // EGL took ownership of the passed FD as eglCreateSyncKHR succeeded, so + // cancelling the cleanup. + std::move(fd_cleanup).Cancel(); + + return EglSync(display, egl_sync); +} + +EglSync::EglSync(EglSync&& sync) { *this = std::move(sync); } + +EglSync& EglSync::operator=(EglSync&& sync) { + if (this != &sync) { + Invalidate(); + + using std::swap; + sync_ = std::exchange(sync.sync_, EGL_NO_SYNC_KHR); + display_ = std::exchange(sync.display_, EGL_NO_DISPLAY); + } + return *this; +} + +void EglSync::Invalidate() { + if (sync_ == EGL_NO_SYNC_KHR || display_ == EGL_NO_DISPLAY) { + return; + } + + const absl::Status egl_sync_support = CheckEglSyncSupported(display_); + if (!egl_sync_support.ok()) { + ABSL_LOG(DFATAL) << "Attempt to destroy an EGL sync: " << egl_sync_support; + return; + } + + // Needs extension: EGL_KHR_fence_sync (EGL) / GL_OES_EGL_sync (OpenGL ES). + // Note: we're doing nothing when the function pointer is nullptr, or the + // call returns EGL_FALSE. + const EGLBoolean result = eglDestroySyncKHR(display_, sync_); + if (result == EGL_FALSE) { + ABSL_LOG(DFATAL) << "eglDestroySyncKHR failed: " << GetEglError(); + } + sync_ = EGL_NO_SYNC_KHR; +} + +absl::Status EglSync::WaitOnGpu() { + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display_)); + + const EGLint result = eglWaitSyncKHR(display_, sync_, 0); + RET_CHECK_EQ(result, EGL_TRUE) << "eglWaitSyncKHR failed: " << GetEglError(); + return absl::OkStatus(); +} + +absl::Status EglSync::Wait() { + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display_)); + + const EGLint result = eglClientWaitSyncKHR( + display_, sync_, EGL_SYNC_FLUSH_COMMANDS_BIT_KHR, EGL_FOREVER_KHR); + RET_CHECK_EQ(result, EGL_CONDITION_SATISFIED_KHR) + << "eglClientWaitSyncKHR failed: " << GetEglError(); + return absl::OkStatus(); +} + +absl::StatusOr EglSync::DupNativeFd() { + MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display_)); + + const int fd = eglDupNativeFenceFDANDROID(display_, sync_); + RET_CHECK_NE(fd, EGL_NO_NATIVE_FENCE_FD_ANDROID) + << "eglDupNativeFenceFDANDROID failed: " << GetEglError(); + return UniqueFd(fd); +} + +absl::StatusOr EglSync::IsSignaled() { + EGLint status; + const EGLBoolean success = + eglGetSyncAttribKHR(display_, sync_, EGL_SYNC_STATUS_KHR, &status); + RET_CHECK_EQ(success, EGL_TRUE) + << "eglGetSyncAttribKHR failed: " << GetEglError(); + return status == EGL_SIGNALED_KHR; +} + +} // namespace mediapipe diff --git a/mediapipe/gpu/egl_sync.h b/mediapipe/gpu/egl_sync.h new file mode 100644 index 0000000000..b42ad67140 --- /dev/null +++ b/mediapipe/gpu/egl_sync.h @@ -0,0 +1,69 @@ +#ifndef MEDIAPIPE_GPU_EGL_SYNC_H_ +#define MEDIAPIPE_GPU_EGL_SYNC_H_ + +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/unique_fd.h" +#include "mediapipe/gpu/gl_base.h" + +namespace mediapipe { + +// RAII wrapper for EGL sync object. +class EglSync { + public: + // Creates a fence in OpenGL command stream. This sync is enqueued and *not* + // flushed. + static absl::StatusOr Create(EGLDisplay display); + + // Creates a native fence in OpenGL command stream. This sync is enqueued and + // *not* flushed. + static absl::StatusOr CreateNative(EGLDisplay display); + + // Create a native fence in OpenGL command stream based on a native fence FD. + static absl::StatusOr CreateNative(EGLDisplay display, + const UniqueFd& native_fence_fd); + + // Move-only + EglSync(EglSync&& sync); + EglSync& operator=(EglSync&& sync); + + EglSync(const EglSync&) = delete; + EglSync& operator=(const EglSync&) = delete; + + ~EglSync() { Invalidate(); } + + // Causes GPU to block and wait until this sync has been signaled. + // This call does not block and returns immediately. + absl::Status WaitOnGpu(); + + // Causes CPU to block and wait until this sync has been signaled. + absl::Status Wait(); + + // Returns the EGLDisplay on which this instance was created. + EGLDisplay display() const { return display_; } + + // Returns the EGLSyncKHR wrapped by this instance. + EGLSyncKHR sync() const { return sync_; } + + // Returns true if this EGL sync is signaled. + absl::StatusOr IsSignaled(); + + // Duplicates the file descriptor stored in native EGL fence sync. + absl::StatusOr DupNativeFd(); + + private: + EglSync(EGLDisplay display, EGLSyncKHR sync) + : display_(display), sync_(sync) {} + + void Invalidate(); + + EGLDisplay display_; + EGLSyncKHR sync_ = EGL_NO_SYNC_KHR; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_GPU_EGL_SYNC_H_ diff --git a/mediapipe/gpu/gl_base.h b/mediapipe/gpu/gl_base.h index 12a04e0bbf..a16bcffa3b 100644 --- a/mediapipe/gpu/gl_base.h +++ b/mediapipe/gpu/gl_base.h @@ -55,6 +55,7 @@ #define HAS_EGL 1 #include +// TODO: b/377324183 - add #include #include #if defined(__ANDROID__) From 0f27e9dec11f5b21031246c40e954de6d15b9865 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 5 Nov 2024 11:23:40 -0800 Subject: [PATCH 032/126] Internal change. PiperOrigin-RevId: 693419022 --- mediapipe/java/com/google/mediapipe/framework/AssetCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/java/com/google/mediapipe/framework/AssetCache.java b/mediapipe/java/com/google/mediapipe/framework/AssetCache.java index 21fea061d7..21bbcd9aba 100644 --- a/mediapipe/java/com/google/mediapipe/framework/AssetCache.java +++ b/mediapipe/java/com/google/mediapipe/framework/AssetCache.java @@ -133,7 +133,7 @@ public synchronized String getAbsolutePathFromAsset(String assetPath) { inStream = assetManager.open(assetPath); writeStreamToFile(inStream, destinationFile); } catch (IOException ioe) { - logger.atSevere().log("Unable to unpack: %s", assetPath); + logger.atSevere().withCause(ioe).log("Unable to unpack: %s", assetPath); try { if (inStream != null) { inStream.close(); From 05292354d3034919b1196b61ccdd41966950b5cd Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 5 Nov 2024 15:57:32 -0800 Subject: [PATCH 033/126] No public description PiperOrigin-RevId: 693505976 --- mediapipe/framework/formats/BUILD | 1 + mediapipe/framework/formats/unique_fd.cc | 118 +++++++++++++++++++++++ mediapipe/framework/formats/unique_fd.h | 97 +------------------ 3 files changed, 123 insertions(+), 93 deletions(-) create mode 100644 mediapipe/framework/formats/unique_fd.cc diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index 3e74db39cc..9eaba8c0e2 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -300,6 +300,7 @@ cc_test( cc_library( name = "unique_fd", + srcs = ["unique_fd.cc"], hdrs = ["unique_fd.h"], deps = [ "//mediapipe/framework/port:ret_check", diff --git a/mediapipe/framework/formats/unique_fd.cc b/mediapipe/framework/formats/unique_fd.cc new file mode 100644 index 0000000000..38f2aed338 --- /dev/null +++ b/mediapipe/framework/formats/unique_fd.cc @@ -0,0 +1,118 @@ +#include "mediapipe/framework/formats/unique_fd.h" + +#include + +#include "absl/base/attributes.h" +#include "absl/log/absl_log.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/port/ret_check.h" + +#if (__ANDROID_API__ >= 29) && defined(__BIONIC__) && !defined(NDEBUG) +#define MEDIAPIPE_UNIQUE_FD_USE_FDSAN 1 + +#include + +#include + +#endif // (__ANDROID_API__ >= 29) && defined(__BIONIC__) && !defined(NDEBUG) + +namespace mediapipe { + +namespace { + +#if defined(MEDIAPIPE_UNIQUE_FD_USE_FDSAN) +// Address of the object is used as tag. +uint64_t Tag(UniqueFd* fd) { return reinterpret_cast(fd); } + +// These functions are marked with __attribute__((weak)), so that their +// availability can be determined at runtime. These wrappers will use them +// if available, and fall back to no-ops or regular close on devices older +// than API level 29 or non-bionic or non-production builds. +void FdsanExchangeTag(int fd, uint64_t old_tag, uint64_t new_tag) { + if (android_fdsan_exchange_owner_tag) { + android_fdsan_exchange_owner_tag(fd, old_tag, new_tag); + } +} + +void FdsanClose(int fd, uint64_t tag) { + if (android_fdsan_close_with_tag) { + if (android_fdsan_close_with_tag(fd, tag) != 0) { + ABSL_LOG(ERROR) << "Failed to close fd: " << fd; + } + return; + } + if (::close(fd) != 0) { + ABSL_LOG(ERROR) << "Failed to close fd: " << fd; + } +} +#endif // MEDIAPIPE_UNIQUE_FD_USE_FDSAN + +} // namespace + +UniqueFd& UniqueFd::operator=(UniqueFd&& move) { + if (this == &move) { + return *this; + } + + Reset(); + + if (move.fd_ != -1) { + fd_ = move.fd_; + move.fd_ = -1; +#if defined(MEDIAPIPE_UNIQUE_FD_USE_FDSAN) + // Acquire ownership from the moved-from object. + FdsanExchangeTag(fd_, Tag(&move), Tag(this)); +#endif // MEDIAPIPE_UNIQUE_FD_USE_FDSAN + } + + return *this; +} + +absl::StatusOr UniqueFd::Dup() const { + RET_CHECK(IsValid()); + int dup_fd = dup(Get()); + return UniqueFd(dup_fd); +} + +// Releases ownership of the file descriptor and returns it. +ABSL_MUST_USE_RESULT int UniqueFd::Release() { + if (!IsValid()) { + return -1; + } + + int fd = fd_; + fd_ = -1; +#if defined(MEDIAPIPE_UNIQUE_FD_USE_FDSAN) + // Release ownership. + FdsanExchangeTag(fd, Tag(this), 0); +#endif // MEDIAPIPE_UNIQUE_FD_USE_FDSAN + return fd; +} + +// Closes a wrapped file descriptor and resets the wrapper. +void UniqueFd::Reset(int new_fd) { + if (IsValid()) { +#if defined(MEDIAPIPE_UNIQUE_FD_USE_FDSAN) + FdsanClose(fd_, Tag(this)); +#else + if (::close(fd_) != 0) { + ABSL_LOG(ERROR) << "Failed to close fd: " << fd_; + } +#endif // MEDIAPIPE_UNIQUE_FD_USE_FDSAN + fd_ = -1; + } + + if (new_fd != -1) { + fd_ = new_fd; +#if defined(MEDIAPIPE_UNIQUE_FD_USE_FDSAN) + // Acquire ownership of the presumably unowned fd. + FdsanExchangeTag(fd_, 0, Tag(this)); +#endif // MEDIAPIPE_UNIQUE_FD_USE_FDSAN + } +} + +} // namespace mediapipe + +#ifdef MEDIAPIPE_UNIQUE_FD_USE_FDSAN +#undef MEDIAPIPE_UNIQUE_FD_USE_FDSAN +#endif diff --git a/mediapipe/framework/formats/unique_fd.h b/mediapipe/framework/formats/unique_fd.h index 015628d9c4..3f1c63d7c1 100644 --- a/mediapipe/framework/formats/unique_fd.h +++ b/mediapipe/framework/formats/unique_fd.h @@ -1,19 +1,10 @@ #ifndef MEDIAPIPE_FRAMEWORK_FORMATS_ANDROID_UNIQUE_FD_H_ #define MEDIAPIPE_FRAMEWORK_FORMATS_ANDROID_UNIQUE_FD_H_ -#include - #include #include "absl/base/attributes.h" -#include "absl/log/absl_log.h" #include "absl/status/statusor.h" -#include "mediapipe/framework/port/ret_check.h" - -#if (__ANDROID_API__ >= 29) && defined(__BIONIC__) && !defined(NDEBUG) -#define MEDIAPIPE_USE_FDSAN 1 -#include -#endif // (__ANDROID_API__ >= 29) && defined(__BIONIC__) && !defined(NDEBUG) namespace mediapipe { @@ -38,103 +29,23 @@ class UniqueFd { ~UniqueFd() { Reset(); } UniqueFd& operator=(const UniqueFd& copy) = delete; - UniqueFd& operator=(UniqueFd&& move) { - if (this == &move) { - return *this; - } - - Reset(); - - if (move.fd_ != -1) { - fd_ = move.fd_; - move.fd_ = -1; -#if defined(MEDIAPIPE_USE_FDSAN) - // Acquire ownership from the moved-from object. - FdsanExchangeTag(fd_, move.Tag(), Tag()); -#endif // MEDIAPIPE_USE_FDSAN - } - - return *this; - } - + UniqueFd& operator=(UniqueFd&& move); // Returns a non-owned file descriptor. int Get() const { return fd_; } // Checks if a valid file descriptor is wrapped. bool IsValid() const { return fd_ >= 0; } - absl::StatusOr Dup() const { - RET_CHECK(IsValid()); - int dup_fd = dup(Get()); - return UniqueFd(dup_fd); - } + absl::StatusOr Dup() const; // Releases ownership of the file descriptor and returns it. - ABSL_MUST_USE_RESULT int Release() { - if (!IsValid()) { - return -1; - } - - int fd = fd_; - fd_ = -1; -#if defined(MEDIAPIPE_USE_FDSAN) - // Release ownership. - FdsanExchangeTag(fd, Tag(), 0); -#endif // MEDIAPIPE_USE_FDSAN - return fd; - } + ABSL_MUST_USE_RESULT int Release(); // Closes a wrapped file descriptor and resets the wrapper. - void Reset(int new_fd = -1) { - if (IsValid()) { -#if defined(MEDIAPIPE_USE_FDSAN) - FdsanClose(fd_, Tag()); -#else - if (::close(fd_) != 0) { - ABSL_LOG(ERROR) << "Failed to close fd: " << fd_; - } -#endif // MEDIAPIPE_USE_FDSAN - fd_ = -1; - } - - if (new_fd != -1) { - fd_ = new_fd; -#if defined(MEDIAPIPE_USE_FDSAN) - // Acquire ownership of the presumably unowned fd. - FdsanExchangeTag(fd_, 0, Tag()); -#endif // MEDIAPIPE_USE_FDSAN - } - } + void Reset(int new_fd = -1); private: int fd_ = -1; - -#if defined(MEDIAPIPE_USE_FDSAN) - // Address of the object is used as tag. - uint64_t Tag() { return reinterpret_cast(this); } - - // These functions are marked with __attribute__((weak)), so that their - // availability can be determined at runtime. These wrappers will use them - // if available, and fall back to no-ops or regular close on devices older - // than API level 29 or non-bionic or non-production builds. - static void FdsanExchangeTag(int fd, uint64_t old_tag, uint64_t new_tag) { - if (android_fdsan_exchange_owner_tag) { - android_fdsan_exchange_owner_tag(fd, old_tag, new_tag); - } - } - - static void FdsanClose(int fd, uint64_t tag) { - if (android_fdsan_close_with_tag) { - if (android_fdsan_close_with_tag(fd, tag) != 0) { - ABSL_LOG(ERROR) << "Failed to close fd: " << fd; - } - return; - } - if (::close(fd) != 0) { - ABSL_LOG(ERROR) << "Failed to close fd: " << fd; - } - } -#endif // MEDIAPIPE_USE_FDSAN }; } // namespace mediapipe From 223cdb153d9ee3248a2d8d9c0b61fd64a2e27b88 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 6 Nov 2024 10:24:11 -0800 Subject: [PATCH 034/126] Internal change. PiperOrigin-RevId: 693775771 --- mediapipe/tasks/ios/genai/inference/BUILD | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/genai/inference/BUILD b/mediapipe/tasks/ios/genai/inference/BUILD index fe591690b3..fad3e3d439 100644 --- a/mediapipe/tasks/ios/genai/inference/BUILD +++ b/mediapipe/tasks/ios/genai/inference/BUILD @@ -16,7 +16,17 @@ load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") licenses(["notice"]) -package(default_visibility = ["//mediapipe/tasks:internal"]) +package(default_visibility = [ + ":clients", + "//mediapipe/tasks:internal", +]) + +package_group( + name = "clients", + packages = [ + "//googlemac/iPhone/Home/OnDeviceML/...", + ], +) swift_library( name = "LlmInference", From 4e1ee5d7e8952ec20aa52edc2fa2c8c0fe1d98b1 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 7 Nov 2024 10:51:47 -0800 Subject: [PATCH 035/126] Fix GlContext includes PiperOrigin-RevId: 694174981 --- mediapipe/gpu/BUILD | 2 ++ mediapipe/gpu/gl_context.cc | 18 +++++++++++++++--- mediapipe/gpu/gl_context.h | 7 +++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 567d893c76..2d37e5ab85 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -226,7 +226,9 @@ cc_library( "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/synchronization", ] + select({ "//conditions:default": [], diff --git a/mediapipe/gpu/gl_context.cc b/mediapipe/gpu/gl_context.cc index 69d0e7bbc4..0be605db55 100644 --- a/mediapipe/gpu/gl_context.cc +++ b/mediapipe/gpu/gl_context.cc @@ -16,22 +16,34 @@ #include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include #include +#include "absl/base/attributes.h" #include "absl/base/dynamic_annotations.h" +#include "absl/base/thread_annotations.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/status/status.h" +#include "absl/strings/numbers.h" #include "absl/strings/str_format.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/port.h" // IWYU pragma: keep #include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_builder.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/gpu/gl_base.h" #include "mediapipe/gpu/gl_context_internal.h" #include "mediapipe/gpu/gpu_buffer_format.h" diff --git a/mediapipe/gpu/gl_context.h b/mediapipe/gpu/gl_context.h index 19dbce2429..e491bb9b06 100644 --- a/mediapipe/gpu/gl_context.h +++ b/mediapipe/gpu/gl_context.h @@ -18,11 +18,18 @@ #include #include +#include #include #include +#include +#include "absl/base/attributes.h" +#include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/executor.h" #include "mediapipe/framework/mediapipe_profiling.h" From e552e279c0608a445e5dfc1f18dd17ea169c01ba Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 7 Nov 2024 15:46:43 -0800 Subject: [PATCH 036/126] Add EglSyncPoint/CreateEglSyncPoint. PiperOrigin-RevId: 694273478 --- mediapipe/gpu/BUILD | 28 +++++++++-- mediapipe/gpu/egl_base.h | 15 ++++++ mediapipe/gpu/egl_errors.cc | 2 +- mediapipe/gpu/egl_sync.cc | 11 ++++- mediapipe/gpu/egl_sync.h | 8 +-- mediapipe/gpu/egl_sync_point.cc | 88 +++++++++++++++++++++++++++++++++ mediapipe/gpu/egl_sync_point.h | 17 +++++++ 7 files changed, 159 insertions(+), 10 deletions(-) create mode 100644 mediapipe/gpu/egl_base.h create mode 100644 mediapipe/gpu/egl_sync_point.cc create mode 100644 mediapipe/gpu/egl_sync_point.h diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 2d37e5ab85..fb8f4b2b72 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -125,6 +125,13 @@ cc_library( }), ) +cc_library( + name = "egl_base", + textual_hdrs = ["egl_base.h"], + visibility = ["//visibility:public"], + deps = [":gl_base"], +) + cc_library( name = "gl_base_hdr", hdrs = ["gl_base.h"], @@ -219,6 +226,7 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "//mediapipe/framework/port:threadpool", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:dynamic_annotations", "@com_google_absl//absl/debugging:leak_check", "@com_google_absl//absl/log:absl_check", @@ -849,7 +857,7 @@ cc_library( hdrs = ["egl_errors.h"], visibility = ["//visibility:public"], deps = [ - ":gl_base", + ":egl_base", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", ], @@ -861,13 +869,12 @@ cc_library( hdrs = ["egl_sync.h"], visibility = ["//visibility:public"], deps = [ + ":egl_base", ":egl_errors", - ":gl_base", "//mediapipe/framework/deps:no_destructor", "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", - "//third_party/GL:EGL_headers", "@com_google_absl//absl/cleanup", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", @@ -876,6 +883,21 @@ cc_library( ], ) +cc_library( + name = "egl_sync_point", + srcs = ["egl_sync_point.cc"], + hdrs = ["egl_sync_point.h"], + visibility = ["//visibility:public"], + deps = [ + ":egl_base", + ":egl_sync", + ":gl_context", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], +) + cc_library( name = "gl_texture_util", srcs = ["gl_texture_util.cc"], diff --git a/mediapipe/gpu/egl_base.h b/mediapipe/gpu/egl_base.h new file mode 100644 index 0000000000..5192c9e457 --- /dev/null +++ b/mediapipe/gpu/egl_base.h @@ -0,0 +1,15 @@ +#ifndef MEDIAPIPE_GPU_EGL_BASE_H_ +#define MEDIAPIPE_GPU_EGL_BASE_H_ + +#include "mediapipe/gpu/gl_base.h" + +#if defined(HAS_EGL) + +// TODO: b/377324183 - merge into gl_base.h + +#include +#include + +#endif // defined(HAS_EGL) + +#endif // MEDIAPIPE_GPU_EGL_BASE_H_ diff --git a/mediapipe/gpu/egl_errors.cc b/mediapipe/gpu/egl_errors.cc index e3f9be3725..8b3de1205f 100644 --- a/mediapipe/gpu/egl_errors.cc +++ b/mediapipe/gpu/egl_errors.cc @@ -2,7 +2,7 @@ #include "absl/status/status.h" #include "absl/strings/str_cat.h" -#include "mediapipe/gpu/gl_base.h" +#include "mediapipe/gpu/egl_base.h" namespace mediapipe { diff --git a/mediapipe/gpu/egl_sync.cc b/mediapipe/gpu/egl_sync.cc index dbfdb7a353..56cf886fe8 100644 --- a/mediapipe/gpu/egl_sync.cc +++ b/mediapipe/gpu/egl_sync.cc @@ -1,7 +1,5 @@ #include "mediapipe/gpu/egl_sync.h" -#include -#include #include #include @@ -16,6 +14,7 @@ #include "mediapipe/framework/formats/unique_fd.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/gpu/egl_base.h" #include "mediapipe/gpu/egl_errors.h" namespace mediapipe { @@ -142,6 +141,14 @@ absl::StatusOr EglSync::CreateNative(EGLDisplay display, return EglSync(display, egl_sync); } +bool EglSync::IsSupported(EGLDisplay display) { + return CheckEglSyncSupported(display).ok(); +} + +bool EglSync::IsNativeSupported(EGLDisplay display) { + return CheckEglNativeSyncSupported(display).ok(); +} + EglSync::EglSync(EglSync&& sync) { *this = std::move(sync); } EglSync& EglSync::operator=(EglSync&& sync) { diff --git a/mediapipe/gpu/egl_sync.h b/mediapipe/gpu/egl_sync.h index b42ad67140..a4566b033b 100644 --- a/mediapipe/gpu/egl_sync.h +++ b/mediapipe/gpu/egl_sync.h @@ -1,13 +1,10 @@ #ifndef MEDIAPIPE_GPU_EGL_SYNC_H_ #define MEDIAPIPE_GPU_EGL_SYNC_H_ -#include -#include - #include "absl/status/status.h" #include "absl/status/statusor.h" #include "mediapipe/framework/formats/unique_fd.h" -#include "mediapipe/gpu/gl_base.h" +#include "mediapipe/gpu/egl_base.h" namespace mediapipe { @@ -26,6 +23,9 @@ class EglSync { static absl::StatusOr CreateNative(EGLDisplay display, const UniqueFd& native_fence_fd); + static bool IsSupported(EGLDisplay display); + static bool IsNativeSupported(EGLDisplay display); + // Move-only EglSync(EglSync&& sync); EglSync& operator=(EglSync&& sync); diff --git a/mediapipe/gpu/egl_sync_point.cc b/mediapipe/gpu/egl_sync_point.cc new file mode 100644 index 0000000000..f11520d973 --- /dev/null +++ b/mediapipe/gpu/egl_sync_point.cc @@ -0,0 +1,88 @@ +#include "mediapipe/gpu/egl_sync_point.h" + +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/gpu/egl_sync.h" +#include "mediapipe/gpu/gl_context.h" + +namespace mediapipe { + +namespace { + +class EglFenceSyncPoint : public GlSyncPoint { + public: + explicit EglFenceSyncPoint(std::shared_ptr gl_context, + EglSync egl_sync) + : GlSyncPoint(std::move(gl_context)), egl_sync_(std::move(egl_sync)) {} + + ~EglFenceSyncPoint() override { + gl_context_->RunWithoutWaiting( + [ptr = new EglSync(std::move(egl_sync_))]() { delete ptr; }); + } + + EglFenceSyncPoint(const EglFenceSyncPoint&) = delete; + EglFenceSyncPoint& operator=(const EglFenceSyncPoint&) = delete; + + void Wait() override { + if (GlContext::IsAnyContextCurrent()) { + WaitInternal(); + } + // Fall back to GL context used during sync creation. + gl_context_->Run([this] { WaitInternal(); }); + } + + void WaitInternal() { + absl::Status result = egl_sync_.Wait(); + if (!result.ok()) { + ABSL_LOG(DFATAL) << "EGL sync Wait failed: " << result; + } + } + + void WaitOnGpu() override { + if (!GlContext::IsAnyContextCurrent()) { + ABSL_LOG(DFATAL) << "WaitOnGpu without current context."; + } + + absl::Status result = egl_sync_.WaitOnGpu(); + if (!result.ok()) { + ABSL_LOG(DFATAL) << "EGL sync WaitOnGpu failed: " << result; + } + } + + bool IsReady() override { + if (GlContext::IsAnyContextCurrent()) { + return IsReadyInternal(); + } + + // Fall back to GL context used during sync creation. + bool ready = false; + gl_context_->Run([this, &ready] { ready = IsReadyInternal(); }); + return ready; + } + + bool IsReadyInternal() { + absl::StatusOr is_ready = egl_sync_.IsSignaled(); + if (!is_ready.ok()) { + ABSL_LOG(DFATAL) << "EGL sync IsSignaled failed: " << is_ready.status(); + return false; + } + return *is_ready; + } + + private: + EglSync egl_sync_; +}; + +} // namespace + +absl::StatusOr> CreateEglSyncPoint( + std::shared_ptr gl_context, EglSync egl_sync) { + return std::make_unique(std::move(gl_context), + std::move(egl_sync)); +} + +} // namespace mediapipe diff --git a/mediapipe/gpu/egl_sync_point.h b/mediapipe/gpu/egl_sync_point.h new file mode 100644 index 0000000000..717d0cf41d --- /dev/null +++ b/mediapipe/gpu/egl_sync_point.h @@ -0,0 +1,17 @@ +#ifndef MEDIAPIPE_GPU_EGL_SYNC_POINT_H_ +#define MEDIAPIPE_GPU_EGL_SYNC_POINT_H_ + +#include + +#include "absl/status/statusor.h" +#include "mediapipe/gpu/egl_sync.h" +#include "mediapipe/gpu/gl_context.h" + +namespace mediapipe { + +absl::StatusOr> CreateEglSyncPoint( + std::shared_ptr gl_context, EglSync egl_sync); + +} // namespace mediapipe + +#endif // MEDIAPIPE_GPU_EGL_SYNC_POINT_H_ From 80814bae0da8b00e96eb62b748ab3cea77b7fe2e Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 8 Nov 2024 06:59:26 -0800 Subject: [PATCH 037/126] No public description PiperOrigin-RevId: 694482368 --- mediapipe/tasks/web/components/processors/BUILD | 12 ++++++------ ...er_options.test.ts => classifier_options_test.ts} | 0 ...fier_result.test.ts => classifier_result_test.ts} | 0 ...ction_result.test.ts => detection_result_test.ts} | 0 ...dder_options.test.ts => embedder_options_test.ts} | 0 ...bedder_result.test.ts => embedder_result_test.ts} | 0 ...ndmark_result.test.ts => landmark_result_test.ts} | 0 7 files changed, 6 insertions(+), 6 deletions(-) rename mediapipe/tasks/web/components/processors/{classifier_options.test.ts => classifier_options_test.ts} (100%) rename mediapipe/tasks/web/components/processors/{classifier_result.test.ts => classifier_result_test.ts} (100%) rename mediapipe/tasks/web/components/processors/{detection_result.test.ts => detection_result_test.ts} (100%) rename mediapipe/tasks/web/components/processors/{embedder_options.test.ts => embedder_options_test.ts} (100%) rename mediapipe/tasks/web/components/processors/{embedder_result.test.ts => embedder_result_test.ts} (100%) rename mediapipe/tasks/web/components/processors/{landmark_result.test.ts => landmark_result_test.ts} (100%) diff --git a/mediapipe/tasks/web/components/processors/BUILD b/mediapipe/tasks/web/components/processors/BUILD index e44e933beb..d986bf7262 100644 --- a/mediapipe/tasks/web/components/processors/BUILD +++ b/mediapipe/tasks/web/components/processors/BUILD @@ -17,7 +17,7 @@ mediapipe_ts_library( mediapipe_ts_library( name = "classifier_options_test_lib", testonly = True, - srcs = ["classifier_options.test.ts"], + srcs = ["classifier_options_test.ts"], deps = [ ":classifier_options", "//mediapipe/tasks/cc/components/processors/proto:classifier_options_jspb_proto", @@ -43,7 +43,7 @@ mediapipe_ts_library( mediapipe_ts_library( name = "classifier_result_test_lib", testonly = True, - srcs = ["classifier_result.test.ts"], + srcs = ["classifier_result_test.ts"], deps = [ ":classifier_result", "//mediapipe/framework/formats:classification_jspb_proto", @@ -68,7 +68,7 @@ mediapipe_ts_library( mediapipe_ts_library( name = "detection_result_test_lib", testonly = True, - srcs = ["detection_result.test.ts"], + srcs = ["detection_result_test.ts"], deps = [ ":detection_result", "//mediapipe/framework/formats:detection_jspb_proto", @@ -89,7 +89,7 @@ mediapipe_ts_library( mediapipe_ts_library( name = "embedder_result_test_lib", testonly = True, - srcs = ["embedder_result.test.ts"], + srcs = ["embedder_result_test.ts"], deps = [ ":embedder_result", "//mediapipe/tasks/cc/components/containers/proto:embeddings_jspb_proto", @@ -113,7 +113,7 @@ mediapipe_ts_library( mediapipe_ts_library( name = "embedder_options_test_lib", testonly = True, - srcs = ["embedder_options.test.ts"], + srcs = ["embedder_options_test.ts"], deps = [ ":embedder_options", "//mediapipe/tasks/cc/components/processors/proto:embedder_options_jspb_proto", @@ -141,7 +141,7 @@ mediapipe_ts_library( mediapipe_ts_library( name = "landmark_result_test_lib", testonly = True, - srcs = ["landmark_result.test.ts"], + srcs = ["landmark_result_test.ts"], deps = [":landmark_result"], ) diff --git a/mediapipe/tasks/web/components/processors/classifier_options.test.ts b/mediapipe/tasks/web/components/processors/classifier_options_test.ts similarity index 100% rename from mediapipe/tasks/web/components/processors/classifier_options.test.ts rename to mediapipe/tasks/web/components/processors/classifier_options_test.ts diff --git a/mediapipe/tasks/web/components/processors/classifier_result.test.ts b/mediapipe/tasks/web/components/processors/classifier_result_test.ts similarity index 100% rename from mediapipe/tasks/web/components/processors/classifier_result.test.ts rename to mediapipe/tasks/web/components/processors/classifier_result_test.ts diff --git a/mediapipe/tasks/web/components/processors/detection_result.test.ts b/mediapipe/tasks/web/components/processors/detection_result_test.ts similarity index 100% rename from mediapipe/tasks/web/components/processors/detection_result.test.ts rename to mediapipe/tasks/web/components/processors/detection_result_test.ts diff --git a/mediapipe/tasks/web/components/processors/embedder_options.test.ts b/mediapipe/tasks/web/components/processors/embedder_options_test.ts similarity index 100% rename from mediapipe/tasks/web/components/processors/embedder_options.test.ts rename to mediapipe/tasks/web/components/processors/embedder_options_test.ts diff --git a/mediapipe/tasks/web/components/processors/embedder_result.test.ts b/mediapipe/tasks/web/components/processors/embedder_result_test.ts similarity index 100% rename from mediapipe/tasks/web/components/processors/embedder_result.test.ts rename to mediapipe/tasks/web/components/processors/embedder_result_test.ts diff --git a/mediapipe/tasks/web/components/processors/landmark_result.test.ts b/mediapipe/tasks/web/components/processors/landmark_result_test.ts similarity index 100% rename from mediapipe/tasks/web/components/processors/landmark_result.test.ts rename to mediapipe/tasks/web/components/processors/landmark_result_test.ts From 5f2772466301ccf20fb26a8b0126802cec07032a Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 8 Nov 2024 07:03:21 -0800 Subject: [PATCH 038/126] No public description PiperOrigin-RevId: 694483477 --- mediapipe/tasks/web/vision/core/BUILD | 2 +- .../{vision_task_runner.test.ts => vision_task_runner_test.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename mediapipe/tasks/web/vision/core/{vision_task_runner.test.ts => vision_task_runner_test.ts} (100%) diff --git a/mediapipe/tasks/web/vision/core/BUILD b/mediapipe/tasks/web/vision/core/BUILD index 46f9a7c959..867444a0f4 100644 --- a/mediapipe/tasks/web/vision/core/BUILD +++ b/mediapipe/tasks/web/vision/core/BUILD @@ -138,7 +138,7 @@ mediapipe_ts_library( mediapipe_ts_library( name = "vision_task_runner_test_lib", testonly = True, - srcs = ["vision_task_runner.test.ts"], + srcs = ["vision_task_runner_test.ts"], deps = [ ":image_processing_options", ":vision_task_options", diff --git a/mediapipe/tasks/web/vision/core/vision_task_runner.test.ts b/mediapipe/tasks/web/vision/core/vision_task_runner_test.ts similarity index 100% rename from mediapipe/tasks/web/vision/core/vision_task_runner.test.ts rename to mediapipe/tasks/web/vision/core/vision_task_runner_test.ts From 163a1e081bbdd9efd174b9ab01d72cb1aafef0a9 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 8 Nov 2024 07:03:59 -0800 Subject: [PATCH 039/126] No public description PiperOrigin-RevId: 694483682 --- mediapipe/tasks/web/components/utils/BUILD | 2 +- .../{cosine_similarity.test.ts => cosine_similarity_test.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename mediapipe/tasks/web/components/utils/{cosine_similarity.test.ts => cosine_similarity_test.ts} (100%) diff --git a/mediapipe/tasks/web/components/utils/BUILD b/mediapipe/tasks/web/components/utils/BUILD index af49cc730b..3b84751e1d 100644 --- a/mediapipe/tasks/web/components/utils/BUILD +++ b/mediapipe/tasks/web/components/utils/BUILD @@ -14,7 +14,7 @@ mediapipe_ts_library( mediapipe_ts_library( name = "cosine_similarity_test_lib", testonly = True, - srcs = ["cosine_similarity.test.ts"], + srcs = ["cosine_similarity_test.ts"], deps = [ ":cosine_similarity", "//mediapipe/tasks/web/components/containers:embedding_result", diff --git a/mediapipe/tasks/web/components/utils/cosine_similarity.test.ts b/mediapipe/tasks/web/components/utils/cosine_similarity_test.ts similarity index 100% rename from mediapipe/tasks/web/components/utils/cosine_similarity.test.ts rename to mediapipe/tasks/web/components/utils/cosine_similarity_test.ts From 31156b611f86bba59ad98bea1c8edb6930b98914 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 8 Nov 2024 09:56:16 -0800 Subject: [PATCH 040/126] Add traces for EglSync::CreateNative. PiperOrigin-RevId: 694539938 --- mediapipe/gpu/BUILD | 1 + mediapipe/gpu/egl_sync.cc | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index fb8f4b2b72..1c4de271f0 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -875,6 +875,7 @@ cc_library( "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "//mediapipe/framework/profiler:perfetto_profiling", "@com_google_absl//absl/cleanup", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", diff --git a/mediapipe/gpu/egl_sync.cc b/mediapipe/gpu/egl_sync.cc index 56cf886fe8..d6d7567c60 100644 --- a/mediapipe/gpu/egl_sync.cc +++ b/mediapipe/gpu/egl_sync.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/formats/unique_fd.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/framework/profiler/perfetto_profiling.h" #include "mediapipe/gpu/egl_base.h" #include "mediapipe/gpu/egl_errors.h" @@ -104,6 +105,7 @@ absl::StatusOr EglSync::Create(EGLDisplay display) { } absl::StatusOr EglSync::CreateNative(EGLDisplay display) { + MEDIAPIPE_PERFETTO_TRACE_EVENT("EglSync::CreateNative"); MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); @@ -116,6 +118,9 @@ absl::StatusOr EglSync::CreateNative(EGLDisplay display) { absl::StatusOr EglSync::CreateNative(EGLDisplay display, const UniqueFd& native_fence_fd) { + MEDIAPIPE_PERFETTO_TRACE_EVENT( + absl::StrCat("EglSync::CreateNative for FD: ", native_fence_fd.Get())); + RET_CHECK(native_fence_fd.IsValid()); MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); From 39ce92a1bff5a9be7f597d1307765f97f2aefd51 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 8 Nov 2024 13:46:19 -0800 Subject: [PATCH 041/126] Bump MediaPipe version to 0.10.19. PiperOrigin-RevId: 694626412 --- version.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.bzl b/version.bzl index fdca852210..1cb37d18fc 100644 --- a/version.bzl +++ b/version.bzl @@ -2,4 +2,4 @@ # The next version of MediaPipe (e.g. the version that is currently in development). # This version should be bumped after every release. -MEDIAPIPE_FULL_VERSION = "0.10.18" +MEDIAPIPE_FULL_VERSION = "0.10.19" From 4fc767135865bd15a82e297098708c08daee2e86 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 8 Nov 2024 14:48:56 -0800 Subject: [PATCH 042/126] More perfetto tracking for EglSync. PiperOrigin-RevId: 694648896 --- mediapipe/gpu/egl_sync.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mediapipe/gpu/egl_sync.cc b/mediapipe/gpu/egl_sync.cc index d6d7567c60..bd406bc556 100644 --- a/mediapipe/gpu/egl_sync.cc +++ b/mediapipe/gpu/egl_sync.cc @@ -106,6 +106,7 @@ absl::StatusOr EglSync::Create(EGLDisplay display) { absl::StatusOr EglSync::CreateNative(EGLDisplay display) { MEDIAPIPE_PERFETTO_TRACE_EVENT("EglSync::CreateNative"); + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); @@ -189,6 +190,8 @@ void EglSync::Invalidate() { } absl::Status EglSync::WaitOnGpu() { + MEDIAPIPE_PERFETTO_TRACE_EVENT("EglSync::WaitOnGpu"); + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display_)); const EGLint result = eglWaitSyncKHR(display_, sync_, 0); @@ -197,6 +200,8 @@ absl::Status EglSync::WaitOnGpu() { } absl::Status EglSync::Wait() { + MEDIAPIPE_PERFETTO_TRACE_EVENT("EglSync::Wait"); + MP_RETURN_IF_ERROR(CheckEglSyncSupported(display_)); const EGLint result = eglClientWaitSyncKHR( @@ -207,6 +212,8 @@ absl::Status EglSync::Wait() { } absl::StatusOr EglSync::DupNativeFd() { + MEDIAPIPE_PERFETTO_TRACE_EVENT("EglSync::DupNativeFd"); + MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display_)); const int fd = eglDupNativeFenceFDANDROID(display_, sync_); @@ -216,6 +223,8 @@ absl::StatusOr EglSync::DupNativeFd() { } absl::StatusOr EglSync::IsSignaled() { + MEDIAPIPE_PERFETTO_TRACE_EVENT("EglSync::IsSignaled"); + EGLint status; const EGLBoolean success = eglGetSyncAttribKHR(display_, sync_, EGL_SYNC_STATUS_KHR, &status); From 8c3e2d7bb90e2faf5743f4ce8fa3f494855c7dcb Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 11 Nov 2024 16:51:10 -0800 Subject: [PATCH 043/126] Patch for supporting WebGPU .deviceInfo during API migration. PiperOrigin-RevId: 695527094 --- mediapipe/web/graph_runner/graph_runner_webgpu.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mediapipe/web/graph_runner/graph_runner_webgpu.ts b/mediapipe/web/graph_runner/graph_runner_webgpu.ts index 83ce45ff67..596da271e2 100644 --- a/mediapipe/web/graph_runner/graph_runner_webgpu.ts +++ b/mediapipe/web/graph_runner/graph_runner_webgpu.ts @@ -88,9 +88,12 @@ export function SupportWebGpu(Base: TBase) { // Our inference engines can utilize the adapter info to optimize WebGPU // shader performance. Therefore, we attempt to attach that information to // our internal GPUDevice reference. - (device as unknown as GPUDeviceWithAdapterInfo).adapterInfo = - adapter.info; - + // We only apply workaround for browsers/environments where necessary, + // otherwise we'll encounter a runtime error, since this is read-only. + const deviceWithInfo = (device as unknown as GPUDeviceWithAdapterInfo); + if (!deviceWithInfo.adapterInfo) { + deviceWithInfo.adapterInfo = adapter.info; + } return device; } From 72289c9a0f67137ea39163eaf0acec9c5b2fe796 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 11 Nov 2024 21:44:01 -0800 Subject: [PATCH 044/126] No public description PiperOrigin-RevId: 695590672 --- mediapipe/util/audio_decoder.cc | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/mediapipe/util/audio_decoder.cc b/mediapipe/util/audio_decoder.cc index c0697dce42..dfae851d7c 100644 --- a/mediapipe/util/audio_decoder.cc +++ b/mediapipe/util/audio_decoder.cc @@ -38,9 +38,11 @@ extern "C" { #include "libavcodec/avcodec.h" #include "libavformat/avformat.h" +#include "libavformat/version.h" #include "libavutil/avutil.h" #include "libavutil/mem.h" #include "libavutil/samplefmt.h" +#include "libavutil/version.h" } ABSL_FLAG(int64_t, media_decoder_allowed_audio_gap_merge, 5, @@ -208,11 +210,9 @@ absl::Status LogStatus(const absl::Status& status, class AVPacketDeleter { public: - void operator()(void* x) const { - AVPacket* packet = static_cast(x); - if (packet) { - av_free_packet(packet); - delete packet; + void operator()(AVPacket* packet) const { + if (packet != nullptr) { + av_packet_free(&packet); } } }; @@ -241,7 +241,7 @@ absl::Status BasePacketProcessor::GetData(Packet* packet) { absl::Status BasePacketProcessor::Flush() { int64_t last_num_frames_processed; do { - std::unique_ptr av_packet(new AVPacket()); + std::unique_ptr av_packet(av_packet_alloc()); av_init_packet(av_packet.get()); av_packet->size = 0; av_packet->data = nullptr; @@ -592,7 +592,11 @@ int64_t AudioPacketProcessor::MaybeCorrectPtsForRollover(int64_t media_pts) { } // AudioDecoder -AudioDecoder::AudioDecoder() { av_register_all(); } +AudioDecoder::AudioDecoder() { +#if LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(58, 79, 100) + av_register_all(); +#endif +} AudioDecoder::~AudioDecoder() { absl::Status status = Close(); @@ -771,7 +775,7 @@ absl::Status AudioDecoder::FillAudioHeader( } absl::Status AudioDecoder::ProcessPacket() { - std::unique_ptr av_packet(new AVPacket()); + std::unique_ptr av_packet(av_packet_alloc()); av_init_packet(av_packet.get()); av_packet->size = 0; av_packet->data = nullptr; From e86914d825fb994ca4dd0ef45c4d8323ba4da22f Mon Sep 17 00:00:00 2001 From: Tommy Chiang Date: Tue, 12 Nov 2024 02:51:45 -0800 Subject: [PATCH 045/126] Log the Tensor multi-write error message only once. PiperOrigin-RevId: 695658091 --- mediapipe/framework/formats/tensor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/framework/formats/tensor.cc b/mediapipe/framework/formats/tensor.cc index d38e75d542..f0f6aa9a60 100644 --- a/mediapipe/framework/formats/tensor.cc +++ b/mediapipe/framework/formats/tensor.cc @@ -407,7 +407,7 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferWriteView( } AllocateOpenGlBuffer(); if (valid_ != 0) { - ABSL_LOG(ERROR) + ABSL_LOG_FIRST_N(ERROR, 1) << "Tensors are designed for single writes. Multiple writes to a " "Tensor instance are not supported and may lead to undefined " "behavior due to lack of synchronization."; From ceb7d5d92c0a43bf4b87fefd0eb7af42318976f6 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 12 Nov 2024 03:03:43 -0800 Subject: [PATCH 046/126] Improves debug output for Packet::Get() on empty packet PiperOrigin-RevId: 695660916 --- mediapipe/framework/api2/BUILD | 4 ++++ mediapipe/framework/api2/packet.h | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/mediapipe/framework/api2/BUILD b/mediapipe/framework/api2/BUILD index f95f7f9f3d..bb1b01f205 100644 --- a/mediapipe/framework/api2/BUILD +++ b/mediapipe/framework/api2/BUILD @@ -139,15 +139,19 @@ cc_library( hdrs = ["packet.h"], deps = [ ":tuple", + "//mediapipe/framework:legacy_calculator_support", "//mediapipe/framework:packet", + "//mediapipe/framework:port", "//mediapipe/framework:timestamp", "//mediapipe/framework/port:logging", "//mediapipe/framework/port:status", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/meta:type_traits", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/framework/api2/packet.h b/mediapipe/framework/api2/packet.h index 750ef80327..79536578b9 100644 --- a/mediapipe/framework/api2/packet.h +++ b/mediapipe/framework/api2/packet.h @@ -11,16 +11,20 @@ #define MEDIAPIPE_FRAMEWORK_API2_PACKET_H_ #include +#include #include #include #include "absl/base/attributes.h" #include "absl/base/optimization.h" #include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/meta/type_traits.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" #include "mediapipe/framework/api2/tuple.h" +#include "mediapipe/framework/legacy_calculator_support.h" #include "mediapipe/framework/packet.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/status_macros.h" @@ -263,7 +267,18 @@ class Packet : public Packet { Packet At(Timestamp timestamp) &&; const T& Get() const { - ABSL_CHECK(payload_); + if (!payload_) { + // TODO - Remove this check once stack trace symbolization + // works on Android non-apk execution. + const CalculatorContext* calculator_context = + LegacyCalculatorSupport::Scoped::current(); + if (calculator_context) { + ABSL_LOG(FATAL) << absl::StrCat( + "Get() called on empty packet during execution of ", + calculator_context->NodeName(), "."); + } + ABSL_LOG(FATAL) << "Get() called on empty packet."; + } const packet_internal::Holder* typed_payload = payload_->As(); ABSL_CHECK(typed_payload); return typed_payload->data(); From d68d72c559949d2fc0646b9167382ed613ca3efb Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 12 Nov 2024 09:34:50 -0800 Subject: [PATCH 047/126] Enable GpuBufferStorageAhwb ASYNC usage for use case: AhwbView write -> GlTextureView read * Adding AhwbView::SetUsageFence to allow for automatic async synchronization when getting GlTextureView * Adding AhwbView::GetWidthStepBytes to properly fill bytes * Adding GpuBufferStorageAhwb::Reuse to allow reusing storage for new writes * Exposing ReadTexture from gl_texture_buffer.cc to read GL texture data PiperOrigin-RevId: 695766930 --- mediapipe/framework/formats/BUILD | 1 + mediapipe/framework/formats/ahwb_view.h | 50 +++++++++++++++++++++++-- mediapipe/gpu/BUILD | 20 +++++++++- mediapipe/gpu/gl_texture_buffer.cc | 5 ++- mediapipe/gpu/gl_texture_buffer.h | 8 ++++ mediapipe/gpu/gpu_buffer_test.cc | 2 - 6 files changed, 77 insertions(+), 9 deletions(-) diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index 9eaba8c0e2..b5d93530e4 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -130,6 +130,7 @@ cc_library( deps = [ ":hardware_buffer", "//mediapipe/framework:port", + "//mediapipe/framework/formats:unique_fd", "//mediapipe/gpu:gpu_buffer_storage", ], ) diff --git a/mediapipe/framework/formats/ahwb_view.h b/mediapipe/framework/formats/ahwb_view.h index 9438e185ed..8703c9dc53 100644 --- a/mediapipe/framework/formats/ahwb_view.h +++ b/mediapipe/framework/formats/ahwb_view.h @@ -2,8 +2,16 @@ #define MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_ #include "mediapipe/framework/port.h" + #ifdef MEDIAPIPE_GPU_BUFFER_USE_AHWB + +#include +#include + +#include "absl/functional/any_invocable.h" +#include "absl/status/status.h" #include "mediapipe/framework/formats/hardware_buffer.h" +#include "mediapipe/framework/formats/unique_fd.h" #include "mediapipe/gpu/gpu_buffer_storage.h" namespace mediapipe { @@ -16,25 +24,59 @@ namespace mediapipe { // own usage. // The AHWB abstractions in GpuBuffer and Tensor are likely more suitable for // other CPU/GPU uses of AHWBs. + class AhwbView { public: - explicit AhwbView(HardwareBuffer* ahwb) : ahwb_(ahwb) {} + explicit AhwbView(HardwareBuffer* ahwb, int width_step_bytes, + absl::AnyInvocable)> + set_usage_fence_fn) + : ahwb_(ahwb), + width_step_bytes_(width_step_bytes), + set_usage_fence_fn_(std::move(set_usage_fence_fn)) {} // Non-copyable AhwbView(const AhwbView&) = delete; AhwbView& operator=(const AhwbView&) = delete; // Non-movable AhwbView(AhwbView&&) = delete; - // Only supports synchronous usage. All users of GetHandle must finish + // Supports only synchronous read usage - all users of GetHandle must finish // accessing the buffer before this view object is destroyed to avoid race - // conditions. - // TODO: Support asynchronous usage. + // conditions). + // + // Supports async write usage - user must provide a usage fence which is + // signaled when the write is complete. See more details in `SetUsageFence`. + // TODO: Support full async usage. const AHardwareBuffer* GetHandle() const { return ahwb_->GetAHardwareBuffer(); } + int GetWidthStepBytes() const { return width_step_bytes_; } + + // Sets usage fence for this AHWB: + // - fence is not signaled => AHWB is still in use + // - fence is signaled => AHWB is not in use anymore + // + // Example use case: + // - Calculator gets AhwbView for writing where writing is done asynchronously + // and fence is created to indicate write completion. (E.g. TPU/DSP delegate + // is used and can provide a completion fence.) + // - Calculator schedules async write, retrieves the completion fence and sets + // it using `SetUsageFence`. + // - Calculator sends corresponding `GpuBuffer` to a downstream calculator. + // - The downstream calculator gets `GlBufferView` for reading, `GpuBuffer` + // automatically imports and inserts the fence as GL fence sync ensuring + // following GL operations wait for write completion. + // + // TODO: b/376753887 - replace with a dedicated type (MP's Fence) + absl::Status SetUsageFence(std::shared_ptr fence) { + return set_usage_fence_fn_(fence); + } + private: const HardwareBuffer* ahwb_; + const int width_step_bytes_; + absl::AnyInvocable)> + set_usage_fence_fn_; }; namespace internal { diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 1c4de271f0..b4b4f12d42 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -567,11 +567,19 @@ cc_library( "//mediapipe/framework/formats:ahwb_view", "//mediapipe/framework/formats:hardware_buffer", "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:ret_check", + "//mediapipe/util:sync_wait", "//third_party/GL:EGL_headers", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings:str_format", - ], + ] + select({ + "//conditions:default": [], + "//mediapipe:android": [ + ":egl_sync", + ":egl_sync_point", + ], + }), ) mediapipe_proto_library( @@ -1372,13 +1380,23 @@ mediapipe_cc_test( ], requires_full_emulation = True, deps = [ + ":egl_sync", ":gl_texture_buffer", ":gl_texture_util", + ":gl_texture_view", ":gpu_buffer_format", ":gpu_buffer_storage_ahwb", ":gpu_test_base", + "//mediapipe/framework:port", + "//mediapipe/framework/formats:ahwb_view", + "//mediapipe/framework/formats:hardware_buffer", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/tool:test_util", + "//mediapipe/util:sync_wait", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/time", ], ) diff --git a/mediapipe/gpu/gl_texture_buffer.cc b/mediapipe/gpu/gl_texture_buffer.cc index e835a1a55f..db61fb35e6 100644 --- a/mediapipe/gpu/gl_texture_buffer.cc +++ b/mediapipe/gpu/gl_texture_buffer.cc @@ -14,6 +14,7 @@ #include "mediapipe/gpu/gl_texture_buffer.h" +#include #include #include #include @@ -357,8 +358,8 @@ void GlTextureBuffer::ViewDoneWriting(const GlTextureView& view) { #endif // __ANDROID__ } -static void ReadTexture(GlContext& ctx, const GlTextureView& view, - GpuBufferFormat format, void* output, size_t size) { +void ReadTexture(GlContext& ctx, const GlTextureView& view, + GpuBufferFormat format, void* output, size_t size) { // TODO: check buffer size? We could use glReadnPixels where available // (OpenGL ES 3.2, i.e. nowhere). Note that, to fully check that the read // won't overflow the buffer with glReadPixels, we'd also need to check or diff --git a/mediapipe/gpu/gl_texture_buffer.h b/mediapipe/gpu/gl_texture_buffer.h index ab8154a85d..a8753889ef 100644 --- a/mediapipe/gpu/gl_texture_buffer.h +++ b/mediapipe/gpu/gl_texture_buffer.h @@ -18,6 +18,7 @@ #ifndef MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_ #define MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_ +#include #include #include @@ -175,6 +176,13 @@ class GlTextureBuffer std::shared_ptr producer_context_; }; +// Reads `texture_view` into `output`. +// NOTE: It's clients responsibility to allocate `output` properly and provide +// the right `size`. +// NOTE: Must be invoked on a thread with GL context. +void ReadTexture(GlContext& ctx, const GlTextureView& texture_view, + GpuBufferFormat format, void* output, size_t size); + using GlTextureBufferSharedPtr = std::shared_ptr; } // namespace mediapipe diff --git a/mediapipe/gpu/gpu_buffer_test.cc b/mediapipe/gpu/gpu_buffer_test.cc index 7be33e39c6..f166dd4777 100644 --- a/mediapipe/gpu/gpu_buffer_test.cc +++ b/mediapipe/gpu/gpu_buffer_test.cc @@ -22,8 +22,6 @@ #include "mediapipe/framework/tool/test_util.h" #include "mediapipe/gpu/gl_texture_buffer.h" #include "mediapipe/gpu/gl_texture_util.h" -#include "mediapipe/gpu/gpu_buffer_storage_ahwb.h" -#include "mediapipe/gpu/gpu_buffer_storage_image_frame.h" #include "mediapipe/gpu/gpu_test_base.h" #include "stb_image.h" #include "stb_image_write.h" From 68e37931318ac79c815c65a9d99c821108a064d1 Mon Sep 17 00:00:00 2001 From: Joe Zou Date: Tue, 12 Nov 2024 12:00:10 -0800 Subject: [PATCH 048/126] No public description PiperOrigin-RevId: 695820858 --- mediapipe/model_maker/python/text/text_classifier/BUILD | 1 - mediapipe/model_maker/python/text/text_classifier/model_spec.py | 1 - 2 files changed, 2 deletions(-) diff --git a/mediapipe/model_maker/python/text/text_classifier/BUILD b/mediapipe/model_maker/python/text/text_classifier/BUILD index 03be858cfb..c5861ed2d5 100644 --- a/mediapipe/model_maker/python/text/text_classifier/BUILD +++ b/mediapipe/model_maker/python/text/text_classifier/BUILD @@ -65,7 +65,6 @@ py_library( deps = [ ":hyperparameters", ":model_options", - "//mediapipe/model_maker/python/core/utils:file_util", "//mediapipe/model_maker/python/text/core:bert_model_spec", ], ) diff --git a/mediapipe/model_maker/python/text/text_classifier/model_spec.py b/mediapipe/model_maker/python/text/text_classifier/model_spec.py index a34f14fd5c..c22454c43f 100644 --- a/mediapipe/model_maker/python/text/text_classifier/model_spec.py +++ b/mediapipe/model_maker/python/text/text_classifier/model_spec.py @@ -17,7 +17,6 @@ import enum import functools -from mediapipe.model_maker.python.core.utils import file_util from mediapipe.model_maker.python.text.core import bert_model_spec from mediapipe.model_maker.python.text.text_classifier import hyperparameters as hp from mediapipe.model_maker.python.text.text_classifier import model_options as mo From c2b3e070ad392974a1f6b814a795f150790e4ae9 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 12 Nov 2024 13:36:32 -0800 Subject: [PATCH 049/126] No public description PiperOrigin-RevId: 695853815 --- mediapipe/gpu/BUILD | 1 - mediapipe/gpu/egl_sync.cc | 14 -------------- 2 files changed, 15 deletions(-) diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index b4b4f12d42..39f8f2cc1e 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -883,7 +883,6 @@ cc_library( "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", - "//mediapipe/framework/profiler:perfetto_profiling", "@com_google_absl//absl/cleanup", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", diff --git a/mediapipe/gpu/egl_sync.cc b/mediapipe/gpu/egl_sync.cc index bd406bc556..56cf886fe8 100644 --- a/mediapipe/gpu/egl_sync.cc +++ b/mediapipe/gpu/egl_sync.cc @@ -14,7 +14,6 @@ #include "mediapipe/framework/formats/unique_fd.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status_macros.h" -#include "mediapipe/framework/profiler/perfetto_profiling.h" #include "mediapipe/gpu/egl_base.h" #include "mediapipe/gpu/egl_errors.h" @@ -105,8 +104,6 @@ absl::StatusOr EglSync::Create(EGLDisplay display) { } absl::StatusOr EglSync::CreateNative(EGLDisplay display) { - MEDIAPIPE_PERFETTO_TRACE_EVENT("EglSync::CreateNative"); - MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); @@ -119,9 +116,6 @@ absl::StatusOr EglSync::CreateNative(EGLDisplay display) { absl::StatusOr EglSync::CreateNative(EGLDisplay display, const UniqueFd& native_fence_fd) { - MEDIAPIPE_PERFETTO_TRACE_EVENT( - absl::StrCat("EglSync::CreateNative for FD: ", native_fence_fd.Get())); - RET_CHECK(native_fence_fd.IsValid()); MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); @@ -190,8 +184,6 @@ void EglSync::Invalidate() { } absl::Status EglSync::WaitOnGpu() { - MEDIAPIPE_PERFETTO_TRACE_EVENT("EglSync::WaitOnGpu"); - MP_RETURN_IF_ERROR(CheckEglSyncSupported(display_)); const EGLint result = eglWaitSyncKHR(display_, sync_, 0); @@ -200,8 +192,6 @@ absl::Status EglSync::WaitOnGpu() { } absl::Status EglSync::Wait() { - MEDIAPIPE_PERFETTO_TRACE_EVENT("EglSync::Wait"); - MP_RETURN_IF_ERROR(CheckEglSyncSupported(display_)); const EGLint result = eglClientWaitSyncKHR( @@ -212,8 +202,6 @@ absl::Status EglSync::Wait() { } absl::StatusOr EglSync::DupNativeFd() { - MEDIAPIPE_PERFETTO_TRACE_EVENT("EglSync::DupNativeFd"); - MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display_)); const int fd = eglDupNativeFenceFDANDROID(display_, sync_); @@ -223,8 +211,6 @@ absl::StatusOr EglSync::DupNativeFd() { } absl::StatusOr EglSync::IsSignaled() { - MEDIAPIPE_PERFETTO_TRACE_EVENT("EglSync::IsSignaled"); - EGLint status; const EGLBoolean success = eglGetSyncAttribKHR(display_, sync_, EGL_SYNC_STATUS_KHR, &status); From fed2566f8c30ff6490f491822665f6381a3df6fb Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 12 Nov 2024 15:29:58 -0800 Subject: [PATCH 050/126] Add IsSignaled function (the previous SyncWait for checking status triggers unnessary StrFormat) PiperOrigin-RevId: 695892325 --- mediapipe/util/BUILD | 3 +-- mediapipe/util/sync_wait.cc | 27 +++++++++++++++++++++++++++ mediapipe/util/sync_wait.h | 7 +++++++ mediapipe/util/sync_wait_test.cc | 16 ++++++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/mediapipe/util/BUILD b/mediapipe/util/BUILD index 92af0ee0a2..25844a580d 100644 --- a/mediapipe/util/BUILD +++ b/mediapipe/util/BUILD @@ -204,9 +204,8 @@ cc_library( "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", - "@com_google_absl//absl/cleanup", - "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/time", ], diff --git a/mediapipe/util/sync_wait.cc b/mediapipe/util/sync_wait.cc index 4a1df09f21..3e8705a7aa 100644 --- a/mediapipe/util/sync_wait.cc +++ b/mediapipe/util/sync_wait.cc @@ -7,6 +7,7 @@ #include #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_format.h" #include "absl/time/time.h" #include "mediapipe/framework/formats/unique_fd.h" @@ -52,4 +53,30 @@ absl::Status SyncWait(const UniqueFd& fd, absl::Duration timeout) { return SyncWait(fd.Get(), timeout); } +absl::StatusOr IsSignaled(int fd) { + RET_CHECK_GE(fd, 0) << "Invalid file descriptor."; + + struct pollfd fds; + fds.fd = fd; + fds.events = POLLIN; + int ret; + do { + ret = poll(&fds, 1, /*timeout_millis=*/0); + if (ret == 1) { + RET_CHECK((fds.revents & POLLERR) == 0); + RET_CHECK((fds.revents & POLLNVAL) == 0); + return true; + } else if (ret == 0) { + return false; + } + } while (ret == -1 && (errno == EINTR || errno == EAGAIN)); + + return absl::ErrnoToStatus( + errno, absl::StrFormat("Failed to check if fd: %d is signaled.", fd)); +} + +absl::StatusOr IsSignaled(const UniqueFd& fd) { + return IsSignaled(fd.Get()); +} + } // namespace mediapipe diff --git a/mediapipe/util/sync_wait.h b/mediapipe/util/sync_wait.h index 19ce9a435d..ba3d9d4b40 100644 --- a/mediapipe/util/sync_wait.h +++ b/mediapipe/util/sync_wait.h @@ -2,6 +2,7 @@ #define MEDIAPIPE_UTIL_SYNC_WAIT_H_ #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/time/time.h" #include "mediapipe/framework/formats/unique_fd.h" @@ -17,6 +18,12 @@ absl::Status SyncWait(int fd, absl::Duration timeout); // signaled. absl::Status SyncWait(const UniqueFd& fd, absl::Duration timeout); +// Checks if sync represented by `fd` is signaled. +absl::StatusOr IsSignaled(int fd); + +// Checks if sync represented by `fd` is signaled. +absl::StatusOr IsSignaled(const UniqueFd& fd); + } // namespace mediapipe #endif // MEDIAPIPE_UTIL_SYNC_WAIT_H_ diff --git a/mediapipe/util/sync_wait_test.cc b/mediapipe/util/sync_wait_test.cc index 00aefbc2f6..eb5cfc66ee 100644 --- a/mediapipe/util/sync_wait_test.cc +++ b/mediapipe/util/sync_wait_test.cc @@ -90,6 +90,22 @@ TEST(SyncWait, ReportsInvalidFd) { StatusIs(absl::StatusCode::kInternal)); } +TEST(SyncWait, IsSignaledWorks) { + TestTimer timer = CreateTestTimer(absl::Milliseconds(100)); + MP_ASSERT_OK_AND_ASSIGN(bool is_signaled, IsSignaled(timer.fd)); + EXPECT_FALSE(is_signaled); + + MP_ASSERT_OK(SyncWait(timer.fd, absl::InfiniteDuration())); + + MP_ASSERT_OK_AND_ASSIGN(is_signaled, IsSignaled(timer.fd)); + EXPECT_TRUE(is_signaled); +} + +TEST(SyncWait, IsSignaledReportsInvalidFd) { + const int fd = -1; + EXPECT_THAT(IsSignaled(fd), StatusIs(absl::StatusCode::kInternal)); +} + void BM_SyncWaitZeroTimeout(benchmark::State& state) { // Non blocking waits will be used and timer canceled automatically after // benchmark completion. From 48080cc921a3d256352e23389b2715c32fe35774 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 13 Nov 2024 01:02:53 -0800 Subject: [PATCH 051/126] Add type information to error message when accessing an empty packet. PiperOrigin-RevId: 696023846 --- mediapipe/framework/api2/packet.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mediapipe/framework/api2/packet.h b/mediapipe/framework/api2/packet.h index 79536578b9..9abf879793 100644 --- a/mediapipe/framework/api2/packet.h +++ b/mediapipe/framework/api2/packet.h @@ -273,11 +273,14 @@ class Packet : public Packet { const CalculatorContext* calculator_context = LegacyCalculatorSupport::Scoped::current(); if (calculator_context) { - ABSL_LOG(FATAL) << absl::StrCat( - "Get() called on empty packet during execution of ", - calculator_context->NodeName(), "."); + ABSL_LOG(FATAL) << absl::StrCat("Get() called for type ", + MediaPipeTypeStringOrDemangled(), + " on empty packet during execution of ", + calculator_context->NodeName(), "."); } - ABSL_LOG(FATAL) << "Get() called on empty packet."; + ABSL_LOG(FATAL) << absl::StrCat("Get() called for type ", + MediaPipeTypeStringOrDemangled(), + " on empty packet."); } const packet_internal::Holder* typed_payload = payload_->As(); ABSL_CHECK(typed_payload); From 0a4bad81deeb6fd336feca7d3a1b58b672eeea8b Mon Sep 17 00:00:00 2001 From: Joe Zou Date: Wed, 13 Nov 2024 10:49:40 -0800 Subject: [PATCH 052/126] No public description PiperOrigin-RevId: 696193844 --- mediapipe/model_maker/python/core/utils/loss_functions.py | 2 +- .../python/text/text_classifier/text_classifier_test.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mediapipe/model_maker/python/core/utils/loss_functions.py b/mediapipe/model_maker/python/core/utils/loss_functions.py index c63c277163..476623bda2 100644 --- a/mediapipe/model_maker/python/core/utils/loss_functions.py +++ b/mediapipe/model_maker/python/core/utils/loss_functions.py @@ -165,7 +165,7 @@ def __call__(self, y_pred_rank = y_pred.shape.ndims if y_pred_rank - weight_rank == 1: sample_weight = tf.expand_dims(sample_weight, [-1]) - elif weight_rank != 0: + elif weight_rank != 0 and y_pred_rank != weight_rank: raise ValueError(f'Unexpected sample_weights, should be either a scalar' f'or a vector of batch_size:{batch_size.numpy()}') ce = -tf.math.log(y_pred) diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py index 62db36e455..c5c5c7fd3a 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py @@ -149,6 +149,7 @@ def test_create_and_train_bert(self, supported_model): batch_size=1, learning_rate=3e-5, distribution_strategy='off', + class_weights={0: 1.0, 1: 1.0}, ), ) bert_classifier = text_classifier.TextClassifier.create( From 4990d719e884b90d75839f65f87f7cd6b9784277 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 13 Nov 2024 13:40:30 -0800 Subject: [PATCH 053/126] No public description PiperOrigin-RevId: 696255374 --- .../inference/utils/llm_utils/metadata_utils.h | 2 ++ .../genai/inference/utils/llm_utils/model_data.cc | 14 ++++++++++++++ .../genai/inference/utils/llm_utils/model_data.h | 8 ++++++++ .../tasks/python/genai/converter/llm_converter.py | 12 ++++++++++++ 4 files changed, 36 insertions(+) diff --git a/mediapipe/tasks/cc/genai/inference/utils/llm_utils/metadata_utils.h b/mediapipe/tasks/cc/genai/inference/utils/llm_utils/metadata_utils.h index f4e7abfd51..b8a421bae2 100644 --- a/mediapipe/tasks/cc/genai/inference/utils/llm_utils/metadata_utils.h +++ b/mediapipe/tasks/cc/genai/inference/utils/llm_utils/metadata_utils.h @@ -27,6 +27,8 @@ constexpr absl::string_view kLlmModelTypeName = "odml.infra.LlmModelType"; constexpr absl::string_view kLlmBackendName = "backend"; constexpr absl::string_view kSpmVocabName = "spm_vocab_model"; constexpr absl::string_view kLoRARank = "lora_rank"; +constexpr absl::string_view kImageEncoder = "image_encoder"; +constexpr absl::string_view kImageAdapter = "image_adapter"; // Retrieve LlmModelType from tflite flatbuffer metadata. absl::StatusOr GetLlmModelType( diff --git a/mediapipe/tasks/cc/genai/inference/utils/llm_utils/model_data.cc b/mediapipe/tasks/cc/genai/inference/utils/llm_utils/model_data.cc index 03f60f19f0..dc8472f50f 100644 --- a/mediapipe/tasks/cc/genai/inference/utils/llm_utils/model_data.cc +++ b/mediapipe/tasks/cc/genai/inference/utils/llm_utils/model_data.cc @@ -145,6 +145,20 @@ class TfliteModelData : public ModelData { return nullptr; } + absl::StatusOr ReadModel(absl::string_view name) override { + MP_ASSIGN_OR_RETURN(auto data, ReadTensor(name)); + if (!data) { + return ModelWithData{}; + } + auto model = tflite::FlatBufferModel::BuildFromBuffer( + reinterpret_cast(data->GetData().data()), + data->GetData().size()); + return ModelWithData{ + .model = std::move(model), + .data = std::move(data), + }; + } + absl::Status InitLlmParameters() { MP_ASSIGN_OR_RETURN(std::string proto_str, ReadMetadata(llm_parameters_.GetTypeName())); diff --git a/mediapipe/tasks/cc/genai/inference/utils/llm_utils/model_data.h b/mediapipe/tasks/cc/genai/inference/utils/llm_utils/model_data.h index 69f616aa85..94371e7d50 100644 --- a/mediapipe/tasks/cc/genai/inference/utils/llm_utils/model_data.h +++ b/mediapipe/tasks/cc/genai/inference/utils/llm_utils/model_data.h @@ -159,6 +159,14 @@ class ModelData { // Frees the underlying data. virtual void Clear() = 0; + + // Holds the tflite model as well as the backing data. + struct ModelWithData { + std::unique_ptr model; + std::unique_ptr> data; + }; + // Reads a tflite model from the main model. + virtual absl::StatusOr ReadModel(absl::string_view name) = 0; }; // Holds data referring to a set of LoRA weights. diff --git a/mediapipe/tasks/python/genai/converter/llm_converter.py b/mediapipe/tasks/python/genai/converter/llm_converter.py index cff7948b97..e92f241500 100644 --- a/mediapipe/tasks/python/genai/converter/llm_converter.py +++ b/mediapipe/tasks/python/genai/converter/llm_converter.py @@ -48,6 +48,8 @@ class ConversionConfig(object): lora_output_tflite_file: A string indicating the name of the generated tflite file for the LoRA weight. Only applicable when the lora_rank is not zero. + image_encoder_file: A string with the name of the image encoder tflite file. + image_adapter_file: A string with the name of the image adapter tflite file. use_fake_weights: Whether to use fake weights. If set to True, the weights will be filled with zeros. """ @@ -71,6 +73,8 @@ def __init__( lora_ckpt: Optional[str] = None, lora_rank: Optional[int] = None, lora_output_tflite_file: Optional[str] = None, + image_encoder_file: Optional[str] = None, + image_adapter_file: Optional[str] = None, use_fake_weights: bool = False, ): self.input_ckpt = input_ckpt @@ -90,6 +94,8 @@ def __init__( self.combine_file_only = combine_file_only self.vocab_model_file = vocab_model_file self.obfuscate = obfuscate + self.image_encoder_file = image_encoder_file + self.image_adapter_file = image_adapter_file self.use_fake_weights = use_fake_weights if output_tflite_file: parent_dir = os.path.dirname(output_tflite_file) @@ -212,6 +218,8 @@ def combined_weight_bins_to_tflite( lora_rank: Optional[int] = None, lora_weight_path: Optional[str] = None, lora_output_tflite_file: Optional[str] = None, + image_encoder_file: Optional[str] = None, + image_adapter_file: Optional[str] = None, ): """Combines weight files to tflite file.""" if backend == 'cpu': @@ -235,6 +243,8 @@ def combined_weight_bins_to_tflite( 0 if lora_rank is None else lora_rank, '' if lora_weight_path is None else lora_weight_path, '' if lora_output_tflite_file is None else lora_output_tflite_file, + '' if image_encoder_file is None else image_encoder_file, + '' if image_adapter_file is None else image_adapter_file, ) else: raise ValueError('Unsupported backend: %s' % backend) @@ -353,4 +363,6 @@ def convert_checkpoint(config: ConversionConfig) -> None: lora_rank=config.lora_rank, lora_weight_path=config.output_dir, lora_output_tflite_file=config.lora_output_tflite_file, + image_encoder_file=config.image_encoder_file, + image_adapter_file=config.image_adapter_file, ) From e2a1f1ef636636bd5671e9ba1e5f8f9b53aaf241 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 13 Nov 2024 15:05:15 -0800 Subject: [PATCH 054/126] Internal update. PiperOrigin-RevId: 696283989 --- .../vision/face_geometry_expected_out.pbtxt | 2558 ++++++++--------- 1 file changed, 1279 insertions(+), 1279 deletions(-) diff --git a/mediapipe/tasks/testdata/vision/face_geometry_expected_out.pbtxt b/mediapipe/tasks/testdata/vision/face_geometry_expected_out.pbtxt index d7e8652f41..df6eaaec35 100644 --- a/mediapipe/tasks/testdata/vision/face_geometry_expected_out.pbtxt +++ b/mediapipe/tasks/testdata/vision/face_geometry_expected_out.pbtxt @@ -3,2343 +3,2343 @@ mesh { vertex_type: VERTEX_PT primitive_type: TRIANGLE - vertex_buffer: -0.05458094 + vertex_buffer: -0.054581404 vertex_buffer: -3.9104233 - vertex_buffer: 6.2336426 + vertex_buffer: 6.233639 vertex_buffer: 0.499977 vertex_buffer: 0.652534 - vertex_buffer: -0.08718063 - vertex_buffer: -2.5412483 + vertex_buffer: -0.08718109 + vertex_buffer: -2.5412502 vertex_buffer: 7.949688 vertex_buffer: 0.500026 vertex_buffer: 0.547487 - vertex_buffer: -0.062314644 - vertex_buffer: -2.6207771 - vertex_buffer: 6.3720016 + vertex_buffer: -0.062315032 + vertex_buffer: -2.620779 + vertex_buffer: 6.371998 vertex_buffer: 0.499974 vertex_buffer: 0.602372 - vertex_buffer: -0.5262091 - vertex_buffer: -0.13145256 + vertex_buffer: -0.52620995 + vertex_buffer: -0.13145065 vertex_buffer: 6.791649 vertex_buffer: 0.482113 vertex_buffer: 0.471979 - vertex_buffer: -0.07805602 - vertex_buffer: -1.9441948 + vertex_buffer: -0.078056484 + vertex_buffer: -1.9441929 vertex_buffer: 8.094444 vertex_buffer: 0.500151 vertex_buffer: 0.527156 - vertex_buffer: -0.058286533 - vertex_buffer: -1.0193558 - vertex_buffer: 7.732723 + vertex_buffer: -0.05828686 + vertex_buffer: -1.0193539 + vertex_buffer: 7.7327194 vertex_buffer: 0.49991 vertex_buffer: 0.498253 - vertex_buffer: 0.015007198 - vertex_buffer: 1.6476421 + vertex_buffer: 0.015007228 + vertex_buffer: 1.6476498 vertex_buffer: 5.7056847 vertex_buffer: 0.499523 vertex_buffer: 0.401062 - vertex_buffer: -4.176094 - vertex_buffer: 2.6367798 - vertex_buffer: 2.9135933 + vertex_buffer: -4.1760993 + vertex_buffer: 2.6367893 + vertex_buffer: 2.9135857 vertex_buffer: 0.289712 vertex_buffer: 0.380764 - vertex_buffer: 0.03712082 - vertex_buffer: 2.985258 - vertex_buffer: 5.0371017 + vertex_buffer: 0.037120968 + vertex_buffer: 2.9852676 + vertex_buffer: 5.037098 vertex_buffer: 0.499955 vertex_buffer: 0.312398 - vertex_buffer: 0.06068772 - vertex_buffer: 3.787365 + vertex_buffer: 0.06068799 + vertex_buffer: 3.7873764 vertex_buffer: 5.021179 vertex_buffer: 0.499987 vertex_buffer: 0.269919 - vertex_buffer: 0.121769845 - vertex_buffer: 8.058422 + vertex_buffer: 0.12177047 + vertex_buffer: 8.058437 vertex_buffer: 3.8209877 vertex_buffer: 0.500023 vertex_buffer: 0.10705 - vertex_buffer: -0.04902646 - vertex_buffer: -4.1000786 + vertex_buffer: -0.049026906 + vertex_buffer: -4.1000805 vertex_buffer: 6.1800804 vertex_buffer: 0.500023 vertex_buffer: 0.666234 - vertex_buffer: -0.03831458 - vertex_buffer: -4.228405 + vertex_buffer: -0.038315058 + vertex_buffer: -4.228407 vertex_buffer: 6.0491524 vertex_buffer: 0.500016 vertex_buffer: 0.679224 - vertex_buffer: -0.05387768 - vertex_buffer: -4.2712173 - vertex_buffer: 5.8285065 + vertex_buffer: -0.053878143 + vertex_buffer: -4.2712193 + vertex_buffer: 5.8285027 vertex_buffer: 0.500023 vertex_buffer: 0.692348 - vertex_buffer: -0.053183556 + vertex_buffer: -0.053184003 vertex_buffer: -5.543394 - vertex_buffer: 5.0727234 + vertex_buffer: 5.0727158 vertex_buffer: 0.499977 vertex_buffer: 0.695278 - vertex_buffer: -0.059314266 - vertex_buffer: -5.8140745 - vertex_buffer: 5.2457848 + vertex_buffer: -0.059314743 + vertex_buffer: -5.8140755 + vertex_buffer: 5.245777 vertex_buffer: 0.499977 vertex_buffer: 0.705934 - vertex_buffer: -0.046299785 - vertex_buffer: -6.184594 - vertex_buffer: 5.4517365 + vertex_buffer: -0.04630032 + vertex_buffer: -6.184597 + vertex_buffer: 5.4517326 vertex_buffer: 0.499977 vertex_buffer: 0.719385 - vertex_buffer: -0.0507226 - vertex_buffer: -6.5362883 - vertex_buffer: 5.412071 + vertex_buffer: -0.050723135 + vertex_buffer: -6.536292 + vertex_buffer: 5.4120636 vertex_buffer: 0.499977 vertex_buffer: 0.737019 - vertex_buffer: -0.0614883 - vertex_buffer: -7.299967 - vertex_buffer: 5.0314445 + vertex_buffer: -0.061488867 + vertex_buffer: -7.2999697 + vertex_buffer: 5.031437 vertex_buffer: 0.499968 vertex_buffer: 0.781371 - vertex_buffer: -0.080724046 - vertex_buffer: -2.802185 + vertex_buffer: -0.08072452 + vertex_buffer: -2.8021832 vertex_buffer: 7.659096 vertex_buffer: 0.499816 vertex_buffer: 0.562981 - vertex_buffer: -0.7109383 + vertex_buffer: -0.7109394 vertex_buffer: -2.4929829 vertex_buffer: 6.7737694 vertex_buffer: 0.473773 vertex_buffer: 0.57391 - vertex_buffer: -7.1076756 - vertex_buffer: 6.3111076 + vertex_buffer: -7.1076837 + vertex_buffer: 6.311123 vertex_buffer: -0.36587524 vertex_buffer: 0.104907 vertex_buffer: 0.254141 - vertex_buffer: -2.470239 - vertex_buffer: 1.9644203 - vertex_buffer: 3.584652 + vertex_buffer: -2.4702423 + vertex_buffer: 1.964428 + vertex_buffer: 3.5846405 vertex_buffer: 0.36593 vertex_buffer: 0.409576 - vertex_buffer: -3.0842545 - vertex_buffer: 1.9237556 - vertex_buffer: 3.5713196 + vertex_buffer: -3.0842586 + vertex_buffer: 1.9237652 + vertex_buffer: 3.571312 vertex_buffer: 0.338758 vertex_buffer: 0.413025 - vertex_buffer: -3.7028773 - vertex_buffer: 1.9847145 - vertex_buffer: 3.4238892 + vertex_buffer: -3.702882 + vertex_buffer: 1.9847221 + vertex_buffer: 3.4238815 vertex_buffer: 0.31112 vertex_buffer: 0.40946 - vertex_buffer: -4.5404453 - vertex_buffer: 2.4034252 - vertex_buffer: 2.868042 + vertex_buffer: -4.540451 + vertex_buffer: 2.4034386 + vertex_buffer: 2.8680344 vertex_buffer: 0.274658 vertex_buffer: 0.389131 - vertex_buffer: -1.9506176 - vertex_buffer: 2.0792522 + vertex_buffer: -1.95062 + vertex_buffer: 2.0792599 vertex_buffer: 3.545063 vertex_buffer: 0.393362 vertex_buffer: 0.403706 - vertex_buffer: -3.3600674 - vertex_buffer: 3.3302898 - vertex_buffer: 3.677475 + vertex_buffer: -3.3600717 + vertex_buffer: 3.3302975 + vertex_buffer: 3.6774712 vertex_buffer: 0.345234 vertex_buffer: 0.344011 - vertex_buffer: -2.6583462 - vertex_buffer: 3.3343391 - vertex_buffer: 3.6318817 + vertex_buffer: -2.6583493 + vertex_buffer: 3.3343468 + vertex_buffer: 3.631878 vertex_buffer: 0.370094 vertex_buffer: 0.346076 - vertex_buffer: -4.0172 - vertex_buffer: 3.2417145 - vertex_buffer: 3.5189857 + vertex_buffer: -4.017204 + vertex_buffer: 3.241726 + vertex_buffer: 3.5189743 vertex_buffer: 0.319322 vertex_buffer: 0.347265 - vertex_buffer: -4.4652123 - vertex_buffer: 3.105711 - vertex_buffer: 3.2689667 + vertex_buffer: -4.465218 + vertex_buffer: 3.1057186 + vertex_buffer: 3.2689629 vertex_buffer: 0.297903 vertex_buffer: 0.353591 - vertex_buffer: -5.148801 - vertex_buffer: 2.13056 - vertex_buffer: 2.6144447 + vertex_buffer: -5.1488075 + vertex_buffer: 2.1305676 + vertex_buffer: 2.614441 vertex_buffer: 0.247792 vertex_buffer: 0.41081 - vertex_buffer: -2.5560007 - vertex_buffer: -8.121601 - vertex_buffer: 4.312229 + vertex_buffer: -2.5560045 + vertex_buffer: -8.121606 + vertex_buffer: 4.3122253 vertex_buffer: 0.396889 vertex_buffer: 0.842755 - vertex_buffer: -4.4293623 - vertex_buffer: 2.7792645 - vertex_buffer: 2.6971169 + vertex_buffer: -4.429368 + vertex_buffer: 2.779272 + vertex_buffer: 2.6971092 vertex_buffer: 0.280098 vertex_buffer: 0.3756 - vertex_buffer: -7.3019996 - vertex_buffer: 3.2599068 - vertex_buffer: -0.24933624 + vertex_buffer: -7.3020077 + vertex_buffer: 3.2599182 + vertex_buffer: -0.24934387 vertex_buffer: 0.10631 vertex_buffer: 0.399956 - vertex_buffer: -5.9285183 - vertex_buffer: 2.6113186 - vertex_buffer: 1.935009 + vertex_buffer: -5.9285254 + vertex_buffer: 2.6113281 + vertex_buffer: 1.9350014 vertex_buffer: 0.209925 vertex_buffer: 0.391353 - vertex_buffer: -3.3364365 - vertex_buffer: -0.7940159 - vertex_buffer: 4.6608505 + vertex_buffer: -3.336441 + vertex_buffer: -0.79401016 + vertex_buffer: 4.6608467 vertex_buffer: 0.355808 vertex_buffer: 0.534406 - vertex_buffer: -1.073098 - vertex_buffer: -3.7692108 + vertex_buffer: -1.0730996 + vertex_buffer: -3.769209 vertex_buffer: 6.153408 vertex_buffer: 0.471751 vertex_buffer: 0.650404 - vertex_buffer: -1.0261879 + vertex_buffer: -1.0261894 vertex_buffer: -4.1541195 - vertex_buffer: 5.910328 + vertex_buffer: 5.910324 vertex_buffer: 0.474155 vertex_buffer: 0.680192 - vertex_buffer: -2.0449853 + vertex_buffer: -2.0449882 vertex_buffer: -3.767992 - vertex_buffer: 5.6874237 + vertex_buffer: 5.687416 vertex_buffer: 0.439785 vertex_buffer: 0.657229 - vertex_buffer: -2.7272358 - vertex_buffer: -3.7194214 - vertex_buffer: 4.9273148 + vertex_buffer: -2.7272398 + vertex_buffer: -3.7194176 + vertex_buffer: 4.927311 vertex_buffer: 0.414617 vertex_buffer: 0.666541 - vertex_buffer: -1.8418827 - vertex_buffer: -4.0302353 - vertex_buffer: 5.497223 + vertex_buffer: -1.8418853 + vertex_buffer: -4.0302334 + vertex_buffer: 5.497219 vertex_buffer: 0.450374 vertex_buffer: 0.680861 - vertex_buffer: -2.4798682 - vertex_buffer: -3.8946476 - vertex_buffer: 4.8089104 + vertex_buffer: -2.4798715 + vertex_buffer: -3.8946457 + vertex_buffer: 4.808899 vertex_buffer: 0.428771 vertex_buffer: 0.682691 - vertex_buffer: -3.4621422 + vertex_buffer: -3.4621468 vertex_buffer: -4.7935104 - vertex_buffer: 3.6339455 + vertex_buffer: 3.6339378 vertex_buffer: 0.374971 vertex_buffer: 0.727805 - vertex_buffer: -0.6082711 - vertex_buffer: -2.526701 + vertex_buffer: -0.6082723 + vertex_buffer: -2.526699 vertex_buffer: 7.89365 vertex_buffer: 0.486717 vertex_buffer: 0.547629 - vertex_buffer: -0.62934816 + vertex_buffer: -0.62934923 vertex_buffer: -1.9188957 vertex_buffer: 8.013584 vertex_buffer: 0.485301 vertex_buffer: 0.527395 - vertex_buffer: -5.3245864 - vertex_buffer: 3.4533978 - vertex_buffer: 3.129799 + vertex_buffer: -5.324592 + vertex_buffer: 3.4534073 + vertex_buffer: 3.129795 vertex_buffer: 0.257765 vertex_buffer: 0.31449 - vertex_buffer: -1.8589399 - vertex_buffer: 0.70103836 - vertex_buffer: 4.5687027 + vertex_buffer: -1.8589423 + vertex_buffer: 0.701046 + vertex_buffer: 4.568695 vertex_buffer: 0.401223 vertex_buffer: 0.455172 - vertex_buffer: -2.21105 - vertex_buffer: -1.6402969 + vertex_buffer: -2.211053 + vertex_buffer: -1.6402931 vertex_buffer: 6.246376 vertex_buffer: 0.429819 vertex_buffer: 0.548615 - vertex_buffer: -2.2373412 - vertex_buffer: -1.2092209 - vertex_buffer: 5.9900475 + vertex_buffer: -2.2373445 + vertex_buffer: -1.209219 + vertex_buffer: 5.9900436 vertex_buffer: 0.421352 vertex_buffer: 0.533741 - vertex_buffer: -5.2615614 - vertex_buffer: -0.38985252 - vertex_buffer: 3.8133812 + vertex_buffer: -5.2615685 + vertex_buffer: -0.3898449 + vertex_buffer: 3.8133736 vertex_buffer: 0.276896 vertex_buffer: 0.532057 - vertex_buffer: -0.57935333 - vertex_buffer: -1.0133495 + vertex_buffer: -0.5793542 + vertex_buffer: -1.0133476 vertex_buffer: 7.52845 vertex_buffer: 0.48337 vertex_buffer: 0.499587 - vertex_buffer: -3.619615 - vertex_buffer: 3.6594543 - vertex_buffer: 4.1930656 + vertex_buffer: -3.6196196 + vertex_buffer: 3.6594658 + vertex_buffer: 4.193058 vertex_buffer: 0.337212 vertex_buffer: 0.282883 - vertex_buffer: -4.5898757 - vertex_buffer: 3.6197815 - vertex_buffer: 3.7516823 + vertex_buffer: -4.589881 + vertex_buffer: 3.619793 + vertex_buffer: 3.7516785 vertex_buffer: 0.296392 vertex_buffer: 0.293243 - vertex_buffer: -6.3526697 - vertex_buffer: 7.150507 - vertex_buffer: 0.71819305 + vertex_buffer: -6.352678 + vertex_buffer: 7.150524 + vertex_buffer: 0.7181854 vertex_buffer: 0.169295 vertex_buffer: 0.193814 - vertex_buffer: -1.0129352 - vertex_buffer: 3.025011 + vertex_buffer: -1.0129362 + vertex_buffer: 3.0250225 vertex_buffer: 4.796524 vertex_buffer: 0.44758 vertex_buffer: 0.30261 - vertex_buffer: -2.0709233 - vertex_buffer: 3.2252083 - vertex_buffer: 3.4418488 + vertex_buffer: -2.0709255 + vertex_buffer: 3.2252178 + vertex_buffer: 3.4418411 vertex_buffer: 0.39239 vertex_buffer: 0.353888 - vertex_buffer: -4.02202 - vertex_buffer: -3.7557068 - vertex_buffer: 3.5507584 + vertex_buffer: -4.0220246 + vertex_buffer: -3.7557049 + vertex_buffer: 3.5507507 vertex_buffer: 0.35449 vertex_buffer: 0.696784 - vertex_buffer: -6.623052 - vertex_buffer: -3.412384 - vertex_buffer: -0.97278595 + vertex_buffer: -6.6230597 + vertex_buffer: -3.4123821 + vertex_buffer: -0.9727936 vertex_buffer: 0.067305 vertex_buffer: 0.730105 - vertex_buffer: -1.7937524 - vertex_buffer: -1.979456 - vertex_buffer: 6.017292 + vertex_buffer: -1.7937549 + vertex_buffer: -1.979454 + vertex_buffer: 6.017288 vertex_buffer: 0.442739 vertex_buffer: 0.572826 - vertex_buffer: -1.2062981 + vertex_buffer: -1.2062998 vertex_buffer: -2.2445698 vertex_buffer: 6.1706963 vertex_buffer: 0.457098 vertex_buffer: 0.584792 - vertex_buffer: -3.57981 - vertex_buffer: -3.7382832 - vertex_buffer: 3.4588318 + vertex_buffer: -3.5798144 + vertex_buffer: -3.7382813 + vertex_buffer: 3.458828 vertex_buffer: 0.381974 vertex_buffer: 0.694711 - vertex_buffer: -3.3970556 - vertex_buffer: -3.7821655 - vertex_buffer: 3.5801582 + vertex_buffer: -3.3970602 + vertex_buffer: -3.7821636 + vertex_buffer: 3.5801468 vertex_buffer: 0.392389 vertex_buffer: 0.694203 - vertex_buffer: -4.9694285 - vertex_buffer: 4.1729107 - vertex_buffer: 3.340042 + vertex_buffer: -4.9694343 + vertex_buffer: 4.172926 + vertex_buffer: 3.3400345 vertex_buffer: 0.277076 vertex_buffer: 0.271932 - vertex_buffer: -2.315681 + vertex_buffer: -2.3156843 vertex_buffer: -1.7466087 - vertex_buffer: 5.9153595 + vertex_buffer: 5.9153557 vertex_buffer: 0.422552 vertex_buffer: 0.563233 - vertex_buffer: -2.4251802 - vertex_buffer: 3.4840698 - vertex_buffer: 4.546856 + vertex_buffer: -2.4251833 + vertex_buffer: 3.4840775 + vertex_buffer: 4.5468445 vertex_buffer: 0.385919 vertex_buffer: 0.281364 - vertex_buffer: -2.5284352 - vertex_buffer: 4.002569 - vertex_buffer: 4.621155 + vertex_buffer: -2.5284379 + vertex_buffer: 4.0025826 + vertex_buffer: 4.621151 vertex_buffer: 0.383103 vertex_buffer: 0.25584 - vertex_buffer: -3.5193088 - vertex_buffer: 7.9853344 - vertex_buffer: 2.9795952 + vertex_buffer: -3.5193126 + vertex_buffer: 7.9853497 + vertex_buffer: 2.9795876 vertex_buffer: 0.331431 vertex_buffer: 0.119714 - vertex_buffer: -5.6371226 - vertex_buffer: 5.7293034 - vertex_buffer: 2.1390076 + vertex_buffer: -5.6371293 + vertex_buffer: 5.7293186 + vertex_buffer: 2.1390038 vertex_buffer: 0.229924 vertex_buffer: 0.232003 - vertex_buffer: -2.971123 - vertex_buffer: 6.184988 + vertex_buffer: -2.9711266 + vertex_buffer: 6.1849995 vertex_buffer: 3.8050308 vertex_buffer: 0.364501 vertex_buffer: 0.189114 - vertex_buffer: -5.761087 - vertex_buffer: 3.9773064 - vertex_buffer: 2.541832 + vertex_buffer: -5.761094 + vertex_buffer: 3.9773197 + vertex_buffer: 2.5418282 vertex_buffer: 0.229622 vertex_buffer: 0.299541 - vertex_buffer: -6.4531307 - vertex_buffer: 5.1509743 - vertex_buffer: 1.08535 + vertex_buffer: -6.453139 + vertex_buffer: 5.1509914 + vertex_buffer: 1.0853424 vertex_buffer: 0.173287 vertex_buffer: 0.278748 - vertex_buffer: -1.0653346 - vertex_buffer: -3.9987717 - vertex_buffer: 6.0787888 + vertex_buffer: -1.0653361 + vertex_buffer: -3.9987736 + vertex_buffer: 6.078785 vertex_buffer: 0.472879 vertex_buffer: 0.666198 - vertex_buffer: -1.9331014 + vertex_buffer: -1.9331044 vertex_buffer: -3.916172 - vertex_buffer: 5.592533 + vertex_buffer: 5.5925255 vertex_buffer: 0.446828 vertex_buffer: 0.668527 - vertex_buffer: -2.5978663 + vertex_buffer: -2.5978699 vertex_buffer: -3.812233 - vertex_buffer: 4.909237 + vertex_buffer: 4.9092293 vertex_buffer: 0.422762 vertex_buffer: 0.67389 - vertex_buffer: -1.6448693 - vertex_buffer: -2.058361 - vertex_buffer: 5.9337997 + vertex_buffer: -1.6448718 + vertex_buffer: -2.0583591 + vertex_buffer: 5.933792 vertex_buffer: 0.445308 vertex_buffer: 0.580066 - vertex_buffer: -3.4851413 - vertex_buffer: -3.7586155 - vertex_buffer: 3.5274696 + vertex_buffer: -3.4851453 + vertex_buffer: -3.7586174 + vertex_buffer: 3.5274658 vertex_buffer: 0.388103 vertex_buffer: 0.693961 - vertex_buffer: -3.066307 + vertex_buffer: -3.066311 vertex_buffer: -4.4653883 - vertex_buffer: 3.8333206 + vertex_buffer: 3.8333168 vertex_buffer: 0.403039 vertex_buffer: 0.70654 - vertex_buffer: -3.3134317 - vertex_buffer: -3.8092175 - vertex_buffer: 3.5993462 + vertex_buffer: -3.313436 + vertex_buffer: -3.8092194 + vertex_buffer: 3.5993423 vertex_buffer: 0.403629 vertex_buffer: 0.693953 - vertex_buffer: -1.3705826 - vertex_buffer: -2.1870499 + vertex_buffer: -1.3705846 + vertex_buffer: -2.187048 vertex_buffer: 6.929592 vertex_buffer: 0.460042 vertex_buffer: 0.557139 - vertex_buffer: -2.3757484 + vertex_buffer: -2.3757515 vertex_buffer: -3.9299736 - vertex_buffer: 4.7485847 + vertex_buffer: 4.748581 vertex_buffer: 0.431158 vertex_buffer: 0.692366 - vertex_buffer: -1.7447376 + vertex_buffer: -1.7447399 vertex_buffer: -4.0353317 - vertex_buffer: 5.2845726 + vertex_buffer: 5.284565 vertex_buffer: 0.452182 vertex_buffer: 0.692366 - vertex_buffer: -0.97020155 - vertex_buffer: -4.186224 - vertex_buffer: 5.687355 + vertex_buffer: -0.97020316 + vertex_buffer: -4.186222 + vertex_buffer: 5.687351 vertex_buffer: 0.475387 vertex_buffer: 0.692366 - vertex_buffer: -1.1198573 - vertex_buffer: -7.185463 - vertex_buffer: 4.9469795 + vertex_buffer: -1.119859 + vertex_buffer: -7.185467 + vertex_buffer: 4.946968 vertex_buffer: 0.465828 vertex_buffer: 0.77919 - vertex_buffer: -1.040087 - vertex_buffer: -6.4274054 - vertex_buffer: 5.3075333 + vertex_buffer: -1.0400888 + vertex_buffer: -6.427408 + vertex_buffer: 5.3075294 vertex_buffer: 0.472329 vertex_buffer: 0.736226 - vertex_buffer: -1.0275493 - vertex_buffer: -6.0652637 - vertex_buffer: 5.31258 + vertex_buffer: -1.027551 + vertex_buffer: -6.0652666 + vertex_buffer: 5.3125763 vertex_buffer: 0.473087 vertex_buffer: 0.717857 - vertex_buffer: -0.9899579 + vertex_buffer: -0.9899597 vertex_buffer: -5.712208 - vertex_buffer: 5.0865936 + vertex_buffer: 5.086586 vertex_buffer: 0.473122 vertex_buffer: 0.704626 - vertex_buffer: -0.922419 - vertex_buffer: -5.449852 - vertex_buffer: 4.9381943 + vertex_buffer: -0.9224205 + vertex_buffer: -5.449853 + vertex_buffer: 4.938183 vertex_buffer: 0.473033 vertex_buffer: 0.695278 - vertex_buffer: -2.3016758 + vertex_buffer: -2.3016791 vertex_buffer: -4.864131 - vertex_buffer: 4.200348 + vertex_buffer: 4.2003365 vertex_buffer: 0.427942 vertex_buffer: 0.695278 - vertex_buffer: -2.4079256 - vertex_buffer: -4.9618464 - vertex_buffer: 4.3218575 + vertex_buffer: -2.4079297 + vertex_buffer: -4.9618473 + vertex_buffer: 4.32185 vertex_buffer: 0.426479 vertex_buffer: 0.70354 - vertex_buffer: -2.539156 - vertex_buffer: -5.145029 - vertex_buffer: 4.4079857 + vertex_buffer: -2.5391598 + vertex_buffer: -5.145028 + vertex_buffer: 4.407978 vertex_buffer: 0.423162 vertex_buffer: 0.711846 - vertex_buffer: -2.626017 + vertex_buffer: -2.626021 vertex_buffer: -5.3240175 - vertex_buffer: 4.311016 + vertex_buffer: 4.3110046 vertex_buffer: 0.418309 vertex_buffer: 0.720063 - vertex_buffer: -3.234158 - vertex_buffer: -3.0173168 - vertex_buffer: 4.740303 + vertex_buffer: -3.2341623 + vertex_buffer: -3.017315 + vertex_buffer: 4.740299 vertex_buffer: 0.390095 vertex_buffer: 0.639573 - vertex_buffer: -7.579906 - vertex_buffer: 0.5933876 - vertex_buffer: -2.304924 + vertex_buffer: -7.5799155 + vertex_buffer: 0.59339714 + vertex_buffer: -2.3049393 vertex_buffer: 0.013954 vertex_buffer: 0.560034 - vertex_buffer: -0.07802668 - vertex_buffer: -2.6862774 - vertex_buffer: 6.828209 + vertex_buffer: -0.07802707 + vertex_buffer: -2.6862755 + vertex_buffer: 6.828205 vertex_buffer: 0.499914 vertex_buffer: 0.580147 - vertex_buffer: -2.7698386 - vertex_buffer: -4.4213257 - vertex_buffer: 3.7819366 + vertex_buffer: -2.7698421 + vertex_buffer: -4.421324 + vertex_buffer: 3.7819328 vertex_buffer: 0.4132 vertex_buffer: 0.6954 - vertex_buffer: -2.9227178 - vertex_buffer: -4.4475775 - vertex_buffer: 3.845951 + vertex_buffer: -2.9227223 + vertex_buffer: -4.4475765 + vertex_buffer: 3.8459435 vertex_buffer: 0.409626 vertex_buffer: 0.701823 - vertex_buffer: -1.0003803 + vertex_buffer: -1.0003818 vertex_buffer: -2.492979 vertex_buffer: 6.198204 vertex_buffer: 0.46808 vertex_buffer: 0.601535 - vertex_buffer: -2.1424727 - vertex_buffer: -1.9390526 - vertex_buffer: 5.46101 + vertex_buffer: -2.1424758 + vertex_buffer: -1.9390469 + vertex_buffer: 5.461006 vertex_buffer: 0.422729 vertex_buffer: 0.585985 - vertex_buffer: -1.0979404 - vertex_buffer: -2.3909836 - vertex_buffer: 6.2144356 + vertex_buffer: -1.097942 + vertex_buffer: -2.3909798 + vertex_buffer: 6.214432 vertex_buffer: 0.46308 vertex_buffer: 0.593784 - vertex_buffer: -2.5645185 - vertex_buffer: 0.45451736 - vertex_buffer: 4.4030495 + vertex_buffer: -2.5645218 + vertex_buffer: 0.45452118 + vertex_buffer: 4.403042 vertex_buffer: 0.37212 vertex_buffer: 0.473414 - vertex_buffer: -3.6344466 - vertex_buffer: 0.15174484 - vertex_buffer: 4.2981873 + vertex_buffer: -3.6344514 + vertex_buffer: 0.15174866 + vertex_buffer: 4.2981834 vertex_buffer: 0.334562 vertex_buffer: 0.496073 - vertex_buffer: -2.4170141 - vertex_buffer: -1.3526745 - vertex_buffer: 5.642761 + vertex_buffer: -2.4170175 + vertex_buffer: -1.3526707 + vertex_buffer: 5.6427574 vertex_buffer: 0.411671 vertex_buffer: 0.546965 - vertex_buffer: -5.1738615 - vertex_buffer: 7.6968765 - vertex_buffer: 1.9025688 + vertex_buffer: -5.1738667 + vertex_buffer: 7.696892 + vertex_buffer: 1.9025612 vertex_buffer: 0.242176 vertex_buffer: 0.147676 - vertex_buffer: -4.5046306 - vertex_buffer: 6.0956154 - vertex_buffer: 3.0134354 + vertex_buffer: -4.5046363 + vertex_buffer: 6.0956287 + vertex_buffer: 3.0134315 vertex_buffer: 0.290777 vertex_buffer: 0.201446 - vertex_buffer: -3.840843 - vertex_buffer: 4.263878 - vertex_buffer: 3.9735641 + vertex_buffer: -3.8408475 + vertex_buffer: 4.2638893 + vertex_buffer: 3.9735603 vertex_buffer: 0.327338 vertex_buffer: 0.256527 - vertex_buffer: -2.8679767 - vertex_buffer: -5.7141867 - vertex_buffer: 4.0249977 + vertex_buffer: -2.8679805 + vertex_buffer: -5.7141886 + vertex_buffer: 4.02499 vertex_buffer: 0.39951 vertex_buffer: 0.748921 - vertex_buffer: -1.1987953 - vertex_buffer: 3.787548 + vertex_buffer: -1.1987965 + vertex_buffer: 3.7875595 vertex_buffer: 4.9401436 vertex_buffer: 0.441728 vertex_buffer: 0.261676 - vertex_buffer: -1.5314405 - vertex_buffer: 6.164812 - vertex_buffer: 4.248989 + vertex_buffer: -1.5314423 + vertex_buffer: 6.1648254 + vertex_buffer: 4.2489853 vertex_buffer: 0.429765 vertex_buffer: 0.187834 - vertex_buffer: -1.8419411 - vertex_buffer: 8.065584 + vertex_buffer: -1.8419428 + vertex_buffer: 8.065599 vertex_buffer: 3.6402245 vertex_buffer: 0.412198 vertex_buffer: 0.108901 - vertex_buffer: -4.211664 - vertex_buffer: 2.1525154 - vertex_buffer: 3.1534004 + vertex_buffer: -4.211669 + vertex_buffer: 2.1525269 + vertex_buffer: 3.1533928 vertex_buffer: 0.288955 vertex_buffer: 0.398952 - vertex_buffer: -5.888899 - vertex_buffer: 1.8484287 - vertex_buffer: 2.299553 + vertex_buffer: -5.888906 + vertex_buffer: 1.8484383 + vertex_buffer: 2.2995453 vertex_buffer: 0.218937 vertex_buffer: 0.435411 - vertex_buffer: -1.6175318 - vertex_buffer: 2.2701035 - vertex_buffer: 3.4401398 + vertex_buffer: -1.6175336 + vertex_buffer: 2.270111 + vertex_buffer: 3.440136 vertex_buffer: 0.412782 vertex_buffer: 0.39897 - vertex_buffer: -5.159606 - vertex_buffer: 3.0025787 - vertex_buffer: 2.737915 + vertex_buffer: -5.1596127 + vertex_buffer: 3.0025902 + vertex_buffer: 2.7379074 vertex_buffer: 0.257135 vertex_buffer: 0.35544 - vertex_buffer: -1.3773215 - vertex_buffer: 0.9951744 - vertex_buffer: 4.782261 + vertex_buffer: -1.377323 + vertex_buffer: 0.9951782 + vertex_buffer: 4.782257 vertex_buffer: 0.427685 vertex_buffer: 0.437961 - vertex_buffer: -1.7201958 - vertex_buffer: -1.7159519 + vertex_buffer: -1.720198 + vertex_buffer: -1.7159481 vertex_buffer: 6.867691 vertex_buffer: 0.44834 vertex_buffer: 0.536936 - vertex_buffer: -6.6515293 - vertex_buffer: 1.5498962 - vertex_buffer: 1.696167 + vertex_buffer: -6.6515374 + vertex_buffer: 1.5499058 + vertex_buffer: 1.6961594 vertex_buffer: 0.17856 vertex_buffer: 0.457554 - vertex_buffer: -5.4059763 - vertex_buffer: 1.3487549 - vertex_buffer: 2.9207191 + vertex_buffer: -5.4059834 + vertex_buffer: 1.3487625 + vertex_buffer: 2.9207115 vertex_buffer: 0.247308 vertex_buffer: 0.457194 - vertex_buffer: -4.583771 - vertex_buffer: 1.0079613 - vertex_buffer: 3.5124168 + vertex_buffer: -4.583777 + vertex_buffer: 1.0079708 + vertex_buffer: 3.512413 vertex_buffer: 0.286267 vertex_buffer: 0.467675 - vertex_buffer: -3.3820121 - vertex_buffer: 0.9262314 - vertex_buffer: 3.8521194 + vertex_buffer: -3.3820162 + vertex_buffer: 0.926239 + vertex_buffer: 3.8521156 vertex_buffer: 0.332828 vertex_buffer: 0.460712 - vertex_buffer: -2.477497 - vertex_buffer: 1.0757065 - vertex_buffer: 3.953865 + vertex_buffer: -2.4775004 + vertex_buffer: 1.075716 + vertex_buffer: 3.9538612 vertex_buffer: 0.368756 vertex_buffer: 0.447207 - vertex_buffer: -1.8026321 - vertex_buffer: 1.2566261 - vertex_buffer: 4.115402 + vertex_buffer: -1.8026342 + vertex_buffer: 1.2566338 + vertex_buffer: 4.1153984 vertex_buffer: 0.398964 vertex_buffer: 0.432655 - vertex_buffer: -0.5538964 - vertex_buffer: 1.5615921 - vertex_buffer: 5.4430733 + vertex_buffer: -0.553897 + vertex_buffer: 1.5615997 + vertex_buffer: 5.4430695 vertex_buffer: 0.47641 vertex_buffer: 0.405806 - vertex_buffer: -6.6932607 - vertex_buffer: 0.20049477 - vertex_buffer: 2.0159836 + vertex_buffer: -6.6932683 + vertex_buffer: 0.20050049 + vertex_buffer: 2.015976 vertex_buffer: 0.189241 vertex_buffer: 0.523924 - vertex_buffer: -5.6793036 - vertex_buffer: 3.1260548 - vertex_buffer: 2.4527283 + vertex_buffer: -5.6793103 + vertex_buffer: 3.1260643 + vertex_buffer: 2.4527206 vertex_buffer: 0.228962 vertex_buffer: 0.348951 - vertex_buffer: -0.4061323 - vertex_buffer: -2.7714462 - vertex_buffer: 7.6199875 + vertex_buffer: -0.4061332 + vertex_buffer: -2.7714443 + vertex_buffer: 7.6199837 vertex_buffer: 0.490726 vertex_buffer: 0.562401 - vertex_buffer: -1.9291446 - vertex_buffer: -0.004880905 - vertex_buffer: 4.950474 + vertex_buffer: -1.929147 + vertex_buffer: -0.0048770905 + vertex_buffer: 4.95047 vertex_buffer: 0.40467 vertex_buffer: 0.485133 - vertex_buffer: -7.719159 - vertex_buffer: 3.9393787 - vertex_buffer: -2.2868347 + vertex_buffer: -7.7191677 + vertex_buffer: 3.939392 + vertex_buffer: -2.2868423 vertex_buffer: 0.019469 vertex_buffer: 0.401564 - vertex_buffer: -1.3079104 - vertex_buffer: 1.5567074 - vertex_buffer: 4.179184 + vertex_buffer: -1.307912 + vertex_buffer: 1.556715 + vertex_buffer: 4.1791763 vertex_buffer: 0.426243 vertex_buffer: 0.420431 - vertex_buffer: -2.5166872 - vertex_buffer: -1.1328773 - vertex_buffer: 4.9170837 + vertex_buffer: -2.5166903 + vertex_buffer: -1.1328754 + vertex_buffer: 4.91708 vertex_buffer: 0.396993 vertex_buffer: 0.548797 - vertex_buffer: -4.802832 - vertex_buffer: 2.761158 - vertex_buffer: 2.5843391 + vertex_buffer: -4.8028374 + vertex_buffer: 2.7611675 + vertex_buffer: 2.5843315 vertex_buffer: 0.26647 vertex_buffer: 0.376977 - vertex_buffer: -1.7231433 - vertex_buffer: -1.1866455 - vertex_buffer: 6.5593147 + vertex_buffer: -1.7231458 + vertex_buffer: -1.1866398 + vertex_buffer: 6.559307 vertex_buffer: 0.439121 vertex_buffer: 0.518958 - vertex_buffer: -7.232868 - vertex_buffer: -1.3291702 - vertex_buffer: -1.7797775 + vertex_buffer: -7.2328777 + vertex_buffer: -1.3291626 + vertex_buffer: -1.7797928 vertex_buffer: 0.032314 vertex_buffer: 0.644357 - vertex_buffer: -1.6283221 - vertex_buffer: 2.525772 - vertex_buffer: 3.2109795 + vertex_buffer: -1.6283238 + vertex_buffer: 2.5257816 + vertex_buffer: 3.2109756 vertex_buffer: 0.419054 vertex_buffer: 0.387155 - vertex_buffer: -1.1273737 - vertex_buffer: -1.0895672 - vertex_buffer: 7.164959 + vertex_buffer: -1.1273751 + vertex_buffer: -1.0895634 + vertex_buffer: 7.164955 vertex_buffer: 0.462783 vertex_buffer: 0.505747 - vertex_buffer: -5.139789 + vertex_buffer: -5.139796 vertex_buffer: -5.5858536 - vertex_buffer: 1.9622345 + vertex_buffer: 1.9622269 vertex_buffer: 0.238979 vertex_buffer: 0.779745 - vertex_buffer: -5.005826 + vertex_buffer: -5.005832 vertex_buffer: -6.508588 - vertex_buffer: 1.2313309 + vertex_buffer: 1.2313194 vertex_buffer: 0.198221 vertex_buffer: 0.831938 - vertex_buffer: -7.366962 - vertex_buffer: 0.3376236 - vertex_buffer: -0.16316223 + vertex_buffer: -7.3669715 + vertex_buffer: 0.33763313 + vertex_buffer: -0.16316986 vertex_buffer: 0.10755 vertex_buffer: 0.540755 - vertex_buffer: -5.9915476 - vertex_buffer: -4.4671774 - vertex_buffer: 1.3122635 + vertex_buffer: -5.9915557 + vertex_buffer: -4.4671764 + vertex_buffer: 1.3122559 vertex_buffer: 0.18361 vertex_buffer: 0.740257 - vertex_buffer: -6.912982 - vertex_buffer: 4.4011097 - vertex_buffer: 0.22525787 + vertex_buffer: -6.91299 + vertex_buffer: 4.401125 + vertex_buffer: 0.22525024 vertex_buffer: 0.13441 vertex_buffer: 0.333683 - vertex_buffer: -2.616919 - vertex_buffer: -9.00794 - vertex_buffer: 4.171379 + vertex_buffer: -2.6169233 + vertex_buffer: -9.007946 + vertex_buffer: 4.1713715 vertex_buffer: 0.385764 vertex_buffer: 0.883154 - vertex_buffer: -0.33355132 - vertex_buffer: -2.6564388 - vertex_buffer: 6.7840347 + vertex_buffer: -0.3335521 + vertex_buffer: -2.656437 + vertex_buffer: 6.784031 vertex_buffer: 0.490967 vertex_buffer: 0.579378 - vertex_buffer: -2.5573 - vertex_buffer: -0.3414135 - vertex_buffer: 4.7602386 + vertex_buffer: -2.557304 + vertex_buffer: -0.34140968 + vertex_buffer: 4.760235 vertex_buffer: 0.382385 vertex_buffer: 0.508573 - vertex_buffer: -6.5110135 - vertex_buffer: 2.69775 - vertex_buffer: 1.3233795 + vertex_buffer: -6.511022 + vertex_buffer: 2.6977615 + vertex_buffer: 1.3233719 vertex_buffer: 0.174399 vertex_buffer: 0.397671 - vertex_buffer: -3.575697 - vertex_buffer: 2.4028053 - vertex_buffer: 3.2938614 + vertex_buffer: -3.5757012 + vertex_buffer: 2.4028168 + vertex_buffer: 3.2938538 vertex_buffer: 0.318785 vertex_buffer: 0.396235 - vertex_buffer: -3.063976 - vertex_buffer: 2.3301315 + vertex_buffer: -3.0639794 + vertex_buffer: 2.330141 vertex_buffer: 3.4520035 vertex_buffer: 0.343364 vertex_buffer: 0.400597 - vertex_buffer: -3.1599689 - vertex_buffer: -4.557003 - vertex_buffer: 3.7644958 + vertex_buffer: -3.1599734 + vertex_buffer: -4.557002 + vertex_buffer: 3.7644844 vertex_buffer: 0.3961 vertex_buffer: 0.710217 - vertex_buffer: -6.6972475 - vertex_buffer: -1.218895 - vertex_buffer: 1.9558678 + vertex_buffer: -6.6972566 + vertex_buffer: -1.2188911 + vertex_buffer: 1.9558601 vertex_buffer: 0.187885 vertex_buffer: 0.588538 - vertex_buffer: -1.4729099 - vertex_buffer: -10.254865 - vertex_buffer: 4.3090057 + vertex_buffer: -1.4729123 + vertex_buffer: -10.254872 + vertex_buffer: 4.308998 vertex_buffer: 0.430987 vertex_buffer: 0.944065 - vertex_buffer: -3.3332095 - vertex_buffer: -8.75649 - vertex_buffer: 2.8909073 + vertex_buffer: -3.3332138 + vertex_buffer: -8.756494 + vertex_buffer: 2.8909035 vertex_buffer: 0.318993 vertex_buffer: 0.898285 - vertex_buffer: -4.0895295 - vertex_buffer: -7.7846384 - vertex_buffer: 2.1102676 + vertex_buffer: -4.0895343 + vertex_buffer: -7.7846413 + vertex_buffer: 2.11026 vertex_buffer: 0.266248 vertex_buffer: 0.869701 - vertex_buffer: 0.09045166 - vertex_buffer: 6.096033 + vertex_buffer: 0.090452164 + vertex_buffer: 6.0960445 vertex_buffer: 4.42157 vertex_buffer: 0.500023 vertex_buffer: 0.190576 - vertex_buffer: -0.13197118 - vertex_buffer: -10.441204 - vertex_buffer: 4.505329 + vertex_buffer: -0.13197199 + vertex_buffer: -10.44121 + vertex_buffer: 4.5053177 vertex_buffer: 0.499977 vertex_buffer: 0.954453 - vertex_buffer: -2.5613708 - vertex_buffer: 2.3542728 - vertex_buffer: 3.479847 + vertex_buffer: -2.561374 + vertex_buffer: 2.3542805 + vertex_buffer: 3.4798431 vertex_buffer: 0.36617 vertex_buffer: 0.398822 - vertex_buffer: -2.0838258 - vertex_buffer: 2.4268627 - vertex_buffer: 3.3835602 + vertex_buffer: -2.0838282 + vertex_buffer: 2.4268703 + vertex_buffer: 3.3835564 vertex_buffer: 0.393207 vertex_buffer: 0.395537 - vertex_buffer: -1.7622774 - vertex_buffer: 2.4867592 - vertex_buffer: 3.2451553 + vertex_buffer: -1.762279 + vertex_buffer: 2.4867668 + vertex_buffer: 3.2451515 vertex_buffer: 0.410373 vertex_buffer: 0.39108 - vertex_buffer: -6.192421 - vertex_buffer: 3.5102692 - vertex_buffer: 1.7944756 + vertex_buffer: -6.192429 + vertex_buffer: 3.5102787 + vertex_buffer: 1.7944717 vertex_buffer: 0.194993 vertex_buffer: 0.342102 - vertex_buffer: -2.2021916 - vertex_buffer: 2.8239002 - vertex_buffer: 3.3703003 + vertex_buffer: -2.202194 + vertex_buffer: 2.8239098 + vertex_buffer: 3.3702965 vertex_buffer: 0.388665 vertex_buffer: 0.362284 - vertex_buffer: -2.7161317 - vertex_buffer: 2.9166946 + vertex_buffer: -2.716135 + vertex_buffer: 2.9167042 vertex_buffer: 3.4998627 vertex_buffer: 0.365962 vertex_buffer: 0.355971 - vertex_buffer: -3.1931787 - vertex_buffer: 2.9413223 + vertex_buffer: -3.1931822 + vertex_buffer: 2.9413319 vertex_buffer: 3.4861336 vertex_buffer: 0.343364 vertex_buffer: 0.355357 - vertex_buffer: -3.6920981 - vertex_buffer: 2.897976 + vertex_buffer: -3.6921024 + vertex_buffer: 2.8979855 vertex_buffer: 3.3511238 vertex_buffer: 0.318785 vertex_buffer: 0.35834 - vertex_buffer: -4.030469 - vertex_buffer: 2.838482 + vertex_buffer: -4.0304737 + vertex_buffer: 2.8384895 vertex_buffer: 3.1473236 vertex_buffer: 0.301415 vertex_buffer: 0.363156 - vertex_buffer: -7.534716 - vertex_buffer: 5.377041 - vertex_buffer: -1.4887314 + vertex_buffer: -7.5347247 + vertex_buffer: 5.377056 + vertex_buffer: -1.4887466 vertex_buffer: 0.058133 vertex_buffer: 0.319076 - vertex_buffer: -3.9188557 - vertex_buffer: 2.5209522 - vertex_buffer: 3.0982513 + vertex_buffer: -3.918861 + vertex_buffer: 2.5209599 + vertex_buffer: 3.0982437 vertex_buffer: 0.301415 vertex_buffer: 0.387449 - vertex_buffer: -0.056284398 - vertex_buffer: -3.0145187 + vertex_buffer: -0.05628477 + vertex_buffer: -3.0145168 vertex_buffer: 6.172512 vertex_buffer: 0.499988 vertex_buffer: 0.618434 - vertex_buffer: -2.4927456 - vertex_buffer: -2.8486595 - vertex_buffer: 5.3483696 + vertex_buffer: -2.4927492 + vertex_buffer: -2.8486557 + vertex_buffer: 5.348366 vertex_buffer: 0.415838 vertex_buffer: 0.624196 - vertex_buffer: -1.7608254 - vertex_buffer: -1.9913807 - vertex_buffer: 6.25832 + vertex_buffer: -1.7608279 + vertex_buffer: -1.9913769 + vertex_buffer: 6.258316 vertex_buffer: 0.445682 vertex_buffer: 0.566077 - vertex_buffer: -1.0984011 - vertex_buffer: -2.9661674 - vertex_buffer: 6.087311 + vertex_buffer: -1.0984027 + vertex_buffer: -2.9661636 + vertex_buffer: 6.087303 vertex_buffer: 0.465844 vertex_buffer: 0.620641 - vertex_buffer: 0.034968972 - vertex_buffer: 2.4771328 - vertex_buffer: 5.1017494 + vertex_buffer: 0.03496912 + vertex_buffer: 2.4771404 + vertex_buffer: 5.1017456 vertex_buffer: 0.499923 vertex_buffer: 0.351524 - vertex_buffer: -4.2818146 - vertex_buffer: -6.8262396 - vertex_buffer: 2.5633278 + vertex_buffer: -4.2818203 + vertex_buffer: -6.8262415 + vertex_buffer: 2.5633202 vertex_buffer: 0.288719 vertex_buffer: 0.819946 - vertex_buffer: -3.4656432 - vertex_buffer: -7.906004 - vertex_buffer: 3.3282852 + vertex_buffer: -3.4656482 + vertex_buffer: -7.906007 + vertex_buffer: 3.3282738 vertex_buffer: 0.335279 vertex_buffer: 0.85282 - vertex_buffer: -1.483537 - vertex_buffer: -9.706334 - vertex_buffer: 4.882229 + vertex_buffer: -1.4835393 + vertex_buffer: -9.706341 + vertex_buffer: 4.882221 vertex_buffer: 0.440512 vertex_buffer: 0.902419 - vertex_buffer: -5.8149533 - vertex_buffer: -5.1643486 - vertex_buffer: 0.16642761 + vertex_buffer: -5.81496 + vertex_buffer: -5.1643467 + vertex_buffer: 0.16641235 vertex_buffer: 0.128294 vertex_buffer: 0.791941 - vertex_buffer: -1.818965 - vertex_buffer: 2.6498566 + vertex_buffer: -1.8189665 + vertex_buffer: 2.649866 vertex_buffer: 3.2792091 vertex_buffer: 0.408772 vertex_buffer: 0.373894 - vertex_buffer: -0.98224825 - vertex_buffer: 0.49705887 - vertex_buffer: 5.730484 + vertex_buffer: -0.9822496 + vertex_buffer: 0.4970646 + vertex_buffer: 5.73048 vertex_buffer: 0.455607 vertex_buffer: 0.451801 - vertex_buffer: -0.11441803 - vertex_buffer: -9.953 - vertex_buffer: 5.0419807 + vertex_buffer: -0.114418834 + vertex_buffer: -9.953008 + vertex_buffer: 5.041973 vertex_buffer: 0.499877 vertex_buffer: 0.90899 - vertex_buffer: -2.4964733 - vertex_buffer: -9.630109 - vertex_buffer: 3.7255173 + vertex_buffer: -2.4964771 + vertex_buffer: -9.630114 + vertex_buffer: 3.7255058 vertex_buffer: 0.375437 vertex_buffer: 0.924192 - vertex_buffer: -7.1446133 - vertex_buffer: -1.3387032 - vertex_buffer: 0.109550476 + vertex_buffer: -7.1446214 + vertex_buffer: -1.3386974 + vertex_buffer: 0.10954285 vertex_buffer: 0.11421 vertex_buffer: 0.615022 - vertex_buffer: -1.6615427 + vertex_buffer: -1.6615449 vertex_buffer: -5.223667 - vertex_buffer: 4.611267 + vertex_buffer: 4.6112633 vertex_buffer: 0.448662 vertex_buffer: 0.695278 - vertex_buffer: -1.7678626 - vertex_buffer: -5.3901997 - vertex_buffer: 4.7363663 + vertex_buffer: -1.7678653 + vertex_buffer: -5.3902016 + vertex_buffer: 4.7363586 vertex_buffer: 0.44802 vertex_buffer: 0.704632 - vertex_buffer: -1.8549478 - vertex_buffer: -5.672799 - vertex_buffer: 4.8468857 + vertex_buffer: -1.8549504 + vertex_buffer: -5.672801 + vertex_buffer: 4.846882 vertex_buffer: 0.447112 vertex_buffer: 0.715808 - vertex_buffer: -1.914007 - vertex_buffer: -5.997034 - vertex_buffer: 4.790905 + vertex_buffer: -1.9140096 + vertex_buffer: -5.997035 + vertex_buffer: 4.7908974 vertex_buffer: 0.444832 vertex_buffer: 0.730794 - vertex_buffer: -2.1178837 - vertex_buffer: -6.5948715 - vertex_buffer: 4.522606 + vertex_buffer: -2.1178868 + vertex_buffer: -6.5948734 + vertex_buffer: 4.5225983 vertex_buffer: 0.430012 vertex_buffer: 0.766809 - vertex_buffer: -3.028949 - vertex_buffer: -3.8013363 - vertex_buffer: 4.1697693 + vertex_buffer: -3.0289533 + vertex_buffer: -3.8013344 + vertex_buffer: 4.169758 vertex_buffer: 0.406787 vertex_buffer: 0.685673 - vertex_buffer: -3.1446614 + vertex_buffer: -3.1446655 vertex_buffer: -3.7468567 - vertex_buffer: 4.1317177 + vertex_buffer: 4.13171 vertex_buffer: 0.400738 vertex_buffer: 0.681069 - vertex_buffer: -3.2364228 + vertex_buffer: -3.2364273 vertex_buffer: -3.6879635 - vertex_buffer: 4.140358 + vertex_buffer: 4.1403503 vertex_buffer: 0.3924 vertex_buffer: 0.677703 - vertex_buffer: -3.808137 - vertex_buffer: -3.2578716 - vertex_buffer: 4.065346 + vertex_buffer: -3.808142 + vertex_buffer: -3.2578678 + vertex_buffer: 4.065342 vertex_buffer: 0.367856 vertex_buffer: 0.663919 - vertex_buffer: -5.896332 - vertex_buffer: -1.8010445 - vertex_buffer: 3.2329674 + vertex_buffer: -5.8963394 + vertex_buffer: -1.8010387 + vertex_buffer: 3.2329636 vertex_buffer: 0.247923 vertex_buffer: 0.601333 - vertex_buffer: -0.9425743 - vertex_buffer: 1.3286476 - vertex_buffer: 5.0629654 + vertex_buffer: -0.94257545 + vertex_buffer: 1.3286552 + vertex_buffer: 5.062958 vertex_buffer: 0.45277 vertex_buffer: 0.42085 - vertex_buffer: -1.1735758 - vertex_buffer: 2.8160534 - vertex_buffer: 3.6443062 + vertex_buffer: -1.1735771 + vertex_buffer: 2.816063 + vertex_buffer: 3.6442986 vertex_buffer: 0.436392 vertex_buffer: 0.359887 - vertex_buffer: -1.5453613 - vertex_buffer: 2.8436775 + vertex_buffer: -1.5453631 + vertex_buffer: 2.843687 vertex_buffer: 3.3741112 vertex_buffer: 0.416164 vertex_buffer: 0.368714 - vertex_buffer: -2.933605 - vertex_buffer: -3.8272305 - vertex_buffer: 4.132 + vertex_buffer: -2.933609 + vertex_buffer: -3.8272285 + vertex_buffer: 4.1319923 vertex_buffer: 0.413386 vertex_buffer: 0.692366 - vertex_buffer: -5.9479156 - vertex_buffer: -3.431078 - vertex_buffer: 2.3189926 + vertex_buffer: -5.947923 + vertex_buffer: -3.431076 + vertex_buffer: 2.318985 vertex_buffer: 0.228018 vertex_buffer: 0.683572 - vertex_buffer: -0.6934988 - vertex_buffer: 2.5053654 - vertex_buffer: 4.5957108 + vertex_buffer: -0.69349945 + vertex_buffer: 2.505373 + vertex_buffer: 4.595703 vertex_buffer: 0.468268 vertex_buffer: 0.352671 - vertex_buffer: -2.3358104 - vertex_buffer: -7.2656155 - vertex_buffer: 4.386078 + vertex_buffer: -2.3358138 + vertex_buffer: -7.2656193 + vertex_buffer: 4.386074 vertex_buffer: 0.411362 vertex_buffer: 0.804327 - vertex_buffer: -0.03143847 - vertex_buffer: -0.11379433 - vertex_buffer: 7.009609 + vertex_buffer: -0.03143871 + vertex_buffer: -0.113788605 + vertex_buffer: 7.0096054 vertex_buffer: 0.499989 vertex_buffer: 0.469825 - vertex_buffer: -0.53570485 - vertex_buffer: 0.67900276 - vertex_buffer: 6.160198 + vertex_buffer: -0.53570557 + vertex_buffer: 0.6790085 + vertex_buffer: 6.1601944 vertex_buffer: 0.479154 vertex_buffer: 0.442654 - vertex_buffer: -0.021373034 - vertex_buffer: 0.75474167 + vertex_buffer: -0.021373123 + vertex_buffer: 0.7547493 vertex_buffer: 6.3074684 vertex_buffer: 0.499974 vertex_buffer: 0.439637 - vertex_buffer: -1.508091 - vertex_buffer: -0.4657917 - vertex_buffer: 5.860111 + vertex_buffer: -1.5080929 + vertex_buffer: -0.4657898 + vertex_buffer: 5.8601074 vertex_buffer: 0.432112 vertex_buffer: 0.493589 - vertex_buffer: -0.10210103 - vertex_buffer: -9.131883 - vertex_buffer: 5.220871 + vertex_buffer: -0.1021018 + vertex_buffer: -9.131888 + vertex_buffer: 5.2208633 vertex_buffer: 0.499886 vertex_buffer: 0.866917 - vertex_buffer: -0.07414225 - vertex_buffer: -8.138553 - vertex_buffer: 5.1079254 + vertex_buffer: -0.07414287 + vertex_buffer: -8.138557 + vertex_buffer: 5.107918 vertex_buffer: 0.499913 vertex_buffer: 0.821729 - vertex_buffer: -1.2661052 - vertex_buffer: -7.9413004 - vertex_buffer: 4.9270973 + vertex_buffer: -1.2661073 + vertex_buffer: -7.941305 + vertex_buffer: 4.9270897 vertex_buffer: 0.456549 vertex_buffer: 0.819201 - vertex_buffer: -3.778433 - vertex_buffer: -5.121208 - vertex_buffer: 3.4297066 + vertex_buffer: -3.778438 + vertex_buffer: -5.12121 + vertex_buffer: 3.429699 vertex_buffer: 0.344549 vertex_buffer: 0.745439 - vertex_buffer: -3.004316 - vertex_buffer: -1.5952225 - vertex_buffer: 4.7329597 + vertex_buffer: -3.0043197 + vertex_buffer: -1.5952206 + vertex_buffer: 4.732956 vertex_buffer: 0.378909 vertex_buffer: 0.57401 - vertex_buffer: -3.1108878 - vertex_buffer: -6.3109074 - vertex_buffer: 3.8444405 + vertex_buffer: -3.110892 + vertex_buffer: -6.3109083 + vertex_buffer: 3.844429 vertex_buffer: 0.374293 vertex_buffer: 0.780185 - vertex_buffer: -4.4307637 - vertex_buffer: -1.4898453 - vertex_buffer: 4.4148483 + vertex_buffer: -4.4307694 + vertex_buffer: -1.4898396 + vertex_buffer: 4.4148407 vertex_buffer: 0.319688 vertex_buffer: 0.570738 - vertex_buffer: -3.635557 - vertex_buffer: -2.2050133 - vertex_buffer: 4.4749756 + vertex_buffer: -3.6355615 + vertex_buffer: -2.2050095 + vertex_buffer: 4.474972 vertex_buffer: 0.357155 vertex_buffer: 0.60427 - vertex_buffer: -5.041381 - vertex_buffer: -2.4032383 - vertex_buffer: 3.8168907 + vertex_buffer: -5.041387 + vertex_buffer: -2.4032345 + vertex_buffer: 3.816887 vertex_buffer: 0.295284 vertex_buffer: 0.621581 - vertex_buffer: -1.4315786 - vertex_buffer: -8.901685 - vertex_buffer: 5.0258293 + vertex_buffer: -1.4315811 + vertex_buffer: -8.9016905 + vertex_buffer: 5.0258217 vertex_buffer: 0.44775 vertex_buffer: 0.862477 - vertex_buffer: -1.9901543 - vertex_buffer: -0.60396385 - vertex_buffer: 5.4264526 + vertex_buffer: -1.9901569 + vertex_buffer: -0.60396004 + vertex_buffer: 5.426449 vertex_buffer: 0.410986 vertex_buffer: 0.508723 - vertex_buffer: -4.198247 - vertex_buffer: -5.8466797 - vertex_buffer: 3.0578117 + vertex_buffer: -4.1982517 + vertex_buffer: -5.8466806 + vertex_buffer: 3.057808 vertex_buffer: 0.313951 vertex_buffer: 0.775308 - vertex_buffer: -3.3484576 - vertex_buffer: -7.0401354 - vertex_buffer: 3.6005745 + vertex_buffer: -3.3484623 + vertex_buffer: -7.0401382 + vertex_buffer: 3.6005669 vertex_buffer: 0.354128 vertex_buffer: 0.812553 - vertex_buffer: -4.370291 - vertex_buffer: -4.006874 - vertex_buffer: 3.421772 + vertex_buffer: -4.3702974 + vertex_buffer: -4.006872 + vertex_buffer: 3.4217644 vertex_buffer: 0.324548 vertex_buffer: 0.703993 - vertex_buffer: -6.537344 - vertex_buffer: -2.4688148 - vertex_buffer: 1.8110313 + vertex_buffer: -6.537352 + vertex_buffer: -2.468813 + vertex_buffer: 1.8110199 vertex_buffer: 0.189096 vertex_buffer: 0.6463 - vertex_buffer: -5.0787134 - vertex_buffer: -4.265339 - vertex_buffer: 2.9516716 + vertex_buffer: -5.07872 + vertex_buffer: -4.265337 + vertex_buffer: 2.951664 vertex_buffer: 0.279777 vertex_buffer: 0.714658 - vertex_buffer: -6.688318 - vertex_buffer: -2.9515648 - vertex_buffer: 0.50354004 + vertex_buffer: -6.688326 + vertex_buffer: -2.951559 + vertex_buffer: 0.5035248 vertex_buffer: 0.133823 vertex_buffer: 0.682701 - vertex_buffer: -4.2454205 - vertex_buffer: -2.8282986 - vertex_buffer: 4.063446 + vertex_buffer: -4.2454257 + vertex_buffer: -2.8282967 + vertex_buffer: 4.0634384 vertex_buffer: 0.336768 vertex_buffer: 0.644733 - vertex_buffer: -1.4265988 - vertex_buffer: 0.2535839 + vertex_buffer: -1.4266007 + vertex_buffer: 0.25359154 vertex_buffer: 5.3043365 vertex_buffer: 0.429884 vertex_buffer: 0.466522 - vertex_buffer: -1.5445278 - vertex_buffer: -2.0910187 + vertex_buffer: -1.5445302 + vertex_buffer: -2.0910168 vertex_buffer: 7.042053 vertex_buffer: 0.455528 vertex_buffer: 0.548623 - vertex_buffer: -2.0375082 - vertex_buffer: -1.9200306 - vertex_buffer: 6.35569 + vertex_buffer: -2.0375109 + vertex_buffer: -1.9200287 + vertex_buffer: 6.355686 vertex_buffer: 0.437114 vertex_buffer: 0.558896 - vertex_buffer: -1.1873711 - vertex_buffer: -1.8289967 + vertex_buffer: -1.1873728 + vertex_buffer: -1.8289948 vertex_buffer: 7.5009995 vertex_buffer: 0.467288 vertex_buffer: 0.529925 - vertex_buffer: -1.5844951 - vertex_buffer: 3.1812801 - vertex_buffer: 3.7376862 + vertex_buffer: -1.5844966 + vertex_buffer: 3.1812916 + vertex_buffer: 3.7376823 vertex_buffer: 0.414712 vertex_buffer: 0.33522 - vertex_buffer: -2.5590115 - vertex_buffer: 3.4128933 - vertex_buffer: 3.864193 + vertex_buffer: -2.5590146 + vertex_buffer: 3.4129047 + vertex_buffer: 3.8641853 vertex_buffer: 0.377046 vertex_buffer: 0.322778 - vertex_buffer: -3.4523673 - vertex_buffer: 3.4629288 - vertex_buffer: 3.869194 + vertex_buffer: -3.4523714 + vertex_buffer: 3.4629383 + vertex_buffer: 3.8691864 vertex_buffer: 0.344108 vertex_buffer: 0.320151 - vertex_buffer: -4.252108 - vertex_buffer: 3.39855 - vertex_buffer: 3.6475868 + vertex_buffer: -4.2521133 + vertex_buffer: 3.3985596 + vertex_buffer: 3.647583 vertex_buffer: 0.312876 vertex_buffer: 0.322332 - vertex_buffer: -4.8403144 - vertex_buffer: 3.246275 - vertex_buffer: 3.259838 + vertex_buffer: -4.8403196 + vertex_buffer: 3.2462864 + vertex_buffer: 3.2598343 vertex_buffer: 0.283526 vertex_buffer: 0.33319 - vertex_buffer: -5.280643 - vertex_buffer: 2.652216 - vertex_buffer: 2.3635025 + vertex_buffer: -5.2806497 + vertex_buffer: 2.6522255 + vertex_buffer: 2.3634949 vertex_buffer: 0.241246 vertex_buffer: 0.382786 - vertex_buffer: -7.408005 - vertex_buffer: 1.8267059 - vertex_buffer: -0.24343109 + vertex_buffer: -7.408014 + vertex_buffer: 1.8267155 + vertex_buffer: -0.24343872 vertex_buffer: 0.102986 vertex_buffer: 0.468763 - vertex_buffer: -4.746617 - vertex_buffer: 1.8352318 - vertex_buffer: 2.945816 + vertex_buffer: -4.746623 + vertex_buffer: 1.8352432 + vertex_buffer: 2.9458122 vertex_buffer: 0.267612 vertex_buffer: 0.42456 - vertex_buffer: -4.077169 - vertex_buffer: 1.5753078 - vertex_buffer: 3.3490562 + vertex_buffer: -4.077174 + vertex_buffer: 1.5753174 + vertex_buffer: 3.3490486 vertex_buffer: 0.297879 vertex_buffer: 0.433176 - vertex_buffer: -3.2392907 - vertex_buffer: 1.4697704 - vertex_buffer: 3.6222115 + vertex_buffer: -3.2392948 + vertex_buffer: 1.4697781 + vertex_buffer: 3.6222038 vertex_buffer: 0.333434 vertex_buffer: 0.433878 - vertex_buffer: -2.4502244 - vertex_buffer: 1.5540104 - vertex_buffer: 3.6770782 + vertex_buffer: -2.4502277 + vertex_buffer: 1.554018 + vertex_buffer: 3.6770744 vertex_buffer: 0.366427 vertex_buffer: 0.426116 - vertex_buffer: -1.836082 - vertex_buffer: 1.712101 + vertex_buffer: -1.8360842 + vertex_buffer: 1.7121105 vertex_buffer: 3.73732 vertex_buffer: 0.396012 vertex_buffer: 0.416696 - vertex_buffer: -1.4331815 - vertex_buffer: 1.8971863 + vertex_buffer: -1.4331831 + vertex_buffer: 1.897192 vertex_buffer: 3.7476387 vertex_buffer: 0.420121 vertex_buffer: 0.410228 - vertex_buffer: -7.6809406 - vertex_buffer: 2.3514805 - vertex_buffer: -2.5189133 + vertex_buffer: -7.68095 + vertex_buffer: 2.35149 + vertex_buffer: -2.5189285 vertex_buffer: 0.007561 vertex_buffer: 0.480777 - vertex_buffer: -2.060004 + vertex_buffer: -2.060007 vertex_buffer: -1.9369621 - vertex_buffer: 6.0419006 + vertex_buffer: 6.041897 vertex_buffer: 0.432949 vertex_buffer: 0.569518 - vertex_buffer: -1.007299 - vertex_buffer: -0.26156425 + vertex_buffer: -1.0073003 + vertex_buffer: -0.26155853 vertex_buffer: 6.350731 vertex_buffer: 0.458639 vertex_buffer: 0.479089 - vertex_buffer: -1.0573628 - vertex_buffer: -2.3342552 - vertex_buffer: 7.56081 + vertex_buffer: -1.0573645 + vertex_buffer: -2.3342514 + vertex_buffer: 7.5608063 vertex_buffer: 0.473466 vertex_buffer: 0.545744 - vertex_buffer: -0.734902 + vertex_buffer: -0.73490334 vertex_buffer: -2.5882797 vertex_buffer: 7.3467026 vertex_buffer: 0.476088 vertex_buffer: 0.56383 - vertex_buffer: -1.0897002 - vertex_buffer: -2.3481636 + vertex_buffer: -1.0897019 + vertex_buffer: -2.3481655 vertex_buffer: 7.254524 vertex_buffer: 0.468472 vertex_buffer: 0.555057 - vertex_buffer: -1.9190764 - vertex_buffer: -2.0415 - vertex_buffer: 5.8397217 + vertex_buffer: -1.9190793 + vertex_buffer: -2.0414982 + vertex_buffer: 5.839718 vertex_buffer: 0.433991 vertex_buffer: 0.582362 - vertex_buffer: -0.6197953 - vertex_buffer: -2.7030697 + vertex_buffer: -0.6197964 + vertex_buffer: -2.7030678 vertex_buffer: 7.5222588 vertex_buffer: 0.483518 vertex_buffer: 0.562984 - vertex_buffer: -0.55812526 + vertex_buffer: -0.5581263 vertex_buffer: -2.5777187 vertex_buffer: 6.7721367 vertex_buffer: 0.482483 vertex_buffer: 0.577849 - vertex_buffer: -1.4102633 - vertex_buffer: 2.4753132 + vertex_buffer: -1.4102648 + vertex_buffer: 2.4753227 vertex_buffer: 3.3475037 vertex_buffer: 0.42645 vertex_buffer: 0.389799 - vertex_buffer: -1.1224699 - vertex_buffer: 2.2189388 - vertex_buffer: 3.7918434 + vertex_buffer: -1.1224712 + vertex_buffer: 2.2189503 + vertex_buffer: 3.7918396 vertex_buffer: 0.438999 vertex_buffer: 0.396495 - vertex_buffer: -0.95554155 - vertex_buffer: 1.9475307 - vertex_buffer: 4.352604 + vertex_buffer: -0.9555427 + vertex_buffer: 1.9475403 + vertex_buffer: 4.3526 vertex_buffer: 0.450067 vertex_buffer: 0.400434 - vertex_buffer: -4.2466135 - vertex_buffer: 2.7904205 - vertex_buffer: 2.9572182 + vertex_buffer: -4.2466187 + vertex_buffer: 2.7904282 + vertex_buffer: 2.9572067 vertex_buffer: 0.289712 vertex_buffer: 0.368253 - vertex_buffer: -4.777348 - vertex_buffer: 2.9552498 - vertex_buffer: 2.9142609 + vertex_buffer: -4.777354 + vertex_buffer: 2.9552612 + vertex_buffer: 2.914257 vertex_buffer: 0.27667 vertex_buffer: 0.363373 - vertex_buffer: 0.45843273 - vertex_buffer: -0.12243652 - vertex_buffer: 6.785671 + vertex_buffer: 0.45843312 + vertex_buffer: -0.1224308 + vertex_buffer: 6.7856674 vertex_buffer: 0.517862 vertex_buffer: 0.471948 - vertex_buffer: 4.2937965 - vertex_buffer: 2.7028866 - vertex_buffer: 2.880951 + vertex_buffer: 4.2938013 + vertex_buffer: 2.7028942 + vertex_buffer: 2.880947 vertex_buffer: 0.710288 vertex_buffer: 0.380764 - vertex_buffer: 0.563513 - vertex_buffer: -2.4813404 + vertex_buffer: 0.5635134 + vertex_buffer: -2.4813385 vertex_buffer: 6.769951 vertex_buffer: 0.526227 vertex_buffer: 0.57391 - vertex_buffer: 7.1661444 - vertex_buffer: 6.344303 - vertex_buffer: -0.44289398 + vertex_buffer: 7.166153 + vertex_buffer: 6.3443146 + vertex_buffer: -0.4429016 vertex_buffer: 0.895093 vertex_buffer: 0.254141 - vertex_buffer: 2.565932 - vertex_buffer: 2.0484772 - vertex_buffer: 3.5651283 + vertex_buffer: 2.5659351 + vertex_buffer: 2.0484886 + vertex_buffer: 3.5651245 vertex_buffer: 0.63407 vertex_buffer: 0.409576 - vertex_buffer: 3.1454465 - vertex_buffer: 1.9983158 - vertex_buffer: 3.5477753 + vertex_buffer: 3.1454506 + vertex_buffer: 1.9983253 + vertex_buffer: 3.5477676 vertex_buffer: 0.661242 vertex_buffer: 0.413025 - vertex_buffer: 3.7646618 - vertex_buffer: 2.0593147 - vertex_buffer: 3.3925018 + vertex_buffer: 3.7646666 + vertex_buffer: 2.0593224 + vertex_buffer: 3.392498 vertex_buffer: 0.68888 vertex_buffer: 0.40946 - vertex_buffer: 4.6312976 - vertex_buffer: 2.4678402 - vertex_buffer: 2.835617 + vertex_buffer: 4.631304 + vertex_buffer: 2.4678497 + vertex_buffer: 2.8356094 vertex_buffer: 0.725342 vertex_buffer: 0.389131 - vertex_buffer: 2.0515 - vertex_buffer: 2.162346 - vertex_buffer: 3.5288582 + vertex_buffer: 2.0515027 + vertex_buffer: 2.1623554 + vertex_buffer: 3.5288506 vertex_buffer: 0.60663 vertex_buffer: 0.403705 - vertex_buffer: 3.488614 - vertex_buffer: 3.3744526 + vertex_buffer: 3.4886181 + vertex_buffer: 3.374464 vertex_buffer: 3.65485 vertex_buffer: 0.654766 vertex_buffer: 0.344011 - vertex_buffer: 2.8172188 - vertex_buffer: 3.3635693 - vertex_buffer: 3.6155205 + vertex_buffer: 2.817222 + vertex_buffer: 3.3635788 + vertex_buffer: 3.6155167 vertex_buffer: 0.629906 vertex_buffer: 0.346076 - vertex_buffer: 4.1270013 - vertex_buffer: 3.2975655 - vertex_buffer: 3.4852333 + vertex_buffer: 4.1270065 + vertex_buffer: 3.297575 + vertex_buffer: 3.4852295 vertex_buffer: 0.680678 vertex_buffer: 0.347265 - vertex_buffer: 4.5772557 - vertex_buffer: 3.166031 - vertex_buffer: 3.235653 + vertex_buffer: 4.5772614 + vertex_buffer: 3.1660423 + vertex_buffer: 3.2356453 vertex_buffer: 0.702097 vertex_buffer: 0.353591 - vertex_buffer: 5.2090616 - vertex_buffer: 2.1991062 - vertex_buffer: 2.5853577 + vertex_buffer: 5.209068 + vertex_buffer: 2.1991177 + vertex_buffer: 2.5853539 vertex_buffer: 0.752212 vertex_buffer: 0.410805 - vertex_buffer: 2.4598808 - vertex_buffer: -8.035987 - vertex_buffer: 4.3135834 + vertex_buffer: 2.459883 + vertex_buffer: -8.035992 + vertex_buffer: 4.313572 vertex_buffer: 0.602918 vertex_buffer: 0.842863 - vertex_buffer: 4.5487585 - vertex_buffer: 2.8514423 - vertex_buffer: 2.6670074 + vertex_buffer: 4.5487647 + vertex_buffer: 2.8514538 + vertex_buffer: 2.6669998 vertex_buffer: 0.719902 vertex_buffer: 0.3756 - vertex_buffer: 7.2774625 - vertex_buffer: 3.2908878 - vertex_buffer: -0.31038666 + vertex_buffer: 7.277471 + vertex_buffer: 3.2908974 + vertex_buffer: -0.3103943 vertex_buffer: 0.893693 vertex_buffer: 0.39996 - vertex_buffer: 5.9919224 - vertex_buffer: 2.67066 - vertex_buffer: 1.8980141 + vertex_buffer: 5.9919305 + vertex_buffer: 2.6706696 + vertex_buffer: 1.8980103 vertex_buffer: 0.790082 vertex_buffer: 0.391354 - vertex_buffer: 3.2944977 - vertex_buffer: -0.7652359 - vertex_buffer: 4.665989 + vertex_buffer: 3.2945013 + vertex_buffer: -0.765234 + vertex_buffer: 4.665985 vertex_buffer: 0.643998 vertex_buffer: 0.534488 - vertex_buffer: 0.96783084 - vertex_buffer: -3.7113724 + vertex_buffer: 0.9678316 + vertex_buffer: -3.7113743 vertex_buffer: 6.1592064 vertex_buffer: 0.528249 vertex_buffer: 0.650404 - vertex_buffer: 0.9485325 - vertex_buffer: -4.094248 - vertex_buffer: 5.9200745 + vertex_buffer: 0.9485334 + vertex_buffer: -4.0942497 + vertex_buffer: 5.9200706 vertex_buffer: 0.52585 vertex_buffer: 0.680191 - vertex_buffer: 1.9597192 + vertex_buffer: 1.9597212 vertex_buffer: -3.6362228 - vertex_buffer: 5.690422 + vertex_buffer: 5.6904182 vertex_buffer: 0.560215 vertex_buffer: 0.657229 - vertex_buffer: 2.7033787 + vertex_buffer: 2.7033815 vertex_buffer: -3.5398808 - vertex_buffer: 4.9262657 + vertex_buffer: 4.926262 vertex_buffer: 0.585384 vertex_buffer: 0.666541 - vertex_buffer: 1.7868371 + vertex_buffer: 1.7868391 vertex_buffer: -3.933569 - vertex_buffer: 5.5071907 + vertex_buffer: 5.507183 vertex_buffer: 0.549626 vertex_buffer: 0.680861 - vertex_buffer: 2.472351 - vertex_buffer: -3.7304955 - vertex_buffer: 4.8139343 + vertex_buffer: 2.472354 + vertex_buffer: -3.7304974 + vertex_buffer: 4.8139305 vertex_buffer: 0.571228 vertex_buffer: 0.682692 - vertex_buffer: 3.5094898 - vertex_buffer: -4.677987 - vertex_buffer: 3.6270447 + vertex_buffer: 3.5094936 + vertex_buffer: -4.677991 + vertex_buffer: 3.6270409 vertex_buffer: 0.624852 vertex_buffer: 0.728099 - vertex_buffer: 0.42484653 + vertex_buffer: 0.42484662 vertex_buffer: -2.513298 vertex_buffer: 7.892376 vertex_buffer: 0.51305 vertex_buffer: 0.547282 - vertex_buffer: 0.48307395 - vertex_buffer: -1.9238777 + vertex_buffer: 0.48307416 + vertex_buffer: -1.9238758 vertex_buffer: 8.017586 vertex_buffer: 0.515097 vertex_buffer: 0.527252 - vertex_buffer: 5.405193 - vertex_buffer: 3.5107555 - vertex_buffer: 3.0897903 + vertex_buffer: 5.4052 + vertex_buffer: 3.5107632 + vertex_buffer: 3.0897827 vertex_buffer: 0.742247 vertex_buffer: 0.314507 - vertex_buffer: 1.8350537 - vertex_buffer: 0.72956085 - vertex_buffer: 4.5594444 + vertex_buffer: 1.8350562 + vertex_buffer: 0.7295685 + vertex_buffer: 4.559437 vertex_buffer: 0.598631 vertex_buffer: 0.454979 - vertex_buffer: 2.090765 - vertex_buffer: -1.5839405 - vertex_buffer: 6.243725 + vertex_buffer: 2.0907671 + vertex_buffer: -1.5839367 + vertex_buffer: 6.243721 vertex_buffer: 0.570338 vertex_buffer: 0.548575 - vertex_buffer: 2.1136065 - vertex_buffer: -1.156723 - vertex_buffer: 5.9879227 + vertex_buffer: 2.1136088 + vertex_buffer: -1.1567173 + vertex_buffer: 5.987919 vertex_buffer: 0.578632 vertex_buffer: 0.533623 - vertex_buffer: 5.2506566 - vertex_buffer: -0.35006142 - vertex_buffer: 3.8119774 + vertex_buffer: 5.250663 + vertex_buffer: -0.3500576 + vertex_buffer: 3.8119736 vertex_buffer: 0.723087 vertex_buffer: 0.532054 - vertex_buffer: 0.4557106 - vertex_buffer: -0.9948273 + vertex_buffer: 0.45571092 + vertex_buffer: -0.99482155 vertex_buffer: 7.528843 vertex_buffer: 0.516446 vertex_buffer: 0.499639 - vertex_buffer: 3.7202723 - vertex_buffer: 3.7115192 - vertex_buffer: 4.1818047 + vertex_buffer: 3.7202773 + vertex_buffer: 3.7115288 + vertex_buffer: 4.181797 vertex_buffer: 0.662801 vertex_buffer: 0.282918 - vertex_buffer: 4.683783 - vertex_buffer: 3.6740036 + vertex_buffer: 4.683789 + vertex_buffer: 3.6740131 vertex_buffer: 3.73135 vertex_buffer: 0.703624 vertex_buffer: 0.293271 - vertex_buffer: 6.4526186 - vertex_buffer: 7.177004 - vertex_buffer: 0.64782715 + vertex_buffer: 6.4526267 + vertex_buffer: 7.1770153 + vertex_buffer: 0.6478195 vertex_buffer: 0.830705 vertex_buffer: 0.193814 - vertex_buffer: 1.140491 - vertex_buffer: 3.0603447 - vertex_buffer: 4.7917366 + vertex_buffer: 1.1404927 + vertex_buffer: 3.0603523 + vertex_buffer: 4.791733 vertex_buffer: 0.552386 vertex_buffer: 0.302568 - vertex_buffer: 2.2480388 - vertex_buffer: 3.2713928 + vertex_buffer: 2.2480416 + vertex_buffer: 3.2714005 vertex_buffer: 3.4257584 vertex_buffer: 0.60761 vertex_buffer: 0.353888 - vertex_buffer: 4.12534 + vertex_buffer: 4.1253448 vertex_buffer: -3.6359367 - vertex_buffer: 3.5436134 + vertex_buffer: 3.5436096 vertex_buffer: 0.645429 vertex_buffer: 0.696707 - vertex_buffer: 6.629145 - vertex_buffer: -3.3822517 - vertex_buffer: -1.0385818 + vertex_buffer: 6.629153 + vertex_buffer: -3.3822498 + vertex_buffer: -1.0385971 vertex_buffer: 0.932695 vertex_buffer: 0.730105 - vertex_buffer: 1.641717 - vertex_buffer: -1.9287262 - vertex_buffer: 6.0156593 + vertex_buffer: 1.641719 + vertex_buffer: -1.9287205 + vertex_buffer: 6.0156555 vertex_buffer: 0.557261 vertex_buffer: 0.572826 - vertex_buffer: 1.0766811 - vertex_buffer: -2.2036495 + vertex_buffer: 1.0766822 + vertex_buffer: -2.2036476 vertex_buffer: 6.1743774 vertex_buffer: 0.542902 vertex_buffer: 0.584792 - vertex_buffer: 3.624907 - vertex_buffer: -3.503807 - vertex_buffer: 3.442051 + vertex_buffer: 3.6249113 + vertex_buffer: -3.5038052 + vertex_buffer: 3.4420433 vertex_buffer: 0.618026 vertex_buffer: 0.694711 - vertex_buffer: 3.4461956 - vertex_buffer: -3.5624218 - vertex_buffer: 3.5712547 + vertex_buffer: 3.4461992 + vertex_buffer: -3.56242 + vertex_buffer: 3.571247 vertex_buffer: 0.607591 vertex_buffer: 0.694203 - vertex_buffer: 5.070438 - vertex_buffer: 4.2198524 - vertex_buffer: 3.3080864 + vertex_buffer: 5.070444 + vertex_buffer: 4.219864 + vertex_buffer: 3.3080826 vertex_buffer: 0.722943 vertex_buffer: 0.271963 - vertex_buffer: 2.1962533 - vertex_buffer: -1.6779041 + vertex_buffer: 2.1962554 + vertex_buffer: -1.6779022 vertex_buffer: 5.9193077 vertex_buffer: 0.577414 vertex_buffer: 0.563167 - vertex_buffer: 2.5358021 - vertex_buffer: 3.5196838 - vertex_buffer: 4.539776 + vertex_buffer: 2.5358052 + vertex_buffer: 3.5196915 + vertex_buffer: 4.539768 vertex_buffer: 0.614083 vertex_buffer: 0.281387 - vertex_buffer: 2.6456668 - vertex_buffer: 4.0347195 + vertex_buffer: 2.6456702 + vertex_buffer: 4.034727 vertex_buffer: 4.60878 vertex_buffer: 0.616907 vertex_buffer: 0.255886 - vertex_buffer: 3.675518 - vertex_buffer: 7.9906406 - vertex_buffer: 2.9460526 + vertex_buffer: 3.675523 + vertex_buffer: 7.990658 + vertex_buffer: 2.9460487 vertex_buffer: 0.668509 vertex_buffer: 0.119914 - vertex_buffer: 5.6913533 - vertex_buffer: 5.756613 - vertex_buffer: 2.0977516 + vertex_buffer: 5.6913605 + vertex_buffer: 5.756626 + vertex_buffer: 2.0977478 vertex_buffer: 0.770092 vertex_buffer: 0.232021 - vertex_buffer: 3.0768647 - vertex_buffer: 6.201044 + vertex_buffer: 3.0768685 + vertex_buffer: 6.2010574 vertex_buffer: 3.7766457 vertex_buffer: 0.635536 vertex_buffer: 0.189249 - vertex_buffer: 5.831969 - vertex_buffer: 4.0339775 - vertex_buffer: 2.4737282 + vertex_buffer: 5.831976 + vertex_buffer: 4.033989 + vertex_buffer: 2.4737244 vertex_buffer: 0.770391 vertex_buffer: 0.299556 - vertex_buffer: 6.487762 - vertex_buffer: 5.180357 - vertex_buffer: 1.0317268 + vertex_buffer: 6.4877706 + vertex_buffer: 5.1803684 + vertex_buffer: 1.0317192 vertex_buffer: 0.826722 vertex_buffer: 0.278755 - vertex_buffer: 0.9777184 + vertex_buffer: 0.9777193 vertex_buffer: -3.945898 - vertex_buffer: 6.0843506 + vertex_buffer: 6.084347 vertex_buffer: 0.527121 vertex_buffer: 0.666198 - vertex_buffer: 1.8753169 - vertex_buffer: -3.8008995 - vertex_buffer: 5.5984306 + vertex_buffer: 1.8753189 + vertex_buffer: -3.8008976 + vertex_buffer: 5.598423 vertex_buffer: 0.553172 vertex_buffer: 0.668527 - vertex_buffer: 2.5915039 + vertex_buffer: 2.5915065 vertex_buffer: -3.6574516 - vertex_buffer: 4.915451 + vertex_buffer: 4.915447 vertex_buffer: 0.577238 vertex_buffer: 0.67389 - vertex_buffer: 1.5212762 - vertex_buffer: -1.9988365 - vertex_buffer: 5.9348717 + vertex_buffer: 1.5212778 + vertex_buffer: -1.9988327 + vertex_buffer: 5.934868 vertex_buffer: 0.554692 vertex_buffer: 0.580066 - vertex_buffer: 3.5580156 - vertex_buffer: -3.5292492 - vertex_buffer: 3.51326 + vertex_buffer: 3.5580196 + vertex_buffer: -3.5292473 + vertex_buffer: 3.5132523 vertex_buffer: 0.611897 vertex_buffer: 0.693961 - vertex_buffer: 3.0891697 - vertex_buffer: -4.274349 - vertex_buffer: 3.8229141 + vertex_buffer: 3.0891728 + vertex_buffer: -4.2743473 + vertex_buffer: 3.8229103 vertex_buffer: 0.596961 vertex_buffer: 0.70654 - vertex_buffer: 3.3768747 - vertex_buffer: -3.600422 - vertex_buffer: 3.5875587 + vertex_buffer: 3.3768785 + vertex_buffer: -3.60042 + vertex_buffer: 3.587551 vertex_buffer: 0.596371 vertex_buffer: 0.693953 - vertex_buffer: 1.2161286 - vertex_buffer: -2.1466656 - vertex_buffer: 6.9213715 + vertex_buffer: 1.2161298 + vertex_buffer: -2.1466618 + vertex_buffer: 6.9213676 vertex_buffer: 0.539958 vertex_buffer: 0.557139 - vertex_buffer: 2.3516455 - vertex_buffer: -3.7717247 - vertex_buffer: 4.75354 + vertex_buffer: 2.351648 + vertex_buffer: -3.7717228 + vertex_buffer: 4.753536 vertex_buffer: 0.568842 vertex_buffer: 0.692366 - vertex_buffer: 1.6980877 - vertex_buffer: -3.940525 - vertex_buffer: 5.294464 + vertex_buffer: 1.6980895 + vertex_buffer: -3.9405231 + vertex_buffer: 5.2944603 vertex_buffer: 0.547818 vertex_buffer: 0.692366 - vertex_buffer: 0.8974841 + vertex_buffer: 0.8974848 vertex_buffer: -4.14546 vertex_buffer: 5.696514 vertex_buffer: 0.524613 vertex_buffer: 0.692366 - vertex_buffer: 1.0443137 - vertex_buffer: -7.1271086 - vertex_buffer: 4.956272 + vertex_buffer: 1.0443145 + vertex_buffer: -7.1271124 + vertex_buffer: 4.9562645 vertex_buffer: 0.53409 vertex_buffer: 0.779141 - vertex_buffer: 0.987621 - vertex_buffer: -6.3614016 - vertex_buffer: 5.318268 + vertex_buffer: 0.9876218 + vertex_buffer: -6.3614044 + vertex_buffer: 5.31826 vertex_buffer: 0.527671 vertex_buffer: 0.736226 - vertex_buffer: 0.96364325 - vertex_buffer: -6.0137033 - vertex_buffer: 5.3195267 + vertex_buffer: 0.963644 + vertex_buffer: -6.013706 + vertex_buffer: 5.319519 vertex_buffer: 0.526913 vertex_buffer: 0.717857 - vertex_buffer: 0.92498523 - vertex_buffer: -5.6456995 - vertex_buffer: 5.0970116 + vertex_buffer: 0.9249861 + vertex_buffer: -5.6457005 + vertex_buffer: 5.097004 vertex_buffer: 0.526878 vertex_buffer: 0.704626 - vertex_buffer: 0.8514866 + vertex_buffer: 0.8514873 vertex_buffer: -5.4173384 - vertex_buffer: 4.945057 + vertex_buffer: 4.9450493 vertex_buffer: 0.526967 vertex_buffer: 0.695278 - vertex_buffer: 2.2936535 + vertex_buffer: 2.2936554 vertex_buffer: -4.7114153 - vertex_buffer: 4.1935005 + vertex_buffer: 4.193493 vertex_buffer: 0.572058 vertex_buffer: 0.695278 - vertex_buffer: 2.3988762 - vertex_buffer: -4.7940826 - vertex_buffer: 4.318264 + vertex_buffer: 2.3988786 + vertex_buffer: -4.7940845 + vertex_buffer: 4.31826 vertex_buffer: 0.573521 vertex_buffer: 0.70354 - vertex_buffer: 2.52226 + vertex_buffer: 2.5222626 vertex_buffer: -4.9738216 - vertex_buffer: 4.4091377 + vertex_buffer: 4.40913 vertex_buffer: 0.576838 vertex_buffer: 0.711846 - vertex_buffer: 2.618237 + vertex_buffer: 2.6182396 vertex_buffer: -5.130539 - vertex_buffer: 4.305893 + vertex_buffer: 4.3058853 vertex_buffer: 0.581691 vertex_buffer: 0.720063 - vertex_buffer: 3.2409048 + vertex_buffer: 3.2409084 vertex_buffer: -2.943821 - vertex_buffer: 4.736702 + vertex_buffer: 4.736698 vertex_buffer: 0.609945 vertex_buffer: 0.63991 - vertex_buffer: 7.502926 - vertex_buffer: 0.59869766 - vertex_buffer: -2.361534 + vertex_buffer: 7.5029354 + vertex_buffer: 0.5987072 + vertex_buffer: -2.3615494 vertex_buffer: 0.986046 vertex_buffer: 0.560034 - vertex_buffer: 2.8017242 + vertex_buffer: 2.8017273 vertex_buffer: -4.2447968 - vertex_buffer: 3.7659073 + vertex_buffer: 3.7659035 vertex_buffer: 0.5868 vertex_buffer: 0.6954 - vertex_buffer: 2.929969 - vertex_buffer: -4.253687 - vertex_buffer: 3.836361 + vertex_buffer: 2.9299724 + vertex_buffer: -4.253685 + vertex_buffer: 3.8363533 vertex_buffer: 0.590372 vertex_buffer: 0.701823 - vertex_buffer: 0.8780336 + vertex_buffer: 0.87803423 vertex_buffer: -2.4603996 vertex_buffer: 6.1991234 vertex_buffer: 0.531915 vertex_buffer: 0.601537 - vertex_buffer: 2.025136 + vertex_buffer: 2.0251384 vertex_buffer: -1.8861618 - vertex_buffer: 5.4707375 + vertex_buffer: 5.4707336 vertex_buffer: 0.577268 vertex_buffer: 0.585935 - vertex_buffer: 0.967478 - vertex_buffer: -2.3653927 + vertex_buffer: 0.9674789 + vertex_buffer: -2.3653946 vertex_buffer: 6.2165947 vertex_buffer: 0.536915 vertex_buffer: 0.593786 - vertex_buffer: 2.5313063 - vertex_buffer: 0.5067806 - vertex_buffer: 4.393051 + vertex_buffer: 2.531309 + vertex_buffer: 0.50678635 + vertex_buffer: 4.3930473 vertex_buffer: 0.627543 vertex_buffer: 0.473352 - vertex_buffer: 3.617191 - vertex_buffer: 0.18891716 - vertex_buffer: 4.296589 + vertex_buffer: 3.6171951 + vertex_buffer: 0.18892479 + vertex_buffer: 4.296585 vertex_buffer: 0.665586 vertex_buffer: 0.495951 - vertex_buffer: 2.3044915 - vertex_buffer: -1.2837143 - vertex_buffer: 5.6435738 + vertex_buffer: 2.304494 + vertex_buffer: -1.2837124 + vertex_buffer: 5.64357 vertex_buffer: 0.588354 vertex_buffer: 0.546862 - vertex_buffer: 5.3009076 - vertex_buffer: 7.7115097 - vertex_buffer: 1.8507767 + vertex_buffer: 5.3009152 + vertex_buffer: 7.711525 + vertex_buffer: 1.850769 vertex_buffer: 0.757824 vertex_buffer: 0.147676 - vertex_buffer: 4.5862565 - vertex_buffer: 6.1100426 - vertex_buffer: 2.9832764 + vertex_buffer: 4.586262 + vertex_buffer: 6.1100597 + vertex_buffer: 2.9832726 vertex_buffer: 0.70925 vertex_buffer: 0.201508 - vertex_buffer: 3.9455872 - vertex_buffer: 4.307043 - vertex_buffer: 3.948494 + vertex_buffer: 3.945592 + vertex_buffer: 4.3070507 + vertex_buffer: 3.9484863 vertex_buffer: 0.672684 vertex_buffer: 0.256581 - vertex_buffer: 2.880608 - vertex_buffer: -5.605525 - vertex_buffer: 4.019226 + vertex_buffer: 2.8806114 + vertex_buffer: -5.605528 + vertex_buffer: 4.0192223 vertex_buffer: 0.600409 vertex_buffer: 0.749005 - vertex_buffer: 1.3191733 - vertex_buffer: 3.8142223 - vertex_buffer: 4.931698 + vertex_buffer: 1.3191751 + vertex_buffer: 3.8142319 + vertex_buffer: 4.931694 vertex_buffer: 0.558266 vertex_buffer: 0.261672 - vertex_buffer: 1.6745269 - vertex_buffer: 6.172098 + vertex_buffer: 1.6745294 + vertex_buffer: 6.1721096 vertex_buffer: 4.240856 vertex_buffer: 0.570304 vertex_buffer: 0.187871 - vertex_buffer: 2.0401752 - vertex_buffer: 8.078575 + vertex_buffer: 2.0401783 + vertex_buffer: 8.078587 vertex_buffer: 3.6178894 vertex_buffer: 0.588166 vertex_buffer: 0.109044 - vertex_buffer: 4.2982955 - vertex_buffer: 2.2354507 - vertex_buffer: 3.1172485 + vertex_buffer: 4.2983003 + vertex_buffer: 2.2354584 + vertex_buffer: 3.1172447 vertex_buffer: 0.711045 vertex_buffer: 0.398952 - vertex_buffer: 5.918257 - vertex_buffer: 1.8992882 - vertex_buffer: 2.2622795 + vertex_buffer: 5.9182653 + vertex_buffer: 1.8992939 + vertex_buffer: 2.2622757 vertex_buffer: 0.78107 vertex_buffer: 0.435405 - vertex_buffer: 1.7605395 - vertex_buffer: 2.3266659 - vertex_buffer: 3.42659 + vertex_buffer: 1.7605418 + vertex_buffer: 2.3266754 + vertex_buffer: 3.4265862 vertex_buffer: 0.587247 vertex_buffer: 0.398932 - vertex_buffer: 5.2549534 - vertex_buffer: 3.0604095 - vertex_buffer: 2.6985817 + vertex_buffer: 5.2549596 + vertex_buffer: 3.060419 + vertex_buffer: 2.698578 vertex_buffer: 0.74287 vertex_buffer: 0.355446 - vertex_buffer: 1.3581057 - vertex_buffer: 1.0136013 - vertex_buffer: 4.7732735 + vertex_buffer: 1.3581073 + vertex_buffer: 1.0136089 + vertex_buffer: 4.7732697 vertex_buffer: 0.572156 vertex_buffer: 0.437652 - vertex_buffer: 1.5795965 - vertex_buffer: -1.6822205 - vertex_buffer: 6.861721 + vertex_buffer: 1.5795982 + vertex_buffer: -1.6822186 + vertex_buffer: 6.861717 vertex_buffer: 0.551868 vertex_buffer: 0.53657 - vertex_buffer: 6.6445875 - vertex_buffer: 1.6044445 - vertex_buffer: 1.6377296 + vertex_buffer: 6.6445966 + vertex_buffer: 1.6044521 + vertex_buffer: 1.637722 vertex_buffer: 0.821442 vertex_buffer: 0.457556 - vertex_buffer: 5.438588 - vertex_buffer: 1.4010563 - vertex_buffer: 2.8901978 + vertex_buffer: 5.4385943 + vertex_buffer: 1.4010639 + vertex_buffer: 2.890194 vertex_buffer: 0.752702 vertex_buffer: 0.457182 - vertex_buffer: 4.6051846 - vertex_buffer: 1.0590935 - vertex_buffer: 3.492607 + vertex_buffer: 4.60519 + vertex_buffer: 1.0591011 + vertex_buffer: 3.4926033 vertex_buffer: 0.713757 vertex_buffer: 0.467627 - vertex_buffer: 3.4062254 - vertex_buffer: 0.9862709 + vertex_buffer: 3.4062293 + vertex_buffer: 0.9862766 vertex_buffer: 3.838192 vertex_buffer: 0.667113 vertex_buffer: 0.460673 - vertex_buffer: 2.4928606 - vertex_buffer: 1.1352692 + vertex_buffer: 2.4928637 + vertex_buffer: 1.1352768 vertex_buffer: 3.9427338 vertex_buffer: 0.631101 vertex_buffer: 0.447154 - vertex_buffer: 1.8204823 - vertex_buffer: 1.3184166 - vertex_buffer: 4.103424 + vertex_buffer: 1.8204843 + vertex_buffer: 1.3184261 + vertex_buffer: 4.1034203 vertex_buffer: 0.600862 vertex_buffer: 0.432473 - vertex_buffer: 0.5738768 - vertex_buffer: 1.5734634 + vertex_buffer: 0.57387745 + vertex_buffer: 1.5734692 vertex_buffer: 5.444023 vertex_buffer: 0.523481 vertex_buffer: 0.405627 - vertex_buffer: 6.6679235 - vertex_buffer: 0.22402573 - vertex_buffer: 1.9835739 + vertex_buffer: 6.667932 + vertex_buffer: 0.22403145 + vertex_buffer: 1.9835663 vertex_buffer: 0.810748 vertex_buffer: 0.523926 - vertex_buffer: 5.7407603 - vertex_buffer: 3.1849308 - vertex_buffer: 2.401783 + vertex_buffer: 5.7407665 + vertex_buffer: 3.1849384 + vertex_buffer: 2.4017792 vertex_buffer: 0.771046 vertex_buffer: 0.348959 - vertex_buffer: 0.23790747 - vertex_buffer: -2.7562447 + vertex_buffer: 0.23790738 + vertex_buffer: -2.7562466 vertex_buffer: 7.6178856 vertex_buffer: 0.509127 vertex_buffer: 0.562718 - vertex_buffer: 1.8713548 - vertex_buffer: 0.033107758 + vertex_buffer: 1.8713568 + vertex_buffer: 0.033109665 vertex_buffer: 4.94363 vertex_buffer: 0.595293 vertex_buffer: 0.485024 - vertex_buffer: 7.683481 - vertex_buffer: 3.9597416 - vertex_buffer: -2.3439941 + vertex_buffer: 7.683491 + vertex_buffer: 3.959755 + vertex_buffer: -2.3440018 vertex_buffer: 0.980531 vertex_buffer: 0.401564 - vertex_buffer: 1.3603485 - vertex_buffer: 1.609335 - vertex_buffer: 4.160656 + vertex_buffer: 1.3603501 + vertex_buffer: 1.6093407 + vertex_buffer: 4.160652 vertex_buffer: 0.5735 vertex_buffer: 0.42 - vertex_buffer: 2.412501 - vertex_buffer: -1.0677414 + vertex_buffer: 2.4125037 + vertex_buffer: -1.0677376 vertex_buffer: 4.927147 vertex_buffer: 0.602995 vertex_buffer: 0.548688 - vertex_buffer: 4.92097 - vertex_buffer: 2.8164482 - vertex_buffer: 2.5507278 + vertex_buffer: 4.920976 + vertex_buffer: 2.8164597 + vertex_buffer: 2.550724 vertex_buffer: 0.73353 vertex_buffer: 0.376977 - vertex_buffer: 1.6102984 - vertex_buffer: -1.1498814 - vertex_buffer: 6.5544205 + vertex_buffer: 1.6103003 + vertex_buffer: -1.1498775 + vertex_buffer: 6.5544167 vertex_buffer: 0.560611 vertex_buffer: 0.519017 - vertex_buffer: 7.1972446 - vertex_buffer: -1.315155 - vertex_buffer: -1.8514862 + vertex_buffer: 7.197254 + vertex_buffer: -1.3151512 + vertex_buffer: -1.8515015 vertex_buffer: 0.967686 vertex_buffer: 0.644357 - vertex_buffer: 1.7810719 - vertex_buffer: 2.5902557 - vertex_buffer: 3.1927032 + vertex_buffer: 1.7810742 + vertex_buffer: 2.5902653 + vertex_buffer: 3.1926994 vertex_buffer: 0.580985 vertex_buffer: 0.38716 - vertex_buffer: 1.0172381 - vertex_buffer: -1.0713634 - vertex_buffer: 7.1607246 + vertex_buffer: 1.017239 + vertex_buffer: -1.0713615 + vertex_buffer: 7.160721 vertex_buffer: 0.537728 vertex_buffer: 0.505385 - vertex_buffer: 5.189652 - vertex_buffer: -5.526311 - vertex_buffer: 1.940609 + vertex_buffer: 5.189658 + vertex_buffer: -5.526314 + vertex_buffer: 1.9405975 vertex_buffer: 0.760966 vertex_buffer: 0.779753 - vertex_buffer: 4.9922805 - vertex_buffer: -6.4331474 - vertex_buffer: 1.185112 + vertex_buffer: 4.9922867 + vertex_buffer: -6.4331484 + vertex_buffer: 1.1851006 vertex_buffer: 0.801779 vertex_buffer: 0.831938 - vertex_buffer: 7.3186183 - vertex_buffer: 0.3694744 - vertex_buffer: -0.21674347 + vertex_buffer: 7.318627 + vertex_buffer: 0.36948395 + vertex_buffer: -0.2167511 vertex_buffer: 0.892441 vertex_buffer: 0.540761 - vertex_buffer: 6.056026 - vertex_buffer: -4.434767 - vertex_buffer: 1.2825623 + vertex_buffer: 6.0560336 + vertex_buffer: -4.434766 + vertex_buffer: 1.2825546 vertex_buffer: 0.816351 vertex_buffer: 0.74026 - vertex_buffer: 6.927325 - vertex_buffer: 4.4287663 - vertex_buffer: 0.16318512 + vertex_buffer: 6.927334 + vertex_buffer: 4.4287796 + vertex_buffer: 0.16317749 vertex_buffer: 0.865595 vertex_buffer: 0.333687 - vertex_buffer: 2.4872823 - vertex_buffer: -8.917337 - vertex_buffer: 4.1714897 + vertex_buffer: 2.4872847 + vertex_buffer: -8.917343 + vertex_buffer: 4.1714783 vertex_buffer: 0.614074 vertex_buffer: 0.883246 - vertex_buffer: 0.18507135 + vertex_buffer: 0.18507123 vertex_buffer: -2.6379662 vertex_buffer: 6.7830925 vertex_buffer: 0.508953 vertex_buffer: 0.579438 - vertex_buffer: 2.4989808 - vertex_buffer: -0.297657 - vertex_buffer: 4.7593803 + vertex_buffer: 2.4989836 + vertex_buffer: -0.2976513 + vertex_buffer: 4.7593727 vertex_buffer: 0.617942 vertex_buffer: 0.508316 - vertex_buffer: 6.560847 - vertex_buffer: 2.7619705 - vertex_buffer: 1.2603416 + vertex_buffer: 6.560855 + vertex_buffer: 2.76198 + vertex_buffer: 1.260334 vertex_buffer: 0.825608 vertex_buffer: 0.397675 - vertex_buffer: 3.6496165 - vertex_buffer: 2.498146 - vertex_buffer: 3.2662697 + vertex_buffer: 3.649621 + vertex_buffer: 2.4981575 + vertex_buffer: 3.2662582 vertex_buffer: 0.681215 vertex_buffer: 0.396235 - vertex_buffer: 3.155451 - vertex_buffer: 2.4324074 - vertex_buffer: 3.4271507 + vertex_buffer: 3.1554554 + vertex_buffer: 2.432415 + vertex_buffer: 3.427147 vertex_buffer: 0.656636 vertex_buffer: 0.400597 - vertex_buffer: 3.1980324 - vertex_buffer: -4.3204155 - vertex_buffer: 3.7522202 + vertex_buffer: 3.198036 + vertex_buffer: -4.3204174 + vertex_buffer: 3.7522125 vertex_buffer: 0.6039 vertex_buffer: 0.710217 - vertex_buffer: 6.7209496 - vertex_buffer: -1.2013817 - vertex_buffer: 1.948906 + vertex_buffer: 6.720958 + vertex_buffer: -1.201376 + vertex_buffer: 1.9488983 vertex_buffer: 0.812086 vertex_buffer: 0.588539 - vertex_buffer: 1.2246006 - vertex_buffer: -10.21917 - vertex_buffer: 4.311302 + vertex_buffer: 1.2246015 + vertex_buffer: -10.219176 + vertex_buffer: 4.3112946 vertex_buffer: 0.568013 vertex_buffer: 0.944565 - vertex_buffer: 3.186315 - vertex_buffer: -8.656027 - vertex_buffer: 2.8606186 + vertex_buffer: 3.1863186 + vertex_buffer: -8.656031 + vertex_buffer: 2.8606071 vertex_buffer: 0.681008 vertex_buffer: 0.898285 - vertex_buffer: 4.006108 - vertex_buffer: -7.676532 - vertex_buffer: 2.0705223 + vertex_buffer: 4.006112 + vertex_buffer: -7.6765366 + vertex_buffer: 2.0705109 vertex_buffer: 0.733752 vertex_buffer: 0.869701 - vertex_buffer: 2.6482399 - vertex_buffer: 2.4515076 - vertex_buffer: 3.45673 + vertex_buffer: 2.648243 + vertex_buffer: 2.451519 + vertex_buffer: 3.4567223 vertex_buffer: 0.63383 vertex_buffer: 0.398822 - vertex_buffer: 2.2057068 - vertex_buffer: 2.506771 - vertex_buffer: 3.3655663 + vertex_buffer: 2.2057095 + vertex_buffer: 2.5067787 + vertex_buffer: 3.3655624 vertex_buffer: 0.606793 vertex_buffer: 0.395537 - vertex_buffer: 1.9240649 - vertex_buffer: 2.5685253 - vertex_buffer: 3.2245102 + vertex_buffer: 1.9240671 + vertex_buffer: 2.5685349 + vertex_buffer: 3.2245064 vertex_buffer: 0.58966 vertex_buffer: 0.391062 - vertex_buffer: 6.265646 - vertex_buffer: 3.5524426 - vertex_buffer: 1.7489777 + vertex_buffer: 6.2656546 + vertex_buffer: 3.552454 + vertex_buffer: 1.7489738 vertex_buffer: 0.805016 vertex_buffer: 0.342108 - vertex_buffer: 2.3390827 - vertex_buffer: 2.884201 + vertex_buffer: 2.3390853 + vertex_buffer: 2.8842087 vertex_buffer: 3.3479233 vertex_buffer: 0.611335 vertex_buffer: 0.362284 - vertex_buffer: 2.8314958 - vertex_buffer: 2.9797497 - vertex_buffer: 3.476944 + vertex_buffer: 2.8314993 + vertex_buffer: 2.9797573 + vertex_buffer: 3.4769402 vertex_buffer: 0.634038 vertex_buffer: 0.355971 - vertex_buffer: 3.3190825 - vertex_buffer: 2.999731 - vertex_buffer: 3.4610176 + vertex_buffer: 3.3190868 + vertex_buffer: 2.9997406 + vertex_buffer: 3.4610138 vertex_buffer: 0.656636 vertex_buffer: 0.355357 - vertex_buffer: 3.7897182 - vertex_buffer: 2.961588 - vertex_buffer: 3.317852 + vertex_buffer: 3.789723 + vertex_buffer: 2.9615993 + vertex_buffer: 3.3178482 vertex_buffer: 0.681215 vertex_buffer: 0.35834 - vertex_buffer: 4.145367 - vertex_buffer: 2.891821 - vertex_buffer: 3.1133041 + vertex_buffer: 4.1453724 + vertex_buffer: 2.8918304 + vertex_buffer: 3.1133003 vertex_buffer: 0.698585 vertex_buffer: 0.363156 - vertex_buffer: 7.5618553 - vertex_buffer: 5.420641 - vertex_buffer: -1.5621185 + vertex_buffer: 7.561866 + vertex_buffer: 5.420656 + vertex_buffer: -1.5621338 vertex_buffer: 0.941867 vertex_buffer: 0.319076 - vertex_buffer: 4.0105886 - vertex_buffer: 2.5953503 + vertex_buffer: 4.010593 + vertex_buffer: 2.595358 vertex_buffer: 3.0636559 vertex_buffer: 0.698585 vertex_buffer: 0.387449 - vertex_buffer: 2.4361215 - vertex_buffer: -2.783411 - vertex_buffer: 5.3472176 + vertex_buffer: 2.4361243 + vertex_buffer: -2.783409 + vertex_buffer: 5.3472137 vertex_buffer: 0.584177 vertex_buffer: 0.624107 - vertex_buffer: 1.6050982 - vertex_buffer: -1.9455338 + vertex_buffer: 1.6051 + vertex_buffer: -1.9455318 vertex_buffer: 6.2601547 vertex_buffer: 0.554318 vertex_buffer: 0.566077 - vertex_buffer: 1.0137081 - vertex_buffer: -2.9374046 - vertex_buffer: 6.0877533 + vertex_buffer: 1.013709 + vertex_buffer: -2.9374008 + vertex_buffer: 6.0877495 vertex_buffer: 0.534154 vertex_buffer: 0.62064 - vertex_buffer: 4.2468576 - vertex_buffer: -6.707801 - vertex_buffer: 2.5414963 + vertex_buffer: 4.2468624 + vertex_buffer: -6.7078037 + vertex_buffer: 2.5414886 vertex_buffer: 0.711218 vertex_buffer: 0.819975 - vertex_buffer: 3.3817728 - vertex_buffer: -7.8021345 - vertex_buffer: 3.3166351 + vertex_buffer: 3.3817763 + vertex_buffer: -7.8021374 + vertex_buffer: 3.3166275 vertex_buffer: 0.66463 vertex_buffer: 0.852871 - vertex_buffer: 1.2883689 - vertex_buffer: -9.677936 - vertex_buffer: 4.8929443 + vertex_buffer: 1.28837 + vertex_buffer: -9.677942 + vertex_buffer: 4.8929367 vertex_buffer: 0.5591 vertex_buffer: 0.902632 - vertex_buffer: 5.832577 + vertex_buffer: 5.8325844 vertex_buffer: -5.1295977 - vertex_buffer: 0.12055969 + vertex_buffer: 0.12055206 vertex_buffer: 0.871706 vertex_buffer: 0.791941 - vertex_buffer: 1.9733884 - vertex_buffer: 2.7239323 - vertex_buffer: 3.2599945 + vertex_buffer: 1.9733912 + vertex_buffer: 2.72394 + vertex_buffer: 3.2599869 vertex_buffer: 0.591234 vertex_buffer: 0.373894 - vertex_buffer: 0.93809897 - vertex_buffer: 0.5055809 - vertex_buffer: 5.7244835 + vertex_buffer: 0.9381001 + vertex_buffer: 0.5055866 + vertex_buffer: 5.724476 vertex_buffer: 0.544341 vertex_buffer: 0.451584 - vertex_buffer: 2.3015637 - vertex_buffer: -9.5524845 - vertex_buffer: 3.7121239 + vertex_buffer: 2.3015656 + vertex_buffer: -9.552489 + vertex_buffer: 3.7121162 vertex_buffer: 0.624563 vertex_buffer: 0.924192 - vertex_buffer: 7.139088 - vertex_buffer: -1.3418045 - vertex_buffer: 0.07254028 + vertex_buffer: 7.139097 + vertex_buffer: -1.3418007 + vertex_buffer: 0.072525024 vertex_buffer: 0.88577 vertex_buffer: 0.615029 - vertex_buffer: 1.6386461 - vertex_buffer: -5.1091747 - vertex_buffer: 4.613632 + vertex_buffer: 1.6386478 + vertex_buffer: -5.1091776 + vertex_buffer: 4.6136246 vertex_buffer: 0.551338 vertex_buffer: 0.695278 - vertex_buffer: 1.728884 - vertex_buffer: -5.284889 - vertex_buffer: 4.7355385 + vertex_buffer: 1.7288855 + vertex_buffer: -5.284891 + vertex_buffer: 4.7355347 vertex_buffer: 0.55198 vertex_buffer: 0.704632 - vertex_buffer: 1.8350952 - vertex_buffer: -5.5643196 - vertex_buffer: 4.855278 + vertex_buffer: 1.835097 + vertex_buffer: -5.5643215 + vertex_buffer: 4.855274 vertex_buffer: 0.552888 vertex_buffer: 0.715808 - vertex_buffer: 1.887146 - vertex_buffer: -5.842271 - vertex_buffer: 4.792778 + vertex_buffer: 1.887148 + vertex_buffer: -5.8422728 + vertex_buffer: 4.7927704 vertex_buffer: 0.555168 vertex_buffer: 0.730794 - vertex_buffer: 2.1029322 - vertex_buffer: -6.506687 - vertex_buffer: 4.525856 + vertex_buffer: 2.1029344 + vertex_buffer: -6.50669 + vertex_buffer: 4.5258446 vertex_buffer: 0.569944 vertex_buffer: 0.767035 - vertex_buffer: 3.061017 - vertex_buffer: -3.6268826 - vertex_buffer: 4.165062 + vertex_buffer: 3.0610201 + vertex_buffer: -3.6268806 + vertex_buffer: 4.165058 vertex_buffer: 0.593203 vertex_buffer: 0.685676 - vertex_buffer: 3.1612403 + vertex_buffer: 3.1612434 vertex_buffer: -3.5417175 - vertex_buffer: 4.122753 + vertex_buffer: 4.1227455 vertex_buffer: 0.599262 vertex_buffer: 0.681069 - vertex_buffer: 3.2587636 - vertex_buffer: -3.4861755 - vertex_buffer: 4.125637 + vertex_buffer: 3.2587674 + vertex_buffer: -3.4861774 + vertex_buffer: 4.1256294 vertex_buffer: 0.6076 vertex_buffer: 0.677703 - vertex_buffer: 3.8822236 + vertex_buffer: 3.882228 vertex_buffer: -3.1814423 - vertex_buffer: 4.05299 + vertex_buffer: 4.0529823 vertex_buffer: 0.631938 vertex_buffer: 0.6635 - vertex_buffer: 5.9277196 - vertex_buffer: -1.7721405 + vertex_buffer: 5.9277267 + vertex_buffer: -1.7721386 vertex_buffer: 3.2307892 vertex_buffer: 0.752033 vertex_buffer: 0.601315 - vertex_buffer: 0.9365861 - vertex_buffer: 1.3445549 - vertex_buffer: 5.05389 + vertex_buffer: 0.9365872 + vertex_buffer: 1.3445606 + vertex_buffer: 5.0538864 vertex_buffer: 0.547226 vertex_buffer: 0.420395 - vertex_buffer: 1.3117576 - vertex_buffer: 2.863121 - vertex_buffer: 3.6299706 + vertex_buffer: 1.3117594 + vertex_buffer: 2.8631287 + vertex_buffer: 3.6299667 vertex_buffer: 0.563544 vertex_buffer: 0.359828 - vertex_buffer: 1.710571 - vertex_buffer: 2.8931694 - vertex_buffer: 3.3603477 + vertex_buffer: 1.7105736 + vertex_buffer: 2.893179 + vertex_buffer: 3.360344 vertex_buffer: 0.583841 vertex_buffer: 0.368714 - vertex_buffer: 2.9408464 - vertex_buffer: -3.642933 - vertex_buffer: 4.1286964 + vertex_buffer: 2.9408498 + vertex_buffer: -3.642931 + vertex_buffer: 4.1286926 vertex_buffer: 0.586614 vertex_buffer: 0.692366 - vertex_buffer: 6.0170794 - vertex_buffer: -3.4064217 - vertex_buffer: 2.3072815 + vertex_buffer: 6.0170856 + vertex_buffer: -3.4064198 + vertex_buffer: 2.3072739 vertex_buffer: 0.771915 vertex_buffer: 0.683578 - vertex_buffer: 0.7762793 - vertex_buffer: 2.5255527 - vertex_buffer: 4.588814 + vertex_buffer: 0.7762805 + vertex_buffer: 2.5255604 + vertex_buffer: 4.58881 vertex_buffer: 0.531597 vertex_buffer: 0.352483 - vertex_buffer: 2.2923872 - vertex_buffer: -7.176118 - vertex_buffer: 4.395317 + vertex_buffer: 2.2923894 + vertex_buffer: -7.1761208 + vertex_buffer: 4.3953133 vertex_buffer: 0.588371 vertex_buffer: 0.804441 - vertex_buffer: 0.5088966 - vertex_buffer: 0.68442154 - vertex_buffer: 6.1560097 + vertex_buffer: 0.5088972 + vertex_buffer: 0.68442535 + vertex_buffer: 6.156002 vertex_buffer: 0.520797 vertex_buffer: 0.442565 - vertex_buffer: 1.4301867 - vertex_buffer: -0.42539024 + vertex_buffer: 1.4301882 + vertex_buffer: -0.42538643 vertex_buffer: 5.853489 vertex_buffer: 0.567985 vertex_buffer: 0.493479 - vertex_buffer: 1.1599193 - vertex_buffer: -7.917469 - vertex_buffer: 4.9433784 + vertex_buffer: 1.1599201 + vertex_buffer: -7.917472 + vertex_buffer: 4.9433746 vertex_buffer: 0.543283 vertex_buffer: 0.819255 - vertex_buffer: 3.8333597 - vertex_buffer: -5.020096 - vertex_buffer: 3.4186134 + vertex_buffer: 3.833364 + vertex_buffer: -5.0200977 + vertex_buffer: 3.418602 vertex_buffer: 0.655317 vertex_buffer: 0.745515 - vertex_buffer: 2.9397933 - vertex_buffer: -1.5405922 - vertex_buffer: 4.737316 + vertex_buffer: 2.9397964 + vertex_buffer: -1.5405884 + vertex_buffer: 4.7373123 vertex_buffer: 0.621009 vertex_buffer: 0.574018 - vertex_buffer: 3.1156285 - vertex_buffer: -6.183937 - vertex_buffer: 3.8408585 + vertex_buffer: 3.1156316 + vertex_buffer: -6.183938 + vertex_buffer: 3.840847 vertex_buffer: 0.62556 vertex_buffer: 0.780312 - vertex_buffer: 4.4402833 - vertex_buffer: -1.4682465 - vertex_buffer: 4.4270935 + vertex_buffer: 4.440288 + vertex_buffer: -1.4682484 + vertex_buffer: 4.427086 vertex_buffer: 0.680198 vertex_buffer: 0.570719 - vertex_buffer: 3.6551414 - vertex_buffer: -2.1533604 - vertex_buffer: 4.4729767 + vertex_buffer: 3.6551456 + vertex_buffer: -2.1533585 + vertex_buffer: 4.472973 vertex_buffer: 0.642764 vertex_buffer: 0.604338 - vertex_buffer: 5.071462 + vertex_buffer: 5.0714684 vertex_buffer: -2.356783 - vertex_buffer: 3.8250847 + vertex_buffer: 3.8250809 vertex_buffer: 0.704663 vertex_buffer: 0.62153 - vertex_buffer: 1.2959383 - vertex_buffer: -8.866041 - vertex_buffer: 5.036125 + vertex_buffer: 1.2959392 + vertex_buffer: -8.866048 + vertex_buffer: 5.0361176 vertex_buffer: 0.552012 vertex_buffer: 0.862592 - vertex_buffer: 1.8981676 - vertex_buffer: -0.56002426 - vertex_buffer: 5.423908 + vertex_buffer: 1.8981696 + vertex_buffer: -0.56001854 + vertex_buffer: 5.4239044 vertex_buffer: 0.589072 vertex_buffer: 0.508637 - vertex_buffer: 4.186279 - vertex_buffer: -5.741809 - vertex_buffer: 3.0424232 + vertex_buffer: 4.1862836 + vertex_buffer: -5.741811 + vertex_buffer: 3.0424118 vertex_buffer: 0.685945 vertex_buffer: 0.775357 - vertex_buffer: 3.320751 - vertex_buffer: -6.9206285 - vertex_buffer: 3.599121 + vertex_buffer: 3.3207548 + vertex_buffer: -6.9206305 + vertex_buffer: 3.5991096 vertex_buffer: 0.645735 vertex_buffer: 0.81264 - vertex_buffer: 4.447845 - vertex_buffer: -3.8932133 - vertex_buffer: 3.412632 + vertex_buffer: 4.44785 + vertex_buffer: -3.8932114 + vertex_buffer: 3.4126244 vertex_buffer: 0.675343 vertex_buffer: 0.703978 - vertex_buffer: 6.592035 - vertex_buffer: -2.4481583 - vertex_buffer: 1.7984543 + vertex_buffer: 6.5920415 + vertex_buffer: -2.4481525 + vertex_buffer: 1.7984428 vertex_buffer: 0.810858 vertex_buffer: 0.646305 - vertex_buffer: 5.1553717 - vertex_buffer: -4.20376 - vertex_buffer: 2.9434357 + vertex_buffer: 5.1553783 + vertex_buffer: -4.2037582 + vertex_buffer: 2.943428 vertex_buffer: 0.720122 vertex_buffer: 0.714667 - vertex_buffer: 6.7463055 - vertex_buffer: -2.9502907 - vertex_buffer: 0.47159576 + vertex_buffer: 6.7463136 + vertex_buffer: -2.9502869 + vertex_buffer: 0.47158813 vertex_buffer: 0.866152 vertex_buffer: 0.682705 - vertex_buffer: 4.296149 + vertex_buffer: 4.296153 vertex_buffer: -2.7682056 - vertex_buffer: 4.064972 + vertex_buffer: 4.064968 vertex_buffer: 0.663187 vertex_buffer: 0.644597 - vertex_buffer: 1.3760409 - vertex_buffer: 0.28308105 - vertex_buffer: 5.2958755 + vertex_buffer: 1.3760425 + vertex_buffer: 0.28308678 + vertex_buffer: 5.2958717 vertex_buffer: 0.570082 vertex_buffer: 0.466326 - vertex_buffer: 1.3695476 - vertex_buffer: -2.0489216 - vertex_buffer: 7.035427 + vertex_buffer: 1.369549 + vertex_buffer: -2.0489197 + vertex_buffer: 7.0354233 vertex_buffer: 0.544562 vertex_buffer: 0.548376 - vertex_buffer: 1.8866544 - vertex_buffer: -1.8764362 + vertex_buffer: 1.8866564 + vertex_buffer: -1.8764381 vertex_buffer: 6.3556557 vertex_buffer: 0.562759 vertex_buffer: 0.558785 - vertex_buffer: 1.0421567 - vertex_buffer: -1.8033142 + vertex_buffer: 1.0421576 + vertex_buffer: -1.8033104 vertex_buffer: 7.4998665 vertex_buffer: 0.531987 vertex_buffer: 0.53014 - vertex_buffer: 1.7499065 - vertex_buffer: 3.2145195 + vertex_buffer: 1.749909 + vertex_buffer: 3.2145271 vertex_buffer: 3.722023 vertex_buffer: 0.585271 vertex_buffer: 0.335177 - vertex_buffer: 2.6983814 - vertex_buffer: 3.4560604 + vertex_buffer: 2.6983845 + vertex_buffer: 3.456068 vertex_buffer: 3.8515701 vertex_buffer: 0.622953 vertex_buffer: 0.322779 - vertex_buffer: 3.5685785 - vertex_buffer: 3.4978447 - vertex_buffer: 3.8557777 + vertex_buffer: 3.568583 + vertex_buffer: 3.4978542 + vertex_buffer: 3.855774 vertex_buffer: 0.655896 vertex_buffer: 0.320163 - vertex_buffer: 4.353757 - vertex_buffer: 3.4455433 - vertex_buffer: 3.6230545 + vertex_buffer: 4.3537617 + vertex_buffer: 3.445551 + vertex_buffer: 3.6230507 vertex_buffer: 0.687132 vertex_buffer: 0.322346 - vertex_buffer: 4.9293575 - vertex_buffer: 3.3051414 - vertex_buffer: 3.2275352 + vertex_buffer: 4.9293637 + vertex_buffer: 3.305151 + vertex_buffer: 3.2275314 vertex_buffer: 0.716482 vertex_buffer: 0.333201 - vertex_buffer: 5.3603797 - vertex_buffer: 2.7067642 + vertex_buffer: 5.360387 + vertex_buffer: 2.7067719 vertex_buffer: 2.3262787 vertex_buffer: 0.758757 vertex_buffer: 0.382787 - vertex_buffer: 7.3424206 - vertex_buffer: 1.8563805 - vertex_buffer: -0.3018341 + vertex_buffer: 7.3424306 + vertex_buffer: 1.85639 + vertex_buffer: -0.30184937 vertex_buffer: 0.897013 vertex_buffer: 0.468769 - vertex_buffer: 4.8157883 - vertex_buffer: 1.8869534 - vertex_buffer: 2.9183197 + vertex_buffer: 4.8157945 + vertex_buffer: 1.886961 + vertex_buffer: 2.918312 vertex_buffer: 0.732392 vertex_buffer: 0.424547 - vertex_buffer: 4.117226 - vertex_buffer: 1.6413307 - vertex_buffer: 3.3260117 + vertex_buffer: 4.1172314 + vertex_buffer: 1.6413403 + vertex_buffer: 3.3260078 vertex_buffer: 0.702114 vertex_buffer: 0.433163 - vertex_buffer: 3.2758975 - vertex_buffer: 1.5367126 + vertex_buffer: 3.2759013 + vertex_buffer: 1.5367184 vertex_buffer: 3.6020355 vertex_buffer: 0.666525 vertex_buffer: 0.433866 - vertex_buffer: 2.4952793 - vertex_buffer: 1.6289577 - vertex_buffer: 3.6608086 + vertex_buffer: 2.4952826 + vertex_buffer: 1.6289654 + vertex_buffer: 3.660801 vertex_buffer: 0.633505 vertex_buffer: 0.426088 - vertex_buffer: 1.9153266 - vertex_buffer: 1.7806721 - vertex_buffer: 3.7212715 + vertex_buffer: 1.9153291 + vertex_buffer: 1.7806797 + vertex_buffer: 3.721264 vertex_buffer: 0.603876 vertex_buffer: 0.416587 - vertex_buffer: 1.5041876 - vertex_buffer: 1.947649 - vertex_buffer: 3.7355576 + vertex_buffer: 1.5041896 + vertex_buffer: 1.9476566 + vertex_buffer: 3.73555 vertex_buffer: 0.579658 vertex_buffer: 0.409945 - vertex_buffer: 7.62904 - vertex_buffer: 2.3723297 + vertex_buffer: 7.6290493 + vertex_buffer: 2.3723392 vertex_buffer: -2.5805435 vertex_buffer: 0.99244 vertex_buffer: 0.480777 - vertex_buffer: 1.9259956 - vertex_buffer: -1.8737621 + vertex_buffer: 1.9259976 + vertex_buffer: -1.8737583 vertex_buffer: 6.0489655 vertex_buffer: 0.567192 vertex_buffer: 0.56942 - vertex_buffer: 0.9361145 - vertex_buffer: -0.23275948 + vertex_buffer: 0.9361154 + vertex_buffer: -0.23275757 vertex_buffer: 6.340225 vertex_buffer: 0.541366 vertex_buffer: 0.478899 - vertex_buffer: 0.8889647 - vertex_buffer: -2.3080616 - vertex_buffer: 7.563141 + vertex_buffer: 0.8889656 + vertex_buffer: -2.3080597 + vertex_buffer: 7.5631332 vertex_buffer: 0.526564 vertex_buffer: 0.546118 - vertex_buffer: 0.57550555 + vertex_buffer: 0.575506 vertex_buffer: -2.5688763 vertex_buffer: 7.3445854 vertex_buffer: 0.523913 vertex_buffer: 0.56383 - vertex_buffer: 0.9179589 - vertex_buffer: -2.3308792 - vertex_buffer: 7.255516 + vertex_buffer: 0.9179598 + vertex_buffer: -2.3308773 + vertex_buffer: 7.255512 vertex_buffer: 0.531529 vertex_buffer: 0.555057 - vertex_buffer: 1.8029492 - vertex_buffer: -1.9879112 - vertex_buffer: 5.845417 + vertex_buffer: 1.8029512 + vertex_buffer: -1.9879055 + vertex_buffer: 5.845413 vertex_buffer: 0.566036 vertex_buffer: 0.582329 - vertex_buffer: 0.44685143 + vertex_buffer: 0.44685158 vertex_buffer: -2.682705 - vertex_buffer: 7.525955 + vertex_buffer: 7.5259476 vertex_buffer: 0.516311 vertex_buffer: 0.563054 - vertex_buffer: 0.40712923 - vertex_buffer: -2.5848274 - vertex_buffer: 6.774906 + vertex_buffer: 0.40712938 + vertex_buffer: -2.5848255 + vertex_buffer: 6.7748985 vertex_buffer: 0.517472 vertex_buffer: 0.577877 - vertex_buffer: 1.5503142 - vertex_buffer: 2.5380096 + vertex_buffer: 1.5503162 + vertex_buffer: 2.5380173 vertex_buffer: 3.3313866 vertex_buffer: 0.573595 vertex_buffer: 0.389807 - vertex_buffer: 1.2257941 - vertex_buffer: 2.2664852 - vertex_buffer: 3.7727013 + vertex_buffer: 1.2257959 + vertex_buffer: 2.266489 + vertex_buffer: 3.7726974 vertex_buffer: 0.560698 vertex_buffer: 0.395332 - vertex_buffer: 1.0358994 - vertex_buffer: 1.9741764 - vertex_buffer: 4.3474655 + vertex_buffer: 1.0359008 + vertex_buffer: 1.974184 + vertex_buffer: 4.3474617 vertex_buffer: 0.549756 vertex_buffer: 0.399751 - vertex_buffer: 4.367508 - vertex_buffer: 2.8476372 - vertex_buffer: 2.9235992 + vertex_buffer: 4.3675127 + vertex_buffer: 2.8476448 + vertex_buffer: 2.9235954 vertex_buffer: 0.710288 vertex_buffer: 0.368253 - vertex_buffer: 4.9116855 - vertex_buffer: 3.011652 + vertex_buffer: 4.911691 + vertex_buffer: 3.0116596 vertex_buffer: 2.8815002 vertex_buffer: 0.72333 vertex_buffer: 0.363373 @@ -5041,20 +5041,20 @@ mesh { pose_transform_matrix { rows: 4 cols: 4 - packed_data: 0.99995184 - packed_data: 0.006250852 - packed_data: -0.0075720036 + packed_data: 0.9999519 + packed_data: 0.006250915 + packed_data: -0.007572061 packed_data: 0 - packed_data: -0.0060578818 - packed_data: 0.9996628 - packed_data: 0.025243768 + packed_data: -0.006057945 + packed_data: 0.9996629 + packed_data: 0.025243366 packed_data: 0 - packed_data: 0.0077272463 - packed_data: -0.025196675 - packed_data: 0.9996526 + packed_data: 0.0077273026 + packed_data: -0.025196271 + packed_data: 0.9996527 packed_data: 0 - packed_data: -0.35120884 - packed_data: 21.932339 - packed_data: -64.35148 + packed_data: -0.3512094 + packed_data: 21.932364 + packed_data: -64.351555 packed_data: 1 } From 87c1af48a167d8f32826f0c3ffac4e7ec10a8752 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 14 Nov 2024 02:23:12 -0800 Subject: [PATCH 055/126] Improve logging to allow users to understand 1) which InferenceCalculator backend is used (without extra VLOG flags) and 2) when a model is loaded (including its size). PiperOrigin-RevId: 696444779 --- mediapipe/calculators/tensor/BUILD | 1 + .../calculators/tensor/inference_calculator.cc | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index a6717c4319..ce84f4b18c 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -454,6 +454,7 @@ cc_library_with_tflite( "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", "//mediapipe/framework/tool:subgraph_expansion", "//mediapipe/util:cpu_util", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/mediapipe/calculators/tensor/inference_calculator.cc b/mediapipe/calculators/tensor/inference_calculator.cc index bf60ea8e65..a29e0fb7ec 100644 --- a/mediapipe/calculators/tensor/inference_calculator.cc +++ b/mediapipe/calculators/tensor/inference_calculator.cc @@ -19,6 +19,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" @@ -118,8 +119,16 @@ absl::StatusOr> InferenceCalculator::GetModelAsPacket( CalculatorContext* cc) { const auto& options = cc->Options(); if (!options.model_path().empty()) { - return TfLiteModelLoader::LoadFromPath( - cc->GetResources(), options.model_path(), options.try_mmap_model()); + MP_ASSIGN_OR_RETURN( + auto model, TfLiteModelLoader::LoadFromPath(cc->GetResources(), + options.model_path(), + options.try_mmap_model())); + ABSL_CHECK(!model.IsEmpty()); + VLOG(1) << absl::StrFormat( + "GetModelAsPacket successfully loaded model " + "(path: %s, size: %ld bytes)", + options.model_path(), model.Get()->allocation()->bytes()); + return model; } if (!kSideInModel(cc).IsEmpty()) return kSideInModel(cc); return absl::Status(absl::StatusCode::kNotFound, From f53af03b07aaafb857bcff0559541b4c53e63d3e Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 14 Nov 2024 08:11:49 -0800 Subject: [PATCH 056/126] Internal update PiperOrigin-RevId: 696527818 --- .../face_landmarks_detector_graph.cc | 1 + .../holistic_face_tracking_test.cc | 49 ++++++++----------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_detector_graph.cc b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_detector_graph.cc index 4ab3a40e54..e129076aba 100644 --- a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_detector_graph.cc +++ b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_detector_graph.cc @@ -335,6 +335,7 @@ class SingleFaceLandmarksDetectorGraph : public core::ModelTaskGraph { auto& landmark_projection = graph.AddNode("LandmarkProjectionCalculator"); landmarks_letterbox_removed >> landmark_projection.In(kNormLandmarksTag); face_rect >> landmark_projection.In(kNormRectTag); + image_size >> landmark_projection.In("IMAGE_DIMENSIONS"); Stream projected_landmarks = AllowIf( landmark_projection[Output(kNormLandmarksTag)], presence, graph); diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking_test.cc b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking_test.cc index 314c330b3c..89a41266ac 100644 --- a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking_test.cc +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking_test.cc @@ -82,9 +82,9 @@ constexpr char kImageInStream[] = "image_in"; constexpr char kPoseLandmarksInStream[] = "pose_landmarks_in"; constexpr char kFaceLandmarksOutStream[] = "face_landmarks_out"; constexpr char kRenderedImageOutStream[] = "rendered_image_out"; -constexpr char kFaceDetectorTFLiteName[] = "face_detector.tflite"; -constexpr char kFaceLandmarksDetectorTFLiteName[] = - "face_landmarks_detector.tflite"; +constexpr char kFaceDetectionModelFile[] = "face_detection_short_range.tflite"; +constexpr char kFaceLandmarksModelFile[] = + "facemesh2_lite_iris_faceflag_2023_02_14.tflite"; std::string GetFilePath(absl::string_view filename) { return file::JoinPath("./", kTestDataDirectory, filename); @@ -108,6 +108,16 @@ mediapipe::LandmarksToRenderDataCalculatorOptions GetFaceRendererOptions() { return render_options; } +mediapipe::RectToRenderDataCalculatorOptions GetRectRendererOptions() { + mediapipe::RectToRenderDataCalculatorOptions render_options; + render_options.set_filled(false); + render_options.mutable_color()->set_r(255); + render_options.mutable_color()->set_g(0); + render_options.mutable_color()->set_b(0); + render_options.set_thickness(2); + return render_options; +} + absl::StatusOr> CreateModelAssetBundleResources(const std::string& model_asset_filename) { auto external_model_bundle = std::make_unique(); @@ -127,38 +137,19 @@ absl::StatusOr> CreateTaskRunner() { Stream face_landmarks_from_pose = SplitToRanges(pose_landmarks, {{0, 11}}, graph)[0]; // Create face landmarker model bundle. - MP_ASSIGN_OR_RETURN( - auto model_bundle, - CreateModelAssetBundleResources(GetFilePath("face_landmarker_v2.task"))); face_detector::proto::FaceDetectorGraphOptions detector_options; face_landmarker::proto::FaceLandmarksDetectorGraphOptions landmarks_detector_options; // Set face detection model. - MP_ASSIGN_OR_RETURN(auto face_detector_model_file, - model_bundle->GetFile(kFaceDetectorTFLiteName)); - core::proto::FilePointerMeta face_detection_file_pointer; - face_detection_file_pointer.set_pointer( - reinterpret_cast(face_detector_model_file.data())); - face_detection_file_pointer.set_length(face_detector_model_file.size()); - detector_options.mutable_base_options() - ->mutable_model_asset() - ->mutable_file_pointer_meta() - ->Swap(&face_detection_file_pointer); detector_options.set_num_faces(1); + detector_options.mutable_base_options()->mutable_model_asset()->set_file_name( + GetFilePath(kFaceDetectionModelFile)); // Set face landmarks model. - MP_ASSIGN_OR_RETURN(auto face_landmarks_model_file, - model_bundle->GetFile(kFaceLandmarksDetectorTFLiteName)); - core::proto::FilePointerMeta face_landmarks_detector_file_pointer; - face_landmarks_detector_file_pointer.set_pointer( - reinterpret_cast(face_landmarks_model_file.data())); - face_landmarks_detector_file_pointer.set_length( - face_landmarks_model_file.size()); landmarks_detector_options.mutable_base_options() ->mutable_model_asset() - ->mutable_file_pointer_meta() - ->Swap(&face_landmarks_detector_file_pointer); + ->set_file_name(GetFilePath(kFaceLandmarksModelFile)); // Track holistic face. HolisticFaceTrackingRequest request; @@ -173,10 +164,11 @@ absl::StatusOr> CreateTaskRunner() { auto render_scale = utils::GetRenderScale( image_size, result.debug_output.roi_from_pose, 0.0001, graph); - auto face_landmarks_render_data = utils::RenderLandmarks( - face_landmarks, render_scale, GetFaceRendererOptions(), graph); std::vector> render_list = { - face_landmarks_render_data}; + utils::RenderLandmarks(face_landmarks, render_scale, + GetFaceRendererOptions(), graph), + utils::RenderRect(result.debug_output.roi_from_pose, + GetRectRendererOptions(), graph)}; auto rendered_image = utils::Render( @@ -209,6 +201,7 @@ TEST_F(HolisticFaceTrackingTest, SmokeTest) { holistic_result.pose_landmarks())}})); ASSERT_TRUE(output_packets.find(kFaceLandmarksOutStream) != output_packets.end()); + ASSERT_FALSE(output_packets.find(kFaceLandmarksOutStream)->second.IsEmpty()); auto face_landmarks = output_packets.find(kFaceLandmarksOutStream) ->second.Get(); EXPECT_THAT( From 1718461bc4fbaff9cc213b5ec5525a76df2e7e0f Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 14 Nov 2024 11:10:42 -0800 Subject: [PATCH 057/126] Add SharedFD type PiperOrigin-RevId: 696587436 --- mediapipe/framework/formats/BUILD | 22 +++++++ mediapipe/framework/formats/shared_fd.h | 66 +++++++++++++++++++ mediapipe/framework/formats/shared_fd_test.cc | 64 ++++++++++++++++++ mediapipe/framework/formats/unique_fd_test.cc | 10 +-- mediapipe/util/BUILD | 7 ++ mediapipe/util/fd_test_util.h | 17 +++++ 6 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 mediapipe/framework/formats/shared_fd.h create mode 100644 mediapipe/framework/formats/shared_fd_test.cc create mode 100644 mediapipe/util/fd_test_util.h diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index b5d93530e4..66ca60ac7d 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -319,6 +319,28 @@ cc_test( ":unique_fd", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:status_matchers", + "//mediapipe/util:fd_test_util", + ], +) + +cc_library( + name = "shared_fd", + hdrs = ["shared_fd.h"], + deps = [ + ":unique_fd", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_test( + name = "shared_fd_test", + srcs = ["shared_fd_test.cc"], + deps = [ + ":shared_fd", + ":unique_fd", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:status_matchers", + "//mediapipe/util:fd_test_util", ], ) diff --git a/mediapipe/framework/formats/shared_fd.h b/mediapipe/framework/formats/shared_fd.h new file mode 100644 index 0000000000..1c6ed105b1 --- /dev/null +++ b/mediapipe/framework/formats/shared_fd.h @@ -0,0 +1,66 @@ +#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_SHARED_FD_H_ +#define MEDIAPIPE_FRAMEWORK_FORMATS_SHARED_FD_H_ + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/unique_fd.h" + +namespace mediapipe { + +// Provides a shared ownership for a file descriptor. +// +// File descriptor is closed as soon as last SharedFd is destroyed. +// (Uses `std::shared_ptr` internally and can be used in the same way: copy, +// move, assign/compare with nullptr, use in conditional statements.) +class SharedFd { + public: + // `fd` a valid file descriptor. + explicit SharedFd(UniqueFd fd) + : fd_(std::make_shared(std::move(fd))) {} + + // Constructs empty SharedFd (fd == nullptr evaluates to true) + SharedFd() = default; + + ~SharedFd() = default; + + // Copyable + SharedFd(const SharedFd&) = default; + SharedFd& operator=(const SharedFd&) = default; + + // Moveable + SharedFd(SharedFd&& other) = default; + SharedFd& operator=(SharedFd&& other) = default; + + // Resets this SharedFd object (fd == nullptr will evaluate to true). + SharedFd& operator=(std::nullptr_t other) { + fd_ = other; + return *this; + } + + bool operator==(std::nullptr_t other) const { return fd_ == other; } + bool operator!=(std::nullptr_t other) const { return !operator==(other); }; + + // SharedFd can be used in conditional statements: + // ``` + // if (fd) { + // int raw_fd = fd.Get(); + // } + // ``` + explicit operator bool() const { return operator!=(nullptr); } + + // Gets raw file descriptor for read purposes. + int Get() const { return fd_->Get(); } + + // Duplicates file descriptor. + absl::StatusOr Dup() const { return fd_->Dup(); } + + private: + std::shared_ptr fd_; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_FRAMEWORK_FORMATS_SHARED_FD_H_ diff --git a/mediapipe/framework/formats/shared_fd_test.cc b/mediapipe/framework/formats/shared_fd_test.cc new file mode 100644 index 0000000000..0d8d22e6a8 --- /dev/null +++ b/mediapipe/framework/formats/shared_fd_test.cc @@ -0,0 +1,64 @@ +#include "mediapipe/framework/formats/shared_fd.h" + +#include + +#include "mediapipe/framework/formats/unique_fd.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/util/fd_test_util.h" + +namespace mediapipe { +namespace { + +TEST(SharedFdTest, CanCreateFromUniqueFd) { + int raw_fd = GetValidFd(); + { + auto fd = SharedFd(UniqueFd(raw_fd)); + EXPECT_TRUE(IsFdValid(fd.Get())); + } + EXPECT_FALSE(IsFdValid(raw_fd)); +} + +TEST(SharedFdTest, CanCopyAndMoveFd) { + int raw_fd = GetValidFd(); + auto fd = SharedFd(UniqueFd(raw_fd)); + { + SharedFd copied_fd = fd; + EXPECT_TRUE(IsFdValid(copied_fd.Get())); + } + EXPECT_TRUE(IsFdValid(fd.Get())); + + { + SharedFd moved_fd = std::move(fd); + EXPECT_TRUE(IsFdValid(moved_fd.Get())); + } + EXPECT_FALSE(IsFdValid(raw_fd)); +} + +TEST(SharedFdTest, CanBeAssignedAndComparedWithNullptr) { + SharedFd fd; + EXPECT_FALSE(fd); + EXPECT_EQ(fd, nullptr); + + int raw_fd = GetValidFd(); + fd = SharedFd(UniqueFd(raw_fd)); + + EXPECT_NE(fd, nullptr); + EXPECT_TRUE(fd); + + fd = nullptr; + EXPECT_FALSE(IsFdValid(raw_fd)); + EXPECT_EQ(fd, nullptr); + EXPECT_FALSE(fd); +} + +TEST(SharedFdTest, CanDup) { + int raw_fd = GetValidFd(); + auto fd = SharedFd(UniqueFd(GetValidFd())); + MP_ASSERT_OK_AND_ASSIGN(UniqueFd dup_fd, fd.Dup()); + EXPECT_NE(dup_fd.Get(), raw_fd); + EXPECT_TRUE(IsFdValid(dup_fd.Get())); +} + +} // namespace +} // namespace mediapipe diff --git a/mediapipe/framework/formats/unique_fd_test.cc b/mediapipe/framework/formats/unique_fd_test.cc index 7a6f4556d4..a900b53be9 100644 --- a/mediapipe/framework/formats/unique_fd_test.cc +++ b/mediapipe/framework/formats/unique_fd_test.cc @@ -1,23 +1,15 @@ #include "mediapipe/framework/formats/unique_fd.h" -#include -#include - #include #include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/util/fd_test_util.h" namespace mediapipe { namespace { -// Returns a valid system file descriptor. -int GetValidFd() { return dup(STDOUT_FILENO); } - -// Helper function to check if the file descriptor is valid (still open). -int IsFdValid(int fd) { return fcntl(fd, F_GETFD) != -1; } - TEST(UniqueFdTest, ShouldInitializeInvalidFd) { UniqueFd unique_fd; EXPECT_FALSE(unique_fd.IsValid()); diff --git a/mediapipe/util/BUILD b/mediapipe/util/BUILD index 25844a580d..c388ce5529 100644 --- a/mediapipe/util/BUILD +++ b/mediapipe/util/BUILD @@ -97,6 +97,13 @@ cc_library( }), ) +cc_library( + name = "fd_test_util", + testonly = True, + hdrs = ["fd_test_util.h"], + visibility = ["//mediapipe:__subpackages__"], +) + cc_library( name = "header_util", srcs = ["header_util.cc"], diff --git a/mediapipe/util/fd_test_util.h b/mediapipe/util/fd_test_util.h new file mode 100644 index 0000000000..0d2241d91c --- /dev/null +++ b/mediapipe/util/fd_test_util.h @@ -0,0 +1,17 @@ +#ifndef MEDIAPIPE_UTIL_FD_TEST_UTIL_H_ +#define MEDIAPIPE_UTIL_FD_TEST_UTIL_H_ + +#include +#include + +namespace mediapipe { + +// Returns a valid system file descriptor. +inline int GetValidFd() { return dup(STDOUT_FILENO); } + +// Helper function to check if the file descriptor is valid (still open). +inline int IsFdValid(int fd) { return fcntl(fd, F_GETFD) != -1; } + +} // namespace mediapipe + +#endif // MEDIAPIPE_UTIL_FD_TEST_UTIL_H_ From a1d9f3c67ec60ec8755b6ce74faacc5e9d763624 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 14 Nov 2024 11:16:35 -0800 Subject: [PATCH 058/126] Udpate SyncWait/IsSignaled to work with SharedFd. PiperOrigin-RevId: 696589669 --- mediapipe/util/BUILD | 2 ++ mediapipe/util/sync_wait.cc | 11 ++++++++++ mediapipe/util/sync_wait.h | 9 ++++++++ mediapipe/util/sync_wait_test.cc | 37 +++++++++++++++++++++++++------- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/mediapipe/util/BUILD b/mediapipe/util/BUILD index c388ce5529..2d8f851857 100644 --- a/mediapipe/util/BUILD +++ b/mediapipe/util/BUILD @@ -208,6 +208,7 @@ cc_library( hdrs = ["sync_wait.h"], visibility = ["//mediapipe:__subpackages__"], deps = [ + "//mediapipe/framework/formats:shared_fd", "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", @@ -224,6 +225,7 @@ cc_test( deps = [ ":sync_wait", "//mediapipe/framework:port", + "//mediapipe/framework/formats:shared_fd", "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:benchmark", "//mediapipe/framework/port:gtest_main", diff --git a/mediapipe/util/sync_wait.cc b/mediapipe/util/sync_wait.cc index 3e8705a7aa..72c87b7d59 100644 --- a/mediapipe/util/sync_wait.cc +++ b/mediapipe/util/sync_wait.cc @@ -10,6 +10,7 @@ #include "absl/status/statusor.h" #include "absl/strings/str_format.h" #include "absl/time/time.h" +#include "mediapipe/framework/formats/shared_fd.h" #include "mediapipe/framework/formats/unique_fd.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status_macros.h" @@ -53,6 +54,11 @@ absl::Status SyncWait(const UniqueFd& fd, absl::Duration timeout) { return SyncWait(fd.Get(), timeout); } +absl::Status SyncWait(const SharedFd& fd, absl::Duration timeout) { + RET_CHECK(fd); + return SyncWait(fd.Get(), timeout); +} + absl::StatusOr IsSignaled(int fd) { RET_CHECK_GE(fd, 0) << "Invalid file descriptor."; @@ -79,4 +85,9 @@ absl::StatusOr IsSignaled(const UniqueFd& fd) { return IsSignaled(fd.Get()); } +absl::StatusOr IsSignaled(const SharedFd& fd) { + RET_CHECK(fd); + return IsSignaled(fd.Get()); +} + } // namespace mediapipe diff --git a/mediapipe/util/sync_wait.h b/mediapipe/util/sync_wait.h index ba3d9d4b40..62935a8939 100644 --- a/mediapipe/util/sync_wait.h +++ b/mediapipe/util/sync_wait.h @@ -4,6 +4,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/time/time.h" +#include "mediapipe/framework/formats/shared_fd.h" #include "mediapipe/framework/formats/unique_fd.h" namespace mediapipe { @@ -18,12 +19,20 @@ absl::Status SyncWait(int fd, absl::Duration timeout); // signaled. absl::Status SyncWait(const UniqueFd& fd, absl::Duration timeout); +// `fd` - represents a sync +// `timeout` - wait timeout, pass `absl::InfiniteDuration()` to wait until +// signaled. +absl::Status SyncWait(const SharedFd& fd, absl::Duration timeout); + // Checks if sync represented by `fd` is signaled. absl::StatusOr IsSignaled(int fd); // Checks if sync represented by `fd` is signaled. absl::StatusOr IsSignaled(const UniqueFd& fd); +// Checks if sync represented by `fd` is signaled. +absl::StatusOr IsSignaled(const SharedFd& fd); + } // namespace mediapipe #endif // MEDIAPIPE_UTIL_SYNC_WAIT_H_ diff --git a/mediapipe/util/sync_wait_test.cc b/mediapipe/util/sync_wait_test.cc index eb5cfc66ee..dde1dae66b 100644 --- a/mediapipe/util/sync_wait_test.cc +++ b/mediapipe/util/sync_wait_test.cc @@ -2,9 +2,12 @@ #include +#include + #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/time/time.h" +#include "mediapipe/framework/formats/shared_fd.h" #include "mediapipe/framework/formats/unique_fd.h" #include "mediapipe/framework/port.h" // IWYU pragma: keep (DRIHSTI_OSX) #include "mediapipe/framework/port/benchmark.h" @@ -60,33 +63,39 @@ TestTimer CreateTestTimer(absl::Duration duration) { TEST(SyncWait, WorksWithIndefiniteTimeout) { TestTimer timer = CreateTestTimer(absl::Milliseconds(2)); - MP_EXPECT_OK(mediapipe::SyncWait(timer.fd, absl::InfiniteDuration())); + MP_EXPECT_OK(SyncWait(timer.fd, absl::InfiniteDuration())); +} + +TEST(SyncWait, WorksWithSharedFd) { + TestTimer timer = CreateTestTimer(absl::Milliseconds(2)); + SharedFd fd(std::move(timer).fd); + MP_EXPECT_OK(SyncWait(fd, absl::InfiniteDuration())); } TEST(SyncWait, WorksWithDefiniteTimeout) { TestTimer timer = CreateTestTimer(absl::Milliseconds(5)); - MP_EXPECT_OK(mediapipe::SyncWait(timer.fd, absl::Milliseconds(10))); + MP_EXPECT_OK(SyncWait(timer.fd, absl::Milliseconds(10))); } TEST(SyncWait, WorksWithReadyFd) { TestTimer timer = CreateTestTimer(absl::Milliseconds(5)); // timer.fd is not available for read - MP_EXPECT_OK(mediapipe::SyncWait(timer.fd, absl::InfiniteDuration())); + MP_EXPECT_OK(SyncWait(timer.fd, absl::InfiniteDuration())); // timer.fd is available for read - MP_EXPECT_OK(mediapipe::SyncWait(timer.fd, absl::InfiniteDuration())); - MP_EXPECT_OK(mediapipe::SyncWait(timer.fd, absl::Milliseconds(1))); + MP_EXPECT_OK(SyncWait(timer.fd, absl::InfiniteDuration())); + MP_EXPECT_OK(SyncWait(timer.fd, absl::Milliseconds(1))); } TEST(SyncWait, ReportsTimeout) { TestTimer timer = CreateTestTimer(absl::Milliseconds(100)); - EXPECT_THAT(mediapipe::SyncWait(timer.fd, absl::Milliseconds(5)), + EXPECT_THAT(SyncWait(timer.fd, absl::Milliseconds(5)), StatusIs(absl::StatusCode::kDeadlineExceeded)); } TEST(SyncWait, ReportsInvalidFd) { const int fd = -1; - EXPECT_THAT(mediapipe::SyncWait(fd, absl::InfiniteDuration()), + EXPECT_THAT(SyncWait(fd, absl::InfiniteDuration()), StatusIs(absl::StatusCode::kInternal)); } @@ -101,6 +110,18 @@ TEST(SyncWait, IsSignaledWorks) { EXPECT_TRUE(is_signaled); } +TEST(SyncWait, IsSignaledWorksWithSharedFd) { + TestTimer timer = CreateTestTimer(absl::Milliseconds(100)); + SharedFd fd(std::move(timer).fd); + MP_ASSERT_OK_AND_ASSIGN(bool is_signaled, IsSignaled(fd)); + EXPECT_FALSE(is_signaled); + + MP_ASSERT_OK(SyncWait(fd, absl::InfiniteDuration())); + + MP_ASSERT_OK_AND_ASSIGN(is_signaled, IsSignaled(fd)); + EXPECT_TRUE(is_signaled); +} + TEST(SyncWait, IsSignaledReportsInvalidFd) { const int fd = -1; EXPECT_THAT(IsSignaled(fd), StatusIs(absl::StatusCode::kInternal)); @@ -111,7 +132,7 @@ void BM_SyncWaitZeroTimeout(benchmark::State& state) { // benchmark completion. TestTimer timer = CreateTestTimer(absl::Minutes(1)); for (auto s : state) { - ABSL_CHECK_EQ(mediapipe::SyncWait(timer.fd, absl::ZeroDuration()).code(), + ABSL_CHECK_EQ(SyncWait(timer.fd, absl::ZeroDuration()).code(), absl::StatusCode::kDeadlineExceeded); } } From be739af1fb369f6def432bb20ccdfc4dcb0adbba Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 14 Nov 2024 11:34:25 -0800 Subject: [PATCH 059/126] Enable SharedFd usage in EglSync PiperOrigin-RevId: 696595949 --- mediapipe/gpu/BUILD | 1 + mediapipe/gpu/egl_sync.cc | 29 ++++++++++++++++++++--------- mediapipe/gpu/egl_sync.h | 13 ++++++++++++- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 39f8f2cc1e..5080d86fdf 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -880,6 +880,7 @@ cc_library( ":egl_base", ":egl_errors", "//mediapipe/framework/deps:no_destructor", + "//mediapipe/framework/formats:shared_fd", "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", diff --git a/mediapipe/gpu/egl_sync.cc b/mediapipe/gpu/egl_sync.cc index 56cf886fe8..d55c744b8c 100644 --- a/mediapipe/gpu/egl_sync.cc +++ b/mediapipe/gpu/egl_sync.cc @@ -11,6 +11,7 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/deps/no_destructor.h" +#include "mediapipe/framework/formats/shared_fd.h" #include "mediapipe/framework/formats/unique_fd.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status_macros.h" @@ -115,24 +116,22 @@ absl::StatusOr EglSync::CreateNative(EGLDisplay display) { } absl::StatusOr EglSync::CreateNative(EGLDisplay display, - const UniqueFd& native_fence_fd) { - RET_CHECK(native_fence_fd.IsValid()); + int native_fence_fd) { MP_RETURN_IF_ERROR(CheckEglSyncSupported(display)); MP_RETURN_IF_ERROR(CheckEglNativeSyncSupported(display)); - MP_ASSIGN_OR_RETURN(UniqueFd fd_for_egl, native_fence_fd.Dup()); - // NOTE: it looks like one could rely on `UniqueFd` for the cleanup, but - // there's some clashing on ownership of the FD when passing it to - // eglCreateSyncKHR and then using `UniqueFd::Release`, hence relying on - // absl::Cleanup. - const int fd = fd_for_egl.Release(); + // NOTE: cannot use `UniqueFd`, as there's clashing on ownership of the FD + // when passing it to eglCreateSyncKHR (which takes the ownership of the FD) + // which makes `UniqueFd` to be in an invalid state and there are related + // fdsan issues, hence relying on absl::Cleanup. + const int fd = dup(native_fence_fd); absl::Cleanup fd_cleanup = [fd]() { close(fd); }; const EGLint sync_attribs[] = {EGL_SYNC_NATIVE_FENCE_FD_ANDROID, static_cast(fd), EGL_NONE}; const EGLSyncKHR egl_sync = eglCreateSyncKHR(display, EGL_SYNC_NATIVE_FENCE_ANDROID, sync_attribs); RET_CHECK_NE(egl_sync, EGL_NO_SYNC_KHR) << absl::StrCat( - "CreateNative/eglCreateSyncKHR with original FD: ", native_fence_fd.Get(), + "CreateNative/eglCreateSyncKHR with original FD: ", native_fence_fd, " and dup FD: ", fd, " - failed: ", GetEglError()); // EGL took ownership of the passed FD as eglCreateSyncKHR succeeded, so // cancelling the cleanup. @@ -141,6 +140,18 @@ absl::StatusOr EglSync::CreateNative(EGLDisplay display, return EglSync(display, egl_sync); } +absl::StatusOr EglSync::CreateNative(EGLDisplay display, + const UniqueFd& native_fence_fd) { + RET_CHECK(native_fence_fd.IsValid()); + return CreateNative(display, native_fence_fd.Get()); +} + +absl::StatusOr EglSync::CreateNative(EGLDisplay display, + const SharedFd& native_fence_fd) { + RET_CHECK(native_fence_fd); + return CreateNative(display, native_fence_fd.Get()); +} + bool EglSync::IsSupported(EGLDisplay display) { return CheckEglSyncSupported(display).ok(); } diff --git a/mediapipe/gpu/egl_sync.h b/mediapipe/gpu/egl_sync.h index a4566b033b..81ae5e0701 100644 --- a/mediapipe/gpu/egl_sync.h +++ b/mediapipe/gpu/egl_sync.h @@ -3,6 +3,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "mediapipe/framework/formats/shared_fd.h" #include "mediapipe/framework/formats/unique_fd.h" #include "mediapipe/gpu/egl_base.h" @@ -19,10 +20,14 @@ class EglSync { // *not* flushed. static absl::StatusOr CreateNative(EGLDisplay display); - // Create a native fence in OpenGL command stream based on a native fence FD. + // Creates a native fence in OpenGL command stream based on a native fence FD. static absl::StatusOr CreateNative(EGLDisplay display, const UniqueFd& native_fence_fd); + // Creates a native fence in OpenGL command stream based on a native fence FD. + static absl::StatusOr CreateNative(EGLDisplay display, + const SharedFd& native_fence_fd); + static bool IsSupported(EGLDisplay display); static bool IsNativeSupported(EGLDisplay display); @@ -58,6 +63,12 @@ class EglSync { EglSync(EGLDisplay display, EGLSyncKHR sync) : display_(display), sync_(sync) {} + // `native_fence_fd` - valid native fence FD. + // NOTE: this function duplicates `native_fence_fd` (doesn't take ownership or + // modifies it) + static absl::StatusOr CreateNative(EGLDisplay display, + int native_fence_fd); + void Invalidate(); EGLDisplay display_; From 9b50ea4c37e115a0d6671fbbc8c067bee14d2705 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 14 Nov 2024 11:57:44 -0800 Subject: [PATCH 060/126] Internal change. PiperOrigin-RevId: 696603490 --- mediapipe/framework/formats/BUILD | 2 +- mediapipe/framework/formats/ahwb_view.h | 15 +++++++-------- mediapipe/gpu/BUILD | 2 ++ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index 66ca60ac7d..f73d7e69f1 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -130,7 +130,7 @@ cc_library( deps = [ ":hardware_buffer", "//mediapipe/framework:port", - "//mediapipe/framework/formats:unique_fd", + "//mediapipe/framework/formats:shared_fd", "//mediapipe/gpu:gpu_buffer_storage", ], ) diff --git a/mediapipe/framework/formats/ahwb_view.h b/mediapipe/framework/formats/ahwb_view.h index 8703c9dc53..249a7d91ad 100644 --- a/mediapipe/framework/formats/ahwb_view.h +++ b/mediapipe/framework/formats/ahwb_view.h @@ -11,7 +11,7 @@ #include "absl/functional/any_invocable.h" #include "absl/status/status.h" #include "mediapipe/framework/formats/hardware_buffer.h" -#include "mediapipe/framework/formats/unique_fd.h" +#include "mediapipe/framework/formats/shared_fd.h" #include "mediapipe/gpu/gpu_buffer_storage.h" namespace mediapipe { @@ -27,9 +27,9 @@ namespace mediapipe { class AhwbView { public: - explicit AhwbView(HardwareBuffer* ahwb, int width_step_bytes, - absl::AnyInvocable)> - set_usage_fence_fn) + explicit AhwbView( + HardwareBuffer* ahwb, int width_step_bytes, + absl::AnyInvocable set_usage_fence_fn) : ahwb_(ahwb), width_step_bytes_(width_step_bytes), set_usage_fence_fn_(std::move(set_usage_fence_fn)) {} @@ -68,15 +68,14 @@ class AhwbView { // following GL operations wait for write completion. // // TODO: b/376753887 - replace with a dedicated type (MP's Fence) - absl::Status SetUsageFence(std::shared_ptr fence) { - return set_usage_fence_fn_(fence); + absl::Status SetUsageFence(SharedFd fence) { + return set_usage_fence_fn_(std::move(fence)); } private: const HardwareBuffer* ahwb_; const int width_step_bytes_; - absl::AnyInvocable)> - set_usage_fence_fn_; + absl::AnyInvocable set_usage_fence_fn_; }; namespace internal { diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 5080d86fdf..ef1768d9e9 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -567,6 +567,7 @@ cc_library( "//mediapipe/framework/formats:ahwb_view", "//mediapipe/framework/formats:hardware_buffer", "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:shared_fd", "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:ret_check", "//mediapipe/util:sync_wait", @@ -1391,6 +1392,7 @@ mediapipe_cc_test( "//mediapipe/framework/formats:ahwb_view", "//mediapipe/framework/formats:hardware_buffer", "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:shared_fd", "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/tool:test_util", From 75562225e2b19604036ee72b80db28c5ab7d283a Mon Sep 17 00:00:00 2001 From: Martin Huschenbett Date: Thu, 14 Nov 2024 15:07:06 -0800 Subject: [PATCH 061/126] No public description PiperOrigin-RevId: 696664069 --- mediapipe/model_maker/python/vision/object_detector/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/model_maker/python/vision/object_detector/model.py b/mediapipe/model_maker/python/vision/object_detector/model.py index ea78ca8c62..317d686885 100644 --- a/mediapipe/model_maker/python/vision/object_detector/model.py +++ b/mediapipe/model_maker/python/vision/object_detector/model.py @@ -256,7 +256,7 @@ def export_saved_model(self, save_path: str): # The remaining method overrides are used to train this object detector model # using model.fit(). - def call( + def call( # pytype: disable=annotation-type-mismatch self, images: Union[tf.Tensor, Sequence[tf.Tensor]], image_shape: Optional[tf.Tensor] = None, From d50c562187326c1e67da29aa393357e49f6ffbbe Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 14 Nov 2024 16:19:19 -0800 Subject: [PATCH 062/126] No public description PiperOrigin-RevId: 696685132 --- WORKSPACE | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/WORKSPACE b/WORKSPACE index 21edd7667f..39fc185346 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -814,3 +814,17 @@ http_archive( strip_prefix = "abseil-py-1.4.0", urls = ["https://github.com/abseil/abseil-py/archive/refs/tags/v1.4.0.tar.gz"], ) + +http_archive( + name = "skia", + sha256 = "038d4a21f9c72d71ab49e3a7d7677b39585329465d093a4260b6c73d2f3984d6", + strip_prefix = "skia-ac75382cb971d2f5465b4608a74561ecb68599c5", + urls = ["https://github.com/google/skia/archive/ac75382cb971d2f5465b4608a74561ecb68599c5.zip"], +) + +http_archive( + name = "skia_user_config", + sha256 = "038d4a21f9c72d71ab49e3a7d7677b39585329465d093a4260b6c73d2f3984d6", + strip_prefix = "skia-ac75382cb971d2f5465b4608a74561ecb68599c5/include/config", + urls = ["https://github.com/google/skia/archive/ac75382cb971d2f5465b4608a74561ecb68599c5.zip"], +) From fc79f0b14c30d2a1f7330c3532fdd3059eeb99b2 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 14 Nov 2024 21:05:57 -0800 Subject: [PATCH 063/126] No public description PiperOrigin-RevId: 696751629 --- mediapipe/tasks/python/genai/bundler/BUILD | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mediapipe/tasks/python/genai/bundler/BUILD b/mediapipe/tasks/python/genai/bundler/BUILD index 0ed63b2a59..172b807e88 100644 --- a/mediapipe/tasks/python/genai/bundler/BUILD +++ b/mediapipe/tasks/python/genai/bundler/BUILD @@ -15,11 +15,6 @@ # Placeholder for internal Python strict library and test compatibility macro. # Placeholder for internal Python strict test compatibility macro. -package(default_visibility = [ - "//mediapipe:__subpackages__", - "//third_party/odml:__subpackages__", -]) - py_library( name = "llm_bundler", srcs = ["llm_bundler.py"], From 0cebcc0a5bfe864dc46e2901876c3633367c886e Mon Sep 17 00:00:00 2001 From: Mohammadreza Heydary Date: Fri, 15 Nov 2024 16:14:57 -0800 Subject: [PATCH 064/126] Remove the check for start and stop tokens in the LLM bundler. PiperOrigin-RevId: 697029482 --- .../tasks/python/genai/bundler/llm_bundler.py | 13 -------- .../python/genai/bundler/llm_bundler_test.py | 31 ------------------- 2 files changed, 44 deletions(-) diff --git a/mediapipe/tasks/python/genai/bundler/llm_bundler.py b/mediapipe/tasks/python/genai/bundler/llm_bundler.py index ef13deb6f1..02d73313ba 100644 --- a/mediapipe/tasks/python/genai/bundler/llm_bundler.py +++ b/mediapipe/tasks/python/genai/bundler/llm_bundler.py @@ -93,19 +93,6 @@ def _validate_config(config: BundleConfig): "Please ensure you are passing a valid SentencePiece model." ) from e - encoded_start_token = sp.PieceToId(config.start_token) - if encoded_start_token == sp.unk_id(): - raise ValueError( - f"Failed to encode start token {config.start_token} with tokenizer." - ) - - for stop_token in config.stop_tokens: - encoded_stop_token = sp.PieceToId(stop_token) - if encoded_stop_token == sp.unk_id(): - raise ValueError( - f"Failed to encode stop token {stop_token} with tokenizer." - ) - def create_bundle(config: BundleConfig): """Creates a bundle from the given config.""" diff --git a/mediapipe/tasks/python/genai/bundler/llm_bundler_test.py b/mediapipe/tasks/python/genai/bundler/llm_bundler_test.py index ea43733f1b..91108346ab 100644 --- a/mediapipe/tasks/python/genai/bundler/llm_bundler_test.py +++ b/mediapipe/tasks/python/genai/bundler/llm_bundler_test.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for llm_bundler.""" - import os import string import zipfile @@ -147,35 +145,6 @@ def test_invalid_stop_tokens_raises_value_error(self): with self.assertRaisesRegex(ValueError, "stop_tokens must be non-empty"): llm_bundler.create_bundle(config) - def test_invalid_start_stop_tokens_raises_value_error(self): - tempdir = self.create_tempdir() - sp_file_path = self._create_sp_model(tempdir.full_path) - tflite_file_path = self._create_tflite_model(tempdir.full_path) - output_file = os.path.join(tempdir, "test.task") - config = llm_bundler.BundleConfig( - tflite_model=tflite_file_path, - tokenizer_model=sp_file_path, - start_token="invalid_token", - stop_tokens=[self.EOS], - output_filename=output_file, - ) - with self.assertRaisesRegex( - ValueError, "Failed to encode start token invalid_token with tokenizer" - ): - llm_bundler.create_bundle(config) - - config = llm_bundler.BundleConfig( - tflite_model=tflite_file_path, - tokenizer_model=sp_file_path, - start_token=self.BOS, - stop_tokens=["invalid_token"], - output_filename=output_file, - ) - with self.assertRaisesRegex( - ValueError, "Failed to encode stop token invalid_token with tokenizer" - ): - llm_bundler.create_bundle(config) - def test_invalid_tokenizer_model_raises_value_error(self): tempdir = self.create_tempdir() sp_file_path = self._create_sp_model(tempdir.full_path, corrupt=True) From cccc5d242b0460a7361321c563aaf5e1d87c2cd6 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 18 Nov 2024 10:14:19 -0800 Subject: [PATCH 065/126] Adding VLOG overrides - MediaPipe utilizes VLOG heavily, but it's not straightforward for how to enable this when running an Android app. VLOG overrides allow to relatively quickly enable VLOGs for various modules within MediaPipe. PiperOrigin-RevId: 697671583 --- mediapipe/framework/BUILD | 17 ++++++++ mediapipe/framework/calculator_graph.cc | 2 + mediapipe/framework/vlog_overrides.cc | 55 +++++++++++++++++++++++++ mediapipe/framework/vlog_overrides.h | 32 ++++++++++++++ 4 files changed, 106 insertions(+) create mode 100644 mediapipe/framework/vlog_overrides.cc create mode 100644 mediapipe/framework/vlog_overrides.h diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index 2930cf68d7..a49cf27084 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -358,6 +358,7 @@ cc_library( ":thread_pool_executor_cc_proto", ":timestamp", ":validated_graph_config", + ":vlog_overrides", "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", @@ -1926,3 +1927,19 @@ cc_library( ":memory_manager", ], ) + +cc_library( + name = "vlog_overrides", + srcs = ["vlog_overrides.cc"], + hdrs = ["vlog_overrides.h"], + visibility = ["//visibility:private"], + deps = [ + "//mediapipe/framework/deps:no_destructor", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/log:globals", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", + ], +) diff --git a/mediapipe/framework/calculator_graph.cc b/mediapipe/framework/calculator_graph.cc index a64c295ac9..3fe9f8d933 100644 --- a/mediapipe/framework/calculator_graph.cc +++ b/mediapipe/framework/calculator_graph.cc @@ -76,6 +76,7 @@ #include "mediapipe/framework/tool/validate.h" #include "mediapipe/framework/tool/validate_name.h" #include "mediapipe/framework/validated_graph_config.h" +#include "mediapipe/framework/vlog_overrides.h" #include "mediapipe/gpu/gpu_service.h" #include "mediapipe/gpu/graph_support.h" #include "mediapipe/util/cpu_util.h" @@ -145,6 +146,7 @@ CalculatorGraph::CalculatorGraph(CalculatorContext* cc) // TODO b/368015341- Use factory method to avoid CHECK in constructor. ABSL_CHECK_OK(DisallowServiceDefaultInitialization()); } + SetVLogOverrides(); } CalculatorGraph::CalculatorGraph(CalculatorGraphConfig config) diff --git a/mediapipe/framework/vlog_overrides.cc b/mediapipe/framework/vlog_overrides.cc new file mode 100644 index 0000000000..169f276f47 --- /dev/null +++ b/mediapipe/framework/vlog_overrides.cc @@ -0,0 +1,55 @@ +#include "mediapipe/framework/vlog_overrides.h" + +// Template to temporary enable VLOG overrides in code: +// #define MEDIAPIPE_VLOG_VMODULE "calculator_graph*=5,southbound*=5" +// #define MEDIAPIPE_VLOG_V 1 + +#if defined(MEDIAPIPE_VLOG_V) || defined(MEDIAPIPE_VLOG_VMODULE) + +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/log/globals.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "mediapipe/framework/deps/no_destructor.h" + +#endif // defined(MEDIAPIPE_VLOG_V) || defined(MEDIAPIPE_VLOG_VMODULE) + +namespace mediapipe { + +void SetVLogOverrides() { +#if defined(MEDIAPIPE_VLOG_V) + ABSL_LOG(INFO) << absl::StrFormat("Setting global VLOG level: %d", + MEDIAPIPE_VLOG_V); + absl::SetGlobalVLogLevel(MEDIAPIPE_VLOG_V); +#endif // defined(MEDIAPIPE_VLOG_V) + +#if defined(MEDIAPIPE_VLOG_VMODULE) + static NoDestructor>> kVModuleMapping( + []() { + constexpr absl::string_view kVModule = MEDIAPIPE_VLOG_VMODULE; + std::vector parts = + absl::StrSplit(kVModule, absl::ByAnyChar(",=")); + ABSL_CHECK_EQ(parts.size() % 2, 0) + << "Invalid MEDIAPIPE_VLOG_VMODULE: " << kVModule; + std::vector> result; + for (int i = 0; i < parts.size(); i += 2) { + result.push_back({parts[i], std::stoi(parts[i + 1])}); + } + return result; + }()); + + ABSL_LOG(INFO) << "Setting VLOG levels..."; + for (const auto& [key, value] : *kVModuleMapping) { + ABSL_LOG(INFO) << absl::StrFormat("Setting [%s] to level: %d", key, value); + absl::SetVLogLevel(key, value); + } +#endif // defined(MEDIAPIPE_VLOG_VMODULE) +} + +} // namespace mediapipe diff --git a/mediapipe/framework/vlog_overrides.h b/mediapipe/framework/vlog_overrides.h new file mode 100644 index 0000000000..82168088a5 --- /dev/null +++ b/mediapipe/framework/vlog_overrides.h @@ -0,0 +1,32 @@ +#ifndef MEDIAPIPE_FRAMEWORK_VLOG_OVERRIDES_H_ +#define MEDIAPIPE_FRAMEWORK_VLOG_OVERRIDES_H_ + +namespace mediapipe { + +// If possible, rely on --v / --vmodule to set VLOG level and modules. +// +// However, in cases when --v / --vmodule cannot be used (e.g. running an +// Android app and enabling VLOGs), MediaPipe allows to set VLOG --v / --vmodule +// overrides for debugging purposes which are applied when `CalculatorGraph` is +// created. +// +// Overrides: +// - MEDIAPIPE_VLOG_V (define and provide value you provide for --v) +// - MEDIAPIPE_VLOG_VMODULE (define and provide value you provide for --vmodule) +// +// You can set overrides by adding: +// ``` +// --copt=-DMEDIAPIPE_VLOG_VMODULE=\"*calculator*=5\" +// ``` +// with your desired module patterns and VLOG levels (see more details for +// --vmodule) to your build command. +// +// IMPORTANT: mind that adding the above to your build command will trigger +// rebuild of the whole binary including dependencies. So, considering vlog +// overrides exist for debugging purposes only, it is faster to simply modify +// `vlog_overrides.cc` adding MEDIAPIPE_VLOG_V/VMODULE at the very top. +void SetVLogOverrides(); + +} // namespace mediapipe + +#endif // MEDIAPIPE_FRAMEWORK_VLOG_OVERRIDES_H_ From 6842c7df46b84d97056f6eabaa0280e65f0fcd51 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 18 Nov 2024 10:48:31 -0800 Subject: [PATCH 066/126] Updating Troubleshooting with VLOG info. PiperOrigin-RevId: 697684512 --- docs/getting_started/troubleshooting.md | 33 +++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/getting_started/troubleshooting.md b/docs/getting_started/troubleshooting.md index d69ccbb6dd..182e09b71c 100644 --- a/docs/getting_started/troubleshooting.md +++ b/docs/getting_started/troubleshooting.md @@ -270,6 +270,37 @@ calculators designed specifically for this purpose such as [`FlowLimiterCalculator`] as described in [`How to process realtime input streams`]. +## VLOG is your friend + +MediaPipe uses `VLOG` in many places to log important events for debugging +purposes, while not affecting performance if logging is not enabled. + +See more about `VLOG` on [abseil `VLOG`] + +Mind that `VLOG` can be spammy if you enable it globally e.g. (using `--v` +flag). The solution `--vmodule` flag that allows different levels to be set for +different source files. + +In cases when `--v` / `--vmodule` cannot be used (e.g. running an Android app), +MediaPipe allows to set `VLOG` `--v` / `--vmodule` flags overrides for debugging +purposes which are applied when `CalculatorGraph` is created. + +Overrides: + +- `MEDIAPIPE_VLOG_V`: define and provide value you provide for `--v` +- `MEDIAPIPE_VLOG_VMODULE`: define and provide value you provide for `--vmodule` + +You can set overrides by adding: +`--copt=-DMEDIAPIPE_VLOG_VMODULE=\"*calculator*=5\"` + +with your desired module patterns and `VLOG` levels (see more details for +`--vmodule` at [abseil `VLOG`]) to your build command. + +IMPORTANT: mind that adding the above to your build command will trigger rebuild +of the whole binary including dependencies. So, considering `VLOG` overrides +exist for debugging purposes only, it is faster to simply modify +[`vlog_overrides.cc`] adding `MEDIAPIPE_VLOG_V/VMODULE` at the very top. + [`CalculatorGraphConfig`]: https://github.com/google-ai-edge/mediapipe/tree/master/mediapipe/framework/calculator.proto [`CalculatorGraphConfig::max_queue_size`]: https://github.com/google-ai-edge/mediapipe/tree/master/mediapipe/framework/calculator.proto [`CalculatorGraphConfig::report_deadlock`]: https://github.com/google-ai-edge/mediapipe/tree/master/mediapipe/framework/calculator.proto @@ -282,6 +313,8 @@ calculators designed specifically for this purpose such as [`CalculatorBase::Close`]: https://github.com/google-ai-edge/mediapipe/tree/master/mediapipe/framework/calculator_base.h [`FlowLimiterCalculator`]: https://github.com/google-ai-edge/mediapipe/tree/master/mediapipe/calculators/core/flow_limiter_calculator.cc [`How to process realtime input streams`]: faq.md#how-to-process-realtime-input-streams +[`vlog_overrides.cc`]: https://github.com/google-ai-edge/mediapipe/tree/master/mediapipe/framework/vlog_overrides.cc +[abseil `VLOG`]: https://abseil.io/docs/cpp/guides/logging#VLOG ## Unsupported flags during build From 85f79f8b09384adc37f0dbbb9c2ab572b7747206 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 19 Nov 2024 03:30:53 -0800 Subject: [PATCH 067/126] No public description PiperOrigin-RevId: 697950751 --- .../framework/stream_handler/early_close_input_stream_handler.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/mediapipe/framework/stream_handler/early_close_input_stream_handler.cc b/mediapipe/framework/stream_handler/early_close_input_stream_handler.cc index 3a7dd86783..7adf0fb75a 100644 --- a/mediapipe/framework/stream_handler/early_close_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/early_close_input_stream_handler.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/stream_handler/early_close_input_stream_handler.h" #include +#include #include "absl/log/absl_check.h" #include "absl/strings/substitute.h" From 34f53d1f1e6e3c36825da5da2f60d1c3ad94f3c9 Mon Sep 17 00:00:00 2001 From: Alan Kelly Date: Tue, 19 Nov 2024 04:41:53 -0800 Subject: [PATCH 068/126] Slice only the tokens which are needed for the next stage of the LLM pipeline. This increases encoder performance by up to 30% for long sequence lengths. PiperOrigin-RevId: 697967093 --- WORKSPACE | 14 +++++------ .../utils/xnn_utils/graph_builder.cc | 8 +++--- .../inference/utils/xnn_utils/graph_builder.h | 2 +- .../cc/genai/inference/utils/xnn_utils/llm.cc | 25 +++++++++++++------ 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 39fc185346..a1d06c46ec 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -228,13 +228,13 @@ http_archive( ], ) -# XNNPACK on 2024-10-30 +# XNNPACK on 2024-11-18 http_archive( name = "XNNPACK", # `curl -L | shasum -a 256` - sha256 = "7587e5b9272c9d37ea6c9feac46568858dd153c99aa2fbd9c7744fa6415ca297", - strip_prefix = "XNNPACK-92b6d8722c34198be870ee956ffe504a11f44093", - url = "https://github.com/google/XNNPACK/archive/92b6d8722c34198be870ee956ffe504a11f44093.zip", + sha256 = "af30fe2b301330a7e19cd422acf22991de3c1f5d91dda58e9ee67544d608fa51", + strip_prefix = "XNNPACK-dc1549a7141c7a9496ae160bb27b8700f0f6e1f1", + url = "https://github.com/google/XNNPACK/archive/dc1549a7141c7a9496ae160bb27b8700f0f6e1f1.zip", ) # 2020-07-09 @@ -268,10 +268,10 @@ http_archive( # KleidiAI is needed to get the best possible performance out of XNNPack http_archive( name = "KleidiAI", - sha256 = "ccdb61c0c5df9174128f433f82430b1afa8a809cf17f43ecae9a1eec9c2dfabf", - strip_prefix = "kleidiai-0dadcdb307f4e5ac98a42e4d4888aad7c23edaf3", + sha256 = "ad37707084a6d4ff41be10cbe8540c75bea057ba79d0de6c367c1bfac6ba0852", + strip_prefix = "kleidiai-40a926833857fb64786e02f97703e42b1537cb57", urls = [ - "https://gitlab.arm.com/kleidi/kleidiai/-/archive/0dadcdb307f4e5ac98a42e4d4888aad7c23edaf3/kleidiai-0dadcdb307f4e5ac98a42e4d4888aad7c23edaf3.zip", + "https://gitlab.arm.com/kleidi/kleidiai/-/archive/40a926833857fb64786e02f97703e42b1537cb57/kleidiai-40a926833857fb64786e02f97703e42b1537cb57.zip" ], ) diff --git a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.cc b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.cc index 384b1d9dcd..df75149c34 100644 --- a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.cc +++ b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.cc @@ -484,11 +484,11 @@ absl::StatusOr> XnnGraphBuilder::Slice( } absl::StatusOr> XnnGraphBuilder::Slice( - std::shared_ptr input, size_t axis, size_t offset, size_t length) { + std::shared_ptr input, size_t axis, int64_t offset, size_t length) { const auto& input_dims = input->dims; - Tensor::DimsType offsets(input_dims.size(), 0); + std::vector offsets(input_dims.size(), 0); offsets[axis] = offset; - Tensor::DimsType output_dims = input_dims; + std::vector output_dims = input_dims; output_dims[axis] = length; Tensor::DimsType inferrable_output_dims(input_dims.size(), 0); inferrable_output_dims[axis] = length; @@ -499,7 +499,7 @@ absl::StatusOr> XnnGraphBuilder::Slice( build_steps_.push_back([input, output, offsets, inferrable_output_dims]( xnn_subgraph_t subgraph) -> absl::Status { RET_CHECK_EQ(xnn_status_success, - xnn_define_static_slice( + xnn_define_static_slice_v2( subgraph, offsets.size(), offsets.data(), inferrable_output_dims.data(), input->tensor_id(subgraph), output->tensor_id(subgraph), /*flags=*/0)); diff --git a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.h b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.h index a7f1e0950f..909cae2d81 100644 --- a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.h +++ b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.h @@ -185,7 +185,7 @@ class XnnGraphBuilder { // dimensions unchanged. For instance, for input A = [B, M, N] and axis = 1, // the output slice would be [B, offset:offset+length, N]. absl::StatusOr> Slice(std::shared_ptr input, - size_t axis, size_t offset, + size_t axis, int64_t offset, size_t length); // Concatenate two input tensors along the provided axis. Both input tensors diff --git a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm.cc b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm.cc index 8ef3b92dce..e53197d288 100644 --- a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm.cc +++ b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -349,13 +350,6 @@ absl::Status Llm::AddInputTokens( transformer_input()->tensor_id(owned_subgraph_.get()), transformer_input()->dims.size(), transformer_input()->dims.data())); - logits_output()->Resize(Tensor::DimsType{ - batch_input_ids.size(), input_seq_len, llm_params_.voc_size_V}); - RET_CHECK_EQ( - xnn_status_success, - xnn_reshape_external_value( - runtime_.get(), logits_output()->tensor_id(owned_subgraph_.get()), - logits_output()->dims.size(), logits_output()->dims.data())); for (auto& kv_cache : kv_cache()) { auto key = kv_cache.k_cache; auto value = kv_cache.v_cache; @@ -373,6 +367,14 @@ absl::Status Llm::AddInputTokens( value->dims.size(), value->dims.data())); } RET_CHECK_EQ(xnn_status_success, xnn_reshape_runtime(runtime_.get())); + size_t num_output_dims = 0; + std::vector output_dims(3); + RET_CHECK_EQ( + xnn_status_success, + xnn_get_external_value_shape( + runtime_.get(), logits_output()->tensor_id(owned_subgraph_.get()), + &num_output_dims, output_dims.data())); + logits_output()->Resize(output_dims); } for (auto& kv_cache : kv_cache()) { @@ -662,9 +664,16 @@ absl::StatusOr> LlmBuilder::PostProcess( ApplyNorm(transformer_out, weights.final_norm_weight, llm_params_.final_norm)); RET_CHECK(weights.softmax_linear); + + int64_t slice_size = llm_params_.draft_size_G + 1; + // The KV caches have been filled, we only need to compute the tokens which + // will be used for computation or output. + MP_ASSIGN_OR_RETURN(auto slice, + Slice(transformer_out, /*axis=*/1, /*offset=*/-slice_size, + /*length=*/slice_size)); MP_ASSIGN_OR_RETURN( auto logits_output, - FullConn(transformer_out, weights.softmax_linear, weights.softmax_bias)); + FullConn(slice, weights.softmax_linear, weights.softmax_bias)); return logits_output; } From 5861b192dff3b026a69cf8528c94a4a541267f78 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 19 Nov 2024 07:14:55 -0800 Subject: [PATCH 069/126] Adds DebugInputStreamHandler. PiperOrigin-RevId: 698004893 --- mediapipe/framework/BUILD | 1 + mediapipe/framework/graph_output_stream.cc | 2 +- mediapipe/framework/input_stream_handler.cc | 35 +++++++++++++++++++++ mediapipe/framework/input_stream_handler.h | 7 +++++ mediapipe/framework/input_stream_manager.cc | 1 + mediapipe/framework/input_stream_manager.h | 3 +- 6 files changed, 46 insertions(+), 3 deletions(-) diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index a49cf27084..dc717d0e21 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -786,6 +786,7 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/framework/tool:tag_map", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/framework/graph_output_stream.cc b/mediapipe/framework/graph_output_stream.cc index e456c6535d..62956cdada 100644 --- a/mediapipe/framework/graph_output_stream.cc +++ b/mediapipe/framework/graph_output_stream.cc @@ -16,7 +16,7 @@ #include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" -#include "mediapipe/framework/port/status.h" +#include "mediapipe/framework/port/status_macros.h" namespace mediapipe { diff --git a/mediapipe/framework/input_stream_handler.cc b/mediapipe/framework/input_stream_handler.cc index e222c2e6cc..6f44568d82 100644 --- a/mediapipe/framework/input_stream_handler.cc +++ b/mediapipe/framework/input_stream_handler.cc @@ -14,7 +14,12 @@ #include "mediapipe/framework/input_stream_handler.h" +#include +#include + #include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/substitute.h" #include "mediapipe/framework/collection_item_id.h" @@ -145,6 +150,19 @@ std::string InputStreamHandler::DebugStreamNames() const { ">"); } +std::string InputStreamHandler::DebugStreamName(CollectionItemId id) const { + const auto tag_map = input_stream_managers_.TagMap(); + const std::string& stream_name = tag_map->Names()[id.value()]; + const auto& [stream_tag, stream_idx] = tag_map->TagAndIndexFromId(id); + return absl::StrCat(stream_tag, ":", stream_idx, ":", stream_name); +} + +std::string InputStreamHandler::GetNodeName() const { + const auto* calculator_context = + calculator_context_manager_->GetDefaultCalculatorContext(); + return calculator_context ? calculator_context->NodeName() : ""; +} + bool InputStreamHandler::ScheduleInvocations(int max_allowance, Timestamp* input_bound) { *input_bound = Timestamp::Unset(); @@ -409,18 +427,35 @@ void SyncSet::FillInputSet(Timestamp input_timestamp, InputStreamShardSet* input_set) { ABSL_CHECK(input_timestamp.IsAllowedInStream()); ABSL_CHECK(input_set); + std::vector streams_with_missing_packets; for (CollectionItemId id : stream_ids_) { const auto& stream = input_stream_handler_->input_stream_managers_.Get(id); int num_packets_dropped = 0; bool stream_is_done = false; Packet current_packet = stream->PopPacketAtTimestamp( input_timestamp, &num_packets_dropped, &stream_is_done); + if (current_packet.IsEmpty()) { + // Track the streams that have no packets at the current timestamp. + streams_with_missing_packets.push_back( + input_stream_handler_->DebugStreamName(id)); + } ABSL_CHECK_EQ(num_packets_dropped, 0) << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", num_packets_dropped, stream->Name()); input_stream_handler_->AddPacketToShard( &input_set->Get(id), std::move(current_packet), stream_is_done); } + + const std::string node_name = input_stream_handler_->GetNodeName(); + if (!streams_with_missing_packets.empty()) { + VLOG(1) << absl::StrCat( + node_name, ": Filled input set at ts: ", input_timestamp.DebugString(), + " with MISSING packets in input streams: ", + absl::StrJoin(streams_with_missing_packets, ", "), "."); + } else { + VLOG(1) << absl::StrCat( + node_name, ": Filled input set at ts: ", input_timestamp.DebugString()); + } } void SyncSet::FillInputBounds(InputStreamShardSet* input_set) { diff --git a/mediapipe/framework/input_stream_handler.h b/mediapipe/framework/input_stream_handler.h index bf74a63d26..2e707af672 100644 --- a/mediapipe/framework/input_stream_handler.h +++ b/mediapipe/framework/input_stream_handler.h @@ -150,6 +150,13 @@ class InputStreamHandler { // Returns a string that concatenates the stream names of all managed streams. std::string DebugStreamNames() const; + // Return the stream name for an input stream in the format: + // stream_tag:stream_index:stream_name. + std::string DebugStreamName(CollectionItemId id) const; + + // Returns the node name of the calculator node. + std::string GetNodeName() const; + // Keeps scheduling new invocations until 1) the node is not ready or 2) the // max number of invocations that are allowed to be scheduled is reached. // Returns true if at least one invocation has been scheduled. diff --git a/mediapipe/framework/input_stream_manager.cc b/mediapipe/framework/input_stream_manager.cc index 5acd571e13..8cd7f6baa6 100644 --- a/mediapipe/framework/input_stream_manager.cc +++ b/mediapipe/framework/input_stream_manager.cc @@ -302,6 +302,7 @@ Packet InputStreamManager::PopPacketAtTimestamp(Timestamp timestamp, if (current_timestamp != timestamp) { // The timestamp bound reported when no packet is sent. Timestamp bound = MinTimestampOrBoundHelper(); + // Generate empty packet at the timestamp bound. packet = Packet().At(bound.PreviousAllowedInStream()); ++(*num_packets_dropped); } diff --git a/mediapipe/framework/input_stream_manager.h b/mediapipe/framework/input_stream_manager.h index b97e11e7ab..27f148edbf 100644 --- a/mediapipe/framework/input_stream_manager.h +++ b/mediapipe/framework/input_stream_manager.h @@ -22,11 +22,10 @@ #include #include "absl/base/thread_annotations.h" +#include "absl/status/status.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/packet.h" #include "mediapipe/framework/packet_type.h" -#include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/timestamp.h" namespace mediapipe { From 13541a976b7ef0c2c3b9bb4fe8e775329e0e0750 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 19 Nov 2024 18:58:09 -0800 Subject: [PATCH 070/126] No public description PiperOrigin-RevId: 698220962 --- mediapipe/tasks/cc/genai/inference/proto/llm_params.proto | 4 ++++ .../tasks/cc/genai/inference/proto/transformer_params.proto | 4 ++++ mediapipe/tasks/python/genai/converter/llm_converter.py | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/mediapipe/tasks/cc/genai/inference/proto/llm_params.proto b/mediapipe/tasks/cc/genai/inference/proto/llm_params.proto index ea98937486..e613198f50 100644 --- a/mediapipe/tasks/cc/genai/inference/proto/llm_params.proto +++ b/mediapipe/tasks/cc/genai/inference/proto/llm_params.proto @@ -94,4 +94,8 @@ message LlmParameters { optional string system_role_token = 12; optional string model_role_token = 13; optional string end_role_token = 14; + + // If this model includes a submodel, these params can be used to load the + // submodel. + optional TransformerParameters submodel_transformer_parameters = 16; } diff --git a/mediapipe/tasks/cc/genai/inference/proto/transformer_params.proto b/mediapipe/tasks/cc/genai/inference/proto/transformer_params.proto index 7b9bdf4d84..c87f758e82 100644 --- a/mediapipe/tasks/cc/genai/inference/proto/transformer_params.proto +++ b/mediapipe/tasks/cc/genai/inference/proto/transformer_params.proto @@ -194,4 +194,8 @@ message TransformerParameters { // Vision parameters int32 vision_tokens_num = 26; + + // The number of stacks that are treated as "extra", which may have slightly + // different loading behavior. + int32 num_extra_stacks = 27; } diff --git a/mediapipe/tasks/python/genai/converter/llm_converter.py b/mediapipe/tasks/python/genai/converter/llm_converter.py index e92f241500..81ec5bfa36 100644 --- a/mediapipe/tasks/python/genai/converter/llm_converter.py +++ b/mediapipe/tasks/python/genai/converter/llm_converter.py @@ -50,6 +50,7 @@ class ConversionConfig(object): zero. image_encoder_file: A string with the name of the image encoder tflite file. image_adapter_file: A string with the name of the image adapter tflite file. + submodel_type: Name of submodel, e.g. GEMMA_2B. use_fake_weights: Whether to use fake weights. If set to True, the weights will be filled with zeros. """ @@ -75,6 +76,7 @@ def __init__( lora_output_tflite_file: Optional[str] = None, image_encoder_file: Optional[str] = None, image_adapter_file: Optional[str] = None, + submodel_type: Optional[str] = None, use_fake_weights: bool = False, ): self.input_ckpt = input_ckpt @@ -96,6 +98,7 @@ def __init__( self.obfuscate = obfuscate self.image_encoder_file = image_encoder_file self.image_adapter_file = image_adapter_file + self.submodel_type = submodel_type self.use_fake_weights = use_fake_weights if output_tflite_file: parent_dir = os.path.dirname(output_tflite_file) @@ -220,6 +223,7 @@ def combined_weight_bins_to_tflite( lora_output_tflite_file: Optional[str] = None, image_encoder_file: Optional[str] = None, image_adapter_file: Optional[str] = None, + submodel_type: Optional[str] = None, ): """Combines weight files to tflite file.""" if backend == 'cpu': @@ -245,6 +249,7 @@ def combined_weight_bins_to_tflite( '' if lora_output_tflite_file is None else lora_output_tflite_file, '' if image_encoder_file is None else image_encoder_file, '' if image_adapter_file is None else image_adapter_file, + '' if submodel_type is None else submodel_type, ) else: raise ValueError('Unsupported backend: %s' % backend) @@ -365,4 +370,5 @@ def convert_checkpoint(config: ConversionConfig) -> None: lora_output_tflite_file=config.lora_output_tflite_file, image_encoder_file=config.image_encoder_file, image_adapter_file=config.image_adapter_file, + submodel_type=config.submodel_type, ) From 7a70aac0f0e57c7ac3fda5a5ad1e351cf7bbfaf4 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 20 Nov 2024 01:16:12 -0800 Subject: [PATCH 071/126] Delete YUVImage copy and move operations PiperOrigin-RevId: 698298660 --- mediapipe/framework/formats/yuv_image.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mediapipe/framework/formats/yuv_image.h b/mediapipe/framework/formats/yuv_image.h index 4aeb75d6b0..30a0f303f5 100644 --- a/mediapipe/framework/formats/yuv_image.h +++ b/mediapipe/framework/formats/yuv_image.h @@ -118,6 +118,12 @@ class YUVImage { YUVImage() = default; ~YUVImage() { Clear(); } + // YUVImage is neither copyable nor movable. + YUVImage(const YUVImage&) = delete; + YUVImage& operator=(const YUVImage&) = delete; + YUVImage(YUVImage&&) = delete; + YUVImage& operator=(YUVImage&&) = delete; + // Convenience constructor YUVImage(libyuv::FourCC fourcc, // std::unique_ptr data_location, // From 17fbbebdefaeae4876af8bc050ffadd6558f09d1 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 20 Nov 2024 03:32:28 -0800 Subject: [PATCH 072/126] Adds GetGraphRuntimeInfo methods which generates runtime debugging information about the state of InputStreams. PiperOrigin-RevId: 698330776 --- mediapipe/framework/BUILD | 16 +++- mediapipe/framework/calculator_graph.cc | 13 +++ mediapipe/framework/calculator_graph.h | 6 ++ mediapipe/framework/calculator_graph_test.cc | 7 +- mediapipe/framework/calculator_node.cc | 60 +++++++++++-- mediapipe/framework/calculator_node.h | 21 +++++ mediapipe/framework/graph_runtime_info.proto | 62 +++++++++++++ mediapipe/framework/input_stream_handler.cc | 62 +++++++++---- mediapipe/framework/input_stream_handler.h | 5 +- mediapipe/framework/tool/BUILD | 15 ++++ .../tool/graph_runtime_info_utils.cc | 86 +++++++++++++++++++ .../framework/tool/graph_runtime_info_utils.h | 31 +++++++ 12 files changed, 356 insertions(+), 28 deletions(-) create mode 100644 mediapipe/framework/graph_runtime_info.proto create mode 100644 mediapipe/framework/tool/graph_runtime_info_utils.cc create mode 100644 mediapipe/framework/tool/graph_runtime_info_utils.h diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index dc717d0e21..64112cd255 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -334,6 +334,7 @@ cc_library( ":delegating_executor", ":executor", ":graph_output_stream", + ":graph_runtime_info_cc_proto", ":graph_service", ":graph_service_manager", ":input_stream_manager", @@ -359,8 +360,8 @@ cc_library( ":timestamp", ":validated_graph_config", ":vlog_overrides", + "//mediapipe/framework/deps:clock", "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", "//mediapipe/framework/port:map_util", "//mediapipe/framework/port:ret_check", @@ -387,6 +388,7 @@ cc_library( "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", ], ) @@ -425,6 +427,7 @@ cc_library( ":calculator_context_manager", ":calculator_state", ":counter_factory", + ":graph_runtime_info_cc_proto", ":graph_service_manager", ":input_side_packet_handler", ":input_stream_handler", @@ -441,6 +444,7 @@ cc_library( ":stream_handler_cc_proto", ":timestamp", ":validated_graph_config", + "//mediapipe/framework/deps:clock", "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", @@ -454,12 +458,14 @@ cc_library( "//mediapipe/framework/tool:tag_map", "//mediapipe/framework/tool:validate_name", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/cleanup", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", ], ) @@ -647,6 +653,12 @@ cc_library( ], ) +mediapipe_proto_library( + name = "graph_runtime_info_proto", + srcs = ["graph_runtime_info.proto"], + visibility = ["//visibility:public"], +) + cc_library( name = "graph_service", hdrs = ["graph_service.h"], @@ -785,6 +797,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:tag_map", + "@com_google_absl//absl/log", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", @@ -1620,6 +1633,7 @@ cc_test( "//mediapipe/calculators/core:counting_source_calculator", "//mediapipe/calculators/core:mux_calculator", "//mediapipe/calculators/core:pass_through_calculator", + "//mediapipe/framework/deps:clock", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:ret_check", diff --git a/mediapipe/framework/calculator_graph.cc b/mediapipe/framework/calculator_graph.cc index 3fe9f8d933..6cacd8117b 100644 --- a/mediapipe/framework/calculator_graph.cc +++ b/mediapipe/framework/calculator_graph.cc @@ -37,10 +37,12 @@ #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" #include "absl/synchronization/mutex.h" +#include "absl/time/time.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_base.h" #include "mediapipe/framework/counter_factory.h" #include "mediapipe/framework/delegating_executor.h" +#include "mediapipe/framework/deps/clock.h" #include "mediapipe/framework/executor.h" #include "mediapipe/framework/graph_output_stream.h" #include "mediapipe/framework/graph_service_manager.h" @@ -914,6 +916,17 @@ absl::Status CalculatorGraph::WaitForObservedOutput() { return scheduler_.WaitForObservedOutput(); } +absl::StatusOr CalculatorGraph::GetGraphRuntimeInfo() { + RET_CHECK(initialized_); + GraphRuntimeInfo info; + for (const auto& node : nodes_) { + *info.add_calculator_infos() = node->GetStreamMonitoringInfo(); + } + const absl::Time time_now = mediapipe::Clock::RealClock()->TimeNow(); + info.set_capture_time_unix_us(absl::ToUnixMicros(time_now)); + return info; +} + absl::Status CalculatorGraph::AddPacketToInputStream( absl::string_view stream_name, const Packet& packet) { return AddPacketToInputStreamInternal(stream_name, packet); diff --git a/mediapipe/framework/calculator_graph.h b/mediapipe/framework/calculator_graph.h index 19032fbc2f..a128d1b551 100644 --- a/mediapipe/framework/calculator_graph.h +++ b/mediapipe/framework/calculator_graph.h @@ -40,6 +40,7 @@ #include "mediapipe/framework/counter_factory.h" #include "mediapipe/framework/executor.h" #include "mediapipe/framework/graph_output_stream.h" +#include "mediapipe/framework/graph_runtime_info.pb.h" #include "mediapipe/framework/graph_service.h" #include "mediapipe/framework/graph_service_manager.h" #include "mediapipe/framework/mediapipe_profiling.h" @@ -257,6 +258,11 @@ class CalculatorGraph { // Quick non-locking means of checking if the graph has encountered an error. bool HasError() const { return has_error_; } + // Returns debugging information about the graph transient state, including + // information about all input streams and their timestamp bounds. This method + // is thread safe and can be called from any thread. + absl::StatusOr GetGraphRuntimeInfo(); + // Add a Packet to a graph input stream based on the graph input stream add // mode. If the mode is ADD_IF_NOT_FULL, the packet will not be added if any // queue exceeds max_queue_size specified by the graph config and will return diff --git a/mediapipe/framework/calculator_graph_test.cc b/mediapipe/framework/calculator_graph_test.cc index 745271e53b..ef59bf6388 100644 --- a/mediapipe/framework/calculator_graph_test.cc +++ b/mediapipe/framework/calculator_graph_test.cc @@ -43,6 +43,7 @@ #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/counter_factory.h" +#include "mediapipe/framework/deps/clock.h" #include "mediapipe/framework/executor.h" #include "mediapipe/framework/input_stream_handler.h" #include "mediapipe/framework/lifetime_tracker.h" @@ -70,7 +71,6 @@ #include "mediapipe/gpu/gpu_service.h" namespace mediapipe { - namespace { constexpr char kCounter2Tag[] = "COUNTER2"; @@ -83,8 +83,9 @@ constexpr char kOutputTag[] = "OUTPUT"; constexpr char kInputTag[] = "INPUT"; constexpr char kSelectTag[] = "SELECT"; -using testing::ElementsAre; -using testing::HasSubstr; +using ::mediapipe::Clock; +using ::testing::ElementsAre; +using ::testing::HasSubstr; // Pass packets through. Note that it calls SetOffset() in Process() // instead of Open(). diff --git a/mediapipe/framework/calculator_node.cc b/mediapipe/framework/calculator_node.cc index 6767cb874d..2a7d56c934 100644 --- a/mediapipe/framework/calculator_node.cc +++ b/mediapipe/framework/calculator_node.cc @@ -17,22 +17,28 @@ #include #include #include +#include #include #include +#include +#include "absl/cleanup/cleanup.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" #include "absl/synchronization/mutex.h" +#include "absl/time/time.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_base.h" #include "mediapipe/framework/calculator_state.h" #include "mediapipe/framework/counter_factory.h" +#include "mediapipe/framework/deps/clock.h" #include "mediapipe/framework/graph_service_manager.h" #include "mediapipe/framework/input_stream_manager.h" #include "mediapipe/framework/mediapipe_profiling.h" @@ -54,6 +60,7 @@ namespace mediapipe { +using ::mediapipe::Clock; namespace { const PacketType* GetPacketType(const PacketTypeSet& packet_type_set, @@ -116,7 +123,11 @@ std::unique_ptr RemoveOmittedPacketTypes( } // namespace -CalculatorNode::CalculatorNode() {} +CalculatorNode::CalculatorNode() { + absl::Time now = Clock::RealClock()->TimeNow(); + last_process_start_ts_ = now; + last_process_finish_ts_ = now; +} Timestamp CalculatorNode::SourceProcessOrder( const CalculatorContext* cc) const { @@ -218,6 +229,29 @@ absl::Status CalculatorNode::Initialize( return InitializeInputStreams(input_stream_managers, output_stream_managers); } +CalculatorRuntimeInfo CalculatorNode::GetStreamMonitoringInfo() const { + CalculatorRuntimeInfo calulator_info; + calulator_info.set_calculator_name(DebugName()); + { + absl::MutexLock lock(&runtime_info_mutex_); + calulator_info.set_last_process_start_unix_us( + absl::ToUnixMicros(last_process_start_ts_)); + calulator_info.set_last_process_finish_unix_us( + absl::ToUnixMicros(last_process_finish_ts_)); + } + const auto monitoring_info = input_stream_handler_->GetMonitoringInfo(); + for (const auto& [stream_name, queue_size, num_packets_added, + minimum_timestamp_or_bound] : monitoring_info) { + auto* stream_info = calulator_info.add_input_stream_infos(); + stream_info->set_stream_name(stream_name); + stream_info->set_queue_size(queue_size); + stream_info->set_number_of_packets_added(num_packets_added); + stream_info->set_minimum_timestamp_or_bound( + minimum_timestamp_or_bound.Value()); + } + return calulator_info; +} + absl::Status CalculatorNode::InitializeOutputSidePackets( const PacketTypeSet& output_side_packet_types, OutputSidePacketImpl* output_side_packets) { @@ -669,14 +703,14 @@ void CalculatorNode::SchedulingLoop() { max_allowance = max_in_flight_ - current_in_flight_; } while (true) { - Timestamp input_bound; - // input_bound is set to a meaningful value iff the latest readiness of the - // node is kNotReady when ScheduleInvocations() returns. - input_stream_handler_->ScheduleInvocations(max_allowance, &input_bound); - if (input_bound != Timestamp::Unset()) { + // last_timestamp_bound_ is set to a meaningful value iff the latest + // readiness of the node is kNotReady when ScheduleInvocations() returns. + input_stream_handler_->ScheduleInvocations(max_allowance, + &last_timestamp_bound_); + if (last_timestamp_bound_ != Timestamp::Unset()) { // Updates the minimum timestamp for which a new packet could possibly // arrive. - output_stream_handler_->UpdateTaskTimestampBound(input_bound); + output_stream_handler_->UpdateTaskTimestampBound(last_timestamp_bound_); } { @@ -805,6 +839,18 @@ std::string CalculatorNode::DebugName() const { // TODO: Split this function. absl::Status CalculatorNode::ProcessNode( CalculatorContext* calculator_context) { + // Update calculator runtime info. + { + absl::MutexLock lock(&runtime_info_mutex_); + last_process_start_ts_ = Clock::RealClock()->TimeNow(); + } + absl::Cleanup last_process_finish_ts_cleanup([this]() { + { + absl::MutexLock lock(&runtime_info_mutex_); + last_process_finish_ts_ = Clock::RealClock()->TimeNow(); + } + }); + if (IsSource()) { // This is a source Calculator. if (Closed()) { diff --git a/mediapipe/framework/calculator_node.h b/mediapipe/framework/calculator_node.h index 1340461169..26b644dec1 100644 --- a/mediapipe/framework/calculator_node.h +++ b/mediapipe/framework/calculator_node.h @@ -28,13 +28,16 @@ #include #include "absl/base/macros.h" +#include "absl/base/thread_annotations.h" #include "absl/status/status.h" #include "absl/synchronization/mutex.h" +#include "absl/time/time.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_base.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_context_manager.h" #include "mediapipe/framework/calculator_state.h" +#include "mediapipe/framework/graph_runtime_info.pb.h" #include "mediapipe/framework/graph_service_manager.h" #include "mediapipe/framework/input_side_packet_handler.h" #include "mediapipe/framework/input_stream_handler.h" @@ -239,6 +242,14 @@ class CalculatorNode { return node_type_info_->Contract(); } + // Returns the last timestamp bound used to schedule this node. + Timestamp GetLastTimestampBound() const { return last_timestamp_bound_; } + + // Returns the stream monitoring info for this node consisting of a vector of + // tuples of input stream name, queue size, number of packets added, and + // minimum timestamp or bound. + CalculatorRuntimeInfo GetStreamMonitoringInfo() const; + private: // Sets up the output side packets from the main flat array. absl::Status InitializeOutputSidePackets( @@ -376,6 +387,16 @@ class CalculatorNode { const ValidatedGraphConfig* validated_graph_ = nullptr; const NodeTypeInfo* node_type_info_ = nullptr; + + // Keeps track of the latest timestamp bound used to schedule this node. + Timestamp last_timestamp_bound_ = Timestamp::Unset(); + + // Keeps track of the runtime info for this node. + mutable absl::Mutex runtime_info_mutex_; + absl::Time last_process_start_ts_ ABSL_GUARDED_BY(runtime_info_mutex_) = + absl::InfinitePast(); + absl::Time last_process_finish_ts_ ABSL_GUARDED_BY(runtime_info_mutex_) = + absl::InfinitePast(); }; } // namespace mediapipe diff --git a/mediapipe/framework/graph_runtime_info.proto b/mediapipe/framework/graph_runtime_info.proto new file mode 100644 index 0000000000..1488a33f7d --- /dev/null +++ b/mediapipe/framework/graph_runtime_info.proto @@ -0,0 +1,62 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package mediapipe; + +option java_package = "com.google.mediapipe.proto"; +option java_outer_classname = "GraphRuntimeInfoProto"; + +// The runtime info for an input stream. +message StreamRuntimeInfo { + // The name of the stream in the format "TAG:index:stream_name" + string stream_name = 1; + + // The number of packets in the queue. + int32 queue_size = 2; + + // The total number of packets added to the queue. + int32 number_of_packets_added = 3; + + // The minimum timestamp or timestanp bound of the stream. + int64 minimum_timestamp_or_bound = 4; +} + +// The runtime info for a calculator. +message CalculatorRuntimeInfo { + // The name of the calculator. + string calculator_name = 1; + + // The last time when the Calculator::Process was started. + int64 last_process_start_unix_us = 2; + + // The last time when the Calculator::Process was finished. + int64 last_process_finish_unix_us = 3; + + // The timestamp bound of the calculator. + int64 timestamp_bound = 4; + + // The runtime info for each input stream of the calculator. + repeated StreamRuntimeInfo input_stream_infos = 5; +} + +// The runtime info for the whole graph. +message GraphRuntimeInfo { + // The time when the runtime info was captured. + int64 capture_time_unix_us = 1; + + // The runtime info for each calculator in the graph. + repeated CalculatorRuntimeInfo calculator_infos = 2; +} diff --git a/mediapipe/framework/input_stream_handler.cc b/mediapipe/framework/input_stream_handler.cc index 6f44568d82..ee4a86adf2 100644 --- a/mediapipe/framework/input_stream_handler.cc +++ b/mediapipe/framework/input_stream_handler.cc @@ -14,11 +14,15 @@ #include "mediapipe/framework/input_stream_handler.h" +#include +#include #include +#include #include #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "absl/log/log.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/substitute.h" @@ -27,8 +31,41 @@ #include "mediapipe/framework/port/ret_check.h" namespace mediapipe { + +namespace { using SyncSet = InputStreamHandler::SyncSet; +// Helper class to vlog the streams with missing packets during FillInputSet +// calls. +class FillInputSetLogger { + public: + FillInputSetLogger(const std::string& node_name, Timestamp timestamp) + : node_name_(node_name), timestamp_(timestamp) {} + ~FillInputSetLogger() { OutputLogs(); } + + void AddMissingPacketStreamName(const std::string& stream_name) { + missing_streams_.push_back(stream_name); + } + + private: + void OutputLogs() const { + if (!missing_streams_.empty()) { + VLOG(1) << absl::StrCat( + node_name_, ": Filled input set at ts: ", timestamp_.DebugString(), + " with MISSING packets in input streams: ", + absl::StrJoin(missing_streams_, ", "), "."); + } else { + VLOG(1) << absl::StrCat( + node_name_, ": Filled input set at ts: ", timestamp_.DebugString()); + } + } + + const std::string node_name_; + const Timestamp timestamp_; + std::vector missing_streams_; +}; +} // namespace + absl::Status InputStreamHandler::InitializeInputStreamManagers( InputStreamManager* flat_input_stream_managers) { for (CollectionItemId id = input_stream_managers_.BeginId(); @@ -60,13 +97,15 @@ std::vector> InputStreamHandler::GetMonitoringInfo() { std::vector> monitoring_info_vector; - for (auto& stream : input_stream_managers_) { + for (CollectionItemId id = input_stream_managers_.BeginId(); + id < input_stream_managers_.EndId(); ++id) { + const auto& stream = input_stream_managers_.Get(id); if (!stream) { continue; } monitoring_info_vector.emplace_back( std::tuple( - stream->Name(), stream->QueueSize(), stream->NumPacketsAdded(), + DebugStreamName(id), stream->QueueSize(), stream->NumPacketsAdded(), stream->MinTimestampOrBound(nullptr))); } return monitoring_info_vector; @@ -427,6 +466,10 @@ void SyncSet::FillInputSet(Timestamp input_timestamp, InputStreamShardSet* input_set) { ABSL_CHECK(input_timestamp.IsAllowedInStream()); ABSL_CHECK(input_set); + std::optional logger; + if (VLOG_IS_ON(1)) { + logger.emplace(input_stream_handler_->GetNodeName(), input_timestamp); + } std::vector streams_with_missing_packets; for (CollectionItemId id : stream_ids_) { const auto& stream = input_stream_handler_->input_stream_managers_.Get(id); @@ -434,9 +477,9 @@ void SyncSet::FillInputSet(Timestamp input_timestamp, bool stream_is_done = false; Packet current_packet = stream->PopPacketAtTimestamp( input_timestamp, &num_packets_dropped, &stream_is_done); - if (current_packet.IsEmpty()) { + if (current_packet.IsEmpty() && logger.has_value()) { // Track the streams that have no packets at the current timestamp. - streams_with_missing_packets.push_back( + logger->AddMissingPacketStreamName( input_stream_handler_->DebugStreamName(id)); } ABSL_CHECK_EQ(num_packets_dropped, 0) @@ -445,17 +488,6 @@ void SyncSet::FillInputSet(Timestamp input_timestamp, input_stream_handler_->AddPacketToShard( &input_set->Get(id), std::move(current_packet), stream_is_done); } - - const std::string node_name = input_stream_handler_->GetNodeName(); - if (!streams_with_missing_packets.empty()) { - VLOG(1) << absl::StrCat( - node_name, ": Filled input set at ts: ", input_timestamp.DebugString(), - " with MISSING packets in input streams: ", - absl::StrJoin(streams_with_missing_packets, ", "), "."); - } else { - VLOG(1) << absl::StrCat( - node_name, ": Filled input set at ts: ", input_timestamp.DebugString()); - } } void SyncSet::FillInputBounds(InputStreamShardSet* input_set) { diff --git a/mediapipe/framework/input_stream_handler.h b/mediapipe/framework/input_stream_handler.h index 2e707af672..cbf9e04db4 100644 --- a/mediapipe/framework/input_stream_handler.h +++ b/mediapipe/framework/input_stream_handler.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -92,8 +93,8 @@ class InputStreamHandler { // Sets up the InputStreamShardSet by propagating data from the managers. absl::Status SetupInputShards(InputStreamShardSet* input_shards); - // Returns a vector of pairs of stream name and queue size for monitoring - // purpose. + // Returns a vector of tuples of stream name, queue size, number of packets + // added, and minimum timestamp or bound for monitoring purpose. std::vector> GetMonitoringInfo(); // Resets the input stream handler and its underlying input streams for diff --git a/mediapipe/framework/tool/BUILD b/mediapipe/framework/tool/BUILD index 76d00abc60..24cd3cf558 100644 --- a/mediapipe/framework/tool/BUILD +++ b/mediapipe/framework/tool/BUILD @@ -978,6 +978,21 @@ cc_test( ], ) +cc_library( + name = "graph_runtime_info_utils", + srcs = ["graph_runtime_info_utils.cc"], + hdrs = ["graph_runtime_info_utils.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:graph_runtime_info_cc_proto", + "//mediapipe/framework:timestamp", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/time", + ], +) + exports_files( ["build_defs.bzl"], visibility = [ diff --git a/mediapipe/framework/tool/graph_runtime_info_utils.cc b/mediapipe/framework/tool/graph_runtime_info_utils.cc new file mode 100644 index 0000000000..86f28d3183 --- /dev/null +++ b/mediapipe/framework/tool/graph_runtime_info_utils.cc @@ -0,0 +1,86 @@ +#include "mediapipe/framework/tool/graph_runtime_info_utils.h" + +#include +#include + +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/time/time.h" +#include "mediapipe/framework/timestamp.h" + +namespace mediapipe::tool { + +absl::StatusOr GetGraphRuntimeInfoString( + const GraphRuntimeInfo& graph_runtime_info) { + const absl::Time caputure_time = + absl::FromUnixMicros(graph_runtime_info.capture_time_unix_us()); + std::string calculators_runtime_info_str; + std::vector calculators_with_unprocessed_packets; + std::vector running_calculators; + int num_packets_in_input_queues = 0; + for (const auto& calculator_info : graph_runtime_info.calculator_infos()) { + const bool is_idle = calculator_info.last_process_finish_unix_us() >= + calculator_info.last_process_start_unix_us(); + const std::string calculator_state_str = + is_idle ? absl::StrFormat( + "idle for %.2fs", + absl::ToDoubleSeconds( + caputure_time - + absl::FromUnixMicros( + calculator_info.last_process_finish_unix_us()))) + : absl::StrFormat( + "running for %.2fs", + absl::ToDoubleSeconds( + caputure_time - + absl::FromUnixMicros( + calculator_info.last_process_start_unix_us()))); + if (!is_idle) { + running_calculators.push_back(calculator_info.calculator_name()); + } + absl::StrAppend( + &calculators_runtime_info_str, + absl::StrFormat( + "\n%s: (%s, ts bound : %s)", calculator_info.calculator_name(), + calculator_state_str, + Timestamp::CreateNoErrorChecking(calculator_info.timestamp_bound()) + .DebugString())); + bool calculator_has_unprocessed_packets = false; + for (const auto& input_stream_info : calculator_info.input_stream_infos()) { + num_packets_in_input_queues += input_stream_info.queue_size(); + calculator_has_unprocessed_packets |= input_stream_info.queue_size() > 0; + absl::StrAppend( + &calculators_runtime_info_str, " * ", input_stream_info.stream_name(), + " - queue size: ", input_stream_info.queue_size(), + ", total added: ", input_stream_info.number_of_packets_added(), + ", min ts: ", + Timestamp::CreateNoErrorChecking( + input_stream_info.minimum_timestamp_or_bound()) + .DebugString(), + "\n"); + } + if (calculator_has_unprocessed_packets) { + calculators_with_unprocessed_packets.push_back( + calculator_info.calculator_name()); + } + } + const std::string calulators_with_unprocessed_packets_str = + num_packets_in_input_queues > 0 + ? absl::StrCat( + " (in calculators: ", + absl::StrJoin(calculators_with_unprocessed_packets, ", "), ")") + : ""; + const std::string running_calculators_str = + running_calculators.empty() + ? "None" + : absl::StrCat(" (running calculators: ", + absl::StrJoin(running_calculators, ", "), ")"); + return absl::StrFormat( + "Graph runtime info: \nRunning calculators: %s\nNum packets in input " + "queues: %d%s\n%s\n", + running_calculators_str, num_packets_in_input_queues, + calulators_with_unprocessed_packets_str, calculators_runtime_info_str); +} + +} // namespace mediapipe::tool diff --git a/mediapipe/framework/tool/graph_runtime_info_utils.h b/mediapipe/framework/tool/graph_runtime_info_utils.h new file mode 100644 index 0000000000..0418ffa152 --- /dev/null +++ b/mediapipe/framework/tool/graph_runtime_info_utils.h @@ -0,0 +1,31 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_TOOL_GRAPH_RUNTIME_INFO_UTILS_H_ +#define MEDIAPIPE_FRAMEWORK_TOOL_GRAPH_RUNTIME_INFO_UTILS_H_ + +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/graph_runtime_info.pb.h" + +namespace mediapipe::tool { + +// Returns a human readable representation of the graph runtime info. +absl::StatusOr GetGraphRuntimeInfoString( + const GraphRuntimeInfo& graph_runtime_info); + +} // namespace mediapipe::tool + +#endif // MEDIAPIPE_FRAMEWORK_TOOL_GRAPH_RUNTIME_INFO_UTILS_H_ From 8a5c4f90ed2692a900322b347474fad24a37d3eb Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 20 Nov 2024 13:59:39 -0800 Subject: [PATCH 073/126] Add a sample script to run LLM inference on Android via the MediaPipe LLM inference engine. PiperOrigin-RevId: 698512881 --- run_llm_inference.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 run_llm_inference.sh diff --git a/run_llm_inference.sh b/run_llm_inference.sh new file mode 100644 index 0000000000..62f1a440de --- /dev/null +++ b/run_llm_inference.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# This is a simple script to run LLM inference on Android via the MediaPipe +# LLM inference engine. + +MODEL_FILENAME="gemma2-2b-it-cpu-int8.task" +ADB_WORK_DIR="/data/local/tmp" +INPUT_PROMPT="What is the most famous building in Paris?" + +if [ ! -f "${MODEL_FILENAME}" ]; then + echo "Error: ${MODEL_FILENAME} not found." + echo "Please download it from https://www.kaggle.com/models/google/gemma-2/tfLite/gemma2-2b-it-cpu-int8" + exit 1 +fi + +adb push "${MODEL_FILENAME}" "${ADB_WORK_DIR}/${MODEL_FILENAME}" + +# Build the MediaPipe Docker base image. +docker build . --tag=mediapipe + +# Build the LLM inference engine binary and copy to the conencted Android device. +CONTAINER_NAME=mediapipe_$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1) +docker run --name "$CONTAINER_NAME" mediapipe:latest sh -c " + chmod +x setup_android_sdk_and_ndk.sh && \ + ./setup_android_sdk_and_ndk.sh ~/Android/Sdk ~/Android/Ndk r26d \ + --accept-licenses && + bazel build --config android_arm64 --client_env=CC=clang-16 -c opt \ + --copt=-DABSL_FLAGS_STRIP_NAMES=0 \ + --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ + //mediapipe/tasks/cc/genai/inference/c:llm_inference_engine_cpu_main +" +docker cp "$CONTAINER_NAME":/mediapipe/bazel-bin/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine_cpu_main llm_inference_engine_cpu_main +adb push llm_inference_engine_cpu_main "${ADB_WORK_DIR}"/llm_inference_engine_cpu_main + +# Run the inference. +adb shell "taskset f0 ${ADB_WORK_DIR}/llm_inference_engine_cpu_main \ + --model_path='${ADB_WORK_DIR}/${MODEL_FILENAME}' \ + --prompt='${INPUT_PROMPT}'" From 256bb14d3bf0313df1c50b341b93a799056875e6 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 20 Nov 2024 14:35:31 -0800 Subject: [PATCH 074/126] No public description PiperOrigin-RevId: 698524674 --- mediapipe/tasks/cc/genai/inference/c/BUILD | 9 +- .../inference/c/llm_inference_engine_cpu.cc | 347 +++++++++++++++--- 2 files changed, 313 insertions(+), 43 deletions(-) diff --git a/mediapipe/tasks/cc/genai/inference/c/BUILD b/mediapipe/tasks/cc/genai/inference/c/BUILD index 92f9f15216..3db359f473 100644 --- a/mediapipe/tasks/cc/genai/inference/c/BUILD +++ b/mediapipe/tasks/cc/genai/inference/c/BUILD @@ -28,12 +28,13 @@ cc_library( hdrs = ["llm_inference_engine.h"], tags = ["swift_module=MediaPipeTasksGenAIC"], deps = [ + "//mediapipe/framework/deps:file_path", "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "//mediapipe/tasks/cc/core:model_asset_bundle_resources", "//mediapipe/tasks/cc/genai/inference/proto:llm_params_cc_proto", "//mediapipe/tasks/cc/genai/inference/proto:transformer_params_cc_proto", - "//mediapipe/tasks/cc/genai/inference/utils/llm_utils:memory_mapped_file", "//mediapipe/tasks/cc/genai/inference/utils/llm_utils:metadata_utils", "//mediapipe/tasks/cc/genai/inference/utils/llm_utils:model_data", "//mediapipe/tasks/cc/genai/inference/utils/llm_utils:scoped_file", @@ -41,6 +42,8 @@ cc_library( "//mediapipe/tasks/cc/genai/inference/utils/xnn_utils:llm", "//mediapipe/tasks/cc/genai/inference/utils/xnn_utils:llm_builder_factory", "//mediapipe/tasks/cc/genai/inference/utils/xnn_utils:llm_weights", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", @@ -49,6 +52,10 @@ cc_library( "@com_google_absl//absl/strings:string_view", "@com_google_sentencepiece//:sentencepiece_processor", "@org_tensorflow//tensorflow/lite:framework_stable", + "@org_tensorflow//tensorflow/lite/c:common", + "@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate_hdrs_only", + "@org_tensorflow//tensorflow/lite/experimental/genai:genai_ops", + "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", ], ) diff --git a/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine_cpu.cc b/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine_cpu.cc index aaa7cba2b4..40524aee95 100644 --- a/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine_cpu.cc +++ b/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine_cpu.cc @@ -14,27 +14,37 @@ #include +#include +#include #include #include +#include #include +#include #include #include #include +#include #include +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" +#include "mediapipe/framework/deps/file_path.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h" #include "mediapipe/tasks/cc/genai/inference/c/llm_inference_engine.h" #include "mediapipe/tasks/cc/genai/inference/proto/llm_params.pb.h" #include "mediapipe/tasks/cc/genai/inference/proto/transformer_params.pb.h" -#include "mediapipe/tasks/cc/genai/inference/utils/llm_utils/memory_mapped_file.h" #include "mediapipe/tasks/cc/genai/inference/utils/llm_utils/metadata_utils.h" #include "mediapipe/tasks/cc/genai/inference/utils/llm_utils/model_data.h" #include "mediapipe/tasks/cc/genai/inference/utils/llm_utils/scoped_file.h" @@ -42,55 +52,162 @@ #include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm.h" #include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_builder_factory.h" #include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.h" -#include "sentencepiece/src/normalizer.h" // from @com_google_sentencepiece #include "sentencepiece/src/sentencepiece_processor.h" // from @com_google_sentencepiece +#include "sentencepiece/src/util.h" // from @com_google_sentencepiece +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h" +#include "tensorflow/lite/experimental/genai/genai_ops.h" +#include "tensorflow/lite/interpreter.h" +#include "tensorflow/lite/interpreter_builder.h" +#include "tensorflow/lite/kernels/register.h" #include "tensorflow/lite/model_builder.h" namespace { constexpr int kCheckLastKChars = 10; +struct TfLiteLlm { + std::unique_ptr interpreter; + std::unique_ptr resources; +}; + struct LlmInferenceEngineCpu_Engine { - sentencepiece::SentencePieceProcessor* tokenizer; - sentencepiece::normalizer::Normalizer* normalizer; - mediapipe::tasks::genai::xnn_utils::Llm* llm; - int start_token_id; - std::vector stop_tokens; - size_t max_num_tokens; + const sentencepiece::SentencePieceProcessor* tokenizer; + const absl::flat_hash_map* bytes_to_unicode_mapper; + const absl::flat_hash_map* unicode_to_bytes_mapper; + const std::variant llm; + const int start_token_id; + const std::vector stop_tokens; + const size_t max_num_tokens; + ~LlmInferenceEngineCpu_Engine() { delete tokenizer; - if (normalizer != nullptr) { - delete normalizer; + delete bytes_to_unicode_mapper; + delete unicode_to_bytes_mapper; + if (std::holds_alternative(llm)) { + delete std::get(llm); + } else { + delete std::get(llm); } - delete llm; }; }; struct LlmInferenceEngineCpu_Session { const LlmInferenceEngineCpu_Engine* engine; std::string prompt; - int max_num_output_tokens; - int response_count; + int timestep; std::string last_10_char; std::string final_output; std::function cpu_callback; bool early_stop; pthread_t work_id; + int next_token_id; ~LlmInferenceEngineCpu_Session() { pthread_join(work_id, nullptr); }; }; +absl::StatusOr>> +CreateBytesToUnicodeMapper() { + auto bytes_to_unicode_mapper = + std::make_unique>(); + // "!" - "~" + for (int i = 33; i <= 126; i++) { + bytes_to_unicode_mapper->insert({static_cast(i), i}); + } + // "¡" - "¬" + for (int i = 161; i <= 172; i++) { + bytes_to_unicode_mapper->insert({static_cast(i), i}); + } + // "®" - "ÿ" + for (int i = 174; i < 256; i++) { + bytes_to_unicode_mapper->insert({static_cast(i), i}); + } + int n = 0; + for (int b = 0; b < 256; b++) { + if (!bytes_to_unicode_mapper->contains(static_cast(b))) { + bytes_to_unicode_mapper->insert({static_cast(b), 256 + n}); + n += 1; + } + } + return bytes_to_unicode_mapper; +} + +absl::StatusOr>> +CreateUnicodeToBytesMapper() { + MP_ASSIGN_OR_RETURN(auto bytes_to_unicode_mapper, + CreateBytesToUnicodeMapper()); + auto unicode_to_bytes_mapper = + std::make_unique>(); + for (const auto& [key, value] : *bytes_to_unicode_mapper) { + unicode_to_bytes_mapper->insert({value, key}); + } + return unicode_to_bytes_mapper; +} + +std::string MapBytesToUnicode( + absl::string_view prompt, + const absl::flat_hash_map* bytes_to_unicode_mapper) { + std::string converted_prompt = ""; + for (const uint8_t byte : prompt) { + converted_prompt.append(sentencepiece::string_util::UnicodeCharToUTF8( + bytes_to_unicode_mapper->at(byte))); + } + return converted_prompt; +} + +std::string MapUnicodeToBytes( + absl::string_view output, + const absl::flat_hash_map* unicode_to_bytes_mapper) { + sentencepiece::string_util::UnicodeText unicode_text = + sentencepiece::string_util::UTF8ToUnicodeText(output); + std::string converted_output = ""; + for (const int code_point : unicode_text) { + if (!unicode_to_bytes_mapper->contains(code_point)) { + converted_output += code_point; + } else { + converted_output += unicode_to_bytes_mapper->at(code_point); + } + } + return converted_output; +} + void* next_token_function(void* args) { struct LlmInferenceEngineCpu_Session* cpu_session = (struct LlmInferenceEngineCpu_Session*)args; - if (cpu_session->response_count++ < cpu_session->max_num_output_tokens) { + if (cpu_session->timestep < cpu_session->engine->max_num_tokens) { if (cpu_session->early_stop) { return nullptr; } auto token_ids_per_step = std::vector(); - auto status = cpu_session->engine->llm->GetNextToken(&token_ids_per_step); - if (!status.ok()) { - ABSL_LOG(FATAL) << "Failed to generate output: " << status; + if (std::holds_alternative( + cpu_session->engine->llm)) { + auto status = std::get( + cpu_session->engine->llm) + ->GetNextToken(&token_ids_per_step); + if (!status.ok()) { + ABSL_LOG(FATAL) << "Failed to generate output: " << status; + } + } else { + auto llm = std::get(cpu_session->engine->llm); + auto* decode_runner = llm->interpreter->GetSignatureRunner("decode"); + ABSL_CHECK_EQ(decode_runner->AllocateTensors(), kTfLiteOk); + TfLiteTensor* decode_input = decode_runner->input_tensor("args_0"); + TfLiteTensor* decode_input_pos = decode_runner->input_tensor("args_1"); + decode_input->data.i64[0] = + static_cast(cpu_session->next_token_id); + decode_input_pos->data.i64[0] = + static_cast(cpu_session->timestep); + + // logits->dims->data[0] = batch size + // logits->dims->data[1] = sequence length + // logits->dims->data[2] = vocab size + const TfLiteTensor* logits = decode_runner->output_tensor("output_0"); + + ABSL_CHECK_EQ(decode_runner->Invoke(), kTfLiteOk); + + auto max_logit_it = std::max_element( + logits->data.f, logits->data.f + logits->dims->data[2]); + token_ids_per_step.push_back(std::distance(logits->data.f, max_logit_it)); } // For future multithreading support. @@ -98,14 +215,19 @@ void* next_token_function(void* args) { return nullptr; } - if (cpu_session->response_count == cpu_session->max_num_output_tokens) { + if (cpu_session->timestep >= cpu_session->engine->max_num_tokens) { cpu_session->early_stop = true; } + cpu_session->next_token_id = token_ids_per_step[0]; + std::string token = cpu_session->engine->tokenizer->IdToPiece(token_ids_per_step[0]); - if (cpu_session->engine->normalizer != nullptr) { - token = cpu_session->engine->normalizer->Normalize(token); + if (cpu_session->engine->unicode_to_bytes_mapper != nullptr) { + token = MapUnicodeToBytes(token, + cpu_session->engine->unicode_to_bytes_mapper); + } else { + token = absl::StrReplaceAll(token, {{"▁", " "}}); } cpu_session->last_10_char.append(token); @@ -133,6 +255,8 @@ void* next_token_function(void* args) { cpu_session->cpu_callback(ready_char); + ++cpu_session->timestep; + next_token_function(args); } return nullptr; @@ -144,27 +268,55 @@ void* start_llm_function(void* args) { std::vector prompt_ids = {}; - auto status = - cpu_session->engine->tokenizer->Encode(cpu_session->prompt, &prompt_ids); + std::string prompt; + if (cpu_session->engine->bytes_to_unicode_mapper != nullptr) { + prompt = MapBytesToUnicode(cpu_session->prompt, + cpu_session->engine->bytes_to_unicode_mapper); + } else { + prompt = cpu_session->prompt; + } + + auto status = cpu_session->engine->tokenizer->Encode(prompt, &prompt_ids); if (!status.ok()) { ABSL_LOG(FATAL) << "Failed to encode input: " << status; } prompt_ids.insert(prompt_ids.begin(), cpu_session->engine->start_token_id); - ABSL_CHECK_OK(cpu_session->engine->llm->SeekTimeStep(0)); - ABSL_CHECK_OK(cpu_session->engine->llm->AddInputTokens({prompt_ids})); + if (std::holds_alternative( + cpu_session->engine->llm)) { + auto llm = std::get( + cpu_session->engine->llm); + ABSL_CHECK_OK(llm->SeekTimeStep(0)); + ABSL_CHECK_OK(llm->AddInputTokens({prompt_ids})); + } else { + auto llm = std::get(cpu_session->engine->llm); + auto* prefill_runner = llm->interpreter->GetSignatureRunner("prefill"); + + ABSL_CHECK_EQ(prefill_runner->AllocateTensors(), kTfLiteOk); + + TfLiteTensor* prefill_input = prefill_runner->input_tensor("args_0"); + TfLiteTensor* prefill_input_pos = prefill_runner->input_tensor("args_1"); + memset(prefill_input->data.data, 0, prefill_input->bytes); + memset(prefill_input_pos->data.data, 0, prefill_input_pos->bytes); + cpu_session->next_token_id = prompt_ids.back(); + prompt_ids.pop_back(); + for (int i = 0; i < prompt_ids.size(); ++i) { + prefill_input->data.i64[i] = static_cast(prompt_ids[i]); + prefill_input_pos->data.i64[i] = static_cast(i); + } + ABSL_CHECK_EQ(prefill_runner->Invoke(), kTfLiteOk); + } - cpu_session->max_num_output_tokens = - cpu_session->engine->max_num_tokens - prompt_ids.size(); + cpu_session->timestep = prompt_ids.size(); next_token_function(args); return nullptr; } -absl::StatusOr -LlmInferenceEngine_CreateEngine_Helper(const LlmModelSettings* model_settings) { +absl::StatusOr> +CreateXnnLlmCpuEngine(const LlmModelSettings* model_settings) { MP_ASSIGN_OR_RETURN(auto model_file, mediapipe::tasks::genai::llm_utils::ScopedFile::Open( model_settings->model_path)); @@ -199,7 +351,6 @@ LlmInferenceEngine_CreateEngine_Helper(const LlmModelSettings* model_settings) { MP_ASSIGN_OR_RETURN(auto spm_model_content, model_data->ReadMetadata("spm_vocab_model")); - model_data.reset(); llm_params.seq_size_T = model_settings->max_num_tokens; @@ -220,24 +371,24 @@ LlmInferenceEngine_CreateEngine_Helper(const LlmModelSettings* model_settings) { auto tokenizer = std::make_unique(); MP_RETURN_IF_ERROR(tokenizer->LoadFromSerializedProto(spm_model_content)); - std::vector prompt_ids; - auto status = tokenizer->Encode("hello", &prompt_ids); - - std::unique_ptr normalizer; - if (tokenizer->model_proto().has_denormalizer_spec() && - tokenizer->model_proto().denormalizer_spec().has_precompiled_charsmap() && - !tokenizer->model_proto() - .denormalizer_spec() - .precompiled_charsmap() - .empty()) { - normalizer = std::make_unique( - tokenizer->model_proto().denormalizer_spec()); + std::unique_ptr> + bytes_to_unicode_mapper; + std::unique_ptr> + unicode_to_bytes_mapper; + // These models uses GPT2 style unicode mapping, which additional mapping is + // needed. + if (model_type == odml::infra::proto::LLM_MODEL_TYPE_STABLELM_4E1T_3B || + model_type == odml::infra::proto::LLM_MODEL_TYPE_FALCON_RW_1B || + model_type == odml::infra::proto::LLM_MODEL_TYPE_PHI_2) { + MP_ASSIGN_OR_RETURN(bytes_to_unicode_mapper, CreateBytesToUnicodeMapper()); + MP_ASSIGN_OR_RETURN(unicode_to_bytes_mapper, CreateUnicodeToBytesMapper()); } std::unique_ptr engine( new LlmInferenceEngineCpu_Engine{ .tokenizer = tokenizer.release(), - .normalizer = normalizer.release(), + .bytes_to_unicode_mapper = bytes_to_unicode_mapper.release(), + .unicode_to_bytes_mapper = unicode_to_bytes_mapper.release(), .llm = llm.release(), .start_token_id = llm_params_proto.start_token_id(), .stop_tokens = @@ -246,6 +397,118 @@ LlmInferenceEngine_CreateEngine_Helper(const LlmModelSettings* model_settings) { .max_num_tokens = model_settings->max_num_tokens, }); + return engine; +} + +// Creates an inference engine from a *.task file. +// This method extracts the TF_LITE_PREFILL_DECODE, TOKENIZER_MODEL and METADATA +// files from the task bundle and initializes the TfLIte XNNPack delegate. +absl::StatusOr> +CreateTfliteLlmCpuEngine(const LlmModelSettings* model_settings) { + auto external_file = + std::make_unique(); + if (model_settings) { + external_file->set_file_name(model_settings->model_path); + } + MP_ASSIGN_OR_RETURN(auto resources, + mediapipe::tasks::core::ModelAssetBundleResources::Create( + "", std::move(external_file))); + const std::vector& files_list = resources->ListFiles(); + const absl::flat_hash_set files_set(files_list.begin(), + files_list.end()); + + std::unique_ptr interpreter; + if (!files_set.contains("TF_LITE_PREFILL_DECODE")) { + return absl::InvalidArgumentError("TF_LITE_PREFILL_DECODE not found."); + } + if (!files_set.contains("TOKENIZER_MODEL")) { + return absl::InvalidArgumentError("TOKENIZER_MODEL not found."); + } + if (!files_set.contains("METADATA")) { + return absl::InvalidArgumentError("METADATA not found."); + } + MP_ASSIGN_OR_RETURN(absl::string_view model_buffer, + resources->GetFile("TF_LITE_PREFILL_DECODE")); + MP_ASSIGN_OR_RETURN(absl::string_view tokenizer_buffer, + resources->GetFile("TOKENIZER_MODEL")); + MP_ASSIGN_OR_RETURN(absl::string_view params_buffer, + resources->GetFile("METADATA")); + auto model = tflite::FlatBufferModel::BuildFromBuffer(model_buffer.data(), + model_buffer.size()); + RET_CHECK(model) << "Failed to build TF_LITE_PREFILL_DECODE model."; + tflite::ops::builtin::BuiltinOpResolver resolver; + // NOTE: We need to manually register optimized OPs for KV-cache and + // Scaled Dot Product Attention (SDPA). + tflite::ops::custom::GenAIOpsRegisterer(&resolver); + tflite::InterpreterBuilder builder(*model, resolver); + RET_CHECK(model_settings); + builder(&interpreter); + RET_CHECK_NE(interpreter, nullptr); + + // RET_CHECK(model_settings->xnnpack_options.has_value()); + auto delegate_options = TfLiteXNNPackDelegateOptionsDefault(); + // Set the number of threads to 4 as default. + delegate_options.num_threads = 4; + // Compute the path for the cache file. + std::string weight_cache_path = model_settings->cache_dir; + if (weight_cache_path != ":nocache") { + if (weight_cache_path.empty()) { + weight_cache_path = + absl::StrCat(model_settings->model_path, ".xnnpack_cache"); + } else { + weight_cache_path = mediapipe::file::JoinPath( + weight_cache_path, + absl::StrCat(mediapipe::file::Basename(model_settings->model_path), + ".xnnpack_cache")); + } + delegate_options.weight_cache_file_path = weight_cache_path.c_str(); + } + RET_CHECK_EQ(interpreter->ModifyGraphWithDelegate( + tflite::Interpreter::TfLiteDelegatePtr( + TfLiteXNNPackDelegateCreate(&delegate_options), + [](TfLiteDelegate* delegate) { + TfLiteXNNPackDelegateDelete(delegate); + })), + kTfLiteOk); + RET_CHECK_EQ(interpreter->SetNumThreads(4), kTfLiteOk); + + auto tflite_llm = std::make_unique( + TfLiteLlm{std::move(interpreter), std::move(resources)}); + + auto tokenizer = std::make_unique(); + MP_RETURN_IF_ERROR(tokenizer->LoadFromSerializedProto(tokenizer_buffer)); + + auto llm_parameters = odml::infra::proto::LlmParameters(); + RET_CHECK(llm_parameters.ParseFromArray(params_buffer.data(), + params_buffer.size())); + + auto start_token_id = tokenizer->PieceToId(llm_parameters.start_token()); + + std::unique_ptr engine( + new LlmInferenceEngineCpu_Engine{ + .tokenizer = tokenizer.release(), + .bytes_to_unicode_mapper = nullptr, + .unicode_to_bytes_mapper = nullptr, + .llm = tflite_llm.release(), + .start_token_id = start_token_id, + .stop_tokens = + std::vector(llm_parameters.stop_tokens().begin(), + llm_parameters.stop_tokens().end()), + .max_num_tokens = model_settings->max_num_tokens, + }); + + return engine; +} + +absl::StatusOr +LlmInferenceEngine_CreateEngine_Helper(const LlmModelSettings* model_settings) { + std::unique_ptr engine; + if (absl::EndsWith(model_settings->model_path, ".tflite")) { + MP_ASSIGN_OR_RETURN(engine, CreateXnnLlmCpuEngine(model_settings)); + } else { + MP_ASSIGN_OR_RETURN(engine, CreateTfliteLlmCpuEngine(model_settings)); + } + return engine.release(); } From 33f0fad3de731449c7f468c45f7b7b42ec072188 Mon Sep 17 00:00:00 2001 From: kuaashish <98159216+kuaashish@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:53:42 +0530 Subject: [PATCH 075/126] Update bot_config.yml Adding kalyan2789g --- .github/bot_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/bot_config.yml b/.github/bot_config.yml index 74a60e4b9b..8049a79100 100644 --- a/.github/bot_config.yml +++ b/.github/bot_config.yml @@ -16,4 +16,4 @@ # A list of assignees assignees: - kuaashish - - ayushgdev + - kalyan2789g From 1077b7366723cc5924530051d24064b6825566e8 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 21 Nov 2024 02:09:14 -0800 Subject: [PATCH 076/126] Add option to set max sequence size in PackMediaSequenceCalculator instead of having it hard coded. PiperOrigin-RevId: 698699219 --- .../pack_media_sequence_calculator.cc | 24 +++++++----- .../pack_media_sequence_calculator.proto | 3 ++ .../pack_media_sequence_calculator_test.cc | 38 ++++++++++++++++++- 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc index 66456345d7..c373cd2236 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include +#include #include #include #include @@ -105,6 +105,8 @@ namespace mpms = mediapipe::mediasequence; // } // } namespace { +constexpr int kMaxProtoBytes = std::numeric_limits::max(); + uint8_t ConvertFloatToByte(const float float_value) { float clamped_value = std::clamp(0.0f, 1.0f, float_value); return static_cast(clamped_value * 255.0 + .5f); @@ -361,15 +363,21 @@ class PackMediaSequenceCalculator : public CalculatorBase { } } - absl::Status VerifySize() { - constexpr int kMaxProtoBytes = INT_MAX; + absl::Status VerifySize(const PackMediaSequenceCalculatorOptions& options) { + if (!options.skip_large_sequences()) { + return absl::OkStatus(); + } + + const int max_bytes = (options.max_sequence_bytes() > 0) + ? options.max_sequence_bytes() + : kMaxProtoBytes; std::string id = mpms::HasExampleId(*sequence_) ? mpms::GetExampleId(*sequence_) : "example"; - RET_CHECK_LT(sequence_->ByteSizeLong(), kMaxProtoBytes) - << "sequence '" << id - << "' would be too many bytes to serialize after adding features."; + RET_CHECK_LT(sequence_->ByteSizeLong(), max_bytes) + << "sequence '" << id << "' with " << sequence_->ByteSizeLong() + << " bytes would be more than " << max_bytes << " bytes."; return absl::OkStatus(); } @@ -381,9 +389,7 @@ class PackMediaSequenceCalculator : public CalculatorBase { options.reconcile_region_annotations(), sequence_.get())); } - if (options.skip_large_sequences()) { - RET_CHECK_OK(VerifySize()); - } + RET_CHECK_OK(VerifySize(options)); if (options.output_only_if_all_present()) { absl::Status status = VerifySequence(); if (!status.ok()) { diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto index cc6c2ffda4..a5c7bbbdf7 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto @@ -65,6 +65,9 @@ message PackMediaSequenceCalculatorOptions { // If true, will return an error status if an output sequence would be too // many bytes to serialize. optional bool skip_large_sequences = 7 [default = true]; + // If > 0, will return an error status if an output sequence would be too + // many bytes to serialize. Otherwise uses int max. + optional int32 max_sequence_bytes = 10 [default = -1]; // If true/false, outputs the SequenceExample at timestamp 0/PostStream. optional bool output_as_zero_timestamp = 8 [default = false]; diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc index d8fbc94d53..fee2a8b48c 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc @@ -85,8 +85,9 @@ class PackMediaSequenceCalculatorTest : public ::testing::Test { const bool replace_instead_of_append, const bool output_as_zero_timestamp = false, const bool add_empty_labels = false, - const std::vector& input_side_packets = { - "SEQUENCE_EXAMPLE:input_sequence"}) { + const std::vector& input_side_packets = + {"SEQUENCE_EXAMPLE:input_sequence"}, + const int32_t max_sequence_bytes = -1) { CalculatorGraphConfig::Node config; config.set_calculator("PackMediaSequenceCalculator"); for (const std::string& side_packet : input_side_packets) { @@ -103,6 +104,7 @@ class PackMediaSequenceCalculatorTest : public ::testing::Test { options->set_replace_data_instead_of_append(replace_instead_of_append); options->set_output_as_zero_timestamp(output_as_zero_timestamp); options->set_add_empty_labels(add_empty_labels); + options->set_max_sequence_bytes(max_sequence_bytes); runner_ = ::absl::make_unique(config); } @@ -1987,5 +1989,37 @@ TEST_F(PackMediaSequenceCalculatorTest, TestTooLargeInputFailsSoftly) { ASSERT_FALSE(runner_->Run().ok()); } +TEST_F(PackMediaSequenceCalculatorTest, SkipLargeSequence) { + SetUpCalculator({"IMAGE:images"}, {}, false, true, false, false, + {"SEQUENCE_EXAMPLE:input_sequence"}, + /*max_sequence_bytes=*/10); + auto input_sequence = ::absl::make_unique(); + std::string test_video_id = "test_video_id"; + mpms::SetClipMediaId(test_video_id, input_sequence.get()); + cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); + std::vector bytes; + ASSERT_TRUE( + cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1})); + OpenCvImageEncoderCalculatorResults encoded_image; + encoded_image.set_encoded_image(bytes.data(), bytes.size()); + encoded_image.set_width(2); + encoded_image.set_height(1); + + int num_images = 2; + for (int i = 0; i < num_images; ++i) { + auto image_ptr = + ::absl::make_unique(encoded_image); + runner_->MutableInputs()->Tag(kImageTag).packets.push_back( + Adopt(image_ptr.release()).At(Timestamp(i))); + } + + runner_->MutableSidePackets()->Tag(kSequenceExampleTag) = + Adopt(input_sequence.release()); + + absl::Status status = runner_->Run(); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("bytes would be more than 10 bytes")); +} + } // namespace } // namespace mediapipe From 2fab0dd938a17433ad68036f109d9b015f92bbff Mon Sep 17 00:00:00 2001 From: Martin Huschenbett Date: Thu, 21 Nov 2024 02:55:33 -0800 Subject: [PATCH 077/126] No public description PiperOrigin-RevId: 698709768 --- mediapipe/python/packet_creator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/python/packet_creator.py b/mediapipe/python/packet_creator.py index 4fff44be4d..1ad6ac42e3 100644 --- a/mediapipe/python/packet_creator.py +++ b/mediapipe/python/packet_creator.py @@ -51,7 +51,7 @@ create_matrix = _packet_creator.create_matrix -def create_image_frame(data: Union[image_frame.ImageFrame, np.ndarray], +def create_image_frame(data: Union[image_frame.ImageFrame, np.ndarray], # pytype: disable=annotation-type-mismatch *, image_format: image_frame.ImageFormat = None, copy: bool = None) -> packet.Packet: @@ -149,7 +149,7 @@ def create_image_frame(data: Union[image_frame.ImageFrame, np.ndarray], # pylint:enable=protected-access -def create_image(data: Union[image.Image, np.ndarray], +def create_image(data: Union[image.Image, np.ndarray], # pytype: disable=annotation-type-mismatch *, image_format: image_frame.ImageFormat = None, copy: bool = None) -> packet.Packet: From df03022cd35377ae333dfba51a478ad492089ef4 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 21 Nov 2024 09:04:25 -0800 Subject: [PATCH 078/126] Update run_llm_inference.sh with recommended models. PiperOrigin-RevId: 698802697 --- run_llm_inference.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/run_llm_inference.sh b/run_llm_inference.sh index 62f1a440de..df3b3393b4 100644 --- a/run_llm_inference.sh +++ b/run_llm_inference.sh @@ -2,6 +2,12 @@ # This is a simple script to run LLM inference on Android via the MediaPipe # LLM inference engine. +# +# This script allows running transformer-based LLM models in *.task or *.bin +# format. We recommend using `gemma2-2b-it-cpu-int8.task` (from +# https://www.kaggle.com/models/google/gemma-2/tfLite/gemma2-2b-it-cpu-int8) or +# the smaller `gemma-1.1-2b-it-cpu-int4.bin` model (from +# https://www.kaggle.com/models/google/gemma/tfLite/gemma-1.1-2b-it-cpu-int4). MODEL_FILENAME="gemma2-2b-it-cpu-int8.task" ADB_WORK_DIR="/data/local/tmp" From f3b8ea3c74a07c6d49f7b0fb06269f1adc134095 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 25 Nov 2024 07:31:04 -0800 Subject: [PATCH 079/126] Allow to read the input frame rate from the header in the input side stream and to limit the frame rate. This way we can use the original frame rate of the video with an upper limit. PiperOrigin-RevId: 699975024 --- .../core/packet_resampler_calculator.cc | 57 +++++-- .../core/packet_resampler_calculator.h | 8 + .../core/packet_resampler_calculator.proto | 13 ++ .../core/packet_resampler_calculator_test.cc | 145 ++++++++++++++++++ 4 files changed, 206 insertions(+), 17 deletions(-) diff --git a/mediapipe/calculators/core/packet_resampler_calculator.cc b/mediapipe/calculators/core/packet_resampler_calculator.cc index 81a68f03f9..06bfa0b4d1 100644 --- a/mediapipe/calculators/core/packet_resampler_calculator.cc +++ b/mediapipe/calculators/core/packet_resampler_calculator.cc @@ -14,10 +14,12 @@ #include "mediapipe/calculators/core/packet_resampler_calculator.h" +#include #include #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "mediapipe/framework/port/ret_check.h" namespace { // Reflect an integer against the lower and upper bound of an interval. @@ -76,6 +78,8 @@ absl::Status PacketResamplerCalculator::GetContract(CalculatorContract* cc) { } cc->Outputs().Get(output_data_id).SetSameAs(&cc->Inputs().Get(input_data_id)); if (cc->Outputs().HasTag(kVideoHeaderTag)) { + RET_CHECK(resampler_options.max_frame_rate() <= 0) + << "VideoHeader output is not supported with max_frame_rate."; cc->Outputs().Tag(kVideoHeaderTag).Set(); } @@ -88,24 +92,13 @@ absl::Status PacketResamplerCalculator::GetContract(CalculatorContract* cc) { return absl::OkStatus(); } -absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) { - const auto resampler_options = - tool::RetrieveOptions(cc->Options(), - cc->InputSidePackets(), "OPTIONS"); - - flush_last_packet_ = resampler_options.flush_last_packet(); - jitter_ = resampler_options.jitter(); - - input_data_id_ = cc->Inputs().GetId("DATA", 0); - if (!input_data_id_.IsValid()) { - input_data_id_ = cc->Inputs().GetId("", 0); +absl::Status PacketResamplerCalculator::UpdateFrameRate( + const PacketResamplerCalculatorOptions& resampler_options, + double frame_rate) { + frame_rate_ = frame_rate; + if (resampler_options.max_frame_rate() > 0) { + frame_rate_ = std::min(frame_rate_, resampler_options.max_frame_rate()); } - output_data_id_ = cc->Outputs().GetId("DATA", 0); - if (!output_data_id_.IsValid()) { - output_data_id_ = cc->Outputs().GetId("", 0); - } - - frame_rate_ = resampler_options.frame_rate(); start_time_ = resampler_options.has_start_time() ? Timestamp(resampler_options.start_time()) : Timestamp::Min(); @@ -125,6 +118,28 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) { RET_CHECK_LE(jitter_usec_, frame_time_usec_); video_header_.frame_rate = frame_rate_; + return absl::OkStatus(); +} + +absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) { + const auto resampler_options = + tool::RetrieveOptions(cc->Options(), + cc->InputSidePackets(), "OPTIONS"); + + flush_last_packet_ = resampler_options.flush_last_packet(); + jitter_ = resampler_options.jitter(); + + input_data_id_ = cc->Inputs().GetId("DATA", 0); + if (!input_data_id_.IsValid()) { + input_data_id_ = cc->Inputs().GetId("", 0); + } + output_data_id_ = cc->Outputs().GetId("DATA", 0); + if (!output_data_id_.IsValid()) { + output_data_id_ = cc->Outputs().GetId("", 0); + } + + RET_CHECK_OK( + UpdateFrameRate(resampler_options, resampler_options.frame_rate())); if (resampler_options.output_header() != PacketResamplerCalculatorOptions::NONE && @@ -150,10 +165,18 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) { } absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) { + const auto resampler_options = + tool::RetrieveOptions(cc->Options(), + cc->InputSidePackets(), "OPTIONS"); + if (cc->InputTimestamp() == Timestamp::PreStream() && cc->Inputs().UsesTags() && cc->Inputs().HasTag(kVideoHeaderTag) && !cc->Inputs().Tag(kVideoHeaderTag).IsEmpty()) { video_header_ = cc->Inputs().Tag(kVideoHeaderTag).Get(); + if (resampler_options.use_input_frame_rate()) { + RET_CHECK_OK( + UpdateFrameRate(resampler_options, video_header_.frame_rate)); + } video_header_.frame_rate = frame_rate_; if (cc->Inputs().Get(input_data_id_).IsEmpty()) { return absl::OkStatus(); diff --git a/mediapipe/calculators/core/packet_resampler_calculator.h b/mediapipe/calculators/core/packet_resampler_calculator.h index 1cf425b5e5..5635d1fb81 100644 --- a/mediapipe/calculators/core/packet_resampler_calculator.h +++ b/mediapipe/calculators/core/packet_resampler_calculator.h @@ -146,6 +146,14 @@ class PacketResamplerCalculator : public CalculatorBase { const mediapipe::PacketResamplerCalculatorOptions& options); private: + // Updates the frame rate of the calculator. + // + // This updates the metadata of the frame rate of the calculator moving + // forward. All already processed packets will be ignored. + absl::Status UpdateFrameRate( + const mediapipe::PacketResamplerCalculatorOptions& resampler_options, + double frame_rate); + std::unique_ptr strategy_; // The timestamp of the first packet received. diff --git a/mediapipe/calculators/core/packet_resampler_calculator.proto b/mediapipe/calculators/core/packet_resampler_calculator.proto index 29ca8082a0..97f717adc7 100644 --- a/mediapipe/calculators/core/packet_resampler_calculator.proto +++ b/mediapipe/calculators/core/packet_resampler_calculator.proto @@ -108,4 +108,17 @@ message PacketResamplerCalculatorOptions { // are included in the output, even if the nearest timestamp is not // between start_time and end_time. optional bool round_limits = 8 [default = false]; + + // If set, the output frame rate is the same as the input frame rate. + // You need to provide the frame rate of the input images in the header in the + // input_side_packet. + // This option only makes sense in combination with max_frame_rate. It will + // hold on to the original frame rate unless it's higher than the + // max_frame_rate. + optional bool use_input_frame_rate = 11 [default = false]; + + // If set, the output frame rate is limited to this value. + // You need to provide the frame rate of the input images in the header in the + // input_side_packet. + optional double max_frame_rate = 12 [default = -1.0]; } diff --git a/mediapipe/calculators/core/packet_resampler_calculator_test.cc b/mediapipe/calculators/core/packet_resampler_calculator_test.cc index d80793da4a..ab74fa5463 100644 --- a/mediapipe/calculators/core/packet_resampler_calculator_test.cc +++ b/mediapipe/calculators/core/packet_resampler_calculator_test.cc @@ -14,6 +14,7 @@ #include "mediapipe/calculators/core/packet_resampler_calculator.h" +#include #include #include #include @@ -271,6 +272,150 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) { } } +TEST(PacketResamplerCalculatorTest, UseInputFrameRate_HeaderHasSameFramerate) { + CalculatorRunner runner(ParseTextProtoOrDie(R"pb( + calculator: "PacketResamplerCalculator" + input_stream: "DATA:in_data" + input_stream: "VIDEO_HEADER:in_video_header" + output_stream: "DATA:out_data" + options { + [mediapipe.PacketResamplerCalculatorOptions.ext] { + use_input_frame_rate: true + frame_rate: 1000.0 + } + } + )pb")); + + for (const int64_t ts : {0, 5000, 10010, 15001, 19990}) { + runner.MutableInputs()->Tag(kDataTag).packets.push_back( + Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts))); + } + VideoHeader video_header_in; + video_header_in.width = 10; + video_header_in.height = 100; + video_header_in.frame_rate = 200.0; + video_header_in.duration = 1.0; + video_header_in.format = ImageFormat::SRGB; + runner.MutableInputs() + ->Tag(kVideoHeaderTag) + .packets.push_back( + Adopt(new VideoHeader(video_header_in)).At(Timestamp::PreStream())); + MP_ASSERT_OK(runner.Run()); + + std::vector expected_frames = {0, 5000, 10010, 15001, 19990}; + std::vector expected_timestamps = {0, 5000, 10000, 15000, 20000}; + EXPECT_EQ(expected_frames.size(), + runner.Outputs().Tag(kDataTag).packets.size()); + EXPECT_EQ(expected_timestamps.size(), + runner.Outputs().Tag(kDataTag).packets.size()); + + int count = 0; + for (const Packet& packet : runner.Outputs().Tag(kDataTag).packets) { + EXPECT_EQ(Timestamp(expected_timestamps[count]), packet.Timestamp()); + const std::string& packet_contents = packet.Get(); + EXPECT_EQ(std::string(absl::StrCat("Frame #", expected_frames[count])), + packet_contents); + ++count; + } +} + +TEST(PacketResamplerCalculatorTest, + UseInputFrameRate_HeaderHasSmallerFramerate) { + CalculatorRunner runner(ParseTextProtoOrDie(R"pb( + calculator: "PacketResamplerCalculator" + input_stream: "DATA:in_data" + input_stream: "VIDEO_HEADER:in_video_header" + output_stream: "DATA:out_data" + options { + [mediapipe.PacketResamplerCalculatorOptions.ext] { + use_input_frame_rate: true + frame_rate: 1000.0 + } + } + )pb")); + + for (const int64_t ts : {0, 5000, 10010, 15001}) { + runner.MutableInputs()->Tag(kDataTag).packets.push_back( + Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts))); + } + VideoHeader video_header_in; + video_header_in.width = 10; + video_header_in.height = 100; + video_header_in.frame_rate = 100.0; + video_header_in.duration = 1.0; + video_header_in.format = ImageFormat::SRGB; + runner.MutableInputs() + ->Tag(kVideoHeaderTag) + .packets.push_back( + Adopt(new VideoHeader(video_header_in)).At(Timestamp::PreStream())); + MP_ASSERT_OK(runner.Run()); + + std::vector expected_frames = {0, 10010, 15001}; + std::vector expected_timestamps = {0, 10000, 20000}; + EXPECT_EQ(expected_frames.size(), + runner.Outputs().Tag(kDataTag).packets.size()); + EXPECT_EQ(expected_timestamps.size(), + runner.Outputs().Tag(kDataTag).packets.size()); + + int count = 0; + for (const Packet& packet : runner.Outputs().Tag(kDataTag).packets) { + EXPECT_EQ(Timestamp(expected_timestamps[count]), packet.Timestamp()); + const std::string& packet_contents = packet.Get(); + EXPECT_EQ(std::string(absl::StrCat("Frame #", expected_frames[count])), + packet_contents); + ++count; + } +} + +TEST(PacketResamplerCalculatorTest, + UseInputFrameRate_MaxFrameRateSmallerThanInput) { + CalculatorRunner runner(ParseTextProtoOrDie(R"pb( + calculator: "PacketResamplerCalculator" + input_stream: "DATA:in_data" + input_stream: "VIDEO_HEADER:in_video_header" + output_stream: "DATA:out_data" + options { + [mediapipe.PacketResamplerCalculatorOptions.ext] { + use_input_frame_rate: true + frame_rate: 1000.0 + max_frame_rate: 50.0 + } + } + )pb")); + + for (const int64_t ts : {0, 5000, 10010, 15001, 20010}) { + runner.MutableInputs()->Tag(kDataTag).packets.push_back( + Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts))); + } + VideoHeader video_header_in; + video_header_in.width = 10; + video_header_in.height = 200; + video_header_in.frame_rate = 100.0; + video_header_in.duration = 1.0; + video_header_in.format = ImageFormat::SRGB; + runner.MutableInputs() + ->Tag(kVideoHeaderTag) + .packets.push_back( + Adopt(new VideoHeader(video_header_in)).At(Timestamp::PreStream())); + MP_ASSERT_OK(runner.Run()); + + std::vector expected_frames = {0, 20010}; + std::vector expected_timestamps = {0, 20000}; + EXPECT_EQ(expected_frames.size(), + runner.Outputs().Tag(kDataTag).packets.size()); + EXPECT_EQ(expected_timestamps.size(), + runner.Outputs().Tag(kDataTag).packets.size()); + + int count = 0; + for (const Packet& packet : runner.Outputs().Tag(kDataTag).packets) { + EXPECT_EQ(Timestamp(expected_timestamps[count]), packet.Timestamp()); + const std::string& packet_contents = packet.Get(); + EXPECT_EQ(std::string(absl::StrCat("Frame #", expected_frames[count])), + packet_contents); + ++count; + } +} + TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepoints) { SimpleRunner runner( "[mediapipe.PacketResamplerCalculatorOptions.ext]: " From 280f3b083398413636fd49029038a59b4e81df61 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 26 Nov 2024 10:06:25 -0800 Subject: [PATCH 080/126] Add stream operator<< for TypeId PiperOrigin-RevId: 700378911 --- mediapipe/framework/tool/type_util.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mediapipe/framework/tool/type_util.h b/mediapipe/framework/tool/type_util.h index 9c955f2a36..aee708d8be 100644 --- a/mediapipe/framework/tool/type_util.h +++ b/mediapipe/framework/tool/type_util.h @@ -16,6 +16,7 @@ #define MEDIAPIPE_FRAMEWORK_TOOL_TYPE_UTIL_H_ #include +#include #include #include #include @@ -35,6 +36,9 @@ class TypeId { std::string name() const { return impl_.name(); } bool operator==(const TypeId& other) const { return impl_ == other.impl_; } bool operator<(const TypeId& other) const { return impl_ < other.impl_; } + friend std::ostream& operator<<(std::ostream& stream, const TypeId& id) { + return stream << id.name(); + } template friend H AbslHashValue(H h, const TypeId& r) { From b9e6a8d6cc98b074bd2eae6000e4a3068724bce2 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 26 Nov 2024 11:08:53 -0800 Subject: [PATCH 081/126] No public description PiperOrigin-RevId: 700399879 --- mediapipe/tasks/cc/vision/image_segmenter/calculators/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/mediapipe/tasks/cc/vision/image_segmenter/calculators/BUILD b/mediapipe/tasks/cc/vision/image_segmenter/calculators/BUILD index 93a49fdf83..a694f9e275 100644 --- a/mediapipe/tasks/cc/vision/image_segmenter/calculators/BUILD +++ b/mediapipe/tasks/cc/vision/image_segmenter/calculators/BUILD @@ -78,6 +78,7 @@ cc_library( "//mediapipe/gpu:shader_util", "//mediapipe/tasks/cc/vision/image_segmenter/proto:segmenter_options_cc_proto", "//mediapipe/tasks/cc/vision/utils:image_utils", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/strings:str_format", ] + select({ From a987a99553f0e04415b7cfe8679ba5314f28bb3c Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 26 Nov 2024 16:52:27 -0800 Subject: [PATCH 082/126] No public description PiperOrigin-RevId: 700499139 --- .../gradle/wrapper/gradle-wrapper.jar | Bin 63721 -> 43462 bytes .../gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar index 7f93135c49b765f8051ef9d0a6055ff8e46073d8..d64cd4917707c1f8861d8cb53dd15194d4248596 100644 GIT binary patch literal 43462 zcma&NWl&^owk(X(xVyW%ySuwf;qI=D6|RlDJ2cR^yEKh!@I- zp9QeisK*rlxC>+~7Dk4IxIRsKBHqdR9b3+fyL=ynHmIDe&|>O*VlvO+%z5;9Z$|DJ zb4dO}-R=MKr^6EKJiOrJdLnCJn>np?~vU-1sSFgPu;pthGwf}bG z(1db%xwr#x)r+`4AGu$j7~u2MpVs3VpLp|mx&;>`0p0vH6kF+D2CY0fVdQOZ@h;A` z{infNyvmFUiu*XG}RNMNwXrbec_*a3N=2zJ|Wh5z* z5rAX$JJR{#zP>KY**>xHTuw?|-Rg|o24V)74HcfVT;WtQHXlE+_4iPE8QE#DUm%x0 zEKr75ur~W%w#-My3Tj`hH6EuEW+8K-^5P62$7Sc5OK+22qj&Pd1;)1#4tKihi=~8C zHiQSst0cpri6%OeaR`PY>HH_;CPaRNty%WTm4{wDK8V6gCZlG@U3$~JQZ;HPvDJcT1V{ z?>H@13MJcCNe#5z+MecYNi@VT5|&UiN1D4ATT+%M+h4c$t;C#UAs3O_q=GxK0}8%8 z8J(_M9bayxN}69ex4dzM_P3oh@ZGREjVvn%%r7=xjkqxJP4kj}5tlf;QosR=%4L5y zWhgejO=vao5oX%mOHbhJ8V+SG&K5dABn6!WiKl{|oPkq(9z8l&Mm%(=qGcFzI=eLu zWc_oCLyf;hVlB@dnwY98?75B20=n$>u3b|NB28H0u-6Rpl((%KWEBOfElVWJx+5yg z#SGqwza7f}$z;n~g%4HDU{;V{gXIhft*q2=4zSezGK~nBgu9-Q*rZ#2f=Q}i2|qOp z!!y4p)4o=LVUNhlkp#JL{tfkhXNbB=Ox>M=n6soptJw-IDI|_$is2w}(XY>a=H52d z3zE$tjPUhWWS+5h=KVH&uqQS=$v3nRs&p$%11b%5qtF}S2#Pc`IiyBIF4%A!;AVoI zXU8-Rpv!DQNcF~(qQnyyMy=-AN~U>#&X1j5BLDP{?K!%h!;hfJI>$mdLSvktEr*89 zdJHvby^$xEX0^l9g$xW-d?J;L0#(`UT~zpL&*cEh$L|HPAu=P8`OQZV!-}l`noSp_ zQ-1$q$R-gDL)?6YaM!=8H=QGW$NT2SeZlb8PKJdc=F-cT@j7Xags+Pr*jPtlHFnf- zh?q<6;)27IdPc^Wdy-mX%2s84C1xZq9Xms+==F4);O`VUASmu3(RlgE#0+#giLh-& zcxm3_e}n4{%|X zJp{G_j+%`j_q5}k{eW&TlP}J2wtZ2^<^E(O)4OQX8FDp6RJq!F{(6eHWSD3=f~(h} zJXCf7=r<16X{pHkm%yzYI_=VDP&9bmI1*)YXZeB}F? z(%QsB5fo*FUZxK$oX~X^69;x~j7ms8xlzpt-T15e9}$4T-pC z6PFg@;B-j|Ywajpe4~bk#S6(fO^|mm1hKOPfA%8-_iGCfICE|=P_~e;Wz6my&)h_~ zkv&_xSAw7AZ%ThYF(4jADW4vg=oEdJGVOs>FqamoL3Np8>?!W#!R-0%2Bg4h?kz5I zKV-rKN2n(vUL%D<4oj@|`eJ>0i#TmYBtYmfla;c!ATW%;xGQ0*TW@PTlGG><@dxUI zg>+3SiGdZ%?5N=8uoLA|$4isK$aJ%i{hECP$bK{J#0W2gQ3YEa zZQ50Stn6hqdfxJ*9#NuSLwKFCUGk@c=(igyVL;;2^wi4o30YXSIb2g_ud$ zgpCr@H0qWtk2hK8Q|&wx)}4+hTYlf;$a4#oUM=V@Cw#!$(nOFFpZ;0lc!qd=c$S}Z zGGI-0jg~S~cgVT=4Vo)b)|4phjStD49*EqC)IPwyeKBLcN;Wu@Aeph;emROAwJ-0< z_#>wVm$)ygH|qyxZaet&(Vf%pVdnvKWJn9`%DAxj3ot;v>S$I}jJ$FLBF*~iZ!ZXE zkvui&p}fI0Y=IDX)mm0@tAd|fEHl~J&K}ZX(Mm3cm1UAuwJ42+AO5@HwYfDH7ipIc zmI;1J;J@+aCNG1M`Btf>YT>~c&3j~Qi@Py5JT6;zjx$cvOQW@3oQ>|}GH?TW-E z1R;q^QFjm5W~7f}c3Ww|awg1BAJ^slEV~Pk`Kd`PS$7;SqJZNj->it4DW2l15}xP6 zoCl$kyEF%yJni0(L!Z&14m!1urXh6Btj_5JYt1{#+H8w?5QI%% zo-$KYWNMJVH?Hh@1n7OSu~QhSswL8x0=$<8QG_zepi_`y_79=nK=_ZP_`Em2UI*tyQoB+r{1QYZCpb?2OrgUw#oRH$?^Tj!Req>XiE#~B|~ z+%HB;=ic+R@px4Ld8mwpY;W^A%8%l8$@B@1m5n`TlKI6bz2mp*^^^1mK$COW$HOfp zUGTz-cN9?BGEp}5A!mDFjaiWa2_J2Iq8qj0mXzk; z66JBKRP{p%wN7XobR0YjhAuW9T1Gw3FDvR5dWJ8ElNYF94eF3ebu+QwKjtvVu4L zI9ip#mQ@4uqVdkl-TUQMb^XBJVLW(-$s;Nq;@5gr4`UfLgF$adIhd?rHOa%D);whv z=;krPp~@I+-Z|r#s3yCH+c1US?dnm+C*)r{m+86sTJusLdNu^sqLrfWed^ndHXH`m zd3#cOe3>w-ga(Dus_^ppG9AC>Iq{y%%CK+Cro_sqLCs{VLuK=dev>OL1dis4(PQ5R zcz)>DjEkfV+MO;~>VUlYF00SgfUo~@(&9$Iy2|G0T9BSP?&T22>K46D zL*~j#yJ?)^*%J3!16f)@Y2Z^kS*BzwfAQ7K96rFRIh>#$*$_Io;z>ux@}G98!fWR@ zGTFxv4r~v)Gsd|pF91*-eaZ3Qw1MH$K^7JhWIdX%o$2kCbvGDXy)a?@8T&1dY4`;L z4Kn+f%SSFWE_rpEpL9bnlmYq`D!6F%di<&Hh=+!VI~j)2mfil03T#jJ_s?}VV0_hp z7T9bWxc>Jm2Z0WMU?`Z$xE74Gu~%s{mW!d4uvKCx@WD+gPUQ zV0vQS(Ig++z=EHN)BR44*EDSWIyT~R4$FcF*VEY*8@l=218Q05D2$|fXKFhRgBIEE zdDFB}1dKkoO^7}{5crKX!p?dZWNz$m>1icsXG2N+((x0OIST9Zo^DW_tytvlwXGpn zs8?pJXjEG;T@qrZi%#h93?FP$!&P4JA(&H61tqQi=opRzNpm zkrG}$^t9&XduK*Qa1?355wd8G2CI6QEh@Ua>AsD;7oRUNLPb76m4HG3K?)wF~IyS3`fXuNM>${?wmB zpVz;?6_(Fiadfd{vUCBM*_kt$+F3J+IojI;9L(gc9n3{sEZyzR9o!_mOwFC#tQ{Q~ zP3-`#uK#tP3Q7~Q;4H|wjZHO8h7e4IuBxl&vz2w~D8)w=Wtg31zpZhz%+kzSzL*dV zwp@{WU4i;hJ7c2f1O;7Mz6qRKeASoIv0_bV=i@NMG*l<#+;INk-^`5w@}Dj~;k=|}qM1vq_P z|GpBGe_IKq|LNy9SJhKOQ$c=5L{Dv|Q_lZl=-ky*BFBJLW9&y_C|!vyM~rQx=!vun z?rZJQB5t}Dctmui5i31C_;_}CEn}_W%>oSXtt>@kE1=JW*4*v4tPp;O6 zmAk{)m!)}34pTWg8{i>($%NQ(Tl;QC@J@FfBoc%Gr&m560^kgSfodAFrIjF}aIw)X zoXZ`@IsMkc8_=w%-7`D6Y4e*CG8k%Ud=GXhsTR50jUnm+R*0A(O3UKFg0`K;qp1bl z7``HN=?39ic_kR|^R^~w-*pa?Vj#7|e9F1iRx{GN2?wK!xR1GW!qa=~pjJb-#u1K8 zeR?Y2i-pt}yJq;SCiVHODIvQJX|ZJaT8nO+(?HXbLefulKKgM^B(UIO1r+S=7;kLJ zcH}1J=Px2jsh3Tec&v8Jcbng8;V-`#*UHt?hB(pmOipKwf3Lz8rG$heEB30Sg*2rx zV<|KN86$soN(I!BwO`1n^^uF2*x&vJ$2d$>+`(romzHP|)K_KkO6Hc>_dwMW-M(#S zK(~SiXT1@fvc#U+?|?PniDRm01)f^#55;nhM|wi?oG>yBsa?~?^xTU|fX-R(sTA+5 zaq}-8Tx7zrOy#3*JLIIVsBmHYLdD}!0NP!+ITW+Thn0)8SS!$@)HXwB3tY!fMxc#1 zMp3H?q3eD?u&Njx4;KQ5G>32+GRp1Ee5qMO0lZjaRRu&{W<&~DoJNGkcYF<5(Ab+J zgO>VhBl{okDPn78<%&e2mR{jwVCz5Og;*Z;;3%VvoGo_;HaGLWYF7q#jDX=Z#Ml`H z858YVV$%J|e<1n`%6Vsvq7GmnAV0wW4$5qQ3uR@1i>tW{xrl|ExywIc?fNgYlA?C5 zh$ezAFb5{rQu6i7BSS5*J-|9DQ{6^BVQ{b*lq`xS@RyrsJN?-t=MTMPY;WYeKBCNg z^2|pN!Q^WPJuuO4!|P@jzt&tY1Y8d%FNK5xK(!@`jO2aEA*4 zkO6b|UVBipci?){-Ke=+1;mGlND8)6+P;8sq}UXw2hn;fc7nM>g}GSMWu&v&fqh

iViYT=fZ(|3Ox^$aWPp4a8h24tD<|8-!aK0lHgL$N7Efw}J zVIB!7=T$U`ao1?upi5V4Et*-lTG0XvExbf!ya{cua==$WJyVG(CmA6Of*8E@DSE%L z`V^$qz&RU$7G5mg;8;=#`@rRG`-uS18$0WPN@!v2d{H2sOqP|!(cQ@ zUHo!d>>yFArLPf1q`uBvY32miqShLT1B@gDL4XoVTK&@owOoD)OIHXrYK-a1d$B{v zF^}8D3Y^g%^cnvScOSJR5QNH+BI%d|;J;wWM3~l>${fb8DNPg)wrf|GBP8p%LNGN# z3EaIiItgwtGgT&iYCFy9-LG}bMI|4LdmmJt@V@% zb6B)1kc=T)(|L@0;wr<>=?r04N;E&ef+7C^`wPWtyQe(*pD1pI_&XHy|0gIGHMekd zF_*M4yi6J&Z4LQj65)S zXwdM{SwUo%3SbPwFsHgqF@V|6afT|R6?&S;lw=8% z3}@9B=#JI3@B*#4s!O))~z zc>2_4Q_#&+5V`GFd?88^;c1i7;Vv_I*qt!_Yx*n=;rj!82rrR2rQ8u5(Ejlo{15P% zs~!{%XJ>FmJ})H^I9bn^Re&38H{xA!0l3^89k(oU;bZWXM@kn$#aoS&Y4l^-WEn-fH39Jb9lA%s*WsKJQl?n9B7_~P z-XM&WL7Z!PcoF6_D>V@$CvUIEy=+Z&0kt{szMk=f1|M+r*a43^$$B^MidrT0J;RI` z(?f!O<8UZkm$_Ny$Hth1J#^4ni+im8M9mr&k|3cIgwvjAgjH z8`N&h25xV#v*d$qBX5jkI|xOhQn!>IYZK7l5#^P4M&twe9&Ey@@GxYMxBZq2e7?`q z$~Szs0!g{2fGcp9PZEt|rdQ6bhAgpcLHPz?f-vB?$dc*!9OL?Q8mn7->bFD2Si60* z!O%y)fCdMSV|lkF9w%x~J*A&srMyYY3{=&$}H zGQ4VG_?$2X(0|vT0{=;W$~icCI{b6W{B!Q8xdGhF|D{25G_5_+%s(46lhvNLkik~R z>nr(&C#5wwOzJZQo9m|U<;&Wk!_#q|V>fsmj1g<6%hB{jGoNUPjgJslld>xmODzGjYc?7JSuA?A_QzjDw5AsRgi@Y|Z0{F{!1=!NES-#*f^s4l0Hu zz468))2IY5dmD9pa*(yT5{EyP^G>@ZWumealS-*WeRcZ}B%gxq{MiJ|RyX-^C1V=0 z@iKdrGi1jTe8Ya^x7yyH$kBNvM4R~`fbPq$BzHum-3Zo8C6=KW@||>zsA8-Y9uV5V z#oq-f5L5}V<&wF4@X@<3^C%ptp6+Ce)~hGl`kwj)bsAjmo_GU^r940Z-|`<)oGnh7 zFF0Tde3>ui?8Yj{sF-Z@)yQd~CGZ*w-6p2U<8}JO-sRsVI5dBji`01W8A&3$?}lxBaC&vn0E$c5tW* zX>5(zzZ=qn&!J~KdsPl;P@bmA-Pr8T*)eh_+Dv5=Ma|XSle6t(k8qcgNyar{*ReQ8 zTXwi=8vr>!3Ywr+BhggHDw8ke==NTQVMCK`$69fhzEFB*4+H9LIvdt-#IbhZvpS}} zO3lz;P?zr0*0$%-Rq_y^k(?I{Mk}h@w}cZpMUp|ucs55bcloL2)($u%mXQw({Wzc~ z;6nu5MkjP)0C(@%6Q_I_vsWrfhl7Zpoxw#WoE~r&GOSCz;_ro6i(^hM>I$8y>`!wW z*U^@?B!MMmb89I}2(hcE4zN2G^kwyWCZp5JG>$Ez7zP~D=J^LMjSM)27_0B_X^C(M z`fFT+%DcKlu?^)FCK>QzSnV%IsXVcUFhFdBP!6~se&xxrIxsvySAWu++IrH;FbcY$ z2DWTvSBRfLwdhr0nMx+URA$j3i7_*6BWv#DXfym?ZRDcX9C?cY9sD3q)uBDR3uWg= z(lUIzB)G$Hr!){>E{s4Dew+tb9kvToZp-1&c?y2wn@Z~(VBhqz`cB;{E4(P3N2*nJ z_>~g@;UF2iG{Kt(<1PyePTKahF8<)pozZ*xH~U-kfoAayCwJViIrnqwqO}7{0pHw$ zs2Kx?s#vQr7XZ264>5RNKSL8|Ty^=PsIx^}QqOOcfpGUU4tRkUc|kc7-!Ae6!+B{o~7nFpm3|G5^=0#Bnm6`V}oSQlrX(u%OWnC zoLPy&Q;1Jui&7ST0~#+}I^&?vcE*t47~Xq#YwvA^6^} z`WkC)$AkNub|t@S!$8CBlwbV~?yp&@9h{D|3z-vJXgzRC5^nYm+PyPcgRzAnEi6Q^gslXYRv4nycsy-SJu?lMps-? zV`U*#WnFsdPLL)Q$AmD|0`UaC4ND07+&UmOu!eHruzV|OUox<+Jl|Mr@6~C`T@P%s zW7sgXLF2SSe9Fl^O(I*{9wsFSYb2l%-;&Pi^dpv!{)C3d0AlNY6!4fgmSgj_wQ*7Am7&$z;Jg&wgR-Ih;lUvWS|KTSg!&s_E9_bXBkZvGiC6bFKDWZxsD$*NZ#_8bl zG1P-#@?OQzED7@jlMJTH@V!6k;W>auvft)}g zhoV{7$q=*;=l{O>Q4a@ ziMjf_u*o^PsO)#BjC%0^h>Xp@;5$p{JSYDt)zbb}s{Kbt!T*I@Pk@X0zds6wsefuU zW$XY%yyRGC94=6mf?x+bbA5CDQ2AgW1T-jVAJbm7K(gp+;v6E0WI#kuACgV$r}6L? zd|Tj?^%^*N&b>Dd{Wr$FS2qI#Ucs1yd4N+RBUQiSZGujH`#I)mG&VKoDh=KKFl4=G z&MagXl6*<)$6P}*Tiebpz5L=oMaPrN+caUXRJ`D?=K9!e0f{@D&cZLKN?iNP@X0aF zE(^pl+;*T5qt?1jRC=5PMgV!XNITRLS_=9{CJExaQj;lt!&pdzpK?8p>%Mb+D z?yO*uSung=-`QQ@yX@Hyd4@CI^r{2oiu`%^bNkz+Nkk!IunjwNC|WcqvX~k=><-I3 zDQdbdb|!v+Iz01$w@aMl!R)koD77Xp;eZwzSl-AT zr@Vu{=xvgfq9akRrrM)}=!=xcs+U1JO}{t(avgz`6RqiiX<|hGG1pmop8k6Q+G_mv zJv|RfDheUp2L3=^C=4aCBMBn0aRCU(DQwX-W(RkRwmLeuJYF<0urcaf(=7)JPg<3P zQs!~G)9CT18o!J4{zX{_e}4eS)U-E)0FAt}wEI(c0%HkxgggW;(1E=>J17_hsH^sP z%lT0LGgbUXHx-K*CI-MCrP66UP0PvGqM$MkeLyqHdbgP|_Cm!7te~b8p+e6sQ_3k| zVcwTh6d83ltdnR>D^)BYQpDKlLk3g0Hdcgz2}%qUs9~~Rie)A-BV1mS&naYai#xcZ z(d{8=-LVpTp}2*y)|gR~;qc7fp26}lPcLZ#=JpYcn3AT9(UIdOyg+d(P5T7D&*P}# zQCYplZO5|7+r19%9e`v^vfSS1sbX1c%=w1;oyruXB%Kl$ACgKQ6=qNWLsc=28xJjg zwvsI5-%SGU|3p>&zXVl^vVtQT3o-#$UT9LI@Npz~6=4!>mc431VRNN8od&Ul^+G_kHC`G=6WVWM z%9eWNyy(FTO|A+@x}Ou3CH)oi;t#7rAxdIXfNFwOj_@Y&TGz6P_sqiB`Q6Lxy|Q{`|fgmRG(k+!#b*M+Z9zFce)f-7;?Km5O=LHV9f9_87; zF7%R2B+$?@sH&&-$@tzaPYkw0;=i|;vWdI|Wl3q_Zu>l;XdIw2FjV=;Mq5t1Q0|f< zs08j54Bp`3RzqE=2enlkZxmX6OF+@|2<)A^RNQpBd6o@OXl+i)zO%D4iGiQNuXd+zIR{_lb96{lc~bxsBveIw6umhShTX+3@ZJ=YHh@ zWY3(d0azg;7oHn>H<>?4@*RQbi>SmM=JrHvIG(~BrvI)#W(EAeO6fS+}mxxcc+X~W6&YVl86W9WFSS}Vz-f9vS?XUDBk)3TcF z8V?$4Q)`uKFq>xT=)Y9mMFVTUk*NIA!0$?RP6Ig0TBmUFrq*Q-Agq~DzxjStQyJ({ zBeZ;o5qUUKg=4Hypm|}>>L=XKsZ!F$yNTDO)jt4H0gdQ5$f|d&bnVCMMXhNh)~mN z@_UV6D7MVlsWz+zM+inZZp&P4fj=tm6fX)SG5H>OsQf_I8c~uGCig$GzuwViK54bcgL;VN|FnyQl>Ed7(@>=8$a_UKIz|V6CeVSd2(P z0Uu>A8A+muM%HLFJQ9UZ5c)BSAv_zH#1f02x?h9C}@pN@6{>UiAp>({Fn(T9Q8B z^`zB;kJ5b`>%dLm+Ol}ty!3;8f1XDSVX0AUe5P#@I+FQ-`$(a;zNgz)4x5hz$Hfbg z!Q(z26wHLXko(1`;(BAOg_wShpX0ixfWq3ponndY+u%1gyX)_h=v1zR#V}#q{au6; z!3K=7fQwnRfg6FXtNQmP>`<;!N137paFS%y?;lb1@BEdbvQHYC{976l`cLqn;b8lp zIDY>~m{gDj(wfnK!lpW6pli)HyLEiUrNc%eXTil|F2s(AY+LW5hkKb>TQ3|Q4S9rr zpDs4uK_co6XPsn_z$LeS{K4jFF`2>U`tbgKdyDne`xmR<@6AA+_hPNKCOR-Zqv;xk zu5!HsBUb^!4uJ7v0RuH-7?l?}b=w5lzzXJ~gZcxRKOovSk@|#V+MuX%Y+=;14i*%{)_gSW9(#4%)AV#3__kac1|qUy!uyP{>?U#5wYNq}y$S9pCc zFc~4mgSC*G~j0u#qqp9 z${>3HV~@->GqEhr_Xwoxq?Hjn#=s2;i~g^&Hn|aDKpA>Oc%HlW(KA1?BXqpxB;Ydx)w;2z^MpjJ(Qi(X!$5RC z*P{~%JGDQqojV>2JbEeCE*OEu!$XJ>bWA9Oa_Hd;y)F%MhBRi*LPcdqR8X`NQ&1L# z5#9L*@qxrx8n}LfeB^J{%-?SU{FCwiWyHp682F+|pa+CQa3ZLzBqN1{)h4d6+vBbV zC#NEbQLC;}me3eeYnOG*nXOJZEU$xLZ1<1Y=7r0(-U0P6-AqwMAM`a(Ed#7vJkn6plb4eI4?2y3yOTGmmDQ!z9`wzbf z_OY#0@5=bnep;MV0X_;;SJJWEf^E6Bd^tVJ9znWx&Ks8t*B>AM@?;D4oWUGc z!H*`6d7Cxo6VuyS4Eye&L1ZRhrRmN6Lr`{NL(wDbif|y&z)JN>Fl5#Wi&mMIr5i;x zBx}3YfF>>8EC(fYnmpu~)CYHuHCyr5*`ECap%t@y=jD>!_%3iiE|LN$mK9>- zHdtpy8fGZtkZF?%TW~29JIAfi2jZT8>OA7=h;8T{{k?c2`nCEx9$r zS+*&vt~2o^^J+}RDG@+9&M^K*z4p{5#IEVbz`1%`m5c2};aGt=V?~vIM}ZdPECDI)47|CWBCfDWUbxBCnmYivQ*0Nu_xb*C>~C9(VjHM zxe<*D<#dQ8TlpMX2c@M<9$w!RP$hpG4cs%AI){jp*Sj|*`m)5(Bw*A0$*i-(CA5#%>a)$+jI2C9r6|(>J8InryENI z$NohnxDUB;wAYDwrb*!N3noBTKPpPN}~09SEL18tkG zxgz(RYU_;DPT{l?Q$+eaZaxnsWCA^ds^0PVRkIM%bOd|G2IEBBiz{&^JtNsODs;5z zICt_Zj8wo^KT$7Bg4H+y!Df#3mbl%%?|EXe!&(Vmac1DJ*y~3+kRKAD=Ovde4^^%~ zw<9av18HLyrf*_>Slp;^i`Uy~`mvBjZ|?Ad63yQa#YK`4+c6;pW4?XIY9G1(Xh9WO8{F-Aju+nS9Vmv=$Ac0ienZ+p9*O%NG zMZKy5?%Z6TAJTE?o5vEr0r>f>hb#2w2U3DL64*au_@P!J!TL`oH2r*{>ffu6|A7tv zL4juf$DZ1MW5ZPsG!5)`k8d8c$J$o;%EIL0va9&GzWvkS%ZsGb#S(?{!UFOZ9<$a| zY|a+5kmD5N&{vRqkgY>aHsBT&`rg|&kezoD)gP0fsNYHsO#TRc_$n6Lf1Z{?+DLziXlHrq4sf(!>O{?Tj;Eh@%)+nRE_2VxbN&&%%caU#JDU%vL3}Cb zsb4AazPI{>8H&d=jUaZDS$-0^AxE@utGs;-Ez_F(qC9T=UZX=>ok2k2 ziTn{K?y~a5reD2A)P${NoI^>JXn>`IeArow(41c-Wm~)wiryEP(OS{YXWi7;%dG9v zI?mwu1MxD{yp_rrk!j^cKM)dc4@p4Ezyo%lRN|XyD}}>v=Xoib0gOcdXrQ^*61HNj z=NP|pd>@yfvr-=m{8$3A8TQGMTE7g=z!%yt`8`Bk-0MMwW~h^++;qyUP!J~ykh1GO z(FZ59xuFR$(WE;F@UUyE@Sp>`aVNjyj=Ty>_Vo}xf`e7`F;j-IgL5`1~-#70$9_=uBMq!2&1l zomRgpD58@)YYfvLtPW}{C5B35R;ZVvB<<#)x%srmc_S=A7F@DW8>QOEGwD6suhwCg z>Pa+YyULhmw%BA*4yjDp|2{!T98~<6Yfd(wo1mQ!KWwq0eg+6)o1>W~f~kL<-S+P@$wx*zeI|1t7z#Sxr5 zt6w+;YblPQNplq4Z#T$GLX#j6yldXAqj>4gAnnWtBICUnA&-dtnlh=t0Ho_vEKwV` z)DlJi#!@nkYV#$!)@>udAU*hF?V`2$Hf=V&6PP_|r#Iv*J$9)pF@X3`k;5})9^o4y z&)~?EjX5yX12O(BsFy-l6}nYeuKkiq`u9145&3Ssg^y{5G3Pse z9w(YVa0)N-fLaBq1`P!_#>SS(8fh_5!f{UrgZ~uEdeMJIz7DzI5!NHHqQtm~#CPij z?=N|J>nPR6_sL7!f4hD_|KH`vf8(Wpnj-(gPWH+ZvID}%?~68SwhPTC3u1_cB`otq z)U?6qo!ZLi5b>*KnYHWW=3F!p%h1;h{L&(Q&{qY6)_qxNfbP6E3yYpW!EO+IW3?@J z);4>g4gnl^8klu7uA>eGF6rIGSynacogr)KUwE_R4E5Xzi*Qir@b-jy55-JPC8c~( zo!W8y9OGZ&`xmc8;=4-U9=h{vCqfCNzYirONmGbRQlR`WWlgnY+1wCXbMz&NT~9*| z6@FrzP!LX&{no2!Ln_3|I==_4`@}V?4a;YZKTdw;vT<+K+z=uWbW(&bXEaWJ^W8Td z-3&1bY^Z*oM<=M}LVt>_j+p=2Iu7pZmbXrhQ_k)ysE9yXKygFNw$5hwDn(M>H+e1&9BM5!|81vd%r%vEm zqxY3?F@fb6O#5UunwgAHR9jp_W2zZ}NGp2%mTW@(hz7$^+a`A?mb8|_G*GNMJ) zjqegXQio=i@AINre&%ofexAr95aop5C+0MZ0m-l=MeO8m3epm7U%vZB8+I+C*iNFM z#T3l`gknX;D$-`2XT^Cg*vrv=RH+P;_dfF++cP?B_msQI4j+lt&rX2)3GaJx%W*Nn zkML%D{z5tpHH=dksQ*gzc|}gzW;lwAbxoR07VNgS*-c3d&8J|;@3t^ zVUz*J*&r7DFRuFVDCJDK8V9NN5hvpgGjwx+5n)qa;YCKe8TKtdnh{I7NU9BCN!0dq zczrBk8pE{{@vJa9ywR@mq*J=v+PG;?fwqlJVhijG!3VmIKs>9T6r7MJpC)m!Tc#>g zMtVsU>wbwFJEfwZ{vB|ZlttNe83)$iz`~#8UJ^r)lJ@HA&G#}W&ZH*;k{=TavpjWE z7hdyLZPf*X%Gm}i`Y{OGeeu^~nB8=`{r#TUrM-`;1cBvEd#d!kPqIgYySYhN-*1;L z^byj%Yi}Gx)Wnkosi337BKs}+5H5dth1JA{Ir-JKN$7zC)*}hqeoD(WfaUDPT>0`- z(6sa0AoIqASwF`>hP}^|)a_j2s^PQn*qVC{Q}htR z5-)duBFXT_V56-+UohKXlq~^6uf!6sA#ttk1o~*QEy_Y-S$gAvq47J9Vtk$5oA$Ct zYhYJ@8{hsC^98${!#Ho?4y5MCa7iGnfz}b9jE~h%EAAv~Qxu)_rAV;^cygV~5r_~?l=B`zObj7S=H=~$W zPtI_m%g$`kL_fVUk9J@>EiBH zOO&jtn~&`hIFMS5S`g8w94R4H40mdNUH4W@@XQk1sr17b{@y|JB*G9z1|CrQjd+GX z6+KyURG3;!*BQrentw{B2R&@2&`2}n(z-2&X7#r!{yg@Soy}cRD~j zj9@UBW+N|4HW4AWapy4wfUI- zZ`gSL6DUlgj*f1hSOGXG0IVH8HxK?o2|3HZ;KW{K+yPAlxtb)NV_2AwJm|E)FRs&& z=c^e7bvUsztY|+f^k7NXs$o1EUq>cR7C0$UKi6IooHWlK_#?IWDkvywnzg&ThWo^? z2O_N{5X39#?eV9l)xI(>@!vSB{DLt*oY!K1R8}_?%+0^C{d9a%N4 zoxHVT1&Lm|uDX%$QrBun5e-F`HJ^T$ zmzv)p@4ZHd_w9!%Hf9UYNvGCw2TTTbrj9pl+T9%-_-}L(tES>Or-}Z4F*{##n3~L~TuxjirGuIY#H7{%$E${?p{Q01 zi6T`n;rbK1yIB9jmQNycD~yZq&mbIsFWHo|ZAChSFPQa<(%d8mGw*V3fh|yFoxOOiWJd(qvVb!Z$b88cg->N=qO*4k~6;R==|9ihg&riu#P~s4Oap9O7f%crSr^rljeIfXDEg>wi)&v*a%7zpz<9w z*r!3q9J|390x`Zk;g$&OeN&ctp)VKRpDSV@kU2Q>jtok($Y-*x8_$2piTxun81@vt z!Vj?COa0fg2RPXMSIo26T=~0d`{oGP*eV+$!0I<(4azk&Vj3SiG=Q!6mX0p$z7I}; z9BJUFgT-K9MQQ-0@Z=^7R<{bn2Fm48endsSs`V7_@%8?Bxkqv>BDoVcj?K#dV#uUP zL1ND~?D-|VGKe3Rw_7-Idpht>H6XRLh*U7epS6byiGvJpr%d}XwfusjH9g;Z98H`x zyde%%5mhGOiL4wljCaWCk-&uE4_OOccb9c!ZaWt4B(wYl!?vyzl%7n~QepN&eFUrw zFIOl9c({``6~QD+43*_tzP{f2x41h(?b43^y6=iwyB)2os5hBE!@YUS5?N_tXd=h( z)WE286Fbd>R4M^P{!G)f;h<3Q>Fipuy+d2q-)!RyTgt;wr$(?9ox3;q+{E*ZQHhOn;lM`cjnu9 zXa48ks-v(~b*;MAI<>YZH(^NV8vjb34beE<_cwKlJoR;k6lJNSP6v}uiyRD?|0w+X@o1ONrH8a$fCxXpf? z?$DL0)7|X}Oc%h^zrMKWc-NS9I0Utu@>*j}b@tJ=ixQSJ={4@854wzW@E>VSL+Y{i z#0b=WpbCZS>kUCO_iQz)LoE>P5LIG-hv9E+oG}DtlIDF>$tJ1aw9^LuhLEHt?BCj& z(O4I8v1s#HUi5A>nIS-JK{v!7dJx)^Yg%XjNmlkWAq2*cv#tHgz`Y(bETc6CuO1VkN^L-L3j_x<4NqYb5rzrLC-7uOv z!5e`GZt%B782C5-fGnn*GhDF$%(qP<74Z}3xx+{$4cYKy2ikxI7B2N+2r07DN;|-T->nU&!=Cm#rZt%O_5c&1Z%nlWq3TKAW0w zQqemZw_ue--2uKQsx+niCUou?HjD`xhEjjQd3%rrBi82crq*~#uA4+>vR<_S{~5ce z-2EIl?~s z1=GVL{NxP1N3%=AOaC}j_Fv=ur&THz zyO!d9kHq|c73kpq`$+t+8Bw7MgeR5~`d7ChYyGCBWSteTB>8WAU(NPYt2Dk`@#+}= zI4SvLlyk#pBgVigEe`?NG*vl7V6m+<}%FwPV=~PvvA)=#ths==DRTDEYh4V5}Cf$z@#;< zyWfLY_5sP$gc3LLl2x+Ii)#b2nhNXJ{R~vk`s5U7Nyu^3yFg&D%Txwj6QezMX`V(x z=C`{76*mNb!qHHs)#GgGZ_7|vkt9izl_&PBrsu@}L`X{95-2jf99K)0=*N)VxBX2q z((vkpP2RneSIiIUEnGb?VqbMb=Zia+rF~+iqslydE34cSLJ&BJW^3knX@M;t*b=EA zNvGzv41Ld_T+WT#XjDB840vovUU^FtN_)G}7v)1lPetgpEK9YS^OWFkPoE{ovj^=@ zO9N$S=G$1ecndT_=5ehth2Lmd1II-PuT~C9`XVePw$y8J#dpZ?Tss<6wtVglm(Ok7 z3?^oi@pPio6l&!z8JY(pJvG=*pI?GIOu}e^EB6QYk$#FJQ%^AIK$I4epJ+9t?KjqA+bkj&PQ*|vLttme+`9G=L% ziadyMw_7-M)hS(3E$QGNCu|o23|%O+VN7;Qggp?PB3K-iSeBa2b}V4_wY`G1Jsfz4 z9|SdB^;|I8E8gWqHKx!vj_@SMY^hLEIbSMCuE?WKq=c2mJK z8LoG-pnY!uhqFv&L?yEuxo{dpMTsmCn)95xanqBrNPTgXP((H$9N${Ow~Is-FBg%h z53;|Y5$MUN)9W2HBe2TD`ct^LHI<(xWrw}$qSoei?}s)&w$;&!14w6B6>Yr6Y8b)S z0r71`WmAvJJ`1h&poLftLUS6Ir zC$bG9!Im_4Zjse)#K=oJM9mHW1{%l8sz$1o?ltdKlLTxWWPB>Vk22czVt|1%^wnN@*!l)}?EgtvhC>vlHm^t+ogpgHI1_$1ox9e;>0!+b(tBrmXRB`PY1vp-R**8N7 zGP|QqI$m(Rdu#=(?!(N}G9QhQ%o!aXE=aN{&wtGP8|_qh+7a_j_sU5|J^)vxq;# zjvzLn%_QPHZZIWu1&mRAj;Sa_97p_lLq_{~j!M9N^1yp3U_SxRqK&JnR%6VI#^E12 z>CdOVI^_9aPK2eZ4h&^{pQs}xsijXgFYRIxJ~N7&BB9jUR1fm!(xl)mvy|3e6-B3j zJn#ajL;bFTYJ2+Q)tDjx=3IklO@Q+FFM}6UJr6km7hj7th9n_&JR7fnqC!hTZoM~T zBeaVFp%)0cbPhejX<8pf5HyRUj2>aXnXBqDJe73~J%P(2C?-RT{c3NjE`)om! zl$uewSgWkE66$Kb34+QZZvRn`fob~Cl9=cRk@Es}KQm=?E~CE%spXaMO6YmrMl%9Q zlA3Q$3|L1QJ4?->UjT&CBd!~ru{Ih^in&JXO=|<6J!&qp zRe*OZ*cj5bHYlz!!~iEKcuE|;U4vN1rk$xq6>bUWD*u(V@8sG^7>kVuo(QL@Ki;yL zWC!FT(q{E8#on>%1iAS0HMZDJg{Z{^!De(vSIq&;1$+b)oRMwA3nc3mdTSG#3uYO_ z>+x;7p4I;uHz?ZB>dA-BKl+t-3IB!jBRgdvAbW!aJ(Q{aT>+iz?91`C-xbe)IBoND z9_Xth{6?(y3rddwY$GD65IT#f3<(0o#`di{sh2gm{dw*#-Vnc3r=4==&PU^hCv$qd zjw;>i&?L*Wq#TxG$mFIUf>eK+170KG;~+o&1;Tom9}}mKo23KwdEM6UonXgc z!6N(@k8q@HPw{O8O!lAyi{rZv|DpgfU{py+j(X_cwpKqcalcqKIr0kM^%Br3SdeD> zHSKV94Yxw;pjzDHo!Q?8^0bb%L|wC;4U^9I#pd5O&eexX+Im{ z?jKnCcsE|H?{uGMqVie_C~w7GX)kYGWAg%-?8|N_1#W-|4F)3YTDC+QSq1s!DnOML3@d`mG%o2YbYd#jww|jD$gotpa)kntakp#K;+yo-_ZF9qrNZw<%#C zuPE@#3RocLgPyiBZ+R_-FJ_$xP!RzWm|aN)S+{$LY9vvN+IW~Kf3TsEIvP+B9Mtm! zpfNNxObWQpLoaO&cJh5>%slZnHl_Q~(-Tfh!DMz(dTWld@LG1VRF`9`DYKhyNv z2pU|UZ$#_yUx_B_|MxUq^glT}O5Xt(Vm4Mr02><%C)@v;vPb@pT$*yzJ4aPc_FZ3z z3}PLoMBIM>q_9U2rl^sGhk1VUJ89=*?7|v`{!Z{6bqFMq(mYiA?%KbsI~JwuqVA9$H5vDE+VocjX+G^%bieqx->s;XWlKcuv(s%y%D5Xbc9+ zc(_2nYS1&^yL*ey664&4`IoOeDIig}y-E~_GS?m;D!xv5-xwz+G`5l6V+}CpeJDi^ z%4ed$qowm88=iYG+(`ld5Uh&>Dgs4uPHSJ^TngXP_V6fPyl~>2bhi20QB%lSd#yYn zO05?KT1z@?^-bqO8Cg`;ft>ilejsw@2%RR7;`$Vs;FmO(Yr3Fp`pHGr@P2hC%QcA|X&N2Dn zYf`MqXdHi%cGR@%y7Rg7?d3?an){s$zA{!H;Ie5exE#c~@NhQUFG8V=SQh%UxUeiV zd7#UcYqD=lk-}sEwlpu&H^T_V0{#G?lZMxL7ih_&{(g)MWBnCZxtXg znr#}>U^6!jA%e}@Gj49LWG@*&t0V>Cxc3?oO7LSG%~)Y5}f7vqUUnQ;STjdDU}P9IF9d9<$;=QaXc zL1^X7>fa^jHBu_}9}J~#-oz3Oq^JmGR#?GO7b9a(=R@fw@}Q{{@`Wy1vIQ#Bw?>@X z-_RGG@wt|%u`XUc%W{J z>iSeiz8C3H7@St3mOr_mU+&bL#Uif;+Xw-aZdNYUpdf>Rvu0i0t6k*}vwU`XNO2he z%miH|1tQ8~ZK!zmL&wa3E;l?!!XzgV#%PMVU!0xrDsNNZUWKlbiOjzH-1Uoxm8E#r`#2Sz;-o&qcqB zC-O_R{QGuynW14@)7&@yw1U}uP(1cov)twxeLus0s|7ayrtT8c#`&2~Fiu2=R;1_4bCaD=*E@cYI>7YSnt)nQc zohw5CsK%m?8Ack)qNx`W0_v$5S}nO|(V|RZKBD+btO?JXe|~^Qqur%@eO~<8-L^9d z=GA3-V14ng9L29~XJ>a5k~xT2152zLhM*@zlp2P5Eu}bywkcqR;ISbas&#T#;HZSf z2m69qTV(V@EkY(1Dk3`}j)JMo%ZVJ*5eB zYOjIisi+igK0#yW*gBGj?@I{~mUOvRFQR^pJbEbzFxTubnrw(Muk%}jI+vXmJ;{Q6 zrSobKD>T%}jV4Ub?L1+MGOD~0Ir%-`iTnWZN^~YPrcP5y3VMAzQ+&en^VzKEb$K!Q z<7Dbg&DNXuow*eD5yMr+#08nF!;%4vGrJI++5HdCFcGLfMW!KS*Oi@=7hFwDG!h2< zPunUEAF+HncQkbfFj&pbzp|MU*~60Z(|Ik%Tn{BXMN!hZOosNIseT?R;A`W?=d?5X zK(FB=9mZusYahp|K-wyb={rOpdn=@;4YI2W0EcbMKyo~-#^?h`BA9~o285%oY zfifCh5Lk$SY@|2A@a!T2V+{^!psQkx4?x0HSV`(w9{l75QxMk!)U52Lbhn{8ol?S) zCKo*7R(z!uk<6*qO=wh!Pul{(qq6g6xW;X68GI_CXp`XwO zxuSgPRAtM8K7}5E#-GM!*ydOOG_{A{)hkCII<|2=ma*71ci_-}VPARm3crFQjLYV! z9zbz82$|l01mv`$WahE2$=fAGWkd^X2kY(J7iz}WGS z@%MyBEO=A?HB9=^?nX`@nh;7;laAjs+fbo!|K^mE!tOB>$2a_O0y-*uaIn8k^6Y zSbuv;5~##*4Y~+y7Z5O*3w4qgI5V^17u*ZeupVGH^nM&$qmAk|anf*>r zWc5CV;-JY-Z@Uq1Irpb^O`L_7AGiqd*YpGUShb==os$uN3yYvb`wm6d=?T*it&pDk zo`vhw)RZX|91^^Wa_ti2zBFyWy4cJu#g)_S6~jT}CC{DJ_kKpT`$oAL%b^!2M;JgT zM3ZNbUB?}kP(*YYvXDIH8^7LUxz5oE%kMhF!rnPqv!GiY0o}NR$OD=ITDo9r%4E>E0Y^R(rS^~XjWyVI6 zMOR5rPXhTp*G*M&X#NTL`Hu*R+u*QNoiOKg4CtNPrjgH>c?Hi4MUG#I917fx**+pJfOo!zFM&*da&G_x)L(`k&TPI*t3e^{crd zX<4I$5nBQ8Ax_lmNRa~E*zS-R0sxkz`|>7q_?*e%7bxqNm3_eRG#1ae3gtV9!fQpY z+!^a38o4ZGy9!J5sylDxZTx$JmG!wg7;>&5H1)>f4dXj;B+@6tMlL=)cLl={jLMxY zbbf1ax3S4>bwB9-$;SN2?+GULu;UA-35;VY*^9Blx)Jwyb$=U!D>HhB&=jSsd^6yw zL)?a|>GxU!W}ocTC(?-%z3!IUhw^uzc`Vz_g>-tv)(XA#JK^)ZnC|l1`@CdX1@|!| z_9gQ)7uOf?cR@KDp97*>6X|;t@Y`k_N@)aH7gY27)COv^P3ya9I{4z~vUjLR9~z1Z z5=G{mVtKH*&$*t0@}-i_v|3B$AHHYale7>E+jP`ClqG%L{u;*ff_h@)al?RuL7tOO z->;I}>%WI{;vbLP3VIQ^iA$4wl6@0sDj|~112Y4OFjMs`13!$JGkp%b&E8QzJw_L5 zOnw9joc0^;O%OpF$Qp)W1HI!$4BaXX84`%@#^dk^hFp^pQ@rx4g(8Xjy#!X%+X5Jd@fs3amGT`}mhq#L97R>OwT5-m|h#yT_-v@(k$q7P*9X~T*3)LTdzP!*B} z+SldbVWrrwQo9wX*%FyK+sRXTa@O?WM^FGWOE?S`R(0P{<6p#f?0NJvnBia?k^fX2 zNQs7K-?EijgHJY}&zsr;qJ<*PCZUd*x|dD=IQPUK_nn)@X4KWtqoJNHkT?ZWL_hF? zS8lp2(q>;RXR|F;1O}EE#}gCrY~#n^O`_I&?&z5~7N;zL0)3Tup`%)oHMK-^r$NT% zbFg|o?b9w(q@)6w5V%si<$!U<#}s#x@0aX-hP>zwS#9*75VXA4K*%gUc>+yzupTDBOKH8WR4V0pM(HrfbQ&eJ79>HdCvE=F z|J>s;;iDLB^3(9}?biKbxf1$lI!*Z%*0&8UUq}wMyPs_hclyQQi4;NUY+x2qy|0J; zhn8;5)4ED1oHwg+VZF|80<4MrL97tGGXc5Sw$wAI#|2*cvQ=jB5+{AjMiDHmhUC*a zlmiZ`LAuAn_}hftXh;`Kq0zblDk8?O-`tnilIh|;3lZp@F_osJUV9`*R29M?7H{Fy z`nfVEIDIWXmU&YW;NjU8)EJpXhxe5t+scf|VXM!^bBlwNh)~7|3?fWwo_~ZFk(22% zTMesYw+LNx3J-_|DM~`v93yXe=jPD{q;li;5PD?Dyk+b? zo21|XpT@)$BM$%F=P9J19Vi&1#{jM3!^Y&fr&_`toi`XB1!n>sbL%U9I5<7!@?t)~ z;&H%z>bAaQ4f$wIzkjH70;<8tpUoxzKrPhn#IQfS%9l5=Iu))^XC<58D!-O z{B+o5R^Z21H0T9JQ5gNJnqh#qH^na|z92=hONIM~@_iuOi|F>jBh-?aA20}Qx~EpDGElELNn~|7WRXRFnw+Wdo`|# zBpU=Cz3z%cUJ0mx_1($X<40XEIYz(`noWeO+x#yb_pwj6)R(__%@_Cf>txOQ74wSJ z0#F3(zWWaR-jMEY$7C*3HJrohc79>MCUu26mfYN)f4M~4gD`}EX4e}A!U}QV8!S47 z6y-U-%+h`1n`*pQuKE%Av0@)+wBZr9mH}@vH@i{v(m-6QK7Ncf17x_D=)32`FOjjo zg|^VPf5c6-!FxN{25dvVh#fog=NNpXz zfB$o+0jbRkHH{!TKhE709f+jI^$3#v1Nmf80w`@7-5$1Iv_`)W^px8P-({xwb;D0y z7LKDAHgX<84?l!I*Dvi2#D@oAE^J|g$3!)x1Ua;_;<@#l1fD}lqU2_tS^6Ht$1Wl} zBESo7o^)9-Tjuz$8YQSGhfs{BQV6zW7dA?0b(Dbt=UnQs&4zHfe_sj{RJ4uS-vQpC zX;Bbsuju4%!o8?&m4UZU@~ZZjeFF6ex2ss5_60_JS_|iNc+R0GIjH1@Z z=rLT9%B|WWgOrR7IiIwr2=T;Ne?30M!@{%Qf8o`!>=s<2CBpCK_TWc(DX51>e^xh8 z&@$^b6CgOd7KXQV&Y4%}_#uN*mbanXq(2=Nj`L7H7*k(6F8s6{FOw@(DzU`4-*77{ zF+dxpv}%mFpYK?>N_2*#Y?oB*qEKB}VoQ@bzm>ptmVS_EC(#}Lxxx730trt0G)#$b zE=wVvtqOct1%*9}U{q<)2?{+0TzZzP0jgf9*)arV)*e!f`|jgT{7_9iS@e)recI#z zbzolURQ+TOzE!ymqvBY7+5NnAbWxvMLsLTwEbFqW=CPyCsmJ}P1^V30|D5E|p3BC5 z)3|qgw@ra7aXb-wsa|l^in~1_fm{7bS9jhVRkYVO#U{qMp z)Wce+|DJ}4<2gp8r0_xfZpMo#{Hl2MfjLcZdRB9(B(A(f;+4s*FxV{1F|4d`*sRNd zp4#@sEY|?^FIJ;tmH{@keZ$P(sLh5IdOk@k^0uB^BWr@pk6mHy$qf&~rI>P*a;h0C{%oA*i!VjWn&D~O#MxN&f@1Po# zKN+ zrGrkSjcr?^R#nGl<#Q722^wbYcgW@{+6CBS<1@%dPA8HC!~a`jTz<`g_l5N1M@9wn9GOAZ>nqNgq!yOCbZ@1z`U_N`Z>}+1HIZxk*5RDc&rd5{3qjRh8QmT$VyS;jK z;AF+r6XnnCp=wQYoG|rT2@8&IvKq*IB_WvS%nt%e{MCFm`&W*#LXc|HrD?nVBo=(8*=Aq?u$sDA_sC_RPDUiQ+wnIJET8vx$&fxkW~kP9qXKt zozR)@xGC!P)CTkjeWvXW5&@2?)qt)jiYWWBU?AUtzAN}{JE1I)dfz~7$;}~BmQF`k zpn11qmObXwRB8&rnEG*#4Xax3XBkKlw(;tb?Np^i+H8m(Wyz9k{~ogba@laiEk;2! zV*QV^6g6(QG%vX5Um#^sT&_e`B1pBW5yVth~xUs#0}nv?~C#l?W+9Lsb_5)!71rirGvY zTIJ$OPOY516Y|_014sNv+Z8cc5t_V=i>lWV=vNu#!58y9Zl&GsMEW#pPYPYGHQ|;vFvd*9eM==$_=vc7xnyz0~ zY}r??$<`wAO?JQk@?RGvkWVJlq2dk9vB(yV^vm{=NVI8dhsX<)O(#nr9YD?I?(VmQ z^r7VfUBn<~p3()8yOBjm$#KWx!5hRW)5Jl7wY@ky9lNM^jaT##8QGVsYeaVywmpv>X|Xj7gWE1Ezai&wVLt3p)k4w~yrskT-!PR!kiyQlaxl(( zXhF%Q9x}1TMt3~u@|#wWm-Vq?ZerK={8@~&@9r5JW}r#45#rWii};t`{5#&3$W)|@ zbAf2yDNe0q}NEUvq_Quq3cTjcw z@H_;$hu&xllCI9CFDLuScEMg|x{S7GdV8<&Mq=ezDnRZAyX-8gv97YTm0bg=d)(>N z+B2FcqvI9>jGtnK%eO%y zoBPkJTk%y`8TLf4)IXPBn`U|9>O~WL2C~C$z~9|0m*YH<-vg2CD^SX#&)B4ngOSG$ zV^wmy_iQk>dfN@Pv(ckfy&#ak@MLC7&Q6Ro#!ezM*VEh`+b3Jt%m(^T&p&WJ2Oqvj zs-4nq0TW6cv~(YI$n0UkfwN}kg3_fp?(ijSV#tR9L0}l2qjc7W?i*q01=St0eZ=4h zyGQbEw`9OEH>NMuIe)hVwYHsGERWOD;JxEiO7cQv%pFCeR+IyhwQ|y@&^24k+|8fD zLiOWFNJ2&vu2&`Jv96_z-Cd5RLgmeY3*4rDOQo?Jm`;I_(+ejsPM03!ly!*Cu}Cco zrQSrEDHNyzT(D5s1rZq!8#?f6@v6dB7a-aWs(Qk>N?UGAo{gytlh$%_IhyL7h?DLXDGx zgxGEBQoCAWo-$LRvM=F5MTle`M})t3vVv;2j0HZY&G z22^iGhV@uaJh(XyyY%} zd4iH_UfdV#T=3n}(Lj^|n;O4|$;xhu*8T3hR1mc_A}fK}jfZ7LX~*n5+`8N2q#rI$ z@<_2VANlYF$vIH$ zl<)+*tIWW78IIINA7Rr7i{<;#^yzxoLNkXL)eSs=%|P>$YQIh+ea_3k z_s7r4%j7%&*NHSl?R4k%1>Z=M9o#zxY!n8sL5>BO-ZP;T3Gut>iLS@U%IBrX6BA3k z)&@q}V8a{X<5B}K5s(c(LQ=%v1ocr`t$EqqY0EqVjr65usa=0bkf|O#ky{j3)WBR(((L^wmyHRzoWuL2~WTC=`yZ zn%VX`L=|Ok0v7?s>IHg?yArBcync5rG#^+u)>a%qjES%dRZoIyA8gQ;StH z1Ao7{<&}6U=5}4v<)1T7t!J_CL%U}CKNs-0xWoTTeqj{5{?Be$L0_tk>M9o8 zo371}S#30rKZFM{`H_(L`EM9DGp+Mifk&IP|C2Zu_)Ghr4Qtpmkm1osCf@%Z$%t+7 zYH$Cr)Ro@3-QDeQJ8m+x6%;?YYT;k6Z0E-?kr>x33`H%*ueBD7Zx~3&HtWn0?2Wt} zTG}*|v?{$ajzt}xPzV%lL1t-URi8*Zn)YljXNGDb>;!905Td|mpa@mHjIH%VIiGx- zd@MqhpYFu4_?y5N4xiHn3vX&|e6r~Xt> zZG`aGq|yTNjv;9E+Txuoa@A(9V7g?1_T5FzRI;!=NP1Kqou1z5?%X~Wwb{trRfd>i z8&y^H)8YnKyA_Fyx>}RNmQIczT?w2J4SNvI{5J&}Wto|8FR(W;Qw#b1G<1%#tmYzQ zQ2mZA-PAdi%RQOhkHy9Ea#TPSw?WxwL@H@cbkZwIq0B!@ns}niALidmn&W?!Vd4Gj zO7FiuV4*6Mr^2xlFSvM;Cp_#r8UaqIzHJQg_z^rEJw&OMm_8NGAY2)rKvki|o1bH~ z$2IbfVeY2L(^*rMRU1lM5Y_sgrDS`Z??nR2lX;zyR=c%UyGb*%TC-Dil?SihkjrQy~TMv6;BMs7P8il`H7DmpVm@rJ;b)hW)BL)GjS154b*xq-NXq2cwE z^;VP7ua2pxvCmxrnqUYQMH%a%nHmwmI33nJM(>4LznvY*k&C0{8f*%?zggpDgkuz&JBx{9mfb@wegEl2v!=}Sq2Gaty0<)UrOT0{MZtZ~j5y&w zXlYa_jY)I_+VA-^#mEox#+G>UgvM!Ac8zI<%JRXM_73Q!#i3O|)lOP*qBeJG#BST0 zqohi)O!|$|2SeJQo(w6w7%*92S})XfnhrH_Z8qe!G5>CglP=nI7JAOW?(Z29;pXJ9 zR9`KzQ=WEhy*)WH>$;7Cdz|>*i>=##0bB)oU0OR>>N<21e4rMCHDemNi2LD>Nc$;& zQRFthpWniC1J6@Zh~iJCoLOxN`oCKD5Q4r%ynwgUKPlIEd#?QViIqovY|czyK8>6B zSP%{2-<;%;1`#0mG^B(8KbtXF;Nf>K#Di72UWE4gQ%(_26Koiad)q$xRL~?pN71ZZ zujaaCx~jXjygw;rI!WB=xrOJO6HJ!!w}7eiivtCg5K|F6$EXa)=xUC za^JXSX98W`7g-tm@uo|BKj39Dl;sg5ta;4qjo^pCh~{-HdLl6qI9Ix6f$+qiZ$}s= zNguKrU;u+T@ko(Vr1>)Q%h$?UKXCY>3se%&;h2osl2D zE4A9bd7_|^njDd)6cI*FupHpE3){4NQ*$k*cOWZ_?CZ>Z4_fl@n(mMnYK62Q1d@+I zr&O))G4hMihgBqRIAJkLdk(p(D~X{-oBUA+If@B}j& zsHbeJ3RzTq96lB7d($h$xTeZ^gP0c{t!Y0c)aQE;$FY2!mACg!GDEMKXFOPI^)nHZ z`aSPJpvV0|bbrzhWWkuPURlDeN%VT8tndV8?d)eN*i4I@u zVKl^6{?}A?P)Fsy?3oi#clf}L18t;TjNI2>eI&(ezDK7RyqFxcv%>?oxUlonv(px) z$vnPzRH`y5A(x!yOIfL0bmgeMQB$H5wenx~!ujQK*nUBW;@Em&6Xv2%s(~H5WcU2R z;%Nw<$tI)a`Ve!>x+qegJnQsN2N7HaKzrFqM>`6R*gvh%O*-%THt zrB$Nk;lE;z{s{r^PPm5qz(&lM{sO*g+W{sK+m3M_z=4=&CC>T`{X}1Vg2PEfSj2x_ zmT*(x;ov%3F?qoEeeM>dUn$a*?SIGyO8m806J1W1o+4HRhc2`9$s6hM#qAm zChQ87b~GEw{ADfs+5}FJ8+|bIlIv(jT$Ap#hSHoXdd9#w<#cA<1Rkq^*EEkknUd4& zoIWIY)sAswy6fSERVm&!SO~#iN$OgOX*{9@_BWFyJTvC%S++ilSfCrO(?u=Dc?CXZ zzCG&0yVR{Z`|ZF0eEApWEo#s9osV>F{uK{QA@BES#&;#KsScf>y zvs?vIbI>VrT<*!;XmQS=bhq%46-aambZ(8KU-wOO2=en~D}MCToB_u;Yz{)1ySrPZ z@=$}EvjTdzTWU7c0ZI6L8=yP+YRD_eMMos}b5vY^S*~VZysrkq<`cK3>>v%uy7jgq z0ilW9KjVDHLv0b<1K_`1IkbTOINs0=m-22c%M~l=^S}%hbli-3?BnNq?b`hx^HX2J zIe6ECljRL0uBWb`%{EA=%!i^4sMcj+U_TaTZRb+~GOk z^ZW!nky0n*Wb*r+Q|9H@ml@Z5gU&W`(z4-j!OzC1wOke`TRAYGZVl$PmQ16{3196( zO*?`--I}Qf(2HIwb2&1FB^!faPA2=sLg(@6P4mN)>Dc3i(B0;@O-y2;lM4akD>@^v z=u>*|!s&9zem70g7zfw9FXl1bpJW(C#5w#uy5!V?Q(U35A~$dR%LDVnq@}kQm13{} zd53q3N(s$Eu{R}k2esbftfjfOITCL;jWa$}(mmm}d(&7JZ6d3%IABCapFFYjdEjdK z&4Edqf$G^MNAtL=uCDRs&Fu@FXRgX{*0<(@c3|PNHa>L%zvxWS={L8%qw`STm+=Rd zA}FLspESSIpE_^41~#5yI2bJ=9`oc;GIL!JuW&7YetZ?0H}$$%8rW@*J37L-~Rsx!)8($nI4 zZhcZ2^=Y+p4YPl%j!nFJA|*M^gc(0o$i3nlphe+~-_m}jVkRN{spFs(o0ajW@f3K{ zDV!#BwL322CET$}Y}^0ixYj2w>&Xh12|R8&yEw|wLDvF!lZ#dOTHM9pK6@Nm-@9Lnng4ZHBgBSrr7KI8YCC9DX5Kg|`HsiwJHg2(7#nS;A{b3tVO?Z% za{m5b3rFV6EpX;=;n#wltDv1LE*|g5pQ+OY&*6qCJZc5oDS6Z6JD#6F)bWxZSF@q% z+1WV;m!lRB!n^PC>RgQCI#D1br_o^#iPk>;K2hB~0^<~)?p}LG%kigm@moD#q3PE+ zA^Qca)(xnqw6x>XFhV6ku9r$E>bWNrVH9fum0?4s?Rn2LG{Vm_+QJHse6xa%nzQ?k zKug4PW~#Gtb;#5+9!QBgyB@q=sk9=$S{4T>wjFICStOM?__fr+Kei1 z3j~xPqW;W@YkiUM;HngG!;>@AITg}vAE`M2Pj9Irl4w1fo4w<|Bu!%rh%a(Ai^Zhi zs92>v5;@Y(Zi#RI*ua*h`d_7;byQSa*v9E{2x$<-_=5Z<7{%)}4XExANcz@rK69T0x3%H<@frW>RA8^swA+^a(FxK| zFl3LD*ImHN=XDUkrRhp6RY5$rQ{bRgSO*(vEHYV)3Mo6Jy3puiLmU&g82p{qr0F?ohmbz)f2r{X2|T2 z$4fdQ=>0BeKbiVM!e-lIIs8wVTuC_m7}y4A_%ikI;Wm5$9j(^Y z(cD%U%k)X>_>9~t8;pGzL6L-fmQO@K; zo&vQzMlgY95;1BSkngY)e{`n0!NfVgf}2mB3t}D9@*N;FQ{HZ3Pb%BK6;5#-O|WI( zb6h@qTLU~AbVW#_6?c!?Dj65Now7*pU{h!1+eCV^KCuPAGs28~3k@ueL5+u|Z-7}t z9|lskE`4B7W8wMs@xJa{#bsCGDFoRSNSnmNYB&U7 zVGKWe%+kFB6kb)e;TyHfqtU6~fRg)f|>=5(N36)0+C z`hv65J<$B}WUc!wFAb^QtY31yNleq4dzmG`1wHTj=c*=hay9iD071Hc?oYoUk|M*_ zU1GihAMBsM@5rUJ(qS?9ZYJ6@{bNqJ`2Mr+5#hKf?doa?F|+^IR!8lq9)wS3tF_9n zW_?hm)G(M+MYb?V9YoX^_mu5h-LP^TL^!Q9Z7|@sO(rg_4+@=PdI)WL(B7`!K^ND- z-uIuVDCVEdH_C@c71YGYT^_Scf_dhB8Z2Xy6vGtBSlYud9vggOqv^L~F{BraSE_t} zIkP+Hp2&nH^-MNEs}^`oMLy11`PQW$T|K(`Bu*(f@)mv1-qY(_YG&J2M2<7k;;RK~ zL{Fqj9yCz8(S{}@c)S!65aF<=&eLI{hAMErCx&>i7OeDN>okvegO87OaG{Jmi<|}D zaT@b|0X{d@OIJ7zvT>r+eTzgLq~|Dpu)Z&db-P4z*`M$UL51lf>FLlq6rfG)%doyp z)3kk_YIM!03eQ8Vu_2fg{+osaEJPtJ-s36R+5_AEG12`NG)IQ#TF9c@$99%0iye+ zUzZ57=m2)$D(5Nx!n)=5Au&O0BBgwxIBaeI(mro$#&UGCr<;C{UjJVAbVi%|+WP(a zL$U@TYCxJ=1{Z~}rnW;7UVb7+ZnzgmrogDxhjLGo>c~MiJAWs&&;AGg@%U?Y^0JhL ze(x6Z74JG6FlOFK(T}SXQfhr}RIFl@QXKnIcXYF)5|V~e-}suHILKT-k|<*~Ij|VF zC;t@=uj=hot~*!C68G8hTA%8SzOfETOXQ|3FSaIEjvBJp(A)7SWUi5!Eu#yWgY+;n zlm<$+UDou*V+246_o#V4kMdto8hF%%Lki#zPh}KYXmMf?hrN0;>Mv%`@{0Qn`Ujp) z=lZe+13>^Q!9zT);H<(#bIeRWz%#*}sgUX9P|9($kexOyKIOc`dLux}c$7It4u|Rl z6SSkY*V~g_B-hMPo_ak>>z@AVQ(_N)VY2kB3IZ0G(iDUYw+2d7W^~(Jq}KY=JnWS( z#rzEa&0uNhJ>QE8iiyz;n2H|SV#Og+wEZv=f2%1ELX!SX-(d3tEj$5$1}70Mp<&eI zCkfbByL7af=qQE@5vDVxx1}FSGt_a1DoE3SDI+G)mBAna)KBG4p8Epxl9QZ4BfdAN zFnF|Y(umr;gRgG6NLQ$?ZWgllEeeq~z^ZS7L?<(~O&$5|y)Al^iMKy}&W+eMm1W z7EMU)u^ke(A1#XCV>CZ71}P}0x)4wtHO8#JRG3MA-6g=`ZM!FcICCZ{IEw8Dm2&LQ z1|r)BUG^0GzI6f946RrBlfB1Vs)~8toZf~7)+G;pv&XiUO(%5bm)pl=p>nV^o*;&T z;}@oZSibzto$arQgfkp|z4Z($P>dTXE{4O=vY0!)kDO* zGF8a4wq#VaFpLfK!iELy@?-SeRrdz%F*}hjKcA*y@mj~VD3!it9lhRhX}5YOaR9$} z3mS%$2Be7{l(+MVx3 z(4?h;P!jnRmX9J9sYN#7i=iyj_5q7n#X(!cdqI2lnr8T$IfOW<_v`eB!d9xY1P=2q&WtOXY=D9QYteP)De?S4}FK6#6Ma z=E*V+#s8>L;8aVroK^6iKo=MH{4yEZ_>N-N z`(|;aOATba1^asjxlILk<4}f~`39dBFlxj>Dw(hMYKPO3EEt1@S`1lxFNM+J@uB7T zZ8WKjz7HF1-5&2=l=fqF-*@>n5J}jIxdDwpT?oKM3s8Nr`x8JnN-kCE?~aM1H!hAE z%%w(3kHfGwMnMmNj(SU(w42OrC-euI>Dsjk&jz3ts}WHqmMpzQ3vZrsXrZ|}+MHA7 z068obeXZTsO*6RS@o3x80E4ok``rV^Y3hr&C1;|ZZ0|*EKO`$lECUYG2gVFtUTw)R z4Um<0ZzlON`zTdvVdL#KFoMFQX*a5wM0Czp%wTtfK4Sjs)P**RW&?lP$(<}q%r68Z zS53Y!d@&~ne9O)A^tNrXHhXBkj~$8j%pT1%%mypa9AW5E&s9)rjF4@O3ytH{0z6riz|@< zB~UPh*wRFg2^7EbQrHf0y?E~dHlkOxof_a?M{LqQ^C!i2dawHTPYUE=X@2(3<=OOxs8qn_(y>pU>u^}3y&df{JarR0@VJn0f+U%UiF=$Wyq zQvnVHESil@d|8&R<%}uidGh7@u^(%?$#|&J$pvFC-n8&A>utA=n3#)yMkz+qnG3wd zP7xCnF|$9Dif@N~L)Vde3hW8W!UY0BgT2v(wzp;tlLmyk2%N|0jfG$%<;A&IVrOI< z!L)o>j>;dFaqA3pL}b-Je(bB@VJ4%!JeX@3x!i{yIeIso^=n?fDX`3bU=eG7sTc%g%ye8$v8P@yKE^XD=NYxTb zbf!Mk=h|otpqjFaA-vs5YOF-*GwWPc7VbaOW&stlANnCN8iftFMMrUdYNJ_Bnn5Vt zxfz@Ah|+4&P;reZxp;MmEI7C|FOv8NKUm8njF7Wb6Gi7DeODLl&G~}G4be&*Hi0Qw z5}77vL0P+7-B%UL@3n1&JPxW^d@vVwp?u#gVcJqY9#@-3X{ok#UfW3<1fb%FT`|)V~ggq z(3AUoUS-;7)^hCjdT0Kf{i}h)mBg4qhtHHBti=~h^n^OTH5U*XMgDLIR@sre`AaB$ zg)IGBET_4??m@cx&c~bA80O7B8CHR7(LX7%HThkeC*@vi{-pL%e)yXp!B2InafbDF zjPXf1mko3h59{lT6EEbxKO1Z5GF71)WwowO6kY|6tjSVSWdQ}NsK2x{>i|MKZK8%Q zfu&_0D;CO-Jg0#YmyfctyJ!mRJp)e#@O0mYdp|8x;G1%OZQ3Q847YWTyy|%^cpA;m zze0(5p{tMu^lDkpe?HynyO?a1$_LJl2L&mpeKu%8YvgRNr=%2z${%WThHG=vrWY@4 zsA`OP#O&)TetZ>s%h!=+CE15lOOls&nvC~$Qz0Ph7tHiP;O$i|eDwpT{cp>+)0-|; zY$|bB+Gbel>5aRN3>c0x)4U=|X+z+{ zn*_p*EQoquRL+=+p;=lm`d71&1NqBz&_ph)MXu(Nv6&XE7(RsS)^MGj5Q?Fwude-(sq zjJ>aOq!7!EN>@(fK7EE#;i_BGvli`5U;r!YA{JRodLBc6-`n8K+Fjgwb%sX;j=qHQ z7&Tr!)!{HXoO<2BQrV9Sw?JRaLXV8HrsNevvnf>Y-6|{T!pYLl7jp$-nEE z#X!4G4L#K0qG_4Z;Cj6=;b|Be$hi4JvMH!-voxqx^@8cXp`B??eFBz2lLD8RRaRGh zn7kUfy!YV~p(R|p7iC1Rdgt$_24i0cd-S8HpG|`@my70g^y`gu%#Tf_L21-k?sRRZHK&at(*ED0P8iw{7?R$9~OF$Ko;Iu5)ur5<->x!m93Eb zFYpIx60s=Wxxw=`$aS-O&dCO_9?b1yKiPCQmSQb>T)963`*U+Ydj5kI(B(B?HNP8r z*bfSBpSu)w(Z3j7HQoRjUG(+d=IaE~tv}y14zHHs|0UcN52fT8V_<@2ep_ee{QgZG zmgp8iv4V{k;~8@I%M3<#B;2R>Ef(Gg_cQM7%}0s*^)SK6!Ym+~P^58*wnwV1BW@eG z4sZLqsUvBbFsr#8u7S1r4teQ;t)Y@jnn_m5jS$CsW1um!p&PqAcc8!zyiXHVta9QC zY~wCwCF0U%xiQPD_INKtTb;A|Zf29(mu9NI;E zc-e>*1%(LSXB`g}kd`#}O;veb<(sk~RWL|f3ljxCnEZDdNSTDV6#Td({6l&y4IjKF z^}lIUq*ZUqgTPumD)RrCN{M^jhY>E~1pn|KOZ5((%F)G|*ZQ|r4zIbrEiV%42hJV8 z3xS)=!X1+=olbdGJ=yZil?oXLct8FM{(6ikLL3E%=q#O6(H$p~gQu6T8N!plf!96| z&Q3=`L~>U0zZh;z(pGR2^S^{#PrPxTRHD1RQOON&f)Siaf`GLj#UOk&(|@0?zm;Sx ztsGt8=29-MZs5CSf1l1jNFtNt5rFNZxJPvkNu~2}7*9468TWm>nN9TP&^!;J{-h)_ z7WsHH9|F%I`Pb!>KAS3jQWKfGivTVkMJLO-HUGM_a4UQ_%RgL6WZvrW+Z4ujZn;y@ zz9$=oO!7qVTaQAA^BhX&ZxS*|5dj803M=k&2%QrXda`-Q#IoZL6E(g+tN!6CA!CP* zCpWtCujIea)ENl0liwVfj)Nc<9mV%+e@=d`haoZ*`B7+PNjEbXBkv=B+Pi^~L#EO$D$ZqTiD8f<5$eyb54-(=3 zh)6i8i|jp(@OnRrY5B8t|LFXFQVQ895n*P16cEKTrT*~yLH6Z4e*bZ5otpRDri&+A zfNbK1D5@O=sm`fN=WzWyse!za5n%^+6dHPGX#8DyIK>?9qyX}2XvBWVqbP%%D)7$= z=#$WulZlZR<{m#gU7lwqK4WS1Ne$#_P{b17qe$~UOXCl>5b|6WVh;5vVnR<%d+Lnp z$uEmML38}U4vaW8>shm6CzB(Wei3s#NAWE3)a2)z@i{4jTn;;aQS)O@l{rUM`J@K& l00vQ5JBs~;vo!vr%%-k{2_Fq1Mn4QF81S)AQ99zk{{c4yR+0b! literal 63721 zcmb5Wb9gP!wgnp7wrv|bwr$&XvSZt}Z6`anZSUAlc9NHKf9JdJ;NJVr`=eI(_pMp0 zy1VAAG3FfAOI`{X1O)&90s;U4K;XLp008~hCjbEC_fbYfS%6kTR+JtXK>nW$ZR+`W ze|#J8f4A@M|F5BpfUJb5h>|j$jOe}0oE!`Zf6fM>CR?!y@zU(cL8NsKk`a z6tx5mAkdjD;J=LcJ;;Aw8p!v#ouk>mUDZF@ zK>yvw%+bKu+T{Nk@LZ;zkYy0HBKw06_IWcMHo*0HKpTsEFZhn5qCHH9j z)|XpN&{`!0a>Vl+PmdQc)Yg4A(AG-z!+@Q#eHr&g<9D?7E)_aEB?s_rx>UE9TUq|? z;(ggJt>9l?C|zoO@5)tu?EV0x_7T17q4fF-q3{yZ^ipUbKcRZ4Qftd!xO(#UGhb2y>?*@{xq%`(-`2T^vc=#< zx!+@4pRdk&*1ht2OWk^Z5IAQ0YTAXLkL{(D*$gENaD)7A%^XXrCchN&z2x+*>o2FwPFjWpeaL=!tzv#JOW#( z$B)Nel<+$bkH1KZv3&-}=SiG~w2sbDbAWarg%5>YbC|}*d9hBjBkR(@tyM0T)FO$# zPtRXukGPnOd)~z=?avu+4Co@wF}1T)-uh5jI<1$HLtyDrVak{gw`mcH@Q-@wg{v^c zRzu}hMKFHV<8w}o*yg6p@Sq%=gkd~;`_VGTS?L@yVu`xuGy+dH6YOwcP6ZE`_0rK% zAx5!FjDuss`FQ3eF|mhrWkjux(Pny^k$u_)dyCSEbAsecHsq#8B3n3kDU(zW5yE|( zgc>sFQywFj5}U*qtF9Y(bi*;>B7WJykcAXF86@)z|0-Vm@jt!EPoLA6>r)?@DIobIZ5Sx zsc@OC{b|3%vaMbyeM|O^UxEYlEMHK4r)V-{r)_yz`w1*xV0|lh-LQOP`OP`Pk1aW( z8DSlGN>Ts|n*xj+%If~+E_BxK)~5T#w6Q1WEKt{!Xtbd`J;`2a>8boRo;7u2M&iOop4qcy<)z023=oghSFV zST;?S;ye+dRQe>ygiJ6HCv4;~3DHtJ({fWeE~$H@mKn@Oh6Z(_sO>01JwH5oA4nvK zr5Sr^g+LC zLt(i&ecdmqsIJGNOSUyUpglvhhrY8lGkzO=0USEKNL%8zHshS>Qziu|`eyWP^5xL4 zRP122_dCJl>hZc~?58w~>`P_s18VoU|7(|Eit0-lZRgLTZKNq5{k zE?V=`7=R&ro(X%LTS*f+#H-mGo_j3dm@F_krAYegDLk6UV{`UKE;{YSsn$ z(yz{v1@p|p!0>g04!eRSrSVb>MQYPr8_MA|MpoGzqyd*$@4j|)cD_%^Hrd>SorF>@ zBX+V<@vEB5PRLGR(uP9&U&5=(HVc?6B58NJT_igiAH*q~Wb`dDZpJSKfy5#Aag4IX zj~uv74EQ_Q_1qaXWI!7Vf@ZrdUhZFE;L&P_Xr8l@GMkhc#=plV0+g(ki>+7fO%?Jb zl+bTy7q{w^pTb{>(Xf2q1BVdq?#f=!geqssXp z4pMu*q;iiHmA*IjOj4`4S&|8@gSw*^{|PT}Aw~}ZXU`6=vZB=GGeMm}V6W46|pU&58~P+?LUs%n@J}CSrICkeng6YJ^M? zS(W?K4nOtoBe4tvBXs@@`i?4G$S2W&;$z8VBSM;Mn9 zxcaEiQ9=vS|bIJ>*tf9AH~m&U%2+Dim<)E=}KORp+cZ^!@wI`h1NVBXu{@%hB2Cq(dXx_aQ9x3mr*fwL5!ZryQqi|KFJuzvP zK1)nrKZ7U+B{1ZmJub?4)Ln^J6k!i0t~VO#=q1{?T)%OV?MN}k5M{}vjyZu#M0_*u z8jwZKJ#Df~1jcLXZL7bnCEhB6IzQZ-GcoQJ!16I*39iazoVGugcKA{lhiHg4Ta2fD zk1Utyc5%QzZ$s3;p0N+N8VX{sd!~l*Ta3|t>lhI&G`sr6L~G5Lul`>m z{!^INm?J|&7X=;{XveF!(b*=?9NAp4y&r&N3(GKcW4rS(Ejk|Lzs1PrxPI_owB-`H zg3(Rruh^&)`TKA6+_!n>RdI6pw>Vt1_j&+bKIaMTYLiqhZ#y_=J8`TK{Jd<7l9&sY z^^`hmi7^14s16B6)1O;vJWOF$=$B5ONW;;2&|pUvJlmeUS&F;DbSHCrEb0QBDR|my zIs+pE0Y^`qJTyH-_mP=)Y+u^LHcuZhsM3+P||?+W#V!_6E-8boP#R-*na4!o-Q1 zVthtYhK{mDhF(&7Okzo9dTi03X(AE{8cH$JIg%MEQca`S zy@8{Fjft~~BdzWC(di#X{ny;!yYGK9b@=b|zcKZ{vv4D8i+`ilOPl;PJl{!&5-0!w z^fOl#|}vVg%=n)@_e1BrP)`A zKPgs`O0EO}Y2KWLuo`iGaKu1k#YR6BMySxQf2V++Wo{6EHmK>A~Q5o73yM z-RbxC7Qdh0Cz!nG+7BRZE>~FLI-?&W_rJUl-8FDIaXoNBL)@1hwKa^wOr1($*5h~T zF;%f^%<$p8Y_yu(JEg=c_O!aZ#)Gjh$n(hfJAp$C2he555W5zdrBqjFmo|VY+el;o z=*D_w|GXG|p0**hQ7~9-n|y5k%B}TAF0iarDM!q-jYbR^us(>&y;n^2l0C%@2B}KM zyeRT9)oMt97Agvc4sEKUEy%MpXr2vz*lb zh*L}}iG>-pqDRw7ud{=FvTD?}xjD)w{`KzjNom-$jS^;iw0+7nXSnt1R@G|VqoRhE%12nm+PH?9`(4rM0kfrZzIK9JU=^$YNyLvAIoxl#Q)xxDz!^0@zZ zSCs$nfcxK_vRYM34O<1}QHZ|hp4`ioX3x8(UV(FU$J@o%tw3t4k1QPmlEpZa2IujG&(roX_q*%e`Hq|);0;@k z0z=fZiFckp#JzW0p+2A+D$PC~IsakhJJkG(c;CqAgFfU0Z`u$PzG~-9I1oPHrCw&)@s^Dc~^)#HPW0Ra}J^=|h7Fs*<8|b13ZzG6MP*Q1dkoZ6&A^!}|hbjM{2HpqlSXv_UUg1U4gn z3Q)2VjU^ti1myodv+tjhSZp%D978m~p& z43uZUrraHs80Mq&vcetqfQpQP?m!CFj)44t8Z}k`E798wxg&~aCm+DBoI+nKq}&j^ zlPY3W$)K;KtEajks1`G?-@me7C>{PiiBu+41#yU_c(dITaqE?IQ(DBu+c^Ux!>pCj zLC|HJGU*v+!it1(;3e`6igkH(VA)-S+k(*yqxMgUah3$@C zz`7hEM47xr>j8^g`%*f=6S5n>z%Bt_Fg{Tvmr+MIsCx=0gsu_sF`q2hlkEmisz#Fy zj_0;zUWr;Gz}$BS%Y`meb(=$d%@Crs(OoJ|}m#<7=-A~PQbyN$x%2iXP2@e*nO0b7AwfH8cCUa*Wfu@b)D_>I*%uE4O3 z(lfnB`-Xf*LfC)E}e?%X2kK7DItK6Tf<+M^mX0Ijf_!IP>7c8IZX%8_#0060P{QMuV^B9i<^E`_Qf0pv9(P%_s8D`qvDE9LK9u-jB}J2S`(mCO&XHTS04Z5Ez*vl^T%!^$~EH8M-UdwhegL>3IQ*)(MtuH2Xt1p!fS4o~*rR?WLxlA!sjc2(O znjJn~wQ!Fp9s2e^IWP1C<4%sFF}T4omr}7+4asciyo3DntTgWIzhQpQirM$9{EbQd z3jz9vS@{aOqTQHI|l#aUV@2Q^Wko4T0T04Me4!2nsdrA8QY1%fnAYb~d2GDz@lAtfcHq(P7 zaMBAGo}+NcE-K*@9y;Vt3*(aCaMKXBB*BJcD_Qnxpt75r?GeAQ}*|>pYJE=uZb73 zC>sv)18)q#EGrTG6io*}JLuB_jP3AU1Uiu$D7r|2_zlIGb9 zjhst#ni)Y`$)!fc#reM*$~iaYoz~_Cy7J3ZTiPm)E?%`fbk`3Tu-F#`{i!l5pNEn5 zO-Tw-=TojYhzT{J=?SZj=Z8#|eoF>434b-DXiUsignxXNaR3 zm_}4iWU$gt2Mw5NvZ5(VpF`?X*f2UZDs1TEa1oZCif?Jdgr{>O~7}-$|BZ7I(IKW`{f;@|IZFX*R8&iT= zoWstN8&R;}@2Ka%d3vrLtR|O??ben;k8QbS-WB0VgiCz;<$pBmIZdN!aalyCSEm)crpS9dcD^Y@XT1a3+zpi-`D}e#HV<} z$Y(G&o~PvL-xSVD5D?JqF3?B9rxGWeb=oEGJ3vRp5xfBPlngh1O$yI95EL+T8{GC@ z98i1H9KhZGFl|;`)_=QpM6H?eDPpw~^(aFQWwyXZ8_EEE4#@QeT_URray*mEOGsGc z6|sdXtq!hVZo=d#+9^@lm&L5|q&-GDCyUx#YQiccq;spOBe3V+VKdjJA=IL=Zn%P} zNk=_8u}VhzFf{UYZV0`lUwcD&)9AFx0@Fc6LD9A6Rd1=ga>Mi0)_QxM2ddCVRmZ0d z+J=uXc(?5JLX3=)e)Jm$HS2yF`44IKhwRnm2*669_J=2LlwuF5$1tAo@ROSU@-y+;Foy2IEl2^V1N;fk~YR z?&EP8#t&m0B=?aJeuz~lHjAzRBX>&x=A;gIvb>MD{XEV zV%l-+9N-)i;YH%nKP?>f`=?#`>B(`*t`aiPLoQM(a6(qs4p5KFjDBN?8JGrf3z8>= zi7sD)c)Nm~x{e<^jy4nTx${P~cwz_*a>%0_;ULou3kHCAD7EYkw@l$8TN#LO9jC( z1BeFW`k+bu5e8Ns^a8dPcjEVHM;r6UX+cN=Uy7HU)j-myRU0wHd$A1fNI~`4;I~`zC)3ul#8#^rXVSO*m}Ag>c%_;nj=Nv$rCZ z*~L@C@OZg%Q^m)lc-kcX&a*a5`y&DaRxh6O*dfhLfF+fU5wKs(1v*!TkZidw*)YBP za@r`3+^IHRFeO%!ai%rxy;R;;V^Fr=OJlpBX;(b*3+SIw}7= zIq$*Thr(Zft-RlY)D3e8V;BmD&HOfX+E$H#Y@B3?UL5L~_fA-@*IB-!gItK7PIgG9 zgWuGZK_nuZjHVT_Fv(XxtU%)58;W39vzTI2n&)&4Dmq7&JX6G>XFaAR{7_3QB6zsT z?$L8c*WdN~nZGiscY%5KljQARN;`w$gho=p006z;n(qIQ*Zu<``TMO3n0{ARL@gYh zoRwS*|Niw~cR!?hE{m*y@F`1)vx-JRfqET=dJ5_(076st(=lFfjtKHoYg`k3oNmo_ zNbQEw8&sO5jAYmkD|Zaz_yUb0rC})U!rCHOl}JhbYIDLzLvrZVw0~JO`d*6f;X&?V=#T@ND*cv^I;`sFeq4 z##H5;gpZTb^0Hz@3C*~u0AqqNZ-r%rN3KD~%Gw`0XsIq$(^MEb<~H(2*5G^<2(*aI z%7}WB+TRlMIrEK#s0 z93xn*Ohb=kWFc)BNHG4I(~RPn-R8#0lqyBBz5OM6o5|>x9LK@%HaM}}Y5goCQRt2C z{j*2TtT4ne!Z}vh89mjwiSXG=%DURar~=kGNNaO_+Nkb+tRi~Rkf!7a$*QlavziD( z83s4GmQ^Wf*0Bd04f#0HX@ua_d8 z23~z*53ePD6@xwZ(vdl0DLc=>cPIOPOdca&MyR^jhhKrdQO?_jJh`xV3GKz&2lvP8 zEOwW6L*ufvK;TN{=S&R@pzV^U=QNk^Ec}5H z+2~JvEVA{`uMAr)?Kf|aW>33`)UL@bnfIUQc~L;TsTQ6>r-<^rB8uoNOJ>HWgqMI8 zSW}pZmp_;z_2O5_RD|fGyTxaxk53Hg_3Khc<8AUzV|ZeK{fp|Ne933=1&_^Dbv5^u zB9n=*)k*tjHDRJ@$bp9mrh}qFn*s}npMl5BMDC%Hs0M0g-hW~P*3CNG06G!MOPEQ_ zi}Qs-6M8aMt;sL$vlmVBR^+Ry<64jrm1EI1%#j?c?4b*7>)a{aDw#TfTYKq+SjEFA z(aJ&z_0?0JB83D-i3Vh+o|XV4UP+YJ$9Boid2^M2en@APw&wx7vU~t$r2V`F|7Qfo z>WKgI@eNBZ-+Og<{u2ZiG%>YvH2L3fNpV9J;WLJoBZda)01Rn;o@){01{7E#ke(7U zHK>S#qZ(N=aoae*4X!0A{)nu0R_sKpi1{)u>GVjC+b5Jyl6#AoQ-1_3UDovNSo`T> z?c-@7XX*2GMy?k?{g)7?Sv;SJkmxYPJPs!&QqB12ejq`Lee^-cDveVWL^CTUldb(G zjDGe(O4P=S{4fF=#~oAu>LG>wrU^z_?3yt24FOx>}{^lCGh8?vtvY$^hbZ)9I0E3r3NOlb9I?F-Yc=r$*~l`4N^xzlV~N zl~#oc>U)Yjl0BxV>O*Kr@lKT{Z09OXt2GlvE38nfs+DD7exl|&vT;)>VFXJVZp9Np zDK}aO;R3~ag$X*|hRVY3OPax|PG`@_ESc8E!mHRByJbZQRS38V2F__7MW~sgh!a>98Q2%lUNFO=^xU52|?D=IK#QjwBky-C>zOWlsiiM&1n z;!&1((Xn1$9K}xabq~222gYvx3hnZPg}VMF_GV~5ocE=-v>V=T&RsLBo&`)DOyIj* zLV{h)JU_y*7SdRtDajP_Y+rBkNN*1_TXiKwHH2&p51d(#zv~s#HwbNy?<+(=9WBvo zw2hkk2Dj%kTFhY+$T+W-b7@qD!bkfN#Z2ng@Pd=i3-i?xYfs5Z*1hO?kd7Sp^9`;Y zM2jeGg<-nJD1er@Pc_cSY7wo5dzQX44=%6rn}P_SRbpzsA{6B+!$3B0#;}qwO37G^ zL(V_5JK`XT?OHVk|{_$vQ|oNEpab*BO4F zUTNQ7RUhnRsU`TK#~`)$icsvKh~(pl=3p6m98@k3P#~upd=k*u20SNcb{l^1rUa)>qO997)pYRWMncC8A&&MHlbW?7i^7M`+B$hH~Y|J zd>FYOGQ;j>Zc2e7R{KK7)0>>nn_jYJy&o@sK!4G>-rLKM8Hv)f;hi1D2fAc$+six2 zyVZ@wZ6x|fJ!4KrpCJY=!Mq0;)X)OoS~{Lkh6u8J`eK%u0WtKh6B>GW_)PVc zl}-k`p09qwGtZ@VbYJC!>29V?Dr>>vk?)o(x?!z*9DJ||9qG-&G~#kXxbw{KKYy}J zQKa-dPt~M~E}V?PhW0R26xdA%1T*%ra6SguGu50YHngOTIv)@N|YttEXo#OZfgtP7;H?EeZZxo<}3YlYxtBq znJ!WFR^tmGf0Py}N?kZ(#=VtpC@%xJkDmfcCoBTxq zr_|5gP?u1@vJZbxPZ|G0AW4=tpb84gM2DpJU||(b8kMOV1S3|(yuwZJ&rIiFW(U;5 zUtAW`O6F6Zy+eZ1EDuP~AAHlSY-+A_eI5Gx)%*uro5tljy}kCZU*_d7)oJ>oQSZ3* zneTn`{gnNC&uJd)0aMBzAg021?YJ~b(fmkwZAd696a=0NzBAqBN54KuNDwa*no(^O z6p05bioXUR^uXjpTol*ppHp%1v9e)vkoUAUJyBx3lw0UO39b0?^{}yb!$yca(@DUn zCquRF?t=Zb9`Ed3AI6|L{eX~ijVH`VzSMheKoP7LSSf4g>md>`yi!TkoG5P>Ofp+n z(v~rW+(5L96L{vBb^g51B=(o)?%%xhvT*A5btOpw(TKh^g^4c zw>0%X!_0`{iN%RbVk+A^f{w-4-SSf*fu@FhruNL##F~sF24O~u zyYF<3el2b$$wZ_|uW#@Ak+VAGk#e|kS8nL1g>2B-SNMjMp^8;-FfeofY2fphFHO!{ z*!o4oTb{4e;S<|JEs<1_hPsmAlVNk?_5-Fp5KKU&d#FiNW~Y+pVFk@Cua1I{T+1|+ zHx6rFMor)7L)krbilqsWwy@T+g3DiH5MyVf8Wy}XbEaoFIDr~y;@r&I>FMW{ z?Q+(IgyebZ)-i4jNoXQhq4Muy9Fv+OxU;9_Jmn+<`mEC#%2Q_2bpcgzcinygNI!&^ z=V$)o2&Yz04~+&pPWWn`rrWxJ&}8khR)6B(--!9Q zubo}h+1T)>a@c)H^i``@<^j?|r4*{;tQf78(xn0g39IoZw0(CwY1f<%F>kEaJ zp9u|IeMY5mRdAlw*+gSN^5$Q)ShM<~E=(c8QM+T-Qk)FyKz#Sw0EJ*edYcuOtO#~Cx^(M7w5 z3)rl#L)rF|(Vun2LkFr!rg8Q@=r>9p>(t3Gf_auiJ2Xx9HmxYTa|=MH_SUlYL`mz9 zTTS$`%;D-|Jt}AP1&k7PcnfFNTH0A-*FmxstjBDiZX?}%u%Yq94$fUT&z6od+(Uk> zuqsld#G(b$G8tus=M!N#oPd|PVFX)?M?tCD0tS%2IGTfh}3YA3f&UM)W$_GNV8 zQo+a(ml2Km4o6O%gKTCSDNq+#zCTIQ1*`TIJh~k6Gp;htHBFnne))rlFdGqwC6dx2+La1&Mnko*352k0y z+tQcwndQlX`nc6nb$A9?<-o|r*%aWXV#=6PQic0Ok_D;q>wbv&j7cKc!w4~KF#-{6 z(S%6Za)WpGIWf7jZ3svNG5OLs0>vCL9{V7cgO%zevIVMH{WgP*^D9ws&OqA{yr|m| zKD4*07dGXshJHd#e%x%J+qmS^lS|0Bp?{drv;{@{l9ArPO&?Q5=?OO9=}h$oVe#3b z3Yofj&Cb}WC$PxmRRS)H%&$1-)z7jELS}!u!zQ?A^Y{Tv4QVt*vd@uj-^t2fYRzQj zfxGR>-q|o$3sGn^#VzZ!QQx?h9`njeJry}@x?|k0-GTTA4y3t2E`3DZ!A~D?GiJup z)8%PK2^9OVRlP(24P^4_<|D=H^7}WlWu#LgsdHzB%cPy|f8dD3|A^mh4WXxhLTVu_ z@abE{6Saz|Y{rXYPd4$tfPYo}ef(oQWZ=4Bct-=_9`#Qgp4ma$n$`tOwq#&E18$B; z@Bp)bn3&rEi0>fWWZ@7k5WazfoX`SCO4jQWwVuo+$PmSZn^Hz?O(-tW@*DGxuf)V1 zO_xm&;NVCaHD4dqt(-MlszI3F-p?0!-e$fbiCeuaw66h^TTDLWuaV<@C-`=Xe5WL) zwooG7h>4&*)p3pKMS3O!4>-4jQUN}iAMQ)2*70?hP~)TzzR?-f@?Aqy$$1Iy8VGG$ zMM?8;j!pUX7QQD$gRc_#+=raAS577ga-w?jd`vCiN5lu)dEUkkUPl9!?{$IJNxQys z*E4e$eF&n&+AMRQR2gcaFEjAy*r)G!s(P6D&TfoApMFC_*Ftx0|D0@E-=B7tezU@d zZ{hGiN;YLIoSeRS;9o%dEua4b%4R3;$SugDjP$x;Z!M!@QibuSBb)HY!3zJ7M;^jw zlx6AD50FD&p3JyP*>o+t9YWW8(7P2t!VQQ21pHJOcG_SXQD;(5aX#M6x##5H_Re>6lPyDCjxr*R(+HE%c&QN+b^tbT zXBJk?p)zhJj#I?&Y2n&~XiytG9!1ox;bw5Rbj~)7c(MFBb4>IiRATdhg zmiEFlj@S_hwYYI(ki{}&<;_7(Z0Qkfq>am z&LtL=2qc7rWguk3BtE4zL41@#S;NN*-jWw|7Kx7H7~_%7fPt;TIX}Ubo>;Rmj94V> zNB1=;-9AR7s`Pxn}t_6^3ahlq53e&!Lh85uG zec0vJY_6e`tg7LgfrJ3k!DjR)Bi#L@DHIrZ`sK=<5O0Ip!fxGf*OgGSpP@Hbbe&$9 z;ZI}8lEoC2_7;%L2=w?tb%1oL0V+=Z`7b=P&lNGY;yVBazXRYu;+cQDKvm*7NCxu&i;zub zAJh#11%?w>E2rf2e~C4+rAb-&$^vsdACs7 z@|Ra!OfVM(ke{vyiqh7puf&Yp6cd6{DptUteYfIRWG3pI+5< zBVBI_xkBAc<(pcb$!Y%dTW(b;B;2pOI-(QCsLv@U-D1XJ z(Gk8Q3l7Ws46Aktuj>|s{$6zA&xCPuXL-kB`CgYMs}4IeyG*P51IDwW?8UNQd+$i~ zlxOPtSi5L|gJcF@DwmJA5Ju8HEJ>o{{upwIpb!f{2(vLNBw`7xMbvcw<^{Fj@E~1( z?w`iIMieunS#>nXlmUcSMU+D3rX28f?s7z;X=se6bo8;5vM|O^(D6{A9*ChnGH!RG zP##3>LDC3jZPE4PH32AxrqPk|yIIrq~`aL-=}`okhNu9aT%q z1b)7iJ)CN=V#Ly84N_r7U^SH2FGdE5FpTO2 z630TF$P>GNMu8`rOytb(lB2};`;P4YNwW1<5d3Q~AX#P0aX}R2b2)`rgkp#zTxcGj zAV^cvFbhP|JgWrq_e`~exr~sIR$6p5V?o4Wym3kQ3HA+;Pr$bQ0(PmADVO%MKL!^q z?zAM8j1l4jrq|5X+V!8S*2Wl@=7*pPgciTVK6kS1Ge zMsd_u6DFK$jTnvVtE;qa+8(1sGBu~n&F%dh(&c(Zs4Fc#A=gG^^%^AyH}1^?|8quj zl@Z47h$){PlELJgYZCIHHL= z{U8O>Tw4x3<1{?$8>k-P<}1y9DmAZP_;(3Y*{Sk^H^A=_iSJ@+s5ktgwTXz_2$~W9>VVZsfwCm@s0sQ zeB50_yu@uS+e7QoPvdCwDz{prjo(AFwR%C?z`EL{1`|coJHQTk^nX=tvs1<0arUOJ z!^`*x&&BvTYmemyZ)2p~{%eYX=JVR?DYr(rNgqRMA5E1PR1Iw=prk=L2ldy3r3Vg@27IZx43+ywyzr-X*p*d@tZV+!U#~$-q=8c zgdSuh#r?b4GhEGNai)ayHQpk>5(%j5c@C1K3(W1pb~HeHpaqijJZa-e6vq_8t-^M^ zBJxq|MqZc?pjXPIH}70a5vt!IUh;l}<>VX<-Qcv^u@5(@@M2CHSe_hD$VG-eiV^V( zj7*9T0?di?P$FaD6oo?)<)QT>Npf6Og!GO^GmPV(Km0!=+dE&bk#SNI+C9RGQ|{~O*VC+tXK3!n`5 zHfl6>lwf_aEVV3`0T!aHNZLsj$paS$=LL(?b!Czaa5bbSuZ6#$_@LK<(7yrrl+80| z{tOFd=|ta2Z`^ssozD9BINn45NxUeCQis?-BKmU*Kt=FY-NJ+)8S1ecuFtN-M?&42 zl2$G>u!iNhAk*HoJ^4v^9#ORYp5t^wDj6|lx~5w45#E5wVqI1JQ~9l?nPp1YINf++ zMAdSif~_ETv@Er(EFBI^@L4BULFW>)NI+ejHFP*T}UhWNN`I)RRS8za? z*@`1>9ZB}An%aT5K=_2iQmfE;GcBVHLF!$`I99o5GO`O%O_zLr9AG18>&^HkG(;=V z%}c!OBQ~?MX(9h~tajX{=x)+!cbM7$YzTlmsPOdp2L-?GoW`@{lY9U3f;OUo*BwRB z8A+nv(br0-SH#VxGy#ZrgnGD(=@;HME;yd46EgWJ`EL%oXc&lFpc@Y}^>G(W>h_v_ zlN!`idhX+OjL+~T?19sroAFVGfa5tX-D49w$1g2g_-T|EpHL6}K_aX4$K=LTvwtlF zL*z}j{f+Uoe7{-px3_5iKPA<_7W=>Izkk)!l9ez2w%vi(?Y;i8AxRNLSOGDzNoqoI zP!1uAl}r=_871(G?y`i&)-7{u=%nxk7CZ_Qh#!|ITec zwQn`33GTUM`;D2POWnkqngqJhJRlM>CTONzTG}>^Q0wUunQyn|TAiHzyX2_%ATx%P z%7gW)%4rA9^)M<_%k@`Y?RbC<29sWU&5;@|9thf2#zf8z12$hRcZ!CSb>kUp=4N#y zl3hE#y6>kkA8VY2`W`g5Ip?2qC_BY$>R`iGQLhz2-S>x(RuWv)SPaGdl^)gGw7tjR zH@;jwk!jIaCgSg_*9iF|a);sRUTq30(8I(obh^|}S~}P4U^BIGYqcz;MPpC~Y@k_m zaw4WG1_vz2GdCAX!$_a%GHK**@IrHSkGoN>)e}>yzUTm52on`hYot7cB=oA-h1u|R ztH$11t?54Qg2L+i33FPFKKRm1aOjKST{l1*(nps`>sv%VqeVMWjl5+Gh+9);hIP8? zA@$?}Sc z3qIRpba+y5yf{R6G(u8Z^vkg0Fu&D-7?1s=QZU`Ub{-!Y`I?AGf1VNuc^L3v>)>i# z{DV9W$)>34wnzAXUiV^ZpYKw>UElrN_5Xj6{r_3| z$X5PK`e5$7>~9Dj7gK5ash(dvs`vwfk}&RD`>04;j62zoXESkFBklYaKm5seyiX(P zqQ-;XxlV*yg?Dhlx%xt!b0N3GHp@(p$A;8|%# zZ5m2KL|{on4nr>2_s9Yh=r5ScQ0;aMF)G$-9-Ca6%wA`Pa)i?NGFA|#Yi?{X-4ZO_ z^}%7%vkzvUHa$-^Y#aA+aiR5sa%S|Ebyn`EV<3Pc?ax_f>@sBZF1S;7y$CXd5t5=WGsTKBk8$OfH4v|0?0I=Yp}7c=WBSCg!{0n)XmiU;lfx)**zZaYqmDJelxk$)nZyx5`x$6R|fz(;u zEje5Dtm|a%zK!!tk3{i9$I2b{vXNFy%Bf{50X!x{98+BsDr_u9i>G5%*sqEX|06J0 z^IY{UcEbj6LDwuMh7cH`H@9sVt1l1#8kEQ(LyT@&+K}(ReE`ux8gb0r6L_#bDUo^P z3Ka2lRo52Hdtl_%+pwVs14=q`{d^L58PsU@AMf(hENumaxM{7iAT5sYmWh@hQCO^ zK&}ijo=`VqZ#a3vE?`7QW0ZREL17ZvDfdqKGD?0D4fg{7v%|Yj&_jcKJAB)>=*RS* zto8p6@k%;&^ZF>hvXm&$PCuEp{uqw3VPG$9VMdW5$w-fy2CNNT>E;>ejBgy-m_6`& z97L1p{%srn@O_JQgFpa_#f(_)eb#YS>o>q3(*uB;uZb605(iqM$=NK{nHY=+X2*G) zO3-_Xh%aG}fHWe*==58zBwp%&`mge<8uq8;xIxOd=P%9EK!34^E9sk|(Zq1QSz-JVeP12Fp)-`F|KY$LPwUE?rku zY@OJ)Z9A!ojfzfeyJ9;zv2EM7ZQB)AR5xGa-tMn^bl)FmoIiVyJ@!~@%{}qXXD&Ns zPnfe5U+&ohKefILu_1mPfLGuapX@btta5C#gPB2cjk5m4T}Nfi+Vfka!Yd(L?-c~5 z#ZK4VeQEXNPc4r$K00Fg>g#_W!YZ)cJ?JTS<&68_$#cZT-ME`}tcwqg3#``3M3UPvn+pi}(VNNx6y zFIMVb6OwYU(2`at$gHba*qrMVUl8xk5z-z~fb@Q3Y_+aXuEKH}L+>eW__!IAd@V}L zkw#s%H0v2k5-=vh$^vPCuAi22Luu3uKTf6fPo?*nvj$9(u)4$6tvF-%IM+3pt*cgs z_?wW}J7VAA{_~!?))?s6{M=KPpVhg4fNuU*|3THp@_(q!b*hdl{fjRVFWtu^1dV(f z6iOux9hi&+UK=|%M*~|aqFK{Urfl!TA}UWY#`w(0P!KMe1Si{8|o))Gy6d7;!JQYhgMYmXl?3FfOM2nQGN@~Ap6(G z3+d_5y@=nkpKAhRqf{qQ~k7Z$v&l&@m7Ppt#FSNzKPZM z8LhihcE6i=<(#87E|Wr~HKvVWhkll4iSK$^mUHaxgy8*K$_Zj;zJ`L$naPj+^3zTi z-3NTaaKnD5FPY-~?Tq6QHnmDDRxu0mh0D|zD~Y=vv_qig5r-cIbCpxlju&8Sya)@{ zsmv6XUSi)@(?PvItkiZEeN*)AE~I_?#+Ja-r8$(XiXei2d@Hi7Rx8+rZZb?ZLa{;@*EHeRQ-YDadz~M*YCM4&F-r;E#M+@CSJMJ0oU|PQ^ z=E!HBJDMQ2TN*Y(Ag(ynAL8%^v;=~q?s4plA_hig&5Z0x_^Oab!T)@6kRN$)qEJ6E zNuQjg|G7iwU(N8pI@_6==0CL;lRh1dQF#wePhmu@hADFd3B5KIH#dx(2A zp~K&;Xw}F_N6CU~0)QpQk7s$a+LcTOj1%=WXI(U=Dv!6 z{#<#-)2+gCyyv=Jw?Ab#PVkxPDeH|sAxyG`|Ys}A$PW4TdBv%zDz z^?lwrxWR<%Vzc8Sgt|?FL6ej_*e&rhqJZ3Y>k=X(^dytycR;XDU16}Pc9Vn0>_@H+ zQ;a`GSMEG64=JRAOg%~L)x*w{2re6DVprNp+FcNra4VdNjiaF0M^*>CdPkt(m150rCue?FVdL0nFL$V%5y6N z%eLr5%YN7D06k5ji5*p4v$UMM)G??Q%RB27IvH7vYr_^3>1D-M66#MN8tWGw>WED} z5AhlsanO=STFYFs)Il_0i)l)f<8qn|$DW7ZXhf5xI;m+7M5-%P63XFQrG9>DMqHc} zsgNU9nR`b}E^mL5=@7<1_R~j@q_2U^3h|+`7YH-?C=vme1C3m`Fe0HC>pjt6f_XMh zy~-i-8R46QNYneL4t@)<0VU7({aUO?aH`z4V2+kxgH5pYD5)wCh75JqQY)jIPN=U6 z+qi8cGiOtXG2tXm;_CfpH9ESCz#i5B(42}rBJJF$jh<1sbpj^8&L;gzGHb8M{of+} zzF^8VgML2O9nxBW7AvdEt90vp+#kZxWf@A)o9f9}vKJy9NDBjBW zSt=Hcs=YWCwnfY1UYx*+msp{g!w0HC<_SM!VL1(I2PE?CS}r(eh?{I)mQixmo5^p# zV?2R!R@3GV6hwTCrfHiK#3Orj>I!GS2kYhk1S;aFBD_}u2v;0HYFq}Iz1Z(I4oca4 zxquja8$+8JW_EagDHf$a1OTk5S97umGSDaj)gH=fLs9>_=XvVj^Xj9a#gLdk=&3tl zfmK9MNnIX9v{?%xdw7568 zNrZ|roYs(vC4pHB5RJ8>)^*OuyNC>x7ad)tB_}3SgQ96+-JT^Qi<`xi=)_=$Skwv~ zdqeT9Pa`LYvCAn&rMa2aCDV(TMI#PA5g#RtV|CWpgDYRA^|55LLN^uNh*gOU>Z=a06qJ;$C9z8;n-Pq=qZnc1zUwJ@t)L;&NN+E5m zRkQ(SeM8=l-aoAKGKD>!@?mWTW&~)uF2PYUJ;tB^my`r9n|Ly~0c%diYzqs9W#FTjy?h&X3TnH zXqA{QI82sdjPO->f=^K^f>N`+B`q9&rN0bOXO79S&a9XX8zund(kW7O76f4dcWhIu zER`XSMSFbSL>b;Rp#`CuGJ&p$s~G|76){d?xSA5wVg##_O0DrmyEYppyBr%fyWbbv zp`K84JwRNP$d-pJ!Qk|(RMr?*!wi1if-9G#0p>>1QXKXWFy)eB3ai)l3601q8!9JC zvU#ZWWDNKq9g6fYs?JQ)Q4C_cgTy3FhgKb8s&m)DdmL5zhNK#8wWg!J*7G7Qhe9VU zha?^AQTDpYcuN!B+#1dE*X{<#!M%zfUQbj=zLE{dW0XeQ7-oIsGY6RbkP2re@Q{}r_$iiH0xU%iN*ST`A)-EH6eaZB$GA#v)cLi z*MpA(3bYk$oBDKAzu^kJoSUsDd|856DApz={3u8sbQV@JnRkp2nC|)m;#T=DvIL-O zI4vh;g7824l}*`_p@MT4+d`JZ2%6NQh=N9bmgJ#q!hK@_<`HQq3}Z8Ij>3%~<*= zcv=!oT#5xmeGI92lqm9sGVE%#X$ls;St|F#u!?5Y7syhx6q#MVRa&lBmmn%$C0QzU z);*ldgwwCmzM3uglr}!Z2G+?& zf%Dpo&mD%2ZcNFiN-Z0f;c_Q;A%f@>26f?{d1kxIJD}LxsQkB47SAdwinfMILZdN3 zfj^HmTzS3Ku5BxY>ANutS8WPQ-G>v4^_Qndy==P3pDm+Xc?>rUHl-4+^%Sp5atOja z2oP}ftw-rqnb}+khR3CrRg^ibi6?QYk1*i^;kQGirQ=uB9Sd1NTfT-Rbv;hqnY4neE5H1YUrjS2m+2&@uXiAo- zrKUX|Ohg7(6F(AoP~tj;NZlV#xsfo-5reuQHB$&EIAhyZk;bL;k9ouDmJNBAun;H& zn;Of1z_Qj`x&M;5X;{s~iGzBQTY^kv-k{ksbE*Dl%Qf%N@hQCfY~iUw!=F-*$cpf2 z3wix|aLBV0b;W@z^%7S{>9Z^T^fLOI68_;l@+Qzaxo`nAI8emTV@rRhEKZ z?*z_{oGdI~R*#<2{bkz$G~^Qef}$*4OYTgtL$e9q!FY7EqxJ2`zk6SQc}M(k(_MaV zSLJnTXw&@djco1~a(vhBl^&w=$fa9{Sru>7g8SHahv$&Bl(D@(Zwxo_3r=;VH|uc5 zi1Ny)J!<(KN-EcQ(xlw%PNwK8U>4$9nVOhj(y0l9X^vP1TA>r_7WtSExIOsz`nDOP zs}d>Vxb2Vo2e5x8p(n~Y5ggAyvib>d)6?)|E@{FIz?G3PVGLf7-;BxaP;c?7ddH$z zA+{~k^V=bZuXafOv!RPsE1GrR3J2TH9uB=Z67gok+u`V#}BR86hB1xl}H4v`F+mRfr zYhortD%@IGfh!JB(NUNSDh+qDz?4ztEgCz&bIG-Wg7w-ua4ChgQR_c+z8dT3<1?uX z*G(DKy_LTl*Ea!%v!RhpCXW1WJO6F`bgS-SB;Xw9#! z<*K}=#wVu9$`Yo|e!z-CPYH!nj7s9dEPr-E`DXUBu0n!xX~&|%#G=BeM?X@shQQMf zMvr2!y7p_gD5-!Lnm|a@z8Of^EKboZsTMk%5VsJEm>VsJ4W7Kv{<|#4f-qDE$D-W>gWT%z-!qXnDHhOvLk=?^a1*|0j z{pW{M0{#1VcR5;F!!fIlLVNh_Gj zbnW(_j?0c2q$EHIi@fSMR{OUKBcLr{Y&$hrM8XhPByyZaXy|dd&{hYQRJ9@Fn%h3p7*VQolBIV@Eq`=y%5BU~3RPa^$a?ixp^cCg z+}Q*X+CW9~TL29@OOng(#OAOd!)e$d%sr}^KBJ-?-X&|4HTmtemxmp?cT3uA?md4% zT8yZ0U;6Rg6JHy3fJae{6TMGS?ZUX6+gGTT{Q{)SI85$5FD{g-eR%O0KMpWPY`4@O zx!hen1*8^E(*}{m^V_?}(b5k3hYo=T+$&M32+B`}81~KKZhY;2H{7O-M@vbCzuX0n zW-&HXeyr1%I3$@ns-V1~Lb@wIpkmx|8I~ob1Of7i6BTNysEwI}=!nU%q7(V_^+d*G z7G;07m(CRTJup!`cdYi93r^+LY+`M*>aMuHJm(A8_O8C#A*$!Xvddgpjx5)?_EB*q zgE8o5O>e~9IiSC@WtZpF{4Bj2J5eZ>uUzY%TgWF7wdDE!fSQIAWCP)V{;HsU3ap?4 znRsiiDbtN7i9hapO;(|Ew>Ip2TZSvK9Z^N21%J?OiA_&eP1{(Pu_=%JjKy|HOardq ze?zK^K zA%sjF64*Wufad%H<) z^|t>e*h+Z1#l=5wHexzt9HNDNXgM=-OPWKd^5p!~%SIl>Fo&7BvNpbf8{NXmH)o{r zO=aBJ;meX1^{O%q;kqdw*5k!Y7%t_30 zy{nGRVc&5qt?dBwLs+^Sfp;f`YVMSB#C>z^a9@fpZ!xb|b-JEz1LBX7ci)V@W+kvQ89KWA0T~Lj$aCcfW#nD5bt&Y_< z-q{4ZXDqVg?|0o)j1%l0^_it0WF*LCn-+)c!2y5yS7aZIN$>0LqNnkujV*YVes(v$ zY@_-!Q;!ZyJ}Bg|G-~w@or&u0RO?vlt5*9~yeoPV_UWrO2J54b4#{D(D>jF(R88u2 zo#B^@iF_%S>{iXSol8jpmsZuJ?+;epg>k=$d`?GSegAVp3n$`GVDvK${N*#L_1`44 z{w0fL{2%)0|E+qgZtjX}itZz^KJt4Y;*8uSK}Ft38+3>j|K(PxIXXR-t4VopXo#9# zt|F{LWr-?34y`$nLBVV_*UEgA6AUI65dYIbqpNq9cl&uLJ0~L}<=ESlOm?Y-S@L*d z<7vt}`)TW#f%Rp$Q}6@3=j$7Tze@_uZO@aMn<|si{?S}~maII`VTjs&?}jQ4_cut9$)PEqMukwoXobzaKx^MV z2fQwl+;LSZ$qy%Tys0oo^K=jOw$!YwCv^ei4NBVauL)tN%=wz9M{uf{IB(BxK|lT*pFkmNK_1tV`nb%jH=a0~VNq2RCKY(rG7jz!-D^k)Ec)yS%17pE#o6&eY+ z^qN(hQT$}5F(=4lgNQhlxj?nB4N6ntUY6(?+R#B?W3hY_a*)hnr4PA|vJ<6p`K3Z5Hy z{{8(|ux~NLUW=!?9Qe&WXMTAkQnLXg(g=I@(VG3{HE13OaUT|DljyWXPs2FE@?`iU z4GQlM&Q=T<4&v@Fe<+TuXiZQT3G~vZ&^POfmI1K2h6t4eD}Gk5XFGpbj1n_g*{qmD6Xy z`6Vv|lLZtLmrnv*{Q%xxtcWVj3K4M%$bdBk_a&ar{{GWyu#ljM;dII;*jP;QH z#+^o-A4np{@|Mz+LphTD0`FTyxYq#wY)*&Ls5o{0z9yg2K+K7ZN>j1>N&;r+Z`vI| zDzG1LJZ+sE?m?>x{5LJx^)g&pGEpY=fQ-4}{x=ru;}FL$inHemOg%|R*ZXPodU}Kh zFEd5#+8rGq$Y<_?k-}r5zgQ3jRV=ooHiF|@z_#D4pKVEmn5CGV(9VKCyG|sT9nc=U zEoT67R`C->KY8Wp-fEcjjFm^;Cg(ls|*ABVHq8clBE(;~K^b+S>6uj70g? z&{XQ5U&!Z$SO7zfP+y^8XBbiu*Cv-yJG|l-oe*!s5$@Lh_KpxYL2sx`B|V=dETN>5K+C+CU~a_3cI8{vbu$TNVdGf15*>D zz@f{zIlorkY>TRh7mKuAlN9A0>N>SV`X)+bEHms=mfYTMWt_AJtz_h+JMmrgH?mZt zm=lfdF`t^J*XLg7v+iS)XZROygK=CS@CvUaJo&w2W!Wb@aa?~Drtf`JV^cCMjngVZ zv&xaIBEo8EYWuML+vxCpjjY^s1-ahXJzAV6hTw%ZIy!FjI}aJ+{rE&u#>rs)vzuxz z+$5z=7W?zH2>Eb32dvgHYZtCAf!=OLY-pb4>Ae79rd68E2LkVPj-|jFeyqtBCCwiW zkB@kO_(3wFq)7qwV}bA=zD!*@UhT`geq}ITo%@O(Z5Y80nEX~;0-8kO{oB6|(4fQh z);73T!>3@{ZobPwRv*W?7m0Ml9GmJBCJd&6E?hdj9lV= z4flNfsc(J*DyPv?RCOx!MSvk(M952PJ-G|JeVxWVjN~SNS6n-_Ge3Q;TGE;EQvZg86%wZ`MB zSMQua(i*R8a75!6$QRO^(o7sGoomb+Y{OMy;m~Oa`;P9Yqo>?bJAhqXxLr7_3g_n>f#UVtxG!^F#1+y@os6x(sg z^28bsQ@8rw%Gxk-stAEPRbv^}5sLe=VMbkc@Jjimqjvmd!3E7+QnL>|(^3!R} zD-l1l7*Amu@j+PWLGHXXaFG0Ct2Q=}5YNUxEQHCAU7gA$sSC<5OGylNnQUa>>l%sM zyu}z6i&({U@x^hln**o6r2s-(C-L50tQvz|zHTqW!ir?w&V23tuYEDJVV#5pE|OJu z7^R!A$iM$YCe?8n67l*J-okwfZ+ZTkGvZ)tVPfR;|3gyFjF)8V zyXXN=!*bpyRg9#~Bg1+UDYCt0 ztp4&?t1X0q>uz;ann$OrZs{5*r`(oNvw=$7O#rD|Wuv*wIi)4b zGtq4%BX+kkagv3F9Id6~-c+1&?zny%w5j&nk9SQfo0k4LhdSU_kWGW7axkfpgR`8* z!?UTG*Zi_baA1^0eda8S|@&F z{)Rad0kiLjB|=}XFJhD(S3ssKlveFFmkN{Vl^_nb!o5M!RC=m)V&v2%e?ZoRC@h3> zJ(?pvToFd`*Zc@HFPL#=otWKwtuuQ_dT-Hr{S%pQX<6dqVJ8;f(o)4~VM_kEQkMR+ zs1SCVi~k>M`u1u2xc}>#D!V&6nOOh-E$O&SzYrjJdZpaDv1!R-QGA141WjQe2s0J~ zQ;AXG)F+K#K8_5HVqRoRM%^EduqOnS(j2)|ctA6Q^=|s_WJYU;Z%5bHp08HPL`YF2 zR)Ad1z{zh`=sDs^&V}J z%$Z$!jd7BY5AkT?j`eqMs%!Gm@T8)4w3GYEX~IwgE~`d|@T{WYHkudy(47brgHXx& zBL1yFG6!!!VOSmDxBpefy2{L_u5yTwja&HA!mYA#wg#bc-m%~8aRR|~AvMnind@zs zy>wkShe5&*un^zvSOdlVu%kHsEo>@puMQ`b1}(|)l~E{5)f7gC=E$fP(FC2=F<^|A zxeIm?{EE!3sO!Gr7e{w)Dx(uU#3WrFZ>ibmKSQ1tY?*-Nh1TDHLe+k*;{Rp!Bmd_m zb#^kh`Y*8l|9Cz2e{;RL%_lg{#^Ar+NH|3z*Zye>!alpt{z;4dFAw^^H!6ING*EFc z_yqhr8d!;%nHX9AKhFQZBGrSzfzYCi%C!(Q5*~hX>)0N`vbhZ@N|i;_972WSx*>LH z87?en(;2_`{_JHF`Sv6Wlps;dCcj+8IJ8ca6`DsOQCMb3n# z3)_w%FuJ3>fjeOOtWyq)ag|PmgQbC-s}KRHG~enBcIwqIiGW8R8jFeBNY9|YswRY5 zjGUxdGgUD26wOpwM#8a!Nuqg68*dG@VM~SbOroL_On0N6QdT9?)NeB3@0FCC?Z|E0 z6TPZj(AsPtwCw>*{eDEE}Gby>0q{*lI+g2e&(YQrsY&uGM{O~}(oM@YWmb*F zA0^rr5~UD^qmNljq$F#ARXRZ1igP`MQx4aS6*MS;Ot(1L5jF2NJ;de!NujUYg$dr# z=TEL_zTj2@>ZZN(NYCeVX2==~=aT)R30gETO{G&GM4XN<+!&W&(WcDP%oL8PyIVUC zs5AvMgh6qr-2?^unB@mXK*Dbil^y-GTC+>&N5HkzXtozVf93m~xOUHn8`HpX=$_v2 z61H;Z1qK9o;>->tb8y%#4H)765W4E>TQ1o0PFj)uTOPEvv&}%(_mG0ISmyhnQV33Z$#&yd{ zc{>8V8XK$3u8}04CmAQ#I@XvtmB*s4t8va?-IY4@CN>;)mLb_4!&P3XSw4pA_NzDb zORn!blT-aHk1%Jpi>T~oGLuh{DB)JIGZ9KOsciWs2N7mM1JWM+lna4vkDL?Q)z_Ct z`!mi0jtr+4*L&N7jk&LodVO#6?_qRGVaucqVB8*us6i3BTa^^EI0x%EREQSXV@f!lak6Wf1cNZ8>*artIJ(ADO*=<-an`3zB4d*oO*8D1K!f z*A@P1bZCNtU=p!742MrAj%&5v%Xp_dSX@4YCw%F|%Dk=u|1BOmo)HsVz)nD5USa zR~??e61sO(;PR)iaxK{M%QM_rIua9C^4ppVS$qCT9j2%?*em?`4Z;4@>I(c%M&#cH z>4}*;ej<4cKkbCAjjDsyKS8rIm90O)Jjgyxj5^venBx&7B!xLmzxW3jhj7sR(^3Fz z84EY|p1NauwXUr;FfZjdaAfh%ivyp+^!jBjJuAaKa!yCq=?T_)R!>16?{~p)FQ3LDoMyG%hL#pR!f@P%*;#90rs_y z@9}@r1BmM-SJ#DeuqCQk=J?ixDSwL*wh|G#us;dd{H}3*-Y7Tv5m=bQJMcH+_S`zVtf;!0kt*(zwJ zs+kedTm!A}cMiM!qv(c$o5K%}Yd0|nOd0iLjus&;s0Acvoi-PFrWm?+q9f^FslxGi z6ywB`QpL$rJzWDg(4)C4+!2cLE}UPCTBLa*_=c#*$b2PWrRN46$y~yST3a2$7hEH= zNjux+wna^AzQ=KEa_5#9Ph=G1{S0#hh1L3hQ`@HrVnCx{!fw_a0N5xV(iPdKZ-HOM za)LdgK}1ww*C_>V7hbQnTzjURJL`S%`6nTHcgS+dB6b_;PY1FsrdE8(2K6FN>37!62j_cBlui{jO^$dPkGHV>pXvW0EiOA zqW`YaSUBWg_v^Y5tPJfWLcLpsA8T zG)!x>pKMpt!lv3&KV!-um= zKCir6`bEL_LCFx4Z5bAFXW$g3Cq`?Q%)3q0r852XI*Der*JNuKUZ`C{cCuu8R8nkt z%pnF>R$uY8L+D!V{s^9>IC+bmt<05h**>49R*#vpM*4i0qRB2uPbg8{{s#9yC;Z18 zD7|4m<9qneQ84uX|J&f-g8a|nFKFt34@Bt{CU`v(SYbbn95Q67*)_Esl_;v291s=9 z+#2F2apZU4Tq=x+?V}CjwD(P=U~d<=mfEFuyPB`Ey82V9G#Sk8H_Ob_RnP3s?)S_3 zr%}Pb?;lt_)Nf>@zX~D~TBr;-LS<1I##8z`;0ZCvI_QbXNh8Iv)$LS=*gHr;}dgb=w5$3k2la1keIm|=7<-JD>)U%=Avl0Vj@+&vxn zt-)`vJxJr88D&!}2^{GPXc^nmRf#}nb$4MMkBA21GzB`-Or`-3lq^O^svO7Vs~FdM zv`NvzyG+0T!P8l_&8gH|pzE{N(gv_tgDU7SWeiI-iHC#0Ai%Ixn4&nt{5y3(GQs)i z&uA;~_0shP$0Wh0VooIeyC|lak__#KVJfxa7*mYmZ22@(<^W}FdKjd*U1CqSjNKW% z*z$5$=t^+;Ui=MoDW~A7;)Mj%ibX1_p4gu>RC}Z_pl`U*{_z@+HN?AF{_W z?M_X@o%w8fgFIJ$fIzBeK=v#*`mtY$HC3tqw7q^GCT!P$I%=2N4FY7j9nG8aIm$c9 zeKTxVKN!UJ{#W)zxW|Q^K!3s;(*7Gbn;e@pQBCDS(I|Y0euK#dSQ_W^)sv5pa%<^o zyu}3d?Lx`)3-n5Sy9r#`I{+t6x%I%G(iewGbvor&I^{lhu-!#}*Q3^itvY(^UWXgvthH52zLy&T+B)Pw;5>4D6>74 zO_EBS)>l!zLTVkX@NDqyN2cXTwsUVao7$HcqV2%t$YzdAC&T)dwzExa3*kt9d(}al zA~M}=%2NVNUjZiO7c>04YH)sRelXJYpWSn^aC$|Ji|E13a^-v2MB!Nc*b+=KY7MCm zqIteKfNkONq}uM;PB?vvgQvfKLPMB8u5+Am=d#>g+o&Ysb>dX9EC8q?D$pJH!MTAqa=DS5$cb+;hEvjwVfF{4;M{5U&^_+r zvZdu_rildI!*|*A$TzJ&apQWV@p{!W`=?t(o0{?9y&vM)V)ycGSlI3`;ps(vf2PUq zX745#`cmT*ra7XECC0gKkpu2eyhFEUb?;4@X7weEnLjXj_F~?OzL1U1L0|s6M+kIhmi%`n5vvDALMagi4`wMc=JV{XiO+^ z?s9i7;GgrRW{Mx)d7rj)?(;|b-`iBNPqdwtt%32se@?w4<^KU&585_kZ=`Wy^oLu9 z?DQAh5z%q;UkP48jgMFHTf#mj?#z|=w= z(q6~17Vn}P)J3M?O)x))%a5+>TFW3No~TgP;f}K$#icBh;rSS+R|}l鯊%1Et zwk~hMkhq;MOw^Q5`7oC{CUUyTw9x>^%*FHx^qJw(LB+E0WBX@{Ghw;)6aA-KyYg8p z7XDveQOpEr;B4je@2~usI5BlFadedX^ma{b{ypd|RNYqo#~d*mj&y`^iojR}s%~vF z(H!u`yx68D1Tj(3(m;Q+Ma}s2n#;O~bcB1`lYk%Irx60&-nWIUBr2x&@}@76+*zJ5 ze&4?q8?m%L9c6h=J$WBzbiTf1Z-0Eb5$IZs>lvm$>1n_Mezp*qw_pr8<8$6f)5f<@ zyV#tzMCs51nTv_5ca`x`yfE5YA^*%O_H?;tWYdM_kHPubA%vy47i=9>Bq) zRQ&0UwLQHeswmB1yP)+BiR;S+Vc-5TX84KUA;8VY9}yEj0eESSO`7HQ4lO z4(CyA8y1G7_C;6kd4U3K-aNOK!sHE}KL_-^EDl(vB42P$2Km7$WGqNy=%fqB+ zSLdrlcbEH=T@W8V4(TgoXZ*G1_aq$K^@ek=TVhoKRjw;HyI&coln|uRr5mMOy2GXP zwr*F^Y|!Sjr2YQXX(Fp^*`Wk905K%$bd03R4(igl0&7IIm*#f`A!DCarW9$h$z`kYk9MjjqN&5-DsH@8xh63!fTNPxWsFQhNv z#|3RjnP$Thdb#Ys7M+v|>AHm0BVTw)EH}>x@_f4zca&3tXJhTZ8pO}aN?(dHo)44Z z_5j+YP=jMlFqwvf3lq!57-SAuRV2_gJ*wsR_!Y4Z(trO}0wmB9%f#jNDHPdQGHFR; zZXzS-$`;7DQ5vF~oSgP3bNV$6Z(rwo6W(U07b1n3UHqml>{=6&-4PALATsH@Bh^W? z)ob%oAPaiw{?9HfMzpGb)@Kys^J$CN{uf*HX?)z=g`J(uK1YO^8~s1(ZIbG%Et(|q z$D@_QqltVZu9Py4R0Ld8!U|#`5~^M=b>fnHthzKBRr=i+w@0Vr^l|W;=zFT#PJ?*a zbC}G#It}rQP^Ait^W&aa6B;+0gNvz4cWUMzpv(1gvfw-X4xJ2Sv;mt;zb2Tsn|kSS zo*U9N?I{=-;a-OybL4r;PolCfiaL=y@o9{%`>+&FI#D^uy#>)R@b^1ue&AKKwuI*` zx%+6r48EIX6nF4o;>)zhV_8(IEX})NGU6Vs(yslrx{5fII}o3SMHW7wGtK9oIO4OM&@@ECtXSICLcPXoS|{;=_yj>hh*%hP27yZwOmj4&Lh z*Nd@OMkd!aKReoqNOkp5cW*lC)&C$P?+H3*%8)6HcpBg&IhGP^77XPZpc%WKYLX$T zsSQ$|ntaVVOoRat$6lvZO(G-QM5s#N4j*|N_;8cc2v_k4n6zx9c1L4JL*83F-C1Cn zaJhd;>rHXB%%ZN=3_o3&Qd2YOxrK~&?1=UuN9QhL$~OY-Qyg&})#ez*8NpQW_*a&kD&ANjedxT0Ar z<6r{eaVz3`d~+N~vkMaV8{F?RBVemN(jD@S8qO~L{rUw#=2a$V(7rLE+kGUZ<%pdr z?$DP|Vg#gZ9S}w((O2NbxzQ^zTot=89!0^~hE{|c9q1hVzv0?YC5s42Yx($;hAp*E zyoGuRyphQY{Q2ee0Xx`1&lv(l-SeC$NEyS~8iil3_aNlnqF_G|;zt#F%1;J)jnPT& z@iU0S;wHJ2$f!juqEzPZeZkjcQ+Pa@eERSLKsWf=`{R@yv7AuRh&ALRTAy z8=g&nxsSJCe!QLchJ=}6|LshnXIK)SNd zRkJNiqHwKK{SO;N5m5wdL&qK`v|d?5<4!(FAsDxR>Ky#0#t$8XCMptvNo?|SY?d8b z`*8dVBlXTUanlh6n)!EHf2&PDG8sXNAt6~u-_1EjPI1|<=33T8 zEnA00E!`4Ave0d&VVh0e>)Dc}=FfAFxpsC1u9ATfQ`-Cu;mhc8Z>2;uyXtqpLb7(P zd2F9<3cXS} znMg?{&8_YFTGRQZEPU-XPq55%51}RJpw@LO_|)CFAt62-_!u_Uq$csc+7|3+TV_!h z+2a7Yh^5AA{q^m|=KSJL+w-EWDBc&I_I1vOr^}P8i?cKMhGy$CP0XKrQzCheG$}G# zuglf8*PAFO8%xop7KSwI8||liTaQ9NCAFarr~psQt)g*pC@9bORZ>m`_GA`_K@~&% zijH0z;T$fd;-Liw8%EKZas>BH8nYTqsK7F;>>@YsE=Rqo?_8}UO-S#|6~CAW0Oz1} z3F(1=+#wrBJh4H)9jTQ_$~@#9|Bc1Pd3rAIA_&vOpvvbgDJOM(yNPhJJq2%PCcMaI zrbe~toYzvkZYQ{ea(Wiyu#4WB#RRN%bMe=SOk!CbJZv^m?Flo5p{W8|0i3`hI3Np# zvCZqY%o258CI=SGb+A3yJe~JH^i{uU`#U#fvSC~rWTq+K`E%J@ zasU07&pB6A4w3b?d?q}2=0rA#SA7D`X+zg@&zm^iA*HVi z009#PUH<%lk4z~p^l0S{lCJk1Uxi=F4e_DwlfHA`X`rv(|JqWKAA5nH+u4Da+E_p+ zVmH@lg^n4ixs~*@gm_dgQ&eDmE1mnw5wBz9Yg?QdZwF|an67Xd*x!He)Gc8&2!urh z4_uXzbYz-aX)X1>&iUjGp;P1u8&7TID0bTH-jCL&Xk8b&;;6p2op_=y^m@Nq*0{#o!!A;wNAFG@0%Z9rHo zcJs?Th>Ny6+hI`+1XoU*ED$Yf@9f91m9Y=#N(HJP^Y@ZEYR6I?oM{>&Wq4|v0IB(p zqX#Z<_3X(&{H+{3Tr|sFy}~=bv+l=P;|sBz$wk-n^R`G3p0(p>p=5ahpaD7>r|>pm zv;V`_IR@tvZreIuv2EM7ZQHhO+qUgw#kOs%*ekY^n|=1#x9&c;Ro&I~{rG-#_3ZB1 z?|9}IFdbP}^DneP*T-JaoYHt~r@EfvnPE5EKUwIxjPbsr$% zfWW83pgWST7*B(o=kmo)74$8UU)v0{@4DI+ci&%=#90}!CZz|rnH+Mz=HN~97G3~@ z;v5(9_2%eca(9iu@J@aqaMS6*$TMw!S>H(b z4(*B!|H|8&EuB%mITr~O?vVEf%(Gr)6E=>H~1VR z&1YOXluJSG1!?TnT)_*YmJ*o_Q@om~(GdrhI{$Fsx_zrkupc#y{DK1WOUR>tk>ZE) ziOLoBkhZZ?0Uf}cm>GsA>Rd6V8@JF)J*EQlQ<=JD@m<)hyElXR0`pTku*3MU`HJn| zIf7$)RlK^pW-$87U;431;Ye4Ie+l~_B3*bH1>*yKzn23cH0u(i5pXV! z4K?{3oF7ZavmmtTq((wtml)m6i)8X6ot_mrE-QJCW}Yn!(3~aUHYG=^fA<^~`e3yc z-NWTb{gR;DOUcK#zPbN^D*e=2eR^_!(!RKkiwMW@@yYtEoOp4XjOGgzi`;=8 zi3`Ccw1%L*y(FDj=C7Ro-V?q)-%p?Ob2ZElu`eZ99n14-ZkEV#y5C+{Pq87Gu3&>g zFy~Wk7^6v*)4pF3@F@rE__k3ikx(hzN3@e*^0=KNA6|jC^B5nf(XaoQaZN?Xi}Rn3 z$8&m*KmWvPaUQ(V<#J+S&zO|8P-#!f%7G+n_%sXp9=J%Z4&9OkWXeuZN}ssgQ#Tcj z8p6ErJQJWZ+fXLCco=RN8D{W%+*kko*2-LEb))xcHwNl~Xmir>kmAxW?eW50Osw3# zki8Fl$#fvw*7rqd?%E?}ZX4`c5-R&w!Y0#EBbelVXSng+kUfeUiqofPehl}$ormli zg%r)}?%=?_pHb9`Cq9Z|B`L8b>(!+8HSX?`5+5mm81AFXfnAt1*R3F z%b2RPIacKAddx%JfQ8l{3U|vK@W7KB$CdLqn@wP^?azRks@x8z59#$Q*7q!KilY-P zHUbs(IFYRGG1{~@RF;Lqyho$~7^hNC`NL3kn^Td%A7dRgr_&`2k=t+}D-o9&C!y^? z6MsQ=tc3g0xkK(O%DzR9nbNB(r@L;1zQrs8mzx&4dz}?3KNYozOW5;=w18U6$G4U2 z#2^qRLT*Mo4bV1Oeo1PKQ2WQS2Y-hv&S|C7`xh6=Pj7MNLC5K-zokZ67S)C;(F0Dd zloDK2_o1$Fmza>EMj3X9je7e%Q`$39Dk~GoOj89-6q9|_WJlSl!!+*{R=tGp z8u|MuSwm^t7K^nUe+^0G3dkGZr3@(X+TL5eah)K^Tn zXEtHmR9UIaEYgD5Nhh(s*fcG_lh-mfy5iUF3xxpRZ0q3nZ=1qAtUa?(LnT9I&~uxX z`pV?+=|-Gl(kz?w!zIieXT}o}7@`QO>;u$Z!QB${a08_bW0_o@&9cjJUXzVyNGCm8 zm=W+$H!;_Kzp6WQqxUI;JlPY&`V}9C$8HZ^m?NvI*JT@~BM=()T()Ii#+*$y@lTZBkmMMda>7s#O(1YZR+zTG@&}!EXFG{ zEWPSDI5bFi;NT>Yj*FjH((=oe%t%xYmE~AGaOc4#9K_XsVpl<4SP@E!TgC0qpe1oi zNpxU2b0(lEMcoibQ-G^cxO?ySVW26HoBNa;n0}CWL*{k)oBu1>F18X061$SP{Gu67 z-v-Fa=Fl^u3lnGY^o5v)Bux}bNZ~ z5pL+7F_Esoun8^5>z8NFoIdb$sNS&xT8_|`GTe8zSXQzs4r^g0kZjg(b0bJvz`g<70u9Z3fQILX1Lj@;@+##bP|FAOl)U^9U>0rx zGi)M1(Hce)LAvQO-pW!MN$;#ZMX?VE(22lTlJrk#pB0FJNqVwC+*%${Gt#r_tH9I_ z;+#)#8cWAl?d@R+O+}@1A^hAR1s3UcW{G+>;X4utD2d9X(jF555}!TVN-hByV6t+A zdFR^aE@GNNgSxxixS2p=on4(+*+f<8xrwAObC)D5)4!z7)}mTpb7&ofF3u&9&wPS< zB62WHLGMhmrmOAgmJ+|c>qEWTD#jd~lHNgT0?t-p{T=~#EMcB| z=AoDKOL+qXCfk~F)-Rv**V}}gWFl>liXOl7Uec_8v)(S#av99PX1sQIVZ9eNLkhq$ zt|qu0b?GW_uo}TbU8!jYn8iJeIP)r@;!Ze_7mj{AUV$GEz6bDSDO=D!&C9!M@*S2! zfGyA|EPlXGMjkH6x7OMF?gKL7{GvGfED=Jte^p=91FpCu)#{whAMw`vSLa`K#atdN zThnL+7!ZNmP{rc=Z>%$meH;Qi1=m1E3Lq2D_O1-X5C;!I0L>zur@tPAC9*7Jeh)`;eec}1`nkRP(%iv-`N zZ@ip-g|7l6Hz%j%gcAM}6-nrC8oA$BkOTz^?dakvX?`^=ZkYh%vUE z9+&)K1UTK=ahYiaNn&G5nHUY5niLGus@p5E2@RwZufRvF{@$hW{;{3QhjvEHMvduO z#Wf-@oYU4ht?#uP{N3utVzV49mEc9>*TV_W2TVC`6+oI)zAjy$KJrr=*q##&kobiQ z1vNbya&OVjK`2pdRrM?LuK6BgrLN7H_3m z!qpNKg~87XgCwb#I=Q&0rI*l$wM!qTkXrx1ko5q-f;=R2fImRMwt5Qs{P*p^z@9ex z`2#v(qE&F%MXlHpdO#QEZyZftn4f05ab^f2vjxuFaat2}jke{j?5GrF=WYBR?gS(^ z9SBiNi}anzBDBRc+QqizTTQuJrzm^bNA~A{j%ugXP7McZqJ}65l10({wk++$=e8O{ zxWjG!Qp#5OmI#XRQQM?n6?1ztl6^D40hDJr?4$Wc&O_{*OfMfxe)V0=e{|N?J#fgE>j9jAajze$iN!*yeF%jJU#G1c@@rm zolGW!j?W6Q8pP=lkctNFdfgUMg92wlM4E$aks1??M$~WQfzzzXtS)wKrr2sJeCN4X zY(X^H_c^PzfcO8Bq(Q*p4c_v@F$Y8cHLrH$`pJ2}=#*8%JYdqsqnGqEdBQMpl!Ot04tUGSXTQdsX&GDtjbWD=prcCT9(+ z&UM%lW%Q3yrl1yiYs;LxzIy>2G}EPY6|sBhL&X&RAQrSAV4Tlh2nITR?{6xO9ujGu zr*)^E`>o!c=gT*_@6S&>0POxcXYNQd&HMw6<|#{eSute2C3{&h?Ah|cw56-AP^f8l zT^kvZY$YiH8j)sk7_=;gx)vx-PW`hbSBXJGCTkpt;ap(}G2GY=2bbjABU5)ty%G#x zAi07{Bjhv}>OD#5zh#$0w;-vvC@^}F! z#X$@)zIs1L^E;2xDAwEjaXhTBw2<{&JkF*`;c3<1U@A4MaLPe{M5DGGkL}#{cHL%* zYMG+-Fm0#qzPL#V)TvQVI|?_M>=zVJr9>(6ib*#z8q@mYKXDP`k&A4A};xMK0h=yrMp~JW{L?mE~ph&1Y1a#4%SO)@{ zK2juwynUOC)U*hVlJU17%llUxAJFuKZh3K0gU`aP)pc~bE~mM!i1mi!~LTf>1Wp< zuG+ahp^gH8g8-M$u{HUWh0m^9Rg@cQ{&DAO{PTMudV6c?ka7+AO& z746QylZ&Oj`1aqfu?l&zGtJnpEQOt;OAFq19MXTcI~`ZcoZmyMrIKDFRIDi`FH)w; z8+*8tdevMDv*VtQi|e}CnB_JWs>fhLOH-+Os2Lh!&)Oh2utl{*AwR)QVLS49iTp{6 z;|172Jl!Ml17unF+pd+Ff@jIE-{Oxv)5|pOm@CkHW?{l}b@1>Pe!l}VccX#xp@xgJ zyE<&ep$=*vT=}7vtvif0B?9xw_3Gej7mN*dOHdQPtW5kA5_zGD zpA4tV2*0E^OUimSsV#?Tg#oiQ>%4D@1F5@AHwT8Kgen$bSMHD3sXCkq8^(uo7CWk`mT zuslYq`6Yz;L%wJh$3l1%SZv#QnG3=NZ=BK4yzk#HAPbqXa92;3K5?0kn4TQ`%E%X} z&>Lbt!!QclYKd6+J7Nl@xv!uD%)*bY-;p`y^ZCC<%LEHUi$l5biu!sT3TGGSTPA21 zT8@B&a0lJHVn1I$I3I1I{W9fJAYc+8 zVj8>HvD}&O`TqU2AAb={?eT;0hyL(R{|h23=4fDSZKC32;wWxsVj`P z3J3{M$PwdH!ro*Cn!D&=jnFR>BNGR<<|I8CI@+@658Dy(lhqbhXfPTVecY@L8%`3Q z1Fux2w?2C3th60jI~%OC9BtpNF$QPqcG+Pz96qZJ71_`0o0w_q7|h&O>`6U+^BA&5 zXd5Zp1Xkw~>M%RixTm&OqpNl8Q+ue=92Op_>T~_9UON?ZM2c0aGm=^A4ejrXj3dV9 zhh_bCt-b9`uOX#cFLj!vhZ#lS8Tc47OH>*)y#{O9?AT~KR9LntM|#l#Dlm^8{nZdk zjMl#>ZM%#^nK2TPzLcKxqx24P7R1FPlBy7LSBrRvx>fE$9AJ;7{PQm~^LBX^k#6Zq zw*Z(zJC|`!6_)EFR}8|n8&&Rbj8y028~P~sFXBFRt+tmqH-S3<%N;C&WGH!f3{7cm zy_fCAb9@HqaXa1Y5vFbxWf%#zg6SI$C+Uz5=CTO}e|2fjWkZ;Dx|84Ow~bkI=LW+U zuq;KSv9VMboRvs9)}2PAO|b(JCEC_A0wq{uEj|3x@}*=bOd zwr{TgeCGG>HT<@Zeq8y}vTpwDg#UBvD)BEs@1KP$^3$sh&_joQPn{hjBXmLPJ{tC) z*HS`*2+VtJO{|e$mM^|qv1R*8i(m1`%)}g=SU#T#0KlTM2RSvYUc1fP+va|4;5}Bfz98UvDCpq7}+SMV&;nX zQw~N6qOX{P55{#LQkrZk(e5YGzr|(B;Q;ju;2a`q+S9bsEH@i1{_Y0;hWYn1-79jl z5c&bytD*k)GqrVcHn6t-7kinadiD>B{Tl`ZY@`g|b~pvHh5!gKP4({rp?D0aFd_cN zhHRo4dd5^S6ViN(>(28qZT6E>??aRhc($kP`>@<+lIKS5HdhjVU;>f7<4))E*5|g{ z&d1}D|vpuV^eRj5j|xx9nwaCxXFG?Qbjn~_WSy=N}P0W>MP zG-F%70lX5Xr$a)2i6?i|iMyM|;Jtf*hO?=Jxj12oz&>P=1#h~lf%#fc73M2_(SUM- zf&qnjS80|_Y0lDgl&I?*eMumUklLe_=Td!9G@eR*tcPOgIShJipp3{A10u(4eT~DY zHezEj8V+7m!knn7)W!-5QI3=IvC^as5+TW1@Ern@yX| z7Nn~xVx&fGSr+L%4iohtS3w^{-H1A_5=r&x8}R!YZvp<2T^YFvj8G_vm}5q;^UOJf ztl=X3iL;;^^a#`t{Ae-%5Oq{?M#s6Npj+L(n-*LMI-yMR{)qki!~{5z{&`-iL}lgW zxo+tnvICK=lImjV$Z|O_cYj_PlEYCzu-XBz&XC-JVxUh9;6*z4fuBG+H{voCC;`~GYV|hj%j_&I zDZCj>Q_0RCwFauYoVMiUSB+*Mx`tg)bWmM^SwMA+?lBg12QUF_x2b)b?qb88K-YUd z0dO}3k#QirBV<5%jL$#wlf!60dizu;tsp(7XLdI=eQs?P`tOZYMjVq&jE)qK*6B^$ zBe>VvH5TO>s>izhwJJ$<`a8fakTL!yM^Zfr2hV9`f}}VVUXK39p@G|xYRz{fTI+Yq z20d=)iwjuG9RB$%$^&8#(c0_j0t_C~^|n+c`Apu|x7~;#cS-s=X1|C*YxX3ailhg_|0`g!E&GZJEr?bh#Tpb8siR=JxWKc{#w7g zWznLwi;zLFmM1g8V5-P#RsM@iX>TK$xsWuujcsVR^7TQ@!+vCD<>Bk9tdCo7Mzgq5 zv8d>dK9x8C@Qoh01u@3h0X_`SZluTb@5o;{4{{eF!-4405x8X7hewZWpz z2qEi4UTiXTvsa(0X7kQH{3VMF>W|6;6iTrrYD2fMggFA&-CBEfSqPlQDxqsa>{e2M z(R5PJ7uOooFc|9GU0ELA%m4&4Ja#cQpNw8i8ACAoK6?-px+oBl_yKmenZut#Xumjz zk8p^OV2KY&?5MUwGrBOo?ki`Sxo#?-Q4gw*Sh0k`@ zFTaYK2;}%Zk-68`#5DXU$2#=%YL#S&MTN8bF+!J2VT6x^XBci6O)Q#JfW{YMz) zOBM>t2rSj)n#0a3cjvu}r|k3od6W(SN}V-cL?bi*Iz-8uOcCcsX0L>ZXjLqk zZu2uHq5B|Kt>e+=pPKu=1P@1r9WLgYFq_TNV1p9pu0erHGd!+bBp!qGi+~4A(RsYN@CyXNrC&hxGmW)u5m35OmWwX`I+0yByglO`}HC4nGE^_HUs^&A(uaM zKPj^=qI{&ayOq#z=p&pnx@@k&I1JI>cttJcu@Ihljt?6p^6{|ds`0MoQwp+I{3l6` zB<9S((RpLG^>=Kic`1LnhpW2=Gu!x`m~=y;A`Qk!-w`IN;S8S930#vBVMv2vCKi}u z6<-VPrU0AnE&vzwV(CFC0gnZYcpa-l5T0ZS$P6(?9AM;`Aj~XDvt;Jua=jIgF=Fm? zdp=M$>`phx%+Gu};;-&7T|B1AcC#L4@mW5SV_^1BRbo6;2PWe$r+npRV`yc;T1mo& z+~_?7rA+(Um&o@Tddl zL_hxvWk~a)yY}%j`Y+200D%9$bWHy&;(yj{jpi?Rtz{J66ANw)UyPOm;t6FzY3$hx zcn)Ir79nhFvNa7^a{SHN7XH*|Vlsx`CddPnA&Qvh8aNhEA;mPVv;Ah=k<*u!Zq^7 z<=xs*iQTQOMMcg|(NA_auh@x`3#_LFt=)}%SQppP{E>mu_LgquAWvh<>L7tf9+~rO znwUDS52u)OtY<~!d$;m9+87aO+&`#2ICl@Y>&F{jI=H(K+@3M1$rr=*H^dye#~TyD z!){#Pyfn+|ugUu}G;a~!&&0aqQ59U@UT3|_JuBlYUpT$2+11;}JBJ`{+lQN9T@QFY z5+`t;6(TS0F?OlBTE!@7D`8#URDNqx2t6`GZ{ZgXeS@v%-eJzZOHz18aS|svxII$a zZeFjrJ*$IwX$f-Rzr_G>xbu@euGl)B7pC&S+CmDJBg$BoV~jxSO#>y z33`bupN#LDoW0feZe0%q8un0rYN|eRAnwDHQ6e_)xBTbtoZtTA=Fvk){q}9Os~6mQ zKB80VI_&6iSq`LnK7*kfHZoeX6?WE}8yjuDn=2#JG$+;-TOA1%^=DnXx%w{b=w}tS zQbU3XxtOI8E(!%`64r2`zog;5<0b4i)xBmGP^jiDZ2%HNSxIf3@wKs~uk4%3Mxz;~ zts_S~E4>W+YwI<-*-$U8*^HKDEa8oLbmqGg?3vewnaNg%Mm)W=)lcC_J+1ov^u*N3 zXJ?!BrH-+wGYziJq2Y#vyry6Z>NPgkEk+Ke`^DvNRdb>Q2Nlr#v%O@<5hbflI6EKE z9dWc0-ORk^T}jP!nkJ1imyjdVX@GrjOs%cpgA8-c&FH&$(4od#x6Y&=LiJZPINVyW z0snY$8JW@>tc2}DlrD3StQmA0Twck~@>8dSix9CyQOALcREdxoM$Sw*l!}bXKq9&r zysMWR@%OY24@e`?+#xV2bk{T^C_xSo8v2ZI=lBI*l{RciPwuE>L5@uhz@{!l)rtVlWC>)6(G)1~n=Q|S!{E9~6*fdpa*n z!()-8EpTdj=zr_Lswi;#{TxbtH$8*G=UM`I+icz7sr_SdnHXrv=?iEOF1UL+*6O;% zPw>t^kbW9X@oEXx<97%lBm-9?O_7L!DeD)Me#rwE54t~UBu9VZ zl_I1tBB~>jm@bw0Aljz8! zXBB6ATG6iByKIxs!qr%pz%wgqbg(l{65DP4#v(vqhhL{0b#0C8mq`bnqZ1OwFV z7mlZZJFMACm>h9v^2J9+^_zc1=JjL#qM5ZHaThH&n zXPTsR8(+)cj&>Un{6v*z?@VTLr{TmZ@-fY%*o2G}*G}#!bmqpoo*Ay@U!JI^Q@7gj;Kg-HIrLj4}#ec4~D2~X6vo;ghep-@&yOivYP zC19L0D`jjKy1Yi-SGPAn94(768Tcf$urAf{)1)9W58P`6MA{YG%O?|07!g9(b`8PXG1B1Sh0?HQmeJtP0M$O$hI z{5G`&9XzYhh|y@qsF1GnHN|~^ru~HVf#)lOTSrv=S@DyR$UKQk zjdEPFDz{uHM&UM;=mG!xKvp;xAGHOBo~>_=WFTmh$chpC7c`~7?36h)7$fF~Ii}8q zF|YXxH-Z?d+Q+27Rs3X9S&K3N+)OBxMHn1u(vlrUC6ckBY@@jl+mgr#KQUKo#VeFm zFwNYgv0<%~Wn}KeLeD9e1$S>jhOq&(e*I@L<=I5b(?G(zpqI*WBqf|Zge0&aoDUsC zngMRA_Kt0>La+Erl=Uv_J^p(z=!?XHpenzn$%EA`JIq#yYF?JLDMYiPfM(&Csr#f{ zdd+LJL1by?xz|D8+(fgzRs~(N1k9DSyK@LJygwaYX8dZl0W!I&c^K?7)z{2is;OkE zd$VK-(uH#AUaZrp=1z;O*n=b?QJkxu`Xsw&7yrX0?(CX=I-C#T;yi8a<{E~?vr3W> zQrpPqOW2M+AnZ&p{hqmHZU-;Q(7?- zP8L|Q0RM~sB0w1w53f&Kd*y}ofx@c z5Y6B8qGel+uT1JMot$nT1!Tim6{>oZzJXdyA+4euOLME?5Fd_85Uk%#E*ln%y{u8Q z$|?|R@Hpb~yTVK-Yr_S#%NUy7EBfYGAg>b({J|5b+j-PBpPy$Ns`PaJin4JdRfOaS zE|<HjH%NuJgsd2wOlv>~y=np%=2)$M9LS|>P)zJ+Fei5vYo_N~B0XCn+GM76 z)Xz3tg*FRVFgIl9zpESgdpWAavvVViGlU8|UFY{{gVJskg*I!ZjWyk~OW-Td4(mZ6 zB&SQreAAMqwp}rjy`HsG({l2&q5Y52<@AULVAu~rWI$UbFuZs>Sc*x+XI<+ez%$U)|a^unjpiW0l0 zj1!K0(b6$8LOjzRqQ~K&dfbMIE=TF}XFAi)$+h}5SD3lo z%%Qd>p9se=VtQG{kQ;N`sI)G^u|DN#7{aoEd zkksYP%_X$Rq08);-s6o>CGJ<}v`qs%eYf+J%DQ^2k68C%nvikRsN?$ap--f+vCS`K z#&~)f7!N^;sdUXu54gl3L=LN>FB^tuK=y2e#|hWiWUls__n@L|>xH{%8lIJTd5`w? zSwZbnS;W~DawT4OwSJVdAylbY+u5S+ZH{4hAi2&}Iv~W(UvHg(1GTZRPz`@{SOqzy z(8g&Dz=$PfRV=6FgxN~zo+G8OoPI&d-thcGVR*_^(R8COTM@bq?fDwY{}WhsQS1AK zF6R1t8!RdFmfocpJ6?9Yv~;WYi~XPgs(|>{5})j!AR!voO7y9&cMPo#80A(`za@t>cx<0;qxM@S*m(jYP)dMXr*?q0E`oL;12}VAep179uEr8c<=D zr5?A*C{eJ`z9Ee;E$8)MECqatHkbHH z&Y+ho0B$31MIB-xm&;xyaFCtg<{m~M-QDbY)fQ>Q*Xibb~8ytxZQ?QMf9!%cV zU0_X1@b4d+Pg#R!`OJ~DOrQz3@cpiGy~XSKjZQQ|^4J1puvwKeScrH8o{bscBsowomu z^f12kTvje`yEI3eEXDHJ6L+O{Jv$HVj%IKb|J{IvD*l6IG8WUgDJ*UGz z3!C%>?=dlfSJ>4U88)V+`U-!9r^@AxJBx8R;)J4Fn@`~k>8>v0M9xp90OJElWP&R5 zM#v*vtT}*Gm1^)Bv!s72T3PB0yVIjJW)H7a)ilkAvoaH?)jjb`MP>2z{%Y?}83 zUIwBKn`-MSg)=?R)1Q0z3b>dHE^)D8LFs}6ASG1|daDly_^lOSy&zIIhm*HXm1?VS=_iacG);_I9c zUQH1>i#*?oPIwBMJkzi_*>HoUe}_4o>2(SHWzqQ=;TyhAHS;Enr7!#8;sdlty&(>d zl%5cjri8`2X^Ds`jnw7>A`X|bl=U8n+3LKLy(1dAu8`g@9=5iw$R0qk)w8Vh_Dt^U zIglK}sn^)W7aB(Q>HvrX=rxB z+*L)3DiqpQ_%~|m=44LcD4-bxO3OO*LPjsh%p(k?&jvLp0py57oMH|*IMa(<|{m1(0S|x)?R-mqJ=I;_YUZA>J z62v*eSK;5w!h8J+6Z2~oyGdZ68waWfy09?4fU&m7%u~zi?YPHPgK6LDwphgaYu%0j zurtw)AYOpYKgHBrkX189mlJ`q)w-f|6>IER{5Lk97%P~a-JyCRFjejW@L>n4vt6#hq;!|m;hNE||LK3nw1{bJOy+eBJjK=QqNjI;Q6;Rp5 z&035pZDUZ#%Oa;&_7x0T<7!RW`#YBOj}F380Bq?MjjEhrvlCATPdkCTTl+2efTX$k zH&0zR1n^`C3ef~^sXzJK-)52(T}uTG%OF8yDhT76L~|^+hZ2hiSM*QA9*D5odI1>& z9kV9jC~twA5MwyOx(lsGD_ggYmztXPD`2=_V|ks_FOx!_J8!zM zTzh^cc+=VNZ&(OdN=y4Juw)@8-85lwf_#VMN!Ed(eQiRiLB2^2e`4dp286h@v@`O%_b)Y~A; zv}r6U?zs&@uD_+(_4bwoy7*uozNvp?bXFoB8?l8yG0qsm1JYzIvB_OH4_2G*IIOwT zVl%HX1562vLVcxM_RG*~w_`FbIc!(T=3>r528#%mwwMK}uEhJ()3MEby zQQjzqjWkwfI~;Fuj(Lj=Ug0y`>~C7`w&wzjK(rPw+Hpd~EvQ-ufQOiB4OMpyUKJhw zqEt~jle9d7S~LI~$6Z->J~QJ{Vdn3!c}g9}*KG^Kzr^(7VI5Gk(mHLL{itj_hG?&K4Ws0+T4gLfi3eu$N=`s36geNC?c zm!~}vG6lx9Uf^5M;bWntF<-{p^bruy~f?sk9 zcETAPQZLoJ8JzMMg<-=ju4keY@SY%Wo?u9Gx=j&dfa6LIAB|IrbORLV1-H==Z1zCM zeZcOYpm5>U2fU7V*h;%n`8 zN95QhfD994={1*<2vKLCNF)feKOGk`R#K~G=;rfq}|)s20&MCa65 zUM?xF5!&e0lF%|U!#rD@I{~OsS_?=;s_MQ_b_s=PuWdC)q|UQ&ea)DMRh5>fpQjXe z%9#*x=7{iRCtBKT#H>#v%>77|{4_slZ)XCY{s3j_r{tdpvb#|r|sbS^dU1x70$eJMU!h{Y7Kd{dl}9&vxQl6Jt1a` zHQZrWyY0?!vqf@u-fxU_@+}u(%Wm>0I#KP48tiAPYY!TdW(o|KtVI|EUB9V`CBBNaBLVih7+yMVF|GSoIQD0Jfb{ z!OXq;(>Z?O`1gap(L~bUcp>Lc@Jl-})^=6P%<~~9ywY=$iu8pJ0m*hOPzr~q`23eX zgbs;VOxxENe0UMVeN*>uCn9Gk!4siN-e>x)pIKAbQz!G)TcqIJ0`JBBaX>1-4_XO_-HCS^vr2vjv#7KltDZdyQ{tlWh4$Gm zB>|O1cBDC)yG(sbnc*@w6e%e}r*|IhpXckx&;sQCwGdKH+3oSG-2)Bf#x`@<4ETAr z0My%7RFh6ZLiZ_;X6Mu1YmXx7C$lSZ^}1h;j`EZd6@%JNUe=btBE z%s=Xmo1Ps?8G`}9+6>iaB8bgjUdXT?=trMu|4yLX^m0Dg{m7rpKNJey|EwHI+nN1e zL^>qN%5Fg)dGs4DO~uwIdXImN)QJ*Jhpj7$fq_^`{3fwpztL@WBB}OwQ#Epo-mqMO zsM$UgpFiG&d#)lzEQ{3Q;)&zTw;SzGOah-Dpm{!q7<8*)Ti_;xvV2TYXa}=faXZy? z3y?~GY@kl)>G&EvEijk9y1S`*=zBJSB1iet>0;x1Ai)*`^{pj0JMs)KAM=@UyOGtO z3y0BouW$N&TnwU6!%zS%nIrnANvZF&vB1~P5_d`x-giHuG zPJ;>XkVoghm#kZXRf>qxxEix;2;D1CC~NrbO6NBX!`&_$iXwP~P*c($EVV|669kDO zKoTLZNF4Cskh!Jz5ga9uZ`3o%7Pv`d^;a=cXI|>y;zC3rYPFLQkF*nv(r>SQvD*## z(Vo%^9g`%XwS0t#94zPq;mYGLKu4LU3;txF26?V~A0xZbU4Lmy`)>SoQX^m7fd^*E z+%{R4eN!rIk~K)M&UEzxp9dbY;_I^c} zOc{wlIrN_P(PPqi51k_$>Lt|X6A^|CGYgKAmoI#Li?;Wq%q~q*L7ehZkUrMxW67Jl zhsb~+U?33QS>eqyN{(odAkbopo=Q$Az?L+NZW>j;#~@wCDX?=L5SI|OxI~7!Pli;e zELMFcZtJY3!|=Gr2L4>z8yQ-{To>(f80*#;6`4IAiqUw`=Pg$%C?#1 z_g@hIGerILSU>=P>z{gM|DS91A4cT@PEIB^hSop!uhMo#2G;+tQSpDO_6nOnPWSLU zS;a9m^DFMXR4?*X=}d7l;nXuHk&0|m`NQn%d?8|Ab3A9l9Jh5s120ibWBdB z$5YwsK3;wvp!Kn@)Qae{ef`0#NwlRpQ}k^r>yos_Ne1;xyKLO?4)t_G4eK~wkUS2A&@_;)K0-03XGBzU+5f+uMDxC z(s8!8!RvdC#@`~fx$r)TKdLD6fWEVdEYtV#{ncT-ZMX~eI#UeQ-+H(Z43vVn%Yj9X zLdu9>o%wnWdvzA-#d6Z~vzj-}V3FQ5;axDIZ;i(95IIU=GQ4WuU{tl-{gk!5{l4_d zvvb&uE{%!iFwpymz{wh?bKr1*qzeZb5f6e6m_ozRF&zux2mlK=v_(_s^R6b5lu?_W4W3#<$zeG~Pd)^!4tzhs}-Sx$FJP>)ZGF(hVTH|C3(U zs0PO&*h_ zNA-&qZpTP$$LtIgfiCn07}XDbK#HIXdmv8zdz4TY;ifNIH-0jy(gMSByG2EF~Th#eb_TueZC` zE?3I>UTMpKQ})=C;6p!?G)M6w^u*A57bD?2X`m3X^6;&4%i_m(uGJ3Z5h`nwxM<)H z$I5m?wN>O~8`BGnZ=y^p6;0+%_0K}Dcg|K;+fEi|qoBqvHj(M&aHGqNF48~XqhtU? z^ogwBzRlOfpAJ+Rw7IED8lRbTdBdyEK$gPUpUG}j-M42xDj_&qEAQEtbs>D#dRd7Y z<&TpSZ(quQDHiCFn&0xsrz~4`4tz!CdL8m~HxZM_agu@IrBpyeL1Ft}V$HX_ZqDPm z-f89)pjuEzGdq-PRu`b1m+qBGY{zr_>{6Ss>F|xHZlJj9dt5HD$u`1*WZe)qEIuDSR)%z+|n zatVlhQ?$w#XRS7xUrFE;Y8vMGhQS5*T{ZnY=q1P?w5g$OKJ#M&e??tAmPWHMj3xhS ziGxapy?kn@$~2%ZY;M8Bc@%$pkl%Rvj!?o%agBvpQ-Q61n9kznC4ttrRNQ4%GFR5u zyv%Yo9~yxQJWJSfj z?#HY$y=O~F|2pZs22pu|_&Ajd+D(Mt!nPUG{|1nlvP`=R#kKH zO*s$r_%ss5h1YO7k0bHJ2CXN)Yd6CHn~W!R=SqkWe=&nAZu(Q1G!xgcUilM@YVei@2@a`8he z9@pM`)VB*=e7-MWgLlXlc)t;fF&-AwM{E-EX}pViFn0I0CNw2bNEnN2dj!^4(^zS3 zobUm1uQnpqk_4q{pl*n06=TfK_C>UgurKFjRXsK_LEn};=79`TB12tv6KzwSu*-C8 z;=~ohDLZylHQ|Mpx-?yql>|e=vI1Z!epyUpAcDCp4T|*RV&X`Q$0ogNwy6mFALo^@ z9=&(9txO8V@E!@6^(W0{*~CT>+-MA~vnJULBxCTUW>X5>r7*eXYUT0B6+w@lzw%n> z_VjJ<2qf|(d6jYq2(x$(ZDf!yVkfnbvNmb5c|hhZ^2TV_LBz`9w!e_V*W_(MiA7|= z&EeIIkw*+$Xd!)j8<@_<}A5;~A_>3JT*kX^@}cDoLd>Qj<`Se^wdUa(j0dp+Tl8EptwBm{9OGsdFEq zM`!pjf(Lm(`$e3FLOjqA5LnN5o!}z{ zNf}rJuZh@yUtq&ErjHeGzX4(!luV!jB&;FAP|!R_QHYw#^Z1LwTePAKJ6X&IDNO#; z)#I@Xnnzyij~C@UH~X51JCgQeF0&hTXnuoElz#m{heZRexWc0k4<>0+ClX7%0 zEBqCCld1tD9Zwkr4{?Nor19#E5-YKfB8d?qgR82-Ow2^AuNevly2*tHA|sK!ybYkX zm-sLQH72P&{vEAW6+z~O5d0qd=xW~rua~5a?ymYFSD@8&gV)E5@RNNBAj^C99+Z5Z zR@Pq55mbCQbz+Mn$d_CMW<-+?TU960agEk1J<>d>0K=pF19yN))a~4>m^G&tc*xR+yMD*S=yip-q=H zIlredHpsJV8H(32@Zxc@bX6a21dUV95Th--8pE6C&3F>pk=yv$yd6@Haw;$v4+Fcb zRwn{Qo@0`7aPa2LQOP}j9v>sjOo5Kqvn|`FLizX zB+@-u4Lw|jsvz{p^>n8Vo8H2peIqJJnMN}A)q6%$Tmig7eu^}K2 zrh$X?T|ZMsoh{6pdw1G$_T<`Ds-G=jc;qcGdK4{?dN2-XxjDNbb(7pk|3JUVCU4y; z)?LXR>f+AAu)JEiti_Zy#z5{RgsC}R(@jl%9YZ>zu~hKQ*AxbvhC378-I@{~#%Y`Z zy=a=9YpewPIC+gkEUUwtUL7|RU7=!^Aa}Mk^6uxOgRGA#JXjWLsjFUnix|Mau{hDT z7mn*z1m5g`vP(#tjT0Zy4eAY(br&!RiiXE=ZI!{sE1#^#%x^Z7t1U)b<;%Y}Q9=5v z;wpDCEZ@OE36TWT=|gxigT@VaW9BvHS05;_P(#s z8zI4XFQys}q)<`tkX$WnSarn{3e!s}4(J!=Yf>+Y>cP3f;vr63f2{|S^`_pWc)^5_!R z*(x-fuBxL51@xe!lnDBKi}Br$c$BMZ3%f2Sa6kLabiBS{pq*yj;q|k(86x`PiC{p6 z_bxCW{>Q2BA8~Ggz&0jkrcU+-$ANBsOop*ms>34K9lNYil@}jC;?cYP(m^P}nR6FV zk(M%48Z&%2Rx$A&FhOEirEhY0(dn;-k(qkTU)sFQ`+-ih+s@A8g?r8Pw+}2;35WYf zi}VO`jS`p(tc)$X$a>-#WXoW!phhatC*$}|rk>|wUU71eUJG^$c6_jwX?iSHM@6__ zvV|6%U*$sSXJu9SX?2%M^kK|}a2QJ8AhF{fuXrHZxXsI~O zGKX45!K7p*MCPEQ=gp?eu&#AW*pR{lhQR##P_*{c_DjMGL|3T3-bSJ(o$|M{ytU}> zAV>wq*uE*qFo9KvnA^@juy{x<-u*#2NvkV={Ly}ysKYB-k`K3@K#^S1Bb$8Y#0L0# z`6IkSG&|Z$ODy|VLS+y5pFJx&8tvPmMd8c9FhCyiU8~k6FwkakUd^(_ml8`rnl>JS zZV){9G*)xBqPz^LDqRwyS6w86#D^~xP4($150M)SOZRe9sn=>V#aG0Iy(_^YcPpIz8QYM-#s+n% z@Jd?xQq?Xk6=<3xSY7XYP$$yd&Spu{A#uafiIfy8gRC`o0nk{ezEDjb=q_qRAlR1d zFq^*9Gn)yTG4b}R{!+3hWQ+u3GT~8nwl2S1lpw`s0X_qpxv)g+JIkVKl${sYf_nV~B>Em>M;RlqGb5WVil(89 zs=ld@|#;dq1*vQGz=7--Br-|l) zZ%Xh@v8>B7P?~}?Cg$q9_={59l%m~O&*a6TKsCMAzG&vD>k2WDzJ6!tc!V)+oxF;h zJH;apM=wO?r_+*#;ulohuP=E>^zon}a$NnlcQ{1$SO*i=jnGVcQa^>QOILc)e6;eNTI>os=eaJ{*^DE+~jc zS}TYeOykDmJ=6O%>m`i*>&pO_S;qMySJIyP=}4E&J%#1zju$RpVAkZbEl+p%?ZP^C z*$$2b4t%a(e+%>a>d_f_<JjxI#J1x;=hPd1zFPx=6T$;;X1TD*2(edZ3f46zaAoW>L53vS_J*N8TMB|n+;LD| zC=GkQPpyDY#Am4l49chDv*gojhRj_?63&&8#doW`INATAo(qY#{q}%nf@eTIXmtU< zdB<7YWfyCmBs|c)cK>1)v&M#!yNj#4d$~pVfDWQc_ke1?fw{T1Nce_b`v|Vp5ig(H zJvRD^+ps46^hLX;=e2!2e;w9y1D@!D$c@Jc&%%%IL=+xzw55&2?darw=9g~>P z9>?Kdc$r?6c$m%x2S$sdpPl>GQZ{rC9mPS63*qjCVa?OIBj!fW zm|g?>CVfGXNjOfcyqImXR_(tXS(F{FcoNzKvG5R$IgGaxC@)i(e+$ME}vPVIhd|mx2IIE+f zM?9opQHIVgBWu)^A|RzXw!^??S!x)SZOwZaJkGjc<_}2l^eSBm!eAJG9T>EC6I_sy z?bxzDIAn&K5*mX)$RQzDA?s)-no-XF(g*yl4%+GBf`##bDXJ==AQk*xmnatI;SsLp zP9XTHq5mmS=iWu~9ES>b%Q=1aMa|ya^vj$@qz9S!ih{T8_PD%Sf_QrNKwgrXw9ldm zHRVR98*{C?_XNpJn{abA!oix_mowRMu^2lV-LPi;0+?-F(>^5#OHX-fPED zCu^l7u3E%STI}c4{J2!)9SUlGP_@!d?5W^QJXOI-Ea`hFMKjR7TluLvzC-ozCPn1`Tpy z!vlv@_Z58ILX6>nDjTp-1LlFMx~-%GA`aJvG$?8*Ihn;mH37eK**rmOEwqegf-Ccx zrIX4;{c~RK>XuTXxYo5kMiWMy)!IC{*DHG@E$hx?RwP@+wuad(P1{@%tRkyJRqD)3 zMHHHZ4boqDn>-=DgR5VlhQTpfVy182Gk;A_S8A1-;U1RR>+$62>(MUx@Nox$vTjHq z%QR=j!6Gdyb5wu7y(YUktwMuW5<@jl?m4cv4BODiT5o8qVdC0MBqGr@-YBIwnpZAY znX9(_uQjP}JJ=!~Ve9#5I~rUnN|P_3D$LqZcvBnywYhjlMSFHm`;u9GPla{5QD7(7*6Tb3Svr8;(nuAd81q$*uq6HC_&~je*Ca7hP4sJp0av{M8480wF zxASi7Qv+~@2U%Nu1Ud;s-G4CTVWIPyx!sg&8ZG0Wq zG_}i3C(6_1>q3w!EH7$Kwq8uBp2F2N7}l65mk1p*9v0&+;th=_E-W)E;w}P(j⁢ zv5o9#E7!G0XmdzfsS{efPNi`1b44~SZ4Z8fuX!I}#8g+(wxzQwUT#Xb2(tbY1+EUhGKoT@KEU9Ktl>_0 z%bjDJg;#*gtJZv!-Zs`?^}v5eKmnbjqlvnSzE@_SP|LG_PJ6CYU+6zY6>92%E+ z=j@TZf-iW4(%U{lnYxQA;7Q!b;^brF8n0D>)`q5>|WDDXLrqYU_tKN2>=#@~OE7grMnNh?UOz-O~6 z6%rHy{#h9K0AT+lDC7q4{hw^|q6*Ry;;L%Q@)Ga}$60_q%D)rv(CtS$CQbpq9|y1e zRSrN4;$Jyl{m5bZw`$8TGvb}(LpY{-cQ)fcyJv7l3S52TLXVDsphtv&aPuDk1OzCA z4A^QtC(!11`IsNx_HnSy?>EKpHJWT^wmS~hc^p^zIIh@9f6U@I2 zC=Mve{j2^)mS#U$e{@Q?SO6%LDsXz@SY+=cK_QMmXBIU)j!$ajc-zLx3V60EXJ!qC zi<%2x8Q24YN+&8U@CIlN zrZkcT9yh%LrlGS9`G)KdP(@9Eo-AQz@8GEFWcb7U=a0H^ZVbLmz{+&M7W(nXJ4sN8 zJLR7eeK(K8`2-}j(T7JsO`L!+CvbueT%izanm-^A1Dn{`1Nw`9P?cq;7no+XfC`K(GO9?O^5zNIt4M+M8LM0=7Gz8UA@Z0N+lg+cX)NfazRu z5D)~HA^(u%w^cz+@2@_#S|u>GpB+j4KzQ^&Wcl9f z&hG#bCA(Yk0D&t&aJE^xME^&E-&xGHhXn%}psEIj641H+Nl-}boj;)Zt*t(4wZ5DN z@GXF$bL=&pBq-#vkTkh>7hl%K5|3 z{`Vn9b$iR-SoGENp}bn4;fR3>9sA%X2@1L3aE9yTra;Wb#_`xWwLSLdfu+PAu+o3| zGVnpzPr=ch{uuoHjtw7+_!L_2;knQ!DuDl0R`|%jr+}jFzXtrHIKc323?JO{l&;VF z*L1+}JU7%QJOg|5|Tc|D8fN zJORAg=_vsy{ak|o);@)Yh8Lkcg@$FG3k@ep36BRa^>~UmnRPziS>Z=`Jb2x*Q#`%A zU*i3&Vg?TluO@X0O;r2Jl6LKLUOVhSqg1*qOt^|8*c7 zo(298@+r$k_wQNGHv{|$tW(T8L+4_`FQ{kEW5Jgg{yf7ey4ss_(SNKfz(N9lx&a;< je(UuV8hP?p&}TPdm1I$XmG#(RzlD&B2izSj9sl%y5~4qc diff --git a/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties index 3fa8f862f7..1af9e0930b 100644 --- a/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties +++ b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From 39cc1d7c72d65b53636b08602887df33ce379bab Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 28 Nov 2024 07:44:56 -0800 Subject: [PATCH 083/126] Enable sharing for InferenceCalculatorGlAdvanced Allows sharing the same instance of GpuInferenceRunner in InferenceCalculatorGlAdvanced between multiple instances in the same graph or other graphs. PiperOrigin-RevId: 701024167 --- mediapipe/calculators/tensor/BUILD | 1 + .../inference_calculator_gl_advanced.cc | 24 +++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index ce84f4b18c..c336c13d98 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -633,6 +633,7 @@ cc_library( ":inference_io_mapper", ":inference_on_disk_cache_helper", ":tensor_span", + "//mediapipe/calculators/tensor:inference_runner", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:mediapipe_profiling", "//mediapipe/framework/api2:packet", diff --git a/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc b/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc index 8661e742c4..07b23af0c1 100644 --- a/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc +++ b/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc @@ -25,6 +25,7 @@ #include "mediapipe/calculators/tensor/inference_calculator.h" #include "mediapipe/calculators/tensor/inference_io_mapper.h" #include "mediapipe/calculators/tensor/inference_on_disk_cache_helper.h" +#include "mediapipe/calculators/tensor/inference_runner.h" #include "mediapipe/calculators/tensor/tensor_span.h" #include "mediapipe/framework/api2/packet.h" #include "mediapipe/framework/calculator_framework.h" @@ -63,17 +64,17 @@ class InferenceCalculatorGlAdvancedImpl private: // Helper class that wraps everything related to GPU inference acceleration. - class GpuInferenceRunner { + class GpuInferenceRunner : public InferenceRunner { public: ~GpuInferenceRunner(); absl::Status Init(CalculatorContext* cc, std::shared_ptr gl_context); - absl::StatusOr> Process( - CalculatorContext* cc, const TensorSpan& input_tensors); + absl::StatusOr> Run( + CalculatorContext* cc, const TensorSpan& input_tensors) override; - const InputOutputTensorNames& GetInputOutputTensorNames() const; + const InputOutputTensorNames& GetInputOutputTensorNames() const override; private: absl::Status InitTFLiteGPURunner( @@ -99,7 +100,7 @@ class InferenceCalculatorGlAdvancedImpl absl::StatusOr> CreateInferenceRunner( CalculatorContext* cc); - std::unique_ptr gpu_inference_runner_; + std::unique_ptr inference_runner_; mediapipe::GlCalculatorHelper gpu_helper_; }; @@ -141,7 +142,7 @@ absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Init( } absl::StatusOr> -InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process( +InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Run( CalculatorContext* cc, const TensorSpan& input_tensors) { std::vector output_tensors; for (int i = 0; i < input_tensors.size(); ++i) { @@ -267,11 +268,10 @@ absl::Status InferenceCalculatorGlAdvancedImpl::UpdateContract( absl::Status InferenceCalculatorGlAdvancedImpl::Open(CalculatorContext* cc) { MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); - gpu_inference_runner_ = std::make_unique(); - MP_RETURN_IF_ERROR( - gpu_inference_runner_->Init(cc, gpu_helper_.GetSharedGlContext())); + + MP_ASSIGN_OR_RETURN(inference_runner_, CreateInferenceRunner(cc)); return InferenceCalculatorNodeImpl::UpdateIoMapping( - cc, gpu_inference_runner_->GetInputOutputTensorNames()); + cc, inference_runner_->GetInputOutputTensorNames()); } absl::StatusOr> InferenceCalculatorGlAdvancedImpl::Process( @@ -279,14 +279,14 @@ absl::StatusOr> InferenceCalculatorGlAdvancedImpl::Process( std::vector output_tensors; MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([&]() -> absl::Status { MP_ASSIGN_OR_RETURN(output_tensors, - gpu_inference_runner_->Process(cc, tensor_span)); + inference_runner_->Run(cc, tensor_span)); return absl::OkStatus(); })); return output_tensors; } absl::Status InferenceCalculatorGlAdvancedImpl::Close(CalculatorContext* cc) { - gpu_inference_runner_.reset(); + inference_runner_.reset(); return absl::OkStatus(); } From beda0d49044d8fc43ae411a9d50e9c7be7e634d7 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 28 Nov 2024 08:27:22 -0800 Subject: [PATCH 084/126] Extract native-to-UTF8 path string conversion; add FormatLastError() PiperOrigin-RevId: 701033706 --- mediapipe/framework/deps/BUILD | 8 +++ mediapipe/framework/deps/file_helpers.cc | 69 +++++++------------ mediapipe/framework/deps/platform_strings.cc | 53 ++++++++++++++ mediapipe/framework/deps/platform_strings.h | 46 +++++++++++++ .../framework/deps/platform_strings_test.cc | 32 +++++++++ 5 files changed, 165 insertions(+), 43 deletions(-) create mode 100644 mediapipe/framework/deps/platform_strings.cc create mode 100644 mediapipe/framework/deps/platform_strings.h create mode 100644 mediapipe/framework/deps/platform_strings_test.cc diff --git a/mediapipe/framework/deps/BUILD b/mediapipe/framework/deps/BUILD index 615f7f1d5a..1b87352367 100644 --- a/mediapipe/framework/deps/BUILD +++ b/mediapipe/framework/deps/BUILD @@ -110,6 +110,13 @@ cc_library( ], ) +cc_library( + name = "platform_strings", + srcs = ["platform_strings.cc"], + hdrs = ["platform_strings.h"], + visibility = ["//visibility:public"], +) + cc_library( name = "file_helpers", srcs = ["file_helpers.cc"], @@ -117,6 +124,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":file_path", + ":platform_strings", "//mediapipe/framework/port:status", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", diff --git a/mediapipe/framework/deps/file_helpers.cc b/mediapipe/framework/deps/file_helpers.cc index 5cfaf09e70..c68c46324e 100644 --- a/mediapipe/framework/deps/file_helpers.cc +++ b/mediapipe/framework/deps/file_helpers.cc @@ -33,7 +33,9 @@ #include #include "absl/status/status.h" +#include "absl/strings/string_view.h" #include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/deps/platform_strings.h" // IWYU pragma: keep #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_builder.h" @@ -93,31 +95,12 @@ class DirectoryListing { struct dirent* next_entry_ = nullptr; }; #else -#if defined(UNICODE) -using PathString = std::wstring; - -PathString Utf8ToNative(const std::string& string) { - std::wstring_convert, wchar_t> converter; - return converter.from_bytes(string.data(), string.data() + string.size()); -} -std::string NativeToUtf8(const PathString& string) { - std::wstring_convert, wchar_t> converter; - return converter.to_bytes(string.data(), string.data() + string.size()); -} -#define FILE_PATH_LITERAL_INTERNAL(x) L##x -#define FILE_PATH_LITERAL(x) FILE_PATH_LITERAL_INTERNAL(x) -#else -using PathString = std::string; -PathString Utf8ToNative(const std::string& string) { return string; } -std::string NativeToUtf8(const PathString& string) { return string; } -#define FILE_PATH_LITERAL(x) x -#endif - class DirectoryListing { public: explicit DirectoryListing(const std::string& directory) : directory_(Utf8ToNative(directory)) { - PathString search_string = directory_ + Utf8ToNative("\\*.*"); + PlatformString search_string = + directory_ + PLATFORM_STRING_LITERAL("\\*.*"); find_handle_ = FindFirstFile(search_string.c_str(), &find_data_); } @@ -134,8 +117,8 @@ class DirectoryListing { // after the one that is returned, if it exists. std::string NextEntry() { if (HasNextEntry()) { - PathString result = - directory_ + Utf8ToNative("\\") + PathString(find_data_.cFileName); + PlatformString result = directory_ + PLATFORM_STRING_LITERAL("\\") + + PlatformString(find_data_.cFileName); ReadNextEntry(); return NativeToUtf8(result); } else { @@ -146,9 +129,10 @@ class DirectoryListing { private: void ReadNextEntry() { int find_result = FindNextFile(find_handle_, &find_data_); - while (find_result != 0 && - (PathString(find_data_.cFileName) == FILE_PATH_LITERAL(".") || - PathString(find_data_.cFileName) == FILE_PATH_LITERAL(".."))) { + while (find_result != 0 && (PlatformString(find_data_.cFileName) == + PLATFORM_STRING_LITERAL(".") || + PlatformString(find_data_.cFileName) == + PLATFORM_STRING_LITERAL(".."))) { find_result = FindNextFile(find_handle_, &find_data_); } @@ -158,7 +142,7 @@ class DirectoryListing { } } - const PathString directory_; + const PlatformString directory_; HANDLE find_handle_ = INVALID_HANDLE_VALUE; WIN32_FIND_DATA find_data_; }; @@ -166,12 +150,12 @@ class DirectoryListing { } // namespace -absl::Status GetContents(absl::string_view file_name, std::string* output, +absl::Status GetContents(absl::string_view path, std::string* output, bool read_as_binary) { - FILE* fp = fopen(file_name.data(), read_as_binary ? "rb" : "r"); + FILE* fp = fopen(std::string(path).c_str(), read_as_binary ? "rb" : "r"); if (fp == NULL) { return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Can't find file: " << file_name; + << "Can't find file: " << path; } output->clear(); @@ -179,8 +163,8 @@ absl::Status GetContents(absl::string_view file_name, std::string* output, char buf[4096]; size_t ret = fread(buf, 1, 4096, fp); if (ret == 0 && ferror(fp)) { - return mediapipe::InternalErrorBuilder(MEDIAPIPE_LOC) - << "Error while reading file: " << file_name; + return mediapipe::UnavailableErrorBuilder(MEDIAPIPE_LOC) + << "Error while reading file: " << path; } output->append(std::string(buf, ret)); } @@ -188,37 +172,36 @@ absl::Status GetContents(absl::string_view file_name, std::string* output, return absl::OkStatus(); } -absl::Status SetContents(absl::string_view file_name, - absl::string_view content) { - FILE* fp = fopen(file_name.data(), "wb"); +absl::Status SetContents(absl::string_view path, absl::string_view content) { + FILE* fp = fopen(std::string(path).c_str(), "wb"); if (fp == NULL) { return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Can't open file: " << file_name; + << "Can't open file: " << path; } fwrite(content.data(), sizeof(char), content.size(), fp); size_t write_error = ferror(fp); if (fclose(fp) != 0 || write_error) { - return mediapipe::InternalErrorBuilder(MEDIAPIPE_LOC) - << "Error while writing file: " << file_name + return mediapipe::UnavailableErrorBuilder(MEDIAPIPE_LOC) + << "Error while writing file: " << path << ". Error message: " << strerror(write_error); } return absl::OkStatus(); } -absl::Status AppendStringToFile(absl::string_view file_name, +absl::Status AppendStringToFile(absl::string_view path, absl::string_view contents) { - FILE* fp = fopen(file_name.data(), "ab"); + FILE* fp = fopen(std::string(path).c_str(), "ab"); if (!fp) { return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Can't open file: " << file_name; + << "Can't open file: " << path; } fwrite(contents.data(), sizeof(char), contents.size(), fp); size_t write_error = ferror(fp); if (fclose(fp) != 0 || write_error) { - return mediapipe::InternalErrorBuilder(MEDIAPIPE_LOC) - << "Error while writing file: " << file_name + return mediapipe::UnavailableErrorBuilder(MEDIAPIPE_LOC) + << "Error while writing file: " << path << ". Error message: " << strerror(write_error); } return absl::OkStatus(); diff --git a/mediapipe/framework/deps/platform_strings.cc b/mediapipe/framework/deps/platform_strings.cc new file mode 100644 index 0000000000..fa8f3c791f --- /dev/null +++ b/mediapipe/framework/deps/platform_strings.cc @@ -0,0 +1,53 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/deps/platform_strings.h" + +#include + +namespace mediapipe { +#ifdef _WIN32 +#include + +std::string FormatLastError() { + DWORD message_id = GetLastError(); + if (message_id == 0) { + return std::string("(no error reported)"); + } + + LPSTR message_buffer = nullptr; + DWORD size = FormatMessage( + /*dwFlags=*/(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS), + /*lpSource=*/NULL, + /*dwMessageId=*/message_id, + /*dwLanguageId=*/MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + /*lpBuffer=*/(LPSTR)&message_buffer, + /*nSize=*/0, + /*Arguments=*/NULL); + if (size == 0) { + return "(error while trying to format the error message)"; + } + + std::string message(message_buffer, size); + LocalFree(message_buffer); + return NativeToUtf8(message); +} +#else +#include +#include + +std::string FormatLastError() { return strerror(errno); } +#endif // _WIN32 +} // namespace mediapipe diff --git a/mediapipe/framework/deps/platform_strings.h b/mediapipe/framework/deps/platform_strings.h new file mode 100644 index 0000000000..eb6e75f56d --- /dev/null +++ b/mediapipe/framework/deps/platform_strings.h @@ -0,0 +1,46 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_DEPS_PLATFORM_STRINGS_H_ +#define MEDIAPIPE_DEPS_PLATFORM_STRINGS_H_ +#include + +namespace mediapipe { +// `PlatformString` represents a native string type on the platform. +// `Utf8ToNative`/`NativeToUtf8` convert between UTF-8 and that type. +#if defined(_WIN32) && defined(UNICODE) +using PlatformString = std::wstring; + +inline PlatformString Utf8ToNative(const std::string& string) { + std::wstring_convert, wchar_t> converter; + return converter.from_bytes(string.data(), string.data() + string.size()); +} +inline std::string NativeToUtf8(const PlatformString& string) { + std::wstring_convert, wchar_t> converter; + return converter.to_bytes(string.data(), string.data() + string.size()); +} +#define PLATFORM_STRING_LITERAL_INTERNAL(x) L##x +#define PLATFORM_STRING_LITERAL(x) PLATFORM_STRING_LITERAL_INTERNAL(x) +#else +using PlatformString = std::string; + +inline PlatformString Utf8ToNative(const std::string& string) { return string; } +inline std::string NativeToUtf8(const PlatformString& string) { return string; } +#define PLATFORM_STRING_LITERAL(x) x +#endif + +// Produces a human-readable message about the last OS error. +std::string FormatLastError(); +} // namespace mediapipe +#endif // MEDIAPIPE_DEPS_PLATFORM_STRINGS_H_ diff --git a/mediapipe/framework/deps/platform_strings_test.cc b/mediapipe/framework/deps/platform_strings_test.cc new file mode 100644 index 0000000000..4f324b29d9 --- /dev/null +++ b/mediapipe/framework/deps/platform_strings_test.cc @@ -0,0 +1,32 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/deps/platform_strings.h" + +#include + +#include "mediapipe/framework/port/gtest.h" +#include "testing/base/public/gunit.h" + +namespace mediapipe { +namespace { + +TEST(PlatformStrings, ThereAndBack) { + const std::string source = "Шчучыншчына"; + const std::string result = NativeToUtf8(Utf8ToNative(source)); + EXPECT_EQ(result, source); +} + +} // namespace +} // namespace mediapipe From d5e3d514d861e6319d77c3adc221a96c05700d19 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 29 Nov 2024 05:22:36 -0800 Subject: [PATCH 085/126] No public description PiperOrigin-RevId: 701262244 --- mediapipe/calculators/audio/BUILD | 3 ++- mediapipe/calculators/tensorflow/BUILD | 1 + mediapipe/calculators/video/tool/BUILD | 1 + .../java/com/google/mediapipe/apps/instantmotiontracking/BUILD | 2 ++ mediapipe/framework/BUILD | 1 + mediapipe/graphs/instant_motion_tracking/calculators/BUILD | 1 + 6 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mediapipe/calculators/audio/BUILD b/mediapipe/calculators/audio/BUILD index 12223f609a..4d9b083297 100644 --- a/mediapipe/calculators/audio/BUILD +++ b/mediapipe/calculators/audio/BUILD @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") # Placeholder: load py_proto_library +load("@com_google_protobuf//:protobuf.bzl", "proto_library") +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") licenses(["notice"]) diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD index 5b8c48ce7e..43b2324b7a 100644 --- a/mediapipe/calculators/tensorflow/BUILD +++ b/mediapipe/calculators/tensorflow/BUILD @@ -13,6 +13,7 @@ # limitations under the License. # +load("@com_google_protobuf//:protobuf.bzl", "proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") # Placeholder: load py_proto_library diff --git a/mediapipe/calculators/video/tool/BUILD b/mediapipe/calculators/video/tool/BUILD index 2a32c680cf..cfa4ddd306 100644 --- a/mediapipe/calculators/video/tool/BUILD +++ b/mediapipe/calculators/video/tool/BUILD @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@com_google_protobuf//:protobuf.bzl", "proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") licenses(["notice"]) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD index 7b9d61d977..2f25b0a8af 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@com_google_protobuf//:protobuf.bzl", "java_lite_proto_library") + licenses(["notice"]) package(default_visibility = ["//visibility:private"]) diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index 64112cd255..d7966e7273 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -13,6 +13,7 @@ # limitations under the License. load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@com_google_protobuf//:protobuf.bzl", "proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") licenses(["notice"]) diff --git a/mediapipe/graphs/instant_motion_tracking/calculators/BUILD b/mediapipe/graphs/instant_motion_tracking/calculators/BUILD index cdfd911d49..0421f957e6 100644 --- a/mediapipe/graphs/instant_motion_tracking/calculators/BUILD +++ b/mediapipe/graphs/instant_motion_tracking/calculators/BUILD @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@com_google_protobuf//:protobuf.bzl", "proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") licenses(["notice"]) From e8b00c5ff6c958b6d011e430c74e42d1caceedb2 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 29 Nov 2024 07:03:24 -0800 Subject: [PATCH 086/126] No public description PiperOrigin-RevId: 701280691 --- mediapipe/calculators/audio/BUILD | 3 +-- mediapipe/calculators/tensorflow/BUILD | 1 - mediapipe/calculators/video/tool/BUILD | 1 - .../java/com/google/mediapipe/apps/instantmotiontracking/BUILD | 2 -- mediapipe/framework/BUILD | 1 - mediapipe/graphs/instant_motion_tracking/calculators/BUILD | 1 - 6 files changed, 1 insertion(+), 8 deletions(-) diff --git a/mediapipe/calculators/audio/BUILD b/mediapipe/calculators/audio/BUILD index 4d9b083297..12223f609a 100644 --- a/mediapipe/calculators/audio/BUILD +++ b/mediapipe/calculators/audio/BUILD @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder: load py_proto_library -load("@com_google_protobuf//:protobuf.bzl", "proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") +# Placeholder: load py_proto_library licenses(["notice"]) diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD index 43b2324b7a..5b8c48ce7e 100644 --- a/mediapipe/calculators/tensorflow/BUILD +++ b/mediapipe/calculators/tensorflow/BUILD @@ -13,7 +13,6 @@ # limitations under the License. # -load("@com_google_protobuf//:protobuf.bzl", "proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") # Placeholder: load py_proto_library diff --git a/mediapipe/calculators/video/tool/BUILD b/mediapipe/calculators/video/tool/BUILD index cfa4ddd306..2a32c680cf 100644 --- a/mediapipe/calculators/video/tool/BUILD +++ b/mediapipe/calculators/video/tool/BUILD @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@com_google_protobuf//:protobuf.bzl", "proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") licenses(["notice"]) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD index 2f25b0a8af..7b9d61d977 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@com_google_protobuf//:protobuf.bzl", "java_lite_proto_library") - licenses(["notice"]) package(default_visibility = ["//visibility:private"]) diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index d7966e7273..64112cd255 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -13,7 +13,6 @@ # limitations under the License. load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("@com_google_protobuf//:protobuf.bzl", "proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") licenses(["notice"]) diff --git a/mediapipe/graphs/instant_motion_tracking/calculators/BUILD b/mediapipe/graphs/instant_motion_tracking/calculators/BUILD index 0421f957e6..cdfd911d49 100644 --- a/mediapipe/graphs/instant_motion_tracking/calculators/BUILD +++ b/mediapipe/graphs/instant_motion_tracking/calculators/BUILD @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@com_google_protobuf//:protobuf.bzl", "proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") licenses(["notice"]) From 2d2932fd2145d144252acd32d11f81153852bda2 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 2 Dec 2024 01:06:45 -0800 Subject: [PATCH 087/126] No public description PiperOrigin-RevId: 701872170 --- mediapipe/calculators/audio/BUILD | 1 + mediapipe/calculators/tensorflow/BUILD | 1 + mediapipe/calculators/video/tool/BUILD | 1 + .../java/com/google/mediapipe/apps/instantmotiontracking/BUILD | 2 ++ mediapipe/framework/BUILD | 1 + mediapipe/graphs/instant_motion_tracking/calculators/BUILD | 1 + 6 files changed, 7 insertions(+) diff --git a/mediapipe/calculators/audio/BUILD b/mediapipe/calculators/audio/BUILD index 12223f609a..d1c69a2ce6 100644 --- a/mediapipe/calculators/audio/BUILD +++ b/mediapipe/calculators/audio/BUILD @@ -14,6 +14,7 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") # Placeholder: load py_proto_library +# Placeholder: load proto_library licenses(["notice"]) diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD index 5b8c48ce7e..80a4d89eef 100644 --- a/mediapipe/calculators/tensorflow/BUILD +++ b/mediapipe/calculators/tensorflow/BUILD @@ -15,6 +15,7 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") # Placeholder: load py_proto_library +# Placeholder: load proto_library licenses(["notice"]) diff --git a/mediapipe/calculators/video/tool/BUILD b/mediapipe/calculators/video/tool/BUILD index 2a32c680cf..d72aab0c31 100644 --- a/mediapipe/calculators/video/tool/BUILD +++ b/mediapipe/calculators/video/tool/BUILD @@ -14,6 +14,7 @@ # limitations under the License. load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") +# Placeholder: load proto_library licenses(["notice"]) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD index 7b9d61d977..b0f0df7d0f 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Placeholder: load java_lite_proto_library + licenses(["notice"]) package(default_visibility = ["//visibility:private"]) diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index 64112cd255..995a9306e7 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -14,6 +14,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") +# Placeholder: load proto_library licenses(["notice"]) diff --git a/mediapipe/graphs/instant_motion_tracking/calculators/BUILD b/mediapipe/graphs/instant_motion_tracking/calculators/BUILD index cdfd911d49..50128d97fc 100644 --- a/mediapipe/graphs/instant_motion_tracking/calculators/BUILD +++ b/mediapipe/graphs/instant_motion_tracking/calculators/BUILD @@ -13,6 +13,7 @@ # limitations under the License. load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") +# Placeholder: load proto_library licenses(["notice"]) From b746d22f7d57a183742b8cdadf5d95c6635e1071 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 3 Dec 2024 02:34:59 -0800 Subject: [PATCH 088/126] No public description PiperOrigin-RevId: 702261726 --- mediapipe/framework/formats/BUILD | 10 + mediapipe/framework/formats/yuv_image.h | 22 ++- mediapipe/framework/formats/yuv_image_test.cc | 171 ++++++++++++++++++ 3 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 mediapipe/framework/formats/yuv_image_test.cc diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index f73d7e69f1..596f72f0c3 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -688,3 +688,13 @@ cc_test( "//mediapipe/framework/port:status", ], ) + +cc_test( + name = "yuv_image_test", + srcs = ["yuv_image_test.cc"], + deps = [ + ":yuv_image", + "//mediapipe/framework/port:gtest_main", + "@libyuv", + ], +) diff --git a/mediapipe/framework/formats/yuv_image.h b/mediapipe/framework/formats/yuv_image.h index 30a0f303f5..8fccd0e90f 100644 --- a/mediapipe/framework/formats/yuv_image.h +++ b/mediapipe/framework/formats/yuv_image.h @@ -15,9 +15,11 @@ #ifndef MEDIAPIPE_FRAMEWORK_FORMATS_YUV_IMAGE_H_ #define MEDIAPIPE_FRAMEWORK_FORMATS_YUV_IMAGE_H_ +#include #include #include #include +#include #include "libyuv/video_common.h" @@ -121,8 +123,24 @@ class YUVImage { // YUVImage is neither copyable nor movable. YUVImage(const YUVImage&) = delete; YUVImage& operator=(const YUVImage&) = delete; - YUVImage(YUVImage&&) = delete; - YUVImage& operator=(YUVImage&&) = delete; + YUVImage(YUVImage&& b) { *this = std::move(b); } + + YUVImage& operator=(YUVImage&& b) { + if (this != &b) { + Clear(); + deallocation_function_ = std::exchange(b.deallocation_function_, nullptr); + fourcc_ = std::exchange(b.fourcc_, libyuv::FOURCC_ANY); + std::swap_ranges(data_, data_ + kMaxNumPlanes, b.data_); + std::swap_ranges(stride_, stride_ + kMaxNumPlanes, b.stride_); + width_ = std::exchange(b.width_, 0); + height_ = std::exchange(b.height_, 0); + bit_depth_ = std::exchange(b.bit_depth_, 0); + matrix_coefficients_ = std::exchange( + b.matrix_coefficients_, COLOR_MATRIX_COEFFICIENTS_UNSPECIFIED); + full_range_ = std::exchange(b.full_range_, false); + } + return *this; + } // Convenience constructor YUVImage(libyuv::FourCC fourcc, // diff --git a/mediapipe/framework/formats/yuv_image_test.cc b/mediapipe/framework/formats/yuv_image_test.cc new file mode 100644 index 0000000000..ded9f0458a --- /dev/null +++ b/mediapipe/framework/formats/yuv_image_test.cc @@ -0,0 +1,171 @@ +#include "mediapipe/framework/formats/yuv_image.h" + +#include +#include + +#include "libyuv/video_common.h" +#include "mediapipe/framework/port/gtest.h" + +namespace mediapipe { +namespace { + +// See: +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone/use-after-move.html +template +void SILENCE_USE_AFTER_MOVE(T&) {} + +TEST(YUVImageTest, TestInitializeAndDestruct) { + uint8_t data0 = 0, data1 = 1, data2 = 2; + const libyuv::FourCC fourcc = libyuv::FOURCC_I420; + const int stride0 = 100, stride1 = 50, stride2 = 50; + const int width = 100, height = 60; + const int bit_depth = 4; + int deallocation_counter = 0; + auto deallocation_function = [&deallocation_counter] { + ++deallocation_counter; + }; + { + YUVImage yuv_image; + yuv_image.Initialize(fourcc, deallocation_function, // + &data0, stride0, // + &data1, stride1, // + &data2, stride2, // + width, height, bit_depth); + + EXPECT_EQ(yuv_image.fourcc(), fourcc); + EXPECT_EQ(yuv_image.data(0), &data0); + EXPECT_EQ(yuv_image.data(1), &data1); + EXPECT_EQ(yuv_image.data(2), &data2); + EXPECT_EQ(yuv_image.stride(0), stride0); + EXPECT_EQ(yuv_image.stride(1), stride1); + EXPECT_EQ(yuv_image.stride(2), stride2); + EXPECT_EQ(yuv_image.width(), width); + EXPECT_EQ(yuv_image.height(), height); + EXPECT_EQ(yuv_image.bit_depth(), bit_depth); + } + EXPECT_EQ(deallocation_counter, 1); +} + +TEST(YUVImageTest, TestMoveConstructor) { + uint8_t data0 = 0, data1 = 1, data2 = 2; + const libyuv::FourCC fourcc = libyuv::FOURCC_I420; + const int stride0 = 100, stride1 = 50, stride2 = 50; + const int width = 100, height = 60; + const int bit_depth = 4; + int deallocation_counter = 0; + auto deallocation_function = [&deallocation_counter] { + ++deallocation_counter; + }; + { + YUVImage yuv_image; + yuv_image.Initialize(fourcc, deallocation_function, // + &data0, stride0, // + &data1, stride1, // + &data2, stride2, // + width, height, bit_depth); + + EXPECT_EQ(yuv_image.fourcc(), fourcc); + EXPECT_EQ(yuv_image.data(0), &data0); + EXPECT_EQ(yuv_image.data(1), &data1); + EXPECT_EQ(yuv_image.data(2), &data2); + EXPECT_EQ(yuv_image.stride(0), stride0); + EXPECT_EQ(yuv_image.stride(1), stride1); + EXPECT_EQ(yuv_image.stride(2), stride2); + EXPECT_EQ(yuv_image.width(), width); + EXPECT_EQ(yuv_image.height(), height); + EXPECT_EQ(yuv_image.bit_depth(), bit_depth); + + YUVImage yuv_image2(std::move(yuv_image)); + + // ClangTidy will complain about accessing yuv_image after it has been moved + // from. The C++ standard says that "moved-from objects shall be placed in a + // valid but unspecified state". These tests are here to ensure that. + SILENCE_USE_AFTER_MOVE(yuv_image); + EXPECT_EQ(yuv_image.fourcc(), libyuv::FOURCC_ANY); + EXPECT_EQ(yuv_image.data(0), nullptr); + EXPECT_EQ(yuv_image.data(1), nullptr); + EXPECT_EQ(yuv_image.data(2), nullptr); + EXPECT_EQ(yuv_image.stride(0), 0); + EXPECT_EQ(yuv_image.stride(1), 0); + EXPECT_EQ(yuv_image.stride(2), 0); + EXPECT_EQ(yuv_image.width(), 0); + EXPECT_EQ(yuv_image.height(), 0); + EXPECT_EQ(yuv_image.bit_depth(), 0); + + EXPECT_EQ(yuv_image2.fourcc(), fourcc); + EXPECT_EQ(yuv_image2.data(0), &data0); + EXPECT_EQ(yuv_image2.data(1), &data1); + EXPECT_EQ(yuv_image2.data(2), &data2); + EXPECT_EQ(yuv_image2.stride(0), stride0); + EXPECT_EQ(yuv_image2.stride(1), stride1); + EXPECT_EQ(yuv_image2.stride(2), stride2); + EXPECT_EQ(yuv_image2.width(), width); + EXPECT_EQ(yuv_image2.height(), height); + EXPECT_EQ(yuv_image2.bit_depth(), bit_depth); + } + EXPECT_EQ(deallocation_counter, 1); +} + +TEST(YUVImageTest, TestMoveAssignment) { + uint8_t data0 = 0, data1 = 1, data2 = 2; + const libyuv::FourCC fourcc = libyuv::FOURCC_I420; + const int stride0 = 100, stride1 = 50, stride2 = 50; + const int width = 100, height = 60; + const int bit_depth = 4; + int deallocation_counter = 0; + auto deallocation_function = [&deallocation_counter] { + ++deallocation_counter; + }; + { + YUVImage yuv_image; + yuv_image.Initialize(fourcc, deallocation_function, // + &data0, stride0, // + &data1, stride1, // + &data2, stride2, // + width, height, bit_depth); + + EXPECT_EQ(yuv_image.fourcc(), fourcc); + EXPECT_EQ(yuv_image.data(0), &data0); + EXPECT_EQ(yuv_image.data(1), &data1); + EXPECT_EQ(yuv_image.data(2), &data2); + EXPECT_EQ(yuv_image.stride(0), stride0); + EXPECT_EQ(yuv_image.stride(1), stride1); + EXPECT_EQ(yuv_image.stride(2), stride2); + EXPECT_EQ(yuv_image.width(), width); + EXPECT_EQ(yuv_image.height(), height); + EXPECT_EQ(yuv_image.bit_depth(), bit_depth); + + YUVImage yuv_image2; + yuv_image2 = std::move(yuv_image); + + // ClangTidy will complain about accessing yuv_image after it has been moved + // from. The C++ standard says that "moved-from objects shall be placed in a + // valid but unspecified state". These tests are here to ensure that. + SILENCE_USE_AFTER_MOVE(yuv_image); + EXPECT_EQ(yuv_image.fourcc(), libyuv::FOURCC_ANY); + EXPECT_EQ(yuv_image.data(0), nullptr); + EXPECT_EQ(yuv_image.data(1), nullptr); + EXPECT_EQ(yuv_image.data(2), nullptr); + EXPECT_EQ(yuv_image.stride(0), 0); + EXPECT_EQ(yuv_image.stride(1), 0); + EXPECT_EQ(yuv_image.stride(2), 0); + EXPECT_EQ(yuv_image.width(), 0); + EXPECT_EQ(yuv_image.height(), 0); + EXPECT_EQ(yuv_image.bit_depth(), 0); + + EXPECT_EQ(yuv_image2.fourcc(), fourcc); + EXPECT_EQ(yuv_image2.data(0), &data0); + EXPECT_EQ(yuv_image2.data(1), &data1); + EXPECT_EQ(yuv_image2.data(2), &data2); + EXPECT_EQ(yuv_image2.stride(0), stride0); + EXPECT_EQ(yuv_image2.stride(1), stride1); + EXPECT_EQ(yuv_image2.stride(2), stride2); + EXPECT_EQ(yuv_image2.width(), width); + EXPECT_EQ(yuv_image2.height(), height); + EXPECT_EQ(yuv_image2.bit_depth(), bit_depth); + } + EXPECT_EQ(deallocation_counter, 1); +} + +} // namespace +} // namespace mediapipe From c1433ba61a4471f30ae7fc7cfc94a85d003f225e Mon Sep 17 00:00:00 2001 From: Alan Kelly Date: Tue, 3 Dec 2024 08:15:56 -0800 Subject: [PATCH 089/126] No public description PiperOrigin-RevId: 702346276 --- .../utils/xnn_utils/graph_builder.cc | 38 ++++++------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.cc b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.cc index df75149c34..1d31e32e4e 100644 --- a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.cc +++ b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.cc @@ -1187,33 +1187,17 @@ absl::StatusOr> XnnGraphBuilder::Clamp( absl::StatusOr> XnnGraphBuilder::Gelu( std::shared_ptr input) { - // x^2 - MP_ASSIGN_OR_RETURN(auto sqr_out, Square(input)); - - // 0.044715 * x^2 - MP_ASSIGN_OR_RETURN(auto sqr_4471, ElementMul(sqr_out, 0.044715)); - - // 1 + 0.044715 * x^2 - MP_ASSIGN_OR_RETURN(auto sqr_4471_1, ElementAdd(sqr_4471, 1.0f)); - - // x + 0.044715 * x^3 - MP_ASSIGN_OR_RETURN(auto x_cube_4471, ElementMul(sqr_4471_1, input)); - - constexpr float sqrt_2_over_pi = 0.7978845608; - MP_ASSIGN_OR_RETURN(auto sqrt_2_over_pi_x_cube_4471, - ElementMul(x_cube_4471, sqrt_2_over_pi)); - - // tanh(x + 0.044715 * x^3) - MP_ASSIGN_OR_RETURN(auto tanh_x_cube_4471, Tanh(sqrt_2_over_pi_x_cube_4471)); - - // 1 + tanh(x + 0.044715 * x^3) - MP_ASSIGN_OR_RETURN(auto tanh_x_cube_4471_1, - ElementAdd(tanh_x_cube_4471, 1.0f)); - - // 0.5 * (1 + [tanh(x + 0.044715 * x^3)]) - MP_ASSIGN_OR_RETURN(auto cdf, ElementMul(tanh_x_cube_4471_1, 0.5)); - - return ElementMul(input, cdf); + MP_ASSIGN_OR_RETURN(auto output, + IntermediateTensor(input->dims, "gelu_output")); + build_steps_.push_back( + [output, input](xnn_subgraph_t subgraph) -> absl::Status { + RET_CHECK_EQ(xnn_status_success, + xnn_define_gelu(subgraph, input->tensor_id(subgraph), + output->tensor_id(subgraph), + /*flags=*/0)); + return absl::Status(); + }); + return output; } absl::StatusOr> XnnGraphBuilder::Sigmoid( From 65d7c909c5bb035fd02e60485f67f8dd7ef6713c Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 4 Dec 2024 03:25:19 -0800 Subject: [PATCH 090/126] Update comment in yuv_image.h PiperOrigin-RevId: 702659382 --- mediapipe/framework/formats/yuv_image.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/framework/formats/yuv_image.h b/mediapipe/framework/formats/yuv_image.h index 8fccd0e90f..1c516c083e 100644 --- a/mediapipe/framework/formats/yuv_image.h +++ b/mediapipe/framework/formats/yuv_image.h @@ -120,7 +120,7 @@ class YUVImage { YUVImage() = default; ~YUVImage() { Clear(); } - // YUVImage is neither copyable nor movable. + // YUVImage is move-only. YUVImage(const YUVImage&) = delete; YUVImage& operator=(const YUVImage&) = delete; YUVImage(YUVImage&& b) { *this = std::move(b); } From c38e59a15c2f9592e9501cc45d4fcb7fd693bfc6 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 4 Dec 2024 23:58:31 -0800 Subject: [PATCH 091/126] No public description PiperOrigin-RevId: 703001728 --- .../model_maker/python/core/data/classification_dataset_test.py | 2 -- mediapipe/model_maker/python/core/data/data_util_test.py | 2 -- mediapipe/model_maker/python/core/data/dataset.py | 1 - mediapipe/model_maker/python/core/data/dataset_test.py | 2 -- mediapipe/model_maker/python/core/tasks/classifier_test.py | 2 -- mediapipe/model_maker/python/core/tasks/custom_model_test.py | 2 -- mediapipe/model_maker/python/core/utils/model_util.py | 2 -- mediapipe/model_maker/python/core/utils/quantization.py | 2 -- mediapipe/model_maker/python/core/utils/test_util.py | 2 -- .../python/text/text_classifier/text_classifier_demo.py | 2 -- .../python/vision/gesture_recognizer/gesture_recognizer_demo.py | 2 -- .../python/vision/image_classifier/image_classifier_demo.py | 2 -- .../python/vision/object_detector/object_detector_demo.py | 2 -- 13 files changed, 25 deletions(-) diff --git a/mediapipe/model_maker/python/core/data/classification_dataset_test.py b/mediapipe/model_maker/python/core/data/classification_dataset_test.py index dfcea7da6a..ad83651ba0 100644 --- a/mediapipe/model_maker/python/core/data/classification_dataset_test.py +++ b/mediapipe/model_maker/python/core/data/classification_dataset_test.py @@ -14,8 +14,6 @@ from typing import Any, List, Tuple, TypeVar -# Dependency imports - import tensorflow as tf from mediapipe.model_maker.python.core.data import classification_dataset diff --git a/mediapipe/model_maker/python/core/data/data_util_test.py b/mediapipe/model_maker/python/core/data/data_util_test.py index 8bed8ef7c6..fbf9a32dc8 100644 --- a/mediapipe/model_maker/python/core/data/data_util_test.py +++ b/mediapipe/model_maker/python/core/data/data_util_test.py @@ -18,8 +18,6 @@ import os -# Dependency imports - from absl import flags import tensorflow as tf diff --git a/mediapipe/model_maker/python/core/data/dataset.py b/mediapipe/model_maker/python/core/data/dataset.py index 30480995df..2db599aa8d 100644 --- a/mediapipe/model_maker/python/core/data/dataset.py +++ b/mediapipe/model_maker/python/core/data/dataset.py @@ -20,7 +20,6 @@ import functools from typing import Any, Callable, Optional, Tuple, TypeVar -# Dependency imports import tensorflow as tf _DatasetT = TypeVar('_DatasetT', bound='Dataset') diff --git a/mediapipe/model_maker/python/core/data/dataset_test.py b/mediapipe/model_maker/python/core/data/dataset_test.py index 7a3f75388f..f57342cded 100644 --- a/mediapipe/model_maker/python/core/data/dataset_test.py +++ b/mediapipe/model_maker/python/core/data/dataset_test.py @@ -16,8 +16,6 @@ from __future__ import division from __future__ import print_function -# Dependency imports - import numpy as np import tensorflow as tf diff --git a/mediapipe/model_maker/python/core/tasks/classifier_test.py b/mediapipe/model_maker/python/core/tasks/classifier_test.py index 2943825acf..ea88268e97 100644 --- a/mediapipe/model_maker/python/core/tasks/classifier_test.py +++ b/mediapipe/model_maker/python/core/tasks/classifier_test.py @@ -14,8 +14,6 @@ import os -# Dependency imports - import tensorflow as tf from mediapipe.model_maker.python.core.tasks import classifier diff --git a/mediapipe/model_maker/python/core/tasks/custom_model_test.py b/mediapipe/model_maker/python/core/tasks/custom_model_test.py index afb418c445..6641d8c649 100644 --- a/mediapipe/model_maker/python/core/tasks/custom_model_test.py +++ b/mediapipe/model_maker/python/core/tasks/custom_model_test.py @@ -18,8 +18,6 @@ import os -# Dependency imports - import tensorflow as tf from mediapipe.model_maker.python.core.tasks import custom_model diff --git a/mediapipe/model_maker/python/core/utils/model_util.py b/mediapipe/model_maker/python/core/utils/model_util.py index 32b509797f..72271d71a8 100644 --- a/mediapipe/model_maker/python/core/utils/model_util.py +++ b/mediapipe/model_maker/python/core/utils/model_util.py @@ -21,8 +21,6 @@ import tempfile from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union -# Dependency imports - import numpy as np import tensorflow as tf diff --git a/mediapipe/model_maker/python/core/utils/quantization.py b/mediapipe/model_maker/python/core/utils/quantization.py index 2a8d92244b..a58f0386ec 100644 --- a/mediapipe/model_maker/python/core/utils/quantization.py +++ b/mediapipe/model_maker/python/core/utils/quantization.py @@ -19,8 +19,6 @@ from typing import Any, Callable, List, Optional, Union -# Dependency imports - import tensorflow as tf from mediapipe.model_maker.python.core.data import dataset as ds diff --git a/mediapipe/model_maker/python/core/utils/test_util.py b/mediapipe/model_maker/python/core/utils/test_util.py index 72fb229c76..04a343777d 100644 --- a/mediapipe/model_maker/python/core/utils/test_util.py +++ b/mediapipe/model_maker/python/core/utils/test_util.py @@ -19,8 +19,6 @@ from typing import Sequence from typing import Dict, List, Union -# Dependency imports - import numpy as np import tensorflow as tf diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier_demo.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier_demo.py index b646a15ad9..6484e70ffb 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier_demo.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier_demo.py @@ -16,8 +16,6 @@ import os import tempfile -# Dependency imports - from absl import app from absl import flags from absl import logging diff --git a/mediapipe/model_maker/python/vision/gesture_recognizer/gesture_recognizer_demo.py b/mediapipe/model_maker/python/vision/gesture_recognizer/gesture_recognizer_demo.py index 0c1d57d2b7..47cc7b3b74 100644 --- a/mediapipe/model_maker/python/vision/gesture_recognizer/gesture_recognizer_demo.py +++ b/mediapipe/model_maker/python/vision/gesture_recognizer/gesture_recognizer_demo.py @@ -19,8 +19,6 @@ import os -# Dependency imports - from absl import app from absl import flags from absl import logging diff --git a/mediapipe/model_maker/python/vision/image_classifier/image_classifier_demo.py b/mediapipe/model_maker/python/vision/image_classifier/image_classifier_demo.py index 31b6e58768..d71c16ef25 100644 --- a/mediapipe/model_maker/python/vision/image_classifier/image_classifier_demo.py +++ b/mediapipe/model_maker/python/vision/image_classifier/image_classifier_demo.py @@ -15,8 +15,6 @@ import os -# Dependency imports - from absl import app from absl import flags from absl import logging diff --git a/mediapipe/model_maker/python/vision/object_detector/object_detector_demo.py b/mediapipe/model_maker/python/vision/object_detector/object_detector_demo.py index 04820796f0..75929277f9 100644 --- a/mediapipe/model_maker/python/vision/object_detector/object_detector_demo.py +++ b/mediapipe/model_maker/python/vision/object_detector/object_detector_demo.py @@ -14,8 +14,6 @@ """Demo for making an object detector model by MediaPipe Model Maker.""" import os -# Dependency imports - from absl import app from absl import flags from absl import logging From 07f4bbb005b2b88a06bb6531d9cfe498b6e4a61e Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 5 Dec 2024 03:17:38 -0800 Subject: [PATCH 092/126] nits: Remove linter warnings, fix unused includes. PiperOrigin-RevId: 703049617 --- mediapipe/framework/BUILD | 5 +---- mediapipe/framework/validated_graph_config.cc | 7 ++----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index 995a9306e7..d344103414 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -1349,7 +1349,6 @@ cc_library( ":legacy_calculator_support", ":packet_generator", ":packet_generator_cc_proto", - ":packet_set", ":packet_type", ":port", ":status_handler", @@ -1358,7 +1357,6 @@ cc_library( ":subgraph", ":thread_pool_executor_cc_proto", "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", "//mediapipe/framework/port:map_util", "//mediapipe/framework/port:ret_check", @@ -1368,12 +1366,11 @@ cc_library( "//mediapipe/framework/tool:name_util", "//mediapipe/framework/tool:status_util", "//mediapipe/framework/tool:subgraph_expansion", - "//mediapipe/framework/tool:validate", "//mediapipe/framework/tool:validate_name", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", ], diff --git a/mediapipe/framework/validated_graph_config.cc b/mediapipe/framework/validated_graph_config.cc index 370701d0c9..ea541f578b 100644 --- a/mediapipe/framework/validated_graph_config.cc +++ b/mediapipe/framework/validated_graph_config.cc @@ -20,7 +20,7 @@ #include "absl/container/flat_hash_set.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" -#include "absl/memory/memory.h" +#include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/substitute.h" @@ -30,16 +30,14 @@ #include "mediapipe/framework/legacy_calculator_support.h" #include "mediapipe/framework/packet_generator.h" #include "mediapipe/framework/packet_generator.pb.h" -#include "mediapipe/framework/packet_set.h" #include "mediapipe/framework/packet_type.h" #include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/core_proto_inc.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/proto_ns.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/source_location.h" -#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_builder.h" +#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/framework/port/topologicalsorter.h" #include "mediapipe/framework/status_handler.h" #include "mediapipe/framework/stream_handler.pb.h" @@ -47,7 +45,6 @@ #include "mediapipe/framework/tool/name_util.h" #include "mediapipe/framework/tool/status_util.h" #include "mediapipe/framework/tool/subgraph_expansion.h" -#include "mediapipe/framework/tool/validate.h" #include "mediapipe/framework/tool/validate_name.h" namespace mediapipe { From d773f3ed3467d9068dc253b3a8243707744208d6 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 5 Dec 2024 03:19:46 -0800 Subject: [PATCH 093/126] internal PiperOrigin-RevId: 703050070 --- mediapipe/framework/BUILD | 2 ++ mediapipe/framework/validated_graph_config.cc | 1 + 2 files changed, 3 insertions(+) diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index d344103414..f468f6361c 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -1357,6 +1357,7 @@ cc_library( ":subgraph", ":thread_pool_executor_cc_proto", "//mediapipe/framework/port:core_proto", + "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:logging", "//mediapipe/framework/port:map_util", "//mediapipe/framework/port:ret_check", @@ -1368,6 +1369,7 @@ cc_library( "//mediapipe/framework/tool:subgraph_expansion", "//mediapipe/framework/tool:validate_name", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", diff --git a/mediapipe/framework/validated_graph_config.cc b/mediapipe/framework/validated_graph_config.cc index ea541f578b..41a88fb4de 100644 --- a/mediapipe/framework/validated_graph_config.cc +++ b/mediapipe/framework/validated_graph_config.cc @@ -405,6 +405,7 @@ absl::Status ValidatedGraphConfig::Initialize( VLOG(1) << "ValidatedGraphConfig produced canonical config:\n" << config_.DebugString(); #endif + initialized_ = true; return absl::OkStatus(); } From b956ea0974ca21757659e72f414263e907f9f15d Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 5 Dec 2024 05:05:49 -0800 Subject: [PATCH 094/126] No public description PiperOrigin-RevId: 703074432 --- mediapipe/framework/api2/BUILD | 4 + mediapipe/framework/api2/builder.h | 199 ++++++++++++++--------- mediapipe/framework/api2/builder_test.cc | 16 +- 3 files changed, 145 insertions(+), 74 deletions(-) diff --git a/mediapipe/framework/api2/BUILD b/mediapipe/framework/api2/BUILD index bb1b01f205..61806f7331 100644 --- a/mediapipe/framework/api2/BUILD +++ b/mediapipe/framework/api2/BUILD @@ -25,10 +25,14 @@ cc_library( "//mediapipe/framework:stream_handler_cc_proto", "//mediapipe/framework/port:any_proto", "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:source_location", + "//mediapipe/framework/port:status", "@com_google_absl//absl/container:btree", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", "@com_google_protobuf//:protobuf", ], ) diff --git a/mediapipe/framework/api2/builder.h b/mediapipe/framework/api2/builder.h index 0b307326ad..89a501931e 100644 --- a/mediapipe/framework/api2/builder.h +++ b/mediapipe/framework/api2/builder.h @@ -13,6 +13,7 @@ #include "absl/container/btree_map.h" #include "absl/log/absl_check.h" #include "absl/memory/memory.h" +#include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "google/protobuf/message_lite.h" @@ -23,6 +24,8 @@ #include "mediapipe/framework/mediapipe_options.pb.h" #include "mediapipe/framework/port/any_proto.h" #include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_builder.h" +#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/framework/stream_handler.pb.h" namespace mediapipe { @@ -62,26 +65,25 @@ class TagIndexMap { return map_[tag]; } - void Visit(std::function fun) const { - for (const auto& tagged : map_) { - TagIndexLocation loc{tagged.first, 0, tagged.second.size()}; - for (const auto& item : tagged.second) { - fun(loc, *item); - ++loc.index; - } - } - } - - void Visit(std::function fun) { + absl::Status Visit( + std::function fun) const { for (auto& tagged : map_) { TagIndexLocation loc{tagged.first, 0, tagged.second.size()}; - for (auto& item : tagged.second) { - fun(loc, item.get()); - ++loc.index; + for (int i = 0; i < tagged.second.size(); ++i) { + auto* item = tagged.second[i].get(); + loc.index = i; + // If the item is nullptr, it means that the connection vector for + // current tag grew by a GetWithAutoGrow() request but hasn't been + // populated yet. + if (item != nullptr) { + MP_RETURN_IF_ERROR(fun(loc, *item)); + } } } + return absl::OkStatus(); } + private: // Note: entries are held by a unique_ptr to ensure pointers remain valid. // Should use absl::flat_hash_map but ordering keys for now. absl::btree_map>> map_; @@ -857,7 +859,7 @@ class Graph { } } - FixUnnamedConnections(); + ABSL_CHECK_OK(FixUnnamedConnections()); ABSL_CHECK_OK(UpdateBoundaryConfig(&config)); for (const std::unique_ptr& node : nodes_) { auto* out_node = config.add_node(); @@ -871,32 +873,41 @@ class Graph { } private: - void FixUnnamedConnections(NodeBase* node, int* unnamed_count) { - node->out_streams_.Visit([&](const TagIndexLocation&, SourceBase* source) { - if (source->name_.empty()) { - source->name_ = absl::StrCat("__stream_", (*unnamed_count)++); - } - }); - node->out_sides_.Visit([&](const TagIndexLocation&, SourceBase* source) { - if (source->name_.empty()) { - source->name_ = absl::StrCat("__side_packet_", (*unnamed_count)++); - } - }); + absl::Status FixUnnamedConnections(NodeBase* node, int* unnamed_count) { + MP_RETURN_IF_ERROR(node->out_streams_.Visit( + [&](const TagIndexLocation& loc, SourceBase& source) -> absl::Status { + if (source.name_.empty()) { + source.name_ = absl::StrCat("__stream_", (*unnamed_count)++); + } + return absl::OkStatus(); + })); + + MP_RETURN_IF_ERROR(node->out_sides_.Visit( + [&](const TagIndexLocation& loc, SourceBase& source) -> absl::Status { + if (source.name_.empty()) { + source.name_ = absl::StrCat("__side_packet_", (*unnamed_count)++); + } + return absl::OkStatus(); + })); + return absl::OkStatus(); } - void FixUnnamedConnections() { + absl::Status FixUnnamedConnections() { int unnamed_count = 0; - FixUnnamedConnections(&graph_boundary_, &unnamed_count); + MP_RETURN_IF_ERROR(FixUnnamedConnections(&graph_boundary_, &unnamed_count)); for (std::unique_ptr& node : nodes_) { - FixUnnamedConnections(node.get(), &unnamed_count); + MP_RETURN_IF_ERROR(FixUnnamedConnections(node.get(), &unnamed_count)); } for (std::unique_ptr& node : packet_gens_) { - node->out_sides_.Visit([&](const TagIndexLocation&, SourceBase* source) { - if (source->name_.empty()) { - source->name_ = absl::StrCat("__side_packet_", unnamed_count++); - } - }); + MP_RETURN_IF_ERROR(node->out_sides_.Visit( + [&](const TagIndexLocation& loc, SourceBase& source) -> absl::Status { + if (source.name_.empty()) { + source.name_ = absl::StrCat("__side_packet_", unnamed_count++); + } + return absl::OkStatus(); + })); } + return absl::OkStatus(); } std::string TaggedName(const TagIndexLocation& loc, absl::string_view name) { @@ -917,24 +928,39 @@ class Graph { absl::Status UpdateNodeConfig(const NodeBase& node, CalculatorGraphConfig::Node* config) { config->set_calculator(node.type_); - node.in_streams_.Visit( - [&](const TagIndexLocation& loc, const DestinationBase& endpoint) { - ABSL_CHECK(endpoint.source != nullptr); + MP_RETURN_IF_ERROR(node.in_streams_.Visit( + [&](const TagIndexLocation& loc, + const DestinationBase& endpoint) -> absl::Status { + RET_CHECK(endpoint.source != nullptr) + << node.type_ << ": Missing source for input stream with tag " + << (loc.tag.empty() ? "(empty)" : loc.tag) << " at index " + << loc.index; config->add_input_stream(TaggedName(loc, endpoint.source->name_)); - }); - node.out_streams_.Visit( - [&](const TagIndexLocation& loc, const SourceBase& endpoint) { + return absl::OkStatus(); + })); + MP_RETURN_IF_ERROR(node.out_streams_.Visit( + [&](const TagIndexLocation& loc, + const SourceBase& endpoint) -> absl::Status { config->add_output_stream(TaggedName(loc, endpoint.name_)); - }); - node.in_sides_.Visit([&](const TagIndexLocation& loc, - const DestinationBase& endpoint) { - ABSL_CHECK(endpoint.source != nullptr); - config->add_input_side_packet(TaggedName(loc, endpoint.source->name_)); - }); - node.out_sides_.Visit( - [&](const TagIndexLocation& loc, const SourceBase& endpoint) { + return absl::OkStatus(); + })); + MP_RETURN_IF_ERROR(node.in_sides_.Visit( + [&](const TagIndexLocation& loc, + const DestinationBase& endpoint) -> absl::Status { + RET_CHECK(endpoint.source != nullptr) + << node.type_ + << ": Missing source for input side packet stream with tag " + << loc.tag << " at index " << loc.index; + config->add_input_side_packet( + TaggedName(loc, endpoint.source->name_)); + return absl::OkStatus(); + })); + MP_RETURN_IF_ERROR( + node.out_sides_.Visit([&](const TagIndexLocation& loc, + const SourceBase& endpoint) -> absl::Status { config->add_output_side_packet(TaggedName(loc, endpoint.name_)); - }); + return absl::OkStatus(); + })); if (node.calculator_option_.has_value()) { *config->mutable_options() = *node.calculator_option_; } @@ -966,15 +992,24 @@ class Graph { absl::Status UpdateNodeConfig(const PacketGenerator& node, PacketGeneratorConfig* config) { config->set_packet_generator(node.type_); - node.in_sides_.Visit([&](const TagIndexLocation& loc, - const DestinationBase& endpoint) { - ABSL_CHECK(endpoint.source != nullptr); - config->add_input_side_packet(TaggedName(loc, endpoint.source->name_)); - }); - node.out_sides_.Visit( - [&](const TagIndexLocation& loc, const SourceBase& endpoint) { + MP_RETURN_IF_ERROR(node.in_sides_.Visit( + [&](const TagIndexLocation& loc, + const DestinationBase& endpoint) -> absl::Status { + RET_CHECK(endpoint.source != nullptr) + << node.type_ + << ": Missing source for input side packet stream with tag " + << (loc.tag.empty() ? "(empty)" : loc.tag) << " at index " + << loc.index; + config->add_input_side_packet( + TaggedName(loc, endpoint.source->name_)); + return absl::OkStatus(); + })); + MP_RETURN_IF_ERROR( + node.out_sides_.Visit([&](const TagIndexLocation& loc, + const SourceBase& endpoint) -> absl::Status { config->add_output_side_packet(TaggedName(loc, endpoint.name_)); - }); + return absl::OkStatus(); + })); if (node.options_used_) { *config->mutable_options() = node.options_; } @@ -983,25 +1018,43 @@ class Graph { // For special boundary node. absl::Status UpdateBoundaryConfig(CalculatorGraphConfig* config) { - graph_boundary_.in_streams_.Visit( - [&](const TagIndexLocation& loc, const DestinationBase& endpoint) { - ABSL_CHECK(endpoint.source != nullptr); + MP_RETURN_IF_ERROR(graph_boundary_.in_streams_.Visit( + [&](const TagIndexLocation& loc, + const DestinationBase& endpoint) -> absl::Status { + RET_CHECK(endpoint.source != nullptr) + << type_ << ": Missing source for graph output stream with tag " + << (loc.tag.empty() ? "(empty)" : loc.tag) << " at index " + << loc.index; config->add_output_stream(TaggedName(loc, endpoint.source->name_)); - }); - graph_boundary_.out_streams_.Visit( - [&](const TagIndexLocation& loc, const SourceBase& endpoint) { + return absl::OkStatus(); + })); + MP_RETURN_IF_ERROR(graph_boundary_.out_streams_.Visit( + [&](const TagIndexLocation& loc, + const SourceBase& endpoint) -> absl::Status { config->add_input_stream(TaggedName(loc, endpoint.name_)); - }); - graph_boundary_.in_sides_.Visit([&](const TagIndexLocation& loc, - const DestinationBase& endpoint) { - ABSL_CHECK(endpoint.source != nullptr); - config->add_output_side_packet(TaggedName(loc, endpoint.source->name_)); - }); - graph_boundary_.out_sides_.Visit( - [&](const TagIndexLocation& loc, const SourceBase& endpoint) { + return absl::OkStatus(); + })); + MP_RETURN_IF_ERROR(graph_boundary_.in_sides_.Visit( + [&](const TagIndexLocation& loc, + const DestinationBase& endpoint) -> absl::Status { + RET_CHECK(endpoint.source != nullptr) + << type_ + << ": Missing source for graph output side packet stream with " + "tag " + << (loc.tag.empty() ? "(empty)" : loc.tag) << " at index " + << loc.index; + config->add_output_side_packet( + TaggedName(loc, endpoint.source->name_)); + return absl::OkStatus(); + })); + MP_RETURN_IF_ERROR(graph_boundary_.out_sides_.Visit( + [&](const TagIndexLocation& loc, + const SourceBase& endpoint) -> absl::Status { config->add_input_side_packet(TaggedName(loc, endpoint.name_)); - }); - return {}; + + return absl::OkStatus(); + })); + return absl::OkStatus(); } std::string type_; diff --git a/mediapipe/framework/api2/builder_test.cc b/mediapipe/framework/api2/builder_test.cc index da3d7a82ad..c282b84643 100644 --- a/mediapipe/framework/api2/builder_test.cc +++ b/mediapipe/framework/api2/builder_test.cc @@ -1,6 +1,6 @@ #include "mediapipe/framework/api2/builder.h" -#include +#include #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" @@ -903,5 +903,19 @@ TEST(CastTest, FromAnyToAny) { [[maybe_unused]] Stream int_dest = any_inp.Cast(); } +TEST(BuilderTest, CrashWithUsefulMessageIfSkippingInputSource) { + Graph graph; + + auto& multi_node = graph.AddNode("MultiInputsOutputs"); + Stream base = graph.In("IN").SetName("base"); + // We only connect to the second input. Missing source for input stream at + // index 0. + base >> multi_node.In(1); + + EXPECT_DEATH(graph.GetConfig(), + testing::HasSubstr("MultiInputsOutputs: Missing source for " + "input stream with tag (empty) at index 0")); +} + } // namespace } // namespace mediapipe::api2::builder From 66832444565bd5a5a83b64ffc305cd510e4536c5 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 5 Dec 2024 11:17:25 -0800 Subject: [PATCH 095/126] Introduce `shadow_copy` parameter to `PathToResourceAsFile` PiperOrigin-RevId: 703182436 --- mediapipe/util/BUILD | 1 - mediapipe/util/resource_util.h | 12 ++++++------ mediapipe/util/resource_util_android.cc | 17 ++++++++++++----- mediapipe/util/resource_util_apple.cc | 5 +++-- mediapipe/util/resource_util_default.cc | 5 +++-- mediapipe/util/resource_util_emscripten.cc | 6 ++++-- mediapipe/util/resource_util_windows.cc | 5 +++-- 7 files changed, 31 insertions(+), 20 deletions(-) diff --git a/mediapipe/util/BUILD b/mediapipe/util/BUILD index 2d8f851857..d8cccf0b46 100644 --- a/mediapipe/util/BUILD +++ b/mediapipe/util/BUILD @@ -281,7 +281,6 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:singleton", "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", diff --git a/mediapipe/util/resource_util.h b/mediapipe/util/resource_util.h index 40f6839b84..c8a0e93914 100644 --- a/mediapipe/util/resource_util.h +++ b/mediapipe/util/resource_util.h @@ -28,9 +28,9 @@ namespace mediapipe { // - If the input path is an absolute path, it is returned as-is. // - If the input path is relative, it is searched in a platform-specific // location: -// - On Android, we look for an asset with the given relative path; if -// it exists, it is copied to the file system (using the AssetCache), -// and a path to that file is returned. +// - On Android with `shadow_copy`, we look for an asset with the given +// relative path; if it exists, it is copied to the file system (using +// the AssetCache), and a path to that file is returned. // - On iOS, we look for a resource with the given relative path in the // application bundle. // @@ -38,9 +38,9 @@ namespace mediapipe { // Note: This function should be used by code that needs a resource to be // accessible as a normal file, usually to call an existing API that only // accepts file paths. Code that can access data as a stream or as a buffer -// should read from an asset directly on Android; an API for this will be -// provided later. TODO. -absl::StatusOr PathToResourceAsFile(const std::string& path); +// should use the Resources API (see below). +absl::StatusOr PathToResourceAsFile(const std::string& path, + bool shadow_copy = true); // DEPRECATED: use `CalculatorContext::GetResources` and // `SubgraphContext::GetResources` which allow for fine grained per graph diff --git a/mediapipe/util/resource_util_android.cc b/mediapipe/util/resource_util_android.cc index 8678b97312..431b9dca3d 100644 --- a/mediapipe/util/resource_util_android.cc +++ b/mediapipe/util/resource_util_android.cc @@ -15,11 +15,13 @@ #include #include "absl/log/absl_log.h" +#include "absl/status/statusor.h" #include "absl/strings/match.h" +#include "absl/strings/str_cat.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/singleton.h" -#include "mediapipe/framework/port/statusor.h" +#include "mediapipe/framework/port/status_builder.h" #include "mediapipe/util/android/asset_manager_util.h" #include "mediapipe/util/android/file/base/helpers.h" @@ -27,7 +29,11 @@ namespace mediapipe { namespace { absl::StatusOr PathToResourceAsFileInternal( - const std::string& path) { + const std::string& path, bool shadow_copy) { + if (!shadow_copy) { + return absl::UnavailableError(absl::StrCat( + "Not copying asset '", path, "' due to `shadow_copy == false`")); + } return Singleton::get()->CachedFileFromAsset(path); } } // namespace @@ -65,7 +71,8 @@ absl::Status DefaultGetResourceContents(const std::string& path, } } // namespace internal -absl::StatusOr PathToResourceAsFile(const std::string& path) { +absl::StatusOr PathToResourceAsFile(const std::string& path, + bool shadow_copy) { // Return full path. if (absl::StartsWith(path, "/")) { return path; @@ -73,7 +80,7 @@ absl::StatusOr PathToResourceAsFile(const std::string& path) { // Try to load a relative path or a base filename as is. { - auto status_or_path = PathToResourceAsFileInternal(path); + auto status_or_path = PathToResourceAsFileInternal(path, shadow_copy); if (status_or_path.ok()) { ABSL_LOG(INFO) << "Successfully loaded: " << path; return status_or_path; @@ -86,7 +93,7 @@ absl::StatusOr PathToResourceAsFile(const std::string& path) { RET_CHECK(last_slash_idx != std::string::npos) << path << " doesn't have a slash in it"; // Make sure it's a path. auto base_name = path.substr(last_slash_idx + 1); - auto status_or_path = PathToResourceAsFileInternal(base_name); + auto status_or_path = PathToResourceAsFileInternal(base_name, shadow_copy); if (status_or_path.ok()) { ABSL_LOG(INFO) << "Successfully loaded: " << base_name; return status_or_path; diff --git a/mediapipe/util/resource_util_apple.cc b/mediapipe/util/resource_util_apple.cc index b78be35824..d2ba375b4b 100644 --- a/mediapipe/util/resource_util_apple.cc +++ b/mediapipe/util/resource_util_apple.cc @@ -18,10 +18,10 @@ #include #include "absl/log/absl_log.h" +#include "absl/status/statusor.h" #include "absl/strings/match.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/statusor.h" #include "mediapipe/util/resource_util.h" namespace mediapipe { @@ -55,7 +55,8 @@ absl::Status DefaultGetResourceContents(const std::string& path, } } // namespace internal -absl::StatusOr PathToResourceAsFile(const std::string& path) { +absl::StatusOr PathToResourceAsFile(const std::string& path, + bool /*shadow_copy*/) { // Return full path. if (absl::StartsWith(path, "/")) { return path; diff --git a/mediapipe/util/resource_util_default.cc b/mediapipe/util/resource_util_default.cc index 3ebbd1f34b..9197295481 100644 --- a/mediapipe/util/resource_util_default.cc +++ b/mediapipe/util/resource_util_default.cc @@ -15,9 +15,9 @@ #include #include "absl/flags/flag.h" +#include "absl/status/statusor.h" #include "mediapipe/framework/deps/file_path.h" #include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/statusor.h" ABSL_FLAG( std::string, resource_root_dir, "", @@ -38,7 +38,8 @@ absl::Status DefaultGetResourceContents(const std::string& path, } } // namespace internal -absl::StatusOr PathToResourceAsFile(const std::string& path) { +absl::StatusOr PathToResourceAsFile(const std::string& path, + bool /*shadow_copy*/) { if (absl::StartsWith(path, "/")) { return path; } diff --git a/mediapipe/util/resource_util_emscripten.cc b/mediapipe/util/resource_util_emscripten.cc index 1243ad115f..ca3d386dc4 100644 --- a/mediapipe/util/resource_util_emscripten.cc +++ b/mediapipe/util/resource_util_emscripten.cc @@ -16,14 +16,16 @@ #include #include "absl/log/absl_log.h" +#include "absl/status/statusor.h" #include "absl/strings/str_format.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/statusor.h" +#include "mediapipe/util/resource_util.h" namespace mediapipe { -absl::StatusOr PathToResourceAsFile(const std::string& path) { +absl::StatusOr PathToResourceAsFile(const std::string& path, + bool /*shadow_copy*/) { if (absl::StartsWith(path, "/")) { return path; } diff --git a/mediapipe/util/resource_util_windows.cc b/mediapipe/util/resource_util_windows.cc index 6d620e58c4..e28094eef3 100644 --- a/mediapipe/util/resource_util_windows.cc +++ b/mediapipe/util/resource_util_windows.cc @@ -15,10 +15,10 @@ #include #include "absl/flags/flag.h" +#include "absl/status/statusor.h" #include "mediapipe/framework/deps/file_path.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/singleton.h" -#include "mediapipe/framework/port/statusor.h" #include "tools/cpp/runfiles/runfiles.h" ABSL_FLAG( @@ -70,7 +70,8 @@ absl::Status DefaultGetResourceContents(const std::string& path, } // namespace internal -absl::StatusOr PathToResourceAsFile(const std::string& path) { +absl::StatusOr PathToResourceAsFile(const std::string& path, + bool /*shadow_copy*/) { std::string qualified_path = path; if (absl::StartsWith(qualified_path, "./")) { qualified_path = "mediapipe" + qualified_path.substr(1); From f4f40e5e766540710cedce260f4a43b8751875dc Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 5 Dec 2024 13:45:35 -0800 Subject: [PATCH 096/126] Fix header includes after refactoring PiperOrigin-RevId: 703237773 --- mediapipe/framework/deps/file_helpers.cc | 3 --- mediapipe/framework/deps/platform_strings.h | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mediapipe/framework/deps/file_helpers.cc b/mediapipe/framework/deps/file_helpers.cc index c68c46324e..25a1b61b8f 100644 --- a/mediapipe/framework/deps/file_helpers.cc +++ b/mediapipe/framework/deps/file_helpers.cc @@ -19,9 +19,6 @@ #ifdef _WIN32 #include #include - -#include -#include #else #include #endif // _WIN32 diff --git a/mediapipe/framework/deps/platform_strings.h b/mediapipe/framework/deps/platform_strings.h index eb6e75f56d..6ff4fdbb2f 100644 --- a/mediapipe/framework/deps/platform_strings.h +++ b/mediapipe/framework/deps/platform_strings.h @@ -14,6 +14,8 @@ #ifndef MEDIAPIPE_DEPS_PLATFORM_STRINGS_H_ #define MEDIAPIPE_DEPS_PLATFORM_STRINGS_H_ +#include +#include #include namespace mediapipe { From 29e87ad23f2803b5dc6c0f998e7c24a0405006fe Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 5 Dec 2024 14:28:27 -0800 Subject: [PATCH 097/126] Migrate away from status builders PiperOrigin-RevId: 703253302 --- mediapipe/framework/deps/file_helpers.cc | 35 ++++++++++-------------- mediapipe/framework/deps/file_helpers.h | 8 +++--- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/mediapipe/framework/deps/file_helpers.cc b/mediapipe/framework/deps/file_helpers.cc index 25a1b61b8f..b889867203 100644 --- a/mediapipe/framework/deps/file_helpers.cc +++ b/mediapipe/framework/deps/file_helpers.cc @@ -14,8 +14,6 @@ #include "mediapipe/framework/deps/file_helpers.h" -#include "absl/strings/str_cat.h" - #ifdef _WIN32 #include #include @@ -30,12 +28,10 @@ #include #include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/deps/file_path.h" #include "mediapipe/framework/deps/platform_strings.h" // IWYU pragma: keep -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_builder.h" #include "mediapipe/framework/port/status_macros.h" namespace mediapipe { @@ -151,8 +147,7 @@ absl::Status GetContents(absl::string_view path, std::string* output, bool read_as_binary) { FILE* fp = fopen(std::string(path).c_str(), read_as_binary ? "rb" : "r"); if (fp == NULL) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Can't find file: " << path; + return absl::InvalidArgumentError(absl::StrCat("Can't find file: ", path)); } output->clear(); @@ -160,8 +155,8 @@ absl::Status GetContents(absl::string_view path, std::string* output, char buf[4096]; size_t ret = fread(buf, 1, 4096, fp); if (ret == 0 && ferror(fp)) { - return mediapipe::UnavailableErrorBuilder(MEDIAPIPE_LOC) - << "Error while reading file: " << path; + return absl::UnavailableError( + absl::StrCat("Error while reading file: ", path)); } output->append(std::string(buf, ret)); } @@ -172,16 +167,15 @@ absl::Status GetContents(absl::string_view path, std::string* output, absl::Status SetContents(absl::string_view path, absl::string_view content) { FILE* fp = fopen(std::string(path).c_str(), "wb"); if (fp == NULL) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Can't open file: " << path; + return absl::InvalidArgumentError(absl::StrCat("Can't open file: ", path)); } fwrite(content.data(), sizeof(char), content.size(), fp); size_t write_error = ferror(fp); if (fclose(fp) != 0 || write_error) { - return mediapipe::UnavailableErrorBuilder(MEDIAPIPE_LOC) - << "Error while writing file: " << path - << ". Error message: " << strerror(write_error); + return absl::UnavailableError( + absl::StrCat("Error while writing file: ", path, + ". Error message: ", strerror(write_error))); } return absl::OkStatus(); } @@ -190,16 +184,15 @@ absl::Status AppendStringToFile(absl::string_view path, absl::string_view contents) { FILE* fp = fopen(std::string(path).c_str(), "ab"); if (!fp) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Can't open file: " << path; + return absl::InvalidArgumentError(absl::StrCat("Can't open file: ", path)); } fwrite(contents.data(), sizeof(char), contents.size(), fp); size_t write_error = ferror(fp); if (fclose(fp) != 0 || write_error) { - return mediapipe::UnavailableErrorBuilder(MEDIAPIPE_LOC) - << "Error while writing file: " << path - << ". Error message: " << strerror(write_error); + return absl::UnavailableError( + absl::StrCat("Error while writing file: ", path, + ". Error message: ", strerror(write_error))); } return absl::OkStatus(); } @@ -252,7 +245,7 @@ absl::Status Exists(absl::string_view file_name) { } switch (errno) { case EACCES: - return mediapipe::PermissionDeniedError("Insufficient permissions."); + return absl::PermissionDeniedError("Insufficient permissions."); default: return absl::NotFoundError( absl::StrCat("The path does not exist: ", file_name)); @@ -294,7 +287,7 @@ absl::Status RecursivelyCreateDir(absl::string_view path) { if (mkdir(std::string(path)) != 0) { switch (errno) { case EACCES: - return mediapipe::PermissionDeniedError("Insufficient permissions."); + return absl::PermissionDeniedError("Insufficient permissions."); default: return absl::UnavailableError("Failed to create directory."); } diff --git a/mediapipe/framework/deps/file_helpers.h b/mediapipe/framework/deps/file_helpers.h index 2725d6c1d2..146ea5daf1 100644 --- a/mediapipe/framework/deps/file_helpers.h +++ b/mediapipe/framework/deps/file_helpers.h @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef MEDIAPIPE_DEPS_FILE_HELPERS_H_ -#define MEDIAPIPE_DEPS_FILE_HELPERS_H_ +#ifndef MEDIAPIPE_FRAMEWORK_DEPS_FILE_HELPERS_H_ +#define MEDIAPIPE_FRAMEWORK_DEPS_FILE_HELPERS_H_ +#include "absl/status/status.h" #include "absl/strings/match.h" -#include "mediapipe/framework/port/status.h" namespace mediapipe { namespace file { @@ -46,4 +46,4 @@ absl::Status RecursivelyCreateDir(absl::string_view path); } // namespace file } // namespace mediapipe -#endif // MEDIAPIPE_DEPS_FILE_HELPERS_H_ +#endif // MEDIAPIPE_FRAMEWORK_DEPS_FILE_HELPERS_H_ From 5fbb13d333889b19a34f6937697ef5cb29ccc4e4 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 6 Dec 2024 02:56:11 -0800 Subject: [PATCH 098/126] Avoids the creation of two "default" GpuExecutor instances PiperOrigin-RevId: 703430623 --- mediapipe/gpu/gpu_shared_data_internal.cc | 23 +++++++++++++++++++---- mediapipe/gpu/graph_support.h | 1 - 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/mediapipe/gpu/gpu_shared_data_internal.cc b/mediapipe/gpu/gpu_shared_data_internal.cc index a6838d238a..bf8086b49f 100644 --- a/mediapipe/gpu/gpu_shared_data_internal.cc +++ b/mediapipe/gpu/gpu_shared_data_internal.cc @@ -42,6 +42,17 @@ namespace mediapipe { +namespace { + +inline constexpr char kGpuExecutorName[] = "__gpu"; + +// Returns the executor name from a context key . +std::string GetExecutorNameFromContextKey(const std::string& context_key) { + return absl::StrCat(kGpuExecutorName, "_", context_key); +} + +} // namespace + #if __APPLE__ static constexpr bool kGlContextUseDedicatedThread = false; #elif defined(__EMSCRIPTEN__) @@ -130,7 +141,9 @@ GpuResources::GpuResources(std::shared_ptr gl_context, #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER { gl_key_context_->insert({SharedContextKey(), gl_context}); - named_executors_[kGpuExecutorName] = + const std::string executor_name = + GetExecutorNameFromContextKey(SharedContextKey()); + named_executors_[executor_name] = std::make_shared(gl_context.get()); #if __APPLE__ #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER @@ -144,7 +157,9 @@ GpuResources::GpuResources(std::shared_ptr gl_context, absl::StatusOr> GpuResources::GetDefaultGpuExecutor() const { - const auto it = named_executors_.find(kGpuExecutorName); + const std::string executor_name = + GetExecutorNameFromContextKey(SharedContextKey()); + const auto it = named_executors_.find(executor_name); RET_CHECK(it != named_executors_.end()) << "Can't find default gpu executor."; return it->second; } @@ -209,8 +224,8 @@ absl::Status GpuResources::PrepareGpuNode(CalculatorNode* node) { GetOrCreateGlContext(context_key)); if (kGlContextUseDedicatedThread) { - std::string executor_name = - absl::StrCat(kGpuExecutorName, "_", context_key); + const std::string executor_name = + GetExecutorNameFromContextKey(context_key); node->SetExecutor(executor_name); if (!ContainsKey(named_executors_, executor_name)) { named_executors_.emplace( diff --git a/mediapipe/gpu/graph_support.h b/mediapipe/gpu/graph_support.h index df8c27dff6..df20ef829b 100644 --- a/mediapipe/gpu/graph_support.h +++ b/mediapipe/gpu/graph_support.h @@ -20,7 +20,6 @@ namespace mediapipe { inline constexpr char kGpuSharedTagName[] = "GPU_SHARED"; inline constexpr char kGpuSharedSidePacketName[] = "gpu_shared"; -inline constexpr char kGpuExecutorName[] = "__gpu"; } // namespace mediapipe From 531d544ce597209d228b907a0719a28e055d9edc Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 6 Dec 2024 04:03:24 -0800 Subject: [PATCH 099/126] Adds and integrates GraphRuntimeInfoLogger into CalculatorGraph. PiperOrigin-RevId: 703446598 --- mediapipe/framework/BUILD | 1 + mediapipe/framework/calculator.proto | 16 +++++ mediapipe/framework/calculator_graph.cc | 17 +++++ mediapipe/framework/calculator_graph.h | 9 +++ mediapipe/framework/tool/BUILD | 35 +++++++++++ .../tool/graph_runtime_info_logger.cc | 62 +++++++++++++++++++ .../tool/graph_runtime_info_logger.h | 51 +++++++++++++++ .../tool/graph_runtime_info_logger_test.cc | 42 +++++++++++++ 8 files changed, 233 insertions(+) create mode 100644 mediapipe/framework/tool/graph_runtime_info_logger.cc create mode 100644 mediapipe/framework/tool/graph_runtime_info_logger.h create mode 100644 mediapipe/framework/tool/graph_runtime_info_logger_test.cc diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index f468f6361c..f5fc73815c 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -369,6 +369,7 @@ cc_library( "//mediapipe/framework/port:source_location", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:fill_packet_set", + "//mediapipe/framework/tool:graph_runtime_info_logger", "//mediapipe/framework/tool:packet_generator_wrapper_calculator", "//mediapipe/framework/tool:status_util", "//mediapipe/framework/tool:tag_map", diff --git a/mediapipe/framework/calculator.proto b/mediapipe/framework/calculator.proto index 9984ee2505..daa2d594e6 100644 --- a/mediapipe/framework/calculator.proto +++ b/mediapipe/framework/calculator.proto @@ -217,6 +217,19 @@ message ProfilerConfig { string calculator_filter = 18; } +// Configuration for the runtime info logger. It collects runtime information +// and statistics about calculators and their input streams at the configured +// capture rate and writes them to LOG(INFO). It can be used to inspect a +// stalled graph by understanding which calculators are waiting for input +// packets to triger their Process() method. +message GraphRuntimeInfoConfig { + // If true, the runtime info logger is enabled and runs in the background. + bool enable_graph_runtime_info = 1; + // The period in seconds at which the runtime info logger is updated. The + // default value is 10 secods. + uint32 capture_period_msec = 2; +} + // Describes the topology and function of a MediaPipe Graph. The graph of // Nodes must be a Directed Acyclic Graph (DAG) except as annotated by // "back_edge" in InputStreamInfo. Use a mediapipe::CalculatorGraph object to @@ -392,6 +405,9 @@ message CalculatorGraphConfig { // calculators from running. If false, max_queue_size for an input stream // is adjusted when throttling prevents all calculators from running. bool report_deadlock = 21; + // Enable the collection of runtime information and statistics about + // calculators and their input streams. + GraphRuntimeInfoConfig runtime_info = 22; // Config for this graph's InputStreamHandler. // If unspecified, the framework will automatically install the default // handler, which works as follows. diff --git a/mediapipe/framework/calculator_graph.cc b/mediapipe/framework/calculator_graph.cc index 6cacd8117b..ed6d438d37 100644 --- a/mediapipe/framework/calculator_graph.cc +++ b/mediapipe/framework/calculator_graph.cc @@ -460,6 +460,23 @@ absl::Status CalculatorGraph::Initialize( #endif initialized_ = true; + + // Emscripten only supports single threaded applications. + const auto& runtime_info_logger_config = + validated_graph_->Config().runtime_info(); +#if !defined(__EMSCRIPTEN__) + if (runtime_info_logger_config.enable_graph_runtime_info()) { + MP_RETURN_IF_ERROR(graph_runtime_info_logger_.StartInBackground( + runtime_info_logger_config, + [this]() { return GetGraphRuntimeInfo(); })); + } +#else + // TODO - remove once graph runtime infos are supported in + // Emscripten. + if (runtime_info_logger_config.enable_graph_runtime_info()) { + ABSL_LOG(WARNING) << "Graph runtime infos are not supported in Emscripten."; + } +#endif // defined(__EMSCRIPTEN__) return absl::OkStatus(); } diff --git a/mediapipe/framework/calculator_graph.h b/mediapipe/framework/calculator_graph.h index a128d1b551..bc11ae9fdc 100644 --- a/mediapipe/framework/calculator_graph.h +++ b/mediapipe/framework/calculator_graph.h @@ -58,6 +58,10 @@ #include "mediapipe/framework/timestamp.h" #include "mediapipe/framework/validated_graph_config.h" +#if !defined(__EMSCRIPTEN__) +#include "mediapipe/framework/tool/graph_runtime_info_logger.h" +#endif // !defined(__EMSCRIPTEN__) + namespace mediapipe { #if !MEDIAPIPE_DISABLE_GPU @@ -770,6 +774,11 @@ class CalculatorGraph { std::shared_ptr profiler_; internal::Scheduler scheduler_; + +#if !defined(__EMSCRIPTEN__) + // Collects runtime information about the graph in the background. + tool::GraphRuntimeInfoLogger graph_runtime_info_logger_; +#endif // !defined(__EMSCRIPTEN__) }; } // namespace mediapipe diff --git a/mediapipe/framework/tool/BUILD b/mediapipe/framework/tool/BUILD index 24cd3cf558..1b05e71e85 100644 --- a/mediapipe/framework/tool/BUILD +++ b/mediapipe/framework/tool/BUILD @@ -978,6 +978,41 @@ cc_test( ], ) +cc_library( + name = "graph_runtime_info_logger", + srcs = ["graph_runtime_info_logger.cc"], + hdrs = ["graph_runtime_info_logger.h"], + visibility = ["//visibility:public"], + deps = [ + ":graph_runtime_info_utils", + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/framework:graph_runtime_info_cc_proto", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:threadpool", + "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + ], +) + +cc_test( + name = "graph_runtime_info_logger_test", + size = "small", + srcs = ["graph_runtime_info_logger_test.cc"], + deps = [ + ":graph_runtime_info_logger", + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:status_matchers", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + ], +) + cc_library( name = "graph_runtime_info_utils", srcs = ["graph_runtime_info_utils.cc"], diff --git a/mediapipe/framework/tool/graph_runtime_info_logger.cc b/mediapipe/framework/tool/graph_runtime_info_logger.cc new file mode 100644 index 0000000000..5d85292ad4 --- /dev/null +++ b/mediapipe/framework/tool/graph_runtime_info_logger.cc @@ -0,0 +1,62 @@ +#include "mediapipe/framework/tool/graph_runtime_info_logger.h" + +#include +#include + +#include "absl/functional/any_invocable.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/time/time.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/tool/graph_runtime_info_utils.h" + +namespace mediapipe::tool { + +constexpr absl::Duration kDefaultCaptureInterval = absl::Seconds(10); + +GraphRuntimeInfoLogger::GraphRuntimeInfoLogger() + : thread_pool_("GraphRuntimeInfoLogger", 1) {} + +GraphRuntimeInfoLogger::~GraphRuntimeInfoLogger() { Stop(); }; + +absl::Status GraphRuntimeInfoLogger::StartInBackground( + const mediapipe::GraphRuntimeInfoConfig& config, + absl::AnyInvocable()> + get_runtime_info_fn) { + get_runtime_info_fn_ = std::move(get_runtime_info_fn); + RET_CHECK(!is_running_.HasBeenNotified()); + ABSL_CHECK_EQ(thread_pool_.num_threads(), 1); + thread_pool_.StartWorkers(); + absl::Duration interval = + config.capture_period_msec() > 0 + ? absl::Milliseconds(config.capture_period_msec()) + : kDefaultCaptureInterval; + thread_pool_.Schedule([this, interval]() mutable { + is_running_.Notify(); + while (!shutdown_signal_.HasBeenNotified()) { + const auto runtime_info = get_runtime_info_fn_(); + if (!runtime_info.ok()) { + ABSL_LOG(DFATAL) << "Failed to get graph runtime info: " + << runtime_info.status(); + return; + } + const auto runtime_info_str = GetGraphRuntimeInfoString(*runtime_info); + if (!runtime_info_str.ok()) { + ABSL_LOG(DFATAL) << "Failed to render graph runtime info: " + << runtime_info_str.status(); + return; + } + ABSL_LOG(INFO) << *runtime_info_str; + shutdown_signal_.WaitForNotificationWithTimeout(interval); + } + }); + is_running_.WaitForNotification(); + return absl::OkStatus(); +} + +void GraphRuntimeInfoLogger::Stop() { shutdown_signal_.Notify(); } + +} // namespace mediapipe::tool diff --git a/mediapipe/framework/tool/graph_runtime_info_logger.h b/mediapipe/framework/tool/graph_runtime_info_logger.h new file mode 100644 index 0000000000..401b58cc68 --- /dev/null +++ b/mediapipe/framework/tool/graph_runtime_info_logger.h @@ -0,0 +1,51 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_TOOL_GRAPH_RUNTIME_INFO_LOGGER_H_ +#define MEDIAPIPE_FRAMEWORK_TOOL_GRAPH_RUNTIME_INFO_LOGGER_H_ + +#include "absl/functional/any_invocable.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/synchronization/notification.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/graph_runtime_info.pb.h" +#include "mediapipe/framework/port/threadpool.h" + +namespace mediapipe::tool { + +// Periodically collects the graph runtime info and output it to LOG(INFO). +class GraphRuntimeInfoLogger { + public: + GraphRuntimeInfoLogger(); + ~GraphRuntimeInfoLogger(); + + // Starts the collector in the background. Can be called only once. + absl::Status StartInBackground( + const mediapipe::GraphRuntimeInfoConfig& config, + absl::AnyInvocable()> + get_runtime_info_fn); + + private: + void Stop(); + + absl::Notification shutdown_signal_; + absl::Notification is_running_; + ThreadPool thread_pool_; + absl::AnyInvocable()> get_runtime_info_fn_; +}; + +} // namespace mediapipe::tool + +#endif // MEDIAPIPE_FRAMEWORK_TOOL_GRAPH_RUNTIME_INFO_LOGGER_H_ diff --git a/mediapipe/framework/tool/graph_runtime_info_logger_test.cc b/mediapipe/framework/tool/graph_runtime_info_logger_test.cc new file mode 100644 index 0000000000..d9b56b88a3 --- /dev/null +++ b/mediapipe/framework/tool/graph_runtime_info_logger_test.cc @@ -0,0 +1,42 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/tool/graph_runtime_info_logger.h" + +#include "absl/synchronization/notification.h" +#include "absl/time/time.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/status_matchers.h" + +namespace mediapipe::tool { +namespace { + +TEST(GraphRuntimeInfoLoggerTest, ShouldCaptureRuntimeInfo) { + mediapipe::GraphRuntimeInfoConfig config; + config.set_enable_graph_runtime_info(true); + + absl::Notification callback_called; + GraphRuntimeInfoLogger logger; + MP_ASSERT_OK(logger.StartInBackground(config, [&]() { + callback_called.Notify(); + return GraphRuntimeInfo(); + })); + EXPECT_TRUE( + callback_called.WaitForNotificationWithTimeout(absl::Seconds(10))); +} + +} // namespace +} // namespace mediapipe::tool From f264ea854335696605d68417b6e20971f3824abe Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 6 Dec 2024 04:07:30 -0800 Subject: [PATCH 100/126] nit: don't overwrite InitializeDefaultExecutor argument "use_application_thread" PiperOrigin-RevId: 703447942 --- mediapipe/framework/calculator_graph.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mediapipe/framework/calculator_graph.cc b/mediapipe/framework/calculator_graph.cc index ed6d438d37..6c391b7d66 100644 --- a/mediapipe/framework/calculator_graph.cc +++ b/mediapipe/framework/calculator_graph.cc @@ -395,6 +395,11 @@ absl::Status CalculatorGraph::InitializeExecutors() { MEDIAPIPE_CHECK_OK(SetExecutorInternal( executor_config.name(), std::shared_ptr(executor))); } +#ifdef __EMSCRIPTEN__ + // Emscripten runs the application single threaded and therefore requires to + // use the application thread. + use_application_thread = true; +#endif // __EMSCRIPTEN__ if (!mediapipe::ContainsKey(executors_, "")) { MP_RETURN_IF_ERROR(InitializeDefaultExecutor(default_executor_options, @@ -408,7 +413,9 @@ absl::Status CalculatorGraph::InitializeDefaultExecutor( const ThreadPoolExecutorOptions* default_executor_options, bool use_application_thread) { #ifdef __EMSCRIPTEN__ - use_application_thread = true; + // Emscripten runs the application single threaded and therefore requires to + // use the application thread. + RET_CHECK(use_application_thread); #endif // __EMSCRIPTEN__ // If specified, run synchronously on the calling thread. if (use_application_thread) { From 12e338981f6356d6527a3ab789bba278bc5fad34 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 6 Dec 2024 06:22:46 -0800 Subject: [PATCH 101/126] Add Name() access to the source names in api2. PiperOrigin-RevId: 703479142 --- mediapipe/framework/api2/builder.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mediapipe/framework/api2/builder.h b/mediapipe/framework/api2/builder.h index 89a501931e..b2d7feb22e 100644 --- a/mediapipe/framework/api2/builder.h +++ b/mediapipe/framework/api2/builder.h @@ -232,6 +232,8 @@ class SourceImpl { return !(*this == other); } + const std::string& Name() const { return base_->name_; } + Src& SetName(const char* name) { base_->name_ = std::string(name); return *this; From a567a5f0ea3d5b2335228e21e688b8296c0bfe29 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 6 Dec 2024 07:38:53 -0800 Subject: [PATCH 102/126] No public description PiperOrigin-RevId: 703496331 --- mediapipe/framework/calculator_graph.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mediapipe/framework/calculator_graph.cc b/mediapipe/framework/calculator_graph.cc index 6c391b7d66..dfaf0230c7 100644 --- a/mediapipe/framework/calculator_graph.cc +++ b/mediapipe/framework/calculator_graph.cc @@ -468,16 +468,18 @@ absl::Status CalculatorGraph::Initialize( initialized_ = true; +#if !defined(__EMSCRIPTEN__) // Emscripten only supports single threaded applications. const auto& runtime_info_logger_config = validated_graph_->Config().runtime_info(); -#if !defined(__EMSCRIPTEN__) if (runtime_info_logger_config.enable_graph_runtime_info()) { MP_RETURN_IF_ERROR(graph_runtime_info_logger_.StartInBackground( runtime_info_logger_config, [this]() { return GetGraphRuntimeInfo(); })); } #else + const auto& runtime_info_logger_config = + validated_graph_->Config().runtime_info(); // TODO - remove once graph runtime infos are supported in // Emscripten. if (runtime_info_logger_config.enable_graph_runtime_info()) { From 0c4adef0caa9ce71c89f74ad37116a0d23f463bd Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 6 Dec 2024 08:14:26 -0800 Subject: [PATCH 103/126] No public description PiperOrigin-RevId: 703504749 --- mediapipe/gpu/BUILD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index ef1768d9e9..c8fa2ad81e 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -570,10 +570,16 @@ cc_library( "//mediapipe/framework/formats:shared_fd", "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", "//mediapipe/util:sync_wait", "//third_party/GL:EGL_headers", + "@com_google_absl//absl/base", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/time", ] + select({ "//conditions:default": [], "//mediapipe:android": [ From 0bd500eb2936b8e9457fa8d9a92f11b91686222f Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 6 Dec 2024 08:52:56 -0800 Subject: [PATCH 104/126] Add memory mapping and locking to file helpers PiperOrigin-RevId: 703514385 --- mediapipe/framework/BUILD | 6 + mediapipe/framework/deps/BUILD | 27 ++++ mediapipe/framework/deps/file_helpers.cc | 186 +++++++++++++++++++++- mediapipe/framework/deps/file_helpers.h | 8 + mediapipe/framework/deps/mlock_helpers.cc | 53 ++++++ mediapipe/framework/deps/mlock_helpers.h | 11 ++ mediapipe/framework/deps/mmapped_file.h | 46 ++++++ mediapipe/framework/port/BUILD | 1 + 8 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 mediapipe/framework/deps/mlock_helpers.cc create mode 100644 mediapipe/framework/deps/mlock_helpers.h create mode 100644 mediapipe/framework/deps/mmapped_file.h diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index f5fc73815c..7ecac979ba 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -681,9 +681,15 @@ cc_library( hdrs = ["resources.h"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework/deps:mlock_helpers", + "//mediapipe/framework/deps:mmapped_file", + "//mediapipe/framework/port:file_helpers", + "//mediapipe/framework/port:logging", + "//mediapipe/framework/port:status", "//mediapipe/framework/tool:status_util", "//mediapipe/util:resource_util", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/mediapipe/framework/deps/BUILD b/mediapipe/framework/deps/BUILD index 1b87352367..da9835037d 100644 --- a/mediapipe/framework/deps/BUILD +++ b/mediapipe/framework/deps/BUILD @@ -117,6 +117,15 @@ cc_library( visibility = ["//visibility:public"], ) +cc_library( + name = "mmapped_file", + hdrs = ["mmapped_file.h"], + deps = [ + "//mediapipe/framework/port:logging", + "@com_google_absl//absl/status", + ], +) + cc_library( name = "file_helpers", srcs = ["file_helpers.cc"], @@ -124,6 +133,24 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":file_path", + ":mmapped_file", + ":platform_strings", + "//mediapipe/framework/formats:unique_fd", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/base:config", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "mlock_helpers", + srcs = ["mlock_helpers.cc"], + hdrs = ["mlock_helpers.h"], + visibility = ["//visibility:public"], + deps = [ ":platform_strings", "//mediapipe/framework/port:status", "@com_google_absl//absl/status", diff --git a/mediapipe/framework/deps/file_helpers.cc b/mediapipe/framework/deps/file_helpers.cc index b889867203..d444b452a4 100644 --- a/mediapipe/framework/deps/file_helpers.cc +++ b/mediapipe/framework/deps/file_helpers.cc @@ -17,27 +17,36 @@ #ifdef _WIN32 #include #include +#include #else #include +#include +#include #endif // _WIN32 #include #include #include #include +#include #include +#include +#include "absl/base/config.h" +#include "absl/cleanup/cleanup.h" // IWYU pragma: keep #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/deps/mmapped_file.h" #include "mediapipe/framework/deps/platform_strings.h" // IWYU pragma: keep +#include "mediapipe/framework/formats/unique_fd.h" #include "mediapipe/framework/port/status_macros.h" namespace mediapipe { namespace file { namespace { - // Helper class that returns all entries (files, directories) in a directory, // except "." and "..". Example usage: // @@ -197,6 +206,181 @@ absl::Status AppendStringToFile(absl::string_view path, return absl::OkStatus(); } +#ifdef _WIN32 +class WindowsMMap : public MemoryMappedFile { + public: + WindowsMMap(std::string path, const void* base_address, size_t length, + HANDLE file_handle, HANDLE mapping_handle) + : MemoryMappedFile(std::move(path), base_address, length), + file_handle_(file_handle), + mapping_handle_(mapping_handle) {} + + virtual absl::Status Close() override; + + private: + const HANDLE file_handle_; + const HANDLE mapping_handle_; +}; + +absl::StatusOr> MMapFile( + absl::string_view path) { + std::string name_string = std::string(path); + const HANDLE file_handle = CreateFile( + /*lpFileName=*/Utf8ToNative(name_string).c_str(), + /*dwDesiredAccess=*/GENERIC_READ, + /*dwShareMode=*/FILE_SHARE_READ, + /*lpSecurityAttributes=*/NULL, + /*dwCreationDisposition=*/OPEN_EXISTING, + /*dwFlagsAndAttributes=*/FILE_ATTRIBUTE_NORMAL, + /*hTemplateFile=*/NULL); + if (file_handle == INVALID_HANDLE_VALUE) { + return absl::UnavailableError( + absl::StrCat("Failed to open the file '", path, + "' for reading: ", FormatLastError())); + } + absl::Cleanup file_closer = [file_handle] { CloseHandle(file_handle); }; + + // We're calling `CreateFileMappingA` regardless of `UNICODE` because we don't + // pass the `lpName` string parameter. + const HANDLE mapping_handle = CreateFileMappingA( + /*hFile=*/file_handle, + /*lpFileMappingAttributes=*/NULL, + /*flProtect=*/PAGE_READONLY, + /*dwMaximumSizeHigh=*/0, // If `dwMaximumSize{Low,High} are zero, + /*dwMaximumSizeLow=*/0, // the maximum mapping size is the file size. + /*lpName=*/NULL); + if (mapping_handle == INVALID_HANDLE_VALUE) { + return absl::UnavailableError( + absl::StrCat("Failed to create a memory mapping for the file '", path, + "': ", FormatLastError())); + } + absl::Cleanup mapping_closer = [mapping_handle] { + CloseHandle(mapping_handle); + }; + + const LPVOID base_address = MapViewOfFile( + /*hFileMappingObject=*/mapping_handle, + /*dwDesiredAccess=*/FILE_MAP_READ, + /*dwFileOffsetHigh=*/0, + /*dwFileOffsetLow=*/0, + /*dwNumberOfBytesToMap=*/0 // Extends to the file end. + ); + if (base_address == NULL) { + return absl::UnavailableError(absl::StrCat( + "Failed to memory-map the file '", path, "': ", FormatLastError())); + } + + LARGE_INTEGER large_length; + const BOOL success = GetFileSizeEx(file_handle, &large_length); + if (!success) { + return absl::UnavailableError( + absl::StrCat("Failed to determine the size of the file '", path, + "': ", FormatLastError())); + } + const size_t length = static_cast(large_length.QuadPart); + + std::move(file_closer).Cancel(); + std::move(mapping_closer).Cancel(); + + return std::make_unique(std::move(name_string), base_address, + length, file_handle, mapping_handle); +} + +absl::Status WindowsMMap::Close() { + BOOL success = UnmapViewOfFile(BaseAddress()); + if (!success) { + return absl::UnavailableError(absl::StrCat( + "Failed to unmap the file '", Path(), "': ", FormatLastError())); + } + success = CloseHandle(mapping_handle_); + if (!success) { + return absl::UnavailableError( + absl::StrCat("Failed to close the memory mapping for file '", Path(), + "': " << FormatLastError())); + } + success = CloseHandle(file_handle_); + if (!success) { + return absl::UnavailableError(absl::StrCat( + "Failed to close the file '", Path(), "': ", FormatLastError())); + } + return absl::OkStatus(); +} +#elif ABSL_HAVE_MMAP +class PosixMMap : public MemoryMappedFile { + public: + PosixMMap(std::string path, const void* base_address, size_t length, + UniqueFd&& fd) + : MemoryMappedFile(path, base_address, length), + unique_fd_(std::move(fd)) {} + + absl::Status Close() override; + + private: + UniqueFd unique_fd_; +}; + +absl::StatusOr> MMapFile( + absl::string_view path) { + std::string name_string = std::string(path); + const int fd = open(name_string.c_str(), O_RDONLY); + if (fd < 0) { + return absl::UnavailableError(absl::StrCat( + "Couldn't open file '", path, "' for reading: ", FormatLastError())); + } + UniqueFd unique_fd(fd); + + struct stat file_stat; + const int status = fstat(unique_fd.Get(), &file_stat); + if (status < 0) { + return absl::UnavailableError( + absl::StrCat("Couldn't stat file '", path, "': ", FormatLastError())); + } + size_t length = file_stat.st_size; + + const void* base_address = + mmap(nullptr, length, PROT_READ, /*flags=*/0, unique_fd.Get(), + /*offset=*/0); + if (base_address == nullptr) { + return absl::UnavailableError(absl::StrCat( + "Couldn't map file '", path, "' into memory: ", FormatLastError())); + } + + return std::make_unique(std::move(name_string), base_address, + length, std::move(unique_fd)); +} + +absl::Status PosixMMap::Close() { + int status = munmap(const_cast(BaseAddress()), Length()); + if (status < 0) { + return absl::UnavailableError(absl::StrCat( + "Couldn't unmap file '", Path(), "' from memory: ", FormatLastError())); + } + + status = close(unique_fd_.Release()); + if (status < 0) { + return absl::UnavailableError(absl::StrCat("Couldn't close file '", Path(), + "': ", FormatLastError())); + } + return absl::OkStatus(); +} +#else // _WIN32 / ABSL_HAVE_MMAP +absl::StatusOr> MMapFile( + absl::string_view path) { + return absl::UnavailableError(absl::StrCat( + "No supported memory-mapping mechanism is provided for file '", path, + "'")); +} + +absl::Status LockMemory(const void* base_address, size_t length) { + return absl::UnavailableError("Locking memory unsupported"); +} + +absl::Status UnlockMemory(const void* base_address, size_t length) { + return absl::UnavailableError( + "Shouldn't attempt unlocking memory where locking is not supported"); +} +#endif // _WIN32 / ABSL_HAVE_MMAP + absl::Status MatchInTopSubdirectories(const std::string& parent_directory, const std::string& file_name, std::vector* results) { diff --git a/mediapipe/framework/deps/file_helpers.h b/mediapipe/framework/deps/file_helpers.h index 146ea5daf1..6724fa5427 100644 --- a/mediapipe/framework/deps/file_helpers.h +++ b/mediapipe/framework/deps/file_helpers.h @@ -16,7 +16,9 @@ #define MEDIAPIPE_FRAMEWORK_DEPS_FILE_HELPERS_H_ #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/match.h" +#include "mediapipe/framework/deps/mmapped_file.h" namespace mediapipe { namespace file { @@ -29,6 +31,12 @@ absl::Status SetContents(absl::string_view file_name, absl::Status AppendStringToFile(absl::string_view file_name, absl::string_view contents); +absl::StatusOr> MMapFile( + absl::string_view path); + +absl::Status LockMemory(const void* base_address, size_t length); +absl::Status UnlockMemory(const void* base_address, size_t length); + absl::Status MatchInTopSubdirectories(const std::string& parent_directory, const std::string& file_name, std::vector* results); diff --git a/mediapipe/framework/deps/mlock_helpers.cc b/mediapipe/framework/deps/mlock_helpers.cc new file mode 100644 index 0000000000..63a994f6f4 --- /dev/null +++ b/mediapipe/framework/deps/mlock_helpers.cc @@ -0,0 +1,53 @@ +#include "mediapipe/framework/deps/mlock_helpers.h" + +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "mediapipe/framework/deps/platform_strings.h" + +namespace mediapipe { +#ifdef _WIN32 +absl::Status LockMemory(const void* base_address, size_t length) { + BOOL status = VirtualLock(const_cast(base_address), length); + if (!status) { + return absl::UnavailableError( + absl::StrCat("Failed to lock pages in memory: ", FormatLastError())); + } + return absl::OkStatus(); +} + +absl::Status UnlockMemory(const void* base_address, size_t length) { + BOOL status = VirtualUnlock(const_cast(base_address), length); + if (!status) { + return absl::UnavailableError( + absl::StrCat("Failed to unlock memory pages: ", FormatLastError())); + } + return absl::OkStatus(); +} +#else // _WIN32 +absl::Status LockMemory(const void* base_address, size_t length) { + int status = mlock(base_address, length); + if (status < 0) { + return absl::UnavailableError( + absl::StrCat("Failed to lock pages in memory: ", FormatLastError())); + } + return absl::OkStatus(); +} + +absl::Status UnlockMemory(const void* base_address, size_t length) { + int status = munlock(base_address, length); + if (status < 0) { + return absl::UnavailableError( + absl::StrCat("Failed to unlock memory pages: ", FormatLastError())); + } + return absl::OkStatus(); +} +#endif // _WIN32 +} // namespace mediapipe diff --git a/mediapipe/framework/deps/mlock_helpers.h b/mediapipe/framework/deps/mlock_helpers.h new file mode 100644 index 0000000000..1f4a1ab585 --- /dev/null +++ b/mediapipe/framework/deps/mlock_helpers.h @@ -0,0 +1,11 @@ +#ifndef MEDIAPIPE_FRAMEWORK_DEPS_MLOCK_HELPERS_H_ +#define MEDIAPIPE_FRAMEWORK_DEPS_MLOCK_HELPERS_H_ +#include "absl/status/status.h" + +namespace mediapipe { +// Uses `mlock`/`VirtualLock` to pin memory pages. +absl::Status LockMemory(const void* base_address, size_t length); +// Unlocks a previously locked memory region. +absl::Status UnlockMemory(const void* base_address, size_t length); +} // namespace mediapipe +#endif // MEDIAPIPE_FRAMEWORK_DEPS_MLOCK_HELPERS_H_ diff --git a/mediapipe/framework/deps/mmapped_file.h b/mediapipe/framework/deps/mmapped_file.h new file mode 100644 index 0000000000..3409762f9f --- /dev/null +++ b/mediapipe/framework/deps/mmapped_file.h @@ -0,0 +1,46 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "absl/status/status.h" +#include "mediapipe/framework/port/logging.h" + +#ifndef MEDIAPIPE_FRAMEWORK_DEPS_MMAPPED_FILE_H_ +#define MEDIAPIPE_FRAMEWORK_DEPS_MMAPPED_FILE_H_ +namespace mediapipe { +namespace file { +class MemoryMappedFile { + public: + MemoryMappedFile(std::string path, const void* base_address, size_t length) + : path_(std::move(path)), base_address_(base_address), length_(length) {} + + virtual absl::Status Close() = 0; + + virtual ~MemoryMappedFile() = default; + + const std::string& Path() const { return path_; } + const void* BaseAddress() const { return base_address_; } + size_t Length() const { return length_; } + + private: + std::string path_; + const void* base_address_; + size_t length_; +}; +} // namespace file +} // namespace mediapipe +#endif // MEDIAPIPE_FRAMEWORK_DEPS_MMAPPED_FILE_H_ diff --git a/mediapipe/framework/port/BUILD b/mediapipe/framework/port/BUILD index 31c8223044..985a4655f0 100644 --- a/mediapipe/framework/port/BUILD +++ b/mediapipe/framework/port/BUILD @@ -113,6 +113,7 @@ cc_library( ":status", "//mediapipe/framework/deps:file_helpers", "//mediapipe/framework/deps:file_path", + "//mediapipe/framework/deps:mmapped_file", ], ) From 180e7de25fcb5ff5475b53be86978b63b18dd22b Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 6 Dec 2024 14:47:46 -0800 Subject: [PATCH 105/126] Fix Windows build PiperOrigin-RevId: 703623007 --- mediapipe/framework/deps/mlock_helpers.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mediapipe/framework/deps/mlock_helpers.cc b/mediapipe/framework/deps/mlock_helpers.cc index 63a994f6f4..cacd62e3b3 100644 --- a/mediapipe/framework/deps/mlock_helpers.cc +++ b/mediapipe/framework/deps/mlock_helpers.cc @@ -3,6 +3,9 @@ #include #ifdef _WIN32 +// clang-format off +#include // Must come before other Windows headers +// clang-format on #include #else #include From d2e9caead598087ffd493bfbffce351b92891e5c Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 6 Dec 2024 15:29:09 -0800 Subject: [PATCH 106/126] Fix Windows build, part 2 PiperOrigin-RevId: 703634571 --- mediapipe/framework/deps/BUILD | 6 ++++-- mediapipe/framework/deps/file_helpers.cc | 4 +++- mediapipe/framework/deps/mlock_helpers.cc | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mediapipe/framework/deps/BUILD b/mediapipe/framework/deps/BUILD index da9835037d..2dc600ada7 100644 --- a/mediapipe/framework/deps/BUILD +++ b/mediapipe/framework/deps/BUILD @@ -135,14 +135,16 @@ cc_library( ":file_path", ":mmapped_file", ":platform_strings", - "//mediapipe/framework/formats:unique_fd", "//mediapipe/framework/port:status", "@com_google_absl//absl/base:config", "@com_google_absl//absl/cleanup", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - ], + ] + select({ + "//mediapipe:windows": [], + "//conditions:default": ["//mediapipe/framework/formats:unique_fd"], + }), ) cc_library( diff --git a/mediapipe/framework/deps/file_helpers.cc b/mediapipe/framework/deps/file_helpers.cc index d444b452a4..32b55fcf1f 100644 --- a/mediapipe/framework/deps/file_helpers.cc +++ b/mediapipe/framework/deps/file_helpers.cc @@ -41,7 +41,9 @@ #include "mediapipe/framework/deps/file_path.h" #include "mediapipe/framework/deps/mmapped_file.h" #include "mediapipe/framework/deps/platform_strings.h" // IWYU pragma: keep +#ifndef _WIN32 #include "mediapipe/framework/formats/unique_fd.h" +#endif // !_WIN32 #include "mediapipe/framework/port/status_macros.h" namespace mediapipe { @@ -296,7 +298,7 @@ absl::Status WindowsMMap::Close() { if (!success) { return absl::UnavailableError( absl::StrCat("Failed to close the memory mapping for file '", Path(), - "': " << FormatLastError())); + "': ", FormatLastError())); } success = CloseHandle(file_handle_); if (!success) { diff --git a/mediapipe/framework/deps/mlock_helpers.cc b/mediapipe/framework/deps/mlock_helpers.cc index cacd62e3b3..0d335f5698 100644 --- a/mediapipe/framework/deps/mlock_helpers.cc +++ b/mediapipe/framework/deps/mlock_helpers.cc @@ -4,7 +4,7 @@ #ifdef _WIN32 // clang-format off -#include // Must come before other Windows headers +#include // Must come before other Windows headers. // clang-format on #include #else From afebe9d36ac0f76a725e49d1fe638035bcecfa7c Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 6 Dec 2024 17:35:35 -0800 Subject: [PATCH 107/126] Support memory mapping in resources. PiperOrigin-RevId: 703664311 --- mediapipe/framework/deps/file_helpers.cc | 16 +++-- mediapipe/framework/resources.cc | 84 +++++++++++++++++++++--- mediapipe/framework/resources.h | 20 ++++++ mediapipe/framework/resources_test.cc | 22 +++++++ 4 files changed, 130 insertions(+), 12 deletions(-) diff --git a/mediapipe/framework/deps/file_helpers.cc b/mediapipe/framework/deps/file_helpers.cc index 32b55fcf1f..a20a129aa6 100644 --- a/mediapipe/framework/deps/file_helpers.cc +++ b/mediapipe/framework/deps/file_helpers.cc @@ -49,6 +49,10 @@ namespace mediapipe { namespace file { namespace { +size_t RoundUp(size_t size, size_t align) { + return (size + align - 1) & ~(align - 1); +} + // Helper class that returns all entries (files, directories) in a directory, // except "." and "..". Example usage: // @@ -158,7 +162,7 @@ absl::Status GetContents(absl::string_view path, std::string* output, bool read_as_binary) { FILE* fp = fopen(std::string(path).c_str(), read_as_binary ? "rb" : "r"); if (fp == NULL) { - return absl::InvalidArgumentError(absl::StrCat("Can't find file: ", path)); + return absl::NotFoundError(absl::StrCat("Can't find file: ", path)); } output->clear(); @@ -340,9 +344,9 @@ absl::StatusOr> MMapFile( size_t length = file_stat.st_size; const void* base_address = - mmap(nullptr, length, PROT_READ, /*flags=*/0, unique_fd.Get(), + mmap(nullptr, length, PROT_READ, /*flags=*/MAP_SHARED, unique_fd.Get(), /*offset=*/0); - if (base_address == nullptr) { + if (base_address == MAP_FAILED) { return absl::UnavailableError(absl::StrCat( "Couldn't map file '", path, "' into memory: ", FormatLastError())); } @@ -352,7 +356,11 @@ absl::StatusOr> MMapFile( } absl::Status PosixMMap::Close() { - int status = munmap(const_cast(BaseAddress()), Length()); + // `munmap` length should be a multiple of page size. + const int page_size = sysconf(_SC_PAGESIZE); + const size_t aligned_length = + RoundUp(Length(), static_cast(page_size)); + int status = munmap(const_cast(BaseAddress()), aligned_length); if (status < 0) { return absl::UnavailableError(absl::StrCat( "Couldn't unmap file '", Path(), "' from memory: ", FormatLastError())); diff --git a/mediapipe/framework/resources.cc b/mediapipe/framework/resources.cc index e8daefa1e1..475646f7c8 100644 --- a/mediapipe/framework/resources.cc +++ b/mediapipe/framework/resources.cc @@ -6,10 +6,15 @@ #include #include "absl/container/flat_hash_map.h" +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "mediapipe/framework/deps/mlock_helpers.h" +#include "mediapipe/framework/deps/mmapped_file.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/tool/status_util.h" #include "mediapipe/util/resource_util.h" @@ -35,30 +40,75 @@ class NoCleanupResource : public Resource { NoCleanupResource(const void* data, size_t length) : Resource(data, length) {} }; +class MMapResource : public Resource { + public: + MMapResource(std::unique_ptr mmapped_file, + bool mlocked) + : Resource(mmapped_file->BaseAddress(), mmapped_file->Length()), + mmapped_file_(std::move(mmapped_file)), + mlocked_(mlocked) {} + + ~MMapResource() override { + if (mlocked_) { + auto status = + UnlockMemory(mmapped_file_->BaseAddress(), mmapped_file_->Length()); + if (!status.ok()) { + ABSL_LOG(DFATAL) << status; + } + } + auto status = mmapped_file_->Close(); + if (!status.ok()) { + ABSL_LOG(DFATAL) << status; + } + } + + private: + std::unique_ptr mmapped_file_; + bool mlocked_; +}; + class DefaultResources : public Resources { public: absl::StatusOr> Get( absl::string_view resource_id, const Options& options) const final { - // First try to load resource as is. - std::string path(resource_id); + const std::string path(resource_id); + if (options.mmap_mode.has_value()) { + const MMapMode mode = options.mmap_mode.value(); + // Try to resolve `resource_id` into a path. + const absl::StatusOr resolved_path = + PathToResourceAsFile(path, /*shadow_copy=*/false); + if (resolved_path.ok()) { + auto status_or_mmap = + MakeMMapResource(path, + /*mlock=*/mode == MMapMode::kMMapAndMLock); + if (status_or_mmap.ok() || mode != MMapMode::kMMapOrRead) { + return status_or_mmap; + } + } else if (mode != MMapMode::kMMapOrRead) { + return resolved_path.status(); + } + } + + // Try to load the resource as is. std::string output; - absl::Status status = + const absl::Status status = GetResourceContents(path, &output, options.read_as_binary); if (status.ok()) { return MakeStringResource(std::move(output)); } - // Try to resolve resource_id. - absl::StatusOr resolved_path = PathToResourceAsFile(path); - if (!resolved_path.ok() || resolved_path.value() == path) { + // Try the path resolution again, this time possibly with shadow copying. + const absl::StatusOr resolved_path_maybe_shadow = + PathToResourceAsFile(path, /*shadow_copy=*/true); + if (!resolved_path_maybe_shadow.ok()) { return tool::CombinedStatus( absl::StrCat("Failed to load resource: ", resource_id), - {status, resolved_path.status()}); + {status, resolved_path_maybe_shadow.status()}); } // Try to load by resolved path. absl::Status status_for_resolved = GetResourceContents( - resolved_path.value(), &output, options.read_as_binary); + resolved_path_maybe_shadow.value(), &output, options.read_as_binary); if (status_for_resolved.ok()) { return MakeStringResource(std::move(output)); } @@ -104,6 +154,24 @@ std::unique_ptr MakeNoCleanupResource(const void* data, return std::make_unique(data, length); } +absl::StatusOr> MakeMMapResource( + absl::string_view path, bool mlock) { + auto mmap_or_error = file::MMapFile(path); + if (!mmap_or_error.ok()) { + return mmap_or_error.status(); + } + std::unique_ptr mmap = std::move(*mmap_or_error); + + if (mlock) { + auto status = LockMemory(mmap->BaseAddress(), mmap->Length()); + if (!status.ok()) { + return absl::UnavailableError(absl::StrCat("Locking memory for file '", + path, "' failed: ", status)); + } + } + return std::make_unique(std::move(mmap), mlock); +} + std::unique_ptr CreateDefaultResources() { return std::make_unique(); } diff --git a/mediapipe/framework/resources.h b/mediapipe/framework/resources.h index 9124ca9fd8..0920236877 100644 --- a/mediapipe/framework/resources.h +++ b/mediapipe/framework/resources.h @@ -60,6 +60,21 @@ std::unique_ptr MakeStringResource(std::string&& s); std::unique_ptr MakeNoCleanupResource(const void* data, size_t length); +// Creates a resource by memory-mapping the file at `path`. +absl::StatusOr> MakeMMapResource( + absl::string_view path, bool mlock); + +enum class MMapMode { + // Map the file contents into memory when supported, read otherwise. + kMMapOrRead, + // Fail if memory mapping is not available. + kMMap, + // Like `kMMap` with additional memory-locking of the mapped pages. + // This makes sure the data is resident in memory (never swapped) but comes + // with increased memory usage and takes time to perform the initial read. + kMMapAndMLock, +}; + // Represents an interface to load resources in calculators and subgraphs. // // Should be accessed through `CalculatorContext::GetResources` and @@ -71,6 +86,11 @@ class Resources { public: struct Options { bool read_as_binary = true; + + // If specified, attempt memory-mapping file-based resources in the given + // mode. Otherwise the file contents are read into memory. + // Memory-mapped files are always `read_as_binary`. + std::optional mmap_mode; }; virtual ~Resources() = default; diff --git a/mediapipe/framework/resources_test.cc b/mediapipe/framework/resources_test.cc index 83e922e69a..e0cb3a7e56 100644 --- a/mediapipe/framework/resources_test.cc +++ b/mediapipe/framework/resources_test.cc @@ -39,6 +39,28 @@ TEST(Resources, CanCreateDefaultResourcesAndReadFileContents) { EXPECT_EQ(resource->ToStringView(), "File system calculator contents\n"); } +TEST(Resources, CanCreateDefaultResourcesAndReadFileContentsWithMMap) { + std::unique_ptr resources = CreateDefaultResources(); + + Resources::Options options{.mmap_mode = MMapMode::kMMap}; + MP_ASSERT_OK_AND_ASSIGN( + std::unique_ptr resource, + resources->Get("mediapipe/framework/testdata/resource_calculator.data", + options)); + EXPECT_EQ(resource->ToStringView(), "File system calculator contents\n"); +} + +TEST(Resources, CanCreateDefaultResourcesAndReadFileContentsWithMMapAndMLock) { + std::unique_ptr resources = CreateDefaultResources(); + + Resources::Options options{.mmap_mode = MMapMode::kMMapAndMLock}; + MP_ASSERT_OK_AND_ASSIGN( + std::unique_ptr resource, + resources->Get("mediapipe/framework/testdata/resource_calculator.data", + options)); + EXPECT_EQ(resource->ToStringView(), "File system calculator contents\n"); +} + TEST(Resources, CanReadFileContentsByUnresolvedId) { absl::SetFlag(&FLAGS_resource_root_dir, "mediapipe/framework/testdata"); std::unique_ptr resources = CreateDefaultResources(); From 09adf69d9590e6581f589c31a79edef211583774 Mon Sep 17 00:00:00 2001 From: Tommy Chiang Date: Mon, 9 Dec 2024 01:16:41 -0800 Subject: [PATCH 108/126] Fix typo PiperOrigin-RevId: 704184963 --- mediapipe/calculators/tensor/inference_calculator.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/calculators/tensor/inference_calculator.proto b/mediapipe/calculators/tensor/inference_calculator.proto index 2ce82a6a0c..0be14b7f58 100644 --- a/mediapipe/calculators/tensor/inference_calculator.proto +++ b/mediapipe/calculators/tensor/inference_calculator.proto @@ -283,7 +283,7 @@ message InferenceCalculatorOptions { TensorNamesMap input_tensor_names_map = 3; } - // Maps the output tensors of the TfLite model to the the output tensors of + // Maps the output tensors of the TfLite model to the output tensors of // the InferenceCalculator. Values are model I/O tensor indices or tensor // names. oneof OutputTensorMap { From b94f5264d3f855314d5a800b9b5ae2ba91fa938a Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 9 Dec 2024 05:55:05 -0800 Subject: [PATCH 109/126] No public description PiperOrigin-RevId: 704253935 --- .../cc/genai/inference/utils/xnn_utils/BUILD | 2 ++ .../inference/utils/xnn_utils/llm_weights.cc | 25 +++++++++++++++---- .../inference/utils/xnn_utils/llm_weights.h | 6 +++-- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/BUILD b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/BUILD index de621718b9..b3fce26ff4 100644 --- a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/BUILD +++ b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/BUILD @@ -146,6 +146,8 @@ cc_library( "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@org_tensorflow//tensorflow/compiler/mlir/lite/schema:schema_fbs", + "@org_tensorflow//tensorflow/lite:model_builder", ], ) diff --git a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.cc b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.cc index b4fec60a92..07259e62d1 100644 --- a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.cc +++ b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -38,6 +39,8 @@ #include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/tflite_weight_accessor.h" #include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/utils.h" #include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/xnn_tensor.h" +#include "tensorflow/compiler/mlir/lite/schema/schema_generated.h" +#include "tensorflow/lite/model_builder.h" namespace mediapipe::tasks::genai::xnn_utils { @@ -486,8 +489,9 @@ absl::StatusOr LlmWeightsLoader::LoadWeights() { return result; } -DefaultLlmWeightsLoader::DefaultLlmWeightsLoader(absl::string_view weight_path, - const LlmParams& params) +DefaultLlmWeightsLoader::DefaultLlmWeightsLoader( + absl::string_view weight_path, const LlmParams& params, + std::shared_ptr flat_buffer_model) : LlmWeightsLoader(nullptr, params) { xnn_weights_cache_ = std::make_shared( params.cache_dir.empty() @@ -497,9 +501,20 @@ DefaultLlmWeightsLoader::DefaultLlmWeightsLoader(absl::string_view weight_path, absl::StrCat(mediapipe::file::Basename(weight_path), ".cache"))); ABSL_CHECK_OK(xnn_weights_cache_->Initialize()); - weight_accessor_ = std::make_unique( - std::make_shared(weight_path), - xnn_weights_cache_.get()); + if (flat_buffer_model != nullptr) { + const tflite::Model* model = flat_buffer_model->GetModel(); + std::shared_ptr tflite_model( + model, [](const tflite::Model*) { /* No deletion needed */ }); + char* data = const_cast( + reinterpret_cast(flat_buffer_model->allocation()->base())); + weight_accessor_ = std::make_unique( + std::make_shared(tflite_model, data), + xnn_weights_cache_.get()); + } else { + weight_accessor_ = std::make_unique( + std::make_shared(weight_path), + xnn_weights_cache_.get()); + } } } // namespace mediapipe::tasks::genai::xnn_utils diff --git a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.h b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.h index 56fba37047..ec1ad4d4db 100644 --- a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.h +++ b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.h @@ -31,6 +31,7 @@ #include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.h" #include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/pack_weights_cache.h" #include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/xnn_tensor.h" +#include "tensorflow/lite/model_builder.h" namespace mediapipe::tasks::genai::xnn_utils { @@ -264,8 +265,9 @@ class DefaultLlmWeightsLoader : public LlmWeightsLoader { DefaultLlmWeightsLoader(std::unique_ptr weight_accessor, const LlmParams& params) : LlmWeightsLoader(std::move(weight_accessor), params) {} - DefaultLlmWeightsLoader(absl::string_view weight_path, - const LlmParams& params); + DefaultLlmWeightsLoader( + absl::string_view weight_path, const LlmParams& params, + std::shared_ptr flat_buffer_model = nullptr); std::shared_ptr GetXnnWeightsCache() override { return xnn_weights_cache_; From 178f05e18ab5584519fbc836b56d82568cf3dfc0 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 9 Dec 2024 12:48:10 -0800 Subject: [PATCH 110/126] Update WASM files for 0.10.19 release PiperOrigin-RevId: 704384199 --- third_party/wasm_files.bzl | 80 +++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/third_party/wasm_files.bzl b/third_party/wasm_files.bzl index 0426a17631..4dfd1cfa46 100644 --- a/third_party/wasm_files.bzl +++ b/third_party/wasm_files.bzl @@ -12,120 +12,120 @@ def wasm_files(): http_file( name = "com_google_mediapipe_wasm_audio_wasm_internal_js", - sha256 = "dcf29ade023e427c0b5c75701b35c9c0f1bf3b222f1cf197bffff3c5bc821564", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.js?generation=1729631783325350"], + sha256 = "9f8d59a241abaa0d3b69dad5a094b843e9f0fdf6d2b7349d3d40541e9679725e", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.js?generation=1733776155050960"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_internal_wasm", - sha256 = "22ca6087b4e3f7d2c82a4cbc5ecfbe3da77790823154057fdebc626d335bc606", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.wasm?generation=1729631784951205"], + sha256 = "a57c300fa8fe6756396c1718ddbe4d134e1361e973087ce192bcdab3eea528d1", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.wasm?generation=1733776156973681"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_nosimd_internal_js", - sha256 = "863b7c9c3424210b47df34504d811e5731c0da7fb1ce447b417b2f8a41904d24", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.js?generation=1729631786438352"], + sha256 = "b9cd5366d4b460d58f151b02ce0ec5784e13130ffb67396cbb532a52ad14c966", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.js?generation=1733776158667129"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_nosimd_internal_wasm", - sha256 = "ae88030aa97bf5a16f172802e1e98de3385dbeac5e2c903315d4a97ba1e1ec4b", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.wasm?generation=1729631788203650"], + sha256 = "cdd5c603a5225d85dbb30944fa1e66c46a76790ec246682c4f3d88c571b5a3a6", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.wasm?generation=1733776160452697"], ) http_file( name = "com_google_mediapipe_wasm_genai_experimental_wasm_internal_js", - sha256 = "2fe752a463559b6611fa340788c91b774adeaa8130a5df1c7a9ded1faf7346c6", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_experimental_wasm_internal.js?generation=1729631789833117"], + sha256 = "d717a50336544581e619ef6470bef6b6cbb80419ba7628cb1d7894bcb78b134f", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_experimental_wasm_internal.js?generation=1733776162207511"], ) http_file( name = "com_google_mediapipe_wasm_genai_experimental_wasm_internal_wasm", - sha256 = "5817c6ca0f9243d2a1a50a325461021fc22cf43f0547dfd08d0fe748a56b2386", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_experimental_wasm_internal.wasm?generation=1729631791598882"], + sha256 = "856430599596575fdde3418e87ab774bc5e5cb74fd2d258c167ef27eedf62f20", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_experimental_wasm_internal.wasm?generation=1733776163952500"], ) http_file( name = "com_google_mediapipe_wasm_genai_experimental_wasm_nosimd_internal_js", - sha256 = "a533f74126602e016d422c40891a3029fcab7739012b2c61c82294125e25b79f", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_experimental_wasm_nosimd_internal.js?generation=1729631793098394"], + sha256 = "6cd43b71667383e643069365b5c76d3fa9a4684b7c53b0342501b2298a4acf36", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_experimental_wasm_nosimd_internal.js?generation=1733776165676349"], ) http_file( name = "com_google_mediapipe_wasm_genai_experimental_wasm_nosimd_internal_wasm", - sha256 = "cc19e6c3659865a095354481817797d98f14a21e4c4bf4046feac5905d1b40c4", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_experimental_wasm_nosimd_internal.wasm?generation=1729631794847288"], + sha256 = "e22418a0e8f3b2781137d2f78bf03088f075512183149695d8d13109e2607e48", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_experimental_wasm_nosimd_internal.wasm?generation=1733776167401717"], ) http_file( name = "com_google_mediapipe_wasm_genai_wasm_internal_js", - sha256 = "27876af95fca95bfc31e480cd8a8c39b3c583885631184ac6b3902823bc7bf30", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_wasm_internal.js?generation=1729631796541863"], + sha256 = "762a2bdc0cf50598cfb136212e5f4ccd948dd1f8818328f18dc0d6954e34303c", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_wasm_internal.js?generation=1733776169085687"], ) http_file( name = "com_google_mediapipe_wasm_genai_wasm_internal_wasm", - sha256 = "7ea22cf096a95d74656374d2c93326f1b3e55ce05f4e9d8c5af15e27fe36917e", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_wasm_internal.wasm?generation=1729631798322940"], + sha256 = "dc42f8170316ab700cadfc39c0dc65872ea20d72f610b634a1e7850ae2a5449b", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_wasm_internal.wasm?generation=1733776170938658"], ) http_file( name = "com_google_mediapipe_wasm_genai_wasm_nosimd_internal_js", - sha256 = "bfecf8ad4e6c1f4468730d508ecdf7a659dd66878f75eb84f05442d73252d191", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_wasm_nosimd_internal.js?generation=1729631799937929"], + sha256 = "7945ddee65bf96a7fc4d0ee0e12314cbf4da4c948ed11f874c627bb0aa69e10e", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_wasm_nosimd_internal.js?generation=1733776172955953"], ) http_file( name = "com_google_mediapipe_wasm_genai_wasm_nosimd_internal_wasm", - sha256 = "0627903513f58ddd0df058be8877f52499842d37f9d1acdb7fc063e7ed0adc50", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_wasm_nosimd_internal.wasm?generation=1729631801590775"], + sha256 = "3bdecd9c4e978b9b912f40e8a174eadd3702eea55f1f7d4255c4e50cf021cbb3", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/genai_wasm_nosimd_internal.wasm?generation=1733776174810127"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_internal_js", - sha256 = "5bc536c7d0de0ea284f90aebcbf3a2e1a4254a94f988cd6b0c80932af9377e1a", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.js?generation=1729631803074796"], + sha256 = "903ddd3412782ce598655b1c052156577da9918e6daf34b8958d730aca685d61", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.js?generation=1733776176490879"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_internal_wasm", - sha256 = "89db81068af5254c863f2d2a758d4c80ba58a8ce3667d460d099e99ceefae375", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.wasm?generation=1729631804845629"], + sha256 = "fc5e0e540e48e94b00d95f5d29228c2c087cc7e61e77c9b28cdae350d89fdd6b", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.wasm?generation=1733776178221074"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_nosimd_internal_js", - sha256 = "a05a51517f2c31a30bad54ca95f8006d634fbed0f553f62518ffd9fa73e7ae89", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.js?generation=1729631806522140"], + sha256 = "643302ccf7ae8bc455b7a67fe0ec726018a49a509395dbc50c5ec03784e69165", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.js?generation=1733776179901587"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_nosimd_internal_wasm", - sha256 = "5c24f8ffe5b15735983c226a3a731fc9195b97ec23a9274ab82d58bf9d484bb4", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.wasm?generation=1729631808337308"], + sha256 = "e9cbe88bbb169a5afa1cea60d7f3bc08badc0bb192b3d20c51781827eec11210", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.wasm?generation=1733776181647902"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_internal_js", - sha256 = "2b120e1c7272905719f7893e5f09e033ead468b46b510e6b490d93e3d94ec69c", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.js?generation=1729631809992099"], + sha256 = "4a97e2520ba506c680ecd6ba6acfb146888afa0e2746d57f205352bc6ebb82eb", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.js?generation=1733776183245106"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_internal_wasm", - sha256 = "35d67ac01df034a04a38cb0533d6438595bcc65c485aed61dd47c86c6c3839cd", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.wasm?generation=1729631811756873"], + sha256 = "f00ec4731faa23b3e714d00e88d4d10e2df5c0a427d3a2b4ae6e3526fdd14ef7", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.wasm?generation=1733776185044729"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_nosimd_internal_js", - sha256 = "d206ba4e27c42a5863b4001c6d9366345f4507979cd6efd087a149ef7781ccd3", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.js?generation=1729631813404093"], + sha256 = "927def7b465c51b86e4b3060f93646aca4e27121f4b8fc0483786e407ea9cf1f", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.js?generation=1733776186780167"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_nosimd_internal_wasm", - sha256 = "17c2bff095305fcae98faa0817cbf72f13df9baf76f2067992a8624ef54d18cb", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.wasm?generation=1729631815062139"], + sha256 = "3821ea9b1f7fb8c549ef2a064ef5c85750bf375c545a49fd6eea0df44a95f1f4", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.wasm?generation=1733776188639956"], ) From a107b3174e08a1f64df9a2f7bf4598a19d80f4b8 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 9 Dec 2024 20:13:15 -0800 Subject: [PATCH 111/126] No public description PiperOrigin-RevId: 704520753 --- .../util/detections_deduplicate_calculator.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mediapipe/calculators/util/detections_deduplicate_calculator.cc b/mediapipe/calculators/util/detections_deduplicate_calculator.cc index a31585b883..0c13317d99 100644 --- a/mediapipe/calculators/util/detections_deduplicate_calculator.cc +++ b/mediapipe/calculators/util/detections_deduplicate_calculator.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include +#include #include #include #include @@ -47,7 +48,7 @@ struct BoundingBoxEq { } // namespace -// This Calculator deduplicates the bunding boxes with exactly the same +// This Calculator deduplicates the bounding boxes with exactly the same // coordinates, and folds the labels into a single Detection proto. Note // non-maximum-suppression remove the overlapping bounding boxes within a class, // while the deduplication operation merges bounding boxes from different @@ -73,7 +74,9 @@ class DetectionsDeduplicateCalculator : public Node { absl::Status Process(mediapipe::CalculatorContext* cc) { const std::vector& raw_detections = kIn(cc).Get(); - absl::flat_hash_map bbox_to_detections; std::vector deduplicated_detections; @@ -87,8 +90,8 @@ class DetectionsDeduplicateCalculator : public Node { detection.location_data().bounding_box())) { // The bbox location already exists. Merge the detection labels into // the existing detection proto. - Detection& deduplicated_detection = - *bbox_to_detections[detection.location_data().bounding_box()]; + Detection& deduplicated_detection = deduplicated_detections + [bbox_to_detections[detection.location_data().bounding_box()]]; deduplicated_detection.mutable_score()->MergeFrom(detection.score()); deduplicated_detection.mutable_label()->MergeFrom(detection.label()); deduplicated_detection.mutable_label_id()->MergeFrom( @@ -100,7 +103,7 @@ class DetectionsDeduplicateCalculator : public Node { // detection vector. deduplicated_detections.push_back(detection); bbox_to_detections[detection.location_data().bounding_box()] = - &deduplicated_detections.back(); + deduplicated_detections.size() - 1; } } kOut(cc).Send(std::move(deduplicated_detections)); From 43dea0ac400d859d7f39ed264b1eb293b69c7182 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 9 Dec 2024 23:11:14 -0800 Subject: [PATCH 112/126] No public description PiperOrigin-RevId: 704564655 --- .../text/custom_ops/sentencepiece/optimized_encoder_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/cc/text/custom_ops/sentencepiece/optimized_encoder_test.cc b/mediapipe/tasks/cc/text/custom_ops/sentencepiece/optimized_encoder_test.cc index f2bc27144c..9803e61b43 100644 --- a/mediapipe/tasks/cc/text/custom_ops/sentencepiece/optimized_encoder_test.cc +++ b/mediapipe/tasks/cc/text/custom_ops/sentencepiece/optimized_encoder_test.cc @@ -34,8 +34,8 @@ namespace mediapipe::tflite_operations::sentencepiece { namespace internal { -tensorflow::Status TFReadFileToString(const std::string& filepath, - std::string* data) { +absl::Status TFReadFileToString(const std::string& filepath, + std::string* data) { return tensorflow::ReadFileToString(tensorflow::Env::Default(), filepath, data); } From 45d2f9510da21bbd66d0ada5021aa8daa95f6a40 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 10 Dec 2024 10:35:25 -0800 Subject: [PATCH 113/126] No public description PiperOrigin-RevId: 704763055 --- docs/getting_started/troubleshooting.md | 67 +++++++++++++++++-- mediapipe/framework/BUILD | 4 +- mediapipe/framework/scheduler.cc | 8 ++- mediapipe/framework/scheduler_queue.cc | 32 +++++---- mediapipe/framework/scheduler_queue.h | 12 ++-- .../tool/graph_runtime_info_logger.h | 2 +- third_party/opencv_macos.BUILD | 21 +++--- 7 files changed, 109 insertions(+), 37 deletions(-) diff --git a/docs/getting_started/troubleshooting.md b/docs/getting_started/troubleshooting.md index 182e09b71c..ca6f226534 100644 --- a/docs/getting_started/troubleshooting.md +++ b/docs/getting_started/troubleshooting.md @@ -17,7 +17,7 @@ nav_order: 10 [https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) as the primary developer documentation site for MediaPipe as of April 3, 2023.* ----- +-------------------------------------------------------------------------------- ## Missing Python binary path @@ -113,9 +113,10 @@ ERROR: Could not find a version that satisfies the requirement mediapipe ERROR: No matching distribution found for mediapipe ``` -after running `pip install mediapipe` usually indicates that there is no qualified MediaPipe Python for your system. -Please note that MediaPipe Python PyPI officially supports the **64-bit** -version of Python 3.7 to 3.10 on the following OS: +after running `pip install mediapipe` usually indicates that there is no +qualified MediaPipe Python for your system. Please note that MediaPipe Python +PyPI officially supports the **64-bit** version of Python 3.7 to 3.10 on the +following OS: - x86_64 Linux - x86_64 macOS 10.15+ @@ -270,6 +271,58 @@ calculators designed specifically for this purpose such as [`FlowLimiterCalculator`] as described in [`How to process realtime input streams`]. +## Monitor calculator inputs and timestamp settlements + +Debugging MediaPipe calculators often requires a deep understanding of the data +flow and timestamp synchronization. Incoming packets to calculators are first +buffered in input queues per stream to be synchronized by the assigned +`InputStreamHandler`. The `InputStreamHandler` job is to determine the input +packet set for a settled timestamp, which puts the calculator into a “ready” +state, followed by triggering a Calculator::Process call with the determined +packet set as input. + +The `DebugInputStreamHandler` can be used to track incoming packets and +timestamp settlements in real-time in the application's LOG(INFO) output. It can +be assigned to specific calculators via the Calculator's input_stream_handler or +graph globally via the `CalculatorGraphConfig`'s input_stream_handler field. + +During the graph execution, incoming packets generate LOG messages which reveal +the timestamp and type of the packet, followed by the current state of all input +queues: + +``` +[INFO] SomeCalculator: Adding packet (ts:2, type:int) to stream INPUT_B:0:input_b +[INFO] SomeCalculator: INPUT_A:0:input_a num_packets: 0 min_ts: 2 +[INFO] SomeCalculator: INPUT_B:0:input_b num_packets: 1 min_ts: 2 +``` + +In addition, it enables the monitoring of timestamp settlement events (in case +the `DefaultInputStreamHandler` is applied). This can help to reveal an +unexpected timestamp bound increase on input streams resulting in a +Calculator::Process call with an incomplete input set resulting in empty packets +on (potentially required) input streams. + +*Example scenario:* + +``` +node { + calculator: "SomeCalculator" + input_stream: "INPUT_A:a" + input_stream: "INPUT_B:b" + ... +} +``` + +Given a calculator with two inputs, receiving an incoming packet with timestamp +1 on stream A followed by an input packet with timestamp 2 on stream B. The +timestamp bound increase to 2 on stream B with pending input packet on stream A +at timestamp 1 triggers the Calculator::Process call with an incomplete input +set for timestamp 1. In this case, the `DefaultInputStreamHandler` outputs: + +``` +[INFO] SomeCalculator: Filled input set at ts: 1 with MISSING packets in input streams: INPUT_B:0:input_b. +``` + ## VLOG is your friend MediaPipe uses `VLOG` in many places to log important events for debugging @@ -287,8 +340,9 @@ purposes which are applied when `CalculatorGraph` is created. Overrides: -- `MEDIAPIPE_VLOG_V`: define and provide value you provide for `--v` -- `MEDIAPIPE_VLOG_VMODULE`: define and provide value you provide for `--vmodule` +- `MEDIAPIPE_VLOG_V`: define and provide value you provide for `--v` +- `MEDIAPIPE_VLOG_VMODULE`: define and provide value you provide for + `--vmodule` You can set overrides by adding: `--copt=-DMEDIAPIPE_VLOG_VMODULE=\"*calculator*=5\"` @@ -326,4 +380,3 @@ To disable support for `avxvnniint8`, add the following to you `.bazelrc`: ``` build --define=xnn_enable_avxvnniint8=false ``` - diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index 7ecac979ba..dd6e8f828e 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -1189,12 +1189,12 @@ cc_library( ":calculator_node", ":executor", "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/synchronization", ], ) diff --git a/mediapipe/framework/scheduler.cc b/mediapipe/framework/scheduler.cc index 36effe0165..8e925d9156 100644 --- a/mediapipe/framework/scheduler.cc +++ b/mediapipe/framework/scheduler.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/scheduler.h" +#include #include #include #include @@ -21,6 +22,7 @@ #include "absl/log/absl_check.h" #include "absl/memory/memory.h" +#include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/calculator_graph.h" #include "mediapipe/framework/executor.h" @@ -36,8 +38,10 @@ namespace mediapipe { namespace internal { +inline constexpr absl::string_view kDefaultQueueName = "default_queue"; + Scheduler::Scheduler(CalculatorGraph* graph) - : graph_(graph), shared_(), default_queue_(&shared_) { + : graph_(graph), shared_(), default_queue_(kDefaultQueueName, &shared_) { shared_.error_callback = std::bind(&CalculatorGraph::RecordError, graph_, std::placeholders::_1); default_queue_.SetIdleCallback(std::bind(&Scheduler::QueueIdleStateChanged, @@ -90,7 +94,7 @@ absl::Status Scheduler::SetNonDefaultExecutor(const std::string& name, "be called after the scheduler " "has started"; auto inserted = non_default_queues_.emplace( - name, absl::make_unique(&shared_)); + name, absl::make_unique(name, &shared_)); RET_CHECK(inserted.second) << "SetNonDefaultExecutor must be called only once for the executor \"" << name << "\""; diff --git a/mediapipe/framework/scheduler_queue.cc b/mediapipe/framework/scheduler_queue.cc index 557d7e40e8..74dc2b1d3f 100644 --- a/mediapipe/framework/scheduler_queue.cc +++ b/mediapipe/framework/scheduler_queue.cc @@ -14,17 +14,15 @@ #include "mediapipe/framework/scheduler_queue.h" -#include +#include #include -#include #include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/calculator_node.h" #include "mediapipe/framework/executor.h" -#include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/status.h" #ifdef __APPLE__ #define AUTORELEASEPOOL @autoreleasepool @@ -97,7 +95,7 @@ void SchedulerQueue::Reset() { void SchedulerQueue::SetExecutor(Executor* executor) { executor_ = executor; } bool SchedulerQueue::IsIdle() { - VLOG(3) << "Scheduler queue empty: " << queue_.empty() + VLOG(3) << "Scheduler queue (" << queue_name_ << ") empty: " << queue_.empty() << ", # of pending tasks: " << num_pending_tasks_; return queue_.empty() && num_pending_tasks_ == 0; } @@ -140,7 +138,8 @@ void SchedulerQueue::AddItemToQueue(Item&& item) { was_idle = IsIdle(); queue_.push(item); ++num_tasks_to_add_; - VLOG(4) << node->DebugName() << " was added to the scheduler queue."; + VLOG(4) << node->DebugName() << " was added to the scheduler queue (" + << queue_name_ << ")"; // Now grab the tasks to execute while still holding the lock. This will // gather any waiting tasks, in addition to the one we just added. @@ -235,13 +234,15 @@ void SchedulerQueue::RunNextTask() { void SchedulerQueue::RunCalculatorNode(CalculatorNode* node, CalculatorContext* cc) { - VLOG(3) << "Running " << node->DebugName(); + VLOG(3) << "Running " << node->DebugName() << " on queue (" << queue_name_ + << ")"; // If we are in the process of stopping the graph (due to tool::StatusStop() // from a non-source node or due to CalculatorGraph::CloseAllPacketSources), // we should not run any more sources. Close the node if it is a source. if (shared_->stopping && node->IsSource()) { - VLOG(4) << "Closing " << node->DebugName() << " due to StatusStop()."; + VLOG(4) << "Closing " << node->DebugName() + << " due to StatusStop() on queue (" << queue_name_ << ")."; int64_t start_time = shared_->timer.StartNode(); // It's OK to not reset/release the prepared CalculatorContext since a // source node always reuses the same CalculatorContext and Close() doesn't @@ -252,7 +253,8 @@ void SchedulerQueue::RunCalculatorNode(CalculatorNode* node, shared_->timer.EndNode(start_time); if (!result.ok()) { VLOG(3) << node->DebugName() - << " had an error while closing due to StatusStop()!"; + << " had an error while closing due to StatusStop()! on queue (" + << queue_name_ << ")"; shared_->error_callback(result); } } else { @@ -273,23 +275,27 @@ void SchedulerQueue::RunCalculatorNode(CalculatorNode* node, shared_->stopping = true; } else { // If we have an error in this calculator. - VLOG(3) << node->DebugName() << " had an error!"; + VLOG(3) << node->DebugName() << " had an error on queue (" + << queue_name_ << ")!"; shared_->error_callback(result); } } } - VLOG(4) << "Done running " << node->DebugName(); + VLOG(4) << "Done running " << node->DebugName() << " on queue (" + << queue_name_ << ")"; node->EndScheduling(); } void SchedulerQueue::OpenCalculatorNode(CalculatorNode* node) { - VLOG(3) << "Opening " << node->DebugName(); + VLOG(3) << "Opening " << node->DebugName() << " on queue (" << queue_name_ + << ")"; int64_t start_time = shared_->timer.StartNode(); const absl::Status result = node->OpenNode(); shared_->timer.EndNode(start_time); if (!result.ok()) { - VLOG(3) << node->DebugName() << " had an error!"; + VLOG(3) << node->DebugName() << " had an error on queue (" << queue_name_ + << ")!"; shared_->error_callback(result); return; } diff --git a/mediapipe/framework/scheduler_queue.h b/mediapipe/framework/scheduler_queue.h index 27b6829895..2268f81401 100644 --- a/mediapipe/framework/scheduler_queue.h +++ b/mediapipe/framework/scheduler_queue.h @@ -15,14 +15,14 @@ #ifndef MEDIAPIPE_FRAMEWORK_SCHEDULER_QUEUE_H_ #define MEDIAPIPE_FRAMEWORK_SCHEDULER_QUEUE_H_ -#include #include #include -#include #include +#include #include -#include "absl/base/macros.h" +#include "absl/base/thread_annotations.h" +#include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/executor.h" @@ -76,7 +76,8 @@ class SchedulerQueue : public TaskQueue { bool is_open_node_ = false; // True if the task should run OpenNode(). }; - explicit SchedulerQueue(SchedulerShared* shared) : shared_(shared) {} + explicit SchedulerQueue(absl::string_view queue_name, SchedulerShared* shared) + : queue_name_(queue_name), shared_(shared) {} // Sets the executor that will run the nodes. Must be called before the // scheduler is started. @@ -145,6 +146,9 @@ class SchedulerQueue : public TaskQueue { // Checks whether the queue has no queued nodes or pending tasks. bool IsIdle() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Queue name for logging purposes. + const std::string queue_name_; + Executor* executor_ = nullptr; IdleCallback idle_callback_; diff --git a/mediapipe/framework/tool/graph_runtime_info_logger.h b/mediapipe/framework/tool/graph_runtime_info_logger.h index 401b58cc68..44594dfdf1 100644 --- a/mediapipe/framework/tool/graph_runtime_info_logger.h +++ b/mediapipe/framework/tool/graph_runtime_info_logger.h @@ -42,8 +42,8 @@ class GraphRuntimeInfoLogger { absl::Notification shutdown_signal_; absl::Notification is_running_; - ThreadPool thread_pool_; absl::AnyInvocable()> get_runtime_info_fn_; + ThreadPool thread_pool_; }; } // namespace mediapipe::tool diff --git a/third_party/opencv_macos.BUILD b/third_party/opencv_macos.BUILD index 7ed8ab8f28..4139f188e7 100644 --- a/third_party/opencv_macos.BUILD +++ b/third_party/opencv_macos.BUILD @@ -9,26 +9,31 @@ exports_files(["LICENSE"]) # Example configurations: # -# To configure OpenCV 3, obtain the path of OpenCV 3 from Homebrew: +# # OpenCV 3 +# To configure OpenCV 3, obtain the path of OpenCV 3 from Homebrew. The +# following commands show the output of the command with version 3.4.16_10: # # $ brew ls opencv@3 | grep version.hpp # $ /opt/homebrew/Cellar/opencv@3/3.4.16_10/include/opencv2/core/version.hpp # # Then set path in "macos_opencv" rule in the WORKSPACE file to -# "/opt/homebrew/Cellar" and the PREFIX below to "opencv@3/3.4.16_10". +# "/opt/homebrew/Cellar" and the PREFIX below to "opencv/" (e.g. +# "opencv/3.4.16_10" for the example above). # -# -# To configure OpenCV 4, obtain the path of OpenCV 4 from Homebrew: +# # OpenCV 4 +# To configure OpenCV 4, obtain the path of OpenCV 4 from Homebrew. The +# following commands show the output of the command with version 4.10.0_12: # # $ brew ls opencv | grep version.hpp # $ /opt/homebrew/Cellar/opencv/4.10.0_12/include/opencv4/opencv2/core/version.hpp # $ /opt/homebrew/Cellar/opencv/4.10.0_12/include/opencv4/opencv2/dnn/version.hpp # # Then set path in "macos_opencv" rule in the WORKSPACE file to -# "/opt/homebrew/Cellar" and the PREFIX below to "opencv/4.10.0_12". For OpenCV -# 4, you will also need to adjust the include paths. The header search path -# should be "include/opencv4/opencv2/**/*.h*" and the include prefix needs to -# be set to "include/opencv4". +# "/opt/homebrew/Cellar" and the PREFIX below to "opencv/" (e.g. +# "opencv/4.10.0_12" for the example above). For OpenCV 4, you will also need to +# adjust the include paths. The header search path should be +# "include/opencv4/opencv2/**/*.h*" and the include prefix needs to be set to +# "include/opencv4". PREFIX = "opt/opencv@3" From adc16aeb81ac9907baa020b524b1f5c8eba770d0 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 11 Dec 2024 10:37:17 -0800 Subject: [PATCH 114/126] Add vision modality to the C API PiperOrigin-RevId: 705157449 --- .../genai/inference/c/llm_inference_engine.h | 18 ++++++++++++++++++ .../inference/c/llm_inference_engine_cpu.cc | 7 +++++++ .../sources/LlmInference+Session.swift | 4 +++- .../genai/inference/sources/LlmInference.swift | 2 ++ .../com/google/mediapipe/tasks/core/jni/llm.cc | 4 ++++ 5 files changed, 34 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine.h b/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine.h index b4d33cb618..91f9a08c70 100644 --- a/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine.h +++ b/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine.h @@ -58,6 +58,12 @@ typedef struct { // Path to the model artifact. const char* model_path; + // Path to the vision encoder to use for vision modality. Optional. + const char* vision_encoder_path; + + // Path to the vision adapter to use for vision modality. Optional. + const char* vision_adapter_path; + // Directory path for storing model related tokenizer and cache weights. the // user is responsible for providing the directory that can be writable by the // program. @@ -121,6 +127,13 @@ typedef struct { // Path to the LoRA tflite flatbuffer file. Optional. // This is only compatible with GPU models. const char* lora_path; + + // Whether to configure the graph to include the token cost calculator, + // which allows users to only compute the cost of a prompt. + bool include_token_cost_calculator; + + // Whether to configure the graph to include the vision modality. + bool enable_vision_modality; } LlmSessionConfig; // LlmResponseContext is the return type for @@ -166,6 +179,11 @@ ODML_EXPORT void LlmInferenceEngine_Session_Delete( ODML_EXPORT int LlmInferenceEngine_Session_AddQueryChunk( LlmInferenceEngine_Session* session, const char* input, char** error_msg); +// Adds an SKBitmap to the session. +ODML_EXPORT int LlmInferenceEngine_Session_AddImage( + LlmInferenceEngine_Session* session, const void* sk_bitmap, + char** error_msg); + // Return the generated output based on the previously added query chunks in // sync mode. ODML_EXPORT LlmResponseContext diff --git a/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine_cpu.cc b/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine_cpu.cc index 40524aee95..7c1e5bd233 100644 --- a/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine_cpu.cc +++ b/mediapipe/tasks/cc/genai/inference/c/llm_inference_engine_cpu.cc @@ -584,6 +584,13 @@ int LlmInferenceEngine_Session_AddQueryChunk( return 0; } +ODML_EXPORT int LlmInferenceEngine_Session_AddImage( + LlmInferenceEngine_Session* session, const void* sk_bitmap, + char** error_msg) { + *error_msg = strdup("Not implemented"); + return 12; +} + LlmResponseContext LlmInferenceEngine_Session_PredictSync( LlmInferenceEngine_Session* session) { LlmInferenceEngine_Session_PredictAsync( diff --git a/mediapipe/tasks/ios/genai/inference/sources/LlmInference+Session.swift b/mediapipe/tasks/ios/genai/inference/sources/LlmInference+Session.swift index 3fe740de2a..a01b80e20c 100644 --- a/mediapipe/tasks/ios/genai/inference/sources/LlmInference+Session.swift +++ b/mediapipe/tasks/ios/genai/inference/sources/LlmInference+Session.swift @@ -54,7 +54,9 @@ extension LlmInference { topp: options.topp, temperature: options.temperature, random_seed: options.randomSeed, - lora_path: nil) + lora_path: nil, + include_token_cost_calculator: true, + enable_vision_modality: false) /// If `loraPath` is != nil, modify session config with the corresponding C string and invoke /// the method to create session runner within the scope where the C String of the `loraPath` diff --git a/mediapipe/tasks/ios/genai/inference/sources/LlmInference.swift b/mediapipe/tasks/ios/genai/inference/sources/LlmInference.swift index 312bb23b2f..f5d7cd940c 100644 --- a/mediapipe/tasks/ios/genai/inference/sources/LlmInference.swift +++ b/mediapipe/tasks/ios/genai/inference/sources/LlmInference.swift @@ -65,6 +65,8 @@ import MediaPipeTasksGenAIC try options.supportedLoraRanks.withUnsafeMutableBufferPointer { supportedLoraRanks in let modelSetting = LlmModelSettings( model_path: modelPath, + vision_encoder_path: nil, + vision_adapter_path: nil, cache_dir: cacheDirectory, max_num_tokens: options.maxTokens, num_decode_steps_per_sync: numberOfDecodeStepsPerSync, diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc index 18d5ceb2c8..e0f3db7d84 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc @@ -43,6 +43,8 @@ LlmModelSettings ParseModelSettings(void* bytes, int size) { LlmModelSettings output; output.model_path = strdup(input.model_path().c_str()); + output.vision_encoder_path = nullptr; + output.vision_adapter_path = nullptr; output.cache_dir = strdup(input.cache_dir().c_str()); output.sequence_batch_size = input.sequence_batch_size(); output.num_decode_steps_per_sync = input.num_decode_steps_per_sync(); @@ -74,6 +76,8 @@ LlmSessionConfig ParseSessionConfig(void* bytes, int size) { if (input.has_lora_path()) { output.lora_path = strdup(input.lora_path().c_str()); } + output.include_token_cost_calculator = true; + output.enable_vision_modality = false; return output; } From 911eb327697a51a919d582fe03b01d0b6f77ab40 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 11 Dec 2024 16:13:31 -0800 Subject: [PATCH 115/126] Bump MediaPipe version to 0.10.20. PiperOrigin-RevId: 705275737 --- version.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.bzl b/version.bzl index 1cb37d18fc..6ffe9b5520 100644 --- a/version.bzl +++ b/version.bzl @@ -2,4 +2,4 @@ # The next version of MediaPipe (e.g. the version that is currently in development). # This version should be bumped after every release. -MEDIAPIPE_FULL_VERSION = "0.10.19" +MEDIAPIPE_FULL_VERSION = "0.10.20" From ad6c03ecd984fcda149d64310692766ccdd2d4f5 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 12 Dec 2024 08:52:27 -0800 Subject: [PATCH 116/126] Add Vision Modality to the MediaPipe LLM JNI Layer PiperOrigin-RevId: 705521252 --- .../com/google/mediapipe/tasks/core/jni/BUILD | 2 + .../google/mediapipe/tasks/core/jni/llm.cc | 76 ++++++++++++++++++- .../com/google/mediapipe/tasks/core/jni/llm.h | 25 ++++++ .../tasks/core/jni/proto/llm_options.proto | 25 ++++++ 4 files changed, 124 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD index 6b09bcbe70..91238b512e 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD @@ -67,8 +67,10 @@ cc_library( "//mediapipe/java/com/google/mediapipe/framework/jni:class_registry", "//mediapipe/java/com/google/mediapipe/framework/jni:jni_util", "//mediapipe/tasks/cc/genai/inference/c:llm_inference_engine_hdr", # needed with ENABLE_ODML_MAVEN_BUILD + "//mediapipe/tasks/cc/genai/inference/proto:tflite_delegate_options_cc_proto", "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto:llm_options_cc_proto", "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto:llm_response_context_cc_proto", + "//third_party/skia/HEAD:core", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc index e0f3db7d84..b232033f53 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc @@ -16,7 +16,9 @@ #include +#include #include +#include #include #include "absl/log/absl_log.h" @@ -25,8 +27,13 @@ #include "mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/jni_util.h" #include "mediapipe/tasks/cc/genai/inference/c/llm_inference_engine.h" +#include "mediapipe/tasks/cc/genai/inference/proto/tflite_delegate_options.pb.h" #include "mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto/llm_options.pb.h" #include "mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto/llm_response_context.pb.h" +#include "third_party/skia/HEAD/include/core/SkAlphaType.h" +#include "third_party/skia/HEAD/include/core/SkBitmap.h" +#include "third_party/skia/HEAD/include/core/SkImage.h" +#include "third_party/skia/HEAD/include/core/SkImageInfo.h" namespace { @@ -37,14 +44,22 @@ using mediapipe::android::JStringToStdString; using mediapipe::android::ThrowIfError; using mediapipe::java::GetJNIEnv; +const bool kDefaultIncludeTokenCostCalculator = true; + LlmModelSettings ParseModelSettings(void* bytes, int size) { LlmModelSettingsProto input; input.ParseFromArray(bytes, size); LlmModelSettings output; output.model_path = strdup(input.model_path().c_str()); - output.vision_encoder_path = nullptr; - output.vision_adapter_path = nullptr; + output.vision_encoder_path = + input.vision_model_settings().has_encoder_path() + ? strdup(input.vision_model_settings().encoder_path().c_str()) + : nullptr; + output.vision_adapter_path = + input.vision_model_settings().has_adapter_path() + ? strdup(input.vision_model_settings().adapter_path().c_str()) + : nullptr; output.cache_dir = strdup(input.cache_dir().c_str()); output.sequence_batch_size = input.sequence_batch_size(); output.num_decode_steps_per_sync = input.num_decode_steps_per_sync(); @@ -57,6 +72,8 @@ LlmModelSettings ParseModelSettings(void* bytes, int size) { for (int i = 0; i < input.supported_lora_ranks_size(); ++i) { output.supported_lora_ranks[i] = input.supported_lora_ranks(i); } + } else { + output.supported_lora_ranks = nullptr; } output.llm_activation_data_type = kLlmActivationDataTypeDefault; output.num_draft_tokens = 0; @@ -76,14 +93,20 @@ LlmSessionConfig ParseSessionConfig(void* bytes, int size) { if (input.has_lora_path()) { output.lora_path = strdup(input.lora_path().c_str()); } - output.include_token_cost_calculator = true; - output.enable_vision_modality = false; + output.include_token_cost_calculator = + input.graph_config().has_include_token_cost_calculator() + ? input.graph_config().include_token_cost_calculator() + : kDefaultIncludeTokenCostCalculator; + output.enable_vision_modality = input.graph_config().enable_vision_modality(); return output; } void FreeModelSettings(LlmModelSettings* model_settings) { delete model_settings->model_path; + delete model_settings->vision_adapter_path; + delete model_settings->vision_encoder_path; delete model_settings->cache_dir; + delete[] model_settings->supported_lora_ranks; model_settings->model_path = nullptr; model_settings->cache_dir = nullptr; } @@ -211,6 +234,20 @@ JNIEXPORT void JNICALL JNI_METHOD(nativeAddQueryChunk)(JNIEnv* env, jclass thiz, } } +JNIEXPORT void JNICALL JNI_METHOD(nativeAddImage)(JNIEnv* env, jclass thiz, + jlong session_handle, + jlong image_handle) { + char* error_msg = nullptr; + int error_code = LlmInferenceEngine_Session_AddImage( + reinterpret_cast(session_handle), + reinterpret_cast(image_handle), &error_msg); + if (error_code) { + ThrowIfError(env, absl::InternalError( + absl::StrCat("Failed to add image:, %s", error_msg))); + free(error_msg); + } +} + JNIEXPORT jbyteArray JNICALL JNI_METHOD(nativePredictSync)(JNIEnv* env, jclass thiz, jlong session_handle) { LlmResponseContext response_context = LlmInferenceEngine_Session_PredictSync( @@ -259,3 +296,34 @@ JNIEXPORT jint JNICALL JNI_METHOD(nativeSizeInTokens)(JNIEnv* env, jclass thiz, } return size; } + +JNIEXPORT jlong JNICALL JNI_METHOD(nativeCreateSkBitmap)( + JNIEnv* env, jclass thiz, jobject byte_buffer, jint width, jint height, + jint color_type, jint alpha_type) { + const int64_t buffer_size = env->GetDirectBufferCapacity(byte_buffer); + void* buffer_data = env->GetDirectBufferAddress(byte_buffer); + if (buffer_data == nullptr || buffer_size < 0) { + ThrowIfError(env, absl::InternalError("Cannot get direct access to the " + "input buffer. It should be created " + "using allocateDirect.")); + } + + SkColorType sk_color_type = static_cast(color_type); + SkAlphaType sk_alpha_type = static_cast(alpha_type); + SkImageInfo imageInfo = + SkImageInfo::Make(width, height, sk_color_type, sk_alpha_type); + + auto bitmap = std::make_unique(); + bool success = + bitmap->installPixels(imageInfo, buffer_data, imageInfo.minRowBytes()); + if (!success) { + ThrowIfError(env, absl::InternalError("Cannot initialize SkBitmap.")); + } + + return reinterpret_cast(bitmap.release()); +} + +JNIEXPORT void JNICALL JNI_METHOD(nativeDeleteSkBitmap)(JNIEnv*, jclass, + jlong bitmap_handle) { + delete reinterpret_cast(bitmap_handle); +} diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.h b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.h index cadd82a236..6e67c0d240 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.h +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.h @@ -69,6 +69,14 @@ JNIEXPORT void JNICALL JNI_METHOD(nativeDeleteSession)(JNIEnv *, jclass, jlong); JNIEXPORT void JNICALL JNI_METHOD(nativeAddQueryChunk)(JNIEnv *, jclass, jlong, jstring); +/* + * Class: com_google_mediapipe_tasks_core_LlmTaskRunner + * Method: nativeAddImage + * Signature: (JLL)V + */ +JNIEXPORT void JNICALL JNI_METHOD(nativeAddImage)(JNIEnv *, jclass, jlong, + jlong); + /* * Class: com_google_mediapipe_tasks_core_LlmTaskRunner * Method: nativePredictSync @@ -109,6 +117,23 @@ JNIEXPORT void JNICALL JNI_METHOD(nativePredictAsync)(JNIEnv *, jclass, jlong, JNIEXPORT jint JNICALL JNI_METHOD(nativeSizeInTokens)(JNIEnv *, jclass, jlong, jstring); +/* + * Class: com_google_mediapipe_tasks_core_LlmTaskRunner + * Method: nativeCreateSkBitmap + * Signature: (Ljava/nio/ByteBuffer;IIII)J + */ +JNIEXPORT jlong JNICALL JNI_METHOD(nativeCreateSkBitmap)(JNIEnv *, jclass, + jobject, jint, jint, + jint, jint); + +/* + * Class: com_google_mediapipe_tasks_core_LlmTaskRunner + * Method: nativeDeleteSkBitmap + * Signature: (J)V + */ +JNIEXPORT void JNICALL JNI_METHOD(nativeDeleteSkBitmap)(JNIEnv *, jclass, + jlong); + #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto/llm_options.proto b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto/llm_options.proto index 22c647e6a2..e444230313 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto/llm_options.proto +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto/llm_options.proto @@ -34,6 +34,20 @@ message LlmSessionConfig { // The absolute path to the LoRA model asset bundle stored locally on the // device. This is only compatible with GPU models. optional string lora_path = 4; + + // Parameters to customize the graph. + message GraphConfig { + // Whether to configure the graph to include the token cost calculator, + // which allows users to only compute the cost of a prompt. + optional bool include_token_cost_calculator = 1; + + // Whether to configure the graph to include the vision modality. Only one + // of enable_vision_modality or enable_audio_modality can be true currently. + optional bool enable_vision_modality = 2; + } + + // Parameters to customize the graph. + optional GraphConfig graph_config = 5; } // Configurable model parameters for creating an LLM inference engine. @@ -72,4 +86,15 @@ message LlmModelSettings { // means only greedy decoding is supported for any sessions created with this // engine. uint32 max_top_k = 8; + + // A container for vision model related settings. + message VisionModelSettings { + // Path to the vision encoder model file. + optional string encoder_path = 1; + + // Path to the vision adapter model file. + optional string adapter_path = 2; + } + + optional VisionModelSettings vision_model_settings = 9; } From 96e3a694a1a5e82828bd2ca467e1fa07f9772550 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 12 Dec 2024 09:15:46 -0800 Subject: [PATCH 117/126] Add vision modality to the Java LLM API PiperOrigin-RevId: 705528226 --- .../com/google/mediapipe/tasks/core/BUILD | 1 + .../mediapipe/tasks/core/LlmTaskRunner.java | 186 ++++++++++++++++++ .../genai/llminference/GraphOptions.java | 42 ++++ .../genai/llminference/LlmInference.java | 26 ++- .../llminference/LlmInferenceSession.java | 27 +++ .../llminference/VisionModelOptions.java | 36 ++++ 6 files changed, 313 insertions(+), 5 deletions(-) create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/GraphOptions.java create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/VisionModelOptions.java diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD index 8b7e18aff6..656f92c60e 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD @@ -85,6 +85,7 @@ android_library( deps = [ ":core_java", ":logging", + "//mediapipe/java/com/google/mediapipe/framework/image", "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto:llm_options_java_proto_lite", "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto:llm_response_context_java_proto_lite", "//third_party/java/protobuf:protobuf_lite", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/LlmTaskRunner.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/LlmTaskRunner.java index c735969be1..0e83f21dfa 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/LlmTaskRunner.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/LlmTaskRunner.java @@ -15,6 +15,14 @@ package com.google.mediapipe.tasks.core; import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.PixelFormat; +import android.media.Image; +import com.google.mediapipe.framework.image.BitmapExtractor; +import com.google.mediapipe.framework.image.ByteBufferExtractor; +import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.framework.image.MPImageProperties; +import com.google.mediapipe.framework.image.MediaImageExtractor; import com.google.mediapipe.tasks.core.OutputHandler.ProgressListener; import com.google.mediapipe.tasks.core.jni.proto.LlmOptionsProto.LlmModelSettings; import com.google.mediapipe.tasks.core.jni.proto.LlmOptionsProto.LlmSessionConfig; @@ -22,6 +30,7 @@ import com.google.mediapipe.tasks.core.logging.TasksStatsDummyLogger; import com.google.mediapipe.tasks.core.logging.TasksStatsLogger; import com.google.protobuf.InvalidProtocolBufferException; +import java.nio.ByteBuffer; import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; @@ -38,6 +47,109 @@ public final class LlmTaskRunner implements AutoCloseable { private final TasksStatsLogger statsLogger; private final AtomicBoolean isProcessing; + /** + * Describes how pixel bits encode color. A pixel may be an alpha mask, a grayscale, RGB, or ARGB. + * + *

This matches the SkColorType enum in https://api.skia.org/SkColorType_8h.html. + */ + private enum SkColorType { + /** Uninitialized. */ + UNKNOWN(0), + /** Pixel with alpha in 8-bit byte. */ + ALPHA_8(1), + /** Pixel with 5 bits red, 6 bits green, 5 bits blue, in 16-bit word. */ + RGB_565(2), + /** Pixel with 4 bits for alpha, red, green, blue; in 16-bit word. */ + ARGB_4444(3), + /** Pixel with 8 bits for red, green, blue, alpha; in 32-bit word. */ + RGBA_8888(4), + /** Pixel with 8 bits each for red, green, blue; in 32-bit word. */ + RGB_888X(5), + /** Pixel with 8 bits for blue, green, red, alpha; in 32-bit word. */ + BGRA_8888(6), + /** 10 bits for red, green, blue; 2 bits for alpha; in 32-bit word. */ + RGBA_1010102(7), + /** 10 bits for blue, green, red; 2 bits for alpha; in 32-bit word. */ + BGRA_1010102(8), + /** Pixel with 10 bits each for red, green, blue; in 32-bit word. */ + RGB_101010X(9), + /** Pixel with 10 bits each for blue, green, red; in 32-bit word. */ + BGR101010X(10), + /** Pixel with 10 bits each for blue, green, red; in 32-bit word, extended range. */ + BGR_101010X_XR(11), + /** Pixel with 10 bits each for blue, green, red, alpha; in 64-bit word, extended range. */ + BGRA_10101010_XR(12), + /** + * Pixel with 10 used bits (most significant) followed by 6 unused bits for red, green, blue, + * alpha; in 64-bit word. + */ + RGBA_10X6(13), + /** Pixel with grayscale level in 8-bit byte. */ + GRAY_8(14), + /** Pixel with half floats in [0,1] for red, green, blue, alpha; in 64-bit word. */ + RGBA_F16NORM(15), + /** Pixel with half floats for red, green, blue, alpha; in 64-bit word. */ + RGBA_F16(16), + /** Pixel with half floats for red, green, blue; in 64-bit word. */ + RGB_F16F16F16X(17), + /** Pixel using C float for red, green, blue, alpha; in 128-bit word. */ + RGBA_F32(18), + /** Pixel with a uint8_t for red and green. */ + R8G8_UNORM(19), + /** Pixel with a half float for alpha. */ + A16_FLOAT(20), + /** Pixel with a half float for red and green. */ + R16G16_FLOAT(21), + /** Pixel with a little endian uint16_t for alpha. */ + A16_UNORM(22), + /** Pixel with a little endian uint16_t for red and green. */ + R16G16_UNORM(23), + /** Pixel with a little endian uint16_t for red, green, blue and alpha. */ + R16G16B16A16_UNORM(24), + /** Pixel with 8 bits for red, green, blue, alpha; in 32-bit word, gamma encoded. */ + SRGBA_8888(25), + /** Pixel with a uint8_t for red. */ + R8_UNORM(26); + + private final int value; + + SkColorType(int value) { + this.value = value; + } + + /** Returns the integer value associated with this color type. */ + int getValue() { + return value; + } + } + + /** + * Describes how to interpret the alpha component of a pixel. A pixel may be opaque, or alpha, + * describing multiple levels of transparency. + * + *

This matches the SkColorType enum in https://api.skia.org/SkAlphaType_8h.html. + */ + private enum SkAlphaType { + UNIITALIZED(0), + /** Pixel is opaque */ + OPAQUE(1), + /** Pixel components are premultiplied by alpha */ + PREMULTIPLIED(2), + /** Pixel components are independent of alpha */ + UNPREMULTIPLIED(3); + + private final int value; + + SkAlphaType(int value) { + this.value = value; + } + + /** Returns the integer value associated with this alpha type. */ + int getValue() { + return value; + } + }; + /** The session to use for LLM inference calls. */ public static final class LlmSession { private final long sessionHandle; @@ -79,6 +191,21 @@ public void addQueryChunk(LlmSession session, String input) { nativeAddQueryChunk(session.sessionHandle, input); } + /** Adds a new image to the session context. */ + public void addImage(LlmSession session, MPImage input) { + validateState(); + long imageHandle = createImage(input); + try { + // TODO: Remove this dummy chunk. + // Since AddImage cannot distinguish if start_id is being added, + // use a dummy chunk to make sure the start_id is being added properly. + nativeAddQueryChunk(session.sessionHandle, ""); + nativeAddImage(session.sessionHandle, imageHandle); + } finally { + nativeDeleteSkBitmap(imageHandle); + } + } + /** Invokes the LLM with the given session and waits for the result. */ public List predictSync(LlmSession session) { validateState(); @@ -160,6 +287,58 @@ public void close() { } } + private long createImage(MPImage image) { + MPImageProperties properties = image.getContainedImageProperties().get(0); + + SkAlphaType skAlphaType = SkAlphaType.OPAQUE; + ByteBuffer buffer; + SkColorType skColorType; + + int width = image.getWidth(); + int height = image.getHeight(); + + if (properties.getStorageType() == MPImage.STORAGE_TYPE_BYTEBUFFER) { + buffer = ByteBufferExtractor.extract(image); + + switch (properties.getImageFormat()) { + case MPImage.IMAGE_FORMAT_RGBA: + skColorType = SkColorType.RGBA_8888; + break; + case MPImage.IMAGE_FORMAT_RGB: + skColorType = SkColorType.RGB_888X; + break; + case MPImage.IMAGE_FORMAT_ALPHA: + skColorType = SkColorType.ALPHA_8; + break; + default: + throw new UnsupportedOperationException( + "Unsupported MediaPipe Image image format: " + properties.getImageFormat()); + } + } else if (properties.getStorageType() == MPImage.STORAGE_TYPE_BITMAP) { + Bitmap bitmap = BitmapExtractor.extract(image); + if (bitmap.getConfig() != Bitmap.Config.ARGB_8888) { + throw new UnsupportedOperationException("Bitmap must use ARGB_8888 config."); + } + skColorType = SkColorType.RGBA_8888; + + buffer = ByteBuffer.allocateDirect(bitmap.getByteCount()); + bitmap.copyPixelsToBuffer(buffer); + } else if (properties.getStorageType() == MPImage.STORAGE_TYPE_MEDIA_IMAGE) { + Image mediaImage = MediaImageExtractor.extract(image); + if (mediaImage.getFormat() != PixelFormat.RGBA_8888) { + throw new UnsupportedOperationException("Android media image must use RGBA_8888 config."); + } + buffer = mediaImage.getPlanes()[0].getBuffer(); + skColorType = SkColorType.RGBA_8888; + } else { + throw new UnsupportedOperationException( + "Unsupported Image container type: " + properties.getStorageType()); + } + + return nativeCreateSkBitmap( + buffer, width, height, skColorType.getValue(), skAlphaType.getValue()); + } + private void validateState() { if (isProcessing.get()) { throw new IllegalStateException("Previous invocation still processing. Wait for done=true."); @@ -187,4 +366,11 @@ private void validateState() { private static native void nativePredictAsync(long sessionPointer, long callbackContextHandle); private static native int nativeSizeInTokens(long sessionPointer, String input); + + private static native long nativeCreateSkBitmap( + ByteBuffer buffer, int width, int height, int colorType, int alphaType); + + private static native void nativeAddImage(long sessionPointer, long imagePointer); + + private static native void nativeDeleteSkBitmap(long imagePointer); } diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/GraphOptions.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/GraphOptions.java new file mode 100644 index 0000000000..c041917cae --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/GraphOptions.java @@ -0,0 +1,42 @@ +package com.google.mediapipe.tasks.genai.llminference; + +import com.google.auto.value.AutoValue; + +/** Configuration for the inference graph. */ +@AutoValue +public abstract class GraphOptions { + + /** + * Returns whether to configure the graph to include the token cost calculator, which allows users + * to only compute the cost of a prompt. + */ + public abstract boolean includeTokenCostCalculator(); + + /** Returns whether to configure the graph to include the vision modality. */ + public abstract boolean enableVisionModality(); + + /** Returns a new {@link Builder} instance. */ + public static Builder builder() { + return new AutoValue_GraphOptions.Builder() + .setIncludeTokenCostCalculator(true) + .setEnableVisionModality(false); + } + + /** Builder for {@link GraphConfig}. */ + @AutoValue.Builder + public abstract static class Builder { + /** Sets whether to configure the graph to include the token cost calculator. */ + public abstract Builder setIncludeTokenCostCalculator(boolean includeTokenCostCalculator); + + /** Sets whether to configure the graph to include the vision modality. */ + public abstract Builder setEnableVisionModality(boolean enableVisionModality); + + /** AutoValue generated builder method. */ + abstract GraphOptions autoBuild(); + + /** Builds a new {@link GraphConfig} instance. */ + public GraphOptions build() { + return autoBuild(); + } + } +} diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInference.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInference.java index dba1e5b955..194fe6bdba 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInference.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInference.java @@ -35,7 +35,7 @@ public final class LlmInference implements AutoCloseable { /** Creates an LlmInference Task. */ public static LlmInference createFromOptions(Context context, LlmInferenceOptions options) { // Configure LLM model settings. - LlmModelSettings modelSettings = + LlmModelSettings.Builder modelSettings = LlmModelSettings.newBuilder() .setModelPath(options.modelPath()) .setCacheDir(context.getCacheDir().getAbsolutePath()) @@ -43,10 +43,20 @@ public static LlmInference createFromOptions(Context context, LlmInferenceOption .setMaxTokens(options.maxTokens()) .setMaxTopK(options.maxTopK()) .setNumberOfSupportedLoraRanks(options.supportedLoraRanks().size()) - .addAllSupportedLoraRanks(options.supportedLoraRanks()) - .build(); + .addAllSupportedLoraRanks(options.supportedLoraRanks()); - return new LlmInference(context, STATS_TAG, modelSettings, options.resultListener()); + if (options.visionModelOptions().isPresent()) { + VisionModelOptions visionModelOptions = options.visionModelOptions().get(); + + LlmModelSettings.VisionModelSettings.Builder visionModelSettings = + LlmModelSettings.VisionModelSettings.newBuilder(); + visionModelOptions.getEncoderPath().ifPresent(visionModelSettings::setEncoderPath); + visionModelOptions.getAdapterPath().ifPresent(visionModelSettings::setAdapterPath); + + modelSettings.setVisionModelSettings(visionModelSettings.build()); + } + + return new LlmInference(context, STATS_TAG, modelSettings.build(), options.resultListener()); } /** Constructor to initialize an {@link LlmInference}. */ @@ -196,9 +206,12 @@ public abstract static class Builder { */ public abstract Builder setMaxTopK(int maxTopK); - /** The supported lora ranks for the base model. Used by GPU only. */ + /** Sets the supported lora ranks for the base model. Used by GPU only. */ public abstract Builder setSupportedLoraRanks(List supportedLoraRanks); + /** Sets the model options to use for vision modality. */ + public abstract Builder setVisionModelOptions(VisionModelOptions visionModelOptions); + abstract LlmInferenceOptions autoBuild(); /** Validates and builds the {@link ImageGeneratorOptions} instance. */ @@ -232,6 +245,9 @@ public final LlmInferenceOptions build() { /** The error listener to use for the {@link LlmInference#generateAsync} API. */ public abstract Optional errorListener(); + /** The model options to for vision modality. */ + public abstract Optional visionModelOptions(); + /** Returns a new builder with the same values as this instance. */ public abstract Builder toBuilder(); diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInferenceSession.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInferenceSession.java index 2d070cd207..d862667293 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInferenceSession.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/LlmInferenceSession.java @@ -1,6 +1,7 @@ package com.google.mediapipe.tasks.genai.llminference; import com.google.auto.value.AutoValue; +import com.google.mediapipe.framework.image.MPImage; import com.google.mediapipe.tasks.core.LlmTaskRunner; import com.google.mediapipe.tasks.core.LlmTaskRunner.LlmSession; import com.google.mediapipe.tasks.core.jni.proto.LlmOptionsProto.LlmSessionConfig; @@ -37,6 +38,16 @@ public static LlmInferenceSession createFromOptions( sessionConfig.setLoraPath(""); } + if (options.graphOptions().isPresent()) { + GraphOptions graphOptions = options.graphOptions().get(); + LlmSessionConfig.GraphConfig graphConfig = + LlmSessionConfig.GraphConfig.newBuilder() + .setIncludeTokenCostCalculator(graphOptions.includeTokenCostCalculator()) + .setEnableVisionModality(graphOptions.enableVisionModality()) + .build(); + sessionConfig.setGraphConfig(graphConfig); + } + LlmTaskRunner taskRunner = llmInference.getTaskRunner(); LlmSession session = taskRunner.createSession(sessionConfig.build()); return new LlmInferenceSession(taskRunner, session); @@ -60,6 +71,16 @@ public void addQueryChunk(String inputText) { taskRunner.addQueryChunk(session, inputText); } + /** + * Add an image to the session. + * + * @param image a MediaPipe {@link MPImage} object for processing. + * @throws MediaPipeException if there is an internal error. + */ + public void addImage(MPImage image) { + taskRunner.addImage(session, image); + } + /** * Generates a response based on the previously added query chunks synchronously. Use {@link * #addQueryChunk(String)} to add at least one query chunk before calling this function. @@ -170,6 +191,9 @@ public abstract static class Builder { */ public abstract Builder setLoraPath(String loraPath); + /** Sets the parameters to customize the graph. */ + public abstract Builder setGraphOptions(GraphOptions graphOptions); + abstract LlmInferenceSessionOptions autoBuild(); /** Validates and builds the {@link LlmInferenceSessionOptions} instance. */ @@ -196,6 +220,9 @@ public final LlmInferenceSessionOptions build() { */ public abstract Optional loraPath(); + /** Returns the parameters to customize the graph. */ + public abstract Optional graphOptions(); + /** Instantiates a new LlmInferenceOptions builder. */ public static Builder builder() { return new AutoValue_LlmInferenceSession_LlmInferenceSessionOptions.Builder() diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/VisionModelOptions.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/VisionModelOptions.java new file mode 100644 index 0000000000..1effe75ab8 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/genai/llminference/VisionModelOptions.java @@ -0,0 +1,36 @@ +package com.google.mediapipe.tasks.genai.llminference; + +import com.google.auto.value.AutoValue; +import java.util.Optional; + +/** Options for configuring vision modality */ +@AutoValue +public abstract class VisionModelOptions { + /** Returns the path to the vision encoder model file. */ + public abstract Optional getEncoderPath(); + + /** Path to the vision adapter model file. */ + public abstract Optional getAdapterPath(); + + /** Builder for {@link VisionModelOptions}. */ + @AutoValue.Builder + public abstract static class Builder { + /** Sets the path to the vision encoder model file. */ + public abstract Builder setEncoderPath(String encoderPath); + + /** Sets the to the vision adapter model file. */ + public abstract Builder setAdapterPath(String adapterPath); + + abstract VisionModelOptions autoBuild(); + + /** Validates and builds the {@link VisionModelOptions} instance. */ + public final VisionModelOptions build() { + return autoBuild(); + } + } + + /** Instantiates a new VisionModelOption builder. */ + public static Builder builder() { + return new AutoValue_VisionModelOptions.Builder(); + } +} From cc8bfd7187e506b983dc7d8be107c5e93e351c4c Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 12 Dec 2024 14:28:17 -0800 Subject: [PATCH 118/126] No public description PiperOrigin-RevId: 705631706 --- .../cc/genai/inference/calculators/llm_gpu_calculator.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mediapipe/tasks/cc/genai/inference/calculators/llm_gpu_calculator.proto b/mediapipe/tasks/cc/genai/inference/calculators/llm_gpu_calculator.proto index 5c87657e69..5c18a5f3f2 100644 --- a/mediapipe/tasks/cc/genai/inference/calculators/llm_gpu_calculator.proto +++ b/mediapipe/tasks/cc/genai/inference/calculators/llm_gpu_calculator.proto @@ -67,4 +67,7 @@ message LlmGpuCalculatorOptions { reserved 30; proto.SamplerParameters sampler_params = 31; + + // Whether the audio model is loaded in streaming mode or not. + bool audio_streaming_enabled = 32; } From 91baa9b5c32f25bf6abaab6a8d797bf737be315d Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 13 Dec 2024 07:35:33 -0800 Subject: [PATCH 119/126] Enable log message output for messages larger than 4096 bytes. PiperOrigin-RevId: 705879341 --- mediapipe/framework/BUILD | 16 +++++++++++++ mediapipe/framework/tool/BUILD | 6 +++++ .../tool/graph_runtime_info_logger.cc | 5 ++-- mediapipe/framework/validated_graph_config.cc | 23 +++++++++++-------- mediapipe/framework/vlog_utils.cc | 23 +++++++++++++++++++ mediapipe/framework/vlog_utils.h | 22 ++++++++++++++++++ 6 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 mediapipe/framework/vlog_utils.cc create mode 100644 mediapipe/framework/vlog_utils.h diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index dd6e8f828e..9067b6da3f 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -1363,6 +1363,7 @@ cc_library( ":stream_handler_cc_proto", ":subgraph", ":thread_pool_executor_cc_proto", + ":vlog_utils", "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:logging", @@ -1377,6 +1378,7 @@ cc_library( "//mediapipe/framework/tool:validate_name", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", @@ -1965,3 +1967,17 @@ cc_library( "@com_google_absl//absl/strings:string_view", ], ) + +cc_library( + name = "vlog_utils", + srcs = ["vlog_utils.cc"], + hdrs = ["vlog_utils.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + ], +) diff --git a/mediapipe/framework/tool/BUILD b/mediapipe/framework/tool/BUILD index 1b05e71e85..3106fb90fa 100644 --- a/mediapipe/framework/tool/BUILD +++ b/mediapipe/framework/tool/BUILD @@ -987,13 +987,18 @@ cc_library( ":graph_runtime_info_utils", "//mediapipe/framework:calculator_cc_proto", "//mediapipe/framework:graph_runtime_info_cc_proto", + "//mediapipe/framework:vlog_utils", + "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:threadpool", + "@com_google_absl//absl/base:log_severity", "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/log", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/time", ], @@ -1021,6 +1026,7 @@ cc_library( deps = [ "//mediapipe/framework:graph_runtime_info_cc_proto", "//mediapipe/framework:timestamp", + "//mediapipe/framework:vlog_utils", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", diff --git a/mediapipe/framework/tool/graph_runtime_info_logger.cc b/mediapipe/framework/tool/graph_runtime_info_logger.cc index 5d85292ad4..6ed4704c46 100644 --- a/mediapipe/framework/tool/graph_runtime_info_logger.cc +++ b/mediapipe/framework/tool/graph_runtime_info_logger.cc @@ -12,13 +12,14 @@ #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/tool/graph_runtime_info_utils.h" +#include "mediapipe/framework/vlog_utils.h" namespace mediapipe::tool { constexpr absl::Duration kDefaultCaptureInterval = absl::Seconds(10); GraphRuntimeInfoLogger::GraphRuntimeInfoLogger() - : thread_pool_("GraphRuntimeInfoLogger", 1) {} + : thread_pool_("GraphRuntimeInfoLogger", /*num_threads=*/1) {} GraphRuntimeInfoLogger::~GraphRuntimeInfoLogger() { Stop(); }; @@ -49,7 +50,7 @@ absl::Status GraphRuntimeInfoLogger::StartInBackground( << runtime_info_str.status(); return; } - ABSL_LOG(INFO) << *runtime_info_str; + VlogLargeMessage(/*verbose_level=*/0, *runtime_info_str); shutdown_signal_.WaitForNotificationWithTimeout(interval); } }); diff --git a/mediapipe/framework/validated_graph_config.cc b/mediapipe/framework/validated_graph_config.cc index 41a88fb4de..e28afec3fe 100644 --- a/mediapipe/framework/validated_graph_config.cc +++ b/mediapipe/framework/validated_graph_config.cc @@ -20,6 +20,7 @@ #include "absl/container/flat_hash_set.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "absl/log/log.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -46,6 +47,7 @@ #include "mediapipe/framework/tool/status_util.h" #include "mediapipe/framework/tool/subgraph_expansion.h" #include "mediapipe/framework/tool/validate_name.h" +#include "mediapipe/framework/vlog_utils.h" namespace mediapipe { @@ -325,11 +327,12 @@ absl::Status ValidatedGraphConfig::Initialize( const GraphServiceManager* service_manager) { RET_CHECK(!initialized_) << "ValidatedGraphConfig can be initialized only once."; - -#if !defined(MEDIAPIPE_MOBILE) - VLOG(1) << "ValidatedGraphConfig::Initialize called with config:\n" - << input_config.DebugString(); -#endif + if (VLOG_IS_ON(1)) { + VlogLargeMessage( + /*verbose_level=*/1, + absl::StrCat("ValidatedGraphConfig::Initialize called with config:\n", + input_config.DebugString())); + } config_ = std::move(input_config); MP_RETURN_IF_ERROR( @@ -401,10 +404,12 @@ absl::Status ValidatedGraphConfig::Initialize( MP_RETURN_IF_ERROR(ValidateExecutors()); -#if !defined(MEDIAPIPE_MOBILE) - VLOG(1) << "ValidatedGraphConfig produced canonical config:\n" - << config_.DebugString(); -#endif + if (VLOG_IS_ON(1)) { + VlogLargeMessage( + /*verbose_level=*/1, + absl::StrCat("ValidatedGraphConfig produced canonical config:\n", + config_.DebugString())); + } initialized_ = true; return absl::OkStatus(); diff --git a/mediapipe/framework/vlog_utils.cc b/mediapipe/framework/vlog_utils.cc new file mode 100644 index 0000000000..6faaabb3b0 --- /dev/null +++ b/mediapipe/framework/vlog_utils.cc @@ -0,0 +1,23 @@ +#include "mediapipe/framework/vlog_utils.h" + +#include "absl/log/absl_log.h" +#include "absl/log/log.h" +#include "absl/strings/str_split.h" // IWYU pragma: keep +#include "absl/strings/string_view.h" +#include "mediapipe/framework/port/logging.h" + +namespace mediapipe { + +void VlogLargeMessage(int verbose_level, absl::string_view message) { +#if defined(MEDIAPIPE_MOBILE) + if (message.size() > 4096) { + for (const auto& line : absl::StrSplit(message, '\n')) { + VLOG(verbose_level) << line; + } + return; + } +#endif + VLOG(verbose_level) << message; +} + +} // namespace mediapipe diff --git a/mediapipe/framework/vlog_utils.h b/mediapipe/framework/vlog_utils.h new file mode 100644 index 0000000000..2d953963ff --- /dev/null +++ b/mediapipe/framework/vlog_utils.h @@ -0,0 +1,22 @@ +#ifndef MEDIAPIPE_FRAMEWORK_VLOG_UTILS_H_ +#define MEDIAPIPE_FRAMEWORK_VLOG_UTILS_H_ + +#include "absl/strings/string_view.h" + +namespace mediapipe { + +// Helper to log a message with a large number of lines on mobile (Android). +// +// On Android, the logcat will truncate the log if the message is larger than +// 4096 bytes. This function splits the message by new lines and logs each +// line separately. To ensure the log message is only generated when VLOG is +// turned on, use this function in a VLOG_IS_ON() block: +// if (VLOG_IS_ON(1)) { +// VlogLargeMessage( +// /*verbose_level=*/1, GenerateDebugString()); +// } +void VlogLargeMessage(int verbose_level, absl::string_view message); + +} // namespace mediapipe + +#endif // MEDIAPIPE_FRAMEWORK_VLOG_UTILS_H_ From 3714d4ea1e5c17e5c3038531c12936e405c2d85b Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 13 Dec 2024 09:44:14 -0800 Subject: [PATCH 120/126] Internal PiperOrigin-RevId: 705912456 --- .../tasks/java/com/google/mediapipe/tasks/core/jni/BUILD | 2 +- .../tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD index 91238b512e..57e7b7e298 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD @@ -70,10 +70,10 @@ cc_library( "//mediapipe/tasks/cc/genai/inference/proto:tflite_delegate_options_cc_proto", "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto:llm_options_cc_proto", "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto:llm_response_context_cc_proto", - "//third_party/skia/HEAD:core", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", + "@skia//:core", ] + select({ "//conditions:default": [ "//mediapipe/tasks/cc/genai/inference/c:libllm_inference_engine_cpu", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc index b232033f53..4c22f55666 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc @@ -24,16 +24,16 @@ #include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" +#include "include/core/SkAlphaType.h" // from @skia +#include "include/core/SkBitmap.h" // from @skia +#include "include/core/SkImage.h" // from @skia +#include "include/core/SkImageInfo.h" // from @skia #include "mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/jni_util.h" #include "mediapipe/tasks/cc/genai/inference/c/llm_inference_engine.h" #include "mediapipe/tasks/cc/genai/inference/proto/tflite_delegate_options.pb.h" #include "mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto/llm_options.pb.h" #include "mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto/llm_response_context.pb.h" -#include "third_party/skia/HEAD/include/core/SkAlphaType.h" -#include "third_party/skia/HEAD/include/core/SkBitmap.h" -#include "third_party/skia/HEAD/include/core/SkImage.h" -#include "third_party/skia/HEAD/include/core/SkImageInfo.h" namespace { From 92939c69f164f618ed8c23f2fa12038b5b0aceaa Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 13 Dec 2024 10:40:57 -0800 Subject: [PATCH 121/126] No public description PiperOrigin-RevId: 705930362 --- .../cc/genai/inference/utils/xnn_utils/llm_weights.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.h b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.h index ec1ad4d4db..ae5fd1b335 100644 --- a/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.h +++ b/mediapipe/tasks/cc/genai/inference/utils/xnn_utils/llm_weights.h @@ -28,7 +28,6 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "mediapipe/tasks/cc/genai/inference/proto/llm_params.pb.h" -#include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/graph_builder.h" #include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/pack_weights_cache.h" #include "mediapipe/tasks/cc/genai/inference/utils/xnn_utils/xnn_tensor.h" #include "tensorflow/lite/model_builder.h" @@ -238,11 +237,6 @@ class LlmWeightsLoader { LlmParams& llm_params() { return params_; } const LlmParams& llm_params() const { return params_; } - // Returns the XnnWeightsCache that could work with weights loader, if any. - virtual std::shared_ptr GetXnnWeightsCache() { - return nullptr; - } - protected: virtual absl::StatusOr LoadSelfAttention( int layer_id); @@ -269,10 +263,6 @@ class DefaultLlmWeightsLoader : public LlmWeightsLoader { absl::string_view weight_path, const LlmParams& params, std::shared_ptr flat_buffer_model = nullptr); - std::shared_ptr GetXnnWeightsCache() override { - return xnn_weights_cache_; - } - private: std::shared_ptr xnn_weights_cache_; }; From ab1de4fced96c18b79eda4ab407b04eb08301ea4 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 13 Dec 2024 12:20:55 -0800 Subject: [PATCH 122/126] Remove unused Proto dependency PiperOrigin-RevId: 705964263 --- mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD | 1 - mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD index 57e7b7e298..94aad6fe4a 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/BUILD @@ -67,7 +67,6 @@ cc_library( "//mediapipe/java/com/google/mediapipe/framework/jni:class_registry", "//mediapipe/java/com/google/mediapipe/framework/jni:jni_util", "//mediapipe/tasks/cc/genai/inference/c:llm_inference_engine_hdr", # needed with ENABLE_ODML_MAVEN_BUILD - "//mediapipe/tasks/cc/genai/inference/proto:tflite_delegate_options_cc_proto", "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto:llm_options_cc_proto", "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto:llm_response_context_cc_proto", "@com_google_absl//absl/log:absl_log", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc index 4c22f55666..89ba4d3563 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/llm.cc @@ -31,7 +31,6 @@ #include "mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/jni_util.h" #include "mediapipe/tasks/cc/genai/inference/c/llm_inference_engine.h" -#include "mediapipe/tasks/cc/genai/inference/proto/tflite_delegate_options.pb.h" #include "mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto/llm_options.pb.h" #include "mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni/proto/llm_response_context.pb.h" From 80b1f432164565c8669be3a4c19b94d38abe46ca Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 16 Dec 2024 15:03:50 -0800 Subject: [PATCH 123/126] Bump MP version for release 0.10.21. PiperOrigin-RevId: 706847793 --- version.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.bzl b/version.bzl index 6ffe9b5520..146c5d9fc6 100644 --- a/version.bzl +++ b/version.bzl @@ -2,4 +2,4 @@ # The next version of MediaPipe (e.g. the version that is currently in development). # This version should be bumped after every release. -MEDIAPIPE_FULL_VERSION = "0.10.20" +MEDIAPIPE_FULL_VERSION = "0.10.21" From 033534ea805716743f2287659dbe9160adb3bd4a Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 17 Dec 2024 01:28:08 -0800 Subject: [PATCH 124/126] Enable the option of exporting a model with a fixed batch size. PiperOrigin-RevId: 706998697 --- .../python/text/text_classifier/text_classifier.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py index 07811d8baa..a881562cf7 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py @@ -859,6 +859,7 @@ def export_model( self, model_name: str = "model.tflite", quantization_config: Optional[quantization.QuantizationConfig] = None, + batch_size: int | None = None, ): """Converts and saves the model to a TFLite file with metadata included. @@ -873,6 +874,8 @@ def export_model( model_name: File name to save TFLite model with metadata. The full export path is {self._hparams.export_dir}/{model_name}. quantization_config: The configuration for model quantization. + batch_size: Inference batch size to use for the TFlite model. Default is + None, which means the batch size is dynamic. """ tf.io.gfile.makedirs(self._hparams.export_dir) tflite_file = os.path.join(self._hparams.export_dir, model_name) @@ -883,16 +886,19 @@ def export_model( shape=(self._model_options.seq_len,), dtype=tf.int32, name="input_word_ids", + batch_size=batch_size, ), input_mask=tf.keras.layers.Input( shape=(self._model_options.seq_len,), dtype=tf.int32, name="input_mask", + batch_size=batch_size, ), input_type_ids=tf.keras.layers.Input( shape=(self._model_options.seq_len,), dtype=tf.int32, name="input_type_ids", + batch_size=batch_size, ), ) output = self._model(constant_len_inputs) From bd154ba09d10f9c0454917b0d47c07366f942610 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 17 Dec 2024 02:39:24 -0800 Subject: [PATCH 125/126] Fix incorrect name in ValidateRequiredSidePacketTypes status message. PiperOrigin-RevId: 707016548 --- mediapipe/framework/validated_graph_config.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/framework/validated_graph_config.cc b/mediapipe/framework/validated_graph_config.cc index e28afec3fe..e5d44b1efe 100644 --- a/mediapipe/framework/validated_graph_config.cc +++ b/mediapipe/framework/validated_graph_config.cc @@ -1090,7 +1090,7 @@ absl::Status ValidatedGraphConfig::ValidateRequiredSidePacketTypes( } if (!statuses.empty()) { return tool::CombinedStatus( - "ValidateRequiredSidePackets failed to validate: ", statuses); + "ValidateRequiredSidePacketTypes failed to validate: ", statuses); } return absl::OkStatus(); } From d12c81054c2b1d5e2a1a67238faed033e5e8281d Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 17 Dec 2024 06:53:22 -0800 Subject: [PATCH 126/126] Delegate memory-mapping the model file to the resource system PiperOrigin-RevId: 707080341 --- mediapipe/util/tflite/BUILD | 3 -- mediapipe/util/tflite/tflite_model_loader.cc | 55 ++++---------------- mediapipe/util/tflite/tflite_model_loader.h | 1 - 3 files changed, 10 insertions(+), 49 deletions(-) diff --git a/mediapipe/util/tflite/BUILD b/mediapipe/util/tflite/BUILD index 587e1407bb..9013c716af 100644 --- a/mediapipe/util/tflite/BUILD +++ b/mediapipe/util/tflite/BUILD @@ -154,15 +154,12 @@ cc_library_with_tflite( ], visibility = ["//visibility:public"], deps = [ - ":error_reporter", "//mediapipe/framework:resources", "//mediapipe/framework/api2:packet", - "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/util:resource_util", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", diff --git a/mediapipe/util/tflite/tflite_model_loader.cc b/mediapipe/util/tflite/tflite_model_loader.cc index 90359624b2..9d144418d0 100644 --- a/mediapipe/util/tflite/tflite_model_loader.cc +++ b/mediapipe/util/tflite/tflite_model_loader.cc @@ -15,75 +15,40 @@ #include "mediapipe/util/tflite/tflite_model_loader.h" #include +#include #include -#include -#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/api2/packet.h" -#include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status_macros.h" #include "mediapipe/framework/resources.h" -#include "mediapipe/util/resource_util.h" -#include "mediapipe/util/tflite/error_reporter.h" -#include "tensorflow/lite/allocation.h" #include "tensorflow/lite/model_builder.h" namespace mediapipe { -using ::mediapipe::util::tflite::ErrorReporter; -using ::tflite::Allocation; using ::tflite::FlatBufferModel; -using ::tflite::MMAPAllocation; absl::StatusOr> TfLiteModelLoader::LoadFromPath( const Resources& resources, const std::string& path, bool try_mmap) { std::string model_path = path; - bool file_exists = file::Exists(model_path).ok(); - if (!file_exists) { - // TODO: get rid of manual resolving with PathToResourceAsFile - // as soon as it's incorporated into GetResourceContents. - absl::StatusOr resolved_model_path = - mediapipe::PathToResourceAsFile(model_path); - if (resolved_model_path.ok()) { - VLOG(2) << "Loading the model from " << model_path; - model_path = *std::move(resolved_model_path); - file_exists = true; - } - } - - // Try to memory map file if available. Falls back to loading from buffer on - // error. - if (file_exists && try_mmap && MMAPAllocation::IsSupported()) { - ErrorReporter error_reporter; - std::unique_ptr allocation = - std::make_unique(model_path.c_str(), &error_reporter); - - if (!error_reporter.HasError()) { - auto model = FlatBufferModel::BuildFromAllocation(std::move(allocation)); - if (model) { - return api2::MakePacket( - model.release(), [](FlatBufferModel* model) { delete model; }); - } - } - - ABSL_LOG(WARNING) << "Failed to memory map model from path '" << model_path - << "'; falling back to loading from buffer. Error: " - << error_reporter.message(); - } - // Load model resource. - MP_ASSIGN_OR_RETURN(std::unique_ptr model_resource, - resources.Get(model_path)); + MP_ASSIGN_OR_RETURN( + std::unique_ptr model_resource, + resources.Get( + model_path, + Resources::Options{ + .mmap_mode = try_mmap ? std::make_optional(MMapMode::kMMapOrRead) + : std::nullopt})); absl::string_view model_view = model_resource->ToStringView(); auto model = FlatBufferModel::VerifyAndBuildFromBuffer(model_view.data(), model_view.size()); - RET_CHECK(model) << "Failed to load model from path " << model_path; + RET_CHECK(model) << "Failed to load model from path (resource ID) " + << model_path; return api2::MakePacket( model.release(), [model_resource = model_resource.release()]( FlatBufferModel* model) mutable { diff --git a/mediapipe/util/tflite/tflite_model_loader.h b/mediapipe/util/tflite/tflite_model_loader.h index f7fd119f3d..1389ad694b 100644 --- a/mediapipe/util/tflite/tflite_model_loader.h +++ b/mediapipe/util/tflite/tflite_model_loader.h @@ -19,7 +19,6 @@ #include #include -#include "absl/base/attributes.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "mediapipe/framework/api2/packet.h"