Skip to content

Commit

Permalink
RenderEngineVtk (and GltfClient) support in-memory meshes (RobotLocom…
Browse files Browse the repository at this point in the history 
…otion#21954)
  • Loading branch information
@SeanCurtis-TRI @RussTedrake
SeanCurtis-TRI authored and RussTedrake committed Dec 15, 2024
1 parent b43c0eb commit 5ae7f1d
Show file tree
Hide file tree
Showing 13 changed files with 626 additions and 206 deletions.
5 changes: 4 additions & 1 deletion geometry/render_gltf_client/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,11 @@ drake_cc_library(
":internal_merge_gltf",
":internal_render_client",
"//common:essential",
"//common:find_resource",
"//geometry/render:render_camera",
"//geometry/render_vtk:internal_render_engine_vtk",
"//systems/sensors:image",
"@common_robotics_utilities",
"@nlohmann_internal//:nlohmann",
"@vtk_internal//:vtkCommonCore",
"@vtk_internal//:vtkCommonMath",
Expand All @@ -138,7 +140,7 @@ drake_cc_library(
"//common:drake_export",
"//common:essential",
"//common:find_resource",
"@common_robotics_utilities",
"//geometry:mesh_source",
"@nlohmann_internal//:nlohmann",
],
)
Expand Down Expand Up @@ -236,6 +238,7 @@ drake_cc_googletest(
name = "internal_render_engine_gltf_client_test",
data = [
":test_resources",
"//geometry/render:test_models",
],
tags = vtk_test_tags(),
deps = [
Expand Down
126 changes: 47 additions & 79 deletions geometry/render_gltf_client/internal_merge_gltf.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,9 @@
#include <string>
#include <utility>

#include <common_robotics_utilities/base64_helpers.hpp>
#include <fmt/format.h>

#include "drake/common/drake_assert.h"
#include "drake/common/find_resource.h"
#include "drake/common/fmt_ostream.h"
#include "drake/common/ssize.h"
#include "drake/common/text_logging.h"
Expand Down Expand Up @@ -78,7 +76,7 @@ void MaybeOffsetNamedIndexInTree(json* j_ptr, std::string_view name,
// Merges *names* of extensions from j2 into j1 (preventing duplicates). This
// should not be used for merging an "extensions" property which contains
// arbitrary objects.
void MergeExtensionNames(json* j1, json&& j2, const std::string& array_name) {
void MergeExtensionNames(json* j1, json&& j2, const string& array_name) {
if (j2.contains(array_name)) {
json& extensions1 = (*j1)[array_name];
for (auto& extension2 : j2[array_name]) {
Expand Down Expand Up @@ -118,28 +116,28 @@ std::string_view to_string(ContainerType x) {
}

// Attempts to merge the key-value pairs from j2 into j1. To merge successfully,
// all every key in j2 that appears in j1 must have the same value. If
// successful, every key-value pair in j2 is in j1 upon return.
// every key in j2 that appears in j1 must have the same value. If successful,
// every key-value pair in j2 is in j1 upon return.
//
// Note: a "successful" merge could still leave j1 unchanged, because either
// there was nothing in j2, or j2's contents were already present in j1.
//
// @throws if merge was not successful.
// @pre j1->is_object() && j2.is_object().
void MergeTrees(json* j1, json&& j2, const std::string& blob_name,
ContainerType container_type,
const std::filesystem::path& j2_path, MergeRecord* record) {
void MergeTrees(json* j1, json&& j2, const string& blob_name,
ContainerType container_type, const string& j2_name,
MergeRecord* record) {
DRAKE_DEMAND(j1->is_object() && j2.is_object());
// First confirm there are no collisions.
for (auto& [key, value] : j2.items()) {
if (j1->contains(key)) {
if ((*j1)[key] != value) {
const std::filesystem::path& j1_path = record->FindSourcePath(*j1);
const string& j1_name = record->FindSourceName(*j1);
throw std::runtime_error(fmt::format(
"Error in merging '{}.{}.{}'; two glTF files have different "
"values. '{}' defines it as {}, but '{}' defines it as {}.",
to_string(container_type), blob_name, key, j1_path.string(),
fmt_streamed((*j1)[key]), j2_path.string(), fmt_streamed(value)));
to_string(container_type), blob_name, key, j1_name,
fmt_streamed((*j1)[key]), j2_name, fmt_streamed(value)));
}
}
}
Expand All @@ -148,16 +146,16 @@ void MergeTrees(json* j1, json&& j2, const std::string& blob_name,
for (auto& [key, value] : j2.items()) {
if (j1->contains(key)) continue;
(*j1)[key] = std::move(value);
record->AddElementTree((*j1)[key], j2_path);
record->AddElementTree((*j1)[key], j2_name);
}
}

// Merge user data. This can be used for merging the arbitrary collection of
// objects contained in either "extras" or "extensions". As documented, if there
// is a collision between the two json trees, we will throw with helpful info.
void MergeBlobs(json* j1, json&& j2, const std::string& blob_name,
ContainerType container_type,
const std::filesystem::path& j2_path, MergeRecord* record) {
void MergeBlobs(json* j1, json&& j2, const string& blob_name,
ContainerType container_type, const string& j2_name,
MergeRecord* record) {
if (j2.contains(blob_name)) {
json& blob2 = j2[blob_name];
if (blob2.is_null()) {
Expand All @@ -172,19 +170,19 @@ void MergeBlobs(json* j1, json&& j2, const std::string& blob_name,
// We can simply take blob2 verbatim (whether object or primitive).
// 3. In all other cases, merging is not possible.
if (blob1.is_object() && blob2.is_object()) {
MergeTrees(&blob1, std::move(blob2), blob_name, container_type, j2_path,
MergeTrees(&blob1, std::move(blob2), blob_name, container_type, j2_name,
record);
} else if (blob1.is_null()) {
// j1 doesn't have the blob, go ahead and replace it with j2's.
blob1 = std::move(blob2);
record->AddElementTree(blob1, j2_path);
record->AddElementTree(blob1, j2_name);
} else {
const std::filesystem::path& j1_path = record->FindSourcePath(blob1);
const string& j1_name = record->FindSourceName(blob1);
throw std::runtime_error(fmt::format(
"Error in merging '{}.{}'. To merge, the must both be objects. "
"'{}' has {} and '{}' has {}.",
to_string(container_type), blob_name, j1_path.string(),
blob1.is_object() ? "an object" : "a primitive", j2_path.string(),
to_string(container_type), blob_name, j1_name,
blob1.is_object() ? "an object" : "a primitive", j2_name,
blob2.is_object() ? "an object" : "a primitive"));
}
}
Expand All @@ -195,56 +193,26 @@ void MergeBlobs(json* j1, json&& j2, const std::string& blob_name,
// the root glTF node and Scene nodes. All other nodes simply get concatenated
// to lists.
void MergeExtrasAndExtensions(json* j1, json&& j2, ContainerType container_type,
const std::filesystem::path& j2_path,
MergeRecord* record) {
MergeBlobs(j1, std::move(j2), "extras", container_type, j2_path, record);
MergeBlobs(j1, std::move(j2), "extensions", container_type, j2_path, record);
}

// If `item_inout` has a field named `uri` and it is not a `data:` URI, replaces
// the field's value with a base64-encoded `data:` URI.
//
// In glTF 2.0, URIs can only appear in two places:
// "images": [ { "uri": "some.png" } ]
// "buffers": [ { "uri": "some.bin", "byteLength": 1024 } ]
//
// When merging a glTF, we expect that our images- and buffers-handling logic
// must call this function as a subroutine.
void MaybeEmbedDataUri(nlohmann::json* item_inout,
const std::filesystem::path& base_path) {
DRAKE_DEMAND(item_inout != nullptr);
nlohmann::json& item = *item_inout;
if (!item.contains("uri")) {
return;
}
const std::string_view uri = item["uri"].template get<std::string_view>();
if (uri.substr(0, 5) == "data:") {
return;
}
const std::string content = ReadFileOrThrow(base_path / uri);
item["uri"] =
fmt::format("data:application/octet-stream;base64,{}",
common_robotics_utilities::base64_helpers::Encode(
std::vector<uint8_t>(content.begin(), content.end())));
const string& j2_name, MergeRecord* record) {
MergeBlobs(j1, std::move(j2), "extras", container_type, j2_name, record);
MergeBlobs(j1, std::move(j2), "extensions", container_type, j2_name, record);
}

} // namespace

MergeRecord::MergeRecord(std::filesystem::path initial_path) {
source_paths_.push_back(std::move(initial_path));
MergeRecord::MergeRecord(string initial_name) {
source_names_.push_back(std::move(initial_name));
}

const std::filesystem::path& MergeRecord::FindSourcePath(
const json& element) const {
const string& MergeRecord::FindSourceName(const json& element) const {
const auto iter = merged_trees_.find(&element);
DRAKE_DEMAND(iter != merged_trees_.end());
return source_paths_.at(iter->second);
return source_names_.at(iter->second);
}

void MergeRecord::AddElementTree(const json& root,
const std::filesystem::path& source_path) {
const int source_index = ssize(source_paths_);
source_paths_.push_back(source_path);
void MergeRecord::AddElementTree(const json& root, const string& source_name) {
const int source_index = ssize(source_names_);
source_names_.push_back(source_name);
// Recursively register all of the json elements in the tree rooted at
// `subtree`.
std::function<void(const json&)> add_tree_recurse = [&](const json& subtree) {
Expand All @@ -258,9 +226,14 @@ void MergeRecord::AddElementTree(const json& root,
add_tree_recurse(root);
}

json ReadJsonFile(const std::filesystem::path& json_path) {
std::ifstream f(json_path);
return json::parse(f);
json ReadJsonFile(const MeshSource& mesh_source) {
if (mesh_source.is_path()) {
std::ifstream f(mesh_source.path());
return json::parse(f);
} else {
DRAKE_DEMAND(mesh_source.is_in_memory());
return json::parse(mesh_source.in_memory().mesh_file.contents());
}
}

json GltfMatrixFromEigenMatrix(const Matrix4<double>& matrix) {
Expand All @@ -286,8 +259,7 @@ Matrix4<double> EigenMatrixFromGltfMatrix(const json& matrix_json) {
return T;
}

void MergeDefaultScenes(json* j1, json&& j2,
const std::filesystem::path& j2_path,
void MergeDefaultScenes(json* j1, json&& j2, const string& j2_name,
MergeRecord* record) {
int index_1 = j1->contains("scene") ? (*j1)["scene"].get<int>() : 0;
int index_2 = j2.contains("scene") ? j2["scene"].get<int>() : 0;
Expand All @@ -304,7 +276,7 @@ void MergeDefaultScenes(json* j1, json&& j2,
}
}
MergeExtrasAndExtensions(&scene_1, std::move(scene_2),
ContainerType::kDefaultScene, j2_path, record);
ContainerType::kDefaultScene, j2_name, record);
}

void MergeNodes(json* j1, json&& j2) {
Expand Down Expand Up @@ -410,12 +382,11 @@ void MergeBufferViews(json* j1, json&& j2) {
}
}

void MergeBuffers(json* j1, json&& j2,
const std::filesystem::path& j2_base_path) {
void MergeBuffers(json* j1, json&& j2) {
if (j2.contains("buffers")) {
json& buffers = (*j1)["buffers"];
for (auto& buffer : j2["buffers"]) {
MaybeEmbedDataUri(&buffer, j2_base_path);
// Buffers can simply be copied over.
buffers.push_back(std::move(buffer));
}
}
Expand All @@ -435,15 +406,13 @@ void MergeTextures(json* j1, json&& j2) {
}
}

void MergeImages(json* j1, json&& j2,
const std::filesystem::path& j2_base_path) {
void MergeImages(json* j1, json&& j2) {
if (j2.contains("images")) {
json& images = (*j1)["images"];
// Offsets to update used indices.
const int buf_offset = ArraySize(*j1, "bufferViews");
for (auto& image : j2["images"]) {
MaybeOffsetNamedIndex(&image, "bufferView", buf_offset);
MaybeEmbedDataUri(&image, j2_base_path);
images.push_back(std::move(image));
}
}
Expand All @@ -463,37 +432,36 @@ void MergeSamplers(json* j1, json&& j2) {
}
}

void MergeGltf(json* j1, json&& j2, const std::filesystem::path& j2_path,
void MergeGltf(json* j1, json&& j2, const string& j2_name,
MergeRecord* record) {
json& asset1 = (*j1)["asset"];
json& asset2 = j2["asset"];
DRAKE_DEMAND(!(asset1.is_null() || asset2.is_null()));
const std::filesystem::path j2_directory = j2_path.parent_path();

asset1["generator"] = "Drake glTF merger";
// TODO(SeanCurtis-TRI): We're not doing anything to the copyright. Should we?
DRAKE_DEMAND(asset1["version"].get<string>() == "2.0");
DRAKE_DEMAND(asset2["version"].get<string>() == "2.0");
MergeExtrasAndExtensions(j1, std::move(j2), ContainerType::kGltf, j2_path,
MergeExtrasAndExtensions(j1, std::move(j2), ContainerType::kGltf, j2_name,
record);
MergeExtrasAndExtensions(&asset1, std::move(asset2), ContainerType::kAsset,
j2_path, record);
j2_name, record);

// Don't change the order. Because we mutate j1 as we go, we need to make sure
// we only mutate something after we've processed everything that depends on
// it.
// https://github.com/KhronosGroup/glTF-Tutorials/blob/master/gltfTutorial/gltfTutorial_002_BasicGltfStructure.md
MergeDefaultScenes(j1, std::move(j2), j2_path, record);
MergeDefaultScenes(j1, std::move(j2), j2_name, record);
MergeNodes(j1, std::move(j2));
MergeMeshes(j1, std::move(j2));
MergeMaterials(j1, std::move(j2));
MergeCameras(j1, std::move(j2));
MergeAccessors(j1, std::move(j2));
MergeTextures(j1, std::move(j2));
MergeImages(j1, std::move(j2), j2_directory);
MergeImages(j1, std::move(j2));
MergeSamplers(j1, std::move(j2));
MergeBufferViews(j1, std::move(j2));
MergeBuffers(j1, std::move(j2), j2_directory);
MergeBuffers(j1, std::move(j2));
MergeExtensionsUsed(j1, std::move(j2));
MergeExtensionsRequired(j1, std::move(j2));

Expand Down
Loading

0 comments on commit 5ae7f1d

Please sign in to comment.