diff --git a/geometry/render_gltf_client/BUILD.bazel b/geometry/render_gltf_client/BUILD.bazel index ae2b29d826b1..58a74ccf66dd 100644 --- a/geometry/render_gltf_client/BUILD.bazel +++ b/geometry/render_gltf_client/BUILD.bazel @@ -117,9 +117,11 @@ drake_cc_library( ":internal_merge_gltf", ":internal_render_client", "//common:essential", + "//common:find_resource", "//geometry/render:render_camera", "//geometry/render_vtk:internal_render_engine_vtk", "//systems/sensors:image", + "@common_robotics_utilities", "@nlohmann_internal//:nlohmann", "@vtk_internal//:vtkCommonCore", "@vtk_internal//:vtkCommonMath", @@ -138,7 +140,7 @@ drake_cc_library( "//common:drake_export", "//common:essential", "//common:find_resource", - "@common_robotics_utilities", + "//geometry:mesh_source", "@nlohmann_internal//:nlohmann", ], ) @@ -236,6 +238,7 @@ drake_cc_googletest( name = "internal_render_engine_gltf_client_test", data = [ ":test_resources", + "//geometry/render:test_models", ], tags = vtk_test_tags(), deps = [ diff --git a/geometry/render_gltf_client/internal_merge_gltf.cc b/geometry/render_gltf_client/internal_merge_gltf.cc index 2a95c850effb..47714be04857 100644 --- a/geometry/render_gltf_client/internal_merge_gltf.cc +++ b/geometry/render_gltf_client/internal_merge_gltf.cc @@ -6,11 +6,9 @@ #include #include -#include #include #include "drake/common/drake_assert.h" -#include "drake/common/find_resource.h" #include "drake/common/fmt_ostream.h" #include "drake/common/ssize.h" #include "drake/common/text_logging.h" @@ -78,7 +76,7 @@ void MaybeOffsetNamedIndexInTree(json* j_ptr, std::string_view name, // Merges *names* of extensions from j2 into j1 (preventing duplicates). This // should not be used for merging an "extensions" property which contains // arbitrary objects. -void MergeExtensionNames(json* j1, json&& j2, const std::string& array_name) { +void MergeExtensionNames(json* j1, json&& j2, const string& array_name) { if (j2.contains(array_name)) { json& extensions1 = (*j1)[array_name]; for (auto& extension2 : j2[array_name]) { @@ -118,28 +116,28 @@ std::string_view to_string(ContainerType x) { } // Attempts to merge the key-value pairs from j2 into j1. To merge successfully, -// all every key in j2 that appears in j1 must have the same value. If -// successful, every key-value pair in j2 is in j1 upon return. +// every key in j2 that appears in j1 must have the same value. If successful, +// every key-value pair in j2 is in j1 upon return. // // Note: a "successful" merge could still leave j1 unchanged, because either // there was nothing in j2, or j2's contents were already present in j1. // // @throws if merge was not successful. // @pre j1->is_object() && j2.is_object(). -void MergeTrees(json* j1, json&& j2, const std::string& blob_name, - ContainerType container_type, - const std::filesystem::path& j2_path, MergeRecord* record) { +void MergeTrees(json* j1, json&& j2, const string& blob_name, + ContainerType container_type, const string& j2_name, + MergeRecord* record) { DRAKE_DEMAND(j1->is_object() && j2.is_object()); // First confirm there are no collisions. for (auto& [key, value] : j2.items()) { if (j1->contains(key)) { if ((*j1)[key] != value) { - const std::filesystem::path& j1_path = record->FindSourcePath(*j1); + const string& j1_name = record->FindSourceName(*j1); throw std::runtime_error(fmt::format( "Error in merging '{}.{}.{}'; two glTF files have different " "values. '{}' defines it as {}, but '{}' defines it as {}.", - to_string(container_type), blob_name, key, j1_path.string(), - fmt_streamed((*j1)[key]), j2_path.string(), fmt_streamed(value))); + to_string(container_type), blob_name, key, j1_name, + fmt_streamed((*j1)[key]), j2_name, fmt_streamed(value))); } } } @@ -148,16 +146,16 @@ void MergeTrees(json* j1, json&& j2, const std::string& blob_name, for (auto& [key, value] : j2.items()) { if (j1->contains(key)) continue; (*j1)[key] = std::move(value); - record->AddElementTree((*j1)[key], j2_path); + record->AddElementTree((*j1)[key], j2_name); } } // Merge user data. This can be used for merging the arbitrary collection of // objects contained in either "extras" or "extensions". As documented, if there // is a collision between the two json trees, we will throw with helpful info. -void MergeBlobs(json* j1, json&& j2, const std::string& blob_name, - ContainerType container_type, - const std::filesystem::path& j2_path, MergeRecord* record) { +void MergeBlobs(json* j1, json&& j2, const string& blob_name, + ContainerType container_type, const string& j2_name, + MergeRecord* record) { if (j2.contains(blob_name)) { json& blob2 = j2[blob_name]; if (blob2.is_null()) { @@ -172,19 +170,19 @@ void MergeBlobs(json* j1, json&& j2, const std::string& blob_name, // We can simply take blob2 verbatim (whether object or primitive). // 3. In all other cases, merging is not possible. if (blob1.is_object() && blob2.is_object()) { - MergeTrees(&blob1, std::move(blob2), blob_name, container_type, j2_path, + MergeTrees(&blob1, std::move(blob2), blob_name, container_type, j2_name, record); } else if (blob1.is_null()) { // j1 doesn't have the blob, go ahead and replace it with j2's. blob1 = std::move(blob2); - record->AddElementTree(blob1, j2_path); + record->AddElementTree(blob1, j2_name); } else { - const std::filesystem::path& j1_path = record->FindSourcePath(blob1); + const string& j1_name = record->FindSourceName(blob1); throw std::runtime_error(fmt::format( "Error in merging '{}.{}'. To merge, the must both be objects. " "'{}' has {} and '{}' has {}.", - to_string(container_type), blob_name, j1_path.string(), - blob1.is_object() ? "an object" : "a primitive", j2_path.string(), + to_string(container_type), blob_name, j1_name, + blob1.is_object() ? "an object" : "a primitive", j2_name, blob2.is_object() ? "an object" : "a primitive")); } } @@ -195,56 +193,26 @@ void MergeBlobs(json* j1, json&& j2, const std::string& blob_name, // the root glTF node and Scene nodes. All other nodes simply get concatenated // to lists. void MergeExtrasAndExtensions(json* j1, json&& j2, ContainerType container_type, - const std::filesystem::path& j2_path, - MergeRecord* record) { - MergeBlobs(j1, std::move(j2), "extras", container_type, j2_path, record); - MergeBlobs(j1, std::move(j2), "extensions", container_type, j2_path, record); -} - -// If `item_inout` has a field named `uri` and it is not a `data:` URI, replaces -// the field's value with a base64-encoded `data:` URI. -// -// In glTF 2.0, URIs can only appear in two places: -// "images": [ { "uri": "some.png" } ] -// "buffers": [ { "uri": "some.bin", "byteLength": 1024 } ] -// -// When merging a glTF, we expect that our images- and buffers-handling logic -// must call this function as a subroutine. -void MaybeEmbedDataUri(nlohmann::json* item_inout, - const std::filesystem::path& base_path) { - DRAKE_DEMAND(item_inout != nullptr); - nlohmann::json& item = *item_inout; - if (!item.contains("uri")) { - return; - } - const std::string_view uri = item["uri"].template get(); - if (uri.substr(0, 5) == "data:") { - return; - } - const std::string content = ReadFileOrThrow(base_path / uri); - item["uri"] = - fmt::format("data:application/octet-stream;base64,{}", - common_robotics_utilities::base64_helpers::Encode( - std::vector(content.begin(), content.end()))); + const string& j2_name, MergeRecord* record) { + MergeBlobs(j1, std::move(j2), "extras", container_type, j2_name, record); + MergeBlobs(j1, std::move(j2), "extensions", container_type, j2_name, record); } } // namespace -MergeRecord::MergeRecord(std::filesystem::path initial_path) { - source_paths_.push_back(std::move(initial_path)); +MergeRecord::MergeRecord(string initial_name) { + source_names_.push_back(std::move(initial_name)); } -const std::filesystem::path& MergeRecord::FindSourcePath( - const json& element) const { +const string& MergeRecord::FindSourceName(const json& element) const { const auto iter = merged_trees_.find(&element); DRAKE_DEMAND(iter != merged_trees_.end()); - return source_paths_.at(iter->second); + return source_names_.at(iter->second); } -void MergeRecord::AddElementTree(const json& root, - const std::filesystem::path& source_path) { - const int source_index = ssize(source_paths_); - source_paths_.push_back(source_path); +void MergeRecord::AddElementTree(const json& root, const string& source_name) { + const int source_index = ssize(source_names_); + source_names_.push_back(source_name); // Recursively register all of the json elements in the tree rooted at // `subtree`. std::function add_tree_recurse = [&](const json& subtree) { @@ -258,9 +226,14 @@ void MergeRecord::AddElementTree(const json& root, add_tree_recurse(root); } -json ReadJsonFile(const std::filesystem::path& json_path) { - std::ifstream f(json_path); - return json::parse(f); +json ReadJsonFile(const MeshSource& mesh_source) { + if (mesh_source.is_path()) { + std::ifstream f(mesh_source.path()); + return json::parse(f); + } else { + DRAKE_DEMAND(mesh_source.is_in_memory()); + return json::parse(mesh_source.in_memory().mesh_file.contents()); + } } json GltfMatrixFromEigenMatrix(const Matrix4& matrix) { @@ -286,8 +259,7 @@ Matrix4 EigenMatrixFromGltfMatrix(const json& matrix_json) { return T; } -void MergeDefaultScenes(json* j1, json&& j2, - const std::filesystem::path& j2_path, +void MergeDefaultScenes(json* j1, json&& j2, const string& j2_name, MergeRecord* record) { int index_1 = j1->contains("scene") ? (*j1)["scene"].get() : 0; int index_2 = j2.contains("scene") ? j2["scene"].get() : 0; @@ -304,7 +276,7 @@ void MergeDefaultScenes(json* j1, json&& j2, } } MergeExtrasAndExtensions(&scene_1, std::move(scene_2), - ContainerType::kDefaultScene, j2_path, record); + ContainerType::kDefaultScene, j2_name, record); } void MergeNodes(json* j1, json&& j2) { @@ -410,12 +382,11 @@ void MergeBufferViews(json* j1, json&& j2) { } } -void MergeBuffers(json* j1, json&& j2, - const std::filesystem::path& j2_base_path) { +void MergeBuffers(json* j1, json&& j2) { if (j2.contains("buffers")) { json& buffers = (*j1)["buffers"]; for (auto& buffer : j2["buffers"]) { - MaybeEmbedDataUri(&buffer, j2_base_path); + // Buffers can simply be copied over. buffers.push_back(std::move(buffer)); } } @@ -435,15 +406,13 @@ void MergeTextures(json* j1, json&& j2) { } } -void MergeImages(json* j1, json&& j2, - const std::filesystem::path& j2_base_path) { +void MergeImages(json* j1, json&& j2) { if (j2.contains("images")) { json& images = (*j1)["images"]; // Offsets to update used indices. const int buf_offset = ArraySize(*j1, "bufferViews"); for (auto& image : j2["images"]) { MaybeOffsetNamedIndex(&image, "bufferView", buf_offset); - MaybeEmbedDataUri(&image, j2_base_path); images.push_back(std::move(image)); } } @@ -463,37 +432,36 @@ void MergeSamplers(json* j1, json&& j2) { } } -void MergeGltf(json* j1, json&& j2, const std::filesystem::path& j2_path, +void MergeGltf(json* j1, json&& j2, const string& j2_name, MergeRecord* record) { json& asset1 = (*j1)["asset"]; json& asset2 = j2["asset"]; DRAKE_DEMAND(!(asset1.is_null() || asset2.is_null())); - const std::filesystem::path j2_directory = j2_path.parent_path(); asset1["generator"] = "Drake glTF merger"; // TODO(SeanCurtis-TRI): We're not doing anything to the copyright. Should we? DRAKE_DEMAND(asset1["version"].get() == "2.0"); DRAKE_DEMAND(asset2["version"].get() == "2.0"); - MergeExtrasAndExtensions(j1, std::move(j2), ContainerType::kGltf, j2_path, + MergeExtrasAndExtensions(j1, std::move(j2), ContainerType::kGltf, j2_name, record); MergeExtrasAndExtensions(&asset1, std::move(asset2), ContainerType::kAsset, - j2_path, record); + j2_name, record); // Don't change the order. Because we mutate j1 as we go, we need to make sure // we only mutate something after we've processed everything that depends on // it. // https://github.com/KhronosGroup/glTF-Tutorials/blob/master/gltfTutorial/gltfTutorial_002_BasicGltfStructure.md - MergeDefaultScenes(j1, std::move(j2), j2_path, record); + MergeDefaultScenes(j1, std::move(j2), j2_name, record); MergeNodes(j1, std::move(j2)); MergeMeshes(j1, std::move(j2)); MergeMaterials(j1, std::move(j2)); MergeCameras(j1, std::move(j2)); MergeAccessors(j1, std::move(j2)); MergeTextures(j1, std::move(j2)); - MergeImages(j1, std::move(j2), j2_directory); + MergeImages(j1, std::move(j2)); MergeSamplers(j1, std::move(j2)); MergeBufferViews(j1, std::move(j2)); - MergeBuffers(j1, std::move(j2), j2_directory); + MergeBuffers(j1, std::move(j2)); MergeExtensionsUsed(j1, std::move(j2)); MergeExtensionsRequired(j1, std::move(j2)); diff --git a/geometry/render_gltf_client/internal_merge_gltf.h b/geometry/render_gltf_client/internal_merge_gltf.h index 235d3f860889..8e5f5564d2e1 100644 --- a/geometry/render_gltf_client/internal_merge_gltf.h +++ b/geometry/render_gltf_client/internal_merge_gltf.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include @@ -8,14 +8,15 @@ #include "drake/common/drake_export.h" #include "drake/common/eigen_types.h" +#include "drake/geometry/mesh_source.h" namespace drake { namespace geometry { namespace render_gltf_client { namespace internal { -/* Returns the parsed result of the indicated json file. */ -nlohmann::json ReadJsonFile(const std::filesystem::path& json_path); +/* Returns the parsed result of the json file indicated by the given source. */ +nlohmann::json ReadJsonFile(const MeshSource& mesh_source); /* Creates the glTF matrix array from the given 4x4 matrix. @returns A json list of 16 number values. */ @@ -37,33 +38,33 @@ Matrix4 EigenMatrixFromGltfMatrix(const nlohmann::json& matrix_json); we can effectively report the source of the conflict. */ class DRAKE_NO_EXPORT MergeRecord { public: - /* The initial merge record should be created with the path of the initial - target glTF's source; it need not be a path to an existing file. */ - explicit MergeRecord(std::filesystem::path initial_path); + /* The initial merge record should be created with the name of the initial + target glTF's source. */ + explicit MergeRecord(std::string initial_name); - /* Finds the source path for the given element. + /* Finds the source name for the given element. @pre element is part of this merge record. */ - const std::filesystem::path& FindSourcePath( - const nlohmann::json& element) const; + const std::string& FindSourceName(const nlohmann::json& element) const; /* Adds the json tree with the given `root` to the record associated with the - given `source_name`. - @pre `source_name` is not in the record already. */ + given `source_name`. `source_name` will appear in error messages if there + is a problem with an element, so should be name that is meaningful to the + end user. */ void AddElementTree(const nlohmann::json& root, - const std::filesystem::path& source_path); + const std::string& source_name); private: /* A map from a json pointer in the target glTF structure to the index of the - source path from which it came. The mapped values should all be valid - indices into `source_paths_`. + source name from which it came. The mapped values should all be valid + indices into `source_names_`. Note: This works because nlohmann::json is linked-list-esque. Each node is allocated on the heap and they don't move just because additional children get included. */ std::unordered_map merged_trees_; - /* The paths of all sources contributing to the composition of j1. */ - std::vector source_paths_; + /* The names of all sources contributing to the composition of j1. */ + std::vector source_names_; }; /* Merges the default scene from j2 into j1's default scene. @@ -72,14 +73,18 @@ class DRAKE_NO_EXPORT MergeRecord { property. If undefined, it is interpreted as zero. It will also attempt to merge the extras and extensions between the two scenes. - @param j1 The glTF root element. - @param j2 The glTF root element. + @param j1 The glTF root element. + @param j2 The glTF root element. + @param j2_name A label to use for referring to the glTF in j2 in error + messages if problems are encountered while processing j2. + If multiple glTF files are merged, each should have a unique + name. + @param record The accumulated record of the merge. @pre Both j1 and j2 have a valid default scene. @throws if there are merge conflicts between the scenes' "extra" or "extensions" data. */ void MergeDefaultScenes(nlohmann::json* j1, nlohmann::json&& j2, - const std::filesystem::path& j2_path, - MergeRecord* record); + const std::string& j2_name, MergeRecord* record); /* Merges the "extensionsUsed" array from j2 into j1. */ void MergeExtensionsUsed(nlohmann::json* j1, nlohmann::json&& j2); @@ -105,20 +110,14 @@ void MergeAccessors(nlohmann::json* j1, nlohmann::json&& j2); /* Merges the "bufferViews" array from j2 into j1. */ void MergeBufferViews(nlohmann::json* j1, nlohmann::json&& j2); -/* Merges the "buffers" array from j2 into j1. As part of that process, converts -any `uri`s with relative files into embedded `data:`. The `j2_directory` is the -base path for resolving relative filenames against. */ -void MergeBuffers(nlohmann::json* j1, nlohmann::json&& j2, - const std::filesystem::path& j2_directory); +/* Merges the "buffers" array from j2 into j1. */ +void MergeBuffers(nlohmann::json* j1, nlohmann::json&& j2); /* Merges the "textures" array from j2 into j1. */ void MergeTextures(nlohmann::json* j1, nlohmann::json&& j2); -/* Merges the "images" array from j2 into j1. As part of that process, converts -any `uri`s with relative files into embedded `data:`. The `j2_directory` is the -base path for resolving relative filenames against. */ -void MergeImages(nlohmann::json* j1, nlohmann::json&& j2, - const std::filesystem::path& j2_directory); +/* Merges the "images" array from j2 into j1. */ +void MergeImages(nlohmann::json* j1, nlohmann::json&& j2); /* Merges the "samplers" array from j2 into j1. */ void MergeSamplers(nlohmann::json* j1, nlohmann::json&& j2); @@ -158,25 +157,28 @@ void MergeSamplers(nlohmann::json* j1, nlohmann::json&& j2); 2. We attempt to merge the "extras" and "extensions" at the glTF level, and in the "asset" and merged Scene properties. If there is any problem in merging, we throw an exception detailing the problem. - 3. When j2 refers to external resources (i.e., has `uri`s with relative paths) - those resources will be embedded directly into j1 using `data:` uris. This - is important for two reasons: - a. We expect to send j1 over the network, in which case filesystem paths - wouldn't make any sense. - b. We have no idea what filesystem path j1 lives at, so we can't do the - path algebra to convert the j2-relative paths to j1-relative paths. - Even if we added j1's path as a new argument, the path algebra would - still be somewhat difficult to get correct 100% of the time. - Note that to ensure that j1 never refers to external files, the caller must - ensure that it didn't have any to begin with. Our only promise is to not add - any new external files to j1; we don't touch anything that's already there. + 3. This process makes no effort to distinguish between data and file URIs. They + simply get merged as is; no action can be taken here to reconcile file URIs + in j2 with the non-existent location of j1. The caller takes full + responsibility to handle this in one of two ways: + a. make j2 an embedded glTF (so there are no file URIs to resolve), or + b. track the files referenced by file URI and place them relative to j1's + final, on-disk location. + Note: RenderEngineGltfClient converts all file uris into data uris before + invoking this, implementing option (a). This explicitly excludes skin data, animation, and morph target elements (although the underlying data contained in buffers remains). + @param[in/out] j1 The glTF's json structure to merge into. + @param[in] j2 The glTF's json to merge from. + @param[in] j2_name A label to use for referring to the glTF in j2 in error + messages if problems are encountered while processing j2. + @param[out] record The records of the origins of components in the merged + glTF json. @pre Both j1 and j2 indicate version 2.0 glTF files. */ void MergeGltf(nlohmann::json* j1, nlohmann::json&& j2, - const std::filesystem::path& j2_path, MergeRecord* record); + const std::string& j2_name, MergeRecord* record); } // namespace internal } // namespace render_gltf_client diff --git a/geometry/render_gltf_client/internal_render_engine_gltf_client.cc b/geometry/render_gltf_client/internal_render_engine_gltf_client.cc index 55e30fb2f983..e51084c51088 100644 --- a/geometry/render_gltf_client/internal_render_engine_gltf_client.cc +++ b/geometry/render_gltf_client/internal_render_engine_gltf_client.cc @@ -8,14 +8,18 @@ #include #include #include +#include // To ease build system upkeep, we annotate VTK includes with their deps. +#include #include // vtkRenderingCore #include // vtkIOExport #include // vtkCommonMath #include // vtkCommonCore +#include "drake/common/find_resource.h" #include "drake/common/never_destroyed.h" +#include "drake/common/overloaded.h" #include "drake/common/ssize.h" #include "drake/common/text_logging.h" @@ -164,8 +168,8 @@ std::string GetSceneFileName(ImageType image_type, int64_t scene_id) { translation, *and* scale components. */ std::map> FindRootNodes(const nlohmann::json& gltf) { std::map> roots; - std::set indices; if (gltf.contains("nodes")) { + std::set indices; // Cull children. const nlohmann::json& nodes = gltf["nodes"]; const int node_count = ssize(nodes); @@ -490,7 +494,7 @@ void RenderEngineGltfClient::ExportScene(const std::string& export_path, const Rgba color = RenderEngine::MakeRgbFromLabel(record.label); ChangeToLabelMaterials(&temp, color); } - MergeGltf(&gltf, std::move(temp), record.path.string(), &merge_record); + MergeGltf(&gltf, std::move(temp), record.name, &merge_record); } // TODO(SeanCurtis-TRI): Update materials for label images. Because the gltf @@ -531,37 +535,99 @@ bool RenderEngineGltfClient::DoRemoveGeometry(GeometryId id) { } } -void RenderEngineGltfClient::ImplementGeometry(const Convex& convex, - void* user_data) { - ImplementMesh(convex.filename(), convex.scale(), user_data); -} - void RenderEngineGltfClient::ImplementGeometry(const Mesh& mesh, void* user_data) { - ImplementMesh(mesh.filename(), mesh.scale(), user_data); -} - -void RenderEngineGltfClient::ImplementMesh( - const std::filesystem::path& mesh_path, double scale, void* user_data) { auto& data = *static_cast(user_data); - const std::string extension = Mesh(mesh_path.string()).extension(); + const std::string extension = mesh.extension(); if (extension == ".obj") { - data.accepted = ImplementObj(mesh_path, scale, data); + // This invokes RenderEngineVtk::ImplementObj(). + data.accepted = ImplementObj(mesh, data); } else if (extension == ".gltf") { - data.accepted = ImplementGltf(mesh_path, scale, data); + data.accepted = ImplementGltf(mesh, data); } else { static const logging::Warn one_time( - "RenderEngineGltfClient only supports Mesh/Convex specifications which " - "use .obj or .gltf files. Mesh specifications using other mesh types " + "RenderEngineGltfClient only supports Mesh specifications which use " + ".obj or .gltf files. Mesh specifications using other mesh types " "(e.g., .stl, .dae, etc.) will be ignored."); data.accepted = false; } } +namespace { + +// If `item_inout` has a field named `uri` and it is not a `data:` URI, replaces +// the field's value with a base64-encoded `data:` URI. +// +// In glTF 2.0, URIs can only appear in two places: +// "images": [ { "uri": "some.png" } ] +// "buffers": [ { "uri": "some.bin", "byteLength": 1024 } ] +// +// As documented on MergeGltf(), this is how RenderEngineGltfClient converts +// external resources to embedded ata URIs. +void MaybeEmbedDataUri(nlohmann::json* item_inout, + const MeshSource& mesh_source, + std::string_view array_name) { + DRAKE_DEMAND(item_inout != nullptr); + nlohmann::json& item = *item_inout; + if (!item.contains("uri")) { + return; + } + const std::string_view uri = item["uri"].template get(); + if (uri.substr(0, 5) == "data:") { + return; + } + std::string content; + if (mesh_source.is_path()) { + content = ReadFileOrThrow(mesh_source.path().parent_path() / uri); + } else { + DRAKE_DEMAND(mesh_source.is_in_memory()); + const auto file_source_iter = + mesh_source.in_memory().supporting_files.find(uri); + if (file_source_iter == mesh_source.in_memory().supporting_files.end()) { + throw std::runtime_error(fmt::format( + "RenderEngineGltfClient cannot add an in-memory Mesh. The Mesh's " + "glTF ('{}') file names a uri ('{}') for {} that is not contained " + "within the supporting files.", + mesh_source.in_memory().mesh_file.filename_hint(), uri, array_name)); + } + content = std::visit(overloaded{[](const fs::path& path) { + return ReadFileOrThrow(path); + }, + [](const MemoryFile& file) { + return file.contents(); + }}, + file_source_iter->second); + } + + // Note: content may still be empty; we'll defer to the server to handle it. + + item["uri"] = + fmt::format("data:application/octet-stream;base64,{}", + common_robotics_utilities::base64_helpers::Encode( + std::vector(content.begin(), content.end()))); +} + +void EmbedFileUris(nlohmann::json* gltf_ptr, const MeshSource& mesh_source) { + nlohmann::json& gltf = *gltf_ptr; + // Iterate through buffers and images. + for (std::string_view array_name : {"buffers", "images"}) { + auto& array = gltf[array_name]; + for (size_t i = 0; i < array.size(); ++i) { + MaybeEmbedDataUri(&array[i], mesh_source, array_name); + } + } +} + +} // namespace + bool RenderEngineGltfClient::ImplementGltf( - const std::filesystem::path& gltf_path, double scale, - const RenderEngineVtk::RegistrationData& data) { - nlohmann::json mesh_data = ReadJsonFile(gltf_path); + const Mesh& mesh, const RenderEngineVtk::RegistrationData& data) { + nlohmann::json mesh_data = ReadJsonFile(mesh.source()); + + // We'll end up merging this glTF into the VTK-generated glTF and broadcasting + // it over a wire. The idea of "file-relative" URIs becomes meaningless at + // that point. So, we'll simply convert all file URIs to data URIs. + EmbedFileUris(&mesh_data, mesh.source()); // TODO(SeanCurtis-TRI) What to do about a gltf that has no materials? We need // to apply the same logic of the data.properties as we do to OBJ. We'll @@ -569,12 +635,14 @@ bool RenderEngineGltfClient::ImplementGltf( // of materials. std::map> root_nodes = FindRootNodes(mesh_data); - SetRootPoses(&mesh_data, root_nodes, data.X_WG, scale, true); + SetRootPoses(&mesh_data, root_nodes, data.X_WG, mesh.scale(), true); DRAKE_DEMAND(!gltfs_.contains(data.id)); + const MeshSource& mesh_source = mesh.source(); + const std::string gltf_name = mesh_source.description(); gltfs_.insert({data.id, - {gltf_path, std::move(mesh_data), std::move(root_nodes), scale, - GetRenderLabelOrThrow(data.properties)}}); + {gltf_name, std::move(mesh_data), std::move(root_nodes), + mesh.scale(), GetRenderLabelOrThrow(data.properties)}}); return true; } diff --git a/geometry/render_gltf_client/internal_render_engine_gltf_client.h b/geometry/render_gltf_client/internal_render_engine_gltf_client.h index e28199bd3bde..2b9dcc035c45 100644 --- a/geometry/render_gltf_client/internal_render_engine_gltf_client.h +++ b/geometry/render_gltf_client/internal_render_engine_gltf_client.h @@ -94,11 +94,8 @@ class DRAKE_NO_EXPORT RenderEngineGltfClient const math::RigidTransformd& X_WG) override; bool DoRemoveGeometry(GeometryId id) override; using RenderEngineVtk::ImplementGeometry; - void ImplementGeometry(const Convex& convex, void* user_data) override; void ImplementGeometry(const Mesh& mesh, void* user_data) override; - void ImplementMesh(const std::filesystem::path& mesh_path, double scale, - void* user_data); /* Adds a .gltf to the scene for the id currently being reified (data->id). Returns true if added, false if ignored (for whatever reason). @@ -108,14 +105,16 @@ class DRAKE_NO_EXPORT RenderEngineGltfClient common extensions. So, by injecting the files directly into the exported glTF file, we maintain whatever declarations the source glTF had without the lossy filter provided by VTK. */ - bool ImplementGltf(const std::filesystem::path& gltf_path, double scale, + bool ImplementGltf(const Mesh& mesh, const RenderEngineVtk::RegistrationData& data); std::unique_ptr render_client_; struct GltfRecord { - // The path for the .gltf file. - std::filesystem::path path; + // The name for the .gltf file. + // If the glTF came from disk, it will be the file path, otherwise the + // filename hint associated with the in-memory mesh. + std::string name; // The contents of a glTF file registered as Mesh or Convex. nlohmann::json contents; // The root nodes of the gltf file represented as a mapping from the node's diff --git a/geometry/render_gltf_client/test/internal_merge_gltf_test.cc b/geometry/render_gltf_client/test/internal_merge_gltf_test.cc index 150f1360b96e..463edf04ae51 100644 --- a/geometry/render_gltf_client/test/internal_merge_gltf_test.cc +++ b/geometry/render_gltf_client/test/internal_merge_gltf_test.cc @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -22,6 +21,8 @@ namespace render_gltf_client { namespace internal { namespace { +namespace fs = std::filesystem; + using nlohmann::json; using std::string; using std::vector; @@ -577,7 +578,7 @@ class MergeTest : public testing::TestWithParam { static vector MergeBuffersCases() { vector cases; auto merge = [](json* j1, json&& j2) { - MergeBuffers(j1, std::move(j2), "/unused/dir"); + MergeBuffers(j1, std::move(j2)); }; VerbatimCopy(&cases, "buffers", merge); return cases; @@ -594,7 +595,7 @@ class MergeTest : public testing::TestWithParam { static vector MergeImageCases() { vector cases; auto merge = [](json* j1, json&& j2) { - MergeImages(j1, std::move(j2), "/unused/dir"); + MergeImages(j1, std::move(j2)); }; BumpIndex(&cases, "images", "bufferViews", "bufferView", merge); VerbatimCopy(&cases, "images", merge); @@ -820,34 +821,13 @@ TEST_P(MergeFailureTest, Evaluate) { test_case.expected_error); } -/* Check that a buffer with several relative URIs is converted to data URIs. */ -GTEST_TEST(MergeGltf, BufferDataUri) { - json target = ReadJsonFile(FindResourceOrThrow( - "drake/geometry/render_gltf_client/test/red_box.gltf")); - const std::filesystem::path source_path = - FindResourceOrThrow("drake/geometry/render/test/meshes/cube3.gltf"); - MergeRecord record("test_target"); - EXPECT_NO_THROW( - MergeGltf(&target, ReadJsonFile(source_path), source_path, &record)); - - // Expect one buffer URI from each of the two gltf files. - ASSERT_EQ(target["buffers"].size(), 2); - EXPECT_THAT(target["buffers"][0]["uri"], testing::StartsWith("data:")); - EXPECT_THAT(target["buffers"][1]["uri"], testing::StartsWith("data:")); - - // Expect two image URIs, both from cube3.gltf. - ASSERT_EQ(target["images"].size(), 2); - EXPECT_THAT(target["images"][0]["uri"], testing::StartsWith("data:")); - EXPECT_THAT(target["images"][1]["uri"], testing::StartsWith("data:")); -} - /* Simply call MergeGltf on real glTF files and makes sure it doesn't throw. */ GTEST_TEST(MergeGltf, Smoke) { - json target = ReadJsonFile(FindResourceOrThrow( - "drake/geometry/render_gltf_client/test/red_box.gltf")); + json target = ReadJsonFile(fs::path(FindResourceOrThrow( + "drake/geometry/render_gltf_client/test/red_box.gltf"))); - json source = ReadJsonFile(FindResourceOrThrow( - "drake/geometry/render_gltf_client/test/textured_green_box.gltf")); + json source = ReadJsonFile(fs::path(FindResourceOrThrow( + "drake/geometry/render_gltf_client/test/textured_green_box.gltf"))); MergeRecord record("test_target"); EXPECT_NO_THROW( @@ -859,7 +839,7 @@ GTEST_TEST(MergeGltf, Smoke) { if (const char* dir = std::getenv("TEST_UNDECLARED_OUTPUTS_DIR")) { /* target now contains the merged result. */ json& merged = target; - std::ofstream f(std::filesystem::path(dir) / "merged.gltf"); + std::ofstream f(fs::path(dir) / "merged.gltf"); // Set the stream so the json gets "pretty-formatted" with a 2-space // indent. f << std::setw(2); diff --git a/geometry/render_gltf_client/test/internal_render_engine_gltf_client_test.cc b/geometry/render_gltf_client/test/internal_render_engine_gltf_client_test.cc index f2f744d11e8d..158d80ccb1a6 100644 --- a/geometry/render_gltf_client/test/internal_render_engine_gltf_client_test.cc +++ b/geometry/render_gltf_client/test/internal_render_engine_gltf_client_test.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -14,6 +15,7 @@ #include "drake/common/find_resource.h" #include "drake/common/fmt_eigen.h" +#include "drake/common/memory_file.h" #include "drake/common/scope_exit.h" #include "drake/common/temp_directory.h" #include "drake/common/test_utilities/eigen_matrix_compare.h" @@ -304,9 +306,9 @@ class RenderEngineGltfClientGltfTest : public ::testing::Test { const std::filesystem::path temp_dir_; }; -/* RenderEngineGltfClient departs from RenderEngineVtk in registering Mesh and - Convex. It defers to RenderEngineVtk for meshes with .obj extensions and - handles everything else. +/* RenderEngineGltfClient departs from RenderEngineVtk in registering Mesh. It + defers to RenderEngineVtk for meshes with .obj extensions and handles + everything else. We do limited testing of the obj case, merely confirm that registration reports true and the geometry id reports as registered. @@ -319,11 +321,6 @@ TEST_F(RenderEngineGltfClientGltfTest, RegisteringMeshes) { // also a warning, but we're not testing for that). EXPECT_FALSE(engine_.RegisterVisual(GeometryId::get_new_id(), Mesh("some.stl"), properties_, X_WG_)); - // This is the only time we'll explicitly test Convex. Given Mesh and Convex - // yield similar results *here*, we'll assume they'll be similar for - // successful registration as well. - EXPECT_FALSE(engine_.RegisterVisual(GeometryId::get_new_id(), - Convex("some.stl"), properties_, X_WG_)); // Objs and glTFs are both accepted. AddObj(); @@ -354,6 +351,171 @@ TEST_F(RenderEngineGltfClientGltfTest, RegisteringMeshes) { // elaborate. It is tested below in a dedicated test. } +/* RenderEngineGltfClient should accept in-memory Mesh shapes. */ +TEST_F(RenderEngineGltfClientGltfTest, InMemoryMeshes) { + // The same .obj used in AddObj(). + const std::string obj_path = + FindResourceOrThrow("drake/geometry/render_gltf_client/test/tri.obj"); + const GeometryId obj_id = GeometryId::get_new_id(); + EXPECT_TRUE(engine_.RegisterVisual( + obj_id, Mesh(InMemoryMesh{MemoryFile::Make(obj_path)}), properties_, + X_WG_)); + + // The same .gltf used in AddGltf(). + const std::string gltf_path = FindResourceOrThrow( + "drake/geometry/render_gltf_client/test/tri_tree.gltf"); + const GeometryId gltf_id = GeometryId::get_new_id(); + EXPECT_TRUE(engine_.RegisterVisual( + gltf_id, Mesh(InMemoryMesh{MemoryFile::Make(gltf_path)}, scale_), + properties_, X_WG_)); + + const std::map& gltfs = + Tester::gltfs(engine_); + ASSERT_TRUE(gltfs.contains(gltf_id)); + const Tester::GltfRecord& record = gltfs.at(gltf_id); + EXPECT_EQ(record.scale, scale_); + EXPECT_EQ(record.label, label_); + // There are three nodes in the gltf, but only two root nodes. One of the + // root nodes is empty. + EXPECT_EQ(record.contents["nodes"].size(), 3); + EXPECT_EQ(record.root_nodes.size(), 2); + // The root node data extracted directly from tri_tree.gltf. + const std::map> expected_roots{ + {1, RigidTransformd(Vector3d(1, 3, -2)).GetAsMatrix4()}, // root_tri + {2, RigidTransformd(Vector3d(-1, -3, 2)).GetAsMatrix4()}}; // empty_root + for (const int index : {1, 2}) { + EXPECT_TRUE( + CompareMatrices(record.root_nodes.at(index), expected_roots.at(index))) + << "Index: " << index; + } +} + +/* If a file referenced in a file glTF file uri is missing, we should throw. */ +TEST_F(RenderEngineGltfClientGltfTest, GltfMissingSupportingFile) { + const fs::path src_path = + FindResourceOrThrow("drake/geometry/render/test/meshes/cube2.gltf"); + const fs::path src_dir = src_path.parent_path(); + const fs::path temp_dir = temp_directory(); + const fs::path gltf_path = temp_dir / "cube2.gltf"; + fs::copy_file(src_path, gltf_path); + + const Mesh disk_mesh(gltf_path); + const GeometryId disk_id = GeometryId::get_new_id(); + DRAKE_EXPECT_THROWS_MESSAGE( + engine_.RegisterVisual(disk_id, disk_mesh, properties_, X_WG_), + "Error reading from.*cube2.bin.*"); + + const Mesh memory_mesh(InMemoryMesh{MemoryFile::Make(gltf_path)}); + const GeometryId memory_id = GeometryId::get_new_id(); + DRAKE_EXPECT_THROWS_MESSAGE( + engine_.RegisterVisual(memory_id, memory_mesh, properties_, X_WG_), + ".*cube2.bin.*not contained within the supporting files."); +} + +/* RenderEngineGltfClient has the responsibility of turning file URIs into + data URIs and this must work with in-memory meshes as well. + + In this case, we pass it an embedded glTF file and confirm the file doesn't + change; everything is already a data uri. */ +TEST_F(RenderEngineGltfClientGltfTest, InMemoryDataUrisPreserved) { + // tri_tree.gltf contains all the data in a URI. + const fs::path gltf_path = FindResourceOrThrow( + "drake/geometry/render_gltf_client/test/tri_tree.gltf"); + const GeometryId disk_id = GeometryId::get_new_id(); + EXPECT_TRUE(engine_.RegisterVisual(disk_id, Mesh(gltf_path.string()), + properties_, X_WG_)); + const GeometryId memory_id = GeometryId::get_new_id(); + InMemoryMesh gltf{MemoryFile::Make(gltf_path)}; + EXPECT_TRUE( + engine_.RegisterVisual(memory_id, Mesh(gltf), properties_, X_WG_)); + + const std::map& gltfs = + Tester::gltfs(engine_); + ASSERT_TRUE(gltfs.contains(disk_id)); + ASSERT_TRUE(gltfs.contains(memory_id)); + const json& disk_contents = gltfs.at(disk_id).contents; + const json& memory_contents = gltfs.at(memory_id).contents; + const json ref_contents = json::parse(gltf.mesh_file.contents()); + + // The two json structures should be *equivalent*. However, we can't + // compare directly because the engine can make small changes (e.g., + // transformation matrices). So, we'll compare images and buffers directly. + EXPECT_EQ(disk_contents.value("images", json{}), + ref_contents.value("images", json{})); + EXPECT_EQ(disk_contents.value("buffers", json{}), + ref_contents.value("buffers", json{})); + + EXPECT_EQ(memory_contents.value("images", json{}), + ref_contents.value("images", json{})); + EXPECT_EQ(memory_contents.value("buffers", json{}), + ref_contents.value("buffers", json{})); +} + +/* RenderEngineGltfClient has the responsibility of turning file URIs into + data URIs and this must work with in-memory meshes as well. + + In this case, the glTF file references external files. We'll confirm they get + converted. */ +TEST_F(RenderEngineGltfClientGltfTest, InMemorySupportingFilesToDataUris) { + // cube3.gltf has two images and one .bin file. So, we'll see that they all + // get processed. + { + const std::filesystem::path dir("drake/geometry/render/test/meshes"); + const std::filesystem::path gltf_path = + FindResourceOrThrow(dir / "cube3.gltf"); + const std::string gltf_content = ReadFileOrThrow(gltf_path); + + string_map files; + for (const char* file_name : + {"cube3_normal.png", "cube3_divot.png", "cube3.bin"}) { + files.insert( + {file_name, MemoryFile::Make(FindResourceOrThrow(dir / file_name))}); + } + + const GeometryId disk_id = GeometryId::get_new_id(); + EXPECT_TRUE( + engine_.RegisterVisual(disk_id, Mesh(gltf_path), properties_, X_WG_)); + + const GeometryId memory_id = GeometryId::get_new_id(); + EXPECT_TRUE(engine_.RegisterVisual( + memory_id, Mesh(InMemoryMesh{MemoryFile::Make(gltf_path), files}), + properties_, X_WG_)); + + const std::map& gltfs = + Tester::gltfs(engine_); + ASSERT_TRUE(gltfs.contains(disk_id)); + ASSERT_TRUE(gltfs.contains(memory_id)); + + const json& disk_contents = gltfs.at(disk_id).contents; + const json& memory_contents = gltfs.at(memory_id).contents; + const json ref_contents = json::parse(gltf_content); + + auto make_data_uri = [](const std::string& content) { + return fmt::format( + "data:application/octet-stream;base64,{}", + common_robotics_utilities::base64_helpers::Encode( + std::vector(content.begin(), content.end()))); + }; + + for (std::string_view array : {"images", "buffers"}) { + for (size_t i = 0; i < ref_contents[array].size(); ++i) { + const std::string& name = + ref_contents[array][i]["uri"].get(); + const FileSource& file_source = files[name]; + const MemoryFile* memory_file = std::get_if(&file_source); + DRAKE_DEMAND(memory_file != nullptr); + const std::string ref_string = make_data_uri(memory_file->contents()); + for (const json* test : {&disk_contents, &memory_contents}) { + SCOPED_TRACE(fmt::format("{}[{}] from {}", array, i, + test == &disk_contents ? "disk" : "memory")); + auto uri = (*test)[array][i]["uri"].get(); + EXPECT_EQ(uri, ref_string); + } + } + } + } +} + /* Currently, our server expects us to output a z-up gltf (because (a) Drake is z-up and (b) VTK does not correct it). The input gltf files are y-up. Therefore, we have to do some special math to apply a z-up pose to a y-up diff --git a/geometry/render_vtk/BUILD.bazel b/geometry/render_vtk/BUILD.bazel index ad2ae7bbb0c9..32c0c11047a9 100644 --- a/geometry/render_vtk/BUILD.bazel +++ b/geometry/render_vtk/BUILD.bazel @@ -59,6 +59,7 @@ drake_cc_library( ":internal_render_engine_vtk_base", ":internal_vtk_util", "//common", + "//geometry:vtk_gltf_uri_loader", "//geometry/proximity:polygon_to_triangle_mesh", "//geometry/render:render_engine", "//geometry/render:render_mesh", @@ -145,6 +146,7 @@ drake_cc_googletest( ":internal_render_engine_vtk", "//common:find_resource", "//common/test_utilities", + "//geometry:read_gltf_to_memory", "//geometry/test_utilities:dummy_render_engine", "//math:geometric_transform", "//systems/sensors:image_io", diff --git a/geometry/render_vtk/internal_render_engine_vtk.cc b/geometry/render_vtk/internal_render_engine_vtk.cc index 68ca4038eb07..c9496f026e0b 100644 --- a/geometry/render_vtk/internal_render_engine_vtk.cc +++ b/geometry/render_vtk/internal_render_engine_vtk.cc @@ -48,6 +48,7 @@ #include "drake/geometry/render/shaders/depth_shaders.h" #include "drake/geometry/render_vtk/internal_render_engine_vtk_base.h" #include "drake/geometry/render_vtk/internal_vtk_util.h" +#include "drake/geometry/vtk_gltf_uri_loader.h" #include "drake/math/rotation_matrix.h" #include "drake/systems/sensors/vtk_diagnostic_event_observer.h" @@ -66,6 +67,7 @@ using geometry::internal::LoadRenderMeshesFromObj; using geometry::internal::MakeDiffuseMaterial; using geometry::internal::RenderMaterial; using geometry::internal::RenderMesh; +using geometry::internal::VtkGltfUriLoader; using math::RigidTransformd; using math::RotationMatrixd; using render::ColorRenderCamera; @@ -262,9 +264,9 @@ void RenderEngineVtk::ImplementGeometry(const Mesh& mesh, void* user_data) { const std::string extension = mesh.extension(); if (extension == ".obj") { - data.accepted = ImplementObj(mesh.filename(), mesh.scale(), data); + data.accepted = ImplementObj(mesh, data); } else if (extension == ".gltf") { - data.accepted = ImplementGltf(mesh.filename(), mesh.scale(), data); + data.accepted = ImplementGltf(mesh, data); } else { static const logging::Warn one_time( "RenderEngineVtk only supports Mesh specifications which use " @@ -557,17 +559,17 @@ void RenderEngineVtk::ImplementRenderMesh(RenderMesh&& mesh, double scale, ImplementPolyData(transform_filter.GetPointer(), material, data); } -bool RenderEngineVtk::ImplementObj(const std::filesystem::path& file_name, - double scale, const RegistrationData& data) { +bool RenderEngineVtk::ImplementObj(const Mesh& mesh, + const RegistrationData& data) { std::vector meshes = LoadRenderMeshesFromObj( - file_name, data.properties, default_diffuse_, diagnostic_); + mesh.source(), data.properties, default_diffuse_, diagnostic_); for (auto& render_mesh : meshes) { - ImplementRenderMesh(std::move(render_mesh), scale, data); + ImplementRenderMesh(std::move(render_mesh), mesh.scale(), data); } return true; } -bool RenderEngineVtk::ImplementGltf(const std::string& file_name, double scale, +bool RenderEngineVtk::ImplementGltf(const Mesh& mesh, const RegistrationData& data) { vtkNew observer; observer->set_diagnostic(&diagnostic_); @@ -580,14 +582,24 @@ bool RenderEngineVtk::ImplementGltf(const std::string& file_name, double scale, // importer (see systems/sensors/image_io_load.cc). vtkNew importer; observe(importer); - importer->SetFileName(file_name.c_str()); + const MeshSource& mesh_source = mesh.source(); + if (mesh_source.is_path()) { + importer->SetFileName(mesh_source.path().c_str()); + } else { + vtkNew uri_loader; + uri_loader->SetMeshSource(&mesh_source); + vtkSmartPointer gltf_stream = + uri_loader->MakeGltfStream(); + importer->SetInputStream(gltf_stream, uri_loader, /* binary= */ false); + } importer->Update(); auto* renderer = importer->GetRenderer(); DRAKE_DEMAND(renderer != nullptr); if (renderer->VisibleActorCount() == 0) { - log()->warn("No visible meshes found in glTF file: {}", file_name); + log()->warn("No visible meshes found in glTF file: '{}'", + mesh.source().description()); return false; } @@ -599,7 +611,7 @@ bool RenderEngineVtk::ImplementGltf(const std::string& file_name, double scale, // This includes the rotation from y-up to z-up and the requested scale. const RigidTransformd X_GF(RotationMatrixd::MakeXRotation(M_PI / 2)); vtkSmartPointer T_GF_transform = - ConvertToVtkTransform(X_GF, scale); + ConvertToVtkTransform(X_GF, mesh.scale()); vtkMatrix4x4* T_GF = T_GF_transform->GetMatrix(); // Color. @@ -609,7 +621,7 @@ bool RenderEngineVtk::ImplementGltf(const std::string& file_name, double scale, "Drake materials have been assigned to a glTF file. glTF defines its " "own materials, so post hoc materials will be ignored and should be " "removed from the model specification. glTF file: '{}'", - file_name); + mesh.source().description()); } const RenderLabel label = GetRenderLabelOrThrow(data.properties); @@ -932,8 +944,6 @@ void RenderEngineVtk::InitializePipelines() { void RenderEngineVtk::ImplementPolyData(vtkPolyDataAlgorithm* source, const RenderMaterial& material, const RegistrationData& data) { - // Parsing via VTK should never require an image to be flipped. - DRAKE_DEMAND(material.flip_y == false); std::array, kNumPipelines> actors{ vtkSmartPointer::New(), vtkSmartPointer::New(), vtkSmartPointer::New()}; diff --git a/geometry/render_vtk/internal_render_engine_vtk.h b/geometry/render_vtk/internal_render_engine_vtk.h index f9288ea345f6..e1a324b103b5 100644 --- a/geometry/render_vtk/internal_render_engine_vtk.h +++ b/geometry/render_vtk/internal_render_engine_vtk.h @@ -200,13 +200,11 @@ class DRAKE_NO_EXPORT RenderEngineVtk : public render::RenderEngine, // Adds an .obj to the scene for the id currently being reified (data->id). // Returns true if added, false if ignored (for whatever reason). - bool ImplementObj(const std::filesystem::path& file_name, double scale, - const RegistrationData& data); + bool ImplementObj(const Mesh& mesh, const RegistrationData& data); // Adds a .gltf to the scene for the id currently being reified (data->id). // Returns true if added, false if ignored (for whatever reason). - bool ImplementGltf(const std::string& file_name, double scale, - const RegistrationData& data); + bool ImplementGltf(const Mesh& mesh, const RegistrationData& data); private: friend class RenderEngineVtkTester; diff --git a/geometry/render_vtk/test/internal_render_engine_vtk_test.cc b/geometry/render_vtk/test/internal_render_engine_vtk_test.cc index 872cd205c264..e0e0966b01e1 100644 --- a/geometry/render_vtk/test/internal_render_engine_vtk_test.cc +++ b/geometry/render_vtk/test/internal_render_engine_vtk_test.cc @@ -1,6 +1,7 @@ #include "drake/geometry/render_vtk/internal_render_engine_vtk.h" #include +#include #include #include #include @@ -25,6 +26,7 @@ #include "drake/common/test_utilities/eigen_matrix_compare.h" #include "drake/common/test_utilities/expect_no_throw.h" #include "drake/common/test_utilities/expect_throws_message.h" +#include "drake/geometry/read_gltf_to_memory.h" #include "drake/geometry/shape_specification.h" #include "drake/math/rigid_transform.h" #include "drake/math/rotation_matrix.h" @@ -691,6 +693,77 @@ TEST_F(RenderEngineVtkTest, MeshTest) { } } +// Repeats various mesh-based tests, but this time the meshes are loaded from +// memory. We render the scene twice: once with the on-disk mesh and once with +// the in-memory mesh to confirm they are rendered the same. +TEST_F(RenderEngineVtkTest, InMemoryMesh) { + // Pose the camera so we can see three sides of the cubes. + const RotationMatrixd R_WR(math::RollPitchYawd(-0.75 * M_PI, 0, M_PI_4)); + const RigidTransformd X_WR(R_WR, + R_WR * -Vector3d(0, 0, 1.5 * kDefaultDistance)); + Init(X_WR, true); + + const GeometryId id = GeometryId::get_new_id(); + PerceptionProperties props; + props.AddProperty("label", "id", RenderLabel(17)); + auto do_test = [this, id, &props](std::string_view file_prefix, + const Mesh& file_mesh, + const Mesh& memory_mesh) { + renderer_->RemoveGeometry(id); + renderer_->RegisterVisual(id, file_mesh, props, RigidTransformd::Identity(), + false); + ImageRgba8U file_image(kWidth, kHeight); + Render(__LINE__, fmt::format("{}_file", file_prefix), nullptr, nullptr, + &file_image, nullptr, nullptr); + + renderer_->RemoveGeometry(id); + renderer_->RegisterVisual(id, memory_mesh, props, + RigidTransformd::Identity(), false); + ImageRgba8U memory_image(kWidth, kHeight); + Render(__LINE__, fmt::format("{}_memory", file_prefix), nullptr, nullptr, + &memory_image, nullptr, nullptr); + + EXPECT_TRUE(file_image == memory_image) << fmt::format( + "The glTF file loaded from disk didn't match that loaded from memory. " + "Check the bazel-testlogs for the saved images with the prefix '{}'.", + file_prefix); + }; + + // cube1.gltf has all internal data; this confirms that data uris are + // preserved. + { + const fs::path path = + FindResourceOrThrow("drake/geometry/render/test/meshes/cube1.gltf"); + InMemoryMesh mesh_data = ReadGltfToMemory(path); + do_test("embedded_gltf", Mesh(path.string()), Mesh(std::move(mesh_data))); + } + + // cube2.gltf uses all external files; confirming that file uris work. + { + const fs::path path = + FindResourceOrThrow("drake/geometry/render/test/meshes/cube2.gltf"); + InMemoryMesh mesh_data = ReadGltfToMemory(path); + do_test("file_uri_gltf", Mesh(path.string()), Mesh(std::move(mesh_data))); + } + + // rainbow_box.obj has some faces colored by texture, some by material. The + // rendering includes faces of both types so we can tell if the right + // materials and textures are getting loaded in the right way. + { + const fs::path obj_path = FindResourceOrThrow( + "drake/geometry/render/test/meshes/rainbow_box.obj"); + const fs::path mtl_path = FindResourceOrThrow( + "drake/geometry/render/test/meshes/rainbow_box.mtl"); + const fs::path png_path = FindResourceOrThrow( + "drake/geometry/render/test/meshes/rainbow_stripes.png"); + do_test("textured_obj", Mesh(obj_path.string()), + Mesh(InMemoryMesh{ + MemoryFile::Make(obj_path), + {{"rainbow_box.mtl", MemoryFile::Make(mtl_path)}, + {"rainbow_stripes.png", MemoryFile::Make(png_path)}}})); + } +} + // A simple regression test to make sure that we are supporting all of the // texture types that glTF supports. To that end, we have a special glTF file // that we'll render and test the resulting image against a reference image. diff --git a/tools/workspace/vtk_internal/patches/gltf_importer_from_stream.patch b/tools/workspace/vtk_internal/patches/gltf_importer_from_stream.patch new file mode 100644 index 000000000000..f1a350008c8f --- /dev/null +++ b/tools/workspace/vtk_internal/patches/gltf_importer_from_stream.patch @@ -0,0 +1,154 @@ +vtkGLTFDocumentLoader can "load" a glTF from a stream (with the appropriate +URI loader). vtkGLTFImporter cannot. This changes the importer so its source +can be configured to be an input stream; it *does* potentially require a custom +vtkURILoader to resolve file URIs in the glTF file, though. + +While it is clearly desirable to upstream this feature, it is not clear if this +adheres to how kitware would accomplish the same end. + +--- IO/Import/vtkGLTFImporter.h ++++ IO/Import/vtkGLTFImporter.h +@@ -48,7 +48,9 @@ + + #include "vtkIOImportModule.h" // For export macro + #include "vtkImporter.h" ++#include "vtkResourceStream.h" + #include "vtkSmartPointer.h" // For SmartPointer ++#include "vtkURILoader.h" + + #include // For map + #include // For vector +@@ -75,6 +77,23 @@ public: + vtkGetFilePathMacro(FileName); + ///@} + ++ /** ++ * Sets the glTF source from a stream. If the stream and filename are set, the ++ * FileName is ignored. If a stream is provided, the `uri_loader` must be ++ * capable of resolving URIs. Note: if the stream contains only data URIs, no ++ * a default vtkUriLoader is sufficient. ++ * ++ * Setting both `stream` and `uri_loader` to null will revert the importer to ++ * use FileName. ++ * ++ * @param stream The data stream of the main .gltf/glb file. ++ * @param uri_loader The loader to resolve non-data URIs in the glTF file. ++ * @param binary If true, the data stream contains the contents of a .glb ++ * file. Value doesn't matter when the other parameters are ++ * null. ++ */ ++ void SetInputStream(vtkResourceStream* stream, vtkURILoader* uri_loader, bool binary); ++ + /** + * glTF defines multiple camera objects, but no default behavior for which camera should be + * used. The importer will by default apply the asset's first camera. This accessor lets you use +@@ -156,6 +175,9 @@ protected: + void ApplySkinningMorphing(); + + char* FileName = nullptr; ++ vtkSmartPointer FileStream; ++ vtkSmartPointer StreamURILoader; ++ bool StreamIsBinary{}; + + std::map> Cameras; + std::map> Textures; + +--- IO/Import/vtkGLTFImporter.cxx ++++ IO/Import/vtkGLTFImporter.cxx +@@ -375,13 +375,26 @@ void vtkGLTFImporter::InitializeLoader() + this->Loader = vtkSmartPointer::New(); + } + ++//------------------------------------------------------------------------------ ++void vtkGLTFImporter::SetInputStream(vtkResourceStream* stream, ++ vtkURILoader* uri_loader, bool binary) { ++ this->FileStream = stream; ++ this->StreamURILoader = uri_loader; ++ this->StreamIsBinary = binary; ++ // Both should be null or not null. ++ if ((this->FileStream == nullptr) != (this->StreamURILoader == nullptr)) { ++ vtkErrorMacro("When setting the input stream, the stream and the uri " ++ "loader must be set or cleared simultaneously."); ++ } ++} ++ + //------------------------------------------------------------------------------ + int vtkGLTFImporter::ImportBegin() + { + // Make sure we have a file to read. +- if (!this->FileName) ++ if (!this->FileStream && !this->FileName) + { +- vtkErrorMacro("A FileName must be specified."); ++ vtkErrorMacro("Neither FileName nor FileStream has been specified."); + return 0; + } + +@@ -398,21 +411,44 @@ int vtkGLTFImporter::ImportBegin() + + // Check extension + std::vector glbBuffer; +- std::string extension = vtksys::SystemTools::GetFilenameLastExtension(this->FileName); +- if (extension == ".glb") ++ if (this->FileName != nullptr) + { +- if (!this->Loader->LoadFileBuffer(this->FileName, glbBuffer)) ++ // this->FileName is defined. ++ std::string extension = vtksys::SystemTools::GetFilenameLastExtension(this->FileName); ++ if (extension == ".glb") + { +- vtkErrorMacro("Error loading binary data"); ++ if (!this->Loader->LoadFileBuffer(this->FileName, glbBuffer)) ++ { ++ vtkErrorMacro("Error loading binary data"); ++ return 0; ++ } ++ } ++ ++ if (!this->Loader->LoadModelMetaDataFromFile(this->FileName)) ++ { ++ vtkErrorMacro("Error loading model metadata"); + return 0; + } + } +- +- if (!this->Loader->LoadModelMetaDataFromFile(this->FileName)) ++ else + { +- vtkErrorMacro("Error loading model metadata"); +- return 0; ++ // this->FileStream is defined. ++ if (this->StreamIsBinary) ++ { ++ if (!this->Loader->LoadStreamBuffer(this->FileStream, glbBuffer)) ++ { ++ vtkErrorMacro("Error loading binary data"); ++ return 0; ++ } ++ } ++ ++ if (!this->Loader->LoadModelMetaDataFromStream(this->FileStream, this->StreamURILoader)) ++ { ++ vtkErrorMacro("Error loading model metadata"); ++ return 0; ++ } + } ++ + if (!this->Loader->LoadModelData(glbBuffer)) + { + vtkErrorMacro("Error loading model data"); +@@ -936,7 +972,13 @@ bool vtkGLTFImporter::GetTemporalInformation(vtkIdType animationIndex, double fr + void vtkGLTFImporter::PrintSelf(ostream& os, vtkIndent indent) + { + this->Superclass::PrintSelf(os, indent); +- os << indent << "File Name: " << (this->FileName ? this->FileName : "(none)") << "\n"; ++ os << indent; ++ if (this->FileStream != nullptr) { ++ os << "FileStream (" << (this->StreamIsBinary ? "binary" : "ascii") << ")"; ++ } else { ++ os << "File Name: " << (this->FileName ? this->FileName : "(none)"); ++ } ++ os << "\n"; + } + + //------------------------------------------------------------------------------ diff --git a/tools/workspace/vtk_internal/repository.bzl b/tools/workspace/vtk_internal/repository.bzl index 6ab9227c5324..d815c3b81b76 100644 --- a/tools/workspace/vtk_internal/repository.bzl +++ b/tools/workspace/vtk_internal/repository.bzl @@ -181,6 +181,7 @@ def vtk_internal_repository( ":patches/fix_illumination_bugs.patch", ":patches/gltf_selected_load.patch", ":patches/io_geometry_gltf_default_scene.patch", + ":patches/gltf_importer_from_stream.patch", ":patches/io_image_formats.patch", ":patches/mr11117.patch", ":patches/nerf_pegtl.patch",