From 95deceaa461ae4428d44894f799b745316481aa4 Mon Sep 17 00:00:00 2001 From: cDc Date: Mon, 23 Oct 2023 12:43:36 +0300 Subject: [PATCH] texture: add multi texture support (#1074) --- apps/TextureMesh/TextureMesh.cpp | 6 +- apps/Viewer/Scene.cpp | 104 ++++--- apps/Viewer/Scene.h | 2 +- libs/Common/Types.inl | 6 +- libs/MVS/Mesh.cpp | 506 ++++++++++++++++++------------- libs/MVS/Mesh.h | 20 +- libs/MVS/RectsBinPack.cpp | 113 +++---- libs/MVS/RectsBinPack.h | 24 +- libs/MVS/Scene.h | 2 +- libs/MVS/SceneTexture.cpp | 173 ++++++----- 10 files changed, 545 insertions(+), 411 deletions(-) diff --git a/apps/TextureMesh/TextureMesh.cpp b/apps/TextureMesh/TextureMesh.cpp index 351c2b3a0..5c6e0ebfb 100644 --- a/apps/TextureMesh/TextureMesh.cpp +++ b/apps/TextureMesh/TextureMesh.cpp @@ -68,6 +68,7 @@ unsigned nOrthoMapResolution; unsigned nArchiveType; int nProcessPriority; unsigned nMaxThreads; +int nMaxTextureSize; String strExportType; String strConfigFileName; boost::program_options::variables_map vm; @@ -125,6 +126,7 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("sharpness-weight", boost::program_options::value(&OPT::fSharpnessWeight)->default_value(0.5f), "amount of sharpness to be applied on the texture (0 - disabled)") ("orthographic-image-resolution", boost::program_options::value(&OPT::nOrthoMapResolution)->default_value(0), "orthographic image resolution to be generated from the textured mesh - the mesh is expected to be already geo-referenced or at least properly oriented (0 - disabled)") ("ignore-mask-label", boost::program_options::value(&OPT::nIgnoreMaskLabel)->default_value(-1), "label value to ignore in the image mask, stored in the MVS scene or next to each image with '.mask.png' extension (-1 - auto estimate mask for lens distortion, -2 - disabled)") + ("max-texture-size", boost::program_options::value(&OPT::nMaxTextureSize)->default_value(8192), "maximum texture size, split it in multiple textures of this size if needed (0 - unbounded)") ; // hidden options, allowed both on command line and @@ -286,7 +288,7 @@ int main(int argc, LPCTSTR* argv) return EXIT_FAILURE; } const String baseFileName(MAKE_PATH_SAFE(Util::getFileFullName(OPT::strOutputFileName))); - if (OPT::nOrthoMapResolution && !scene.mesh.textureDiffuse.empty()) { + if (OPT::nOrthoMapResolution && !scene.mesh.HasTexture()) { // the input mesh is already textured and an orthographic projection was requested goto ProjectOrtho; } @@ -311,7 +313,7 @@ int main(int argc, LPCTSTR* argv) TD_TIMER_START(); if (!scene.TextureMesh(OPT::nResolutionLevel, OPT::nMinResolution, OPT::minCommonCameras, OPT::fOutlierThreshold, OPT::fRatioDataSmoothness, OPT::bGlobalSeamLeveling, OPT::bLocalSeamLeveling, OPT::nTextureSizeMultiple, OPT::nRectPackingHeuristic, Pixel8U(OPT::nColEmpty), - OPT::fSharpnessWeight, OPT::nIgnoreMaskLabel, views)) + OPT::fSharpnessWeight, OPT::nIgnoreMaskLabel, OPT::nMaxTextureSize, views)) return EXIT_FAILURE; VERBOSE("Mesh texturing completed: %u vertices, %u faces (%s)", scene.mesh.vertices.GetSize(), scene.mesh.faces.GetSize(), TD_TIMER_GET_FMT().c_str()); diff --git a/apps/Viewer/Scene.cpp b/apps/Viewer/Scene.cpp index d837502e1..d4765ac32 100644 --- a/apps/Viewer/Scene.cpp +++ b/apps/Viewer/Scene.cpp @@ -125,8 +125,7 @@ SEACAVE::Thread Scene::thread; Scene::Scene(ARCHIVE_TYPE _nArchiveType) : nArchiveType(_nArchiveType), - listPointCloud(0), - listMesh(0) + listPointCloud(0) { } Scene::~Scene() @@ -171,9 +170,10 @@ void Scene::ReleasePointCloud() } void Scene::ReleaseMesh() { - if (listMesh) { - glDeleteLists(listMesh, 1); - listMesh = 0; + if (!listMeshes.empty()) { + for (GLuint listMesh: listMeshes) + glDeleteLists(listMesh, 1); + listMeshes.Release(); } } @@ -294,18 +294,21 @@ bool Scene::Open(LPCTSTR fileName, LPCTSTR geometryFileName) // init and load texture if (scene.mesh.HasTexture()) { - Image& image = textures.AddEmpty(); - ASSERT(image.idx == NO_ID); - #if 0 - cv::flip(scene.mesh.textureDiffuse, scene.mesh.textureDiffuse, 0); - image.SetImage(scene.mesh.textureDiffuse); - scene.mesh.textureDiffuse.release(); - #else // preserve texture, used only to be able to export the mesh - Image8U3 textureDiffuse; - cv::flip(scene.mesh.textureDiffuse, textureDiffuse, 0); - image.SetImage(textureDiffuse); - #endif - image.GenerateMipmap(); + FOREACH(i, scene.mesh.texturesDiffuse) { + Image& image = textures.emplace_back(); + ASSERT(image.idx == NO_ID); + #if 0 + Image8U3& textureDiffuse = scene.mesh.texturesDiffuse[i]; + cv::flip(textureDiffuse, textureDiffuse, 0); + image.SetImage(textureDiffuse); + textureDiffuse.release(); + #else // preserve texture, used only to be able to export the mesh + Image8U3 textureDiffuse; + cv::flip(scene.mesh.texturesDiffuse[i], textureDiffuse, 0); + image.SetImage(textureDiffuse); + #endif + image.GenerateMipmap(); + } } // init display lists @@ -423,15 +426,15 @@ void Scene::CompilePointCloud() glNewList(listPointCloud, GL_COMPILE); ASSERT((window.sparseType&(Window::SPR_POINTS|Window::SPR_LINES)) != 0); // compile point-cloud - if (!scene.pointcloud.IsEmpty() && (window.sparseType&Window::SPR_POINTS) != 0) { + if ((window.sparseType&Window::SPR_POINTS) != 0) { ASSERT_ARE_SAME_TYPE(float, MVS::PointCloud::Point::Type); glBegin(GL_POINTS); glColor3f(1.f,1.f,1.f); FOREACH(i, scene.pointcloud.points) { - if (!scene.pointcloud.pointViews.IsEmpty() && + if (!scene.pointcloud.pointViews.empty() && scene.pointcloud.pointViews[i].size() < window.minViews) continue; - if (!scene.pointcloud.colors.IsEmpty()) { + if (!scene.pointcloud.colors.empty()) { const MVS::PointCloud::Color& c = scene.pointcloud.colors[i]; glColor3ub(c.r,c.g,c.b); } @@ -452,31 +455,37 @@ void Scene::CompileMesh() scene.mesh.ComputeNormalFaces(); // translate, normalize and flip Y axis of the texture coordinates MVS::Mesh::TexCoordArr normFaceTexcoords; - if (scene.mesh.HasTexture()) + if (scene.mesh.HasTexture() && window.bRenderTexture) scene.mesh.FaceTexcoordsNormalize(normFaceTexcoords, true); - listMesh = glGenLists(1); - glNewList(listMesh, GL_COMPILE); - // compile mesh - ASSERT_ARE_SAME_TYPE(float, MVS::Mesh::Vertex::Type); - ASSERT_ARE_SAME_TYPE(float, MVS::Mesh::Normal::Type); - ASSERT_ARE_SAME_TYPE(float, MVS::Mesh::TexCoord::Type); - glColor3f(1.f, 1.f, 1.f); - glBegin(GL_TRIANGLES); - FOREACH(i, scene.mesh.faces) { - const MVS::Mesh::Face& face = scene.mesh.faces[i]; - const MVS::Mesh::Normal& n = scene.mesh.faceNormals[i]; - glNormal3fv(n.ptr()); - for (int j = 0; j < 3; ++j) { - if (!normFaceTexcoords.empty() && window.bRenderTexture) { - const MVS::Mesh::TexCoord& t = normFaceTexcoords[i * 3 + j]; - glTexCoord2fv(t.ptr()); + MVS::Mesh::TexIndex texIdx(0); + do { + GLuint& listMesh = listMeshes.emplace_back(glGenLists(1)); + listMesh = glGenLists(1); + glNewList(listMesh, GL_COMPILE); + // compile mesh + ASSERT_ARE_SAME_TYPE(float, MVS::Mesh::Vertex::Type); + ASSERT_ARE_SAME_TYPE(float, MVS::Mesh::Normal::Type); + ASSERT_ARE_SAME_TYPE(float, MVS::Mesh::TexCoord::Type); + glColor3f(1.f, 1.f, 1.f); + glBegin(GL_TRIANGLES); + FOREACH(idxFace, scene.mesh.faces) { + if (!scene.mesh.faceTexindices.empty() && scene.mesh.faceTexindices[idxFace] != texIdx) + continue; + const MVS::Mesh::Face& face = scene.mesh.faces[idxFace]; + const MVS::Mesh::Normal& n = scene.mesh.faceNormals[idxFace]; + glNormal3fv(n.ptr()); + for (int j = 0; j < 3; ++j) { + if (!normFaceTexcoords.empty()) { + const MVS::Mesh::TexCoord& t = normFaceTexcoords[idxFace*3 + j]; + glTexCoord2fv(t.ptr()); + } + const MVS::Mesh::Vertex& p = scene.mesh.vertices[face[j]]; + glVertex3fv(p.ptr()); } - const MVS::Mesh::Vertex& p = scene.mesh.vertices[face[j]]; - glVertex3fv(p.ptr()); } - } - glEnd(); - glEndList(); + glEnd(); + glEndList(); + } while (++texIdx < scene.mesh.texturesDiffuse.size()); } void Scene::CompileBounds() @@ -513,17 +522,20 @@ void Scene::Draw() glCallList(listPointCloud); } // render mesh - if (listMesh) { + if (!listMeshes.empty()) { glEnable(GL_DEPTH_TEST); glEnable(GL_CULL_FACE); if (!scene.mesh.faceTexcoords.empty() && window.bRenderTexture) { glEnable(GL_TEXTURE_2D); - textures.front().Bind(); - glCallList(listMesh); + FOREACH(i, listMeshes) { + textures[i].Bind(); + glCallList(listMeshes[i]); + } glDisable(GL_TEXTURE_2D); } else { glEnable(GL_LIGHTING); - glCallList(listMesh); + for (GLuint listMesh: listMeshes) + glCallList(listMesh); glDisable(GL_LIGHTING); } } diff --git a/apps/Viewer/Scene.h b/apps/Viewer/Scene.h index 48251d31d..e86574889 100644 --- a/apps/Viewer/Scene.h +++ b/apps/Viewer/Scene.h @@ -68,7 +68,7 @@ class Scene Point3fArr obbPoints; GLuint listPointCloud; - GLuint listMesh; + CLISTDEF0IDX(GLuint,MVS::Mesh::TexIndex) listMeshes; // multi-threading static SEACAVE::EventQueue events; // internal events queue (processed by the working threads) diff --git a/libs/Common/Types.inl b/libs/Common/Types.inl index 58b9ca85e..fa02932d7 100644 --- a/libs/Common/Types.inl +++ b/libs/Common/Types.inl @@ -2617,9 +2617,11 @@ void TImage::RasterizeTriangleBary(const TPoint2& v1, const TPoint2& ImageRef boxMinI(FLOOR2INT(boxMin)); ImageRef boxMaxI(CEIL2INT(boxMax)); Base::clip(boxMinI, boxMaxI, size); - // parse all pixels inside the bounding-box + // ignore back oriented triangles (negative area) const T area(EdgeFunction(v1, v2, v3)); - ASSERTM(area < 0, "UV triangle of negative area"); + if (area <= 0) + return; + // parse all pixels inside the bounding-box const T invArea(T(1) / area); for (int y = boxMinI.y; y <= boxMaxI.y; ++y) { for (int x = boxMinI.x; x <= boxMaxI.x; ++x) { diff --git a/libs/MVS/Mesh.cpp b/libs/MVS/Mesh.cpp index 764121d6e..7c5aee748 100644 --- a/libs/MVS/Mesh.cpp +++ b/libs/MVS/Mesh.cpp @@ -117,7 +117,7 @@ void Mesh::ReleaseExtra() faceNormals.Release(); faceFaces.Release(); faceTexcoords.Release(); - textureDiffuse.release(); + texturesDiffuse.Release(); } // ReleaseExtra void Mesh::EmptyExtra() { @@ -128,7 +128,7 @@ void Mesh::EmptyExtra() faceNormals.Empty(); faceFaces.Empty(); faceTexcoords.Empty(); - textureDiffuse.release(); + texturesDiffuse.Empty(); } // EmptyExtra void Mesh::Swap(Mesh& rhs) { @@ -141,7 +141,8 @@ void Mesh::Swap(Mesh& rhs) faceNormals.Swap(rhs.faceNormals); faceFaces.Swap(rhs.faceFaces); faceTexcoords.Swap(rhs.faceTexcoords); - std::swap(textureDiffuse, rhs.textureDiffuse); + faceTexindices.Swap(rhs.faceTexindices); + std::swap(texturesDiffuse, rhs.texturesDiffuse); } // Swap // combine this mesh with the given mesh, without removing duplicate vertices void Mesh::Join(const Mesh& mesh) @@ -1424,40 +1425,54 @@ static const Mesh::TexCoord halfPixel(0.5f, 0.5f); // translate, normalize and flip Y axis of the texture coordinates void Mesh::FaceTexcoordsNormalize(TexCoordArr& newFaceTexcoords, bool flipY) const { - ASSERT(!faceTexcoords.empty() && !textureDiffuse.empty()); - const TexCoord invNorm(1.f/(float)textureDiffuse.cols, 1.f/(float)textureDiffuse.rows); + ASSERT(HasTexture()); + TexCoordArr invNorms(texturesDiffuse.size()); + FOREACH(i, texturesDiffuse) { + ASSERT(!texturesDiffuse[i].empty()); + invNorms[i] = TexCoord(1.f/(float)texturesDiffuse[i].cols, 1.f/(float)texturesDiffuse[i].rows); + } newFaceTexcoords.resize(faceTexcoords.size()); if (flipY) { FOREACH(i, faceTexcoords) { const TexCoord& texcoord = faceTexcoords[i]; + const TexCoord& invNorm = invNorms[GetFaceTextureIndex(i/3)]; newFaceTexcoords[i] = TexCoord( (texcoord.x+halfPixel.x)*invNorm.x, 1.f-(texcoord.y+halfPixel.y)*invNorm.y ); } } else { - FOREACH(i, faceTexcoords) + FOREACH(i, faceTexcoords) { + const TexCoord& invNorm = invNorms[GetFaceTextureIndex(i/3)]; newFaceTexcoords[i] = (faceTexcoords[i]+halfPixel)*invNorm; + } } } // FaceTexcoordsNormalize // flip Y axis, unnormalize and translate back texture coordinates void Mesh::FaceTexcoordsUnnormalize(TexCoordArr& newFaceTexcoords, bool flipY) const { - ASSERT(!faceTexcoords.empty() && !textureDiffuse.empty()); - const TexCoord scale((float)textureDiffuse.cols, (float)textureDiffuse.rows); + ASSERT(HasTexture()); + TexCoordArr scales(texturesDiffuse.size()); + FOREACH(i, texturesDiffuse) { + ASSERT(!texturesDiffuse[i].empty()); + scales[i] = TexCoord((float)texturesDiffuse[i].cols, (float)texturesDiffuse[i].rows); + } newFaceTexcoords.resize(faceTexcoords.size()); if (flipY) { FOREACH(i, faceTexcoords) { const TexCoord& texcoord = faceTexcoords[i]; + const TexCoord& scale = scales[GetFaceTextureIndex(i/3)]; newFaceTexcoords[i] = TexCoord( texcoord.x*scale.x-halfPixel.x, (1.f-texcoord.y)*scale.y-halfPixel.y ); } } else { - FOREACH(i, faceTexcoords) + FOREACH(i, faceTexcoords) { + const TexCoord& scale = scales[GetFaceTextureIndex(i/3)]; newFaceTexcoords[i] = faceTexcoords[i]*scale - halfPixel; + } } } // FaceTexcoordsUnnormalize /*----------------------------------------------------------------*/ @@ -1522,10 +1537,10 @@ namespace BasicPLY { uint8_t num; Mesh::TexCoord* pTex; } tex; - Mesh::TexChunk chunk; + Mesh::TexIndex texId; float weight; static void InitLoadProps(PLY& ply, int elem_count, - Mesh::FaceArr& faces, Mesh::TexCoordArr& faceTexcoords, Mesh::TexChunkArr& faceTextureChunks) + Mesh::FaceArr& faces, Mesh::TexCoordArr& faceTexcoords, Mesh::TexIndexArr& faceTexindices) { PLY::PlyElement* elm = ply.find_element(elem_names[1]); const size_t nMaxProps(SizeOfArray(props)); @@ -1536,7 +1551,7 @@ namespace BasicPLY { switch (p) { case 0: faces.resize((IDX)elem_count); break; case 1: faceTexcoords.resize((IDX)elem_count*3); break; - case 2: faceTextureChunks.resize((IDX)elem_count); break; + case 2: faceTexindices.resize((IDX)elem_count); break; } } } @@ -1563,7 +1578,7 @@ namespace BasicPLY { const PLY::PlyProperty Face::props[] = { {"vertex_indices", PLY::Uint32, PLY::Uint32, offsetof(Face,face.pFace), 1, PLY::Uint8, PLY::Uint8, offsetof(Face,face.num)}, {"texcoord", PLY::Float32, PLY::Float32, offsetof(Face,tex.pTex), 1, PLY::Uint8, PLY::Uint8, offsetof(Face,tex.num)}, - {"texnumber", PLY::Int32, PLY::Uint8, offsetof(Face,chunk), 0, 0, 0, 0}, + {"texnumber", PLY::Int32, PLY::Uint8, offsetof(Face,texId), 0, 0, 0, 0}, {"weight", PLY::Float32, PLY::Float32, offsetof(Face,weight), 0, 0, 0, 0}, }; } // namespace BasicPLY @@ -1638,8 +1653,7 @@ bool Mesh::LoadPLY(const String& fileName) } else if (PLY::equal_strings(BasicPLY::elem_names[1], elem_name)) { ASSERT(faces.size() == (FIndex)elem_count); - TexChunkArr faceTextureChunks; // TODO: replace with mesh texture chunks when implemented - BasicPLY::Face::InitLoadProps(ply, elem_count, faces, faceTexcoords, faceTextureChunks); + BasicPLY::Face::InitLoadProps(ply, elem_count, faces, faceTexcoords, faceTexindices); BasicPLY::Face face; FOREACH(f, faces) { ply.get_element(&face); @@ -1657,18 +1671,19 @@ bool Mesh::LoadPLY(const String& fileName) memcpy(faceTexcoords.data()+f*3, face.tex.pTex, sizeof(TexCoord)*3); delete[] face.tex.pTex; } - if (!faceTextureChunks.empty()) - faceTextureChunks[f] = face.chunk; + if (!faceTexindices.empty()) + faceTexindices[f] = face.texId; } if (!faceTexcoords.empty()) { // load the texture for (const std::string& comment: ply.get_comments()) { if (_tcsncmp(comment.c_str(), _T("TextureFile "), 12) == 0) { const String textureFileName(comment.substr(12)); - textureDiffuse.Load(Util::getFilePath(fileName)+textureFileName); - break; + texturesDiffuse.emplace_back().Load(Util::getFilePath(fileName)+textureFileName); } } + if (texturesDiffuse.size() <= 1) + faceTexindices.Release(); // flip Y axis, unnormalize and translate back texture coordinates TexCoordArr unnormFaceTexcoords; FaceTexcoordsUnnormalize(unnormFaceTexcoords, true); @@ -1692,7 +1707,8 @@ bool Mesh::LoadOBJ(const String& fileName) DEBUG_EXTRA("error: invalid OBJ file"); return false; } - if (model.get_vertices().empty() || model.get_groups().size() != 1 || model.get_groups()[0].faces.empty()) { + + if (model.get_vertices().empty() || model.get_groups().empty()) { DEBUG_EXTRA("error: invalid mesh file"); return false; } @@ -1711,31 +1727,35 @@ bool Mesh::LoadOBJ(const String& fileName) } // store faces - const ObjModel::Group& group = model.get_groups()[0]; - ASSERT(group.faces.size() < std::numeric_limits::max()); - faces.Reserve((FIndex)group.faces.size()); - for (const ObjModel::Face& f: group.faces) { - ASSERT(f.vertices[0] != NO_ID); - faces.emplace_back(f.vertices[0], f.vertices[1], f.vertices[2]); - if (f.texcoords[0] != NO_ID) { - for (int i=0; i<3; ++i) - faceTexcoords.emplace_back(model.get_texcoords()[f.texcoords[i]]); - } - if (f.normals[0] != NO_ID) { - Normal& n = faceNormals.emplace_back(Normal::ZERO); - for (int i=0; i<3; ++i) - n += normalized(model.get_normals()[f.normals[i]]); - normalize(n); + FOREACH(groupIdx, model.get_groups()) { + const auto& group = model.get_groups()[groupIdx]; + ASSERT(group.faces.size() < std::numeric_limits::max()); + faces.reserve((FIndex)group.faces.size()); + for (const ObjModel::Face& f: group.faces) { + ASSERT(f.vertices[0] != NO_ID); + faces.emplace_back(f.vertices[0], f.vertices[1], f.vertices[2]); + if (f.texcoords[0] != NO_ID) { + for (int i=0; i<3; ++i) + faceTexcoords.emplace_back(model.get_texcoords()[f.texcoords[i]]); + faceTexindices.emplace_back((TexIndex)groupIdx); + } + if (f.normals[0] != NO_ID) { + Normal& n = faceNormals.emplace_back(Normal::ZERO); + for (int i=0; i<3; ++i) + n += normalized(model.get_normals()[f.normals[i]]); + normalize(n); + } } + // store texture + ObjModel::MaterialLib::Material* pMaterial(model.GetMaterial(group.material_name)); + if (pMaterial && pMaterial->LoadDiffuseMap()) + texturesDiffuse.emplace_back(pMaterial->diffuse_map); } - // store texture - ObjModel::MaterialLib::Material* pMaterial(model.GetMaterial(group.material_name)); - if (pMaterial && pMaterial->LoadDiffuseMap()) - cv::swap(textureDiffuse, pMaterial->diffuse_map); - // flip Y axis, unnormalize and translate back texture coordinates if (!faceTexcoords.empty()) { + if (texturesDiffuse.size() <= 1) + faceTexindices.Release(); TexCoordArr unnormFaceTexcoords; FaceTexcoordsUnnormalize(unnormFaceTexcoords, true); faceTexcoords.Swap(unnormFaceTexcoords); @@ -1763,7 +1783,7 @@ bool Mesh::LoadGLTF(const String& fileName, bool bBinary) if (!warn.empty()) DEBUG("warning: %s", warn.c_str()); } - + // parse model for (const tinygltf::Mesh& gltfMesh : gltfModel.meshes) { for (const tinygltf::Primitive& gltfPrimitive : gltfMesh.primitives) { @@ -1855,16 +1875,18 @@ bool Mesh::SavePLY(const String& fileName, const cList& comments, bool b ply.append_comment(comment); // export texture file name as comment if needed - String textureFileName; - if (!faceTexcoords.empty() && !textureDiffuse.empty()) { - textureFileName = Util::getFileFullName(fileName)+(bTexLossless?_T(".png"):_T(".jpg")); - ply.append_comment((_T("TextureFile ")+Util::getFileNameExt(textureFileName)).c_str()); + if (HasTexture()) { + FOREACH(texId, texturesDiffuse) { + const String textureFileName(Util::getFileFullName(fileName) + std::to_string(texId).c_str() + (bTexLossless?_T(".png"):_T(".jpg"))); + ply.append_comment((_T("TextureFile ")+Util::getFileNameExt(textureFileName)).c_str()); + texturesDiffuse[texId].Save(textureFileName); + } } // describe what properties go into vertex and face elements ASSERT(vertexNormals.empty() || vertexNormals.size() == vertices.size()); BasicPLY::Vertex::InitSaveProps(ply, (int)vertices.size(), !vertexNormals.empty()); - BasicPLY::Face::InitSaveProps(ply, (int)faces.size(), !faces.empty(), !faceTexcoords.empty(), false); + BasicPLY::Face::InitSaveProps(ply, (int)faces.size(), !faces.empty(), !faceTexcoords.empty(), !faceTexindices.empty()); if (!ply.header_complete()) return false; @@ -1900,12 +1922,10 @@ bool Mesh::SavePLY(const String& fileName, const cList& comments, bool b FOREACH(f, faces) { face.face.pFace = faces.data()+f; face.tex.pTex = normFaceTexcoords.data()+f*3; + if (!faceTexindices.empty()) + face.texId = faceTexindices[f]; ply.put_element(&face); } - - // export the texture - if (!textureDiffuse.empty()) - textureDiffuse.Save(textureFileName); } ASSERT(ply.get_current_element_count() == (int)faces.size()); @@ -1943,27 +1963,31 @@ bool Mesh::SaveOBJ(const String& fileName) const } // store faces - ObjModel::Group& group = model.AddGroup(_T("material_0")); - group.faces.reserve(faces.size()); - FOREACH(idxFace, faces) { - const Face& face = faces[idxFace]; - ObjModel::Face f; - memset(&f, 0xFF, sizeof(ObjModel::Face)); - for (int i=0; i<3; ++i) { - f.vertices[i] = face[i]; - if (!faceTexcoords.empty()) - f.texcoords[i] = idxFace*3+i; - if (!vertexNormals.empty()) - f.normals[i] = face[i]; + FOREACH(idxTexture, texturesDiffuse) { + ObjModel::Group& group = model.AddGroup(_T("material_" + std::to_string(idxTexture))); + group.faces.reserve(faces.size()); + FOREACH(idxFace, faces) { + const auto texIdx = faceTexindices[idxFace]; + if (texIdx != idxTexture) + continue; + + const Face& face = faces[idxFace]; + ObjModel::Face f; + memset(&f, 0xFF, sizeof(ObjModel::Face)); + for (int i=0; i<3; ++i) { + f.vertices[i] = face[i]; + if (!faceTexcoords.empty()) + f.texcoords[i] = idxFace*3+i; + if (!vertexNormals.empty()) + f.normals[i] = face[i]; + } + group.faces.push_back(f); } - group.faces.push_back(f); + ObjModel::MaterialLib::Material* pMaterial(model.GetMaterial(group.material_name)); + ASSERT(pMaterial != NULL); + pMaterial->diffuse_map = texturesDiffuse[idxTexture]; } - // store texture - ObjModel::MaterialLib::Material* pMaterial(model.GetMaterial(group.material_name)); - ASSERT(pMaterial != NULL); - pMaterial->diffuse_map = textureDiffuse; - return model.Save(fileName); } // export the mesh as a GLTF file @@ -1975,136 +1999,148 @@ void ExtendBufferGLTF(const T* src, size_t size, tinygltf::Buffer& dst, size_t& dst.data.resize(byte_offset + byte_length); memcpy(&dst.data[byte_offset], &src[0], byte_length); } + bool Mesh::SaveGLTF(const String& fileName, bool bBinary) const { - ASSERT(!fileName.IsEmpty()); + ASSERT(!fileName.empty()); Util::ensureFolder(fileName); - // store a copy of the mesh if it has texture, in order to convert - // the texture coordinates from per face to per vertex - Mesh meshCompressed; - if (HasTexture()) - ConvertTexturePerVertex(meshCompressed); - const Mesh& mesh(HasTexture() ? meshCompressed : *this); + std::vector meshes; + if (texturesDiffuse.size() > 1) { + meshes = SplitMeshPerTextureBlob(); + for (Mesh& mesh: meshes) { + Mesh convertedMesh; + mesh.ConvertTexturePerVertex(convertedMesh); + mesh.Swap(convertedMesh); + } + } else { + Mesh convertedMesh; + ConvertTexturePerVertex(convertedMesh); + meshes.emplace_back(std::move(convertedMesh)); + } // create GLTF model tinygltf::Model gltfModel; tinygltf::Scene gltfScene; tinygltf::Mesh gltfMesh; - tinygltf::Primitive gltfPrimitive; tinygltf::Buffer gltfBuffer; gltfScene.name = "scene"; gltfMesh.name = "mesh"; - // setup vertices - { - STATIC_ASSERT(3 * sizeof(Vertex::Type) == sizeof(Vertex)); // VertexArr should be continuous - const Box box(GetAABB()); - gltfPrimitive.attributes["POSITION"] = (int)gltfModel.accessors.size(); - tinygltf::Accessor vertexPositionAccessor; - vertexPositionAccessor.name = "vertexPositionAccessor"; - vertexPositionAccessor.bufferView = (int)gltfModel.bufferViews.size(); - vertexPositionAccessor.type = TINYGLTF_TYPE_VEC3; - vertexPositionAccessor.componentType = TINYGLTF_COMPONENT_TYPE_FLOAT; - vertexPositionAccessor.count = mesh.vertices.size(); - vertexPositionAccessor.minValues = {box.ptMin.x(), box.ptMin.y(), box.ptMin.z()}; - vertexPositionAccessor.maxValues = {box.ptMax.x(), box.ptMax.y(), box.ptMax.z()}; - gltfModel.accessors.emplace_back(std::move(vertexPositionAccessor)); - // setup vertices buffer - tinygltf::BufferView vertexPositionBufferView; - vertexPositionBufferView.name = "vertexPositionBufferView"; - vertexPositionBufferView.buffer = (int)gltfModel.buffers.size(); - ExtendBufferGLTF(mesh.vertices.data(), mesh.vertices.size(), gltfBuffer, - vertexPositionBufferView.byteOffset, vertexPositionBufferView.byteLength); - gltfModel.bufferViews.emplace_back(std::move(vertexPositionBufferView)); - } - - // setup faces - { - STATIC_ASSERT(3 * sizeof(Face::Type) == sizeof(Face)); // FaceArr should be continuous - gltfPrimitive.indices = (int)gltfModel.accessors.size(); - tinygltf::Accessor triangleAccessor; - triangleAccessor.name = "triangleAccessor"; - triangleAccessor.bufferView = (int)gltfModel.bufferViews.size(); - triangleAccessor.type = TINYGLTF_TYPE_SCALAR; - triangleAccessor.componentType = TINYGLTF_COMPONENT_TYPE_UNSIGNED_INT; - triangleAccessor.count = mesh.faces.size() * 3; - gltfModel.accessors.emplace_back(std::move(triangleAccessor)); - // setup triangles buffer - tinygltf::BufferView triangleBufferView; - triangleBufferView.name = "triangleBufferView"; - triangleBufferView.buffer = (int)gltfModel.buffers.size(); - ExtendBufferGLTF(mesh.faces.data(), mesh.faces.size(), gltfBuffer, - triangleBufferView.byteOffset, triangleBufferView.byteLength); - gltfModel.bufferViews.emplace_back(std::move(triangleBufferView)); - gltfPrimitive.mode = TINYGLTF_MODE_TRIANGLES; - } - - // setup material - gltfPrimitive.material = (int)gltfModel.materials.size(); - tinygltf::Material gltfMaterial; - gltfMaterial.name = "material"; - gltfMaterial.doubleSided = true; - if (mesh.HasTexture()) { - // setup texture - gltfMaterial.emissiveFactor = std::vector{0,0,0}; - gltfMaterial.pbrMetallicRoughness.baseColorTexture.index = (int)gltfModel.textures.size(); - gltfMaterial.pbrMetallicRoughness.baseColorTexture.texCoord = 0; - gltfMaterial.pbrMetallicRoughness.baseColorFactor = std::vector{1,1,1,1}; - gltfMaterial.pbrMetallicRoughness.metallicFactor = 0; - gltfMaterial.pbrMetallicRoughness.roughnessFactor = 1; - gltfMaterial.extensions = {{"KHR_materials_unlit", {}}}; - gltfModel.extensionsUsed = {"KHR_materials_unlit"}; - // setup texture coordinates accessor - gltfPrimitive.attributes["TEXCOORD_0"] = (int)gltfModel.accessors.size(); - tinygltf::Accessor vertexTexcoordAccessor; - vertexTexcoordAccessor.name = "vertexTexcoordAccessor"; - vertexTexcoordAccessor.bufferView = (int)gltfModel.bufferViews.size(); - vertexTexcoordAccessor.componentType = TINYGLTF_COMPONENT_TYPE_FLOAT; - vertexTexcoordAccessor.count = mesh.faceTexcoords.size(); - vertexTexcoordAccessor.type = TINYGLTF_TYPE_VEC2; - gltfModel.accessors.emplace_back(std::move(vertexTexcoordAccessor)); - // setup texture coordinates - STATIC_ASSERT(2 * sizeof(TexCoord::Type) == sizeof(TexCoord)); // TexCoordArr should be continuous - ASSERT(mesh.vertices.size() == mesh.faceTexcoords.size()); - tinygltf::BufferView vertexTexcoordBufferView; - vertexTexcoordBufferView.name = "vertexTexcoordBufferView"; - vertexTexcoordBufferView.buffer = (int)gltfModel.buffers.size(); - TexCoordArr normFaceTexcoords; - mesh.FaceTexcoordsNormalize(normFaceTexcoords, false); - ExtendBufferGLTF(normFaceTexcoords.data(), normFaceTexcoords.size(), gltfBuffer, - vertexTexcoordBufferView.byteOffset, vertexTexcoordBufferView.byteLength); - gltfModel.bufferViews.emplace_back(std::move(vertexTexcoordBufferView)); - // setup texture - tinygltf::Texture texture; - texture.name = "texture"; - texture.source = (int)gltfModel.images.size(); - texture.sampler = (int)gltfModel.samplers.size(); - gltfModel.textures.emplace_back(std::move(texture)); - // setup texture image - tinygltf::Image image; - image.name = Util::getFileFullName(fileName); - image.width = mesh.textureDiffuse.cols; - image.height = mesh.textureDiffuse.rows; - image.component = 3; - image.bits = 8; - image.pixel_type = TINYGLTF_COMPONENT_TYPE_UNSIGNED_BYTE; - image.mimeType = "image/png"; - image.image.resize(mesh.textureDiffuse.size().area() * 3); - mesh.textureDiffuse.copyTo(cv::Mat(mesh.textureDiffuse.size(), CV_8UC3, image.image.data())); - gltfModel.images.emplace_back(std::move(image)); - // setup texture sampler - tinygltf::Sampler sampler; - sampler.name = "sampler"; - sampler.minFilter = TINYGLTF_TEXTURE_FILTER_LINEAR; - sampler.magFilter = TINYGLTF_TEXTURE_FILTER_LINEAR; - sampler.wrapS = TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE; - sampler.wrapT = TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE; - gltfModel.samplers.emplace_back(std::move(sampler)); - } - gltfModel.materials.emplace_back(std::move(gltfMaterial)); - gltfModel.buffers.emplace_back(std::move(gltfBuffer)); - gltfMesh.primitives.emplace_back(std::move(gltfPrimitive)); + for (size_t meshId = 0; meshId < meshes.size(); meshId++) { + const Mesh& mesh = meshes[meshId]; + ASSERT(mesh.HasTextureCoordinatesPerVertex()); + tinygltf::Primitive gltfPrimitive; + // setup vertices + { + STATIC_ASSERT(3 * sizeof(Vertex::Type) == sizeof(Vertex)); // VertexArr should be continuous + const Box box(GetAABB()); + gltfPrimitive.attributes["POSITION"] = (int)gltfModel.accessors.size(); + tinygltf::Accessor vertexPositionAccessor; + vertexPositionAccessor.name = "vertexPositionAccessor"; + vertexPositionAccessor.bufferView = (int)gltfModel.bufferViews.size(); + vertexPositionAccessor.type = TINYGLTF_TYPE_VEC3; + vertexPositionAccessor.componentType = TINYGLTF_COMPONENT_TYPE_FLOAT; + vertexPositionAccessor.count = mesh.vertices.size(); + vertexPositionAccessor.minValues = {box.ptMin.x(), box.ptMin.y(), box.ptMin.z()}; + vertexPositionAccessor.maxValues = {box.ptMax.x(), box.ptMax.y(), box.ptMax.z()}; + gltfModel.accessors.emplace_back(std::move(vertexPositionAccessor)); + // setup vertices buffer + tinygltf::BufferView vertexPositionBufferView; + vertexPositionBufferView.name = "vertexPositionBufferView"; + vertexPositionBufferView.buffer = (int)gltfModel.buffers.size(); + ExtendBufferGLTF(mesh.vertices.data(), mesh.vertices.size(), gltfBuffer, + vertexPositionBufferView.byteOffset, vertexPositionBufferView.byteLength); + gltfModel.bufferViews.emplace_back(std::move(vertexPositionBufferView)); + } + + // setup faces + { + STATIC_ASSERT(3 * sizeof(Face::Type) == sizeof(Face)); // FaceArr should be continuous + gltfPrimitive.indices = (int)gltfModel.accessors.size(); + tinygltf::Accessor triangleAccessor; + triangleAccessor.name = "triangleAccessor"; + triangleAccessor.bufferView = (int)gltfModel.bufferViews.size(); + triangleAccessor.type = TINYGLTF_TYPE_SCALAR; + triangleAccessor.componentType = TINYGLTF_COMPONENT_TYPE_UNSIGNED_INT; + triangleAccessor.count = mesh.faces.size() * 3; + gltfModel.accessors.emplace_back(std::move(triangleAccessor)); + // setup triangles buffer + tinygltf::BufferView triangleBufferView; + triangleBufferView.name = "triangleBufferView"; + triangleBufferView.buffer = (int)gltfModel.buffers.size(); + ExtendBufferGLTF(mesh.faces.data(), mesh.faces.size(), gltfBuffer, + triangleBufferView.byteOffset, triangleBufferView.byteLength); + gltfModel.bufferViews.emplace_back(std::move(triangleBufferView)); + gltfPrimitive.mode = TINYGLTF_MODE_TRIANGLES; + } + + // setup material + gltfPrimitive.material = (int)gltfModel.materials.size(); + tinygltf::Material gltfMaterial; + gltfMaterial.name = "material"; + gltfMaterial.doubleSided = true; + if (mesh.HasTexture()) { + // setup texture + gltfMaterial.emissiveFactor = std::vector{0,0,0}; + gltfMaterial.pbrMetallicRoughness.baseColorTexture.index = (int)gltfModel.textures.size(); + gltfMaterial.pbrMetallicRoughness.baseColorTexture.texCoord = 0; + gltfMaterial.pbrMetallicRoughness.baseColorFactor = std::vector{1,1,1,1}; + gltfMaterial.pbrMetallicRoughness.metallicFactor = 0; + gltfMaterial.pbrMetallicRoughness.roughnessFactor = 1; + gltfMaterial.extensions = {{"KHR_materials_unlit", {}}}; + gltfModel.extensionsUsed = {"KHR_materials_unlit"}; + // setup texture coordinates accessor + gltfPrimitive.attributes["TEXCOORD_0"] = (int)gltfModel.accessors.size(); + tinygltf::Accessor vertexTexcoordAccessor; + vertexTexcoordAccessor.name = "vertexTexcoordAccessor"; + vertexTexcoordAccessor.bufferView = (int)gltfModel.bufferViews.size(); + vertexTexcoordAccessor.componentType = TINYGLTF_COMPONENT_TYPE_FLOAT; + vertexTexcoordAccessor.count = mesh.faceTexcoords.size(); + vertexTexcoordAccessor.type = TINYGLTF_TYPE_VEC2; + gltfModel.accessors.emplace_back(std::move(vertexTexcoordAccessor)); + // setup texture coordinates + STATIC_ASSERT(2 * sizeof(TexCoord::Type) == sizeof(TexCoord)); // TexCoordArr should be continuous + ASSERT(mesh.vertices.size() == mesh.faceTexcoords.size()); + tinygltf::BufferView vertexTexcoordBufferView; + vertexTexcoordBufferView.name = "vertexTexcoordBufferView"; + vertexTexcoordBufferView.buffer = (int)gltfModel.buffers.size(); + TexCoordArr normFaceTexcoords; + mesh.FaceTexcoordsNormalize(normFaceTexcoords, false); + ExtendBufferGLTF(normFaceTexcoords.data(), normFaceTexcoords.size(), gltfBuffer, + vertexTexcoordBufferView.byteOffset, vertexTexcoordBufferView.byteLength); + gltfModel.bufferViews.emplace_back(std::move(vertexTexcoordBufferView)); + // setup texture + tinygltf::Texture texture; + texture.name = "texture"; + texture.source = (int)gltfModel.images.size(); + texture.sampler = (int)gltfModel.samplers.size(); + gltfModel.textures.emplace_back(std::move(texture)); + // setup texture image + tinygltf::Image image; + image.name = Util::getFileFullName(fileName) + "_" + std::to_string(meshId).c_str(); + image.width = mesh.texturesDiffuse[0].cols; + image.height = mesh.texturesDiffuse[0].rows; + image.component = 3; + image.bits = 8; + image.pixel_type = TINYGLTF_COMPONENT_TYPE_UNSIGNED_BYTE; + image.mimeType = "image/png"; + image.image.resize(mesh.texturesDiffuse[0].size().area() * 3); + mesh.texturesDiffuse[0].copyTo(cv::Mat(mesh.texturesDiffuse[0].size(), CV_8UC3, image.image.data())); + gltfModel.images.emplace_back(std::move(image)); + // setup texture sampler + tinygltf::Sampler sampler; + sampler.name = "sampler"; + sampler.minFilter = TINYGLTF_TEXTURE_FILTER_LINEAR; + sampler.magFilter = TINYGLTF_TEXTURE_FILTER_LINEAR; + sampler.wrapS = TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE; + sampler.wrapT = TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE; + gltfModel.samplers.emplace_back(std::move(sampler)); + } + gltfModel.materials.emplace_back(std::move(gltfMaterial)); + gltfModel.buffers.emplace_back(std::move(gltfBuffer)); + gltfMesh.primitives.emplace_back(std::move(gltfPrimitive)); + } // setup scene node gltfScene.nodes.emplace_back((int)gltfModel.nodes.size()); @@ -4032,8 +4068,10 @@ void Mesh::ConvertTexturePerVertex(Mesh& mesh) const ASSERT(HasTexture()); mesh.vertices = vertices; mesh.faces.resize(faces.size()); - mesh.faceTexcoords.reserve(vertices.size()*3/2); mesh.faceTexcoords.resize(vertices.size()); + if (!faceTexindices.empty()) + mesh.faceTexindices.resize(vertices.size()); + VertexIdxArr mapVertices(vertices.size(), vertices.size()*3/2); mapVertices.Memset(0xff); FOREACH(idxF, faces) { @@ -4042,6 +4080,7 @@ void Mesh::ConvertTexturePerVertex(Mesh& mesh) const // with the same position, but different texture coordinates const Face& face = faces[idxF]; Face& newface = mesh.faces[idxF]; + const TexIndex ti = !faceTexindices.empty() ? faceTexindices[idxF] : 0; for (int i=0; i<3; ++i) { const TexCoord& tc = faceTexcoords[idxF*3+i]; VIndex idxV(face[i]); @@ -4050,6 +4089,8 @@ void Mesh::ConvertTexturePerVertex(Mesh& mesh) const if (idxVT == NO_ID) { // vertex seen for the first time, so just copy it mesh.faceTexcoords[newface[i] = idxVT = idxV] = tc; + if (!faceTexindices.empty()) + mesh.faceTexindices[newface[i] = idxVT = idxV] = ti; break; } // vertex already seen in an other face, check the texture coordinates @@ -4063,6 +4104,8 @@ void Mesh::ConvertTexturePerVertex(Mesh& mesh) const mapVertices.emplace_back(newface[i] = idxVT = mesh.vertices.size()); mesh.vertices.emplace_back(vertices[face[i]]); mesh.faceTexcoords.emplace_back(tc); + if (!faceTexindices.empty()) + mesh.faceTexindices.emplace_back(ti); break; } // continue with the next linked vertex which share the position, @@ -4071,7 +4114,7 @@ void Mesh::ConvertTexturePerVertex(Mesh& mesh) const } } } - mesh.textureDiffuse = textureDiffuse; + mesh.texturesDiffuse = texturesDiffuse; } // ConvertTexturePerVertex /*----------------------------------------------------------------*/ @@ -4213,8 +4256,9 @@ void Mesh::SamplePoints(REAL samplingDensity, unsigned mumPointsTheoretic, Point const TexCoord& TO = faceTexcoords[idxTexCoord+0]; const TexCoord& TA = faceTexcoords[idxTexCoord+1]; const TexCoord& TB = faceTexcoords[idxTexCoord+2]; + const TexIndex& TI = faceTexindices[idxFace]; const TexCoord xt(TO + static_cast(x)*(TA - TO) + static_cast(y)*(TB - TO)); - pointcloud.colors.emplace_back(textureDiffuse.sampleSafe(xt)); + pointcloud.colors.emplace_back(texturesDiffuse[TI].sampleSafe(xt)); } } } @@ -4237,7 +4281,7 @@ void Mesh::Project(const Camera& camera, DepthMap& depthMap) const } void Mesh::Project(const Camera& camera, DepthMap& depthMap, Image8U3& image) const { - ASSERT(!faceTexcoords.empty() && !textureDiffuse.empty()); + ASSERT(!faceTexcoords.empty() && !texturesDiffuse.empty()); struct RasterMesh : TRasterMesh { typedef TRasterMesh Base; const Mesh& mesh; @@ -4260,7 +4304,8 @@ void Mesh::Project(const Camera& camera, DepthMap& depthMap, Image8U3& image) co xt = mesh.faceTexcoords[idxFaceTex+0] * pbary[0]; xt += mesh.faceTexcoords[idxFaceTex+1] * pbary[1]; xt += mesh.faceTexcoords[idxFaceTex+2] * pbary[2]; - image(pt) = mesh.textureDiffuse.sampleSafe(xt); + const auto texIdx = mesh.faceTexindices[idxFaceTex / 3]; + image(pt) = mesh.texturesDiffuse[texIdx].sampleSafe(xt); } } }; @@ -4343,7 +4388,7 @@ void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap) const } void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap, Image8U3& image) const { - ASSERT(!faceTexcoords.empty() && !textureDiffuse.empty()); + ASSERT(!faceTexcoords.empty() && !texturesDiffuse.empty()); struct RasterMesh : TRasterMesh { typedef TRasterMesh Base; const Mesh& mesh; @@ -4369,7 +4414,8 @@ void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap, Image8U3& imag xt = mesh.faceTexcoords[idxFaceTex+0] * bary[0]; xt += mesh.faceTexcoords[idxFaceTex+1] * bary[1]; xt += mesh.faceTexcoords[idxFaceTex+2] * bary[2]; - image(pt) = mesh.textureDiffuse.sampleSafe(xt); + auto texIdx = mesh.faceTexindices[idxFaceTex / 3]; + image(pt) = mesh.texturesDiffuse[texIdx].sampleSafe(xt); } } }; @@ -4386,7 +4432,7 @@ void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap, Image8U3& imag // assuming the mesh is properly oriented, ortho-project it to a camera looking from top to down void Mesh::ProjectOrthoTopDown(unsigned resolution, Image8U3& image, Image8U& mask, Point3& center) const { - ASSERT(!IsEmpty() && !textureDiffuse.empty()); + ASSERT(!IsEmpty() && !texturesDiffuse.empty()); // initialize camera const AABB3f box(vertices.Begin(), vertices.GetSize()); const Point3 size(Vertex(box.GetSize())*1.01f/*border*/); @@ -4485,16 +4531,44 @@ Mesh Mesh::SubMesh(const FaceIdxArr& chunk) const Mesh mesh; mesh.vertices = vertices; mesh.faces.reserve(chunk.size()); - for (FIndex idxFace: chunk) + if (!faceTexcoords.empty()) + mesh.faceTexcoords.reserve(chunk.size()*3); + for (FIndex idxFace: chunk) { mesh.faces.emplace_back(faces[idxFace]); + if (!faceTexcoords.empty()) { + const TexCoord* tri = faceTexcoords.data()+idxFace*3; + for (int i = 0; i < 3; ++i) + mesh.faceTexcoords.emplace_back(tri[i]); + } + } mesh.ListIncidenteFaces(); mesh.RemoveUnreferencedVertices(); - // fix non-manifold vertices and edges mesh.FixNonManifold(); return mesh; } // SubMesh /*----------------------------------------------------------------*/ +// extract one sub-mesh for each texture, i.e. for each value of faceTexindices; +std::vector Mesh::SplitMeshPerTextureBlob() const { + + ASSERT(HasTexture()); + if (texturesDiffuse.size() == 1) + return {*this}; + ASSERT(faceTexindices.size() == faces.size()); + std::vector submeshes; + submeshes.reserve(texturesDiffuse.size()); + FOREACH(texId, texturesDiffuse) { + FaceIdxArr chunk; + FOREACH(idxFace, faceTexindices) { + if (faceTexindices[idxFace] == texId) + chunk.push_back(idxFace); + } + Mesh submesh = SubMesh(chunk); + submesh.texturesDiffuse.emplace_back(texturesDiffuse[texId]); + submeshes.emplace_back(std::move(submesh)); + } + return submeshes; +} // transfer the texture of this mesh to the new mesh; @@ -4511,11 +4585,9 @@ inline Eigen::AlignedBox3f bounding_box(const FaceBox& faceBox) { bool Mesh::TransferTexture(Mesh& mesh, const FaceIdxArr& faceSubsetIndices, unsigned borderSize, unsigned textureSize) { ASSERT(HasTexture() && mesh.HasTexture()); - if (mesh.textureDiffuse.empty()) { - mesh.textureDiffuse.create(textureSize, textureSize); - mesh.textureDiffuse.memset(0); - } - Image8U mask(mesh.textureDiffuse.size(), uint8_t(255)); + if (mesh.texturesDiffuse.empty()) + mesh.texturesDiffuse.emplace_back(textureSize, textureSize).memset(0); + Image8U mask(mesh.texturesDiffuse.back().size(), uint8_t(255)); const FIndex num_faces(faceSubsetIndices.empty() ? mesh.faces.size() : faceSubsetIndices.size()); if (vertices == mesh.vertices && faces == mesh.faces) { // the two meshes are identical, only the texture coordinates are different; @@ -4533,15 +4605,16 @@ bool Mesh::TransferTexture(Mesh& mesh, const FaceIdxArr& faceSubsetIndices, unsi Mesh& meshTrg; Image8U& mask; const TexCoord* tri; - inline cv::Size Size() const { return meshTrg.textureDiffuse.size(); } + const TexIndex texId; + inline cv::Size Size() const { return meshTrg.texturesDiffuse[0].size(); } inline void operator()(const ImageRef& pt, const Point3f& bary) { - ASSERT(meshTrg.textureDiffuse.isInside(pt)); + ASSERT(meshTrg.texturesDiffuse[texId].isInside(pt)); const TexCoord x(tri[0]*bary.x + tri[1]*bary.y + tri[2]*bary.z); - const Pixel8U color(meshRef.textureDiffuse.sample(x)); - meshTrg.textureDiffuse(pt) = color; + const Pixel8U color(meshRef.texturesDiffuse[texId].sample(x)); + meshTrg.texturesDiffuse[texId](pt) = color; mask(pt) = 0; } - } data{*this, mesh, mask, faceTexcoords.data()+idxFace*3}; + } data{*this, mesh, mask, faceTexcoords.data()+idxFace*3, mesh.faceTexindices[idxFace]}; // render triangle and for each pixel interpolate the color // from the triangle corners using barycentric coordinates const TexCoord* tri = mesh.faceTexcoords.data()+idxFace*3; @@ -4645,16 +4718,17 @@ bool Mesh::TransferTexture(Mesh& mesh, const FaceIdxArr& faceSubsetIndices, unsi Mesh& meshTrg; Image8U& mask; const Face& face; - inline cv::Size Size() const { return meshTrg.textureDiffuse.size(); } + const TexIndex texId; + inline cv::Size Size() const { return meshTrg.texturesDiffuse.back().size(); } inline void operator()(const ImageRef& pt, const Point3f& bary) { - ASSERT(meshTrg.textureDiffuse.isInside(pt)); + ASSERT(meshTrg.texturesDiffuse[texId].isInside(pt)); const Vertex X(meshTrg.vertices[face[0]]*bary.x + meshTrg.vertices[face[1]]*bary.y + meshTrg.vertices[face[2]]*bary.z); const Normal N(normalized(meshTrg.vertexNormals[face[0]]*bary.x + meshTrg.vertexNormals[face[1]]*bary.y + meshTrg.vertexNormals[face[2]]*bary.z)); - const Ray3f ray((Vertex(X)), Normal(N)); + const Ray3f ray(X, N); #if USE_MESH_INT == USE_MESH_BF const IntersectRayMesh intRay(meshRef, ray); #elif USE_MESH_INT == USE_MESH_BVH @@ -4670,17 +4744,17 @@ bool Mesh::TransferTexture(Mesh& mesh, const FaceIdxArr& faceSubsetIndices, unsi const Vertex baryRef(CorrectBarycentricCoordinates(BarycentricCoordinatesUV(meshRef.vertices[refFace[0]], meshRef.vertices[refFace[1]], meshRef.vertices[refFace[2]], refX))); const TexCoord* tri = meshRef.faceTexcoords.data()+refIdxFace*3; const TexCoord x(tri[0]*baryRef.x + tri[1]*baryRef.y + tri[2]*baryRef.z); - const Pixel8U color(meshRef.textureDiffuse.sample(x)); - meshTrg.textureDiffuse(pt) = color; + const Pixel8U color(meshRef.texturesDiffuse[texId].sample(x)); + meshTrg.texturesDiffuse.back()(pt) = color; mask(pt) = 0; } } #if USE_MESH_INT == USE_MESH_BF - } data{*this, mesh, mask, mesh.faces[idxFace]}; + } data{*this, mesh, mask, mesh.faces[idxFace], mesh.faceTexindices[idxFace]}; #elif USE_MESH_INT == USE_MESH_BVH - } data{tree, *this, mesh, mask, mesh.faces[idxFace]}; + } data{tree, *this, mesh, mask, mesh.faces[idxFace], mesh.faceTexindices[idxFace]}; #else - } data{octree, *this, mesh, mask, mesh.faces[idxFace]}; + } data{octree, *this, mesh, mask, mesh.faces[idxFace], mesh.faceTexindices[idxFace]}; #endif // render triangle and for each pixel interpolate the color // from the triangle corners using barycentric coordinates @@ -4690,7 +4764,7 @@ bool Mesh::TransferTexture(Mesh& mesh, const FaceIdxArr& faceSubsetIndices, unsi } // fill border if (borderSize > 0) { - ASSERT(mask.size().area() == mesh.textureDiffuse.size().area()); + ASSERT(mask.size().area() == mesh.texturesDiffuse[0].size().area()); const int border(static_cast(borderSize)); CLISTDEF0(int) idx_valid_pixels; idx_valid_pixels.push_back(-1); @@ -4701,13 +4775,13 @@ bool Mesh::TransferTexture(Mesh& mesh, const FaceIdxArr& faceSubsetIndices, unsi idx_valid_pixels.push_back(i); Image32F dists; cv::Mat_ labels; cv::distanceTransform(mask, dists, labels, cv::DIST_L1, 3, cv::DIST_LABEL_PIXEL); - ASSERT(mesh.textureDiffuse.isContinuous()); + ASSERT(mesh.texturesDiffuse[0].isContinuous()); for (int i=0; i(dists(i)); if (dist > 0 && dist <= border) { const int label(labels(i)); const int idx_closest_pixel(idx_valid_pixels[label]); - mesh.textureDiffuse(i) = mesh.textureDiffuse(idx_closest_pixel); + mesh.texturesDiffuse[0](i) = mesh.texturesDiffuse[0](idx_closest_pixel); } } } diff --git a/libs/MVS/Mesh.h b/libs/MVS/Mesh.h index 8d962e11f..2310927a3 100644 --- a/libs/MVS/Mesh.h +++ b/libs/MVS/Mesh.h @@ -71,8 +71,9 @@ class MVS_API Mesh typedef TPoint2 TexCoord; typedef SEACAVE::cList TexCoordArr; - typedef uint8_t TexChunk; - typedef SEACAVE::cList TexChunkArr; + typedef uint8_t TexIndex; + typedef SEACAVE::cList TexIndexArr; + typedef SEACAVE::cList Image8U3Arr; typedef TPoint3 FaceFaces; typedef SEACAVE::cList FaceFacesArr; @@ -130,9 +131,10 @@ class MVS_API Mesh NormalArr faceNormals; // for each face, the normal to it (optional) FaceFacesArr faceFaces; // for each face, the list of adjacent faces, NO_ID for border edges (optional) - TexCoordArr faceTexcoords; // for each face, the texture-coordinates corresponding to the contained vertices (optional) + TexCoordArr faceTexcoords; // for each face, the texture-coordinates corresponding to its vertices, 3x num faces OR for each vertex (optional) + TexIndexArr faceTexindices; // for each face, the corresponding index of the texture (optional) - Image8U3 textureDiffuse; // texture containing the diffuse color (optional) + Image8U3Arr texturesDiffuse; // textures containing the diffuse color (optional) #ifdef _USE_CUDA static CUDA::KernelRT kernelComputeFaceNormal; @@ -150,9 +152,10 @@ class MVS_API Mesh void EmptyExtra(); void Swap(Mesh&); void Join(const Mesh&); - inline bool IsEmpty() const { return vertices.empty(); } + bool IsEmpty() const { return vertices.empty(); } bool IsWatertight(); - inline bool HasTexture() const { ASSERT(faceTexcoords.empty() == textureDiffuse.empty()); return !faceTexcoords.empty(); } + bool HasTexture() const { ASSERT(faces.size()*3 == faceTexcoords.size() || vertices.size() == faceTexcoords.size()); return !faceTexcoords.empty() && !texturesDiffuse.empty(); } + bool HasTextureCoordinatesPerVertex() const { return !faceTexcoords.empty() && vertices.size() == faceTexcoords.size(); } Box GetAABB() const; Box GetAABB(const Box& bound) const; @@ -187,8 +190,10 @@ class MVS_API Mesh void RemoveFaces(FaceIdxArr& facesRemove, bool bUpdateLists=false); void RemoveVertices(VertexIdxArr& vertexRemove, bool bUpdateLists=false); VIndex RemoveUnreferencedVertices(bool bUpdateLists=false); + std::vector SplitMeshPerTextureBlob() const; void ConvertTexturePerVertex(Mesh&) const; + TexIndex GetFaceTextureIndex(FIndex idxF) const { return faceTexindices.empty() ? 0 : faceTexindices[idxF]; } void FaceTexcoordsNormalize(TexCoordArr& newFaceTexcoords, bool flipY=true) const; void FaceTexcoordsUnnormalize(TexCoordArr& newFaceTexcoords, bool flipY=true) const; @@ -264,7 +269,8 @@ class MVS_API Mesh ar & vertexBoundary; ar & faceNormals; ar & faceTexcoords; - ar & textureDiffuse; + ar & faceTexindices; + ar & texturesDiffuse; } #endif }; diff --git a/libs/MVS/RectsBinPack.cpp b/libs/MVS/RectsBinPack.cpp index 3a376fd8f..0f93358c8 100644 --- a/libs/MVS/RectsBinPack.cpp +++ b/libs/MVS/RectsBinPack.cpp @@ -88,12 +88,10 @@ MaxRectsBinPack::Rect MaxRectsBinPack::Insert(int width, int height, FreeRectCho return newNode; } -bool MaxRectsBinPack::Insert(RectArr& rects, FreeRectChoiceHeuristic method) +MaxRectsBinPack::RectWIdxArr MaxRectsBinPack::Insert(RectWIdxArr& unplacedRects, FreeRectChoiceHeuristic method) { - cList indices(rects.GetSize()); - std::iota(indices.Begin(), indices.End(), 0); - RectArr newRects(rects.GetSize()); - while (!rects.IsEmpty()) { + RectWIdxArr placedRects; + while (!unplacedRects.IsEmpty()) { int bestScore1 = std::numeric_limits::max(); int bestScore2 = std::numeric_limits::max(); IDX bestRectIndex = NO_IDX; @@ -108,9 +106,9 @@ bool MaxRectsBinPack::Insert(RectArr& rects, FreeRectChoiceHeuristic method) IDX privBestRectIndex = NO_IDX; Rect privBestNode; #pragma omp for nowait - for (int_t i=0; i<(int_t)rects.GetSize(); ++i) { + for (int_t i=0; i<(int_t)unplacedRects.GetSize(); ++i) { int score1, score2; - Rect newNode(ScoreRect(rects[i].width, rects[i].height, method, score1, score2)); + Rect newNode(ScoreRect(unplacedRects[i].rect.width, unplacedRects[i].rect.height, method, score1, score2)); if (score1 < privBestScore1 || (score1 == privBestScore1 && score2 < privBestScore2)) { privBestScore1 = score1; privBestScore2 = score2; @@ -141,24 +139,18 @@ bool MaxRectsBinPack::Insert(RectArr& rects, FreeRectChoiceHeuristic method) } #endif - // if no place found... + // if no place found, return the placed rectangles list if (bestRectIndex == NO_IDX) { - // restore the original rectangles - FOREACH(j, rects) - newRects[indices[j]] = rects[j]; - rects.Swap(newRects); - return false; + break; } // store rectangle PlaceRect(bestNode); - newRects[indices[bestRectIndex]] = bestNode; - rects.RemoveAt(bestRectIndex); - indices.RemoveAt(bestRectIndex); + placedRects.Insert(RectWIdx{bestNode, unplacedRects[bestRectIndex].patchIdx}); + unplacedRects.RemoveAt(bestRectIndex); } - rects.Swap(newRects); - return true; + return placedRects; } void MaxRectsBinPack::PlaceRect(const Rect &node) @@ -524,6 +516,13 @@ int MaxRectsBinPack::ComputeTextureSize(const RectArr& rects, int mult) // ... as power of two return POWI(2, CEIL2INT(LOGN((float)sizeTex) / LOGN(2.f))); } + +int MaxRectsBinPack::ComputeTextureSize(const RectWIdxArr& rectsWIdx, int mult) { + RectArr rects(rectsWIdx.GetSize()); + FOREACH(i, rectsWIdx) + rects[i] = rectsWIdx[i].rect; + return ComputeTextureSize(rects, mult); +} /*----------------------------------------------------------------*/ @@ -563,7 +562,7 @@ void GuillotineBinPack::Init(int width, int height) freeRectangles.push_back(n); } -bool GuillotineBinPack::Insert(RectArr& rects, bool merge, +GuillotineBinPack::RectWIdxArr GuillotineBinPack::Insert(RectWIdxArr& unplacedRects, bool merge, FreeRectChoiceHeuristic rectChoice, GuillotineSplitHeuristic splitMethod) { // Remember variables about the best packing choice we have made so far during the iteration process. @@ -571,19 +570,18 @@ bool GuillotineBinPack::Insert(RectArr& rects, bool merge, size_t bestRect = 0; bool bestFlipped = false; - // Pack rectangles one at a time until we have cleared the rects array of all rectangles. - // rects will get destroyed in the process. - cList indices(rects.GetSize()); - std::iota(indices.Begin(), indices.End(), 0); - RectArr newRects(rects.GetSize()); - while (!rects.IsEmpty()) { + // Pack rectangles one at a time until we have cleared the rects array of all rectangles or there is no space. + // unplacedRects will get destroyed in the process. + RectWIdxArr placedRects; + while (!unplacedRects.IsEmpty()) { // Stores the penalty score of the best rectangle placement - bigger=worse, smaller=better. int bestScore = std::numeric_limits::max(); for (size_t i = 0; i < freeRectangles.size(); ++i) { - for (size_t j = 0; j < rects.GetSize(); ++j) { + for (size_t j = 0; j < unplacedRects.GetSize(); ++j) { + Rect currentRect = unplacedRects[j].rect; // If this rectangle is a perfect match, we pick it instantly. - if (rects[j].width == freeRectangles[i].width && rects[j].height == freeRectangles[i].height) { + if (currentRect.width == freeRectangles[i].width && currentRect.height == freeRectangles[i].height) { bestFreeRect = i; bestRect = j; bestFlipped = false; @@ -592,7 +590,7 @@ bool GuillotineBinPack::Insert(RectArr& rects, bool merge, break; } // If flipping this rectangle is a perfect match, pick that then. - else if (rects[j].height == freeRectangles[i].width && rects[j].width == freeRectangles[i].height) { + else if (currentRect.height == freeRectangles[i].width && currentRect.width == freeRectangles[i].height) { bestFreeRect = i; bestRect = j; bestFlipped = true; @@ -601,8 +599,8 @@ bool GuillotineBinPack::Insert(RectArr& rects, bool merge, break; } // Try if we can fit the rectangle upright. - else if (rects[j].width <= freeRectangles[i].width && rects[j].height <= freeRectangles[i].height) { - int score = ScoreByHeuristic(rects[j].width, rects[j].height, freeRectangles[i], rectChoice); + else if (currentRect.width <= freeRectangles[i].width && currentRect.height <= freeRectangles[i].height) { + int score = ScoreByHeuristic(currentRect.width, currentRect.height, freeRectangles[i], rectChoice); if (score < bestScore) { bestFreeRect = i; bestRect = j; @@ -611,8 +609,8 @@ bool GuillotineBinPack::Insert(RectArr& rects, bool merge, } } // If not, then perhaps flipping sideways will make it fit? - else if (rects[j].height <= freeRectangles[i].width && rects[j].width <= freeRectangles[i].height) { - int score = ScoreByHeuristic(rects[j].height, rects[j].width, freeRectangles[i], rectChoice); + else if (currentRect.height <= freeRectangles[i].width && currentRect.width <= freeRectangles[i].height) { + int score = ScoreByHeuristic(currentRect.height, currentRect.width, freeRectangles[i], rectChoice); if (score < bestScore) { bestFreeRect = i; bestRect = j; @@ -625,19 +623,15 @@ bool GuillotineBinPack::Insert(RectArr& rects, bool merge, // If we didn't manage to find any rectangle to pack, abort. if (bestScore == std::numeric_limits::max()) { - // restore the original rectangles - FOREACH(j, rects) - newRects[indices[j]] = rects[j]; - rects.Swap(newRects); - return false; + break; } // Otherwise, we're good to go and do the actual packing. Rect newNode; newNode.x = freeRectangles[bestFreeRect].x; newNode.y = freeRectangles[bestFreeRect].y; - newNode.width = rects[bestRect].width; - newNode.height = rects[bestRect].height; + newNode.width = unplacedRects[bestRect].rect.width; + newNode.height = unplacedRects[bestRect].rect.height; if (bestFlipped) std::swap(newNode.width, newNode.height); @@ -647,9 +641,8 @@ bool GuillotineBinPack::Insert(RectArr& rects, bool merge, freeRectangles.erase(freeRectangles.begin() + bestFreeRect); // Remove the rectangle we just packed from the input list. - newRects[indices[bestRect]] = newNode; - rects.RemoveAt(bestRect); - indices.RemoveAt(bestRect); + placedRects.Insert(MaxRectsBinPack::RectWIdx{newNode, unplacedRects[bestRect].patchIdx}); + unplacedRects.RemoveAt(bestRect); // Perform a Rectangle Merge step if desired. if (merge) @@ -661,7 +654,7 @@ bool GuillotineBinPack::Insert(RectArr& rects, bool merge, // Check that we're really producing correct packings here. ASSERT(disjointRects.Add(newNode) == true); } - return true; + return placedRects; } GuillotineBinPack::Rect GuillotineBinPack::Insert(int width, int height, bool merge, FreeRectChoiceHeuristic rectChoice, GuillotineSplitHeuristic splitMethod) @@ -984,12 +977,10 @@ void SkylineBinPack::Init(int width, int height, bool useWasteMap_) } } -bool SkylineBinPack::Insert(RectArr& rects, LevelChoiceHeuristic method) +SkylineBinPack::RectWIdxArr SkylineBinPack::Insert(RectWIdxArr& unplacedRects, LevelChoiceHeuristic method) { - cList indices(rects.GetSize()); - std::iota(indices.Begin(), indices.End(), 0); - RectArr newRects(rects.GetSize()); - while (!rects.IsEmpty()) { + RectWIdxArr placedRects; + while (!unplacedRects.IsEmpty()) { int bestScore1 = std::numeric_limits::max(); int bestScore2 = std::numeric_limits::max(); int bestSkylineIndex = -1; @@ -1005,9 +996,9 @@ bool SkylineBinPack::Insert(RectArr& rects, LevelChoiceHeuristic method) IDX privBestRectIndex = NO_IDX; Rect privBestNode; #pragma omp for nowait - for (int_t i=0; i<(int_t)rects.GetSize(); ++i) { + for (int_t i=0; i<(int_t)unplacedRects.GetSize(); ++i) { int score1, score2, index; - Rect newNode(ScoreRect(rects[i].width, rects[i].height, method, score1, score2, index)); + Rect newNode(ScoreRect(unplacedRects[i].rect.width, unplacedRects[i].rect.height, method, score1, score2, index)); if (score1 < privBestScore1 || (score1 == privBestScore1 && score2 < privBestScore2)) { privBestScore1 = score1; privBestScore2 = score2; @@ -1028,9 +1019,9 @@ bool SkylineBinPack::Insert(RectArr& rects, LevelChoiceHeuristic method) } } #else - FOREACH(i, rects) { + FOREACH(i, unplacedRects) { int score1, score2, index; - Rect newNode(ScoreRect(rects[i].width, rects[i].height, method, score1, score2, index)); + Rect newNode(ScoreRect(unplacedRects[i].rect.width, unplacedRects[i].rect.height, method, score1, score2, index)); if (score1 < bestScore1 || (score1 == bestScore1 && score2 < bestScore2)) { bestNode = newNode; bestScore1 = score1; @@ -1041,13 +1032,9 @@ bool SkylineBinPack::Insert(RectArr& rects, LevelChoiceHeuristic method) } #endif - // if no place found... + // if no place found, give up if (bestRectIndex == NO_IDX) { - // restore the original rectangles - FOREACH(j, rects) - newRects[indices[j]] = rects[j]; - rects.Swap(newRects); - return false; + break; } // Perform the actual packing. @@ -1056,14 +1043,12 @@ bool SkylineBinPack::Insert(RectArr& rects, LevelChoiceHeuristic method) disjointRects.Add(bestNode); #endif AddSkylineLevel(bestSkylineIndex, bestNode); - usedSurfaceArea += rects[bestRectIndex].area(); + usedSurfaceArea += unplacedRects[bestRectIndex].rect.area(); - newRects[indices[bestRectIndex]] = bestNode; - rects.RemoveAt(bestRectIndex); - indices.RemoveAt(bestRectIndex); + placedRects.Insert(MaxRectsBinPack::RectWIdx{bestNode, unplacedRects[bestRectIndex].patchIdx}); + unplacedRects.RemoveAt(bestRectIndex); } - rects.Swap(newRects); - return true; + return placedRects; } SkylineBinPack::Rect SkylineBinPack::Insert(int width, int height, LevelChoiceHeuristic method) diff --git a/libs/MVS/RectsBinPack.h b/libs/MVS/RectsBinPack.h index 390ec1a9f..c1ef9b048 100644 --- a/libs/MVS/RectsBinPack.h +++ b/libs/MVS/RectsBinPack.h @@ -57,8 +57,17 @@ namespace MVS { class MaxRectsBinPack { public: + // A simple rectangle typedef cv::Rect Rect; + // A rectangle that stores an index of origin + typedef struct { + Rect rect; + uint32_t patchIdx; + } RectWIdx; + /// A list of rectangles typedef CLISTDEF0(Rect) RectArr; + /// A list of rectangles along their original indices + typedef CLISTDEF0(RectWIdx) RectWIdxArr; /// Instantiates a bin of size (0,0). Call Init to create a new bin. MaxRectsBinPack(); @@ -84,7 +93,7 @@ class MaxRectsBinPack /// @param rects [IN/OUT] The list of rectangles to insert; the rectangles will be modified with the new coordinates in the process. /// @param method The rectangle placement rule to use when packing. /// returns true if all rectangles were inserted - bool Insert(RectArr& rects, FreeRectChoiceHeuristic method=RectBestShortSideFit); + RectWIdxArr Insert(RectWIdxArr& rects, FreeRectChoiceHeuristic method=RectBestShortSideFit); /// Inserts a single rectangle into the bin, possibly rotated. Rect Insert(int width, int height, FreeRectChoiceHeuristic method=RectBestShortSideFit); @@ -94,6 +103,7 @@ class MaxRectsBinPack /// Computes an approximate texture atlas size. static int ComputeTextureSize(const RectArr& rects, int mult=0); + static int ComputeTextureSize(const RectWIdxArr& rects, int mult=0); /// Returns true if a is contained/on the border in b. static inline bool IsContainedIn(const Rect& a, const Rect& b) { @@ -190,8 +200,12 @@ class DisjointRectCollection class GuillotineBinPack { public: + // A simple rectangle typedef cv::Rect Rect; + /// A list of rectangles typedef CLISTDEF0(Rect) RectArr; + /// A list of rectangles along their original indices + typedef CLISTDEF0(MaxRectsBinPack::RectWIdx) RectWIdxArr; /// The initial bin size will be (0,0). Call Init to set the bin size. GuillotineBinPack(); @@ -243,7 +257,7 @@ class GuillotineBinPack /// @param merge If true, performs Rectangle Merge operations during the packing process. /// @param rectChoice The free rectangle choice heuristic rule to use. /// @param splitMethod The free rectangle split heuristic rule to use. - bool Insert(RectArr& rects, bool merge, + RectWIdxArr Insert(RectWIdxArr& rects, bool merge, FreeRectChoiceHeuristic rectChoice, GuillotineSplitHeuristic splitMethod); /// Computes the ratio of used/total surface area. 0.00 means no space is yet used, 1.00 means the whole bin is used. @@ -311,8 +325,12 @@ class GuillotineBinPack class SkylineBinPack { public: + // A simple rectangle typedef cv::Rect Rect; + /// A list of rectangles typedef CLISTDEF0(Rect) RectArr; + /// A list of rectangles along their original indices + typedef CLISTDEF0(MaxRectsBinPack::RectWIdx) RectWIdxArr; /// Instantiates a bin of size (0,0). Call Init to create a new bin. SkylineBinPack(); @@ -335,7 +353,7 @@ class SkylineBinPack /// Inserts the given list of rectangles in an offline/batch mode, possibly rotated. /// @param rects [in/out] The list of rectangles to insert. This vector will be update in the process. /// @param method The rectangle placement rule to use when packing. - bool Insert(RectArr& rects, LevelChoiceHeuristic method); + RectWIdxArr Insert(RectWIdxArr& rects, LevelChoiceHeuristic method); /// Inserts a single rectangle into the bin, possibly rotated. Rect Insert(int width, int height, LevelChoiceHeuristic method); diff --git a/libs/MVS/Scene.h b/libs/MVS/Scene.h index 8580c623e..8eaa96369 100644 --- a/libs/MVS/Scene.h +++ b/libs/MVS/Scene.h @@ -152,7 +152,7 @@ class MVS_API Scene // Mesh texturing bool TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsigned minCommonCameras=0, float fOutlierThreshold=0.f, float fRatioDataSmoothness=0.3f, bool bGlobalSeamLeveling=true, bool bLocalSeamLeveling=true, unsigned nTextureSizeMultiple=0, unsigned nRectPackingHeuristic=3, Pixel8U colEmpty=Pixel8U(255,127,39), - float fSharpnessWeight=0.5f, int ignoreMaskLabel = -1, const IIndexArr& views=IIndexArr()); + float fSharpnessWeight=0.5f, int ignoreMaskLabel=-1, int maxTextureSize=0, const IIndexArr& views=IIndexArr()); #ifdef _USE_BOOST // implement BOOST serialization diff --git a/libs/MVS/SceneTexture.cpp b/libs/MVS/SceneTexture.cpp index e8404ba50..e15f06169 100644 --- a/libs/MVS/SceneTexture.cpp +++ b/libs/MVS/SceneTexture.cpp @@ -133,6 +133,7 @@ typedef Mesh::VIndex VIndex; typedef Mesh::Face Face; typedef Mesh::FIndex FIndex; typedef Mesh::TexCoord TexCoord; +typedef Mesh::TexIndex TexIndex; typedef int MatIdx; typedef Eigen::Triplet MatEntry; @@ -340,7 +341,7 @@ struct MeshTexture { void CreateSeamVertices(); void GlobalSeamLeveling(); void LocalSeamLeveling(); - void GenerateTexture(bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty, float fSharpnessWeight); + void GenerateTexture(bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty, float fSharpnessWeight, int maxTextureSize); template static inline PIXEL RGB2YCBCR(const PIXEL& v) { @@ -387,7 +388,8 @@ struct MeshTexture { BoolArr& vertexBoundary; // for each vertex, stores if it is at the boundary or not Mesh::FaceFacesArr& faceFaces; // for each face, the list of adjacent faces, NO_ID for border edges (optional) Mesh::TexCoordArr& faceTexcoords; // for each face, the texture-coordinates of the vertices - Image8U3& textureDiffuse; // texture containing the diffuse color + Mesh::TexIndexArr& faceTexindices; // for each face, the texture-coordinates of the vertices + Mesh::Image8U3Arr& texturesDiffuse; // texture containing the diffuse color // constant the entire time Mesh::VertexArr& vertices; @@ -436,7 +438,8 @@ MeshTexture::MeshTexture(Scene& _scene, unsigned _nResolutionLevel, unsigned _nM vertexBoundary(_scene.mesh.vertexBoundary), faceFaces(_scene.mesh.faceFaces), faceTexcoords(_scene.mesh.faceTexcoords), - textureDiffuse(_scene.mesh.textureDiffuse), + faceTexindices(_scene.mesh.faceTexindices), + texturesDiffuse(_scene.mesh.texturesDiffuse), vertices(_scene.mesh.vertices), faces(_scene.mesh.faces), images(_scene.images), @@ -556,7 +559,7 @@ bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThr #endif } rasterer.Clear(); - for (auto idxFace : cameraFaces) { + for (FIndex idxFace : cameraFaces) { rasterer.validFace = true; const Face& facet = faces[idxFace]; rasterer.idxFace = idxFace; @@ -2109,11 +2112,12 @@ void MeshTexture::LocalSeamLeveling() } } -void MeshTexture::GenerateTexture(bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty, float fSharpnessWeight) +void MeshTexture::GenerateTexture(bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty, float fSharpnessWeight, int maxTextureSize) { // project patches in the corresponding view and compute texture-coordinates and bounding-box const int border(2); faceTexcoords.resize(faces.size()*3); + faceTexindices.resize(faces.size()); #ifdef TEXOPT_USE_OPENMP const unsigned numPatches(texturePatches.size()-1); #pragma omp parallel for schedule(dynamic) @@ -2210,84 +2214,115 @@ void MeshTexture::GenerateTexture(bool bGlobalSeamLeveling, bool bLocalSeamLevel // create texture { // arrange texture patches to fit the smallest possible texture image - RectsBinPack::RectArr rects(texturePatches.GetSize()); - FOREACH(i, texturePatches) - rects[i] = texturePatches[i].rect; - int textureSize(RectsBinPack::ComputeTextureSize(rects, nTextureSizeMultiple)); - // increase texture size till all patches fit - while (true) { - TD_TIMER_STARTD(); - bool bPacked(false); + RectsBinPack::RectWIdxArr unplacedRects(texturePatches.size()); + FOREACH(i, texturePatches) { + if (maxTextureSize > 0 && (texturePatches[i].rect.width > maxTextureSize || texturePatches[i].rect.height > maxTextureSize)) { + DEBUG("error: a patch of size %u x %u does not fit the texture", texturePatches[i].rect.width, texturePatches[i].rect.height); + ABORT("the maximum texture size chosen cannot fit a patch"); + } + unplacedRects[i] = {texturePatches[i].rect, i}; + } + + // pack patches: one pack per texture file + CLISTDEF2IDX(RectsBinPack::RectWIdxArr, TexIndex) placedRects; { + // increase texture size till all patches fit const unsigned typeRectsBinPack(nRectPackingHeuristic/100); const unsigned typeSplit((nRectPackingHeuristic-typeRectsBinPack*100)/10); const unsigned typeHeuristic(nRectPackingHeuristic%10); - switch (typeRectsBinPack) { - case 0: { - MaxRectsBinPack pack(textureSize, textureSize); - bPacked = pack.Insert(rects, (MaxRectsBinPack::FreeRectChoiceHeuristic)typeHeuristic); - break; } - case 1: { - SkylineBinPack pack(textureSize, textureSize, typeSplit!=0); - bPacked = pack.Insert(rects, (SkylineBinPack::LevelChoiceHeuristic)typeHeuristic); - break; } - case 2: { - GuillotineBinPack pack(textureSize, textureSize); - bPacked = pack.Insert(rects, false, (GuillotineBinPack::FreeRectChoiceHeuristic)typeHeuristic, (GuillotineBinPack::GuillotineSplitHeuristic)typeSplit); - break; } - default: - ABORT("error: unknown RectsBinPack type"); + int textureSize = 0; + while (!unplacedRects.empty()) { + TD_TIMER_STARTD(); + if (textureSize == 0) { + textureSize = RectsBinPack::ComputeTextureSize(unplacedRects, nTextureSizeMultiple); + if (maxTextureSize > 0 && textureSize > maxTextureSize) + textureSize = maxTextureSize; + } + + RectsBinPack::RectWIdxArr newPlacedRects; + switch (typeRectsBinPack) { + case 0: { + MaxRectsBinPack pack(textureSize, textureSize); + newPlacedRects = pack.Insert(unplacedRects, (MaxRectsBinPack::FreeRectChoiceHeuristic)typeHeuristic); + break; } + case 1: { + SkylineBinPack pack(textureSize, textureSize, typeSplit!=0); + newPlacedRects = pack.Insert(unplacedRects, (SkylineBinPack::LevelChoiceHeuristic)typeHeuristic); + break; } + case 2: { + GuillotineBinPack pack(textureSize, textureSize); + newPlacedRects = pack.Insert(unplacedRects, false, (GuillotineBinPack::FreeRectChoiceHeuristic)typeHeuristic, (GuillotineBinPack::GuillotineSplitHeuristic)typeSplit); + break; } + default: + ABORT("error: unknown RectsBinPack type"); + } + DEBUG_ULTIMATE("\tpacking texture completed: %u initial patches, %u placed patches, %u texture-size, %u textures (%s)", texturePatches.size(), newPlacedRects.GetSize(), textureSize, placedRects.size(), TD_TIMER_GET_FMT().c_str()); + + if (textureSize == maxTextureSize || unplacedRects.empty()) { + // create texture image + placedRects.emplace_back(std::move(newPlacedRects)); + texturesDiffuse.emplace_back(textureSize, textureSize).setTo(cv::Scalar(colEmpty.b, colEmpty.g, colEmpty.r)); + textureSize = 0; + } else { + // try again with a bigger texture + textureSize *= 2; + if (maxTextureSize > 0) + textureSize = std::max(textureSize, maxTextureSize); + unplacedRects.JoinRemove(newPlacedRects); + } } - DEBUG_ULTIMATE("\tpacking texture completed: %u patches, %u texture-size (%s)", rects.size(), textureSize, TD_TIMER_GET_FMT().c_str()); - if (bPacked) - break; - textureSize *= 2; } - // create texture image - textureDiffuse.create(textureSize, textureSize); - textureDiffuse.setTo(cv::Scalar(colEmpty.b, colEmpty.g, colEmpty.r)); #ifdef TEXOPT_USE_OPENMP #pragma omp parallel for schedule(dynamic) - for (int_t i=0; i<(int_t)texturePatches.size(); ++i) { - const uint32_t idxPatch((uint32_t)i); + for (int_t i=0; i<(int_t)placedRects.size(); ++i) { + for (int_t j=0; j<(int_t)placedRects[(TexIndex)i].size(); ++j) { + const TexIndex idxTexture((TexIndex)i); + const uint32_t idxPlacedPatch((uint32_t)j); #else - FOREACH(idxPatch, texturePatches) { + FOREACH(idxTexture, placedRects) { + FOREACH(idxPlacedPatch, placedRects[idxTexture]) { #endif - const TexturePatch& texturePatch = texturePatches[idxPatch]; - const RectsBinPack::Rect& rect = rects[idxPatch]; - // copy patch image - ASSERT((rect.width == texturePatch.rect.width && rect.height == texturePatch.rect.height) || - (rect.height == texturePatch.rect.width && rect.width == texturePatch.rect.height)); - int x(0), y(1); - if (texturePatch.label != NO_ID) { - const Image& imageData = images[texturePatch.label]; - cv::Mat patch(imageData.image(texturePatch.rect)); - if (rect.width != texturePatch.rect.width) { - // flip patch and texture-coordinates - patch = patch.t(); - x = 1; y = 0; + const TexturePatch& texturePatch = texturePatches[placedRects[idxTexture][idxPlacedPatch].patchIdx]; + const RectsBinPack::Rect& rect = placedRects[idxTexture][idxPlacedPatch].rect; + // copy patch image + ASSERT((rect.width == texturePatch.rect.width && rect.height == texturePatch.rect.height) || + (rect.height == texturePatch.rect.width && rect.width == texturePatch.rect.height)); + int x(0), y(1); + if (texturePatch.label != NO_ID) { + const Image& imageData = images[texturePatch.label]; + cv::Mat patch(imageData.image(texturePatch.rect)); + if (rect.width != texturePatch.rect.width) { + // flip patch and texture-coordinates + patch = patch.t(); + x = 1; y = 0; + } + patch.copyTo(texturesDiffuse[idxTexture](rect)); } - patch.copyTo(textureDiffuse(rect)); - } - // compute final texture coordinates - const TexCoord offset(rect.tl()); - for (const FIndex idxFace: texturePatch.faces) { - TexCoord* texcoords = faceTexcoords.data()+idxFace*3; - for (int v=0; v<3; ++v) { - TexCoord& texcoord = texcoords[v]; - texcoord = TexCoord( - texcoord[x]+offset.x, - texcoord[y]+offset.y - ); + // compute final texture coordinates + const TexCoord offset(rect.tl()); + for (const FIndex idxFace: texturePatch.faces) { + TexCoord* texcoords = faceTexcoords.data()+idxFace*3; + faceTexindices[idxFace] = idxTexture; + for (int v=0; v<3; ++v) { + TexCoord& texcoord = texcoords[v]; + texcoord = TexCoord( + texcoord[x]+offset.x, + texcoord[y]+offset.y + ); + } } } } + if (texturesDiffuse.size() == 1) + faceTexindices.Release(); // apply some sharpening if (fSharpnessWeight > 0) { constexpr double sigma = 1.5; - Image8U3 blurryTextureDiffuse; - cv::GaussianBlur(textureDiffuse, blurryTextureDiffuse, cv::Size(), sigma); - cv::addWeighted(textureDiffuse, 1+fSharpnessWeight, blurryTextureDiffuse, -fSharpnessWeight, 0, textureDiffuse); + for (auto &textureDiffuse: texturesDiffuse) { + Image8U3 blurryTextureDiffuse; + cv::GaussianBlur(textureDiffuse, blurryTextureDiffuse, cv::Size(), sigma); + cv::addWeighted(textureDiffuse, 1+fSharpnessWeight, blurryTextureDiffuse, -fSharpnessWeight, 0, textureDiffuse); + } } } } @@ -2298,7 +2333,7 @@ void MeshTexture::GenerateTexture(bool bGlobalSeamLeveling, bool bLocalSeamLevel // - nIgnoreMaskLabel: label value to ignore in the image mask, stored in the MVS scene or next to each image with '.mask.png' extension (-1 - auto estimate mask for lens distortion, -2 - disabled) bool Scene::TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty, float fSharpnessWeight, - int nIgnoreMaskLabel, const IIndexArr& views) + int nIgnoreMaskLabel, int maxTextureSize, const IIndexArr& views) { MeshTexture texture(*this, nResolutionLevel, nMinResolution); @@ -2313,8 +2348,8 @@ bool Scene::TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsi // generate the texture image and atlas { TD_TIMER_STARTD(); - texture.GenerateTexture(bGlobalSeamLeveling, bLocalSeamLeveling, nTextureSizeMultiple, nRectPackingHeuristic, colEmpty, fSharpnessWeight); - DEBUG_EXTRA("Generating texture atlas and image completed: %u patches, %u image size (%s)", texture.texturePatches.GetSize(), mesh.textureDiffuse.width(), TD_TIMER_GET_FMT().c_str()); + texture.GenerateTexture(bGlobalSeamLeveling, bLocalSeamLeveling, nTextureSizeMultiple, nRectPackingHeuristic, colEmpty, fSharpnessWeight, maxTextureSize); + DEBUG_EXTRA("Generating texture atlas and image completed: %u patches, %u image size, %u textures (%s)", texture.texturePatches.GetSize(), mesh.texturesDiffuse[0].width(), mesh.texturesDiffuse.size(), TD_TIMER_GET_FMT().c_str()); } return true;