From 0d1a991bb128ea8adc6e10d60ea36c6a4e458e61 Mon Sep 17 00:00:00 2001 From: cDc Date: Sat, 9 Nov 2024 12:35:20 +0200 Subject: [PATCH] mesh: make TRasterMesh multithread-safe --- apps/Tests/Tests.cpp | 3 + libs/MVS/DepthMap.cpp | 39 ++++++----- libs/MVS/Mesh.cpp | 140 +++++++++++++++++++++++++++++++------- libs/MVS/Mesh.h | 64 +++++++++++------ libs/MVS/SceneRefine.cpp | 10 +-- libs/MVS/SceneTexture.cpp | 12 ++-- 6 files changed, 196 insertions(+), 72 deletions(-) diff --git a/apps/Tests/Tests.cpp b/apps/Tests/Tests.cpp index ebc730cbd..a34a4c4bc 100644 --- a/apps/Tests/Tests.cpp +++ b/apps/Tests/Tests.cpp @@ -104,6 +104,9 @@ bool PipelineTest(bool verbose=false) VERBOSE("ERROR: TestDataset failed cleaning the mesh!"); return false; } + #ifdef _USE_OPENMP + TestMeshProjectionMT(scene.mesh, scene.images[1]); + #endif if (!scene.TextureMesh(0, 0) || !scene.mesh.HasTexture()) { VERBOSE("ERROR: TestDataset failed texturing the mesh!"); return false; diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index c6e85bd7f..99fc24643 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -1107,18 +1107,17 @@ bool MVS::TriangulatePoints2DepthMap( // rasterize triangles onto depthmap struct RasterDepth : TRasterMeshBase { typedef TRasterMeshBase Base; + using Base::Triangle; using Base::camera; using Base::depthMap; - using Base::ptc; - using Base::pti; const Mesh::NormalArr& vertexNormals; NormalMap& normalMap; Mesh::Face face; RasterDepth(const Mesh::NormalArr& _vertexNormals, const Camera& _camera, DepthMap& _depthMap, NormalMap& _normalMap) : Base(_camera, _depthMap), vertexNormals(_vertexNormals), normalMap(_normalMap) {} - inline void operator()(const ImageRef& pt, const Point3f& bary) { - const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(bary)); - const Depth z(ComputeDepth(pbary)); + inline void Raster(const ImageRef& pt, const Triangle& t, const Point3f& bary) { + const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(t, bary)); + const Depth z(ComputeDepth(t, pbary)); ASSERT(z > Depth(0)); depthMap(pt) = z; normalMap(pt) = normalized( @@ -1128,16 +1127,18 @@ bool MVS::TriangulatePoints2DepthMap( ); } }; - RasterDepth rasterer = {mesh.vertexNormals, camera, depthMap, normalMap}; + RasterDepth rasterer {mesh.vertexNormals, camera, depthMap, normalMap}; + RasterDepth::Triangle triangle; + RasterDepth::TriangleRasterizer triangleRasterizer(triangle, rasterer); for (const Mesh::Face& face : mesh.faces) { rasterer.face = face; - rasterer.ptc[0].z = mesh.vertices[face[0]].z; - rasterer.ptc[1].z = mesh.vertices[face[1]].z; - rasterer.ptc[2].z = mesh.vertices[face[2]].z; + triangle.ptc[0].z = mesh.vertices[face[0]].z; + triangle.ptc[1].z = mesh.vertices[face[1]].z; + triangle.ptc[2].z = mesh.vertices[face[2]].z; Image8U::RasterizeTriangleBary( projs[face[0]], projs[face[1]], - projs[face[2]], rasterer); + projs[face[2]], triangleRasterizer); } } return true; @@ -1181,22 +1182,24 @@ bool MVS::TriangulatePoints2DepthMap( using Base::depthMap; RasterDepth(const Camera& _camera, DepthMap& _depthMap) : Base(_camera, _depthMap) {} - inline void operator()(const ImageRef& pt, const Point3f& bary) { - const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(bary)); - const Depth z(ComputeDepth(pbary)); + inline void Raster(const ImageRef& pt, const Triangle& t, const Point3f& bary) { + const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(t, bary)); + const Depth z(ComputeDepth(t, pbary)); ASSERT(z > Depth(0)); depthMap(pt) = z; } }; - RasterDepth rasterer = {camera, depthMap}; + RasterDepth rasterer {camera, depthMap}; + RasterDepth::Triangle triangle; + RasterDepth::TriangleRasterizer triangleRasterizer(triangle, rasterer); for (const Mesh::Face& face : mesh.faces) { - rasterer.ptc[0].z = mesh.vertices[face[0]].z; - rasterer.ptc[1].z = mesh.vertices[face[1]].z; - rasterer.ptc[2].z = mesh.vertices[face[2]].z; + triangle.ptc[0].z = mesh.vertices[face[0]].z; + triangle.ptc[1].z = mesh.vertices[face[1]].z; + triangle.ptc[2].z = mesh.vertices[face[2]].z; Image8U::RasterizeTriangleBary( projs[face[0]], projs[face[1]], - projs[face[2]], rasterer); + projs[face[2]], triangleRasterizer); } } return true; diff --git a/libs/MVS/Mesh.cpp b/libs/MVS/Mesh.cpp index 65eda30d6..1d7419014 100644 --- a/libs/MVS/Mesh.cpp +++ b/libs/MVS/Mesh.cpp @@ -4078,9 +4078,11 @@ void Mesh::Project(const Camera& camera, DepthMap& depthMap) const : Base(_vertices, _camera, _depthMap) {} }; RasterMesh rasterer(vertices, camera, depthMap); + RasterMesh::Triangle triangle; + RasterMesh::TriangleRasterizer triangleRasterizer(triangle, rasterer); rasterer.Clear(); for (const Face& facet: faces) - rasterer.Project(facet); + rasterer.Project(facet, triangleRasterizer); } void Mesh::Project(const Camera& camera, DepthMap& depthMap, Image8U3& image) const { @@ -4097,9 +4099,9 @@ void Mesh::Project(const Camera& camera, DepthMap& depthMap, Image8U3& image) co Base::Clear(); image.memset(0); } - void Raster(const ImageRef& pt, const Point3f& bary) { - const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(bary)); - const Depth z(ComputeDepth(pbary)); + void Raster(const ImageRef& pt, const Triangle& t, const Point3f& bary) { + const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(t, bary)); + const Depth z(ComputeDepth(t, pbary)); ASSERT(z > Depth(0)); Depth& depth = depthMap(pt); if (depth == 0 || depth > z) { @@ -4115,11 +4117,13 @@ void Mesh::Project(const Camera& camera, DepthMap& depthMap, Image8U3& image) co if (image.size() != depthMap.size()) image.create(depthMap.size()); RasterMesh rasterer(*this, camera, depthMap, image); + RasterMesh::Triangle triangle; + RasterMesh::TriangleRasterizer triangleRasterizer(triangle, rasterer); rasterer.Clear(); FOREACH(idxFace, faces) { const Face& facet = faces[idxFace]; rasterer.idxFaceTex = idxFace*3; - rasterer.Project(facet); + rasterer.Project(facet, triangleRasterizer); } } // project mesh to the given camera plane, computing also the normal-map (in camera space) @@ -4138,13 +4142,13 @@ void Mesh::Project(const Camera& camera, DepthMap& depthMap, NormalMap& normalMa Base::Clear(); normalMap.memset(0); } - inline void Project(const Face& facet) { + inline void Project(const Face& facet, TriangleRasterizer& tr) { idxVerts = facet.ptr(); - Base::Project(facet); + Base::Project(facet, tr); } - void Raster(const ImageRef& pt, const Point3f& bary) { - const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(bary)); - const Depth z(ComputeDepth(pbary)); + void Raster(const ImageRef& pt, const Triangle& t, const Point3f& bary) { + const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(t, bary)); + const Depth z(ComputeDepth(t, pbary)); ASSERT(z > Depth(0)); Depth& depth = depthMap(pt); if (depth == Depth(0) || depth > z) { @@ -4160,10 +4164,12 @@ void Mesh::Project(const Camera& camera, DepthMap& depthMap, NormalMap& normalMa if (normalMap.size() != depthMap.size()) normalMap.create(depthMap.size()); RasterMesh rasterer(*this, camera, depthMap, normalMap); + RasterMesh::Triangle triangle; + RasterMesh::TriangleRasterizer triangleRasterizer(triangle, rasterer); rasterer.Clear(); // render the entire mesh for (const Face& facet: faces) - rasterer.Project(facet); + rasterer.Project(facet, triangleRasterizer); } // project mesh to the given camera plane using orthographic projection void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap) const @@ -4172,12 +4178,12 @@ void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap) const typedef TRasterMesh Base; RasterMesh(const VertexArr& _vertices, const Camera& _camera, DepthMap& _depthMap) : Base(_vertices, _camera, _depthMap) {} - inline bool ProjectVertex(const Mesh::Vertex& pt, int v) { - return (ptc[v] = camera.TransformPointW2C(Cast(pt))).z > 0 && - depthMap.isInsideWithBorder(pti[v] = camera.TransformPointOrthoC2I(ptc[v])); + inline bool ProjectVertex(const Mesh::Vertex& pt, int v, Triangle& t) { + return (t.ptc[v] = camera.TransformPointW2C(Cast(pt))).z > 0 && + depthMap.isInsideWithBorder(t.pti[v] = camera.TransformPointOrthoC2I(t.ptc[v])); } - void Raster(const ImageRef& pt, const Point3f& bary) { - const Depth z(ComputeDepth(bary)); + void Raster(const ImageRef& pt, const Triangle& t, const Point3f& bary) { + const Depth z(ComputeDepth(t, bary)); ASSERT(z > Depth(0)); Depth& depth = depthMap(pt); if (depth == 0 || depth > z) @@ -4185,9 +4191,11 @@ void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap) const } }; RasterMesh rasterer(vertices, camera, depthMap); + RasterMesh::Triangle triangle; + RasterMesh::TriangleRasterizer triangleRasterizer(triangle, rasterer); rasterer.Clear(); for (const Face& facet: faces) - rasterer.Project(facet); + rasterer.Project(facet, triangleRasterizer); } void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap, Image8U3& image) const { @@ -4204,12 +4212,12 @@ void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap, Image8U3& imag Base::Clear(); image.memset(0); } - inline bool ProjectVertex(const Mesh::Vertex& pt, int v) { - return (ptc[v] = camera.TransformPointW2C(Cast(pt))).z > 0 && - depthMap.isInsideWithBorder(pti[v] = camera.TransformPointOrthoC2I(ptc[v])); + inline bool ProjectVertex(const Mesh::Vertex& pt, int v, Triangle& t) { + return (t.ptc[v] = camera.TransformPointW2C(Cast(pt))).z > 0 && + depthMap.isInsideWithBorder(t.pti[v] = camera.TransformPointOrthoC2I(t.ptc[v])); } - void Raster(const ImageRef& pt, const Point3f& bary) { - const Depth z(ComputeDepth(bary)); + void Raster(const ImageRef& pt, const Triangle& t, const Point3f& bary) { + const Depth z(ComputeDepth(t, bary)); ASSERT(z > Depth(0)); Depth& depth = depthMap(pt); if (depth == 0 || depth > z) { @@ -4225,11 +4233,13 @@ void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap, Image8U3& imag if (image.size() != depthMap.size()) image.create(depthMap.size()); RasterMesh rasterer(*this, camera, depthMap, image); + RasterMesh::Triangle triangle; + RasterMesh::TriangleRasterizer triangleRasterizer(triangle, rasterer); rasterer.Clear(); FOREACH(idxFace, faces) { const Face& facet = faces[idxFace]; rasterer.idxFaceTex = idxFace*3; - rasterer.Project(facet); + rasterer.Project(facet, triangleRasterizer); } } // assuming the mesh is properly oriented, ortho-project it to a camera looking from top to down @@ -4717,3 +4727,87 @@ bool Mesh::InitKernels(int device) } /*----------------------------------------------------------------*/ #endif + + +#ifdef _USE_OPENMP +// test mesh projection on the image using multi-threaded and single-threaded rasterization +bool MVS::TestMeshProjectionMT(const Mesh& mesh, const Image& image) { + // used to render the mesh + typedef TImage FaceMap; + struct RasterMesh : TRasterMesh { + typedef TRasterMesh Base; + FaceMap& faceMap; + RasterMesh(const Mesh::VertexArr& _vertices, const Camera& _camera, DepthMap& _depthMap, FaceMap& _faceMap) + : Base(_vertices, _camera, _depthMap), faceMap(_faceMap) {} + void Clear() { + Base::Clear(); + faceMap.memset((uint8_t)NO_ID); + } + void Raster(const ImageRef& pt, const Triangle& t, const Point3f& bary, Mesh::FIndex idxFace) { + const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(t, bary)); + const Depth z(ComputeDepth(t, pbary)); + ASSERT(z > Depth(0)); + Depth& depth = depthMap(pt); + if (depth == 0 || depth > z) { + depth = z; + faceMap(pt) = idxFace; + } + } + }; + struct TriangleRasterizer { + RasterMesh* rasterizer; + RasterMesh::Triangle triangle; + Mesh::FIndex idxFace; + inline cv::Size Size() const { + return rasterizer->Size(); + } + inline void operator()(const ImageRef& pt, const Point3f& bary) const { + rasterizer->Raster(pt, triangle, bary, idxFace); + } + }; + // project mesh on the image + DepthMap depthMapMT(image.GetSize()); + FaceMap faceMapMT(image.GetSize()); + { // multi-threaded rasterization + RasterMesh rasterer(mesh.vertices, image.camera, depthMapMT, faceMapMT); + TriangleRasterizer triangleRasterizer{&rasterer}; + rasterer.Clear(); + #pragma omp parallel for firstprivate(triangleRasterizer) schedule(dynamic) + for (int_t i=0; i<(int_t)mesh.faces.size(); ++i) { + const Mesh::FIndex idxFace = (Mesh::FIndex)i; + const Mesh::Face& facet = mesh.faces[idxFace]; + triangleRasterizer.idxFace = idxFace; + rasterer.Project(facet, triangleRasterizer); + } + } + DepthMap depthMapST(image.GetSize()); + FaceMap faceMapST(image.GetSize()); + { // single-threaded rasterization + RasterMesh rasterer(mesh.vertices, image.camera, depthMapST, faceMapST); + TriangleRasterizer triangleRasterizer{&rasterer}; + rasterer.Clear(); + FOREACH(idxFace, mesh.faces) { + const Mesh::Face& facet = mesh.faces[idxFace]; + triangleRasterizer.idxFace = idxFace; + rasterer.Project(facet, triangleRasterizer); + } + } + // compare results + unsigned numDiffDepths(0), numDiffFaces(0); + for (int y = 0; y struct TRasterMeshBase { - const Camera& camera; + typedef DERIVED Rasterizer; + struct Triangle { + Point3 ptc[3]; + Point2f pti[3]; + }; + + const Camera& camera; DepthMap& depthMap; - Point3 ptc[3]; - Point2f pti[3]; - TRasterMeshBase(const Camera& _camera, DepthMap& _depthMap) : camera(_camera), depthMap(_depthMap) {} @@ -304,27 +307,39 @@ struct TRasterMeshBase { return depthMap.size(); } - inline bool ProjectVertex(const Point3f& pt, int v) { - return (ptc[v] = camera.TransformPointW2C(Cast(pt))).z > 0 && - depthMap.isInsideWithBorder(pti[v] = camera.TransformPointC2I(ptc[v])); + inline bool ProjectVertex(const Point3f& pt, int v, Triangle& t) { + return (t.ptc[v] = camera.TransformPointW2C(Cast(pt))).z > 0 && + depthMap.isInsideWithBorder(t.pti[v] = camera.TransformPointC2I(t.ptc[v])); } - inline Point3f PerspectiveCorrectBarycentricCoordinates(const Point3f& bary) { - return SEACAVE::PerspectiveCorrectBarycentricCoordinates(bary, (float)ptc[0].z, (float)ptc[1].z, (float)ptc[2].z); + inline Point3f PerspectiveCorrectBarycentricCoordinates(const Triangle& t, const Point3f& bary) { + return SEACAVE::PerspectiveCorrectBarycentricCoordinates(bary, (float)t.ptc[0].z, (float)t.ptc[1].z, (float)t.ptc[2].z); } - inline float ComputeDepth(const Point3f& pbary) { - return pbary[0]*(float)ptc[0].z + pbary[1]*(float)ptc[1].z + pbary[2]*(float)ptc[2].z; + inline float ComputeDepth(const Triangle& t, const Point3f& pbary) { + return pbary[0]*(float)t.ptc[0].z + pbary[1]*(float)t.ptc[1].z + pbary[2]*(float)t.ptc[2].z; } - void Raster(const ImageRef& pt, const Point3f& bary) { - const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(bary)); - const Depth z(ComputeDepth(pbary)); + void Raster(const ImageRef& pt, const Triangle& t, const Point3f& bary) { + const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(t, bary)); + const Depth z(ComputeDepth(t, pbary)); ASSERT(z > Depth(0)); Depth& depth = depthMap(pt); if (depth == 0 || depth > z) depth = z; } - inline void operator()(const ImageRef& pt, const Point3f& bary) { - static_cast(this)->Raster(pt, bary); + + struct TriangleRasterizer { + Triangle& triangle; + Rasterizer& rasterizer; + TriangleRasterizer(Triangle& t, Rasterizer& r) : triangle(t), rasterizer(r) {} + inline cv::Size Size() const { + return rasterizer.Size(); + } + inline void operator()(const ImageRef& pt, const Point3f& bary) const { + rasterizer.Raster(pt, triangle, bary); + } + }; + inline TriangleRasterizer CreateTriangleRasterizer(Triangle& triangle) { + return TriangleRasterizer(triangle, *static_cast(this)); } }; @@ -332,29 +347,34 @@ struct TRasterMeshBase { template struct TRasterMesh : TRasterMeshBase { typedef TRasterMeshBase Base; + using typename Base::Triangle; using Base::camera; using Base::depthMap; - using Base::ptc; - using Base::pti; - const Mesh::VertexArr& vertices; TRasterMesh(const Mesh::VertexArr& _vertices, const Camera& _camera, DepthMap& _depthMap) : Base(_camera, _depthMap), vertices(_vertices) {} - void Project(const Mesh::Face& facet) { + template + void Project(const Mesh::Face& facet, TriangleRasterizer& tr) { // project face vertices to image plane for (int v=0; v<3; ++v) { // skip face if not completely inside - if (!static_cast(this)->ProjectVertex(vertices[facet[v]], v)) + if (!static_cast(this)->ProjectVertex(vertices[facet[v]], v, tr.triangle)) return; } // draw triangle - Image8U3::RasterizeTriangleBary(pti[0], pti[1], pti[2], *this); + Image8U3::RasterizeTriangleBary(tr.triangle.pti[0], tr.triangle.pti[1], tr.triangle.pti[2], tr); + } + void Project(const Mesh::Face& facet) { + Triangle triangle; + Project(facet, this->CreateTriangleRasterizer(triangle)); } }; + +bool TestMeshProjectionMT(const Mesh& mesh, const Image& image); /*----------------------------------------------------------------*/ diff --git a/libs/MVS/SceneRefine.cpp b/libs/MVS/SceneRefine.cpp index c7ae91790..ab6ad8cba 100644 --- a/libs/MVS/SceneRefine.cpp +++ b/libs/MVS/SceneRefine.cpp @@ -111,9 +111,9 @@ class MeshRefine { faceMap.memset((uint8_t)NO_ID); baryMap.memset(0); } - void Raster(const ImageRef& pt, const Point3f& bary) { - const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(bary)); - const Depth z(ComputeDepth(pbary)); + void Raster(const ImageRef& pt, const Triangle& t, const Point3f& bary) { + const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(t, bary)); + const Depth z(ComputeDepth(t, pbary)); ASSERT(z > Depth(0)); Depth& depth = depthMap(pt); if (depth == 0 || depth > z) { @@ -734,11 +734,13 @@ void MeshRefine::ProjectMesh( baryMap.create(size); // project all triangles on this image and keep the closest ones RasterMesh rasterer(vertices, camera, depthMap, faceMap, baryMap); + RasterMesh::Triangle triangle; + RasterMesh::TriangleRasterizer triangleRasterizer(triangle, rasterer); rasterer.Clear(); for (auto idxFace : cameraFaces) { const Face& facet = faces[idxFace]; rasterer.idxFace = idxFace; - rasterer.Project(facet); + rasterer.Project(facet, triangleRasterizer); } } diff --git a/libs/MVS/SceneTexture.cpp b/libs/MVS/SceneTexture.cpp index 4b3c63e04..22e5afaa7 100644 --- a/libs/MVS/SceneTexture.cpp +++ b/libs/MVS/SceneTexture.cpp @@ -112,9 +112,9 @@ struct MeshTexture { Base::Clear(); faceMap.memset((uint8_t)NO_ID); } - void Raster(const ImageRef& pt, const Point3f& bary) { - const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(bary)); - const Depth z(ComputeDepth(pbary)); + void Raster(const ImageRef& pt, const Triangle& t, const Point3f& bary) { + const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(t, bary)); + const Depth z(ComputeDepth(t, pbary)); ASSERT(z > Depth(0)); Depth& depth = depthMap(pt); if (depth == 0 || depth > z) { @@ -496,6 +496,8 @@ bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThr faceMap.create(imageData.GetSize()); depthMap.create(imageData.GetSize()); RasterMesh rasterer(vertices, imageData.camera, depthMap, faceMap); + RasterMesh::Triangle triangle; + RasterMesh::TriangleRasterizer triangleRasterizer(triangle, rasterer); if (nIgnoreMaskLabel >= 0) { // import mask BitMatrix bmask; @@ -513,9 +515,9 @@ bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThr rasterer.validFace = true; const Face& facet = faces[idxFace]; rasterer.idxFace = idxFace; - rasterer.Project(facet); + rasterer.Project(facet, triangleRasterizer); if (!rasterer.validFace) - rasterer.Project(facet); + rasterer.Project(facet, triangleRasterizer); } // compute the projection area of visible faces #if TEXOPT_FACEOUTLIER != TEXOPT_FACEOUTLIER_NA