Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Viewer color source #1195

Closed
wants to merge 6 commits into from
Closed

Viewer color source #1195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
384da63
mesh: make TRasterMesh multithread-safe
cdcseacave Nov 8, 2024
3424afd
mesh: add multi-threaded mesh projection test
cdcseacave Nov 9, 2024
f320a01
Merge remote-tracking branch 'origin/develop' into develop_justin
cdcseacave Nov 23, 2024
f69109c
Merge remote-tracking branch 'base/develop' into develop_justin
JuSiGabriel Jan 20, 2025
ccd4a30
Add visualization of various confidence maps in color of the point cl…
JuSiGabriel Jan 21, 2025
edb3e89
Refactor and clean
JuSiGabriel Jan 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 32 additions & 5 deletions apps/Viewer/Scene.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -419,8 +419,7 @@ bool Scene::Export(LPCTSTR _fileName, LPCTSTR exportType) const
return bPoints || bMesh;
}

void Scene::CompilePointCloud()
{
void Scene::CompilePointCloud(Window::COLORSOURCE colorSource) {
if (scene.pointcloud.IsEmpty())
return;
ReleasePointCloud();
Expand All @@ -431,14 +430,42 @@ void Scene::CompilePointCloud()
if ((window.sparseType&Window::SPR_POINTS) != 0) {
ASSERT_ARE_SAME_TYPE(float, MVS::PointCloud::Point::Type);
glBegin(GL_POINTS);
glColor3f(1.f,1.f,1.f);
glColor3f(1.f, 1.f, 1.f);
MVS::DepthData depthData;
MVS::DepthMap& depthMap = depthData.depthMap;
MVS::ConfidenceMap confMap;
if (colorSource == Window::COLORSOURCE::COL_DEPTH || colorSource == Window::COLORSOURCE::COL_COMPOSITE || colorSource == Window::COLORSOURCE::COL_NORMAL) {
depthData.Load(sceneName, colorSource == Window::COLORSOURCE::COL_NORMAL ? 3 : 1);
colorSource == Window::COLORSOURCE::COL_NORMAL ? MVS::EstimateConfidenceFromNormal(depthData, confMap, 1) : MVS::EstimateConfidenceFromDepth(depthData, confMap, 1, 3);
}
int cmpt = 0;
FOREACH(i, scene.pointcloud.points) {
if (!scene.pointcloud.pointViews.empty() &&
scene.pointcloud.pointViews[i].size() < window.minViews)
continue;
if (!scene.pointcloud.colors.empty()) {
if (!scene.pointcloud.colors.empty() && colorSource == Window::COLORSOURCE::COL_IMAGE) {
const MVS::PointCloud::Color& c = scene.pointcloud.colors[i];
glColor3ub(c.r,c.g,c.b);
glColor3ub(c.r, c.g, c.b);
}
if (colorSource == Window::COLORSOURCE::COL_DEPTH || colorSource == Window::COLORSOURCE::COL_COMPOSITE || colorSource == Window::COLORSOURCE::COL_NORMAL) {
int j = cmpt/depthMap.cols;
int k = cmpt%depthMap.cols;
while (depthMap(j, k) <= 0) {
cmpt++;
j = cmpt/depthMap.cols;
k = cmpt%depthMap.cols;
}
float confidence = colorSource == Window::COLORSOURCE::COL_COMPOSITE ? 0.7*confMap(j, k) + 0.3*scene.pointcloud.pointWeights[i][0] : confMap(j, k);
Pixel8U c = Pixel8U::gray2color(confidence);
glColor3ub(c.r, c.g, c.b);
cmpt++;
}
if (colorSource == Window::COLORSOURCE::COL_CONFIDENCE) {
if (!scene.pointcloud.pointWeights.empty()) {
float confidence = scene.pointcloud.pointWeights[i][0];
Pixel8U c = Pixel8U::gray2color(confidence);
glColor3ub(c.r, c.g, c.b);
}
}
const MVS::PointCloud::Point& X = scene.pointcloud.points[i];
glVertex3fv(X.ptr());
Expand Down
2 changes: 1 addition & 1 deletion apps/Viewer/Scene.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ class Scene
bool Open(LPCTSTR fileName, LPCTSTR geometryFileName=NULL);
bool Save(LPCTSTR fileName=NULL, bool bRescaleImages=false);
bool Export(LPCTSTR fileName, LPCTSTR exportType=NULL) const;
void CompilePointCloud();
void CompilePointCloud(Window::COLORSOURCE colorSource = Window::COLORSOURCE::COL_IMAGE);
void CompileMesh();
void CompileBounds();
void CropToBounds();
Expand Down
52 changes: 48 additions & 4 deletions apps/Viewer/Window.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,9 @@ void Window::Reset(SPARSE _sparseType, unsigned _minViews)
bRenderBounds = false;
selectionType = SEL_NA;
selectionIdx = NO_IDX;
colorSource = COL_IMAGE;
if (clbkCompilePointCloud != NULL)
clbkCompilePointCloud();
clbkCompilePointCloud(colorSource);
if (clbkCompileMesh != NULL)
clbkCompileMesh();
glfwPostEmptyEvent();
Expand Down Expand Up @@ -236,7 +237,7 @@ void Window::Key(int k, int /*scancode*/, int action, int mod)
if (minViews > 2) {
minViews--;
if (clbkCompilePointCloud != NULL)
clbkCompilePointCloud();
clbkCompilePointCloud(colorSource);
}
} else {
pointSize = MAXF(pointSize-0.5f, 0.5f);
Expand All @@ -248,7 +249,7 @@ void Window::Key(int k, int /*scancode*/, int action, int mod)
if (mod & GLFW_MOD_SHIFT) {
minViews++;
if (clbkCompilePointCloud != NULL)
clbkCompilePointCloud();
clbkCompilePointCloud(colorSource);
} else {
pointSize += 0.5f;
}
Expand Down Expand Up @@ -307,7 +308,7 @@ void Window::Key(int k, int /*scancode*/, int action, int mod)
case SPR_ALL: sparseType = SPR_POINTS; break;
}
if (clbkCompilePointCloud != NULL)
clbkCompilePointCloud();
clbkCompilePointCloud(colorSource);
break;
case GLFW_KEY_R:
if (action == GLFW_RELEASE)
Expand Down Expand Up @@ -366,7 +367,50 @@ void Window::Key(int k, int /*scancode*/, int action, int mod)
cameraBlend = MINF(cameraBlend+0.1f, 1.f);
}
break;
case GLFW_KEY_F1:
if (action == GLFW_RELEASE) {
colorSource = COL_NONE;
if (clbkCompilePointCloud != NULL)
clbkCompilePointCloud(colorSource);
}
break;
case GLFW_KEY_F2:
if (action == GLFW_RELEASE) {
colorSource = COL_IMAGE;
if (clbkCompilePointCloud != NULL)
clbkCompilePointCloud(colorSource);
}
break;
case GLFW_KEY_F3:
if (action == GLFW_RELEASE) {
colorSource = COL_CONFIDENCE;
if (clbkCompilePointCloud != NULL)
clbkCompilePointCloud(colorSource);
}
break;
case GLFW_KEY_F4:
if (action == GLFW_RELEASE) {
colorSource = COL_DEPTH;
if (clbkCompilePointCloud != NULL)
clbkCompilePointCloud(colorSource);
}
break;
case GLFW_KEY_F5:
if (action == GLFW_RELEASE) {
colorSource = COL_COMPOSITE;
if (clbkCompilePointCloud != NULL)
clbkCompilePointCloud(colorSource);
}
break;
case GLFW_KEY_F6:
if (action == GLFW_RELEASE) {
colorSource = COL_NORMAL;
if (clbkCompilePointCloud != NULL)
clbkCompilePointCloud(colorSource);
}
break;
}

}
void Window::Key(GLFWwindow* window, int k, int scancode, int action, int mod)
{
Expand Down
12 changes: 11 additions & 1 deletion apps/Viewer/Window.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,16 @@ class Window
bool bRenderTexture;
bool bRenderBounds;

enum COLORSOURCE {
COL_NONE = 0,
COL_IMAGE,
COL_DEPTH,
COL_CONFIDENCE,
COL_COMPOSITE,
COL_NORMAL
};
COLORSOURCE colorSource;

enum SELECTION {
SEL_NA = 0,
SEL_POINT,
Expand All @@ -102,7 +112,7 @@ class Window
ClbkCenterScene clbkCenterScene;
typedef DELEGATE<void (const Ray3&, int)> ClbkRayScene;
ClbkRayScene clbkRayScene;
typedef DELEGATE<void (void)> ClbkCompilePointCloud;
typedef DELEGATE<void (COLORSOURCE)> ClbkCompilePointCloud;
ClbkCompilePointCloud clbkCompilePointCloud;
typedef DELEGATE<void (void)> ClbkCompileMesh;
ClbkCompileMesh clbkCompileMesh;
Expand Down
72 changes: 72 additions & 0 deletions libs/MVS/DepthMap.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ using namespace MVS;


// D E F I N E S ///////////////////////////////////////////////////
// uncomment to enable multi-threading based on OpenMP
#ifdef _USE_OPENMP
#define DEPTHMAP_USE_OPENMP
#endif

#define DEFVAR_OPTDENSE_string(name, title, desc, ...) DEFVAR_string(OPTDENSE, name, title, desc, __VA_ARGS__)
#define DEFVAR_OPTDENSE_bool(name, title, desc, ...) DEFVAR_bool(OPTDENSE, name, title, desc, __VA_ARGS__)
Expand Down Expand Up @@ -1568,6 +1572,74 @@ bool MVS::EstimateNormalMap(const Matrix3x3f& K, const DepthMap& depthMap, Norma
} // EstimateNormalMap
/*----------------------------------------------------------------*/

// estimate confidence map from depth-map variation in a window;
// the estimated confidence is the mean of the depth differences to the 3 closer pixel to the central pixel
void MVS::EstimateConfidenceFromDepth(const DepthData& depthData, ConfidenceMap& confMap, const int winHalfSize, const int n) {
ASSERT(depthData.dMax > depthData.dMin);
const float dDepth = depthData.dMax - depthData.dMin;
const DepthMap& depthMap = depthData.depthMap;
confMap.create(depthMap.size());
confMap.memset(0);
#ifdef DEPTHMAP_USE_OPENMP
#pragma omp parallel for
#endif
for (int r = 0; r<depthMap.rows; r++)
for (int c = 0; c<depthMap.cols; c++) {
if (depthMap(r, c) <= 0)
continue;
DoubleArr depthDiffValues;
for (int k = -winHalfSize; k<=winHalfSize; k++)
for (int l = -winHalfSize; l<=winHalfSize; l++) {
if (r+k >= 0 && r+k < depthMap.rows && c+l >= 0 && c+l < depthMap.cols && depthMap(r+k, c+l) > 0 && !(k == 0 && l == 0)) {
depthDiffValues.push_back(ABS(depthMap(r, c) - depthMap(r+k, c+l)));
}
}
depthDiffValues.Sort();
const int s = MIN(n, (int)depthDiffValues.size());
float confidence = 0;
for (int k = 0; k < s; k++)
confidence += depthDiffValues[k]/depthDiffValues.GetSize();
confMap(r, c) = EXP(-2000*confidence/dDepth);
}
} // EstimateConfidenceFromDepth
/*----------------------------------------------------------------*/

// estimate confidence map from normal variance in a window
void MVS::EstimateConfidenceFromNormal(const DepthData& depthData, ConfidenceMap& confMap, const int winHalfSize) {
const NormalMap& normalMap = depthData.normalMap;
confMap.create(normalMap.size());
confMap.memset(0);
#ifdef DEPTHMAP_USE_OPENMP
#pragma omp parallel for
#endif
for (int r = 0; r < normalMap.rows; r++) {
for (int c = 0; c< normalMap.cols; c++) {
if (depthData.depthMap(r, c) <= 0)
continue;
Point3f mean(0, 0, 0);

int count = 0;
for (int k = -winHalfSize; k<=winHalfSize; k++) {
for (int l = -winHalfSize; l<=winHalfSize; l++) {
if (r+k >= 0 && r+k < normalMap.rows && c+l>=0 && c+l < normalMap.cols && depthData.depthMap(r+k, c+l) > 0) {
mean += normalMap(r+k, c+l);
count++;
}
}
}
mean /= count;
float theta = 0;
for (int k = -winHalfSize; k<=winHalfSize; k++) {
for (int l = -winHalfSize; l<=winHalfSize; l++) {
if (r+k >= 0 && r+k < normalMap.rows && c+l >= 0 && c+l < normalMap.cols && depthData.depthMap(r+k, c+l) > 0)
theta += SQUARE(ACOS(mean.dot(normalMap(r+k, c+l))));
}
}
confMap(r, c) = SQUARE(1 - theta/(count*ACOS(-1)));
}
}
} // EstimateConfidenceFromNormal
/*----------------------------------------------------------------*/

// save the depth map in our .dmap file format
bool MVS::SaveDepthMap(const String& fileName, const DepthMap& depthMap)
Expand Down
2 changes: 2 additions & 0 deletions libs/MVS/DepthMap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,8 @@ MVS_API void EstimatePointColors(const ImageArr& images, PointCloud& pointcloud)
MVS_API void EstimatePointNormals(const ImageArr& images, PointCloud& pointcloud, int numNeighbors=16/*K-nearest neighbors*/);

MVS_API bool EstimateNormalMap(const Matrix3x3f& K, const DepthMap&, NormalMap&);
MVS_API void EstimateConfidenceFromDepth(const DepthData& depthData, ConfidenceMap& confMap, const int winHalfSize, const int n);
MVS_API void EstimateConfidenceFromNormal(const DepthData& depthData, ConfidenceMap& confMap, const int winHalfSize);

MVS_API bool SaveDepthMap(const String& fileName, const DepthMap& depthMap);
MVS_API bool LoadDepthMap(const String& fileName, DepthMap& depthMap);
Expand Down
Loading