Added "usePrec" option as v1.3

t-taniai · Apr 25, 2018 · e7b9e3d · e7b9e3d
1 parent 6a9eaa6
commit e7b9e3d
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 3 deletions.
diff --git a/EvalToolVC/EvalTool/main.cpp b/EvalToolVC/EvalTool/main.cpp
@@ -12,6 +12,7 @@ using namespace cv;
 cv::Scalar BGCOLOR = cv::Scalar(255, 255, 0);
 cv::Scalar FLBGCOLOR = cv::Scalar(128, 128, 128);
 bool autoFlip = false;
+bool usePrec = false;
 
 void load_data(string dir, cv::Mat& flow1, cv::Mat& flow2, cv::Mat& mask1, cv::Mat& mask2, string& image1 = string(), string& image2 = string())
 {
@@ -51,7 +52,12 @@ cv::Mat_<double> compute_score(cv::Mat maskGT1, cv::Mat flowGT1, cv::Mat mask1,
 
 	if (!mask1.empty())
 	{
-		s.at<double>(0) = (double)cv::countNonZero(maskGT1 & mask1) / cv::countNonZero(maskGT1 | mask1);
+		if (usePrec)
+			// Precision (accurate pixel rate)
+			s.at<double>(0) = 1.0 - (double)cv::countNonZero(maskGT1 ^ mask1) / maskGT1.size().area();
+		else
+			// Intersection-over-union
+			s.at<double>(0) = (double)cv::countNonZero(maskGT1 & mask1) / cv::countNonZero(maskGT1 | mask1);
 	}
 
 	if (!flow1.empty())
@@ -202,7 +208,8 @@ void run_evaluation(string resultDir, string datasetDir)
 		return;
 	}
 
-	fprintf(scoreTable, "%s,%s,%s,%s,%s", "Row", "Src", "Ref", "SegIUR", "Flip");
+	const char* smetric = usePrec ? "SegPrec" : "SegIUR";
+	fprintf(scoreTable, "%s,%s,%s,%s,%s", "Row", "Src", "Ref", smetric, "Flip");
 
 	cv::Mat_<double> thresholds(THRESHOLD, 1);
 	for (int i = 0; i < THRESHOLD; i++) {
@@ -291,7 +298,7 @@ void run_evaluation(string resultDir, string datasetDir)
 	fclose(scoreTable);
 
 	printf("------------- Score Summary ----------------------\n");
-	printf("%8s %8s %8s %8s %8s %8s\n", "IUR", "FA1", "FA2", "FA3", "FA4", "FA5");
+	printf("%8s %8s %8s %8s %8s %8s\n", smetric, "FA1", "FA2", "FA3", "FA4", "FA5");
 	printf("%8.3lf %8.3lf %8.3lf %8.3lf %8.3lf %8.3lf\n", score.at<double>(0), score.at<double>(1), score.at<double>(2), score.at<double>(3), score.at<double>(4), score.at<double>(5));
 
 }
@@ -317,7 +324,9 @@ int main(int argn, char** args)
 	std::string mode = "evaluation";
 	argParser.TryGetArgment("mode", mode);
 	argParser.TryGetArgment("autoFlip", autoFlip); // Use only when cosegmentation methods are not aware which of 0/1 is the foreground label.
+	argParser.TryGetArgment("usePrec", usePrec);
 	std::cout << "Auto flip segmentation mask  : " << (autoFlip ? "on" : "off") << " (Use only when foreground label is not consistent. Enabled by -autoFlip 1)" << std::endl;
+	std::cout << "Evaluate by precision        : " << (usePrec ? "on" : "off") << " (Use precision instead of IUR for segmentation. Enabled by -usePrec 1)" << std::endl;
 
 
 	if (mode == "evaluation")

diff --git a/EvalToolVC/Readme.txt b/EvalToolVC/Readme.txt
@@ -7,6 +7,9 @@ RunVisualization.bat is the main script for visualization.
 By double clicking the bat files, evaluation/visualization will be done on sample results in /TSS_CVPR2016_EvaluationKit/Results.
 By drag-and-dropping the /Results folder (or any folder that has the same structure), you can also evaluate/visualize data in that folder.
 
+RunEvaluationPrec.bat demonstrates how to use the usePrec feature for evaluating segmentation accuracy by precision.
+RunEvaluationAutoFlip.bat demonstrates how to use the autoFlip feature for automatically flipping segmentation labels.
+
 
 ---------
 Requirements for re-compiling:

diff --git a/EvalToolVC/RunEvaluationAutoFlip.bat b/EvalToolVC/RunEvaluationAutoFlip.bat
@@ -0,0 +1,17 @@
+@echo off
+
+set evaltool="%~dp0x64\Release\EvalTool.exe"
+set datasetroot=%~dp0..\Dataset
+
+if [%1]==[] (
+set resultsroot=%~dp0..\Results
+) else (
+set resultsroot=%~1
+)
+
+%evaltool% -resultsDir "%resultsroot%\FG3DCar" -datasetDir "%datasetroot%\FG3DCar" -mode evaluation -autoFlip 1
+%evaltool% -resultsDir "%resultsroot%\JODS" -datasetDir "%datasetroot%\JODS" -mode evaluation -autoFlip 1
+%evaltool% -resultsDir "%resultsroot%\PASCAL" -datasetDir "%datasetroot%\PASCAL" -mode evaluation -autoFlip 1
+
+echo Done.
+pause;
diff --git a/EvalToolVC/RunEvaluationPrec.bat b/EvalToolVC/RunEvaluationPrec.bat
@@ -0,0 +1,17 @@
+@echo off
+
+set evaltool="%~dp0x64\Release\EvalTool.exe"
+set datasetroot=%~dp0..\Dataset
+
+if [%1]==[] (
+set resultsroot=%~dp0..\Results
+) else (
+set resultsroot=%~1
+)
+
+%evaltool% -resultsDir "%resultsroot%\FG3DCar" -datasetDir "%datasetroot%\FG3DCar" -mode evaluation -usePrec 1
+%evaltool% -resultsDir "%resultsroot%\JODS" -datasetDir "%datasetroot%\JODS" -mode evaluation -usePrec 1
+%evaltool% -resultsDir "%resultsroot%\PASCAL" -datasetDir "%datasetroot%\PASCAL" -mode evaluation -usePrec 1
+
+echo Done.
+pause;
diff --git a/Readme.txt b/Readme.txt
@@ -52,6 +52,8 @@ Segmentation accuracy:
 ----------------------
 
 Given the estimated foreground mask and the ground truth foreground masks for each image, the segmentation accuracy is computed in terms of the intersection-over-union ratio metric.
+When a method does not distinguish foreground and background, then use the "autoFlip" feature, which automatically flips labels so as to maximize the scores.
+VC version only: When the "usePrec" commandline option is enabled, then the accuracy is evaluated by precision (ratio of accurate-over-all pixels).
 
 
 --------------
@@ -79,3 +81,4 @@ History:
 06/10/2016 v1.0 Released the evaluation kit
 07/28/2016 v1.1 Added "autoFlip" option
 08/26/2016 v1.2 Added more example results and fixed bat files of EvalToolVC. Released windows binaries using Github releases.
+04/24/2018 v1.3 Added "usePrec" option for VC version