Skip to content

Commit

Permalink
Specify separate validation threshold for exactly matching groundtrut…
Browse files Browse the repository at this point in the history 
…h in 2D QR code detector test

Summary: 2D QR code detector test is failing frequently.  Separating success rate thresholds for successful detection and successfully matching groundtruth data exactly to yield more fine-grained control over test success condition.  Individual module values detected can differ from groundtruth value and still result in successful QR code detection due to error correction.

Reviewed By: enpe

Differential Revision: D64157351

fbshipit-source-id: ff068c3667409c4d18da1c227346c3851b71aeee
  • Loading branch information
@leapally @facebook-github-bot
leapally authored and facebook-github-bot committed Oct 10, 2024
1 parent 6ed0a89 commit c2568c9
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 19 deletions.
49 changes: 32 additions & 17 deletions impl/ocean/test/testcv/testdetector/testqrcodes/TestQRCodeDetector2D.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,26 +258,29 @@ bool TestQRCodeDetector2D::testDetectQRCodesSmallImageSyntheticData(const unsign
{
static_assert(std::is_same_v<Scalar, double> || std::is_same_v<Scalar, float>);

const double validationPrecisionThreshold = std::is_same_v<Scalar, double> ? 0.90 : 0.85;
const double detectionValidationThreshold = std::is_same_v<Scalar, double> ? 0.95 : 0.9;
const double groundtruthComparisonValidationThreshold = std::is_same_v<Scalar, double> ? 0.80 : 0.75;

return testDetectQRCodesSyntheticData_Internal(gaussianFilterSize, testDuration, worker, "SmallImage", validationPrecisionThreshold);
return testDetectQRCodesSyntheticData_Internal(gaussianFilterSize, testDuration, worker, "SmallImage", detectionValidationThreshold, groundtruthComparisonValidationThreshold);
}

bool TestQRCodeDetector2D::testDetectQRCodesLargeImageSyntheticData(const unsigned int gaussianFilterSize, const double testDuration, Worker& worker)
{
static_assert(std::is_same_v<Scalar, double> || std::is_same_v<Scalar, float>);

const double validationPrecisionThreshold = std::is_same_v<Scalar, double> ? 0.85 : 0.80;
const double detectionValidationThreshold = std::is_same_v<Scalar, double> ? 0.90 : 0.85;
const double groundtruthComparisonValidationThreshold = std::is_same_v<Scalar, double> ? 0.75 : 0.70;

return testDetectQRCodesSyntheticData_Internal(gaussianFilterSize, testDuration, worker, "LargeImage", validationPrecisionThreshold, 6u, 20u, 2048u, 4096u);
return testDetectQRCodesSyntheticData_Internal(gaussianFilterSize, testDuration, worker, "LargeImage", detectionValidationThreshold, groundtruthComparisonValidationThreshold, 6u, 20u, 2048u, 4096u);
}

bool TestQRCodeDetector2D::testDetectQRCodesSyntheticData_Internal(const unsigned int gaussianFilterSize, const double testDuration, Worker& worker, const std::string& testLabel, const double validationPrecisionThreshold, const unsigned int moduleSizePixelsMin, const unsigned int moduleSizePixelsMax, const unsigned int imageDimPixelsMin, const unsigned int imageDimPixelsMax)
bool TestQRCodeDetector2D::testDetectQRCodesSyntheticData_Internal(const unsigned int gaussianFilterSize, const double testDuration, Worker& worker, const std::string& testLabel, const double detectionValidationThreshold, const double groundtruthComparisonValidationThreshold, const unsigned int moduleSizePixelsMin, const unsigned int moduleSizePixelsMax, const unsigned int imageDimPixelsMin, const unsigned int imageDimPixelsMax)
{
ocean_assert(gaussianFilterSize == 0u || gaussianFilterSize % 2u == 1u);
ocean_assert(testDuration > 0.0);
ocean_assert(testLabel.end() == std::find_if(testLabel.begin(), testLabel.end(), [](char c) { return !std::isalnum(c); }) && "testLabel must be alphanumeric");
ocean_assert(validationPrecisionThreshold >= 0.0 && validationPrecisionThreshold <= 1.0);
ocean_assert(detectionValidationThreshold >= 0.0 && detectionValidationThreshold <= 1.0);
ocean_assert(groundtruthComparisonValidationThreshold >= 0.0 && groundtruthComparisonValidationThreshold <= detectionValidationThreshold);
ocean_assert(moduleSizePixelsMin >= 1u);
ocean_assert(moduleSizePixelsMax >= moduleSizePixelsMin);
ocean_assert(imageDimPixelsMin >= 0u);
Expand All @@ -287,7 +290,8 @@ bool TestQRCodeDetector2D::testDetectQRCodesSyntheticData_Internal(const unsigne

RandomGenerator randomGenerator;

ValidationPrecision validation(validationPrecisionThreshold, randomGenerator);
ValidationPrecision detectionValidation(detectionValidationThreshold, randomGenerator);
ValidationPrecision groundtruthComparisonValidation(groundtruthComparisonValidationThreshold, randomGenerator);

Timestamp startTimestamp(true);

Expand Down Expand Up @@ -411,7 +415,8 @@ bool TestQRCodeDetector2D::testDetectQRCodesSyntheticData_Internal(const unsigne
{
ocean_assert(false && "This should never happen!");

OCEAN_SET_FAILED(validation);
OCEAN_SET_FAILED(detectionValidation);
OCEAN_SET_FAILED(groundtruthComparisonValidation);
}

if (gaussianFilterSize != 0u)
Expand All @@ -430,24 +435,30 @@ bool TestQRCodeDetector2D::testDetectQRCodesSyntheticData_Internal(const unsigne

for (const bool useWorker : {true, false})
{
ValidationPrecision::ScopedIteration scopedIteration(validation);
ValidationPrecision::ScopedIteration detectionScopedIteration(detectionValidation);
ValidationPrecision::ScopedIteration groundtruthComparisonScopedIteration(groundtruthComparisonValidation);

Worker* workerToUse = useWorker ? &worker : nullptr;

QRCodeDetector2D::Observations observations;
const QRCodes codes = QRCodeDetector2D::detectQRCodes(*anyCamera, frame, &observations, workerToUse);

bool detectionSuccess = codes.size() == 1;
bool groundTruthComparisonSuccess = detectionSuccess && codes[0].isSame(groundtruthCode, false);
const bool detectionSuccess = codes.size() == 1 && codes[0].isSame(groundtruthCode, true);
const bool groundtruthComparisonSuccess = detectionSuccess && codes[0].isSame(groundtruthCode, false);

if (!detectionSuccess || !groundTruthComparisonSuccess)
if (!detectionSuccess)
{
scopedIteration.setInaccurate();
detectionScopedIteration.setInaccurate();
}

if (!groundtruthComparisonSuccess)
{
groundtruthComparisonScopedIteration.setInaccurate();
}

#ifdef OCEAN_TEST_QRCODES_DETECTOR2D_ENABLE_VERBOSE_LOGGING
bool failedAtDetection = !detectionSuccess;
bool failedAtGroundTruthComparison = detectionSuccess && !groundTruthComparisonSuccess;
bool failedAtGroundTruthComparison = detectionSuccess && !groundtruthComparisonSuccess;

// Differentiate cases of ground truth comparison failure where extracted modules in detected QR code is wrong but otherwise decoded data is correct due to error correction.
bool failedAtGroundTruthModulesComparison = failedAtGroundTruthComparison && codes[0].isSame(groundtruthCode, true);
Expand Down Expand Up @@ -541,13 +552,17 @@ bool TestQRCodeDetector2D::testDetectQRCodesSyntheticData_Internal(const unsigne
#endif // #ifdef OCEAN_TEST_QRCODES_DETECTOR2D_ENABLE_VERBOSE_LOGGING
}
}
while (validation.needMoreIterations() || !startTimestamp.hasTimePassed(testDuration));
while (detectionValidation.needMoreIterations() || groundtruthComparisonValidation.needMoreIterations() || !startTimestamp.hasTimePassed(testDuration));

Log::info() << " ";

Log::info() << "Validation: " << validation;
Log::info() << "QR Code Detection Validation: " << detectionValidation;
Log::info() << "Groundtruth Comparison Validation: " << groundtruthComparisonValidation;

return validation.succeeded();
bool success = detectionValidation.succeeded();
success = groundtruthComparisonValidation.succeeded() && success;

return success;
}

} // namespace TestQRCodes
Expand Down
5 changes: 3 additions & 2 deletions impl/ocean/test/testcv/testdetector/testqrcodes/TestQRCodeDetector2D.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,15 +78,16 @@ class OCEAN_TEST_CV_DETECTOR_QRCODES_EXPORT TestQRCodeDetector2D : protected CV:
* @param testDuration Number of seconds for each test, with range (0, infinity)
* @param worker A worker object
* @param testLable Label specified by caller test function, used in log messages and within generated file names. Must be alphanumeric.
* @param validationPrecisionThreshold Proportion of QR code detection attempts that must succeed for the test to pass, range: [0, 1]
* @param detectionValidationThreshold Proportion of QR code detection attempts that must succeed for the test to pass, range: [0, 1]
* @param groundtruthComparisonValidationThreshold Proportion of QR code detection attempts that must succeed with exact match to groundtruth for the test to pass, range: [0, detectionValidationThreshold]
* @param moduleSizePixelsMin Minimum module size in pixels, range: [1, infinity)
* @param moduleSizePixelsMax Maximum module size in pixels, range: [moduleSizePixelsMin, infinity)
* @param imageDimPixelsMin Hard limit on how small width or height of generated image can be, in terms of pixels, range: [0, infinity)
* @param imageDimPixelsMax Soft limit on how large width or height of generated image can be, in terms of pixels, range: [0, infinity)
* @return True, if succeeded
* @remark Images generated may have width or height greater than `imageDimPixelsMax` if the image needs to be larger to fit the QR code generated due to other parameter values
*/
static bool testDetectQRCodesSyntheticData_Internal(const unsigned int gaussianFilterSize, const double testDuration, Worker& worker, const std::string& testLabel, const double validationPrecisionThreshold = 0.99, const unsigned int moduleSizePixelsMin = 6u, const unsigned int moduleSizePixelsMax = 8u, const unsigned int imageDimPixelsMin = 0u, const unsigned int imageDimPixelsMax = 720u);
static bool testDetectQRCodesSyntheticData_Internal(const unsigned int gaussianFilterSize, const double testDuration, Worker& worker, const std::string& testLabel, const double detectionValidationThreshold, const double groundtruthComparisonValidationThreshold, const unsigned int moduleSizePixelsMin = 6u, const unsigned int moduleSizePixelsMax = 8u, const unsigned int imageDimPixelsMin = 0u, const unsigned int imageDimPixelsMax = 720u);
};

} // namespace TestQRCodes
Expand Down

0 comments on commit c2568c9

Please sign in to comment.