Skip to content

Commit

Permalink
EvalRegression - Add a flag to skip generating the R plot. Improve me… (
Browse files Browse the repository at this point in the history 
#603)

* EvalRegression - Add a flag to skip generating the R plot. Improve memory usage for very large input files.

* EvalRegression - Add total row to bottom of correlation 2d histogram table.

* EvalRegression - Move MAE and RMS calculation outside of CorrelationCounter class.

* EvalRegression - Refactor from float* to QList<float>.

* EvalRegression - Create HistogramTable class to move histogram logic out of CorrelationCounter.

* EvalRegression - Small refactoring in EvalRegression code.

* EvalRegression - Small refactor for the HistogramTable object.
  • Loading branch information
@JStehouwer
JStehouwer authored Mar 19, 2024
1 parent d290499 commit 677e20d
Show file tree
Hide file tree
Showing 6 changed files with 158 additions and 53 deletions.
4 changes: 2 additions & 2 deletions app/br/br.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,8 @@ class FakeMain : public QRunnable
check((parc >= 2) && (parc <= 7), "Incorrect parameter count for 'evalLandmarking'.");
br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1, parc >= 6 ? atoi(parv[5]) : 0, parc >= 7 ? atoi(parv[6]) : 5);
} else if (!strcmp(fun, "evalRegression")) {
check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'.");
br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : "");
check(parc >= 2 && parc <= 5, "Incorrect parameter count for 'evalRegression'.");
br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : "", parc >= 5 ? parv[4] : "true");
} else if (!strcmp(fun, "evalKNN")) {
check(parc >= 2 && parc <= 3, "Incorrect parameter count for 'evalKNN'.");
br_eval_knn(parv[0], parv[1], parc > 2 ? parv[2] : "");
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/api_docs/c_api/functions.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,7 @@ Evaluates regression accuracy to disk.

* **function definition:**

void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = "")
void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = "", const char *generate_plots)

* **parameters:**

Expand Down
197 changes: 151 additions & 46 deletions openbr/core/eval.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1092,17 +1092,17 @@ float EvalLandmarking(const QString &predictedGallery, const QString &truthGalle

// Helper class for computing correlation coefficient between two sets of values.
// See Pearson calculation as utilized by python numpy.
class CorrelationCounter
struct CorrelationCounter
{
private:
int num_samples;
double sumX, sumY, sumXX, sumYY, sumXY;
// Variables to compute the correlation
int num_samples = 0;
double sumX = 0.f, sumY = 0.f, sumXX = 0.f, sumYY = 0.f, sumXY = 0.f;
public:
CorrelationCounter() : num_samples(0), sumX(0.0), sumY(0.0), sumXX(0.0), sumYY(0.0), sumXY(0.0) {}

void add_sample(double pred, double gt)
{
num_samples += 1;
num_samples++;

sumX += pred;
sumY += gt;
sumXX += pred * pred;
Expand All @@ -1119,66 +1119,171 @@ class CorrelationCounter
}
};

void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty, QString truthProperty)
// Helper class for computing a 2d histogram (heatmap) between two sets of values.
class HistogramTable
{
qDebug("Evaluating regression of (%s,%s) against ground truth (%s,%s)", qPrintable(predictedGallery), qPrintable(predictedProperty), qPrintable(truthGallery), qPrintable(truthProperty));
private:
// Variables to compute the 2D histogram correlation
float gtMinValue = FLT_MAX, gtMaxValue = FLT_MIN, predMinValue = FLT_MAX, predMaxValue = FLT_MIN;

if (predictedProperty.isEmpty())
predictedProperty = "Regressor";
// If predictedProperty is specified, but truthProperty isn't, copy the value over
// rather than using the default for truthProperty
else if (truthProperty.isEmpty())
truthProperty = predictedProperty;
// Hold a list of the scores
QList<float> gts, preds;
public:
HistogramTable() {
gts = QList<float>();
preds = QList<float>();
}

if (truthProperty.isEmpty())
predictedProperty = "Regressand";
void add_sample(double pred, double gt)
{
gts << gt;
preds << pred;

const TemplateList predicted(TemplateList::fromGallery(predictedGallery));
const TemplateList truth(TemplateList::fromGallery(truthGallery));
if (predicted.size() != truth.size()) qFatal("Input size mismatch.");
if (gt < gtMinValue) gtMinValue = gt;
if (gt > gtMaxValue) gtMaxValue = gt;
if (pred < predMinValue) predMinValue = pred;
if (pred > predMaxValue) predMaxValue = pred;
}

QString getGTValues(QString separator) {
QString values = "";
for (int i = 0; i < gts.size() - 1; i++)
values.append(QString("%1%2").arg(QString::number(gts[i])).arg(separator));
values.append(QString("%1").arg(gts[gts.size()-1]));
return values;
}

QString getPredValues(QString separator) {
QString values = "";
for (int i = 0; i < preds.size() - 1; i++)
values.append(QString("%1%2").arg(QString::number(preds[i])).arg(separator));
values.append(QString("%1").arg(preds[preds.size()-1]));
return values;
}

void print_hist()
{
int num_samples = gts.size();
int histSize = 20;
QVector<int> hist = QVector<int>(histSize*histSize, 0);
for (int i = 0; i < histSize * histSize; i++)
hist[i] = 0;

printf("\nHistogram Table (PRED = Y-axis in [%.3f,%.3f], GT = X-axis in [%.3f,%.3f])\n| Y // X |", predMinValue, predMaxValue, gtMinValue, gtMaxValue);
for (int i = 0; i < num_samples; i++) {
int predBin = MIN(histSize-1, MAX(0, ((int)(histSize*(preds[i]-predMinValue)/(predMaxValue-predMinValue)))));
int gtBin = MIN(histSize-1, MAX(0, ((int)(histSize*(gts[i]-gtMinValue)/(gtMaxValue-gtMinValue)))));
hist[histSize * predBin + gtBin] += 1;
}

for (int i = 0; i < histSize; i++)
printf(" %.2f |", (i / (float) histSize));
printf(" Total |\n| :----: |");
for (int i = 0; i < histSize; i++)
printf(" :----: |");
printf(" :----: |");
for (int i = 0; i < histSize; i++) {
printf("\n| %.2f |", (i / (float) histSize));
float total = 0.f;
for (int j = 0; j < histSize; j++) {
float value = hist[i*histSize+j] * (100.f / num_samples);
total += value;
if (value == 0)
printf(" |");
else if (value >= 10.f)
printf(" %.2f |", value);
else
printf(" %.2f |", value);
}
if (total == 0)
printf(" |");
else if (total >= 10.f)
printf(" %.2f |", total);
else
printf(" %.2f |", total);
}
printf("\n| Total |");
for (int j = 0; j < histSize; j++) {
float total = 0;
for (int i = 0; i < histSize; i++) {
total += hist[i*histSize+j] * (100.f / num_samples);
}
if (total == 0)
printf(" |");
else if (total >= 10.f)
printf(" %.2f |", total);
else
printf(" %.2f |", total);
}
printf(" 100.0 |\n\n");
}
};

float rmsError = 0;
float maeError = 0;
void EvalRegression(const TemplateList predicted, const TemplateList truth, QString predictedProperty, QString truthProperty, bool generatePlot) {
float rms = 0.f, mae = 0.f;
CorrelationCounter cc;
QStringList truthValues, predictedValues;
HistogramTable ht;
for (int i=0; i<predicted.size(); i++) {
if (predicted[i].file.name != truth[i].file.name)
qFatal("Input order mismatch.");

if (predicted[i].file.contains(predictedProperty) && truth[i].file.contains(truthProperty)) {
float pred = predicted[i].file.get<float>(predictedProperty);
float gt = truth[i].file.get<float>(truthProperty);
float difference = pred - gt;

rmsError += pow(difference, 2.f);
maeError += fabsf(difference);
truthValues.append(QString::number(gt));
predictedValues.append(QString::number(pred));

rms += pow(pred - gt, 2.f) / predicted.size();
mae += fabsf(pred - gt) / predicted.size();
cc.add_sample(pred, gt);
ht.add_sample(pred, gt);
}
}

QStringList rSource;
rSource << "# Load libraries" << "library(ggplot2)" << "" << "# Set Data"
<< "Actual <- c(" + truthValues.join(",") + ")"
<< "Predicted <- c(" + predictedValues.join(",") + ")"
<< "data <- data.frame(Actual, Predicted)"
<< "" << "# Construct Plot" << "pdf(\"EvalRegression.pdf\")"
<< "print(qplot(Actual, Predicted, data=data, geom=\"jitter\", alpha=I(2/3)) + geom_abline(intercept=0, slope=1, color=\"forestgreen\", size=I(1)) + geom_smooth(size=I(1), color=\"mediumblue\") + theme_bw())"
<< "print(qplot(Actual, Predicted-Actual, data=data, geom=\"jitter\", alpha=I(2/3)) + geom_abline(intercept=0, slope=0, color=\"forestgreen\", size=I(1)) + geom_smooth(size=I(1), color=\"mediumblue\") + theme_bw())"
<< "dev.off()";


QString rFile = "EvalRegression.R";
QtUtils::writeFile(rFile, rSource);
bool success = QtUtils::runRScript(rFile);
if (success) QtUtils::showFile("EvalRegression.pdf");
if (generatePlot) {
QStringList rSource;
rSource << "# Load libraries" << "library(ggplot2)" << "" << "# Set Data"
<< "Actual <- c(" + ht.getGTValues(",") + ")"
<< "Predicted <- c(" + ht.getPredValues(",") + ")"
<< "data <- data.frame(Actual, Predicted)"
<< "" << "# Construct Plot" << "pdf(\"EvalRegression.pdf\")"
<< "print(qplot(Actual, Predicted, data=data, geom=\"jitter\", alpha=I(2/3)) + geom_abline(intercept=0, slope=1, color=\"forestgreen\", size=I(1)) + geom_smooth(size=I(1), color=\"mediumblue\") + theme_bw())"
<< "print(qplot(Actual, Predicted-Actual, data=data, geom=\"jitter\", alpha=I(2/3)) + geom_abline(intercept=0, slope=0, color=\"forestgreen\", size=I(1)) + geom_smooth(size=I(1), color=\"mediumblue\") + theme_bw())"
<< "dev.off()";


QString rFile = "EvalRegression.R";
QtUtils::writeFile(rFile, rSource);
bool success = QtUtils::runRScript(rFile);
if (success) QtUtils::showFile("EvalRegression.pdf");
}

qDebug("Total Samples = %i", predicted.size());
qDebug("RMS Error = %f", sqrt(rmsError/predicted.size()));
qDebug("MAE = %f", maeError/predicted.size());
qDebug("RMS Error = %f", sqrt(rms));
qDebug("MAE = %f", mae);
qDebug("Correlation (Pearson) = %f", cc.correlation_coefficient());
ht.print_hist();
}

void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty, QString truthProperty, bool generatePlot)
{
qDebug("Evaluating regression of (%s,%s) against ground truth (%s,%s)", qPrintable(predictedGallery), qPrintable(predictedProperty), qPrintable(truthGallery), qPrintable(truthProperty));

if (predictedProperty.isEmpty())
predictedProperty = "Regressor";
// If predictedProperty is specified, but truthProperty isn't, copy the value over
// rather than using the default for truthProperty
else if (truthProperty.isEmpty())
truthProperty = predictedProperty;

if (truthProperty.isEmpty())
predictedProperty = "Regressand";

const TemplateList predicted(TemplateList::fromGallery(predictedGallery));
if (predictedGallery == truthGallery) {
EvalRegression(predicted, predicted, predictedProperty, truthProperty, generatePlot);
} else {
const TemplateList truth = TemplateList::fromGallery(truthGallery);
if (predicted.size() != truth.size()) qFatal("Input size mismatch.");
EvalRegression(predicted, truth, predictedProperty, truthProperty, generatePlot);
}
}

void readKNN(size_t &probeCount, size_t &k, QVector<Candidate> &neighbors, const QString &fileName)
Expand Down
2 changes: 1 addition & 1 deletion openbr/core/eval.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ namespace br
void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0, float relativeMinSize = 0, const QString &label = "", const float true_positive_threshold = 0.5f); // Return average overlap
float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error
void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "", bool generatePlots = true);
void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv = "");
void EvalEER(const QString &predictedXML, const QString gt_property = "", const QString distribution_property = "", const QString &csv = "");
struct Candidate
Expand Down
4 changes: 2 additions & 2 deletions openbr/openbr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,9 @@ float br_eval_landmarking(const char *predicted_gallery, const char *truth_galle
return EvalLandmarking(predicted_gallery, truth_gallery, csv, normalization_index_a, normalization_index_b, sample_index, total_examples);
}

void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property, const char *truth_property)
void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property, const char *truth_property, const char *generate_plots)
{
EvalRegression(predicted_gallery, truth_gallery, predicted_property, truth_property);
EvalRegression(predicted_gallery, truth_gallery, predicted_property, truth_property, strcmp(generate_plots, "true") == 0);
}

void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv)
Expand Down
2 changes: 1 addition & 1 deletion openbr/openbr.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *tru

BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", int normalization_index_a = 0, int normalization_index_b = 1, int sample_index = 0, int total_examples = 5);

BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = "");
BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = "", const char *generate_plots = "true");

BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv = "");

Expand Down

0 comments on commit 677e20d

Please sign in to comment.