Skip to content

Commit

Permalink
added key overflow warning at end of db serialization when at least o…
Browse files Browse the repository at this point in the history
…ne image id/path > 256 char
  • Loading branch information
beniz committed Jul 3, 2020
1 parent 01a6681 commit fa195c0
Showing 1 changed file with 13 additions and 1 deletion.
14 changes: 13 additions & 1 deletion src/backends/caffe/caffeinputconns.cc
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,7 @@ namespace dd
int count = 0;
const int kMaxKeyLength = 256;
char key_cstr[kMaxKeyLength];
bool key_overflow = false;

for (int line_id = 0; line_id < (int)lfiles.size(); ++line_id) {
Datum datum;
Expand All @@ -388,7 +389,9 @@ namespace dd
// sequential
int length = snprintf(key_cstr, kMaxKeyLength, "%08d_%s", line_id,
lfiles[line_id].first.c_str());

if (lfiles[line_id].first.size() > kMaxKeyLength)
key_overflow = true;

// put in db
std::string out;
if(!datum.SerializeToString(&out))
Expand All @@ -407,6 +410,9 @@ namespace dd
txn->Commit();
_logger->info("Processed {} files",count);
}
if (key_overflow)
_logger->warn("Some of the keys in {} have been truncated to fit the 256 max key length requirement",
dbfullname);
}

void ImgCaffeInputFileConn::write_image_to_db_multilabel(const std::string &dbfullname,
Expand All @@ -425,6 +431,7 @@ namespace dd
int count = 0;
const int kMaxKeyLength = 256;
char key_cstr[kMaxKeyLength];
bool key_overflow = false;

for (int line_id = 0; line_id < (int)lfiles.size(); ++line_id) {
Datum datum;
Expand All @@ -449,6 +456,8 @@ namespace dd
// sequential
int length = snprintf(key_cstr, kMaxKeyLength, "%08d_%s", line_id,
lfiles[line_id].first.c_str());
if (lfiles[line_id].first.size() > kMaxKeyLength)
key_overflow = true;

// put in db
std::string out;
Expand All @@ -468,6 +477,9 @@ namespace dd
txn->Commit();
_logger->info("Processed {} files",count);
}
if (key_overflow)
_logger->warn("Some of the keys in {} have been truncated to fit the 256 max key length requirement",
dbfullname);
}

// - fixed size in-memory arrays put down to disk at once
Expand Down

0 comments on commit fa195c0

Please sign in to comment.