- add WDC5 support

CMakeLists.txt

@@ -62,6 +62,7 @@ set(MAIN_SOURCES
         fileReaders/WDC2/DB2Base.cpp
         fileReaders/WDC3/DB2Ver3.cpp
         fileReaders/WDC4/DB2Ver4.cpp
+        fileReaders/WDC5/DB2Ver5.cpp
         exporters/sqlite/CSQLLiteExporter.cpp
         exporters/sqlite/CSQLLiteExporter.h
         fileReaders/DBD/DBDFileStorage.cpp

fileReaders/DBD/DBDFile.cpp

@@ -137,7 +137,7 @@ void DBDFile::parseColumnDefLine(std::string &line) {
 //    columnDefs.insert(, newDef);
 }
 
-void DBDFile::parseColumnBuildDefLine(std::string &line, BuildConfig &buildConfig) {
+void DBDFile::parseColumnBuildDefLine(const std::string &line, BuildConfig &buildConfig) {
     ColumnBuildDef buildDef;
 
     int prefixStart = line.find('$', 0);
@@ -201,7 +201,7 @@ void DBDFile::parseColumnBuildDefLine(std::string &line, BuildConfig &buildConfi
     buildConfig.columns.push_back(buildDef);
 }
 
-bool DBDFile::findBuildConfig(std::string buildVersionString, std::string layout, DBDFile::BuildConfig *&buildConfig_) {
+bool DBDFile::findBuildConfig(const std::string &buildVersionString, const std::string &layout, DBDFile::BuildConfig *&buildConfig_) {
     for (auto &buildConfig :buildConfigs) {
         if (std::find(buildConfig.builds.begin(), buildConfig.builds.end(), buildVersionString) != buildConfig.builds.end()) {
             buildConfig_ = &buildConfig;
@@ -210,7 +210,7 @@ bool DBDFile::findBuildConfig(std::string buildVersionString, std::string layout
     }
     return false;
 }
-bool DBDFile::findBuildConfigByLayout(std::string layout, DBDFile::BuildConfig *&buildConfig_) {
+bool DBDFile::findBuildConfigByLayout(const std::string &layout, DBDFile::BuildConfig *&buildConfig_) {
     for (auto &buildConfig :buildConfigs) {
         if (std::find(buildConfig.layoutHashes.begin(), buildConfig.layoutHashes.end(), layout) != buildConfig.layoutHashes.end()) {
             buildConfig_ = &buildConfig;

fileReaders/DBD/DBDFile.h

@@ -51,16 +51,16 @@ class DBDFile {
     std::unordered_map<std::string, ColumnDef> columnDefs;
     std::vector<BuildConfig> buildConfigs;
 public:
-    bool findBuildConfig(std::string buildVersionString, std::string layout, DBDFile::BuildConfig *&buildConfig);
-    bool findBuildConfigByLayout(std::string layout, DBDFile::BuildConfig *&buildConfig_);
+    bool findBuildConfig(const std::string &buildVersionString, const std::string &layout, DBDFile::BuildConfig *&buildConfig);
+    bool findBuildConfigByLayout(const std::string &layout, DBDFile::BuildConfig *&buildConfig_);
     const ColumnDef *getColumnDef(const std::string &columnName) const {
         return &columnDefs.at(columnName);
     }
 private:
     enum class SectionMode { NONE, COLUMNS, BUILD};
 
     void parseColumnDefLine(std::string &line);
-    void parseColumnBuildDefLine(std::string &line, BuildConfig &buildConfig);
+    void parseColumnBuildDefLine(const std::string &line, BuildConfig &buildConfig);
     void commitBuildConfig(SectionMode currentMode, BuildConfig &buildConfig);
 };
 

fileReaders/WDC3/DB2Ver3.cpp

@@ -31,18 +31,21 @@ void DB2Ver3::process(HFileContent db2File, const std::string &fileName) {
     currentOffset = 0;
     bytesRead = 0;
 
+    WDC3::db2_header header;
     readValue(header);
+    readValue(headerContent);
 
-    readValues(section_headers, header->section_count);
-    readValues(fields, header->total_field_count);
-    fieldInfoLength = header->field_storage_info_size / sizeof(field_storage_info);
+
+    readValues(section_headers, headerContent->section_count);
+    readValues(fields, headerContent->total_field_count);
+    fieldInfoLength = headerContent->field_storage_info_size / sizeof(field_storage_info);
     readValues(field_info, fieldInfoLength);
 
 
     int palleteDataRead = 0;
-    if (header->pallet_data_size > 0) {
-        palleteDataArray.resize(header->field_count);
-        for (int i = 0; i < header->field_count; i++) {
+    if (headerContent->pallet_data_size > 0) {
+        palleteDataArray.resize(headerContent->field_count);
+        for (int i = 0; i < headerContent->field_count; i++) {
             if ((field_info[i].storage_type == field_compression::field_compression_bitpacked_indexed) ||
                 (field_info[i].storage_type == field_compression::field_compression_bitpacked_indexed_array)) {
 
@@ -56,14 +59,14 @@ void DB2Ver3::process(HFileContent db2File, const std::string &fileName) {
         }
     }
 
-    assert(palleteDataRead*4 == header->pallet_data_size);
+    assert(palleteDataRead*4 == headerContent->pallet_data_size);
 
-//    readValues(pallet_data, header->pallet_data_size);
+//    readValues(pallet_data, headerContent->pallet_data_size);
     //Form hashtable for column
 
-    if (header->common_data_size > 0) {
-        commonDataHashMap.resize(header->field_count);
-        for (int i = 0; i < header->field_count; i++) {
+    if (headerContent->common_data_size > 0) {
+        commonDataHashMap.resize(headerContent->field_count);
+        for (int i = 0; i < headerContent->field_count; i++) {
             if (field_info[i].storage_type == field_compression::field_compression_common_data) {
                 int id;
                 uint32_t value;
@@ -79,9 +82,9 @@ void DB2Ver3::process(HFileContent db2File, const std::string &fileName) {
 //    readValues(common_data, );
 
     //Read section
-    sections.resize(header->section_count);
+    sections.resize(headerContent->section_count);
 
-    for (int i = 0; i < header->section_count; i++) {
+    for (int i = 0; i < headerContent->section_count; i++) {
         auto &itemSectionHeader = section_headers[i];
 
         section &section = sections[i];
@@ -90,12 +93,12 @@ void DB2Ver3::process(HFileContent db2File, const std::string &fileName) {
 
         assert(itemSectionHeader.file_offset == currentOffset);
 
-        if (!header->flags.isSparse) {
+        if (!headerContent->flags.isSparse) {
             // Normal records
 
             for (int j = 0; j < itemSectionHeader.record_count; j++) {
                 record_data recordData;
-                readValues(recordData.data, header->record_size);
+                readValues(recordData.data, headerContent->record_size);
 
                 section.records.push_back(recordData);
             }
@@ -117,7 +120,7 @@ void DB2Ver3::process(HFileContent db2File, const std::string &fileName) {
             readValues(section.copy_table, itemSectionHeader.copy_table_count);
         }
 
-        if (header->table_hash == 145293629)
+        if (headerContent->table_hash == 145293629)
             currentOffset+=itemSectionHeader.offset_map_id_count*4;
 
         readValues(section.offset_map, itemSectionHeader.offset_map_id_count);
@@ -148,7 +151,7 @@ void DB2Ver3::process(HFileContent db2File, const std::string &fileName) {
         if (itemSectionHeader.tact_key_hash != 0)
         {
             //Check if section was decrypted properly
-            if (!header->flags.isSparse) {
+            if (!headerContent->flags.isSparse) {
                 section.isEncoded = checkDataIfNonZero(section.records[0].data, currentOffset - itemSectionHeader.file_offset);
             }
             else {
@@ -270,7 +273,7 @@ std::shared_ptr<DB2Ver3::WDC3Record> DB2Ver3::getRecord(const int recordIndex) {
     if (recordId == 0) {
         //Some sections have id_list, but have 0 as an entry there for the record.
         //That's when we need calc recordId this way
-        recordId = header->min_id+recordIndex;
+        recordId = headerContent->min_id + recordIndex;
     }
 
     unsigned char *recordPointer = sectionContent.records[indexWithinSection].data;
@@ -347,7 +350,7 @@ void DB2Ver3::guessFieldSizeForCommon(int fieldSizeBits, int &elementSizeBytes,
 
 std::string DB2Ver3::getLayoutHash() {
     std::stringstream res;
-    res << std::setfill('0') << std::setw(8) << std::hex << header->layout_hash ;
+    res << std::setfill('0') << std::setw(8) << std::hex << headerContent->layout_hash ;
     std::string resStr = res.str();
     std::locale locale;
     auto to_upper = [&locale] (char ch) { return std::use_facet<std::ctype<char>>(locale).toupper(ch); };
@@ -357,11 +360,11 @@ std::string DB2Ver3::getLayoutHash() {
     return resStr;
 }
 
-int DB2Ver3::isSparse() { return header->flags.isSparse; }
+int DB2Ver3::isSparse() { return headerContent->flags.isSparse; }
 
 const field_storage_info * const DB2Ver3::getFieldInfo(uint32_t fieldIndex) const {
-    if (fieldIndex >= header->field_count) {
-        std::cout << "fieldIndex = " << fieldIndex << " is bigger than field count = " << header->field_count << std::endl;
+    if (fieldIndex >= headerContent->field_count) {
+        std::cout << "fieldIndex = " << fieldIndex << " is bigger than field count = " << headerContent->field_count << std::endl;
         return nullptr;
     }
     return &field_info[fieldIndex];
@@ -387,7 +390,7 @@ static inline void fixPaletteValue(WDC3::DB2Ver3::WDCFieldValue &value, int exte
 }
 
 std::vector<WDC3::DB2Ver3::WDCFieldValue> DB2Ver3::WDC3Record::getField(int fieldIndex, int externalArraySize, int externalElemSizeBytes) const {
-    auto const db2Header = db2Class->header;
+    auto const db2Header = db2Class->headerContent;
 
     std::vector<WDC3::DB2Ver3::WDCFieldValue> result = {};
 
@@ -505,21 +508,21 @@ std::vector<WDC3::DB2Ver3::WDCFieldValue> DB2Ver3::WDC3Record::getField(int fiel
 }
 
 std::string DB2Ver3::WDC3Record::readString(int fieldIndex, int fieldElementOffset, int stringOffset) const {
-    const wdc3_db2_header * const db2Header = db2Class->header;
+    const auto * const headerContent = db2Class->headerContent;
 
-    if (fieldIndex >= db2Header->field_count) {
+    if (fieldIndex >= headerContent->field_count) {
         return "!#Invalid field number";
     }
 
     if (stringOffset == 0)
         return ""; //How else would you mark empty string? Only through null offset
 
     uint32_t fieldOffsetIntoGlobalArray =
-            (recordIndex * db2Header->record_size) +
+            (recordIndex * headerContent->record_size) +
             (db2Class->field_info[fieldIndex].field_offset_bits >> 3) + fieldElementOffset;
 
     uint32_t stringOffsetIntoGlobalStringSection =
-            (fieldOffsetIntoGlobalArray + stringOffset) - (db2Header->record_count*db2Header->record_size);
+            (fieldOffsetIntoGlobalArray + stringOffset) - (headerContent->record_count*headerContent->record_size);
 
     int offset = stringOffsetIntoGlobalStringSection; // NOLINT(cppcoreguidelines-narrowing-conversions)
 
@@ -535,7 +538,7 @@ std::string DB2Ver3::WDC3Record::readString(int fieldIndex, int fieldElementOffs
         sectionIndexforStr++;
     }
 
-    if (sectionIndexforStr >= db2Header->section_count) {
+    if (sectionIndexforStr >= headerContent->section_count) {
         return "!#Found invalid section for String, offset = " + std::to_string(offset);
     }
 
@@ -592,7 +595,7 @@ std::string DB2Ver3::WDC3RecordSparse::readNextAsString() {
     std::string result = std::string((char *)&recordPointer[fieldOffset]);
     fieldOffset+=result.length()+1;
 
-    if (fieldOffset > db2Class->header->record_size) {
+    if (fieldOffset > db2Class->headerContent->record_size) {
         return "Reading a field resulted in buffer overflow";
     }
 

fileReaders/WDC3/DB2Ver3.h

@@ -26,8 +26,13 @@ namespace WDC3 {
     };
 
     PACK(
-    struct wdc3_db2_header {
-        uint32_t magic;                  // 'WDC3'
+    struct db2_header {
+        uint32_t magic; // 'WDC3'
+    });
+
+    PACK(
+    struct db2_header_content
+    {
         uint32_t record_count;           // this is for all sections combined now
         uint32_t field_count;
         uint32_t record_size;
@@ -203,7 +208,7 @@ namespace WDC3 {
         std::shared_ptr<WDC3Record> getRecord(int recordIndex);
         std::shared_ptr<WDC3RecordSparse> getSparseRecord(int recordIndex);
 
-        int getRecordCount() { return header->record_count; };
+        int getRecordCount() { return headerContent->record_count; };
         int isSparse();
         bool hasRelationshipField() { return hasRelationShipField; };
 
@@ -215,8 +220,8 @@ namespace WDC3 {
 
         const field_storage_info * const getFieldInfo(uint32_t fieldIndex) const;
 
-        const WDC3::wdc3_db2_header* const getWDCHeader() {
-            return this->header;
+        const auto* const getWDCHeader() {
+            return this->headerContent;
         }
 
         union WDCFieldValue {
@@ -303,7 +308,7 @@ namespace WDC3 {
         }
 
 //fields
-        wdc3_db2_header *header;
+        db2_header_content *headerContent;
         wdc3_section_header *section_headers;
         field_structure *fields;
         field_storage_info *field_info;

fileReaders/WDC4/DB2Ver4.cpp

@@ -31,18 +31,21 @@ void DB2Ver4::process(HFileContent db2File, const std::string &fileName) {
     currentOffset = 0;
     bytesRead = 0;
 
+    WDC3::db2_header header;
     readValue(header);
 
-    readValues(section_headers, header->section_count);
-    readValues(fields, header->total_field_count);
-    fieldInfoLength = header->field_storage_info_size / sizeof(WDC3::field_storage_info);
+    readValue(headerContent);
+
+    readValues(section_headers, headerContent->section_count);
+    readValues(fields, headerContent->total_field_count);
+    fieldInfoLength = headerContent->field_storage_info_size / sizeof(WDC3::field_storage_info);
     readValues(field_info, fieldInfoLength);
 
 
     int palleteDataRead = 0;
-    if (header->pallet_data_size > 0) {
-        palleteDataArray.resize(header->field_count);
-        for (int i = 0; i < header->field_count; i++) {
+    if (headerContent->pallet_data_size > 0) {
+        palleteDataArray.resize(headerContent->field_count);
+        for (int i = 0; i < headerContent->field_count; i++) {
             if ((field_info[i].storage_type == WDC3::field_compression::field_compression_bitpacked_indexed) ||
                 (field_info[i].storage_type == WDC3::field_compression::field_compression_bitpacked_indexed_array)) {
 
@@ -56,14 +59,14 @@ void DB2Ver4::process(HFileContent db2File, const std::string &fileName) {
         }
     }
 
-    assert(palleteDataRead*4 == header->pallet_data_size);
+    assert(palleteDataRead*4 == headerContent->pallet_data_size);
 
-//    readValues(pallet_data, header->pallet_data_size);
+//    readValues(pallet_data, headerContent->pallet_data_size);
     //Form hashtable for column
 
-    if (header->common_data_size > 0) {
-        commonDataHashMap.resize(header->field_count);
-        for (int i = 0; i < header->field_count; i++) {
+    if (headerContent->common_data_size > 0) {
+        commonDataHashMap.resize(headerContent->field_count);
+        for (int i = 0; i < headerContent->field_count; i++) {
             if (field_info[i].storage_type == WDC3::field_compression::field_compression_common_data) {
                 int id;
                 uint32_t value;
@@ -79,7 +82,7 @@ void DB2Ver4::process(HFileContent db2File, const std::string &fileName) {
 //    readValues(common_data, );
 
     int size = 0;
-    for (int i = 0; i < header->section_count; i++) {
+    for (int i = 0; i < headerContent->section_count; i++) {
         auto &itemSectionHeader = section_headers[i];
         if (itemSectionHeader.tact_key_hash == 0) continue;
 
@@ -94,9 +97,9 @@ void DB2Ver4::process(HFileContent db2File, const std::string &fileName) {
     }
 
     //Read section
-    sections.resize(header->section_count);
+    sections.resize(headerContent->section_count);
 
-    for (int i = 0; i < header->section_count; i++) {
+    for (int i = 0; i < headerContent->section_count; i++) {
         auto &itemSectionHeader = section_headers[i];
 
         WDC3::section &section = sections[i];
@@ -105,12 +108,12 @@ void DB2Ver4::process(HFileContent db2File, const std::string &fileName) {
 
         assert(itemSectionHeader.file_offset == currentOffset);
 
-        if (!header->flags.isSparse) {
+        if (!headerContent->flags.isSparse) {
             // Normal records
 
             for (int j = 0; j < itemSectionHeader.record_count; j++) {
                 WDC3::record_data recordData;
-                readValues(recordData.data, header->record_size);
+                readValues(recordData.data, headerContent->record_size);
 
                 section.records.push_back(recordData);
             }
@@ -132,7 +135,7 @@ void DB2Ver4::process(HFileContent db2File, const std::string &fileName) {
             readValues(section.copy_table, itemSectionHeader.copy_table_count);
         }
 
-        if (header->table_hash == 145293629)
+        if (headerContent->table_hash == 145293629)
             currentOffset+=itemSectionHeader.offset_map_id_count*4;
 
         readValues(section.offset_map, itemSectionHeader.offset_map_id_count);
@@ -163,7 +166,7 @@ void DB2Ver4::process(HFileContent db2File, const std::string &fileName) {
         if (itemSectionHeader.tact_key_hash != 0)
         {
             //Check if section was decrypted properly
-            if (!header->flags.isSparse) {
+            if (!headerContent->flags.isSparse) {
                 section.isEncoded = checkDataIfNonZero(section.records[0].data, currentOffset - itemSectionHeader.file_offset);
             }
             else {