Use .ckmame.db to find files in archives.

regress/archive-broken.zip.test

@@ -21,5 +21,6 @@ remove empty archive
 end-of-inline-data
 stderr
 error opening zip archive 'roms/2-48.zip': Not a zip archive
-error opening zip archive 'roms/2-48.zip': Not a zip archive
+error creating zip archive 'roms/2-48.zip': Not a zip archive
+error creating zip archive 'roms/2-48.zip': Not a zip archive
 end-of-inline-data

regress/archive-broken2.zip.test

@@ -10,4 +10,5 @@ game 1-4                                     : not a single file found
 end-of-inline-data
 stderr
 error opening zip archive 'roms/2-48.zip': Not a zip archive
+error opening zip archive 'roms/2-48.zip': Not a zip archive
 end-of-inline-data

regress/dbrestore.cc

@@ -158,7 +158,7 @@ int main(int argc, char *argv[]) {
 
             switch (type) {
                 case DBTYPE_CKMAMEDB:
-                    db = std::make_unique<CkmameDB>(db_fname, ".");
+                    db = std::make_unique<CkmameDB>(db_fname, ".", FILE_NOWHERE);
                     break;
 
                 case DBTYPE_MEMDB:

regress/disk-swap.zip.test

@@ -13,22 +13,16 @@ end-of-inline-data
 stdout
 In game disk:
 rom  04.rom        size       4  crc d87f7e0c: correct
-disk 108-5         sha1 7570a907e20a51cbf6193ec6779b82d1967bb609: missing
+disk 108-5         sha1 7570a907e20a51cbf6193ec6779b82d1967bb609: is in 'roms/disk-2/108-2.chd'
 image 108-5.chd   : needed elsewhere
 save needed file '108-5.chd'
+add 'roms/disk-2/108-2.chd' as '108-5.chd'
 In game disk-2:
 rom  04.rom        size       4  crc d87f7e0c: correct
 disk 108-2         sha1 9fffa910f0ca90f61e1ab3fab0d1da225be992ae: is in 'saved/9fffa910f0ca90f61e1ab3fab0d1da225be992ae-000/108-5.chd'
-image 108-2.chd   : needed elsewhere
-save needed file '108-2.chd'
+image 108-2.chd   : not used
+delete unused file '108-2.chd'
 add 'saved/9fffa910f0ca90f61e1ab3fab0d1da225be992ae-000/108-5.chd' as '108-2.chd'
-In game disk:
-rom  04.rom        size       4  crc d87f7e0c: correct
-disk 108-5         sha1 7570a907e20a51cbf6193ec6779b82d1967bb609: is in 'saved/7570a907e20a51cbf6193ec6779b82d1967bb609-000/108-2.chd'
-add 'saved/7570a907e20a51cbf6193ec6779b82d1967bb609-000/108-2.chd' as '108-5.chd'
-In archive saved/7570a907e20a51cbf6193ec6779b82d1967bb609-000:
-delete used file '108-2.chd'
-remove empty archive
 In archive saved/9fffa910f0ca90f61e1ab3fab0d1da225be992ae-000:
 delete used file '108-5.chd'
 remove empty archive

regress/inparent.vtest

@@ -1,12 +1,13 @@
-description test 2-rom game (1 in parent), both roms in parent - BROKEN
+description test 2-rom game (1 in parent), both roms in parent
 return 0
 arguments -cFv clone-8
 file mame.db mame.db
 file roms/parent-4.zip 2-48-ok.zip
-file roms/.ckmame.db {} <inline.ckmamedb>
-hashes parent-4.zip 08.rom cheap
-end-of-inline-data
+file roms/clone-8.zip {} 1-8-ok.zip
+file roms/.ckmame.db {} <empty.ckmamedb>
 stdout
 In game clone-8:
-game clone-8                                 : not a single file found
+rom  04.rom        size       4  crc d87f7e0c: correct
+rom  08.rom        size       8  crc 3656897d: is in 'roms/parent-4.zip/08.rom'
+add 'roms/parent-4.zip/08.rom' as '08.rom'
 end-of-inline-data

regress/nogood-diskgood.zip.test

@@ -5,7 +5,7 @@ arguments -D ../mamedb-disk-many.db -Fv --report-detailed diskgood-romnogood
 file roms/diskgood-romnogood/108-5.chd 108-5.chd
 file roms/nogood.zip 1-4-ok.zip
 file roms/.ckmame.db {} <inline.ckmamedb>
-ignore nogood.zip
+hashes nogood.zip * cheap
 end-of-inline-data
 stdout
 In game diskgood-romnogood:

regress/rom-dir-in-extra-dirs-subdir-2.vtest

@@ -5,6 +5,9 @@ file mame.db mame.db
 file extra/foo.zip 1-4-foo.zip
 file extradir/1-8.zip 1-8-ok.zip
 file extra/.ckmame.db {} <empty.ckmamedb>
+file extradir/.ckmame.db {} <inline.ckmamedb>
+hashes 1-8.zip * cheap
+end-of-inline-data
 stdout
 In game 1-4:
 rom  04.rom        size       4  crc d87f7e0c: is in 'extra/foo.zip/foo/1-4/04.rom'

regress/rom-from-child-renamed.vtest

@@ -8,19 +8,13 @@ file roms/clone-8.zip 1-4-misnamed.zip {}
 file roms/.ckmame.db {} <empty.ckmamedb>
 stdout
 In game 1-4:
-game 1-4                                     : not a single file found
-In game parent-4:
-game parent-4                                : not a single file found
-In game clone-8:
-game clone-8                                 : not a single file found
-save needed file '04'
-In game 1-4:
-rom  04.rom        size       4  crc d87f7e0c: is in 'saved/d87f7e0c-000.zip/04'
-add 'saved/d87f7e0c-000.zip/04' as '04.rom'
+rom  04.rom        size       4  crc d87f7e0c: is in 'roms/clone-8.zip/04'
+add 'roms/clone-8.zip/04' as '04.rom'
 In game parent-4:
 rom  04.rom        size       4  crc d87f7e0c: is in 'roms/1-4.zip/04.rom'
 add 'roms/1-4.zip/04.rom' as '04.rom'
-In archive saved/d87f7e0c-000.zip:
-delete used file '04'
-remove empty archive
+In game clone-8:
+game clone-8                                 : not a single file found
+file 04            size       4  crc d87f7e0c: not used
+delete unused file '04'
 end-of-inline-data

regress/swap-roms-2.vtest

@@ -11,31 +11,21 @@ hashes grandparent.zip * cheap
 end-of-inline-data
 stdout
 In game grandparent:
-game grandparent                             : not a single file found
+rom  04.rom        size       4  crc d87f7e0c: is in 'roms/parent.zip/04.rom'
 file 0a.rom        size      10  crc 0b4a4cde: needed elsewhere
 save needed file '0a.rom'
+add 'roms/parent.zip/04.rom' as '04.rom'
 In game parent:
-game parent                                  : not a single file found
-file 04.rom        size       4  crc d87f7e0c: needed elsewhere
-save needed file '04.rom'
+rom  08.rom        size       8  crc 3656897d: is in 'roms/child.zip/08.rom'
+file 04.rom        size       4  crc d87f7e0c: not used
+delete unused file '04.rom'
+add 'roms/child.zip/08.rom' as '08.rom'
 In game child:
 rom  0a.rom        size      10  crc 0b4a4cde: is in 'saved/0b4a4cde-000.zip/0a.rom'
-file 08.rom        size       8  crc 3656897d: needed elsewhere
-save needed file '08.rom'
+file 08.rom        size       8  crc 3656897d: not used
+delete unused file '08.rom'
 add 'saved/0b4a4cde-000.zip/0a.rom' as '0a.rom'
-In game grandparent:
-rom  04.rom        size       4  crc d87f7e0c: is in 'saved/d87f7e0c-000.zip/04.rom'
-add 'saved/d87f7e0c-000.zip/04.rom' as '04.rom'
-In game parent:
-rom  08.rom        size       8  crc 3656897d: is in 'saved/3656897d-000.zip/08.rom'
-add 'saved/3656897d-000.zip/08.rom' as '08.rom'
 In archive saved/0b4a4cde-000.zip:
 delete used file '0a.rom'
 remove empty archive
-In archive saved/3656897d-000.zip:
-delete used file '08.rom'
-remove empty archive
-In archive saved/d87f7e0c-000.zip:
-delete used file '04.rom'
-remove empty archive
 end-of-inline-data

regress/swap-roms.vtest

@@ -10,24 +10,19 @@ end-of-inline-data
 stdout
 In game parent:
 rom  04.rom        size       4  crc d87f7e0c: is in 'roms/child.zip/04.rom'
-rom  08.rom        size       8  crc 3656897d: missing
+rom  08.rom        size       8  crc 3656897d: is in 'roms/child.zip/08.rom'
 file 08.rom        size       8  crc 12345678: needed elsewhere
 save needed file '08.rom'
 add 'roms/child.zip/04.rom' as '04.rom'
+add 'roms/child.zip/08.rom' as '08.rom'
 In game child:
 rom  08.rom        size       8  crc 12345678: is in 'saved/12345678-000.zip/08.rom'
 file 04.rom        size       4  crc d87f7e0c: not used
-file 08.rom        size       8  crc 3656897d: needed elsewhere
+file 08.rom        size       8  crc 3656897d: not used
 delete unused file '04.rom'
-save needed file '08.rom'
+delete unused file '08.rom'
 add 'saved/12345678-000.zip/08.rom' as '08.rom'
-In game parent:
-rom  08.rom        size       8  crc 3656897d: is in 'saved/3656897d-000.zip/08.rom'
-add 'saved/3656897d-000.zip/08.rom' as '08.rom'
 In archive saved/12345678-000.zip:
 delete used file '08.rom'
 remove empty archive
-In archive saved/3656897d-000.zip:
-delete used file '08.rom'
-remove empty archive
 end-of-inline-data

regress/uncompressed-ckmamedb-remove-game-2.dir.test

@@ -3,7 +3,8 @@ description test single-rom game (no parent), correct partial ckmame db exists,
 return 0
 arguments --roms-unzipped -D ../mame.db -jFvc 1-4
 file roms/1-8 1-8-ok.zip
-file roms/.ckmame.db 1-4-ok.ckmamedb-dump {}
+#file roms/.ckmame.db 1-4-ok.ckmamedb-dump {}
+file roms/.ckmame.db 1-4-ok.ckmamedb-dump <empty.ckmamedb-unzipped>
 stdout
 In game 1-4:
 game 1-4                                     : not a single file found

src/Archive.cc

@@ -92,7 +92,7 @@ Archive::Archive(ArchiveContentsPtr contents_) :
 
 Archive::Archive(ArchiveType type, const std::string &name_, filetype_t ft, where_t where_, int flags_) : Archive(std::make_shared<ArchiveContents>(type, name_, ft, where_, flags_)) { }
 
-ArchivePtr Archive::open(const ArchiveContentsPtr& contents) {
+ArchivePtr Archive::open(const ArchiveContentsPtr& contents, int flags) {
     ArchivePtr archive;
 
     if (contents->open_archive.expired()) {
@@ -128,7 +128,8 @@ ArchivePtr Archive::open(const ArchiveContentsPtr& contents) {
         //printf("# already open %s\n", archive->name.c_str());
         archive = contents->open_archive.lock();
     }
-
+
+    contents->flags |= flags & (ARCHIVE_FL_MASK | ARCHIVE_FL_HASHTYPES_MASK);
     return archive;
 }
 
@@ -333,7 +334,7 @@ ArchivePtr Archive::open(const std::string &name, filetype_t filetype, where_t w
     auto contents = ArchiveContents::by_name(filetype, archive_name);
 
     if (contents) {
-        return open(contents);
+        return open(contents, flags);
     }
 
     ArchivePtr archive;

src/Archive.h

@@ -152,7 +152,7 @@ class Archive {
     static ArchivePtr open(const std::string &name, filetype_t filetype, where_t where, int flags);
     static ArchivePtr open_toplevel(const std::string &name, filetype_t filetype, where_t where, int flags);
 
-    static ArchivePtr open(const ArchiveContentsPtr& contents);
+    static ArchivePtr open(const ArchiveContentsPtr& contents, int flags = 0);
 
     static bool read_only_mode;
 

src/ArchiveLocation.h

@@ -53,4 +53,15 @@ class ArchiveLocation { // TODO: better name
     bool operator==(const ArchiveLocation &other) const { return name == other.name && filetype == other.filetype; }
 };
 
+
+namespace std {
+template <>
+struct hash<ArchiveLocation> {
+    std::size_t operator()(const ArchiveLocation &k) const {
+        return std::hash<int>()(k.filetype) ^ std::hash<std::string>()(k.name);
+    }
+};
+}
+
+
 #endif // HAD_ARCHIVE_LOCATION_H

src/ArchiveZip.cc

@@ -64,7 +64,7 @@ bool ArchiveZip::ensure_zip() {
     if ((za = zip_open(name.c_str(), zip_flags, &err)) == nullptr) {
 	zip_error_t error;
 	zip_error_init_with_code(&error, err);
-        output.error("error %s zip archive '%s': %s", (contents->flags & ZIP_CREATE ? "creating" : "opening"), name.c_str(), zip_error_strerror(&error));
+        output.error("error %s zip archive '%s': %s", (contents->flags & ARCHIVE_FL_CREATE ? "creating" : "opening"), name.c_str(), zip_error_strerror(&error));
 	zip_error_fini(&error);
 	return false;
     }

src/CkmameCache.cc

@@ -36,11 +36,12 @@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <algorithm>
 #include <filesystem>
 
-#include "globals.h"
-#include "util.h"
-#include "Exception.h"
 #include "Dir.h"
+#include "Exception.h"
+#include "RomDB.h"
+#include "globals.h"
 #include "sighandle.h"
+#include "util.h"
 
 CkmameCachePtr ckmame_cache;
 
@@ -98,35 +99,44 @@ std::string CkmameCache::get_directory_name_for_archive(const std::string &name)
 const CkmameCache::CacheDirectory* CkmameCache::get_directory_for_archive(const std::string &name) {
     for (auto &directory : cache_directories) {
 	if (name.compare(0, directory.name.length(), directory.name) == 0 && (name.length() == directory.name.length() || name[directory.name.length()] == '/')) {
-	    if (!directory.initialized) {
-		directory.initialized = true;
-		if (!configuration.fix_romset) {
-		    std::error_code ec;
-		    if (!std::filesystem::exists(directory.name, ec)) {
-			return nullptr; /* we won't write any files, so DB would remain empty */
-		    }
-		}
-		if (!ensure_dir(directory.name, false)) {
-		    return nullptr;
-		}
-
-		try {
-		    directory.db = std::make_shared<CkmameDB>(directory.name);
-		}
-		catch (std::exception &e) {
-		    output.error_database("can't open rom directory database for '%s': %s", directory.name.c_str(), e.what());
-		    return nullptr;
-		}
-	    }
-	    return &directory;
+            directory.initialize();
+            if (directory.db) {
+                return &directory;
+            }
+            else {
+                return nullptr;
+            }
 	}
     }
 
     return nullptr;
 }
 
+void CkmameCache::CacheDirectory::initialize() {
+    if (initialized) {
+        return;
+    }
+    initialized = true;
+    if (!configuration.fix_romset) {
+        std::error_code ec;
+        if (!std::filesystem::exists(name, ec)) {
+            return; /* we won't write any files, so DB would remain empty */
+        }
+    }
+    if (!ensure_dir(name, false)) {
+        return;
+    }
 
-void CkmameCache::register_directory(const std::string &directory_name) {
+    try {
+        db = std::make_shared<CkmameDB>(name, where);
+    }
+    catch (std::exception& e) {
+        db = {};
+        output.error_database("can't open rom directory database for '%s': %s", name.c_str(), e.what());
+    }
+}
+
+void CkmameCache::register_directory(const std::string &directory_name, where_t where) {
     std::string name;
 
     if (directory_name.empty()) {
@@ -154,7 +164,9 @@ void CkmameCache::register_directory(const std::string &directory_name) {
 	}
     }
 
-    cache_directories.emplace_back(name);
+    // TODO: check that same directory isn't registerd with different where.
+
+    cache_directories.emplace_back(name, where);
 }
 
 
@@ -163,6 +175,13 @@ void CkmameCache::ensure_extra_maps() {
 	return;
     }
 
+    // TODO: this is a hack, to be replaced when we rework the delete lists.
+    // Get superfluous files in ROM directory into ckmamedb.
+    cache_directories[0].initialize();
+    if (cache_directories[0].db) {
+        cache_directories[0].db->refresh();
+    }
+
     /* Opening the archives will register them in the map. */
     extra_map_done = true;
 
@@ -343,6 +362,18 @@ void CkmameCache::used(Archive *a, size_t index) {
 	needed_delete_list->entries.push_back(fl);
 	break;
 
+    case FILE_ROMSET: {
+        if (a->name == configuration.rom_directory) {
+            superfluous_delete_list->entries.push_back(fl);
+            break;
+        }
+        auto name = a->name.substr(configuration.rom_directory.size() + 1);
+        if (!db->game_exists(name)) {
+            superfluous_delete_list->entries.push_back(fl);
+        }
+        break;
+    }
+
     case FILE_SUPERFLUOUS:
 	superfluous_delete_list->entries.push_back(fl);
 	break;
@@ -360,3 +391,16 @@ void CkmameCache::used(Archive *a, size_t index) {
 	break;
     }
 }
+std::vector<CkmameDB::FindResult> CkmameCache::find_file(filetype_t filetype, size_t detector_id, const FileData& rom) {
+    auto results = std::vector<CkmameDB::FindResult>();
+
+    for (auto& cache_directory: cache_directories) {
+        cache_directory.initialize();
+        if (cache_directory.db) {
+            cache_directory.db->find_file(filetype, detector_id, rom, results);
+        }
+    }
+
+//    printf("searching for file '%s', got %lu results\n", rom.name.c_str(), results.size());
+    return results;
+}