diff --git a/src/rgw/driver/sfs/sqlite/dbconn.cc b/src/rgw/driver/sfs/sqlite/dbconn.cc index fbfb1293401c3..1b43023cba9d7 100644 --- a/src/rgw/driver/sfs/sqlite/dbconn.cc +++ b/src/rgw/driver/sfs/sqlite/dbconn.cc @@ -13,6 +13,7 @@ */ #include "dbconn.h" +#include #include #include @@ -30,7 +31,7 @@ namespace rgw::sal::sfs::sqlite { static std::string get_temporary_db_path(CephContext* ctt) { auto rgw_sfs_path = ctt->_conf.get_val("rgw_sfs_data_path"); - auto tmp_db_name = std::string(SCHEMA_DB_NAME) + "_tmp"; + auto tmp_db_name = std::string(DB_FILENAME) + "_tmp"; auto db_path = std::filesystem::path(rgw_sfs_path) / std::string(tmp_db_name); return db_path.string(); } @@ -121,6 +122,7 @@ DBConn::DBConn(CephContext* _cct) storage_pool_mutex(), cct(_cct), profile_enabled(_cct->_conf.get_val("rgw_sfs_sqlite_profile")) { + maybe_rename_database_file(); sqlite3_config(SQLITE_CONFIG_LOG, &sqlite_error_callback, cct); // get_storage() relies on there already being an entry in the pool // for the main thread (i.e. the thread that created the DBConn). @@ -414,4 +416,65 @@ void DBConn::maybe_upgrade_metadata() { } } +void DBConn::maybe_rename_database_file() const { + if (!std::filesystem::exists(getLegacyDBPath(cct))) { + return; + } + if (std::filesystem::exists(getDBPath(cct))) { + return; + } + + lsubdout(cct, rgw_sfs, SFS_LOG_STARTUP) + << fmt::format( + "Migrating legacy database file {} -> {}", getLegacyDBPath(cct), + getDBPath(cct) + ) + << dendl; + + dbapi::sqlite::database src_db(getLegacyDBPath(cct)); + dbapi::sqlite::database dst_db(getDBPath(cct)); + auto state = + std::unique_ptr( + sqlite3_backup_init( + dst_db.connection().get(), "main", src_db.connection().get(), + "main" + ), + sqlite3_backup_finish + ); + + if (!state) { + lsubdout(cct, rgw_sfs, SFS_LOG_ERROR) + << fmt::format( + "Error opening legacy database file {} {}. Please migrate " + "s3gw.db to sfs.db manually", + getLegacyDBPath(cct), sqlite3_errmsg(dst_db.connection().get()) + ) + << dendl; + ceph_abort_msg("sfs database file migration failed"); + } + + int rc = sqlite3_backup_step(state.get(), -1); + if (rc != SQLITE_DONE) { + lsubdout(cct, rgw_sfs, SFS_LOG_ERROR) + << fmt::format( + "Error migrating legacy database file {} {}. Please migrate " + "s3gw.db to sfs.db manually", + getLegacyDBPath(cct), sqlite3_errmsg(dst_db.connection().get()) + ) + << dendl; + ceph_abort_msg("sfs database file migration failed"); + } + + std::error_code ec; // ignore errors + fs::remove(getLegacyDBPath(cct), ec); + fs::remove(getLegacyDBPath(cct) + "-wal", ec); + fs::remove(getLegacyDBPath(cct) + "-shm", ec); + + lsubdout(cct, rgw_sfs, SFS_LOG_STARTUP) + << fmt::format( + "Done migrating legacy database. Continuing startup with {}", + getDBPath(cct) + ) + << dendl; +} } // namespace rgw::sal::sfs::sqlite diff --git a/src/rgw/driver/sfs/sqlite/dbconn.h b/src/rgw/driver/sfs/sqlite/dbconn.h index b81bc515c0ad7..715b11bb70e3b 100644 --- a/src/rgw/driver/sfs/sqlite/dbconn.h +++ b/src/rgw/driver/sfs/sqlite/dbconn.h @@ -41,7 +41,9 @@ constexpr int SFS_METADATA_VERSION = 4; /// minimum required version to upgrade db. constexpr int SFS_METADATA_MIN_VERSION = 4; -constexpr std::string_view SCHEMA_DB_NAME = "s3gw.db"; +constexpr std::string_view LEGACY_DB_FILENAME = "s3gw.db"; +constexpr std::string_view DB_FILENAME = "sfs.db"; +constexpr std::string_view DB_WAL_FILENAME = "sfs.db-wal"; constexpr std::string_view USERS_TABLE = "users"; constexpr std::string_view BUCKETS_TABLE = "buckets"; @@ -293,12 +295,20 @@ class DBConn { static std::string getDBPath(CephContext* cct) { auto rgw_sfs_path = cct->_conf.get_val("rgw_sfs_data_path"); auto db_path = - std::filesystem::path(rgw_sfs_path) / std::string(SCHEMA_DB_NAME); + std::filesystem::path(rgw_sfs_path) / std::string(DB_FILENAME); + return db_path.string(); + } + + static std::string getLegacyDBPath(CephContext* cct) { + auto rgw_sfs_path = cct->_conf.get_val("rgw_sfs_data_path"); + auto db_path = + std::filesystem::path(rgw_sfs_path) / std::string(LEGACY_DB_FILENAME); return db_path.string(); } void check_metadata_is_compatible() const; void maybe_upgrade_metadata(); + void maybe_rename_database_file() const; }; using DBConnRef = std::shared_ptr; diff --git a/src/test/rgw/sfs/test_rgw_sfs_gc.cc b/src/test/rgw/sfs/test_rgw_sfs_gc.cc index 91a4322e7343e..0aa5915a4237e 100644 --- a/src/test/rgw/sfs/test_rgw_sfs_gc.cc +++ b/src/test/rgw/sfs/test_rgw_sfs_gc.cc @@ -58,9 +58,7 @@ class TestSFSGC : public ::testing::Test { } fs::path getDBFullPath(const std::string& base_dir) const { - auto db_full_name = "s3gw.db"; - auto db_full_path = fs::path(base_dir) / db_full_name; - return db_full_path; + return fs::path(base_dir) / DB_FILENAME; } fs::path getDBFullPath() const { return getDBFullPath(getTestDir()); } @@ -73,7 +71,7 @@ class TestSFSGC : public ::testing::Test { [](const std::filesystem::path& path) { return ( std::filesystem::is_regular_file(path) && - !path.filename().string().starts_with("s3gw.db") + !path.filename().string().starts_with(DB_FILENAME) ); } ); diff --git a/src/test/rgw/sfs/test_rgw_sfs_object_state_machine.cc b/src/test/rgw/sfs/test_rgw_sfs_object_state_machine.cc index 24bd7fc550a37..8bea72fe18f8d 100644 --- a/src/test/rgw/sfs/test_rgw_sfs_object_state_machine.cc +++ b/src/test/rgw/sfs/test_rgw_sfs_object_state_machine.cc @@ -78,9 +78,7 @@ class TestSFSObjectStateMachine : public ::testing::Test { } fs::path getDBFullPath(const std::string& base_dir) const { - auto db_full_name = "s3gw.db"; - auto db_full_path = fs::path(base_dir) / db_full_name; - return db_full_path; + return fs::path(base_dir) / sqlite::DB_FILENAME; } fs::path getDBFullPath() const { return getDBFullPath(getTestDir()); } diff --git a/src/test/rgw/sfs/test_rgw_sfs_sfs_bucket.cc b/src/test/rgw/sfs/test_rgw_sfs_sfs_bucket.cc index f3440333506ce..bf849a27ea4a0 100644 --- a/src/test/rgw/sfs/test_rgw_sfs_sfs_bucket.cc +++ b/src/test/rgw/sfs/test_rgw_sfs_sfs_bucket.cc @@ -17,7 +17,7 @@ /* HINT - s3gw.db will create here: /tmp/rgw_sfs_tests + Creates sqlite files in /tmp/rgw_sfs_tests */ using namespace rgw::sal::sfs::sqlite; @@ -43,9 +43,7 @@ class TestSFSBucket : public ::testing::Test { } fs::path getDBFullPath(const std::string& base_dir) const { - auto db_full_name = "s3gw.db"; - auto db_full_path = fs::path(base_dir) / db_full_name; - return db_full_path; + return fs::path(base_dir) / DB_FILENAME; } fs::path getDBFullPath() const { return getDBFullPath(getTestDir()); } diff --git a/src/test/rgw/sfs/test_rgw_sfs_sfs_user.cc b/src/test/rgw/sfs/test_rgw_sfs_sfs_user.cc index 113b3d89f7bf0..201145b52e3d3 100644 --- a/src/test/rgw/sfs/test_rgw_sfs_sfs_user.cc +++ b/src/test/rgw/sfs/test_rgw_sfs_sfs_user.cc @@ -16,7 +16,7 @@ /* HINT - s3gw.db will create here: /tmp/rgw_sfs_tests + Creates sqlite files in /tmp/rgw_sfs_tests */ using namespace rgw::sal::sfs::sqlite; @@ -73,9 +73,7 @@ class TestSFSUser : public ::testing::Test { } fs::path getDBFullPath(const std::string& base_dir) const { - auto db_full_name = "s3gw.db"; - auto db_full_path = fs::path(base_dir) / db_full_name; - return db_full_path; + return fs::path(base_dir) / DB_FILENAME; } fs::path getDBFullPath() const { return getDBFullPath(getTestDir()); } diff --git a/src/test/rgw/sfs/test_rgw_sfs_sqlite_buckets.cc b/src/test/rgw/sfs/test_rgw_sfs_sqlite_buckets.cc index b96b36912bcdc..9fe02301855cb 100644 --- a/src/test/rgw/sfs/test_rgw_sfs_sqlite_buckets.cc +++ b/src/test/rgw/sfs/test_rgw_sfs_sqlite_buckets.cc @@ -66,9 +66,7 @@ class TestSFSSQLiteBuckets : public ::testing::Test { } fs::path getDBFullPath(const std::string& base_dir) const { - auto db_full_name = "s3gw.db"; - auto db_full_path = fs::path(base_dir) / db_full_name; - return db_full_path; + return fs::path(base_dir) / DB_FILENAME; } fs::path getDBFullPath() const { return getDBFullPath(getTestDir()); } diff --git a/src/test/rgw/sfs/test_rgw_sfs_sqlite_lifecycle.cc b/src/test/rgw/sfs/test_rgw_sfs_sqlite_lifecycle.cc index 5f80ed7dc2798..2ca3382b34107 100644 --- a/src/test/rgw/sfs/test_rgw_sfs_sqlite_lifecycle.cc +++ b/src/test/rgw/sfs/test_rgw_sfs_sqlite_lifecycle.cc @@ -36,9 +36,7 @@ class TestSFSSQLiteLifecycle : public ::testing::Test { } fs::path getDBFullPath(const std::string& base_dir) const { - auto db_full_name = "s3gw.db"; - auto db_full_path = fs::path(base_dir) / db_full_name; - return db_full_path; + return fs::path(base_dir) / DB_FILENAME; } fs::path getDBFullPath() const { return getDBFullPath(getTestDir()); } diff --git a/src/test/rgw/sfs/test_rgw_sfs_sqlite_objects.cc b/src/test/rgw/sfs/test_rgw_sfs_sqlite_objects.cc index e94df71a12958..2129826a36e94 100644 --- a/src/test/rgw/sfs/test_rgw_sfs_sqlite_objects.cc +++ b/src/test/rgw/sfs/test_rgw_sfs_sqlite_objects.cc @@ -36,9 +36,7 @@ class TestSFSSQLiteObjects : public ::testing::Test { } fs::path getDBFullPath(const std::string& base_dir) const { - auto db_full_name = "s3gw.db"; - auto db_full_path = fs::path(base_dir) / db_full_name; - return db_full_path; + return fs::path(base_dir) / DB_FILENAME; } fs::path getDBFullPath() const { return getDBFullPath(getTestDir()); } diff --git a/src/test/rgw/sfs/test_rgw_sfs_sqlite_users.cc b/src/test/rgw/sfs/test_rgw_sfs_sqlite_users.cc index e19cc54a65ef5..ca0936c83f62f 100644 --- a/src/test/rgw/sfs/test_rgw_sfs_sqlite_users.cc +++ b/src/test/rgw/sfs/test_rgw_sfs_sqlite_users.cc @@ -36,9 +36,7 @@ class TestSFSSQLiteUsers : public ::testing::Test { } fs::path getDBFullPath(const std::string& base_dir) const { - auto db_full_name = "s3gw.db"; - auto db_full_path = fs::path(base_dir) / db_full_name; - return db_full_path; + return fs::path(base_dir) / DB_FILENAME; } fs::path getDBFullPath() const { return getDBFullPath(getTestDir()); } diff --git a/src/test/rgw/sfs/test_rgw_sfs_sqlite_versioned_objects.cc b/src/test/rgw/sfs/test_rgw_sfs_sqlite_versioned_objects.cc index 908ca529db49b..9827bcfd76fdf 100644 --- a/src/test/rgw/sfs/test_rgw_sfs_sqlite_versioned_objects.cc +++ b/src/test/rgw/sfs/test_rgw_sfs_sqlite_versioned_objects.cc @@ -56,9 +56,7 @@ class TestSFSSQLiteVersionedObjects : public ::testing::Test { } fs::path getDBFullPath(const std::string& base_dir) const { - auto db_full_name = "s3gw.db"; - auto db_full_path = fs::path(base_dir) / db_full_name; - return db_full_path; + return fs::path(base_dir) / DB_FILENAME; } fs::path getDBFullPath() const { return getDBFullPath(getTestDir()); } diff --git a/src/test/rgw/sfs/test_rgw_sfs_wal_checkpoint.cc b/src/test/rgw/sfs/test_rgw_sfs_wal_checkpoint.cc index 8495be007ef89..b682ed3eef5ea 100644 --- a/src/test/rgw/sfs/test_rgw_sfs_wal_checkpoint.cc +++ b/src/test/rgw/sfs/test_rgw_sfs_wal_checkpoint.cc @@ -3,6 +3,7 @@ #include +#include "driver/sfs/sqlite/dbconn.h" #include "rgw/rgw_sal_sfs.h" using namespace rgw::sal::sfs; @@ -79,7 +80,7 @@ class TestSFSWALCheckpoint : public ::testing::Test { size_t num_threads, size_t num_objects ) { std::atomic max_wal_size{0}; - fs::path wal(test_dir / "s3gw.db-wal"); + fs::path wal(test_dir / sqlite::DB_WAL_FILENAME); std::vector threads; for (size_t i = 0; i < num_threads; ++i) { @@ -124,7 +125,7 @@ TEST_F(TestSFSWALCheckpoint, confirm_wal_explosion) { // The fact that we have no size limit set means the WAL // won't be truncated even when the last writer completes, // so it should *still* be huge now. - EXPECT_EQ(fs::file_size(test_dir / "s3gw.db-wal"), max_wal_size); + EXPECT_EQ(fs::file_size(test_dir / sqlite::DB_WAL_FILENAME), max_wal_size); } // This test proves the WAL growth problem has been fixed @@ -143,5 +144,5 @@ TEST_F(TestSFSWALCheckpoint, test_wal_checkpoint) { // Once the writes are all done, the WAL should be finally // truncated to something less than 16MB. - EXPECT_LT(fs::file_size(test_dir / "s3gw.db-wal"), SIZE_1MB * 16); + EXPECT_LT(fs::file_size(test_dir / sqlite::DB_WAL_FILENAME), SIZE_1MB * 16); }