From bcc7c71b957db9e8193bc4c152c52e810594f5cd Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Fri, 29 Sep 2023 12:43:23 -0700 Subject: [PATCH 01/16] possible fix for rocksdb centos compile fail --- src/storage/rocksdb.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index 8aff86959..5f5e53c4c 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -7,6 +7,8 @@ #include "../cluster.h" #include "rocksdbfactor_internal.h" +template class std::basic_string; + static const char keyprefix[] = INTERNAL_KEY_PREFIX; rocksdb::Options DefaultRocksDBOptions(); From d6aeaf99bfece5c1c7a183c0c0020684faf156c3 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Fri, 29 Sep 2023 15:16:03 -0700 Subject: [PATCH 02/16] fix alpine compile --- src/Makefile | 1 + src/server.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index a3cd741f4..fc58b64f5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -358,6 +358,7 @@ endif OS := $(shell cat /etc/os-release | grep ID= | head -n 1 | cut -d'=' -f2) ifeq ($(OS),alpine) FINAL_CXXFLAGS+=-DUNW_LOCAL_ONLY + FINAL_CXXFLAGS+=-DALPINE FINAL_LIBS += -lunwind endif diff --git a/src/server.h b/src/server.h index 7983c6dc7..57b4253f8 100644 --- a/src/server.h +++ b/src/server.h @@ -3923,7 +3923,7 @@ void updateActiveReplicaMastersFromRsi(rdbSaveInfo *rsi); uint64_t getMvccTstamp(); void incrementMvccTstamp(); -#if __GNUC__ >= 7 && !defined(NO_DEPRECATE_FREE) +#if __GNUC__ >= 7 && !defined(NO_DEPRECATE_FREE) && !defined(ALPINE) [[deprecated]] void *calloc(size_t count, size_t size) noexcept; [[deprecated]] From 98ff535583eb7038e069b9f7ba36be482b87aed9 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Tue, 3 Oct 2023 10:19:53 -0700 Subject: [PATCH 03/16] fix comment --- src/readwritelock.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/readwritelock.h b/src/readwritelock.h index 05385e4e7..c14e9ea01 100644 --- a/src/readwritelock.h +++ b/src/readwritelock.h @@ -42,9 +42,9 @@ class readWriteLock { while (m_readCount > 0) m_cv.wait(rm); if (exclusive) { - /* Another thread might have the write lock while we have the read lock - but won't be able to release it until they can acquire the read lock - so release the read lock and try again instead of waiting to avoid deadlock */ + /* Another thread might have the write lock while we have the internal lock + but won't be able to release it until they can acquire the internal lock + so release the internal lock and try again instead of waiting to avoid deadlock */ while(!m_writeLock.try_lock()) m_cv.wait(rm); } From 93496e9e3a3d4c6d7a5ef17a579775a39c093221 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Wed, 4 Oct 2023 16:49:32 -0700 Subject: [PATCH 04/16] fix hashslot enumerate --- src/storage/rocksdb.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index 5f5e53c4c..add36effc 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -27,7 +27,7 @@ bool FInternalKey(const char *key, size_t cch) std::string getPrefix(unsigned int hashslot) { char *hash_char = (char *)&hashslot; - return std::string(hash_char + (sizeof(unsigned int) - 2), 2); + return std::string(hash_char, sizeof(unsigned int)); } std::string prefixKey(const char *key, size_t cchKey) @@ -186,7 +186,7 @@ bool RocksDBStorageProvider::enumerate(callback fn) const if (FInternalKey(it->key().data(), it->key().size())) continue; ++count; - bool fContinue = fn(it->key().data()+2, it->key().size()-2, it->value().data(), it->value().size()); + bool fContinue = fn(it->key().data()+sizeof(unsigned int), it->key().size()-sizeof(unsigned int), it->value().data(), it->value().size()); if (!fContinue) break; } @@ -205,15 +205,17 @@ bool RocksDBStorageProvider::enumerate_hashslot(callback fn, unsigned int hashsl std::string prefix = getPrefix(hashslot); std::unique_ptr it = std::unique_ptr(m_spdb->NewIterator(ReadOptions(), m_spcolfamily.get())); size_t count = 0; - for (it->Seek(prefix.c_str()); it->Valid(); it->Next()) { + for (it->Seek(prefix); it->Valid(); it->Next()) { if (FInternalKey(it->key().data(), it->key().size())) continue; - if (strncmp(it->key().data(),prefix.c_str(),2) != 0) + if (strncmp(it->key().data(),prefix.c_str(),sizeof(unsigned int)) != 0) break; ++count; - bool fContinue = fn(it->key().data()+2, it->key().size()-2, it->value().data(), it->value().size()); + bool fContinue = fn(it->key().data()+sizeof(unsigned int), it->key().size()-sizeof(unsigned int), it->value().data(), it->value().size()); if (!fContinue) break; + serverLog(LL_WARNING, "%d %d", hashslot, *it->key().data()); + serverLog(LL_WARNING, "Found key %d of %d with len %d: %.*s", count, g_pserver->cluster->slots_keys_count[hashslot], (int)it->key().size()-sizeof(unsigned int), (int)it->key().size()-sizeof(unsigned int), it->key().data()+sizeof(unsigned int)); } bool full_iter = !it->Valid() || (strncmp(it->key().data(),prefix.c_str(),2) != 0); if (full_iter && count != g_pserver->cluster->slots_keys_count[hashslot]) @@ -280,7 +282,7 @@ std::vector RocksDBStorageProvider::getEvictionCandidates(unsigned for (it->Seek(randomHashSlot()); it->Valid() && result.size() < count; it->Next()) { if (FInternalKey(it->key().data(), it->key().size())) continue; - result.emplace_back(it->key().data() + 2, it->key().size() - 2); + result.emplace_back(it->key().data() + sizeof(unsigned int), it->key().size() - sizeof(unsigned int)); } } else { std::unique_ptr it = std::unique_ptr(m_spdb->NewIterator(ReadOptions(), m_spexpirecolfamily.get())); From 2fd6410aa568eda9e893beeb5c09af91d0cce6aa Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Wed, 4 Oct 2023 16:53:38 -0700 Subject: [PATCH 05/16] also log prefix --- src/storage/rocksdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index add36effc..1e8f9dc07 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -214,7 +214,7 @@ bool RocksDBStorageProvider::enumerate_hashslot(callback fn, unsigned int hashsl bool fContinue = fn(it->key().data()+sizeof(unsigned int), it->key().size()-sizeof(unsigned int), it->value().data(), it->value().size()); if (!fContinue) break; - serverLog(LL_WARNING, "%d %d", hashslot, *it->key().data()); + serverLog(LL_WARNING, "%d %d %d", hashslot, *it->key().data(), *prefix.c_str()); serverLog(LL_WARNING, "Found key %d of %d with len %d: %.*s", count, g_pserver->cluster->slots_keys_count[hashslot], (int)it->key().size()-sizeof(unsigned int), (int)it->key().size()-sizeof(unsigned int), it->key().data()+sizeof(unsigned int)); } bool full_iter = !it->Valid() || (strncmp(it->key().data(),prefix.c_str(),2) != 0); From 8a1d648f155da3ca772247a3daa360cad0caeef2 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Wed, 4 Oct 2023 16:57:00 -0700 Subject: [PATCH 06/16] cast --- src/storage/rocksdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index 1e8f9dc07..fdfb94dff 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -214,7 +214,7 @@ bool RocksDBStorageProvider::enumerate_hashslot(callback fn, unsigned int hashsl bool fContinue = fn(it->key().data()+sizeof(unsigned int), it->key().size()-sizeof(unsigned int), it->value().data(), it->value().size()); if (!fContinue) break; - serverLog(LL_WARNING, "%d %d %d", hashslot, *it->key().data(), *prefix.c_str()); + serverLog(LL_WARNING, "%d %d %d", hashslot, (unsigned int)*it->key().data(), (unsigned int)*prefix.c_str()); serverLog(LL_WARNING, "Found key %d of %d with len %d: %.*s", count, g_pserver->cluster->slots_keys_count[hashslot], (int)it->key().size()-sizeof(unsigned int), (int)it->key().size()-sizeof(unsigned int), it->key().data()+sizeof(unsigned int)); } bool full_iter = !it->Valid() || (strncmp(it->key().data(),prefix.c_str(),2) != 0); From 756ab91830533c0d4b6d065c12ea41a81b925d59 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Wed, 4 Oct 2023 17:01:06 -0700 Subject: [PATCH 07/16] stored full unsigned int anyways --- src/storage/rocksdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index fdfb94dff..77440ebfb 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -208,7 +208,7 @@ bool RocksDBStorageProvider::enumerate_hashslot(callback fn, unsigned int hashsl for (it->Seek(prefix); it->Valid(); it->Next()) { if (FInternalKey(it->key().data(), it->key().size())) continue; - if (strncmp(it->key().data(),prefix.c_str(),sizeof(unsigned int)) != 0) + if ((unsigned int)*it->key().data() != hashslot) break; ++count; bool fContinue = fn(it->key().data()+sizeof(unsigned int), it->key().size()-sizeof(unsigned int), it->value().data(), it->value().size()); From ae003aa8bc04e44f0f9356b87b7f0c55e24631a0 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Wed, 4 Oct 2023 17:42:32 -0700 Subject: [PATCH 08/16] messed up endians --- src/storage/rocksdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index 77440ebfb..a60c7c1a2 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -208,7 +208,7 @@ bool RocksDBStorageProvider::enumerate_hashslot(callback fn, unsigned int hashsl for (it->Seek(prefix); it->Valid(); it->Next()) { if (FInternalKey(it->key().data(), it->key().size())) continue; - if ((unsigned int)*it->key().data() != hashslot) + if (*(unsigned int *)it->key().data() != hashslot) break; ++count; bool fContinue = fn(it->key().data()+sizeof(unsigned int), it->key().size()-sizeof(unsigned int), it->value().data(), it->value().size()); From 5249668debdb02e3d10840be051fa072841be875 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Wed, 4 Oct 2023 17:48:42 -0700 Subject: [PATCH 09/16] remove debug logs --- src/storage/rocksdb.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index a60c7c1a2..1932c4b3b 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -214,10 +214,8 @@ bool RocksDBStorageProvider::enumerate_hashslot(callback fn, unsigned int hashsl bool fContinue = fn(it->key().data()+sizeof(unsigned int), it->key().size()-sizeof(unsigned int), it->value().data(), it->value().size()); if (!fContinue) break; - serverLog(LL_WARNING, "%d %d %d", hashslot, (unsigned int)*it->key().data(), (unsigned int)*prefix.c_str()); - serverLog(LL_WARNING, "Found key %d of %d with len %d: %.*s", count, g_pserver->cluster->slots_keys_count[hashslot], (int)it->key().size()-sizeof(unsigned int), (int)it->key().size()-sizeof(unsigned int), it->key().data()+sizeof(unsigned int)); } - bool full_iter = !it->Valid() || (strncmp(it->key().data(),prefix.c_str(),2) != 0); + bool full_iter = !it->Valid() || (*(unsigned int *)it->key().data() != hashslot); if (full_iter && count != g_pserver->cluster->slots_keys_count[hashslot]) { printf("WARNING: rocksdb hashslot count mismatch"); From 4a3125b7218b802ac8906423a932fe4059fa029f Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Wed, 4 Oct 2023 17:50:46 -0700 Subject: [PATCH 10/16] update prefix extractor --- src/storage/rocksdbfactory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/rocksdbfactory.cpp b/src/storage/rocksdbfactory.cpp index 7087a0136..f0083d8d7 100644 --- a/src/storage/rocksdbfactory.cpp +++ b/src/storage/rocksdbfactory.cpp @@ -72,7 +72,7 @@ RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, cons rocksdb::DB *db = nullptr; auto options = RocksDbOptions(); - options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(2)); + options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(sizeof(unsigned int))); for (int idb = 0; idb < dbnum; ++idb) { From 0190424a8e11ba9b9bc34b5a8af6f71161a93d47 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Wed, 4 Oct 2023 22:39:10 -0700 Subject: [PATCH 11/16] update alpine docker to latest version --- pkg/docker/Dockerfile_Alpine | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/docker/Dockerfile_Alpine b/pkg/docker/Dockerfile_Alpine index 7ff44a1df..2787eda0b 100644 --- a/pkg/docker/Dockerfile_Alpine +++ b/pkg/docker/Dockerfile_Alpine @@ -1,4 +1,4 @@ -FROM alpine:3.12 +FROM alpine:3.18 # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added RUN addgroup -S -g 1000 keydb && adduser -S -G keydb -u 999 keydb RUN mkdir -p /etc/keydb From 348c78c57f897f50abee2102005555fed9009149 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Thu, 5 Oct 2023 09:40:52 -0700 Subject: [PATCH 12/16] missed a 2 --- src/storage/rocksdbfactory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/rocksdbfactory.cpp b/src/storage/rocksdbfactory.cpp index f0083d8d7..af78e8361 100644 --- a/src/storage/rocksdbfactory.cpp +++ b/src/storage/rocksdbfactory.cpp @@ -196,7 +196,7 @@ IStorage *RocksDBStorageFactory::create(int db, key_load_iterator iter, void *pr printf("\tDatabase %d was not shutdown cleanly, recomputing metrics\n", db); fFirstRealKey = false; if (iter != nullptr) - iter(it->key().data()+2, it->key().size()-2, privdata); + iter(it->key().data()+sizeof(unsigned int), it->key().size()-sizeof(unsigned int), privdata); ++count; } } From e771f081b7028e1323505f95f95d1a8ba6a2a374 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Fri, 6 Oct 2023 11:46:13 -0700 Subject: [PATCH 13/16] update prefix to uint16_t --- src/storage/rocksdb.cpp | 15 ++++++++------- src/storage/rocksdb.h | 2 ++ src/storage/rocksdbfactory.cpp | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index 1932c4b3b..e98408dcf 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -26,8 +26,9 @@ bool FInternalKey(const char *key, size_t cch) std::string getPrefix(unsigned int hashslot) { - char *hash_char = (char *)&hashslot; - return std::string(hash_char, sizeof(unsigned int)); + HASHSLOT_PREFIX_TYPE slot = (HASHSLOT_PREFIX_TYPE)hashslot; + char *hash_char = (char *)&slot; + return std::string(hash_char, HASHSLOT_PREFIX_BYTES); } std::string prefixKey(const char *key, size_t cchKey) @@ -186,7 +187,7 @@ bool RocksDBStorageProvider::enumerate(callback fn) const if (FInternalKey(it->key().data(), it->key().size())) continue; ++count; - bool fContinue = fn(it->key().data()+sizeof(unsigned int), it->key().size()-sizeof(unsigned int), it->value().data(), it->value().size()); + bool fContinue = fn(it->key().data()+HASHSLOT_PREFIX_BYTES, it->key().size()-HASHSLOT_PREFIX_BYTES, it->value().data(), it->value().size()); if (!fContinue) break; } @@ -208,14 +209,14 @@ bool RocksDBStorageProvider::enumerate_hashslot(callback fn, unsigned int hashsl for (it->Seek(prefix); it->Valid(); it->Next()) { if (FInternalKey(it->key().data(), it->key().size())) continue; - if (*(unsigned int *)it->key().data() != hashslot) + if (*(HASHSLOT_PREFIX_TYPE *)it->key().data() != hashslot) break; ++count; - bool fContinue = fn(it->key().data()+sizeof(unsigned int), it->key().size()-sizeof(unsigned int), it->value().data(), it->value().size()); + bool fContinue = fn(it->key().data()+HASHSLOT_PREFIX_BYTES, it->key().size()-HASHSLOT_PREFIX_BYTES, it->value().data(), it->value().size()); if (!fContinue) break; } - bool full_iter = !it->Valid() || (*(unsigned int *)it->key().data() != hashslot); + bool full_iter = !it->Valid() || (*(HASHSLOT_PREFIX_TYPE *)it->key().data() != hashslot); if (full_iter && count != g_pserver->cluster->slots_keys_count[hashslot]) { printf("WARNING: rocksdb hashslot count mismatch"); @@ -280,7 +281,7 @@ std::vector RocksDBStorageProvider::getEvictionCandidates(unsigned for (it->Seek(randomHashSlot()); it->Valid() && result.size() < count; it->Next()) { if (FInternalKey(it->key().data(), it->key().size())) continue; - result.emplace_back(it->key().data() + sizeof(unsigned int), it->key().size() - sizeof(unsigned int)); + result.emplace_back(it->key().data() + HASHSLOT_PREFIX_BYTES, it->key().size() - HASHSLOT_PREFIX_BYTES); } } else { std::unique_ptr it = std::unique_ptr(m_spdb->NewIterator(ReadOptions(), m_spexpirecolfamily.get())); diff --git a/src/storage/rocksdb.h b/src/storage/rocksdb.h index dd6196a55..3cf084115 100644 --- a/src/storage/rocksdb.h +++ b/src/storage/rocksdb.h @@ -7,6 +7,8 @@ #include "../fastlock.h" #define INTERNAL_KEY_PREFIX "\x00\x04\x03\x00\x05\x02\x04" +#define HASHSLOT_PREFIX_TYPE uint16_t +#define HASHSLOT_PREFIX_BYTES sizeof(HASHSLOT_PREFIX_TYPE) static const char count_key[] = INTERNAL_KEY_PREFIX "__keydb__count\1"; static const char version_key[] = INTERNAL_KEY_PREFIX "__keydb__version\1"; static const char meta_key[] = INTERNAL_KEY_PREFIX "__keydb__metadata\1"; diff --git a/src/storage/rocksdbfactory.cpp b/src/storage/rocksdbfactory.cpp index af78e8361..df5c35166 100644 --- a/src/storage/rocksdbfactory.cpp +++ b/src/storage/rocksdbfactory.cpp @@ -72,7 +72,7 @@ RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, cons rocksdb::DB *db = nullptr; auto options = RocksDbOptions(); - options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(sizeof(unsigned int))); + options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(HASHSLOT_PREFIX_BYTES)); for (int idb = 0; idb < dbnum; ++idb) { @@ -196,7 +196,7 @@ IStorage *RocksDBStorageFactory::create(int db, key_load_iterator iter, void *pr printf("\tDatabase %d was not shutdown cleanly, recomputing metrics\n", db); fFirstRealKey = false; if (iter != nullptr) - iter(it->key().data()+sizeof(unsigned int), it->key().size()-sizeof(unsigned int), privdata); + iter(it->key().data()+HASHSLOT_PREFIX_BYTES, it->key().size()-HASHSLOT_PREFIX_BYTES, privdata); ++count; } } From 4c3fc7e2867cbbde6efc232ee416fe9cf36835b3 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Fri, 6 Oct 2023 12:23:25 -0700 Subject: [PATCH 14/16] endian crosscompat --- src/storage/rocksdb.cpp | 6 +++--- src/storage/rocksdb.h | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index e98408dcf..fe2e7cbf5 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -26,7 +26,7 @@ bool FInternalKey(const char *key, size_t cch) std::string getPrefix(unsigned int hashslot) { - HASHSLOT_PREFIX_TYPE slot = (HASHSLOT_PREFIX_TYPE)hashslot; + HASHSLOT_PREFIX_TYPE slot = HASHSLOT_PREFIX_ENDIAN((HASHSLOT_PREFIX_TYPE)hashslot); char *hash_char = (char *)&slot; return std::string(hash_char, HASHSLOT_PREFIX_BYTES); } @@ -209,14 +209,14 @@ bool RocksDBStorageProvider::enumerate_hashslot(callback fn, unsigned int hashsl for (it->Seek(prefix); it->Valid(); it->Next()) { if (FInternalKey(it->key().data(), it->key().size())) continue; - if (*(HASHSLOT_PREFIX_TYPE *)it->key().data() != hashslot) + if (HASHSLOT_PREFIX_RECOVER(*(HASHSLOT_PREFIX_TYPE *)it->key().data()) != hashslot) break; ++count; bool fContinue = fn(it->key().data()+HASHSLOT_PREFIX_BYTES, it->key().size()-HASHSLOT_PREFIX_BYTES, it->value().data(), it->value().size()); if (!fContinue) break; } - bool full_iter = !it->Valid() || (*(HASHSLOT_PREFIX_TYPE *)it->key().data() != hashslot); + bool full_iter = !it->Valid() || (HASHSLOT_PREFIX_RECOVER(*(HASHSLOT_PREFIX_TYPE *)it->key().data()) != hashslot); if (full_iter && count != g_pserver->cluster->slots_keys_count[hashslot]) { printf("WARNING: rocksdb hashslot count mismatch"); diff --git a/src/storage/rocksdb.h b/src/storage/rocksdb.h index 3cf084115..094e085b4 100644 --- a/src/storage/rocksdb.h +++ b/src/storage/rocksdb.h @@ -4,11 +4,14 @@ #include "../IStorage.h" #include #include +#include #include "../fastlock.h" #define INTERNAL_KEY_PREFIX "\x00\x04\x03\x00\x05\x02\x04" #define HASHSLOT_PREFIX_TYPE uint16_t #define HASHSLOT_PREFIX_BYTES sizeof(HASHSLOT_PREFIX_TYPE) +#define HASHSLOT_PREFIX_ENDIAN htole16 +#define HASHSLOT_PREFIX_RECOVER le16toh static const char count_key[] = INTERNAL_KEY_PREFIX "__keydb__count\1"; static const char version_key[] = INTERNAL_KEY_PREFIX "__keydb__version\1"; static const char meta_key[] = INTERNAL_KEY_PREFIX "__keydb__metadata\1"; From 1f976f8e327701262ca23e0158614532181b1e40 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Fri, 6 Oct 2023 13:09:07 -0700 Subject: [PATCH 15/16] fix mac build, use big endian for expire prefix --- src/storage/rocksdb.cpp | 7 +++++-- src/storage/rocksdb.h | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index fe2e7cbf5..eb6a7a7fa 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "../server.h" #include "../cluster.h" #include "rocksdbfactor_internal.h" @@ -230,7 +231,8 @@ void RocksDBStorageProvider::setExpire(const char *key, size_t cchKey, long long { rocksdb::Status status; std::unique_lock l(m_lock); - std::string prefix((const char *)&expire,sizeof(long long)); + long long beExpire = htonll(expire); + std::string prefix((const char *)&beExpire,sizeof(long long)); std::string strKey(key, cchKey); if (m_spbatch != nullptr) status = m_spbatch->Put(m_spexpirecolfamily.get(), rocksdb::Slice(prefix + strKey), rocksdb::Slice(strKey)); @@ -244,7 +246,8 @@ void RocksDBStorageProvider::removeExpire(const char *key, size_t cchKey, long l { rocksdb::Status status; std::unique_lock l(m_lock); - std::string prefix((const char *)&expire,sizeof(long long)); + long long beExpire = htonll(expire); + std::string prefix((const char *)&beExpire,sizeof(long long)); std::string strKey(key, cchKey); std::string fullKey = prefix + strKey; if (!FExpireExists(fullKey)) diff --git a/src/storage/rocksdb.h b/src/storage/rocksdb.h index 094e085b4..564c9a432 100644 --- a/src/storage/rocksdb.h +++ b/src/storage/rocksdb.h @@ -4,7 +4,13 @@ #include "../IStorage.h" #include #include +#ifdef __APPLE__ +#include +#define htole16(x) OSSwapHostToLittleInt16(x) +#define le16toh(x) OSSwapLittleToHostInt16(x) +#else #include +#endif #include "../fastlock.h" #define INTERNAL_KEY_PREFIX "\x00\x04\x03\x00\x05\x02\x04" From 0449579db0f5160e063e8ef5dfbe1c80d488d397 Mon Sep 17 00:00:00 2001 From: Malavan Sotheeswaran Date: Fri, 6 Oct 2023 13:18:02 -0700 Subject: [PATCH 16/16] use endian.h for expires too --- src/storage/rocksdb.cpp | 5 ++--- src/storage/rocksdb.h | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index eb6a7a7fa..79fa88f8c 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include "../server.h" #include "../cluster.h" #include "rocksdbfactor_internal.h" @@ -231,7 +230,7 @@ void RocksDBStorageProvider::setExpire(const char *key, size_t cchKey, long long { rocksdb::Status status; std::unique_lock l(m_lock); - long long beExpire = htonll(expire); + long long beExpire = htobe64(expire); std::string prefix((const char *)&beExpire,sizeof(long long)); std::string strKey(key, cchKey); if (m_spbatch != nullptr) @@ -246,7 +245,7 @@ void RocksDBStorageProvider::removeExpire(const char *key, size_t cchKey, long l { rocksdb::Status status; std::unique_lock l(m_lock); - long long beExpire = htonll(expire); + long long beExpire = htobe64(expire); std::string prefix((const char *)&beExpire,sizeof(long long)); std::string strKey(key, cchKey); std::string fullKey = prefix + strKey; diff --git a/src/storage/rocksdb.h b/src/storage/rocksdb.h index 564c9a432..5ded5b605 100644 --- a/src/storage/rocksdb.h +++ b/src/storage/rocksdb.h @@ -6,8 +6,9 @@ #include #ifdef __APPLE__ #include -#define htole16(x) OSSwapHostToLittleInt16(x) -#define le16toh(x) OSSwapLittleToHostInt16(x) +#define htole16 OSSwapHostToLittleInt16 +#define le16toh OSSwapLittleToHostInt16 +#define htobe64 OSSwapHostToBigInt64 #else #include #endif