diff --git a/CMakeLists.txt b/CMakeLists.txt index 55353915c..6f6860996 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -981,7 +981,7 @@ endfunction() file_glob_recurse(FILES_NEED_FORMAT DIRECTORIES "src" "modules" "python" "test" "benchmark" PATTERNS ".*\\.(cc|cpp|h|hpp|vineyard-mod)$" - EXCLUDE_PATTERNS "(.*\\.vineyard.h$)|(modules/kv-state-cache/radix-tree/radix\.*)" + EXCLUDE_PATTERNS "(.*\\.vineyard.h$)|(modules/kv-state-cache/radix-tree/radix.*)" ) # the `memcpy.h` is borrowed from external project diff --git a/modules/kv-state-cache/ds/kv_state_cache.h b/modules/kv-state-cache/ds/kv_state_cache.h index 9f0284df9..7cb80f202 100644 --- a/modules/kv-state-cache/ds/kv_state_cache.h +++ b/modules/kv-state-cache/ds/kv_state_cache.h @@ -22,7 +22,6 @@ limitations under the License. #include "common/util/status.h" #include "kv-state-cache/ds/kv_state_cache_block.h" #include "kv-state-cache/radix-tree/radix-tree.h" -#include "kv-state-cache/strategy/LRU_strategy.h" #ifndef MODULES_KV_STATE_CACHE_DS_KV_STATE_CACHE_H_ #define MODULES_KV_STATE_CACHE_DS_KV_STATE_CACHE_H_ diff --git a/modules/kv-state-cache/ds/kv_state_cache_block.cc b/modules/kv-state-cache/ds/kv_state_cache_block.cc index f7d32b233..1441d1969 100644 --- a/modules/kv-state-cache/ds/kv_state_cache_block.cc +++ b/modules/kv-state-cache/ds/kv_state_cache_block.cc @@ -195,8 +195,8 @@ void KVStateCacheBlockBuilder::Update(const KV_STATE_WITH_LAYER& kvState, ACQUIRE_BIT_RESOURCE(this->bitmap[index / 64], index % 64); } -short KVStateCacheBlockBuilder::Split(KVStateCacheBlockBuilder* child, - int index) { +int16_t KVStateCacheBlockBuilder::Split(KVStateCacheBlockBuilder* child, + int index) { // TBD VINEYARD_ASSERT(this->layer == child->layer); int childIndex = child->FindEmptySlot(); diff --git a/modules/kv-state-cache/ds/kv_state_cache_block.h b/modules/kv-state-cache/ds/kv_state_cache_block.h index afab749c9..4870201f1 100644 --- a/modules/kv-state-cache/ds/kv_state_cache_block.h +++ b/modules/kv-state-cache/ds/kv_state_cache_block.h @@ -19,6 +19,9 @@ limitations under the License. #include #include #include +#include +#include +#include #include #include "basic/ds/tensor.h" diff --git a/modules/kv-state-cache/strategy/LRU_strategy.cc b/modules/kv-state-cache/strategy/LRU_strategy.cc deleted file mode 100644 index 87d4cdeec..000000000 --- a/modules/kv-state-cache/strategy/LRU_strategy.cc +++ /dev/null @@ -1,124 +0,0 @@ -/** Copyright 2020-2023 Alibaba Group Holding Limited. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -#include "LRU_strategy.h" -#include "common/util/logging.h" - -namespace vineyard { - -void PrintTokenList(std::vector& vector) { - std::string tokens_str = ""; - for (size_t i = 0; i < vector.size(); ++i) { - tokens_str += std::to_string(vector[i]); - } - LOG(INFO) << tokens_str; -} - -void LRUStrategy::PrintLRUList() { - LOG(INFO) << "List:"; - std::shared_ptr node = header; - while (node != nullptr) { - PrintTokenList(node->tokens); - LOG(INFO) << "->"; - node = node->next; - } -} - -LRUStrategy::LRUStrategy(int capacity) { - this->capacity = capacity; - this->header = this->tail = nullptr; - this->current_size = 0; -} - -LRUStrategy::LRUStrategy(const std::vector>& cache_list, - int capacity) { - // TBD -} - -std::shared_ptr LRUStrategy::InsertToHeader( - const std::vector& tokens, std::vector& evicted_tokens) { - if (current_size == capacity) { - std::shared_ptr remove_node = tail; // Remove(); - evicted_tokens = remove_node->tokens; - } - - std::shared_ptr cache_node = std::make_shared(); - cache_node->tokens = tokens; - - if (header == nullptr) { - header = cache_node; - tail = cache_node; - } else { - cache_node->next = header; - header->prev = cache_node; - header = cache_node; - } - - current_size++; - return cache_node; -} - -void LRUStrategy::MoveToHead(std::shared_ptr cache_node) { - if (cache_node == header) { - return; - } - - if (cache_node == tail) { - tail = tail->prev; - tail->next = nullptr; - } else { - cache_node->prev->next = cache_node->next; - cache_node->next->prev = cache_node->prev; - } - - cache_node->next = header; - header->prev = cache_node; - header = cache_node; - cache_node->prev = nullptr; -} - -std::shared_ptr LRUStrategy::Remove() { - std::shared_ptr cache_node = tail; - if (tail->prev != nullptr) { - tail->prev->next = nullptr; - tail = tail->prev; - } else { - header = nullptr; - tail = nullptr; - } - current_size--; - - LOG(INFO) << "Remove token:"; - PrintTokenList(cache_node->tokens); - return cache_node; -} - -void LRUStrategy::Remove(std::shared_ptr cache_node) { - if (cache_node == header) { - header = header->next; - header->prev = nullptr; - } else if (cache_node == tail) { - tail = tail->prev; - tail->next = nullptr; - } else { - cache_node->prev->next = cache_node->next; - cache_node->next->prev = cache_node->prev; - } - current_size--; -} - -std::shared_ptr LRUStrategy::GetHeader() { return header; } - -} // namespace vineyard diff --git a/modules/kv-state-cache/strategy/LRU_strategy.h b/modules/kv-state-cache/strategy/LRU_strategy.h deleted file mode 100644 index fcfc519e5..000000000 --- a/modules/kv-state-cache/strategy/LRU_strategy.h +++ /dev/null @@ -1,67 +0,0 @@ -/** Copyright 2020-2023 Alibaba Group Holding Limited. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -#include -#include - -#include "cache_strategy.h" - -#ifndef MODULES_LRU_STRATEGY_H_ -#define MODULES_LRU_STRATEGY_H_ - -namespace vineyard { - -struct LRUCacheNode { - std::shared_ptr next; - std::shared_ptr prev; - std::vector tokens; -}; - -class LRUStrategy : public CacheStrategy { - private: - int current_size; - - std::shared_ptr header; - - std::shared_ptr tail; - - LRUStrategy(); - - std::shared_ptr Remove(); - - ~LRUStrategy(); - - public: - LRUStrategy(int capacity); - - LRUStrategy(const std::vector>& cache_list, int capacity); - - void MoveToHead(std::shared_ptr cache_node); - - std::shared_ptr InsertToHeader( - const std::vector& tokens, std::vector& evicted_tokens); - - void Remove(std::shared_ptr cache_node); - - std::shared_ptr GetHeader(); - - int GetCapacity() { return capacity; } - - void PrintLRUList(); -}; - -} // namespace vineyard - -#endif \ No newline at end of file diff --git a/modules/kv-state-cache/strategy/cache_strategy.h b/modules/kv-state-cache/strategy/cache_strategy.h deleted file mode 100644 index 36596cd12..000000000 --- a/modules/kv-state-cache/strategy/cache_strategy.h +++ /dev/null @@ -1,31 +0,0 @@ -/** Copyright 2020-2023 Alibaba Group Holding Limited. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -#include -#include - -#ifndef MODULES_CACHE_STRATEGY_H_ -#define MODULES_CACHE_STRATEGY_H_ - -namespace vineyard { - -class CacheStrategy { - protected: - int capacity; -}; - -} // namespace vineyard - -#endif \ No newline at end of file diff --git a/modules/kv-state-cache/utils/kv_state_cache_utils.cc b/modules/kv-state-cache/utils/kv_state_cache_utils.cc index 2f238917f..34a47ef72 100644 --- a/modules/kv-state-cache/utils/kv_state_cache_utils.cc +++ b/modules/kv-state-cache/utils/kv_state_cache_utils.cc @@ -14,13 +14,16 @@ limitations under the License. */ #include +#include +#include +#include #include "client/client.h" #include "common/util/logging.h" #include "kv-state-cache/ds/kv_state_cache.h" -#include "kv_state_cache_utils.h" +#include "kv-state-cache/utils/kv_state_cache_utils.h" -using namespace vineyard; +namespace vineyard { static Client client; static std::shared_ptr kvStateCacheBuilder = nullptr; @@ -254,3 +257,5 @@ void threadFunc() { pthread_mutex_unlock(&syncMutex); } } + +} // namespace vineyard diff --git a/modules/kv-state-cache/utils/kv_state_cache_utils.h b/modules/kv-state-cache/utils/kv_state_cache_utils.h index 1b4c81e8b..a40a12d8a 100644 --- a/modules/kv-state-cache/utils/kv_state_cache_utils.h +++ b/modules/kv-state-cache/utils/kv_state_cache_utils.h @@ -13,10 +13,15 @@ See the License for the specific language governing permissions and limitations under the License. */ +#include +#include + #include "kv-state-cache/ds/kv_state_cache.h" -#ifndef MODULES_KV_STATE_CACHE_UTILS_H_ -#define MODULES_KV_STATE_CACHE_UTILS_H_ +#ifndef MODULES_KV_STATE_CACHE_UTILS_KV_STATE_CACHE_UTILS_H_ +#define MODULES_KV_STATE_CACHE_UTILS_KV_STATE_CACHE_UTILS_H_ + +namespace vineyard { void InitKVStateCache( int dimension = 10, int cacheCapacity = 10, int layer = 1, @@ -37,4 +42,6 @@ void Delete(std::vector token); void CloseKVStateCache(); -#endif \ No newline at end of file +} // namespace vineyard + +#endif // MODULES_KV_STATE_CACHE_UTILS_KV_STATE_CACHE_UTILS_H_ diff --git a/src/common/util/protocols.cc b/src/common/util/protocols.cc index 5c692c0dd..c59b726d8 100644 --- a/src/common/util/protocols.cc +++ b/src/common/util/protocols.cc @@ -2139,8 +2139,7 @@ void WriteInstanceStatusReply(const json& meta, std::string& msg) { Status ReadInstanceStatusReply(const json& root, json& meta) { CHECK_IPC_ERROR(root, command_t::INSTANCE_STATUS_REPLY); - meta = root["meta"].get(); - ; + meta = root["meta"]; return Status::OK(); } @@ -2305,7 +2304,6 @@ void WriteTryReleaseLockRequest(const std::string& key, std::string& msg) { Status ReadTryReleaseLockRequest(const json& root, std::string& key) { CHECK_IPC_ERROR(root, command_t::RELEASE_LOCK_REQUEST); key = root["key"].get(); - ; return Status::OK(); } @@ -2319,7 +2317,6 @@ void WriteTryReleaseLockReply(const bool result, std::string& msg) { Status ReadTryReleaseLockReply(const json& root, bool& result) { CHECK_IPC_ERROR(root, command_t::RELEASE_LOCK_REPLY); result = root["result"].get(); - ; return Status::OK(); } diff --git a/test/distributed_lock_test.cc b/test/distributed_lock_test.cc deleted file mode 100644 index ef81bf2aa..000000000 --- a/test/distributed_lock_test.cc +++ /dev/null @@ -1,66 +0,0 @@ -/** Copyright 2020-2023 Alibaba Group Holding Limited. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -#include -#include - -#include "client/client.h" -#include "common/util/logging.h" - -using namespace vineyard; - -int numThreads = 5; - -static int count = 0; - -void test(int i) { - std::string socket = std::string(getenv("VINEYARD_IPC_SOCKET")); - Client client; - client.Connect(socket); - - bool result; - std::string actural_key_of_lock; - - LOG(INFO) << "Thread: " << i << " try to acquire lock: test"; - client.TryAcquireLock("test", result, actural_key_of_lock); - LOG(INFO) << "Thread: " << i - << " acquire Lock: " << (result == true ? "success" : "fail") - << ", key is :" + actural_key_of_lock; - - if (result) { - count++; - LOG(INFO) << "count: " << count; - - sleep(3); - - LOG(INFO) << "Thread: " << i << " try to release lock: test"; - client.TryReleaseLock(actural_key_of_lock, result); - LOG(INFO) << "Thread: " << i - << " release Lock: " << (result == true ? "success" : "fail"); - } -} - -int main() { - std::thread threads[numThreads]; - for (int i = 0; i < numThreads; i++) { - threads[i] = std::thread(test, i); - } - - for (int i = 0; i < numThreads; i++) { - threads[i].join(); - } - - return 0; -} \ No newline at end of file diff --git a/test/kv_state_cache_multi_test.cc b/test/kv_state_cache_multi_test.cc index dad9764d9..cb67f0ffa 100644 --- a/test/kv_state_cache_multi_test.cc +++ b/test/kv_state_cache_multi_test.cc @@ -58,12 +58,14 @@ int main(int argc, char** argv) { } char* socket_str[2]; - socket_str[0] = (char*) malloc(sockets[0].length() + 1); - socket_str[1] = (char*) malloc(sockets[1].length() + 1); + socket_str[0] = + (char*) malloc(sockets[0].length() + 1); // NOLINT(readability/casting) + socket_str[1] = + (char*) malloc(sockets[1].length() + 1); // NOLINT(readability/casting) memset(socket_str[0], 0, sockets[0].length() + 1); memset(socket_str[1], 0, sockets[1].length() + 1); - strcpy(socket_str[0], sockets[0].c_str()); - strcpy(socket_str[1], sockets[1].c_str()); + strcpy(socket_str[0], sockets[0].c_str()); // NOLINT(readability/casting) + strcpy(socket_str[1], sockets[1].c_str()); // NOLINT(readability/casting) char* args_1[] = {process_name, socket_str[0], arg_0, token_sequence_1, token_sequence_2, token_sequence_3, @@ -79,9 +81,14 @@ int main(int argc, char** argv) { int status; waitpid(pids[i], &status, 0); if (WIFEXITED(status) && WEXITSTATUS(status) != 0) { + free(socket_str[0]); + free(socket_str[1]); return 1; } } + free(socket_str[0]); + free(socket_str[1]); + return 0; } diff --git a/test/kv_state_cache_radix_tree_test.cc b/test/kv_state_cache_radix_tree_test.cc index 2c06f8c54..76fd7f307 100644 --- a/test/kv_state_cache_radix_tree_test.cc +++ b/test/kv_state_cache_radix_tree_test.cc @@ -22,7 +22,7 @@ limitations under the License. #include "common/util/logging.h" #include "kv-state-cache/utils/kv_state_cache_utils.h" -using namespace vineyard; +using namespace vineyard; // NOLINT(build/namespaces) void print_tokens(const std::vector& tokens) { std::string tokens_str = ""; @@ -188,4 +188,4 @@ int main() { LOG(INFO) << "Start to test radix tree split..."; radix_tree_split(); LOG(INFO) << "Finish radix tree split test!"; -} \ No newline at end of file +} diff --git a/test/kv_state_cache_test.cc b/test/kv_state_cache_test.cc index 70ac8be25..e806981e0 100644 --- a/test/kv_state_cache_test.cc +++ b/test/kv_state_cache_test.cc @@ -22,7 +22,7 @@ limitations under the License. #include "common/util/logging.h" #include "kv-state-cache/utils/kv_state_cache_utils.h" -using namespace vineyard; +using namespace vineyard; // NOLINT(build/namespaces) int dimension = 10; int capacity = 20; @@ -80,9 +80,10 @@ generate_kv_state(int token) { std::vector key_state; std::vector value_state; for (int i = 0; i < dimension; ++i) { - key_state.push_back(((double) token) / dimension * (i + 1) + + key_state.push_back((static_cast(token)) / dimension * (i + 1) + currentLayer * 10); - value_state.push_back(((double) token) / dimension * (i + 1) * 2 + + value_state.push_back((static_cast(token)) / dimension * (i + 1) * + 2 + currentLayer * 10); } @@ -102,21 +103,24 @@ void check_kv_state( VINEYARD_ASSERT(iter->second.second.size() == (size_t) dimension); for (int i = 0; i < dimension; ++i) { if (iter->second.first[i] != - ((double) token) / dimension * (i + 1) + iter->first * 10) { + (static_cast(token)) / dimension * (i + 1) + + iter->first * 10) { LOG(INFO) << "token:" << token << " dimension" << dimension << " layer:" << iter->first; LOG(INFO) << "key_state[" << i << "]: " << iter->second.first[i] << ". But is should be " - << ((double) token) / dimension * (i + 1) + iter->first * 10; + << (static_cast(token)) / dimension * (i + 1) + + iter->first * 10; throw std::runtime_error("key_state error!"); } if (iter->second.second[i] != - ((double) token) / dimension * (i + 1) * 2 + iter->first * 10) { + (static_cast(token)) / dimension * (i + 1) * 2 + + iter->first * 10) { LOG(INFO) << "token:" << token << " dimension" << dimension << " layer:" << iter->first; LOG(INFO) << "value_state[" << i << "]: " << iter->second.second[i] << ". But is should be " - << ((double) token) / dimension * (i + 1) * 2 + + << (static_cast(token)) / dimension * (i + 1) * 2 + iter->first * 10; throw std::runtime_error("value_state error!"); } diff --git a/test/rax_diff_test.cc b/test/rax_diff_test.cc deleted file mode 100644 index ce1dcf1bf..000000000 --- a/test/rax_diff_test.cc +++ /dev/null @@ -1,101 +0,0 @@ -/** Copyright 2020-2023 Alibaba Group Holding Limited. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -#include -#include -#include -#include "kv-state-cache/radix-tree/radix.h" - -int key_1[] = {1, 2}; -int key_2[] = {1, 3}; -int key_3[] = {1, 4}; -int key_4[] = {1, 3, 1}; -int key_5[] = {1, 3, 2}; - -void insert(rax* rt, int* key, int len) { - for (int i = 1; i <= len; i++) { - raxInsert(rt, key, i, NULL, NULL); - } -} - -int main(int argc, char** argv) { - rax* rt_1 = raxNew(); - rax* rt_2 = raxNew(); - - int max_node = argc > 1 ? atoi(argv[1]) : 3; - - // raxInsert(rt_2, key_1, 2, NULL, NULL); - // raxInsert(rt_2, key_2, 2, NULL, NULL); - - // raxInsert(rt_1, key_3, 2, NULL, NULL); - // raxInsert(rt_1, key_4, 3, NULL, NULL); - // raxInsert(rt_1, key_5, 3, NULL, NULL); - - insert(rt_1, key_3, 2); - insert(rt_1, key_4, 3); - - sleep(1); - - insert(rt_2, key_1, 2); - insert(rt_2, key_2, 2); - - sleep(1); - - insert(rt_1, key_5, 3); - - raxShow(rt_1); - printf("==============================\n"); - raxShow(rt_2); - printf("==============================\n"); - - testIteRax(rt_1); - printf("==============================\n"); - testIteRax(rt_2); - printf("==============================\n"); - - std::vector> evicted_tokens; - std::set> insert_tokens; - mergeTree(rt_1, rt_2, evicted_tokens, insert_tokens, max_node); - - printf("evicted_tokens:\n"); - for (size_t i = 0; i < evicted_tokens.size(); i++) { - for (size_t j = 0; j < evicted_tokens[i].size(); j++) { - printf("%d ", evicted_tokens[i][j]); - } - printf("\n"); - } - for (size_t i = 0; i < evicted_tokens.size(); i++) { - // void* tree_data; - raxRemove(rt_1, evicted_tokens[i].data(), evicted_tokens[i].size(), NULL, - false); - } - - for (auto it = insert_tokens.begin(); it != insert_tokens.end(); it++) { - raxInsert(rt_1, const_cast(it->data()), it->size(), NULL, NULL, - false); - } - - raxShow(rt_1); - printf("==============================\n"); - raxShow(rt_2); - printf("==============================\n"); - - testIteRax(rt_1); - printf("==============================\n"); - testIteRax(rt_2); - printf("==============================\n"); - - return 0; -} \ No newline at end of file