From f5226eb5d7b4070a80d96aaa0ecdeca4a0e25336 Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Sat, 23 May 2020 00:38:22 +0530 Subject: [PATCH 01/16] Empty Commit --- submodules/fastlog | 2 +- submodules/nlohmann_json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/submodules/fastlog b/submodules/fastlog index 4ac3223..a33aafc 160000 --- a/submodules/fastlog +++ b/submodules/fastlog @@ -1 +1 @@ -Subproject commit 4ac3223876b80d9e8ace5fcfcbffaaa2629bfd7f +Subproject commit a33aafc6cfe74d83ed527ef9a4aa1f6bc53cef17 diff --git a/submodules/nlohmann_json b/submodules/nlohmann_json index ea60d40..456478b 160000 --- a/submodules/nlohmann_json +++ b/submodules/nlohmann_json @@ -1 +1 @@ -Subproject commit ea60d40f4a60a47d3be9560d8f7bc37c163fe47b +Subproject commit 456478b3c50d60100dbb1fb9bc931f370a2c1c28 From f5318d3ac59067e47fbedbc5ea82d0c142121327 Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Sun, 24 May 2020 18:21:06 +0530 Subject: [PATCH 02/16] merged #13 from @viveknigam3003 --- submodules/fastlog | 2 +- submodules/nlohmann_json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/submodules/fastlog b/submodules/fastlog index a33aafc..4ac3223 160000 --- a/submodules/fastlog +++ b/submodules/fastlog @@ -1 +1 @@ -Subproject commit a33aafc6cfe74d83ed527ef9a4aa1f6bc53cef17 +Subproject commit 4ac3223876b80d9e8ace5fcfcbffaaa2629bfd7f diff --git a/submodules/nlohmann_json b/submodules/nlohmann_json index 456478b..ea60d40 160000 --- a/submodules/nlohmann_json +++ b/submodules/nlohmann_json @@ -1 +1 @@ -Subproject commit 456478b3c50d60100dbb1fb9bc931f370a2c1c28 +Subproject commit ea60d40f4a60a47d3be9560d8f7bc37c163fe47b From 0dce9c1270b10a2b4e39ec5d6aabd9a0c3dd17f1 Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Sun, 24 May 2020 23:41:35 +0530 Subject: [PATCH 03/16] Added GoogleTest --- .gitmodules | 5 ++++- submodules/googletest | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) create mode 160000 submodules/googletest diff --git a/.gitmodules b/.gitmodules index 922be36..b68ea32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,4 +8,7 @@ [submodule "submodules/fastlog"] path = submodules/fastlog - url = https://github.com/gabrielefronze/fastlog.git \ No newline at end of file + url = https://github.com/gabrielefronze/fastlog.git +[submodule "submodules/googletest"] + path = submodules/googletest + url = https://github.com/google/googletest diff --git a/submodules/googletest b/submodules/googletest new file mode 160000 index 0000000..011959a --- /dev/null +++ b/submodules/googletest @@ -0,0 +1 @@ +Subproject commit 011959aafddcd30611003de96cfd8d7a7685c700 From 19436bb67f37ae6136a4f3c80b1db5fd246e5442 Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Sun, 24 May 2020 23:43:50 +0530 Subject: [PATCH 04/16] Added Google Test for Utils #15 --- source/utils.cpp | 205 ++++++++++++++++++++++++--------------- submodules/fastlog | 2 +- submodules/nlohmann_json | 2 +- tests/CMakeLists.txt | 15 ++- tests/gtest_utils.cpp | 57 +++++++++++ 5 files changed, 200 insertions(+), 81 deletions(-) create mode 100644 tests/gtest_utils.cpp diff --git a/source/utils.cpp b/source/utils.cpp index c0c6dcc..b4ad2c0 100644 --- a/source/utils.cpp +++ b/source/utils.cpp @@ -9,188 +9,224 @@ #include #include - -std::string to_string(char* contents, size_t size){ - std::string s; - for (size_t i = 0; i < size; i++) { - s = s + contents[i]; - } - return s; +std::string to_string(char *contents, size_t size) +{ + std::string s; + for (size_t i = 0; i < size; i++) + { + s = s + contents[i]; + } + return s; } // Returns true if and only if path is root path -bool is_root_path(const char *path){ - return (strcmp(path, rucio_root_path.c_str()) == 0 || strcmp(path, (rucio_root_path+"/").c_str()) == 0); +bool is_root_path(const char *path) +{ + return (strcmp(path, rucio_root_path.c_str()) == 0 || strcmp(path, (rucio_root_path + "/").c_str()) == 0); } - - // This function returns -1 if path contains no token // 0 if path is root // POSIX format depth in other cases -int path_depth(const char *path, const char token){ +int path_depth(const char *path, const char token) +{ // If path is root return depth=0 immediately - if(is_root_path(path)) { + if (is_root_path(path)) + { return 0; } // Otherwise compute depth - else { + else + { // Count token occurrences auto s = path; int16_t i = 0; size_t size = 0; - for (i = 0; s[i];){ + for (i = 0; s[i];) + { size++; - if(s[i] == token) i++; - else s++; + if (s[i] == token) + i++; + else + s++; } // Remove one to count if just the last char is equal to token (spurious token) // '/scope1' and '/scope1/' should behave the same - if(path[size-1] == token) i--; + if (path[size - 1] == token) + i--; // Returns depth return i; } } -bool is_server_mountpoint(const char *path){ +bool is_server_mountpoint(const char *path) +{ return path_depth(path) == 1; }; // This function returns true is the depth is 1 (e.g. /scope1 or /scope1/) -bool is_main_scope(const char *path){ +bool is_main_scope(const char *path) +{ return path_depth(path) == 2; } -std::vector split(const std::string &s, char delim) { - std::vector elems; - std::stringstream ss(s); - std::string key_or_value; - while(std::getline(ss, key_or_value, delim)) { - elems.emplace_back(key_or_value); - } - return elems; +std::vector split(const std::string &s, char delim) +{ + std::vector elems; + std::stringstream ss(s); + std::string key_or_value; + while (std::getline(ss, key_or_value, delim)) + { + elems.emplace_back(key_or_value); + } + return elems; } -void remove_trailing_token(std::string& path, std::string token){ - if(path.length() - 1 > 0) { - if (path.substr(path.length() - 1) == token) { +void remove_trailing_token(std::string &path, std::string token) +{ + if (path.length() - 1 > 0) + { + if (path.substr(path.length() - 1) == token) + { path.pop_back(); } } } -void remove_leading_token(std::string& path, std::string token){ -// std::cout << "received: " << path < 0) { - if (path.substr(1) == token) { +void remove_leading_token(std::string &path, std::string token) +{ + // std::cout << "received: " << path < 0) + { + if (path.substr(1) == token) + { path.erase(path.begin()); } } } -std::string remove_substring(const std::string& path, const std::string& subs){ +std::string remove_substring(const std::string &path, const std::string &subs) +{ auto path_copy = path; // Search for the rucio root path with trailing "/" - size_t pos = path_copy.find(subs); + size_t pos = path_copy.find(subs); - if (pos != std::string::npos) - { + if (pos != std::string::npos) + { // Erase root path from string - path_copy.erase(pos, subs.length()); - } + path_copy.erase(pos, subs.length()); + } - return path_copy; + return path_copy; } -std::string remove_root_path(const std::string& path){ +std::string remove_root_path(const std::string &path) +{ return remove_substring(path, rucio_root_path); } -std::string extract_server_name(const std::string& path){ +std::string extract_server_name(const std::string &path) +{ auto path_copy = remove_root_path(path); size_t pos = path_copy.find('/'); if (pos != std::string::npos) - { + { // Erase everything after the first "/" - path_copy.erase(pos, path_copy.length()); - } + path_copy.erase(pos, path_copy.length()); + } return std::move(path_copy); } -std::string extract_scope(const std::string& path){ +std::string extract_scope(const std::string &path) +{ auto path_copy = remove_root_path(path); remove_trailing_token(path_copy); - path_copy = remove_substring(path_copy, extract_server_name(path)+'/'); + path_copy = remove_substring(path_copy, extract_server_name(path) + '/'); size_t pos = path_copy.find('/'); if (pos != std::string::npos) - { + { // Erase everything after the first "/" - path_copy.erase(pos, path_copy.length()); - } + path_copy.erase(pos, path_copy.length()); + } return std::move(path_copy); } -std::string extract_name(const std::string& path){ +std::string extract_name(const std::string &path) +{ auto path_copy = path; remove_trailing_token(path_copy); size_t pos = path_copy.find_last_of('/'); if (pos != std::string::npos) - { + { // Erase everything after the first "/" - path_copy.erase(0, pos+1); - } + path_copy.erase(0, pos + 1); + } return std::move(path_copy); } -std::string get_did(const std::string& path){ - return extract_scope(path)+":"+extract_name(path); +std::string get_did(const std::string &path) +{ + return extract_scope(path) + ":" + extract_name(path); } -void split_dids(const std::string &line, std::vector& did_strings){ +void split_dids(const std::string &line, std::vector &did_strings) +{ did_strings.reserve(std::count(line.begin(), line.end(), '{')); std::stringstream stream(line); std::string buffer; - while(getline(stream, buffer, '\n')){ + while (getline(stream, buffer, '\n')) + { coherentize_dids(buffer); - if(not buffer.empty()) { + if (not buffer.empty()) + { did_strings.emplace_back(std::move(buffer)); } } } -void coherentize_dids(std::string &did_string){ - if(did_string.back() != '}'){ - if(did_string_remainder.empty()) { +void coherentize_dids(std::string &did_string) +{ + if (did_string.back() != '}') + { + if (did_string_remainder.empty()) + { did_string_remainder = std::move(did_string); - } else { + } + else + { did_string_remainder.append(std::move(did_string)); } did_string = ""; - }else if(did_string.front() != '{'){ + } + else if (did_string.front() != '{') + { did_string = did_string_remainder + did_string; did_string_remainder = ""; } } -void structurize_did(const std::string& did_str, std::vector& target) { +void structurize_did(const std::string &did_str, std::vector &target) +{ std::vector did_strings_vect; split_dids(did_str, did_strings_vect); - for (auto &sdid : did_strings_vect) { + for (auto &sdid : did_strings_vect) + { - for (const auto &ch : {' ', '}', '{', '"'}) { + for (const auto &ch : {' ', '}', '{', '"'}) + { sdid.erase(std::remove(sdid.begin(), sdid.end(), ch), sdid.end()); } @@ -202,11 +238,16 @@ void structurize_did(const std::string& did_str, std::vector& target) did.scope = key_values[1]; - if (key_values[3] == "FILE") { + if (key_values[3] == "FILE") + { did.type = rucio_data_type::rucio_file; - } else if (key_values[3] == "CONTAINER") { + } + else if (key_values[3] == "CONTAINER") + { did.type = rucio_data_type::rucio_container; - } else if (key_values[3] == "DATASET") { + } + else if (key_values[3] == "DATASET") + { did.type = rucio_data_type::rucio_dataset; } @@ -218,12 +259,15 @@ void structurize_did(const std::string& did_str, std::vector& target) } } -void structurize_container_did(const std::string& did_str, std::vector& target){ +void structurize_container_did(const std::string &did_str, std::vector &target) +{ std::vector did_strings_vect; split_dids(did_str, did_strings_vect); - for (auto &sdid : did_strings_vect) { - for (const auto &ch : {' ', '}', '{', '"'}) { + for (auto &sdid : did_strings_vect) + { + for (const auto &ch : {' ', '}', '{', '"'}) + { sdid.erase(std::remove(sdid.begin(), sdid.end(), ch), sdid.end()); } @@ -235,11 +279,16 @@ void structurize_container_did(const std::string& did_str, std::vector, 2020 +*/ + +#include +#include + +using namespace std; + +TEST(Utils_Test, Test_is_root_path){ + EXPECT_EQ(1, (bool)is_root_path("/")); + EXPECT_EQ(0, (bool)is_root_path("/a")); + EXPECT_EQ(0, (bool)is_root_path("/a/b")); +} + +TEST(Utils_Test, Test_path_depth){ + EXPECT_EQ(0, path_depth("/", '/')); + EXPECT_EQ(1, path_depth("/a",'/')); + EXPECT_EQ(1, path_depth("/a/",'/')); + EXPECT_EQ(2, path_depth("/a/b",'/')); + EXPECT_EQ(6, path_depth("/a/b/c/d/e/f/",'/')); + EXPECT_EQ(-1, path_depth("a", 'a')); +} + +TEST(Utils_Test, Test_string_manipulation){ + EXPECT_EQ("a", extract_server_name("/a/b/c")); + EXPECT_EQ("c", extract_scope("/a/b/c/d")); +} + +TEST(Utils_Test, Test_is_server_mountpoint){ + EXPECT_TRUE(is_server_mountpoint("/server1")); + EXPECT_TRUE(is_server_mountpoint("/server1/")); + EXPECT_FALSE(is_server_mountpoint("/")); + EXPECT_FALSE(is_server_mountpoint("/server1/scope1")); + EXPECT_FALSE(is_server_mountpoint("/server1/scope1/")); + EXPECT_FALSE(is_server_mountpoint("/server1/scope1/name1")); +} + +TEST(Utils_Test, Test_is_main_scope){ + EXPECT_FALSE(is_main_scope("/server1")); + EXPECT_FALSE(is_main_scope("/server1/")); + EXPECT_FALSE(is_main_scope("/")); + EXPECT_TRUE(is_main_scope("/server1/scope1")); + EXPECT_TRUE(is_main_scope("/server1/scope1/")); + EXPECT_FALSE(is_main_scope("/server1/scope1/name1")); +} + +// TEST(Utils_Test, Test_string_manipulation){ +// string test_string = "/server/scope/container/dataset/filename/"; +// EXPECT_NO_THROW(remove_trailing_token(test_string)); +// } + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file From cb18986dda5f5e0181ee05efbf07caf743d82b0e Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Mon, 25 May 2020 13:29:00 +0530 Subject: [PATCH 05/16] Added curl-REST Unit Test #15 --- source/utils.cpp | 9 ++++++--- tests/CMakeLists.txt | 8 +++++++- tests/gtest_curl-REST.cpp | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 tests/gtest_curl-REST.cpp diff --git a/source/utils.cpp b/source/utils.cpp index b4ad2c0..4d3405a 100644 --- a/source/utils.cpp +++ b/source/utils.cpp @@ -1,6 +1,9 @@ -// -// Created by Gabriele Gaetano Fronzé on 2019-10-16. -// +/* +Created by Gabriele Gaetano Fronzé on 2019-10-16. +Authors: +- Gabriele Gaetano Fronzé , 2019 +- Vivek Nigam , 2020 +*/ #include #include diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4e3b4c9..e3eb6d3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,4 +27,10 @@ add_executable( gtest_utils gtest_utils.cpp) -target_link_libraries(gtest_utils PUBLIC utils ${GTEST_LIBRARIES} pthread) \ No newline at end of file +target_link_libraries(gtest_utils PUBLIC utils ${GTEST_LIBRARIES} pthread) + +add_executable( + gtest_curl-REST + gtest_curl-REST.cpp) + +target_link_libraries(gtest_curl-REST PUBLIC curl-wrapper ${GTEST_LIBRARIES} pthread) \ No newline at end of file diff --git a/tests/gtest_curl-REST.cpp b/tests/gtest_curl-REST.cpp new file mode 100644 index 0000000..276d6c3 --- /dev/null +++ b/tests/gtest_curl-REST.cpp @@ -0,0 +1,18 @@ +/* +Authors: +- Vivek Nigam , 2020 +*/ + +#include +#include +#include + +TEST(curl_Test, Test){ + auto res = GET("www.example.com"); + ASSERT_EQ(1, res.payload.size()); +} + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file From c9d92e74cb37dcc896e1816cb9e1a5275fae06ea Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Mon, 25 May 2020 15:24:07 +0530 Subject: [PATCH 06/16] Added more unit tests --- tests/CMakeLists.txt | 9 ++++++++- tests/gtest_REST-API.cpp | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 tests/gtest_REST-API.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e3eb6d3..0c1daf0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -33,4 +33,11 @@ add_executable( gtest_curl-REST gtest_curl-REST.cpp) -target_link_libraries(gtest_curl-REST PUBLIC curl-wrapper ${GTEST_LIBRARIES} pthread) \ No newline at end of file +target_link_libraries(gtest_curl-REST PUBLIC curl-wrapper ${GTEST_LIBRARIES} pthread) + +add_executable( + gtest_REST-API + gtest_REST-API.cpp +) + +target_link_libraries(gtest_REST-API PUBLIC rucio-rest-api-wrapper globals ${GTEST_LIBRARIES} pthread) \ No newline at end of file diff --git a/tests/gtest_REST-API.cpp b/tests/gtest_REST-API.cpp new file mode 100644 index 0000000..1a3be62 --- /dev/null +++ b/tests/gtest_REST-API.cpp @@ -0,0 +1,13 @@ +#include +#include +#include +#include + +using namespace std; + + + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file From 2deebbf910d6fd386f19b3c0fff7ee5c09bbab96 Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Mon, 25 May 2020 18:24:17 +0530 Subject: [PATCH 07/16] From upstream/master --- source/utils.cpp | 209 ++++++++++++++++++----------------------------- 1 file changed, 79 insertions(+), 130 deletions(-) diff --git a/source/utils.cpp b/source/utils.cpp index 35e1228..f8844a2 100644 --- a/source/utils.cpp +++ b/source/utils.cpp @@ -1,9 +1,6 @@ -/* -Created by Gabriele Gaetano Fronzé on 2019-10-16. -Authors: -- Gabriele Gaetano Fronzé , 2019 -- Vivek Nigam , 2020 -*/ +// +// Created by Gabriele Gaetano Fronzé on 2019-10-16. +// #include #include @@ -12,168 +9,146 @@ Created by Gabriele Gaetano Fronzé on 2019-10-16. #include #include -std::string to_string(char *contents, size_t size) -{ - std::string s; - for (size_t i = 0; i < size; i++) - { - s = s + contents[i]; - } - return s; + +std::string to_string(char* contents, size_t size){ + std::string s; + for (size_t i = 0; i < size; i++) { + s = s + contents[i]; + } + return s; } // Returns true if and only if path is root path -bool is_root_path(const char *path) -{ - return (strcmp(path, rucio_root_path.c_str()) == 0 || strcmp(path, (rucio_root_path + "/").c_str()) == 0); +bool is_root_path(const char *path){ + return (strcmp(path, rucio_root_path.c_str()) == 0 || strcmp(path, (rucio_root_path+"/").c_str()) == 0); } + + // This function returns -1 if path contains no token // 0 if path is root // POSIX format depth in other cases -int path_depth(const char *path, const char token) -{ +int path_depth(const char *path, const char token){ // If path is root return depth=0 immediately - if (is_root_path(path)) - { + if(is_root_path(path)) { return 0; } // Otherwise compute depth - else - { + else { // Count token occurrences auto s = path; int16_t i = 0; size_t size = 0; - for (i = 0; s[i];) - { + for (i = 0; s[i];){ size++; - if (s[i] == token) - i++; - else - s++; + if(s[i] == token) i++; + else s++; } // Remove one to count if just the last char is equal to token (spurious token) // '/scope1' and '/scope1/' should behave the same - if (path[size - 1] == token) - i--; + if(path[size-1] == token) i--; // Returns depth return i; } } -bool is_server_mountpoint(const char *path) -{ +bool is_server_mountpoint(const char *path){ return path_depth(path) == 1; }; // This function returns true is the depth is 1 (e.g. /scope1 or /scope1/) -bool is_main_scope(const char *path) -{ +bool is_main_scope(const char *path){ return path_depth(path) == 2; } -std::vector split(const std::string &s, char delim) -{ - std::vector elems; - std::stringstream ss(s); - std::string key_or_value; - while (std::getline(ss, key_or_value, delim)) - { - elems.emplace_back(key_or_value); - } - return elems; +std::vector split(const std::string &s, char delim) { + std::vector elems; + std::stringstream ss(s); + std::string key_or_value; + while(std::getline(ss, key_or_value, delim)) { + elems.emplace_back(key_or_value); + } + return elems; } -void remove_trailing_token(std::string &path, std::string token) -{ - if (path.length() - 1 > 0) - { - if (path.substr(path.length() - 1) == token) - { +void remove_trailing_token(std::string& path, std::string token){ + if(path.length() - 1 > 0) { + if (path.substr(path.length() - 1) == token) { path.pop_back(); } } } -void remove_leading_token(std::string &path, std::string token) -{ - // std::cout << "received: " << path < 0) - { - if (path.substr(1) == token) - { +void remove_leading_token(std::string& path, std::string token){ +// std::cout << "received: " << path < 0) { + if (path.substr(1) == token) { path.erase(path.begin()); } } } -std::string remove_substring(const std::string &path, const std::string &subs) -{ +std::string remove_substring(const std::string& path, const std::string& subs){ auto path_copy = path; // Search for the rucio root path with trailing "/" - size_t pos = path_copy.find(subs); + size_t pos = path_copy.find(subs); - if (pos != std::string::npos) - { + if (pos != std::string::npos) + { // Erase root path from string - path_copy.erase(pos, subs.length()); - } + path_copy.erase(pos, subs.length()); + } - return path_copy; + return path_copy; } -std::string remove_root_path(const std::string &path) -{ +std::string remove_root_path(const std::string& path){ return remove_substring(path, rucio_root_path); } -std::string extract_server_name(const std::string &path) -{ +std::string extract_server_name(const std::string& path){ auto path_copy = remove_root_path(path); size_t pos = path_copy.find('/'); if (pos != std::string::npos) - { + { // Erase everything after the first "/" - path_copy.erase(pos, path_copy.length()); - } + path_copy.erase(pos, path_copy.length()); + } return std::move(path_copy); } -std::string extract_scope(const std::string &path) -{ +std::string extract_scope(const std::string& path){ auto path_copy = remove_root_path(path); remove_trailing_token(path_copy); - path_copy = remove_substring(path_copy, extract_server_name(path) + '/'); + path_copy = remove_substring(path_copy, extract_server_name(path)+'/'); size_t pos = path_copy.find('/'); if (pos != std::string::npos) - { + { // Erase everything after the first "/" - path_copy.erase(pos, path_copy.length()); - } + path_copy.erase(pos, path_copy.length()); + } return std::move(path_copy); } -std::string extract_name(const std::string &path) -{ +std::string extract_name(const std::string& path){ auto path_copy = path; remove_trailing_token(path_copy); size_t pos = path_copy.find_last_of('/'); if (pos != std::string::npos) - { + { // Erase everything after the first "/" - path_copy.erase(0, pos + 1); - } + path_copy.erase(0, pos+1); + } return std::move(path_copy); } @@ -188,53 +163,40 @@ std::string get_did(const std::string& path){ return extract_scope(path)+":"+extract_name(path); } -void split_dids(const std::string &line, std::vector &did_strings) -{ +void split_dids(const std::string &line, std::vector& did_strings){ did_strings.reserve(std::count(line.begin(), line.end(), '{')); std::stringstream stream(line); std::string buffer; - while (getline(stream, buffer, '\n')) - { + while(getline(stream, buffer, '\n')){ coherentize_dids(buffer); - if (not buffer.empty()) - { + if(not buffer.empty()) { did_strings.emplace_back(std::move(buffer)); } } } -void coherentize_dids(std::string &did_string) -{ - if (did_string.back() != '}') - { - if (did_string_remainder.empty()) - { +void coherentize_dids(std::string &did_string){ + if(did_string.back() != '}'){ + if(did_string_remainder.empty()) { did_string_remainder = std::move(did_string); - } - else - { + } else { did_string_remainder.append(std::move(did_string)); } did_string = ""; - } - else if (did_string.front() != '{') - { + }else if(did_string.front() != '{'){ did_string = did_string_remainder + did_string; did_string_remainder = ""; } } -void structurize_did(const std::string &did_str, std::vector &target) -{ +void structurize_did(const std::string& did_str, std::vector& target) { std::vector did_strings_vect; split_dids(did_str, did_strings_vect); - for (auto &sdid : did_strings_vect) - { + for (auto &sdid : did_strings_vect) { - for (const auto &ch : {' ', '}', '{', '"'}) - { + for (const auto &ch : {' ', '}', '{', '"'}) { sdid.erase(std::remove(sdid.begin(), sdid.end(), ch), sdid.end()); } @@ -246,16 +208,11 @@ void structurize_did(const std::string &did_str, std::vector &target) did.scope = key_values[1]; - if (key_values[3] == "FILE") - { + if (key_values[3] == "FILE") { did.type = rucio_data_type::rucio_file; - } - else if (key_values[3] == "CONTAINER") - { + } else if (key_values[3] == "CONTAINER") { did.type = rucio_data_type::rucio_container; - } - else if (key_values[3] == "DATASET") - { + } else if (key_values[3] == "DATASET") { did.type = rucio_data_type::rucio_dataset; } @@ -267,15 +224,12 @@ void structurize_did(const std::string &did_str, std::vector &target) } } -void structurize_container_did(const std::string &did_str, std::vector &target) -{ +void structurize_container_did(const std::string& did_str, std::vector& target){ std::vector did_strings_vect; split_dids(did_str, did_strings_vect); - for (auto &sdid : did_strings_vect) - { - for (const auto &ch : {' ', '}', '{', '"'}) - { + for (auto &sdid : did_strings_vect) { + for (const auto &ch : {' ', '}', '{', '"'}) { sdid.erase(std::remove(sdid.begin(), sdid.end(), ch), sdid.end()); } @@ -287,16 +241,11 @@ void structurize_container_did(const std::string &did_str, std::vector Date: Mon, 25 May 2020 19:39:10 +0530 Subject: [PATCH 08/16] submodules from upstream/master --- submodules/fastlog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/fastlog b/submodules/fastlog index a33aafc..4ac3223 160000 --- a/submodules/fastlog +++ b/submodules/fastlog @@ -1 +1 @@ -Subproject commit a33aafc6cfe74d83ed527ef9a4aa1f6bc53cef17 +Subproject commit 4ac3223876b80d9e8ace5fcfcbffaaa2629bfd7f From 2b80169433b912272da71a351f04b6c6b01f19e6 Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Mon, 25 May 2020 22:26:34 +0530 Subject: [PATCH 09/16] Added more Unit Tests for Utils #15 --- tests/gtest_utils.cpp | 51 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/tests/gtest_utils.cpp b/tests/gtest_utils.cpp index 7ff40c9..f15d585 100644 --- a/tests/gtest_utils.cpp +++ b/tests/gtest_utils.cpp @@ -4,6 +4,7 @@ */ #include +#include #include using namespace std; @@ -20,12 +21,7 @@ TEST(Utils_Test, Test_path_depth){ EXPECT_EQ(1, path_depth("/a/",'/')); EXPECT_EQ(2, path_depth("/a/b",'/')); EXPECT_EQ(6, path_depth("/a/b/c/d/e/f/",'/')); - EXPECT_EQ(-1, path_depth("a", 'a')); -} - -TEST(Utils_Test, Test_string_manipulation){ - EXPECT_EQ("a", extract_server_name("/a/b/c")); - EXPECT_EQ("c", extract_scope("/a/b/c/d")); + EXPECT_EQ(0, path_depth("a", 'a')); } TEST(Utils_Test, Test_is_server_mountpoint){ @@ -46,10 +42,45 @@ TEST(Utils_Test, Test_is_main_scope){ EXPECT_FALSE(is_main_scope("/server1/scope1/name1")); } -// TEST(Utils_Test, Test_string_manipulation){ -// string test_string = "/server/scope/container/dataset/filename/"; -// EXPECT_NO_THROW(remove_trailing_token(test_string)); -// } +TEST(Utils_Test, Test_string_manipulation){ + string test_string = "/server/scope/container/dataset/filename/"; + string exp_string = "/server/scope/container/dataset/filename"; + remove_trailing_token(test_string); + + EXPECT_EQ(test_string, exp_string); + EXPECT_EQ("server", extract_server_name(test_string)); + EXPECT_EQ("scope", extract_scope(test_string)); + EXPECT_EQ("filename", extract_name(test_string)); + EXPECT_EQ("scope:filename", get_did(test_string)); +} + +TEST(Utils_Test, Test_structurize_DID){ + std::string test_did = R"( + { + "scope": "user.root", + "type": "FILE", + "name": "test-19102019.txt", + "parent": null, + "level": 0 + })"; + std::vector did_v; + structurize_did(test_did, did_v); + EXPECT_EQ("user.root", did_v[0].scope); + EXPECT_EQ(rucio_data_type::rucio_file, did_v[0].type); + EXPECT_EQ("test-19102019.txt", did_v[0].name); + EXPECT_EQ("null", did_v[0].parent); + EXPECT_EQ(0, did_v[0].level); +} + +TEST(Utils_Test, Test_tokenize_python_list){ + std::vector expected_scopes = {"a","b","c","d"}; + std::vector scopes; + tokenize_python_list(R"(["a","b","c","d"])", scopes); + EXPECT_EQ("a", scopes[0]); + EXPECT_EQ("b", scopes[1]); + EXPECT_EQ("c", scopes[2]); + EXPECT_EQ("d", scopes[3]); +} int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); From dcf5e991d44d00d74b17f400c13b701d5db06209 Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Tue, 26 May 2020 14:02:30 +0530 Subject: [PATCH 10/16] Added Test_is_hidden in Utils #15 --- tests/gtest_utils.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/gtest_utils.cpp b/tests/gtest_utils.cpp index f15d585..ba15514 100644 --- a/tests/gtest_utils.cpp +++ b/tests/gtest_utils.cpp @@ -48,6 +48,8 @@ TEST(Utils_Test, Test_string_manipulation){ remove_trailing_token(test_string); EXPECT_EQ(test_string, exp_string); + EXPECT_EQ("/server/scope/dataset/filename", remove_substring(test_string, "container/")); + EXPECT_EQ("server/scope/container/dataset/filename", remove_root_path(test_string)); EXPECT_EQ("server", extract_server_name(test_string)); EXPECT_EQ("scope", extract_scope(test_string)); EXPECT_EQ("filename", extract_name(test_string)); @@ -82,6 +84,11 @@ TEST(Utils_Test, Test_tokenize_python_list){ EXPECT_EQ("d", scopes[3]); } +TEST(Utils_Test, Test_is_hidden){ + EXPECT_TRUE(is_hidden("/server/scope/container/dataset/.filename/")); + EXPECT_FALSE(is_hidden("/server/scope/container/dataset/")); +} + int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); From 5c50c73e039b9b8620e7e57d41e9067a9def576f Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Tue, 26 May 2020 20:51:36 +0530 Subject: [PATCH 11/16] Adding gtest_REST-API.cpp #15 --- tests/gtest_REST-API.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/gtest_REST-API.cpp b/tests/gtest_REST-API.cpp index 1a3be62..a898658 100644 --- a/tests/gtest_REST-API.cpp +++ b/tests/gtest_REST-API.cpp @@ -6,6 +6,16 @@ using namespace std; +std::string server_name; + +TEST(RESTAPI_Test, Test_server_connection){ + parse_settings(); + server_name = "rucio-dev-server"; + + EXPECT_NO_THROW(rucio_get_auth_token_userpass(server_name)); + ASSERT_TRUE(rucio_is_token_valid(server_name)); +} + int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); From 7f30f6e43e9b3319a2156002bf91e43acb202cec Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Wed, 27 May 2020 15:48:24 +0530 Subject: [PATCH 12/16] Added REST-API Tests #15 --- tests/gtest_REST-API.cpp | 109 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 105 insertions(+), 4 deletions(-) diff --git a/tests/gtest_REST-API.cpp b/tests/gtest_REST-API.cpp index a898658..7d4d19c 100644 --- a/tests/gtest_REST-API.cpp +++ b/tests/gtest_REST-API.cpp @@ -1,23 +1,124 @@ +/* +Authors: +- Vivek Nigam , 2020 +*/ + #include #include #include +#include #include using namespace std; +using namespace fastlog; +std::string server_name="rucio-dev-server"; -std::string server_name; +TEST(RESTAPI_Test, Test_rucio_ping){ + ASSERT_TRUE(rucio_ping("https://localhost/")); +} TEST(RESTAPI_Test, Test_server_connection){ - parse_settings(); - server_name = "rucio-dev-server"; - + fastlog(INFO, "Servers Found:"); + auto servers = rucio_list_servers(); + if (servers.size()>0){ + for (auto i = servers.begin(); i != servers.end(); ++i) + cout << *i << endl; + } else{ + fastlog(ERROR, "No servers found!"); + exit(-1); + } EXPECT_NO_THROW(rucio_get_auth_token_userpass(server_name)); ASSERT_TRUE(rucio_is_token_valid(server_name)); } +TEST(RESTAPI_Test, Test_server_scopes){ + auto scopes = rucio_list_scopes(server_name); + fastlog(INFO, "Scopes Found: "); + for (auto i = scopes.begin(); i != scopes.end(); ++i) + cout << *i << endl; + ASSERT_TRUE(scopes.size()>=0); +} + +void get_scope_dids(std::string server_short_name, std::string scope_name){ + fastlog(INFO, "Using server: %s | Scope: %s", server_short_name.c_str(), scope_name.c_str()); + std::cout< Date: Wed, 27 May 2020 21:11:08 +0530 Subject: [PATCH 13/16] Improved Test_rucio_get_metalink_replicas #15 --- tests/gtest_REST-API.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/gtest_REST-API.cpp b/tests/gtest_REST-API.cpp index 7d4d19c..80ec146 100644 --- a/tests/gtest_REST-API.cpp +++ b/tests/gtest_REST-API.cpp @@ -93,9 +93,10 @@ bool get_replica_metalinks(const std::string& path){ } TEST(RESTAPI_Test, Test_rucio_get_replicas_metalinks){ - ASSERT_TRUE(get_replica_metalinks("/"+server_name+"/test/container/dataset1/file1")); - ASSERT_TRUE(get_replica_metalinks("/"+server_name+"/test/container/dataset1/file2")); - ASSERT_FALSE(get_replica_metalinks("/"+server_name+"/test/container/dataset1/file6")); + //ASSERT_TRUE(get_replica_metalinks("/"+server_name+"/test/container/dataset1/file1")); + ASSERT_TRUE(not rucio_get_replicas_metalinks("/"+server_name+"/test/container/dataset1/file1").empty()); + ASSERT_FALSE(rucio_get_replicas_metalinks("/"+server_name+"/test/container/dataset1/file1").empty()); + //ASSERT_FALSE(get_replica_metalinks("/"+server_name+"/test/container/dataset1/file6")); } bool get_list_container_dids(const string& server, const string& scope, const string& did_name){ From 3b59560b0bc34afeec7f7d68032fe11e0360bd54 Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Fri, 29 May 2020 17:13:19 +0530 Subject: [PATCH 14/16] Improved REST_API Tests #15 --- source/REST-API.cpp | 3 -- tests/gtest_REST-API.cpp | 99 +++++++++++----------------------------- 2 files changed, 26 insertions(+), 76 deletions(-) diff --git a/source/REST-API.cpp b/source/REST-API.cpp index f80f384..3234991 100644 --- a/source/REST-API.cpp +++ b/source/REST-API.cpp @@ -281,9 +281,6 @@ std::vector rucio_get_replicas_metalinks(const std::string& path){ auto beg_pfn = rses.find('[', 0); auto end_pfn = rses.find(']', beg_pfn + 1); - std::cout << beg_pfn << std::endl; - std::cout << end_pfn << std::endl; - std::vector pfns; while(beg_pfn != std::string::npos && end_pfn != std::string::npos){ diff --git a/tests/gtest_REST-API.cpp b/tests/gtest_REST-API.cpp index 80ec146..5814b42 100644 --- a/tests/gtest_REST-API.cpp +++ b/tests/gtest_REST-API.cpp @@ -19,52 +19,31 @@ TEST(RESTAPI_Test, Test_rucio_ping){ } TEST(RESTAPI_Test, Test_server_connection){ - fastlog(INFO, "Servers Found:"); auto servers = rucio_list_servers(); - if (servers.size()>0){ - for (auto i = servers.begin(); i != servers.end(); ++i) - cout << *i << endl; - } else{ - fastlog(ERROR, "No servers found!"); - exit(-1); - } - EXPECT_NO_THROW(rucio_get_auth_token_userpass(server_name)); + + ASSERT_EQ(server_name, servers[0]); + ASSERT_NO_THROW(rucio_get_auth_token_userpass(server_name)); ASSERT_TRUE(rucio_is_token_valid(server_name)); } TEST(RESTAPI_Test, Test_server_scopes){ auto scopes = rucio_list_scopes(server_name); - fastlog(INFO, "Scopes Found: "); - for (auto i = scopes.begin(); i != scopes.end(); ++i) - cout << *i << endl; ASSERT_TRUE(scopes.size()>=0); -} - -void get_scope_dids(std::string server_short_name, std::string scope_name){ - fastlog(INFO, "Using server: %s | Scope: %s", server_short_name.c_str(), scope_name.c_str()); - std::cout< Date: Fri, 29 May 2020 17:53:48 +0530 Subject: [PATCH 15/16] Added No-server check #15 --- tests/gtest_REST-API.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/gtest_REST-API.cpp b/tests/gtest_REST-API.cpp index 5814b42..b2ac6bd 100644 --- a/tests/gtest_REST-API.cpp +++ b/tests/gtest_REST-API.cpp @@ -20,7 +20,10 @@ TEST(RESTAPI_Test, Test_rucio_ping){ TEST(RESTAPI_Test, Test_server_connection){ auto servers = rucio_list_servers(); - + if (servers.empty()){ + fastlog(ERROR, "No servers found!"); + exit(-1); + } ASSERT_EQ(server_name, servers[0]); ASSERT_NO_THROW(rucio_get_auth_token_userpass(server_name)); ASSERT_TRUE(rucio_is_token_valid(server_name)); @@ -55,6 +58,7 @@ TEST(RESTAPI_Test, Test__is_container){ TEST(RESTAPI_Test, Test_rucio_get_replicas_metalinks){ ASSERT_TRUE(not rucio_get_replicas_metalinks("/"+server_name+"/test/container/dataset1/file1").empty()); ASSERT_FALSE(rucio_get_replicas_metalinks("/"+server_name+"/test/container/dataset1/file1").empty()); + ASSERT_TRUE(not rucio_get_replicas_metalinks("/"+server_name+"/test/container/dataset1/file2").empty()); } TEST(RESTAPI_Test, Test_rucio_list_container_dids){ From 9db040d7b4c9f786fb99cdea99caea45ffb0c36a Mon Sep 17 00:00:00 2001 From: Vivek Nigam Date: Fri, 29 May 2020 23:41:58 +0530 Subject: [PATCH 16/16] Added size checks for scopes/DIDs #15 --- tests/gtest_REST-API.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/gtest_REST-API.cpp b/tests/gtest_REST-API.cpp index b2ac6bd..5d85d37 100644 --- a/tests/gtest_REST-API.cpp +++ b/tests/gtest_REST-API.cpp @@ -31,7 +31,7 @@ TEST(RESTAPI_Test, Test_server_connection){ TEST(RESTAPI_Test, Test_server_scopes){ auto scopes = rucio_list_scopes(server_name); - ASSERT_TRUE(scopes.size()>=0); + ASSERT_TRUE(scopes.size()==5); ASSERT_EQ("data13_hip", scopes[0]); ASSERT_EQ("test", scopes[4]); } @@ -44,12 +44,13 @@ TEST(RESTAPI_Test, Test_scope_dids){ ASSERT_TRUE(ret.empty()); ret = rucio_list_dids("test", server_name); + ASSERT_TRUE(ret.size()>=2); ASSERT_EQ("container", ret[0].name); ASSERT_EQ("dataset3", ret[1].name); ASSERT_NE("dataset1", ret[1].name); } -TEST(RESTAPI_Test, Test__is_container){ +TEST(RESTAPI_Test, Test_is_container){ EXPECT_TRUE(rucio_is_container("/"+server_name+"/test/container")); //EXPECT_FALSE(rucio_is_container("/"+server_name+"/test/container/dataset1")); //EXPECT_FALSE(rucio_is_container("/"+server_name+"/test/dataset3")); @@ -63,14 +64,17 @@ TEST(RESTAPI_Test, Test_rucio_get_replicas_metalinks){ TEST(RESTAPI_Test, Test_rucio_list_container_dids){ auto ret = rucio_list_container_dids("test", "container", server_name); + ASSERT_TRUE(ret.size()==2); ASSERT_EQ("dataset1", ret[0].name); ASSERT_EQ("dataset2", ret[1].name); ASSERT_NE("dataser3", ret[1].name); ret = rucio_list_container_dids("test", "dataset3", server_name); + ASSERT_TRUE(ret.size()==1); ASSERT_EQ("file4", ret[0].name); ret = rucio_list_container_dids("test", "dataset2", server_name); + ASSERT_TRUE(ret.size()==2); ASSERT_EQ("file3", ret[0].name); ASSERT_EQ("file4", ret[1].name); }