Skip to content

Commit

Permalink
[Fix](sk) All sk in log should be encrypted (#43544)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Yukang-Lian
Yukang-Lian authored Nov 12, 2024
1 parent 3d7785d commit 954661c
Show file tree
Hide file tree
Showing 3 changed files with 127 additions and 1 deletion.
45 changes: 44 additions & 1 deletion cloud/src/meta-service/meta_service_helper.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@

#include <brpc/controller.h>
#include <gen_cpp/cloud.pb.h>
#include <openssl/md5.h>

#include <iomanip>
#include <memory>
#include <string>
#include <string_view>
Expand All @@ -29,12 +31,26 @@
#include "common/logging.h"
#include "common/stopwatch.h"
#include "common/util.h"
#include "cpp/sync_point.h"
#include "meta-service/keys.h"
#include "meta-service/txn_kv.h"
#include "meta-service/txn_kv_error.h"
#include "resource-manager/resource_manager.h"

namespace doris::cloud {
inline std::string md5(const std::string& str) {
unsigned char digest[MD5_DIGEST_LENGTH];
MD5_CTX context;
MD5_Init(&context);
MD5_Update(&context, str.c_str(), str.length());
MD5_Final(digest, &context);

std::ostringstream ss;
for (unsigned char i : digest) {
ss << std::setw(2) << std::setfill('0') << std::hex << (int)i;
}
return ss.str();
}

template <class Request>
void begin_rpc(std::string_view func_name, brpc::Controller* ctrl, const Request* req) {
Expand Down Expand Up @@ -101,7 +117,34 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re
LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
<< " status=" << res->status().ShortDebugString()
<< " delete_bitmap_size=" << res->segment_delete_bitmaps_size();

} else if constexpr (std::is_same_v<Response, GetObjStoreInfoResponse> ||
std::is_same_v<Response, GetStageResponse>) {
std::string debug_string = res->DebugString();
// Start position for searching "sk" fields
size_t pos = 0;
// Iterate through the string and find all occurrences of "sk: "
while ((pos = debug_string.find("sk: ", pos)) != std::string::npos) {
// Find the start and end of the "sk" value (assumed to be within quotes)
// Start after the quote
size_t sk_value_start = debug_string.find('\"', pos) + 1;
// End at the next quote
size_t sk_value_end = debug_string.find('\"', sk_value_start);

// Extract the "sk" value
std::string sk_value =
debug_string.substr(sk_value_start, sk_value_end - sk_value_start);
// Encrypt the "sk" value with MD5
std::string encrypted_sk = "md5: " + md5(sk_value);

// Replace the original "sk" value with the encrypted MD5 value
debug_string.replace(sk_value_start, sk_value_end - sk_value_start, encrypted_sk);

// Move the position to the end of the current "sk" field and continue searching
pos = sk_value_end;
}
TEST_SYNC_POINT_CALLBACK("sk_finish_rpc", &debug_string);
LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
<< " response=" << debug_string;
} else {
LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
<< " response=" << res->ShortDebugString();
Expand Down
4 changes: 4 additions & 0 deletions cloud/src/meta-service/meta_service_resource.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ void MetaServiceImpl::get_obj_store_info(google::protobuf::RpcController* contro
GetObjStoreInfoResponse* response,
::google::protobuf::Closure* done) {
RPC_PREPROCESS(get_obj_store_info);
TEST_SYNC_POINT_CALLBACK("obj-store-info_sk_response", &response);
TEST_SYNC_POINT_RETURN_WITH_VOID("obj-store-info_sk_response_return");
// Prepare data
std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
if (cloud_unique_id.empty()) {
Expand Down Expand Up @@ -2600,6 +2602,8 @@ void MetaServiceImpl::get_stage(google::protobuf::RpcController* controller,
const GetStageRequest* request, GetStageResponse* response,
::google::protobuf::Closure* done) {
RPC_PREPROCESS(get_stage);
TEST_SYNC_POINT_CALLBACK("stage_sk_response", &response);
TEST_SYNC_POINT_RETURN_WITH_VOID("stage_sk_response_return");
std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
if (cloud_unique_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
Expand Down
79 changes: 79 additions & 0 deletions cloud/test/meta_service_http_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1456,4 +1456,83 @@ TEST(MetaServiceHttpTest, TxnLazyCommit) {
}
}

TEST(MetaServiceHttpTest, get_stage_response_sk) {
auto sp = SyncPoint::get_instance();
sp->enable_processing();
std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01,
[&](...) { sp->disable_processing(); });

GetStageResponse res;
auto* stage = res.add_stage();
stage->mutable_obj_info()->set_ak("stage-ak");
stage->mutable_obj_info()->set_sk("stage-sk");
auto foo = [res](auto args) { (*(try_any_cast<GetStageResponse**>(args[0])))->CopyFrom(res); };
sp->set_call_back("stage_sk_response", foo);
sp->set_call_back("stage_sk_response_return",
[](auto&& args) { *try_any_cast<bool*>(args.back()) = true; });

auto rate_limiter = std::make_shared<cloud::RateLimiter>();

auto ms = std::make_unique<cloud::MetaServiceImpl>(nullptr, nullptr, rate_limiter);

auto bar = [](auto args) {
std::cout << *try_any_cast<std::string*>(args[0]);

EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("stage-sk") == std::string::npos);
EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
.find("md5: f497d053066fa4b7d3b1f6564597d233") != std::string::npos);
};
sp->set_call_back("sk_finish_rpc", bar);

GetStageResponse res1;
GetStageRequest req1;
brpc::Controller cntl;
ms->get_stage(&cntl, &req1, &res1, nullptr);
}

TEST(MetaServiceHttpTest, get_obj_store_info_response_sk) {
auto sp = SyncPoint::get_instance();
sp->enable_processing();
std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01,
[&](...) { sp->disable_processing(); });

GetObjStoreInfoResponse res;
auto* obj_info = res.add_obj_info();
obj_info->set_ak("obj-store-info-ak1");
obj_info->set_sk("obj-store-info-sk1");
obj_info = res.add_storage_vault()->mutable_obj_info();
obj_info->set_ak("obj-store-info-ak2");
obj_info->set_sk("obj-store-info-sk2");
auto foo = [res](auto args) {
(*(try_any_cast<GetObjStoreInfoResponse**>(args[0])))->CopyFrom(res);
};
sp->set_call_back("obj-store-info_sk_response", foo);
sp->set_call_back("obj-store-info_sk_response_return",
[](auto&& args) { *try_any_cast<bool*>(args.back()) = true; });

auto rate_limiter = std::make_shared<cloud::RateLimiter>();

auto ms = std::make_unique<cloud::MetaServiceImpl>(nullptr, nullptr, rate_limiter);

auto bar = [](auto args) {
std::cout << *try_any_cast<std::string*>(args[0]);

EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("obj-store-info-sk1") ==
std::string::npos);
EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
.find("md5: 35d5a637fd9d45a28207a888b751efc4") != std::string::npos);

EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("obj-store-info-sk2") ==
std::string::npos);
EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
.find("md5: 01d7473ae201a2ecdf1f7c064eb81a95") != std::string::npos);
};
sp->set_call_back("sk_finish_rpc", bar);

GetObjStoreInfoResponse res1;
GetObjStoreInfoRequest req1;
brpc::Controller cntl;
ms->get_obj_store_info(&cntl, &req1, &res1, nullptr);
}

} // namespace doris::cloud

0 comments on commit 954661c

Please sign in to comment.