From 15c89e7eb35f1b0f8b83f7fd91c7bd13649f3081 Mon Sep 17 00:00:00 2001 From: Gustavo Lopes Date: Wed, 3 Jul 2024 18:01:43 +0100 Subject: [PATCH] appsec helper: remote config telemetry metrics --- appsec/src/extension/commands_helpers.c | 1 + appsec/src/helper/client.cpp | 13 +- .../helper/remote_config/client_handler.cpp | 37 +- .../helper/remote_config/client_handler.hpp | 14 + appsec/src/helper/service.hpp | 14 + .../remote_config/client_handler_test.cpp | 55 +-- appsec/tests/integration/build.gradle | 7 + .../appsec/php/docker/AppSecContainer.groovy | 1 + .../php/mock_agent/ConfigV07Handler.groovy | 49 +++ .../appsec/php/mock_agent/InfoHandler.groovy | 3 +- .../php/mock_agent/MockDatadogAgent.groovy | 1 + .../rem_cfg/IntegrityCheckException.groovy | 11 + .../rem_cfg/MissingContentException.groovy | 7 + .../rem_cfg/RemoteConfigRequest.java | 320 ++++++++++++++++++ .../rem_cfg/RemoteConfigResponse.java | 186 ++++++++++ .../src/test/bin/enable_extensions.sh | 1 - .../php/integration/Apache2FpmTests.groovy | 33 +- .../integration/src/test/resources/gdbinit | 4 + 18 files changed, 726 insertions(+), 31 deletions(-) create mode 100644 appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/ConfigV07Handler.groovy create mode 100644 appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/IntegrityCheckException.groovy create mode 100644 appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/MissingContentException.groovy create mode 100644 appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/RemoteConfigRequest.java create mode 100644 appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/RemoteConfigResponse.java create mode 100644 appsec/tests/integration/src/test/resources/gdbinit diff --git a/appsec/src/extension/commands_helpers.c b/appsec/src/extension/commands_helpers.c index 4b5f106736f..3ae618d291c 100644 --- a/appsec/src/extension/commands_helpers.c +++ b/appsec/src/extension/commands_helpers.c @@ -750,6 +750,7 @@ void _handle_telemetry_metric(const char *nonnull key_str, size_t key_len, HANDLE_METRIC("waf.init", DDTRACE_METRIC_TYPE_COUNT); HANDLE_METRIC("waf.input_truncated", DDTRACE_METRIC_TYPE_COUNT); + HANDLE_METRIC("remote_config.first_pull", DDTRACE_METRIC_TYPE_GAUGE); HANDLE_METRIC("remote_config.first_pull", DDTRACE_METRIC_TYPE_GAUGE); HANDLE_METRIC("remote_config.first_pull", DDTRACE_METRIC_TYPE_GAUGE); HANDLE_METRIC( diff --git a/appsec/src/helper/client.cpp b/appsec/src/helper/client.cpp index c1b41faec4e..ed13a4687ef 100644 --- a/appsec/src/helper/client.cpp +++ b/appsec/src/helper/client.cpp @@ -239,6 +239,8 @@ std::shared_ptr client::publish( auto response = std::make_shared(); try { + service_->before_first_publish(); + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) auto res = context_->publish(std::move(command.data)); if (res) { @@ -502,19 +504,27 @@ struct RequestMetricsSubmitter : public metrics::TelemetrySubmitter { void submit_metric( std::string_view name, double value, std::string tags) override { + SPDLOG_TRACE("submit_metric [req]: name={}, value={}, tags={}", name, + value, tags); tel_metrics[name].emplace_back(value, tags); }; void submit_legacy_metric(std::string_view name, double value) override { + SPDLOG_TRACE( + "submit_legacy_metric [req]: name={}, value={}", name, value); metrics[name] = value; }; void submit_legacy_meta(std::string_view name, std::string value) override { + SPDLOG_TRACE( + "submit_legacy_meta [req]: name={}, value={}", name, value); meta[std::string{name}] = value; }; void submit_legacy_meta_copy_key( std::string name, std::string value) override { + SPDLOG_TRACE("submit_legacy_meta_copy_key [req]: name={}, value={}", + name, value); meta[name] = value; } @@ -528,15 +538,12 @@ template void collect_metrics_impl(Response &response, service &service, std::optional &context) { - RequestMetricsSubmitter msubmitter{}; if (context) { context->get_metrics(msubmitter); } service.drain_metrics( [&msubmitter](std::string_view name, double value, std::string tags) { - spdlog::debug( - "submit_metric: name={}, value={}, tags={}", name, value, tags); msubmitter.submit_metric(name, value, std::move(tags)); }); msubmitter.metrics.merge(service.drain_legacy_metrics()); diff --git a/appsec/src/helper/remote_config/client_handler.cpp b/appsec/src/helper/remote_config/client_handler.cpp index 8bc714b5b7c..4837f4ac652 100644 --- a/appsec/src/helper/remote_config/client_handler.cpp +++ b/appsec/src/helper/remote_config/client_handler.cpp @@ -9,6 +9,8 @@ #include "listeners/engine_listener.hpp" #include "listeners/listener.hpp" #include "metrics.hpp" +#include +#include namespace dds::remote_config { @@ -16,10 +18,12 @@ static constexpr std::chrono::milliseconds default_max_interval = 5min; client_handler::client_handler(remote_config::client::ptr &&rc_client, std::shared_ptr service_config, + std::shared_ptr msubmitter, const std::chrono::milliseconds &poll_interval) : service_config_(std::move(service_config)), rc_client_(std::move(rc_client)), poll_interval_(poll_interval), - interval_(poll_interval), max_interval(default_max_interval) + msubmitter_(std::move(msubmitter)), interval_(poll_interval), + max_interval(default_max_interval) { // It starts checking if rc is available rc_action_ = [this] { discover(); }; @@ -56,9 +60,8 @@ client_handler::ptr client_handler::from_settings(service_identifier &&id, } if (eng_settings.rules_file.empty()) { - listeners.emplace_back( - std::make_shared(engine_ptr, - std::move(msubmitter), eng_settings.rules_file_or_default())); + listeners.emplace_back(std::make_shared( + engine_ptr, msubmitter, eng_settings.rules_file_or_default())); } if (listeners.empty()) { @@ -69,7 +72,7 @@ client_handler::ptr client_handler::from_settings(service_identifier &&id, remote_config::settings(rc_settings), std::move(listeners)); return std::make_shared(std::move(rc_client), - std::move(service_config), + std::move(service_config), std::move(msubmitter), std::chrono::milliseconds{rc_settings.poll_interval}); } @@ -102,7 +105,29 @@ void client_handler::handle_error() void client_handler::poll() { try { - rc_client_->poll(); + if (last_success_ != empty_time) { + auto now = std::chrono::steady_clock::now(); + auto elapsed = + std::chrono::duration_cast( + now - last_success_); + msubmitter_->submit_metric("remote_config.last_success"sv, + static_cast(elapsed.count()), {}); + } + + const bool result = rc_client_->poll(); + + auto now = std::chrono::steady_clock::now(); + last_success_ = now; + + auto creation_time = creation_time_.load(std::memory_order_acquire); + if (result && creation_time != empty_time) { + auto elapsed = + std::chrono::duration_cast( + now - creation_time); + msubmitter_->submit_metric("remote_config.first_pull"sv, + static_cast(elapsed.count()), {}); + creation_time_.store(empty_time, std::memory_order_release); + } } catch (dds::remote_config::network_exception & /** e */) { handle_error(); } diff --git a/appsec/src/helper/remote_config/client_handler.hpp b/appsec/src/helper/remote_config/client_handler.hpp index 6536d2cb473..58132188e47 100644 --- a/appsec/src/helper/remote_config/client_handler.hpp +++ b/appsec/src/helper/remote_config/client_handler.hpp @@ -13,6 +13,7 @@ #include "service_identifier.hpp" #include "std_logging.hpp" #include "utils.hpp" +#include #include #include #include @@ -28,6 +29,7 @@ class client_handler { client_handler(remote_config::client::ptr &&rc_client, std::shared_ptr service_config, + std::shared_ptr msubmitter, const std::chrono::milliseconds &poll_interval = 1s); ~client_handler(); @@ -62,6 +64,11 @@ class client_handler { } } + bool has_applied_rc() + { + return creation_time_.load(std::memory_order_acquire) == empty_time; + } + protected: void run(std::future &&exit_signal); void handle_error(); @@ -81,6 +88,13 @@ class client_handler { std::promise exit_; std::thread handler_; + + static constexpr auto empty_time = std::chrono::steady_clock::time_point{}; + + std::shared_ptr msubmitter_; + std::atomic creation_time_{ + std::chrono::steady_clock::now()}; // def value if first poll() done + std::chrono::steady_clock::time_point last_success_{}; }; } // namespace dds::remote_config diff --git a/appsec/src/helper/service.hpp b/appsec/src/helper/service.hpp index c90045259f5..9cf658e48fa 100644 --- a/appsec/src/helper/service.hpp +++ b/appsec/src/helper/service.hpp @@ -14,6 +14,7 @@ #include "service_identifier.hpp" #include "std_logging.hpp" #include "utils.hpp" +#include #include #include #include @@ -47,18 +48,21 @@ class service { void submit_metric(std::string_view metric_name, double value, std::string tags) override { + SPDLOG_TRACE("submit_metric: {} {} {}", metric_name, value, tags); const std::lock_guard lock{pending_metrics_mutex_}; pending_metrics_.emplace_back(metric_name, value, std::move(tags)); } void submit_legacy_metric(std::string_view name, double value) override { + SPDLOG_TRACE("submit_legacy_metric: {} {}", name, value); const std::lock_guard lock{legacy_metrics_mutex_}; legacy_metrics_[name] = value; } void submit_legacy_meta( std::string_view name, std::string value) override { + SPDLOG_TRACE("submit_legacy_meta: {} {}", name, value); const std::lock_guard lock{meta_mutex_}; meta_[std::string{name}] = std::move(value); } @@ -184,6 +188,16 @@ class service { return msubmitter_->drain_legacy_meta(); } + // to be called just before the submitting data to the engine for the first + // time in the request + void before_first_publish() const + { + if (client_handler_ && !client_handler_->has_applied_rc()) { + msubmitter_->submit_metric( + "remote_config.requests_before_running"sv, 1, ""); + } + } + protected: std::shared_ptr engine_{}; std::shared_ptr service_config_{}; diff --git a/appsec/tests/helper/remote_config/client_handler_test.cpp b/appsec/tests/helper/remote_config/client_handler_test.cpp index fb3401ba046..9798794bc4a 100644 --- a/appsec/tests/helper/remote_config/client_handler_test.cpp +++ b/appsec/tests/helper/remote_config/client_handler_test.cpp @@ -17,9 +17,10 @@ class client_handler : public remote_config::client_handler { public: client_handler(remote_config::client::ptr &&rc_client, std::shared_ptr service_config, + std::shared_ptr msubmitter, const std::chrono::milliseconds &poll_interval) : remote_config::client_handler( - std::move(rc_client), service_config, poll_interval) + std::move(rc_client), service_config, msubmitter, poll_interval) {} void set_max_interval(std::chrono::milliseconds new_interval) { @@ -171,6 +172,7 @@ TEST_F(ClientHandlerTest, IfNoProductsAreRequiredRemoteClientIsNotGenerated) TEST_F(ClientHandlerTest, ValidateRCThread) { + auto msubmitter = std::make_shared>(); std::promise poll_call_promise; auto poll_call_future = poll_call_promise.get_future(); std::promise available_call_promise; @@ -184,8 +186,10 @@ TEST_F(ClientHandlerTest, ValidateRCThread) .Times(1) .WillOnce(DoAll(SignalCall(&poll_call_promise), Return(true))); + EXPECT_CALL( + *msubmitter, submit_metric("remote_config.first_pull"sv, _, "")); auto client_handler = remote_config::client_handler( - std::move(rc_client), service_config, 200ms); + std::move(rc_client), service_config, msubmitter, 200ms); client_handler.start(); @@ -196,6 +200,7 @@ TEST_F(ClientHandlerTest, ValidateRCThread) TEST_F(ClientHandlerTest, WhenRcNotAvailableItKeepsDiscovering) { + auto msubmitter = std::make_shared(); auto rc_client = std::make_unique( dds::service_identifier(sid)); EXPECT_CALL(*rc_client, is_remote_config_available) @@ -204,8 +209,8 @@ TEST_F(ClientHandlerTest, WhenRcNotAvailableItKeepsDiscovering) .WillOnce(Return(false)); EXPECT_CALL(*rc_client, poll).Times(0); - auto client_handler = - mock::client_handler(std::move(rc_client), service_config, 500ms); + auto client_handler = mock::client_handler( + std::move(rc_client), service_config, msubmitter, 500ms); client_handler.tick(); client_handler.tick(); @@ -213,6 +218,7 @@ TEST_F(ClientHandlerTest, WhenRcNotAvailableItKeepsDiscovering) TEST_F(ClientHandlerTest, WhenPollFailsItGoesBackToDiscovering) { + auto msubmitter = std::make_shared(); auto rc_client = std::make_unique( dds::service_identifier(sid)); EXPECT_CALL(*rc_client, is_remote_config_available) @@ -223,8 +229,8 @@ TEST_F(ClientHandlerTest, WhenPollFailsItGoesBackToDiscovering) .Times(1) .WillOnce(Throw(dds::remote_config::network_exception("some"))); - auto client_handler = - mock::client_handler(std::move(rc_client), service_config, 500ms); + auto client_handler = mock::client_handler( + std::move(rc_client), service_config, msubmitter, 500ms); client_handler.tick(); client_handler.tick(); client_handler.tick(); @@ -232,6 +238,7 @@ TEST_F(ClientHandlerTest, WhenPollFailsItGoesBackToDiscovering) TEST_F(ClientHandlerTest, WhenDiscoverFailsItStaysOnDiscovering) { + auto msubmitter = std::make_shared(); auto rc_client = std::make_unique( dds::service_identifier(sid)); EXPECT_CALL(*rc_client, is_remote_config_available) @@ -241,8 +248,8 @@ TEST_F(ClientHandlerTest, WhenDiscoverFailsItStaysOnDiscovering) .WillOnce(Throw(dds::remote_config::network_exception("some"))); EXPECT_CALL(*rc_client, poll).Times(0); - auto client_handler = - mock::client_handler(std::move(rc_client), service_config, 50ms); + auto client_handler = mock::client_handler( + std::move(rc_client), service_config, msubmitter, 50ms); client_handler.set_max_interval(100ms); client_handler.tick(); client_handler.tick(); @@ -251,6 +258,7 @@ TEST_F(ClientHandlerTest, WhenDiscoverFailsItStaysOnDiscovering) TEST_F(ClientHandlerTest, ItKeepsPollingWhileNoError) { + auto msubmitter = std::make_shared>(); auto rc_client = std::make_unique( dds::service_identifier(sid)); EXPECT_CALL(*rc_client, is_remote_config_available) @@ -261,25 +269,31 @@ TEST_F(ClientHandlerTest, ItKeepsPollingWhileNoError) .WillOnce(Return(true)) .WillOnce(Return(true)); - auto client_handler = - mock::client_handler(std::move(rc_client), service_config, 500ms); + auto client_handler = mock::client_handler( + std::move(rc_client), service_config, msubmitter, 500ms); + EXPECT_CALL( + *msubmitter, submit_metric("remote_config.first_pull"sv, _, "")); client_handler.tick(); + EXPECT_CALL( + *msubmitter, submit_metric("remote_config.last_success"sv, _, "")); client_handler.tick(); client_handler.tick(); } TEST_F(ClientHandlerTest, ItDoesNotStartIfNoRcClientGiven) { + auto msubmitter = std::make_shared(); auto rc_client = nullptr; - auto client_handler = - remote_config::client_handler(rc_client, service_config, 500ms); + auto client_handler = remote_config::client_handler( + rc_client, service_config, msubmitter, 500ms); EXPECT_FALSE(client_handler.start()); } TEST_F(ClientHandlerTest, ItDoesNotGoOverMaxIfGivenInitialIntervalIsLower) { + auto msubmitter = std::make_shared(); auto rc_client = std::make_unique( dds::service_identifier(sid)); EXPECT_CALL(*rc_client, is_remote_config_available) @@ -287,8 +301,8 @@ TEST_F(ClientHandlerTest, ItDoesNotGoOverMaxIfGivenInitialIntervalIsLower) .WillRepeatedly(Return(false)); auto max_interval = 300ms; - auto client_handler = - mock::client_handler(std::move(rc_client), service_config, 299ms); + auto client_handler = mock::client_handler( + std::move(rc_client), service_config, msubmitter, 299ms); client_handler.set_max_interval(max_interval); client_handler.tick(); @@ -301,6 +315,7 @@ TEST_F(ClientHandlerTest, ItDoesNotGoOverMaxIfGivenInitialIntervalIsLower) TEST_F(ClientHandlerTest, IfInitialIntervalIsHigherThanMaxItBecomesNewMax) { + auto msubmitter = std::make_shared(); auto rc_client = std::make_unique( dds::service_identifier(sid)); EXPECT_CALL(*rc_client, is_remote_config_available) @@ -308,8 +323,8 @@ TEST_F(ClientHandlerTest, IfInitialIntervalIsHigherThanMaxItBecomesNewMax) .WillRepeatedly(Return(false)); auto interval = 200ms; - auto client_handler = - mock::client_handler(std::move(rc_client), service_config, interval); + auto client_handler = mock::client_handler( + std::move(rc_client), service_config, msubmitter, interval); client_handler.set_max_interval(100ms); client_handler.tick(); @@ -322,23 +337,25 @@ TEST_F(ClientHandlerTest, IfInitialIntervalIsHigherThanMaxItBecomesNewMax) TEST_F(ClientHandlerTest, ByDefaultMaxIntervalisFiveMinutes) { + auto msubmitter = std::make_shared(); auto rc_client = std::make_unique( dds::service_identifier(sid)); - auto client_handler = - mock::client_handler(std::move(rc_client), service_config, 200ms); + auto client_handler = mock::client_handler( + std::move(rc_client), service_config, msubmitter, 200ms); EXPECT_EQ(5min, client_handler.get_max_interval()); } TEST_F(ClientHandlerTest, RegisterAndUnregisterRuntimeID) { + auto msubmitter = std::make_shared(); auto rc_client = std::make_unique( dds::service_identifier(sid)); EXPECT_CALL(*rc_client, register_runtime_id).Times(1); EXPECT_CALL(*rc_client, unregister_runtime_id).Times(1); auto client_handler = remote_config::client_handler( - std::move(rc_client), service_config, 200ms); + std::move(rc_client), service_config, msubmitter, 200ms); client_handler.register_runtime_id("something"); client_handler.unregister_runtime_id("something"); diff --git a/appsec/tests/integration/build.gradle b/appsec/tests/integration/build.gradle index 3c508ba1d9c..83b5b9d6d59 100644 --- a/appsec/tests/integration/build.gradle +++ b/appsec/tests/integration/build.gradle @@ -11,6 +11,13 @@ repositories { mavenCentral() } +sourceCompatibility = '11' +targetCompatibility = '11' + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + dependencies { implementation 'ch.qos.logback:logback-classic:1.5.6' implementation 'org.slf4j:jul-to-slf4j:1.7.36' diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy index 9320ece2f07..9ff6a5666b2 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy @@ -213,6 +213,7 @@ class AppSecContainer> extends GenericContain withFileSystemBind(wwwDir, '/test-resources', BindMode.READ_ONLY) withFileSystemBind('src/test/waf/recommended.json', '/etc/recommended.json', BindMode.READ_ONLY) + withFileSystemBind('src/test/resources/gdbinit', '/root/.gdbinit', BindMode.READ_ONLY) withFileSystemBind('src/test/bin/enable_extensions.sh', '/usr/local/bin/enable_extensions.sh', BindMode.READ_ONLY) addVolumeMount("php-appsec-$phpVersion-$phpVariant", '/appsec') diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/ConfigV07Handler.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/ConfigV07Handler.groovy new file mode 100644 index 00000000000..d02573d7b98 --- /dev/null +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/ConfigV07Handler.groovy @@ -0,0 +1,49 @@ +package com.datadog.appsec.php.mock_agent + +import com.datadog.appsec.php.mock_agent.rem_cfg.RemoteConfigResponse +import com.datadog.appsec.php.mock_agent.rem_cfg.RemoteConfigRequest +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.common.collect.Lists +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.javalin.http.Context +import io.javalin.http.Handler +import org.jetbrains.annotations.NotNull + +@Slf4j +@CompileStatic +@Singleton +class ConfigV07Handler implements Handler { + RemoteConfigResponse nextResponse + final List capturedRequests = [] + + @Override + void handle(@NotNull Context context) throws Exception { + RemoteConfigRequest request = context.bodyStreamAsClass(RemoteConfigRequest) + log.debug("Received request with version ${request.client.clientState.targetsVersion}: {}", request.toString()) + synchronized (capturedRequests) { + capturedRequests.add(request) + capturedRequests.notify() + } + if (nextResponse != null) { + context.json(nextResponse) + } else { + context.json([:]) + } + } + + void setNextResponse(RemoteConfigResponse nextResponse) { + this.nextResponse = nextResponse + } + + List drain(long timeoutInMs) { + synchronized (capturedRequests) { + if (capturedRequests.isEmpty()) { + capturedRequests.wait(timeoutInMs) + } + def requests = Lists.newArrayList(capturedRequests) + capturedRequests.clear() + requests + } + } +} diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/InfoHandler.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/InfoHandler.groovy index 3ef291b2957..4e953f3b8a5 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/InfoHandler.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/InfoHandler.groovy @@ -15,7 +15,8 @@ class InfoHandler implements Handler { static class InfoResponse { String version = '7.49.0' List endpoints = [ - '/v0.4/traces' + '/v0.4/traces', + '/v0.7/config', ] } diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockDatadogAgent.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockDatadogAgent.groovy index 159b16ccf0c..3895aaf2d5c 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockDatadogAgent.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockDatadogAgent.groovy @@ -26,6 +26,7 @@ class MockDatadogAgent implements Startable { this.httpServer.put('v0.4/traces', tracesHandler) this.httpServer.get('info', InfoHandler.instance) this.httpServer.post('/telemetry/proxy/api/v2/apmtelemetry', TelemetryHandler.instance) + this.httpServer.post('v0.7/config', ConfigV07Handler.instance) this.httpServer.error(404, ctx -> { log.info("Unmatched request: ${ctx.method()} ${ctx.path()}") }) diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/IntegrityCheckException.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/IntegrityCheckException.groovy new file mode 100644 index 00000000000..26a7e1ef133 --- /dev/null +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/IntegrityCheckException.groovy @@ -0,0 +1,11 @@ +package com.datadog.appsec.php.mock_agent.rem_cfg + +class IntegrityCheckException extends RuntimeException { + IntegrityCheckException(String s) { + super(s) + } + + IntegrityCheckException(String s, Exception e) { + super(s, e) + } +} diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/MissingContentException.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/MissingContentException.groovy new file mode 100644 index 00000000000..1fb28f10f24 --- /dev/null +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/MissingContentException.groovy @@ -0,0 +1,7 @@ +package com.datadog.appsec.php.mock_agent.rem_cfg + +class MissingContentException extends RuntimeException { + MissingContentException(String s) { + super(s) + } +} diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/RemoteConfigRequest.java b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/RemoteConfigRequest.java new file mode 100644 index 00000000000..f2db7ed5c15 --- /dev/null +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/RemoteConfigRequest.java @@ -0,0 +1,320 @@ +package com.datadog.appsec.php.mock_agent.rem_cfg; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +public class RemoteConfigRequest { + + public static RemoteConfigRequest newRequest( + String clientId, + String runtimeId, + String tracerVersion, + Collection productNames, + String serviceName, + List extraServices, + String serviceEnv, + String serviceVersion, + List tags, + ClientInfo.ClientState clientState, + Collection cachedTargetFiles, + long capabilities) { + + ClientInfo.TracerInfo tracerInfo = + new RemoteConfigRequest.ClientInfo.TracerInfo(); + tracerInfo.runtimeId = runtimeId; + tracerInfo.tracerVersion = tracerVersion; + tracerInfo.serviceName = serviceName; + tracerInfo.extraServices = extraServices; + tracerInfo.serviceEnv = serviceEnv; + tracerInfo.serviceVersion = serviceVersion; + tracerInfo.tags = tags; + + ClientInfo clientInfo = + new RemoteConfigRequest.ClientInfo( + clientState, clientId, productNames, tracerInfo, capabilities); + + RemoteConfigRequest rcr = new RemoteConfigRequest(); + rcr.client = clientInfo; + rcr.cachedTargetFiles = cachedTargetFiles; + + return rcr; + } + + private ClientInfo client; + + @JsonProperty("cached_target_files") + private Collection cachedTargetFiles; + + public ClientInfo getClient() { + return this.client; + } + + /** Stores client information for Remote Configuration */ + public static class ClientInfo { + @JsonProperty("state") + public ClientState clientState; + + public String id; + public Collection products; + + @JsonProperty("client_tracer") + public TracerInfo tracerInfo; + + @JsonProperty("client_agent") + public AgentInfo agentInfo = null; // MUST NOT be set + + @JsonProperty("is_tracer") + public boolean isTracer = true; + + @JsonProperty("is_agent") + public Boolean isAgent = null; // MUST NOT be set; + + public byte[] capabilities; + + public ClientInfo() {} + public ClientInfo( + ClientState clientState, + String id, + Collection productNames, + TracerInfo tracerInfo, + final long capabilities) { + this.clientState = clientState; + this.id = id; + this.products = productNames; + this.tracerInfo = tracerInfo; + + // Big-endian encoding of the `long` capabilities, stripping any trailing zero bytes + // (except the first one) + final int size = Math.max(1, Long.BYTES - Long.numberOfLeadingZeros(capabilities) / 8); + this.capabilities = new byte[size]; + for (int i = size - 1; i >= 0; i--) { + this.capabilities[size - i - 1] = (byte) (capabilities >>> (i * 8)); + } + } + + public TracerInfo getTracerInfo() { + return this.tracerInfo; + } + + public static class ClientState { + @JsonProperty("root_version") + public long rootVersion = 1L; + + @JsonProperty("targets_version") + public long targetsVersion; + + @JsonProperty("config_states") + public List configStates = new ArrayList<>(); + + @JsonProperty("has_error") + public boolean hasError; + + public String error; + + @JsonProperty("backend_client_state") + public String backendClientState; + + public void setState( + long targetsVersion, + List configStates, + String error, + String backendClientState) { + this.targetsVersion = targetsVersion; + this.configStates = configStates; + this.error = error; + this.hasError = error != null && !error.isEmpty(); + this.backendClientState = backendClientState; + } + + public static class ConfigState { + public static final int APPLY_STATE_ACKNOWLEDGED = 2; + public static final int APPLY_STATE_ERROR = 3; + + private String id; + private long version; + public String product; + + @JsonProperty("apply_state") + public int applyState; + + @JsonProperty("apply_error") + public String applyError; + + public void setState(String id, long version, String product, String error) { + this.id = id; + this.version = version; + this.product = product; + this.applyState = error == null ? APPLY_STATE_ACKNOWLEDGED : APPLY_STATE_ERROR; + this.applyError = error; + } + + @Override + public String toString() { + return new StringJoiner(", ", ConfigState.class.getSimpleName() + "[", "]") + .add("id='" + id + "'") + .add("version=" + version) + .add("product='" + product + "'") + .add("applyState=" + applyState) + .add("applyError='" + applyError + "'") + .toString(); + } + } + + @Override + public String toString() { + return new StringJoiner(", ", ClientState.class.getSimpleName() + "[", "]") + .add("rootVersion=" + rootVersion) + .add("targetsVersion=" + targetsVersion) + .add("configStates=" + configStates) + .add("hasError=" + hasError) + .add("error='" + error + "'") + .add("backendClientState='" + backendClientState + "'") + .toString(); + } + } + + public static class TracerInfo { + @JsonProperty("runtime_id") + public String runtimeId; + + public String language = "java"; + + public List tags; + + @JsonProperty("tracer_version") + public String tracerVersion; + + @JsonProperty("service") + public String serviceName; + + @JsonProperty("extra_services") + public List extraServices; + + @JsonProperty("env") + public String serviceEnv; + + @JsonProperty("app_version") + public String serviceVersion; + + @Override + public String toString() { + return new StringJoiner(", ", TracerInfo.class.getSimpleName() + "[", "]") + .add("runtimeId='" + runtimeId + "'") + .add("language='" + language + "'") + .add("tags=" + tags) + .add("tracerVersion='" + tracerVersion + "'") + .add("serviceName='" + serviceName + "'") + .add("extraServices=" + extraServices) + .add("serviceEnv='" + serviceEnv + "'") + .add("serviceVersion='" + serviceVersion + "'") + .toString(); + } + } + + private class AgentInfo { + String name; + String version; + + @Override + public String toString() { + return new StringJoiner(", ", AgentInfo.class.getSimpleName() + "[", "]") + .add("name='" + name + "'") + .add("version='" + version + "'") + .toString(); + } + } + + @Override + public String toString() { + return new StringJoiner(", ", ClientInfo.class.getSimpleName() + "[", "]") + .add("clientState=" + clientState) + .add("id='" + id + "'") + .add("products=" + products) + .add("tracerInfo=" + tracerInfo) + .add("agentInfo=" + agentInfo) + .add("isTracer=" + isTracer) + .add("isAgent=" + isAgent) + .add("capabilities=" + Arrays.toString(capabilities)) + .toString(); + } + } + + public static class CachedTargetFile { + public String path; + public long length; + public List hashes; + + public CachedTargetFile() {} + public CachedTargetFile( + String path, long length, Map hashes) { + this.path = path; + this.length = length; + List hashesList = + hashes.entrySet().stream() + .map(e -> new TargetFileHash(e.getKey(), e.getValue())) + .collect(Collectors.toList()); + this.hashes = hashesList; + } + + public boolean hashesMatch(Map hashesMap) { + if (this.hashes == null) { + return false; + } + + if (hashesMap.size() != this.hashes.size()) { + return false; + } + + for (TargetFileHash tfh : hashes) { + String digest = hashesMap.get(tfh.algorithm); + if (!digest.equals(tfh.hash)) { + return false; + } + } + + return true; + } + + public class TargetFileHash { + String algorithm; + String hash; + + TargetFileHash() {} + TargetFileHash(String algorithm, String hash) { + this.algorithm = algorithm; + this.hash = hash; + } + + @Override + public String toString() { + return new StringJoiner(", ", TargetFileHash.class.getSimpleName() + "[", "]") + .add("algorithm='" + algorithm + "'") + .add("hash='" + hash + "'") + .toString(); + } + } + + @Override + public String toString() { + return new StringJoiner(", ", CachedTargetFile.class.getSimpleName() + "[", "]") + .add("path='" + path + "'") + .add("length=" + length) + .add("hashes=" + hashes) + .toString(); + } + } + + @Override + public String toString() { + return new StringJoiner(", ", RemoteConfigRequest.class.getSimpleName() + "[", "]") + .add("client=" + client) + .add("cachedTargetFiles=" + cachedTargetFiles) + .toString(); + } +} diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/RemoteConfigResponse.java b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/RemoteConfigResponse.java new file mode 100644 index 00000000000..0c77f63dc80 --- /dev/null +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/RemoteConfigResponse.java @@ -0,0 +1,186 @@ +package com.datadog.appsec.php.mock_agent.rem_cfg; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class RemoteConfigResponse { + @JsonProperty("client_configs") + public List clientConfigs; + + @JsonDeserialize(using = TargetsDeserializer.class) + private Targets targets; + + @JsonProperty("target_files") + public List targetFiles; + + public Targets.ConfigTarget getTarget(String configKey) { + return this.targets.targetsSigned.targets.get(configKey); + } + + public String getTargetsSignature(String keyId) { + for (Targets.Signature signature : this.targets.signatures) { + if (keyId.equals(signature.keyId)) { + return signature.signature; + } + } + + throw new IntegrityCheckException("Missing signature for key " + keyId); + } + + public Targets.TargetsSigned getTargetsSigned() { + return this.targets.targetsSigned; + } + + public byte[] getFileContents(String configKey) { + + if (targetFiles == null) { + throw new MissingContentException("No content for " + configKey); + } + + try { + for (TargetFile targetFile : this.targetFiles) { + if (!configKey.equals(targetFile.path)) { + continue; + } + + Targets.ConfigTarget configTarget = getTarget(configKey); + String hashStr; + if (configTarget == null + || configTarget.hashes == null + || (hashStr = configTarget.hashes.get("sha256")) == null) { + throw new IntegrityCheckException("No sha256 hash present for " + configKey); + } + BigInteger expectedHash = new BigInteger(hashStr, 16); + + String raw = targetFile.raw; + byte[] decode = Base64.getDecoder().decode(raw); + BigInteger gottenHash = sha256(decode); + if (!expectedHash.equals(gottenHash)) { + throw new IntegrityCheckException( + "File " + + configKey + + " does not " + + "have the expected sha256 hash: Expected " + + expectedHash.toString(16) + + ", but got " + + gottenHash.toString(16)); + } + if (decode.length != configTarget.length) { + throw new IntegrityCheckException( + "File " + + configKey + + " does not " + + "have the expected length: Expected " + + configTarget.length + + ", but got " + + decode.length); + } + + return decode; + } + } catch (IntegrityCheckException e) { + throw e; + } catch (Exception exception) { + throw new IntegrityCheckException( + "Could not get file contents from remote config, file " + configKey, exception); + } + + throw new MissingContentException("No content for " + configKey); + } + + private static BigInteger sha256(byte[] bytes) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hash = digest.digest(bytes); + return new BigInteger(1, hash); + } catch (NoSuchAlgorithmException e) { + throw new UndeclaredThrowableException(e); + } + } + + public List getClientConfigs() { + return this.clientConfigs != null ? this.clientConfigs : Collections.emptyList(); + } + + public static class Targets { + public List signatures; + + @JsonProperty("signed") + public TargetsSigned targetsSigned; + + public static class Signature { + @JsonProperty("keyid") + public String keyId; + + @JsonProperty("sig") + public String signature; + } + + public static class TargetsSigned { + @JsonProperty("_type") + public String type; + + public TargetsCustom custom; + public Instant expires; + + @JsonProperty("spec_version") + public String specVersion; + + public long version; + public Map targets; + + public static class TargetsCustom { + @JsonProperty("opaque_backend_state") + public String opaqueBackendState; + } + } + + public static class ConfigTarget { + public ConfigTargetCustom custom; + public Map hashes; + public long length; + + public static class ConfigTargetCustom { + @JsonProperty("v") + public long version; + } + } + } + + public static class TargetFile { + public String path; + public String raw; + } + + public static class TargetsDeserializer extends JsonDeserializer { + @Override + public Targets deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + String targetsJsonBase64 = node.asText(); + byte[] targetsJsonDecoded = + Base64.getDecoder().decode(targetsJsonBase64.getBytes(StandardCharsets.ISO_8859_1)); + + JsonParser defParser = jsonParser.getCodec().getFactory().createParser(targetsJsonDecoded); + + JsonDeserializer defaultDeserializer = deserializationContext.findRootValueDeserializer( + deserializationContext.constructType(Targets.class)); + return (Targets) defaultDeserializer.deserialize(defParser, deserializationContext); + } + } +} diff --git a/appsec/tests/integration/src/test/bin/enable_extensions.sh b/appsec/tests/integration/src/test/bin/enable_extensions.sh index ff3b67edcfe..2dd36363b4c 100755 --- a/appsec/tests/integration/src/test/bin/enable_extensions.sh +++ b/appsec/tests/integration/src/test/bin/enable_extensions.sh @@ -24,7 +24,6 @@ if [[ -f /appsec/ddappsec.so && -d /project ]]; then echo datadog.appsec.helper_extra_args=--log_level info echo datadog.appsec.helper_path=/appsec/ddappsec-helper echo datadog.appsec.helper_log_file=/tmp/logs/helper.log - echo datadog.appsec.rules=/etc/recommended.json echo datadog.appsec.log_file=/tmp/logs/appsec.log echo datadog.appsec.log_level=debug } >> /etc/php/php.ini diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/Apache2FpmTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/Apache2FpmTests.groovy index 319e162d913..7b5be0f4fc4 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/Apache2FpmTests.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/Apache2FpmTests.groovy @@ -70,10 +70,22 @@ class Apache2FpmTests implements CommonTests { TelemetryHelpers.Metric wafInit TelemetryHelpers.Metric wafReq1 TelemetryHelpers.Metric wafReq2 + TelemetryHelpers.Metric rcFirstPull + TelemetryHelpers.Metric rcRequestsB4Running + TelemetryHelpers.Metric rcLastSuccess List messages = [] def deadline = System.currentTimeSeconds() + 30 - while ((!wafInit || !wafReq1 || !wafReq2) && System.currentTimeSeconds() < deadline) { + def lastHttpReq = System.currentTimeSeconds() + while ((!wafInit || !wafReq1 || !wafReq2 || !rcFirstPull || !rcLastSuccess) && System.currentTimeSeconds() < deadline) { + if (System.currentTimeSeconds() - lastHttpReq > 5) { + lastHttpReq = System.currentTimeSeconds() + // used to flush global (not request-bound) telemetry metrics + def request = container.buildReq('/hello.php').GET().build() + trace = container.traceFromRequest(req, ofString()) { HttpResponse resp -> + assert resp.body().size() > 0 + } + } def telData = container.drainTelemetry(500) messages.addAll( TelemetryHelpers.filterMessages(telData, TelemetryHelpers.GenerateMetrics)) @@ -81,6 +93,9 @@ class Apache2FpmTests implements CommonTests { wafInit = allSeries.find { it.name == 'waf.init' } wafReq1 = allSeries.find { it.name == 'waf.requests' && it.tags.size() == 2 } wafReq2 = allSeries.find { it.name == 'waf.requests' && it.tags.size() == 3 } + rcFirstPull = allSeries.find { it.name == 'remote_config.first_pull' } + rcRequestsB4Running = allSeries.find { it.name == 'remote_config.requests_before_running' } + rcLastSuccess = allSeries.find { it.name == 'remote_config.last_success' } } assert wafInit != null @@ -100,6 +115,22 @@ class Apache2FpmTests implements CommonTests { assert wafReq2 != null assert wafReq2.tags.find { it == 'rule_triggered:true' } + + assert rcFirstPull != null + assert rcFirstPull.namespace == 'appsec' + assert rcFirstPull.points[0][1] > 0 + assert rcFirstPull.type == 'gauge' + + assert rcRequestsB4Running != null + assert rcRequestsB4Running.namespace == 'appsec' + // the first request triggers helper/RC start so it will never be covered + assert rcRequestsB4Running.points[0][1] >= 1 + assert rcRequestsB4Running.type == 'count' + + assert rcLastSuccess != null + assert rcLastSuccess.namespace == 'appsec' + assert rcLastSuccess.points[0][1] > 0 + assert rcLastSuccess.type == 'gauge' } @Test diff --git a/appsec/tests/integration/src/test/resources/gdbinit b/appsec/tests/integration/src/test/resources/gdbinit new file mode 100644 index 00000000000..aaf915b3cc7 --- /dev/null +++ b/appsec/tests/integration/src/test/resources/gdbinit @@ -0,0 +1,4 @@ +set verbose off +set confirm off +handle SIGPIPE nostop print pass +catch throw