From 5b6708bfe469e69405540b1ca566348a4c0d13db Mon Sep 17 00:00:00 2001 From: Nitay Kufert Date: Wed, 17 Apr 2019 16:52:31 +0300 Subject: [PATCH 1/3] feature/removeTagsFromMetricNameDDReporter --- src/main/resources/application.conf | 2 ++ src/main/scala/config/MetricsSettings.scala | 34 +++++++++++++------ .../reporter/RemoraDatadogReporter.scala | 16 ++++----- .../reporter/RemoraDatadogReporterSpec.scala | 21 +++++++++--- 4 files changed, 51 insertions(+), 22 deletions(-) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 5c6d864..0e28b47 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -81,6 +81,8 @@ metrics { port = ${?DATADOG_AGENT_PORT} tracked_consumer_group = [] tracked_consumer_group = ${?DATADOG_CONSUMER_GROUPS} + removeTagsFromMetricName = false + removeTagsFromMetricName = ${?REMOVE_TAGS_FROM_METRIC_NAME} } toregistry { enabled = false diff --git a/src/main/scala/config/MetricsSettings.scala b/src/main/scala/config/MetricsSettings.scala index 2b322f5..efc5284 100644 --- a/src/main/scala/config/MetricsSettings.scala +++ b/src/main/scala/config/MetricsSettings.scala @@ -3,25 +3,39 @@ package config import com.typesafe.config.Config import scala.collection.JavaConverters._ -case class RegistryOptions(enabled: Boolean = true, intervalSeconds : Int) -case class CloudWatch(enabled: Boolean = false, name : String, intervalMinutes: Int) -case class DataDog(enabled: Boolean = false, name : String, intervalMinutes: Int, agentHost: String, agentPort: Int, trackedConsumerGroups: List[String]) +case class RegistryOptions(enabled: Boolean = true, intervalSeconds: Int) +case class CloudWatch(enabled: Boolean = false, name: String, intervalMinutes: Int) + +case class DataDog(enabled: Boolean = false, + name: String, + intervalMinutes: Int, + agentHost: String, + agentPort: Int, + trackedConsumerGroups: List[String], + removeTagsFromMetricName: Boolean) case class MetricsSettings(cloudWatch: CloudWatch, dataDog: DataDog, registryOptions: RegistryOptions) object MetricsSettings { def apply(config: Config): MetricsSettings = MetricsSettings( - CloudWatch(config.getBoolean("metrics.cloudwatch.enabled"), + CloudWatch( + config.getBoolean("metrics.cloudwatch.enabled"), config.getString("metrics.cloudwatch.name"), - config.getInt("metrics.cloudwatch.interval_minutes")), - DataDog(config.getBoolean("metrics.datadog.enabled"), + config.getInt("metrics.cloudwatch.interval_minutes") + ), + DataDog( + config.getBoolean("metrics.datadog.enabled"), config.getString("metrics.datadog.name"), config.getInt("metrics.datadog.interval_minutes"), config.getString("metrics.datadog.host"), config.getInt("metrics.datadog.port"), - config.getStringList("metrics.datadog.tracked_consumer_group").asScala.toList + config.getStringList("metrics.datadog.tracked_consumer_group").asScala.toList, + config.getBoolean("metrics.datadog.removeTagsFromMetricName") ), - RegistryOptions(config.getBoolean("metrics.toregistry.enabled"), - config.getInt("metrics.toregistry.interval_seconds"))) -} \ No newline at end of file + RegistryOptions( + config.getBoolean("metrics.toregistry.enabled"), + config.getInt("metrics.toregistry.interval_seconds") + ) + ) +} diff --git a/src/main/scala/reporter/RemoraDatadogReporter.scala b/src/main/scala/reporter/RemoraDatadogReporter.scala index 6c5d890..bc100a5 100644 --- a/src/main/scala/reporter/RemoraDatadogReporter.scala +++ b/src/main/scala/reporter/RemoraDatadogReporter.scala @@ -11,7 +11,6 @@ import org.coursera.metrics.datadog.{DatadogReporter, MetricNameFormatter} class RemoraDatadogReporter(metricRegistry: MetricRegistry, datadogConfig: DataDog) { - private val transport = new UdpTransport.Builder().withStatsdHost(datadogConfig.agentHost).withPort(datadogConfig.agentPort).build /** @@ -24,12 +23,13 @@ class RemoraDatadogReporter(metricRegistry: MetricRegistry, datadogConfig: DataD } } - private val metricNameFormatter: MetricNameFormatter = new MetricNameFormatter { + private def metricNameFormatter(removeTagsFromMetricName: Boolean): MetricNameFormatter = new MetricNameFormatter { override def format(nameWithPrefix: String, path: String*): String = { RegistryKafkaMetric.decode(nameWithPrefix.replaceFirst(s"${datadogConfig.name}\\.","")) match { case Some(registryKafkaMetric) => - val builder = new TaggedNameBuilder().metricName(nameWithPrefix) - .addTag("topic", registryKafkaMetric.topic) + val builder = new TaggedNameBuilder().metricName( + if (removeTagsFromMetricName) buildNameWithoutTags(registryKafkaMetric) else nameWithPrefix + ).addTag("topic", registryKafkaMetric.topic) .addTag("group", registryKafkaMetric.group) registryKafkaMetric.partition.foreach(p => builder.addTag("partition", p)) builder.build().encode() @@ -38,15 +38,15 @@ class RemoraDatadogReporter(metricRegistry: MetricRegistry, datadogConfig: DataD } } + private def buildNameWithoutTags(registryKafkaMetric: RegistryKafkaMetric): String = + s"${datadogConfig.name}.${registryKafkaMetric.prefix}.${registryKafkaMetric.suffix}" + def startReporter(): Unit = DatadogReporter .forRegistry(metricRegistry) .withPrefix(datadogConfig.name) .withTransport(transport) .filter(kafkaConsumerGroupFilter) - .withMetricNameFormatter(metricNameFormatter) + .withMetricNameFormatter(metricNameFormatter(datadogConfig.removeTagsFromMetricName)) .build .start(datadogConfig.intervalMinutes, TimeUnit.MINUTES) - - } - diff --git a/src/test/scala/reporter/RemoraDatadogReporterSpec.scala b/src/test/scala/reporter/RemoraDatadogReporterSpec.scala index 791b207..cf6c8b3 100644 --- a/src/test/scala/reporter/RemoraDatadogReporterSpec.scala +++ b/src/test/scala/reporter/RemoraDatadogReporterSpec.scala @@ -11,7 +11,8 @@ class RemoraDatadogReporterSpec extends FlatSpec with Matchers with PrivateMetho private val metricRegistry: MetricRegistry = new MetricRegistry private val metric: Metric = mock[Metric] - private val config = DataDog(enabled = true, "test", 1, "localhost", 8125, List.empty) + private val config = DataDog(enabled = true, "test", 1, "localhost", 8125, List.empty, removeTagsFromMetricName = false) + private val configRemoveTags = DataDog(enabled = true, "test", 1, "localhost", 8125, List.empty, removeTagsFromMetricName = true) "Metrics filter" should "match any metric when no filter is given" in { val filter = buildMetricFilter(List.empty) @@ -50,14 +51,26 @@ class RemoraDatadogReporterSpec extends FlatSpec with Matchers with PrivateMetho formatter.format(s"${config.name}.gauge.test_1_faulty_test-consumer__lag") should be(s"${config.name}.gauge.test_1_faulty_test-consumer__lag") } - private def buildMetricFilter(kafkaConsumerList: List[String]): MetricFilter = { - val config = DataDog(enabled = true, "test", 1, "localhost", 8125, kafkaConsumerList) + "Metric name formatter without tags" should "add tag information if metric is well formatted" in { + val formatter = getMetricNameFormatter(configRemoveTags) + + formatter.format(s"${configRemoveTags.name}.gauge.test.1.test-consumer.lag") should be(s"${configRemoveTags.name}.gauge.lag[topic:test,group:test-consumer,partition:1]") + } + + it should "not add partition tag information if no partition" in { + val formatter = getMetricNameFormatter(configRemoveTags) + + formatter.format(s"${configRemoveTags.name}.gauge.test-topic.test-consumer.totalLag") should be(s"${configRemoveTags.name}.gauge.totalLag[topic:test-topic,group:test-consumer]") + } + + private def buildMetricFilter(kafkaConsumerList: List[String], removeTags: Boolean = false): MetricFilter = { + val config = DataDog(enabled = true, "test", 1, "localhost", 8125, kafkaConsumerList, removeTags) val reporter = new RemoraDatadogReporter(metricRegistry, config) reporter invokePrivate PrivateMethod[MetricFilter]('kafkaConsumerGroupFilter)() } private def getMetricNameFormatter(config: DataDog): MetricNameFormatter = { val reporter = new RemoraDatadogReporter(metricRegistry, config) - reporter invokePrivate PrivateMethod[MetricNameFormatter]('metricNameFormatter)() + reporter invokePrivate PrivateMethod[MetricNameFormatter]('metricNameFormatter)(config.removeTagsFromMetricName) } } From 0f1f6b49fcc8f00dfcd86ba11cacd04fee6f83eb Mon Sep 17 00:00:00 2001 From: Nitay Kufert Date: Sun, 28 Apr 2019 12:00:34 +0300 Subject: [PATCH 2/3] feature/removeTagsFromMetricNameDDReporter: rename configurations. post PR --- src/main/resources/application.conf | 4 ++-- src/main/scala/config/MetricsSettings.scala | 4 ++-- src/main/scala/reporter/RemoraDatadogReporter.scala | 6 +++--- src/test/scala/reporter/RemoraDatadogReporterSpec.scala | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 0e28b47..21af876 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -81,8 +81,8 @@ metrics { port = ${?DATADOG_AGENT_PORT} tracked_consumer_group = [] tracked_consumer_group = ${?DATADOG_CONSUMER_GROUPS} - removeTagsFromMetricName = false - removeTagsFromMetricName = ${?REMOVE_TAGS_FROM_METRIC_NAME} + remove_tags_from_metric_name = false + remove_tags_from_metric_name = ${?DATADOG_REMOVE_TAGS_FROM_METRIC_NAME} } toregistry { enabled = false diff --git a/src/main/scala/config/MetricsSettings.scala b/src/main/scala/config/MetricsSettings.scala index efc5284..ef59787 100644 --- a/src/main/scala/config/MetricsSettings.scala +++ b/src/main/scala/config/MetricsSettings.scala @@ -12,7 +12,7 @@ case class DataDog(enabled: Boolean = false, agentHost: String, agentPort: Int, trackedConsumerGroups: List[String], - removeTagsFromMetricName: Boolean) + remove_tags_from_metric_name: Boolean) case class MetricsSettings(cloudWatch: CloudWatch, dataDog: DataDog, registryOptions: RegistryOptions) @@ -31,7 +31,7 @@ object MetricsSettings { config.getString("metrics.datadog.host"), config.getInt("metrics.datadog.port"), config.getStringList("metrics.datadog.tracked_consumer_group").asScala.toList, - config.getBoolean("metrics.datadog.removeTagsFromMetricName") + config.getBoolean("metrics.datadog.remove_tags_from_metric_name") ), RegistryOptions( config.getBoolean("metrics.toregistry.enabled"), diff --git a/src/main/scala/reporter/RemoraDatadogReporter.scala b/src/main/scala/reporter/RemoraDatadogReporter.scala index bc100a5..315d72f 100644 --- a/src/main/scala/reporter/RemoraDatadogReporter.scala +++ b/src/main/scala/reporter/RemoraDatadogReporter.scala @@ -23,12 +23,12 @@ class RemoraDatadogReporter(metricRegistry: MetricRegistry, datadogConfig: DataD } } - private def metricNameFormatter(removeTagsFromMetricName: Boolean): MetricNameFormatter = new MetricNameFormatter { + private def metricNameFormatter(remove_tags_from_metric_name: Boolean): MetricNameFormatter = new MetricNameFormatter { override def format(nameWithPrefix: String, path: String*): String = { RegistryKafkaMetric.decode(nameWithPrefix.replaceFirst(s"${datadogConfig.name}\\.","")) match { case Some(registryKafkaMetric) => val builder = new TaggedNameBuilder().metricName( - if (removeTagsFromMetricName) buildNameWithoutTags(registryKafkaMetric) else nameWithPrefix + if (remove_tags_from_metric_name) buildNameWithoutTags(registryKafkaMetric) else nameWithPrefix ).addTag("topic", registryKafkaMetric.topic) .addTag("group", registryKafkaMetric.group) registryKafkaMetric.partition.foreach(p => builder.addTag("partition", p)) @@ -46,7 +46,7 @@ class RemoraDatadogReporter(metricRegistry: MetricRegistry, datadogConfig: DataD .withPrefix(datadogConfig.name) .withTransport(transport) .filter(kafkaConsumerGroupFilter) - .withMetricNameFormatter(metricNameFormatter(datadogConfig.removeTagsFromMetricName)) + .withMetricNameFormatter(metricNameFormatter(datadogConfig.remove_tags_from_metric_name)) .build .start(datadogConfig.intervalMinutes, TimeUnit.MINUTES) } diff --git a/src/test/scala/reporter/RemoraDatadogReporterSpec.scala b/src/test/scala/reporter/RemoraDatadogReporterSpec.scala index cf6c8b3..0939c5d 100644 --- a/src/test/scala/reporter/RemoraDatadogReporterSpec.scala +++ b/src/test/scala/reporter/RemoraDatadogReporterSpec.scala @@ -11,8 +11,8 @@ class RemoraDatadogReporterSpec extends FlatSpec with Matchers with PrivateMetho private val metricRegistry: MetricRegistry = new MetricRegistry private val metric: Metric = mock[Metric] - private val config = DataDog(enabled = true, "test", 1, "localhost", 8125, List.empty, removeTagsFromMetricName = false) - private val configRemoveTags = DataDog(enabled = true, "test", 1, "localhost", 8125, List.empty, removeTagsFromMetricName = true) + private val config = DataDog(enabled = true, "test", 1, "localhost", 8125, List.empty, remove_tags_from_metric_name = false) + private val configRemoveTags = DataDog(enabled = true, "test", 1, "localhost", 8125, List.empty, remove_tags_from_metric_name = true) "Metrics filter" should "match any metric when no filter is given" in { val filter = buildMetricFilter(List.empty) @@ -71,6 +71,6 @@ class RemoraDatadogReporterSpec extends FlatSpec with Matchers with PrivateMetho private def getMetricNameFormatter(config: DataDog): MetricNameFormatter = { val reporter = new RemoraDatadogReporter(metricRegistry, config) - reporter invokePrivate PrivateMethod[MetricNameFormatter]('metricNameFormatter)(config.removeTagsFromMetricName) + reporter invokePrivate PrivateMethod[MetricNameFormatter]('metricNameFormatter)(config.remove_tags_from_metric_name) } } From 1441579b6755dbc73963963a7bcb2952fdbb0185 Mon Sep 17 00:00:00 2001 From: Nitay Kufert Date: Mon, 29 Apr 2019 15:06:15 +0300 Subject: [PATCH 3/3] feature/removeTagsFromMetricNameDDReporter: fix replaceAll mistake, case class fields with camelCase --- src/main/scala/config/MetricsSettings.scala | 2 +- src/main/scala/reporter/RemoraDatadogReporter.scala | 6 +++--- src/test/scala/reporter/RemoraDatadogReporterSpec.scala | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/scala/config/MetricsSettings.scala b/src/main/scala/config/MetricsSettings.scala index ef59787..88c7189 100644 --- a/src/main/scala/config/MetricsSettings.scala +++ b/src/main/scala/config/MetricsSettings.scala @@ -12,7 +12,7 @@ case class DataDog(enabled: Boolean = false, agentHost: String, agentPort: Int, trackedConsumerGroups: List[String], - remove_tags_from_metric_name: Boolean) + removeTagsFromMetricName: Boolean) case class MetricsSettings(cloudWatch: CloudWatch, dataDog: DataDog, registryOptions: RegistryOptions) diff --git a/src/main/scala/reporter/RemoraDatadogReporter.scala b/src/main/scala/reporter/RemoraDatadogReporter.scala index 315d72f..bc100a5 100644 --- a/src/main/scala/reporter/RemoraDatadogReporter.scala +++ b/src/main/scala/reporter/RemoraDatadogReporter.scala @@ -23,12 +23,12 @@ class RemoraDatadogReporter(metricRegistry: MetricRegistry, datadogConfig: DataD } } - private def metricNameFormatter(remove_tags_from_metric_name: Boolean): MetricNameFormatter = new MetricNameFormatter { + private def metricNameFormatter(removeTagsFromMetricName: Boolean): MetricNameFormatter = new MetricNameFormatter { override def format(nameWithPrefix: String, path: String*): String = { RegistryKafkaMetric.decode(nameWithPrefix.replaceFirst(s"${datadogConfig.name}\\.","")) match { case Some(registryKafkaMetric) => val builder = new TaggedNameBuilder().metricName( - if (remove_tags_from_metric_name) buildNameWithoutTags(registryKafkaMetric) else nameWithPrefix + if (removeTagsFromMetricName) buildNameWithoutTags(registryKafkaMetric) else nameWithPrefix ).addTag("topic", registryKafkaMetric.topic) .addTag("group", registryKafkaMetric.group) registryKafkaMetric.partition.foreach(p => builder.addTag("partition", p)) @@ -46,7 +46,7 @@ class RemoraDatadogReporter(metricRegistry: MetricRegistry, datadogConfig: DataD .withPrefix(datadogConfig.name) .withTransport(transport) .filter(kafkaConsumerGroupFilter) - .withMetricNameFormatter(metricNameFormatter(datadogConfig.remove_tags_from_metric_name)) + .withMetricNameFormatter(metricNameFormatter(datadogConfig.removeTagsFromMetricName)) .build .start(datadogConfig.intervalMinutes, TimeUnit.MINUTES) } diff --git a/src/test/scala/reporter/RemoraDatadogReporterSpec.scala b/src/test/scala/reporter/RemoraDatadogReporterSpec.scala index 0939c5d..cf6c8b3 100644 --- a/src/test/scala/reporter/RemoraDatadogReporterSpec.scala +++ b/src/test/scala/reporter/RemoraDatadogReporterSpec.scala @@ -11,8 +11,8 @@ class RemoraDatadogReporterSpec extends FlatSpec with Matchers with PrivateMetho private val metricRegistry: MetricRegistry = new MetricRegistry private val metric: Metric = mock[Metric] - private val config = DataDog(enabled = true, "test", 1, "localhost", 8125, List.empty, remove_tags_from_metric_name = false) - private val configRemoveTags = DataDog(enabled = true, "test", 1, "localhost", 8125, List.empty, remove_tags_from_metric_name = true) + private val config = DataDog(enabled = true, "test", 1, "localhost", 8125, List.empty, removeTagsFromMetricName = false) + private val configRemoveTags = DataDog(enabled = true, "test", 1, "localhost", 8125, List.empty, removeTagsFromMetricName = true) "Metrics filter" should "match any metric when no filter is given" in { val filter = buildMetricFilter(List.empty) @@ -71,6 +71,6 @@ class RemoraDatadogReporterSpec extends FlatSpec with Matchers with PrivateMetho private def getMetricNameFormatter(config: DataDog): MetricNameFormatter = { val reporter = new RemoraDatadogReporter(metricRegistry, config) - reporter invokePrivate PrivateMethod[MetricNameFormatter]('metricNameFormatter)(config.remove_tags_from_metric_name) + reporter invokePrivate PrivateMethod[MetricNameFormatter]('metricNameFormatter)(config.removeTagsFromMetricName) } }