Add Prefix Option to Prometheus Telemetry Config (linkerd#1657)

* Update Prometheus config docs with 'prefix' key

* Implement prefix for Prometheus as optional config value

* Add tests for prefix for Prometheus metric keys

* Update documentation for Prometheus config key 'prefix'

linkerd/docs/telemetry.md

@@ -22,6 +22,7 @@ experimental | `false` | Set this to `true` to enable the telemeter if it is exp
 telemetry:
 - kind: io.l5d.prometheus
   path: /admin/metrics/prometheus
+  prefix: linkerd_
 ```
 
 kind: `io.l5d.prometheus`
@@ -33,6 +34,7 @@ Exposes admin endpoints:
 Key | Default Value | Description
 --- | ------------- | -----------
 path | `/admin/metrics/prometheus` | HTTP path where linkerd exposes Prometheus metrics
+prefix | No prefix | Prefix for exposed Prometheus metrics
 
 ## InfluxDB
 

telemetry/prometheus/src/main/scala/io/buoyant/telemetry/prometheus/PrometheusTelemeter.scala

@@ -14,7 +14,7 @@ import io.buoyant.telemetry.{Metric, MetricsTree, Telemeter}
  * histogram summaries directly off of the MetricsTree and assumes that stats
  * are being snapshotted at some appropriate interval.
  */
-class PrometheusTelemeter(metrics: MetricsTree, private[prometheus] val handlerPath: String) extends Telemeter with Admin.WithHandlers {
+class PrometheusTelemeter(metrics: MetricsTree, private[prometheus] val handlerPath: String, private[prometheus] val handlerPrefix: String) extends Telemeter with Admin.WithHandlers {
 
   private[prometheus] val handler = Service.mk { request: Request =>
     val response = Response()
@@ -80,7 +80,7 @@ class PrometheusTelemeter(metrics: MetricsTree, private[prometheus] val handlerP
       case _ => (prefix0, labels0)
     }
 
-    val key = escapeKey(prefix1.mkString(":"))
+    val key = escapeKey(handlerPrefix + prefix1.mkString(":"))
 
     tree.metric match {
       case c: Metric.Counter =>

telemetry/prometheus/src/main/scala/io/buoyant/telemetry/prometheus/PrometheusTelemeterInitializer.scala

@@ -12,10 +12,11 @@ class PrometheusTelemeterInitializer extends TelemeterInitializer {
 
 object PrometheusTelemeterInitializer extends PrometheusTelemeterInitializer
 
-class PrometheusConfig(path: Option[String]) extends TelemeterConfig {
+class PrometheusConfig(path: Option[String], prefix: Option[String]) extends TelemeterConfig {
   @JsonIgnore def mk(params: Stack.Params): Telemeter =
     new PrometheusTelemeter(
       params[MetricsTree],
-      path.getOrElse("/admin/metrics/prometheus")
+      path.getOrElse("/admin/metrics/prometheus"),
+      prefix.getOrElse("")
     )
 }

telemetry/prometheus/src/test/scala/io/buoyant/telemetry/prometheus/PrometheusTelemeterInitializerTest.scala

@@ -12,6 +12,7 @@ class PrometheusTelemeterInitializerTest extends FunSuite {
     val yaml =
       """|kind: io.l5d.prometheus
          |path: /metrics
+         |prefix: some_prefix_
          |""".stripMargin
 
     val config = Parser.objectMapper(yaml, Seq(LoadService[TelemeterInitializer]))
@@ -34,4 +35,42 @@ class PrometheusTelemeterInitializerTest extends FunSuite {
     val telemeter = config.mk(Stack.Params.empty).asInstanceOf[PrometheusTelemeter]
     assert(telemeter.handlerPath === "/admin/metrics/prometheus")
   }
+
+  test("io.l5d.prometheus telemeter path") {
+    val yaml =
+      """|kind: io.l5d.prometheus
+         |path: /some/path
+         |""".stripMargin
+
+    val config = Parser.objectMapper(yaml, Seq(LoadService[TelemeterInitializer]))
+      .readValue[TelemeterConfig](yaml)
+
+    val telemeter = config.mk(Stack.Params.empty).asInstanceOf[PrometheusTelemeter]
+    assert(telemeter.handlerPath === "/some/path")
+  }
+
+  test("io.l5d.prometheus telemeter default prefix") {
+    val yaml =
+      """|kind: io.l5d.prometheus
+         |""".stripMargin
+
+    val config = Parser.objectMapper(yaml, Seq(LoadService[TelemeterInitializer]))
+      .readValue[TelemeterConfig](yaml)
+
+    val telemeter = config.mk(Stack.Params.empty).asInstanceOf[PrometheusTelemeter]
+    assert(telemeter.handlerPrefix === "")
+  }
+
+  test("io.l5d.prometheus telemeter prefix") {
+    val yaml =
+      """|kind: io.l5d.prometheus
+         |prefix: some_prefix_
+         |""".stripMargin
+
+    val config = Parser.objectMapper(yaml, Seq(LoadService[TelemeterInitializer]))
+      .readValue[TelemeterConfig](yaml)
+
+    val telemeter = config.mk(Stack.Params.empty).asInstanceOf[PrometheusTelemeter]
+    assert(telemeter.handlerPrefix === "some_prefix_")
+  }
 }

telemetry/prometheus/src/test/scala/io/buoyant/telemetry/prometheus/PrometheusTelemeterTest.scala

@@ -7,11 +7,12 @@ import io.buoyant.test.FunSuite
 class PrometheusTelemeterTest extends FunSuite {
 
   private val prometheusPath = "/admin/metrics/prometheus"
+  private val prometheusPrefix = "linkerd_"
 
   def statsAndHandler = {
     val metrics = MetricsTree()
     val stats = new MetricsTreeStatsReceiver(metrics)
-    val telemeter = new PrometheusTelemeter(metrics, prometheusPath)
+    val telemeter = new PrometheusTelemeter(metrics, prometheusPath, prometheusPrefix)
     val handler = telemeter.handler
     (stats, handler)
   }
@@ -21,21 +22,21 @@ class PrometheusTelemeterTest extends FunSuite {
     val counter = stats.scope("foo", "bar").counter("bas")
     counter.incr()
     val rsp1 = await(handler(Request(prometheusPath))).contentString
-    assert(rsp1 == "foo:bar:bas 1\n")
+    assert(rsp1 == "linkerd_foo:bar:bas 1\n")
     counter.incr()
     val rsp2 = await(handler(Request(prometheusPath))).contentString
-    assert(rsp2 == "foo:bar:bas 2\n")
+    assert(rsp2 == "linkerd_foo:bar:bas 2\n")
   }
 
   test("gauge") {
     val (stats, handler) = statsAndHandler
     var v = 1.0f
     val gauge = stats.scope("foo", "bar").addGauge("bas")(v)
     val rsp1 = await(handler(Request(prometheusPath))).contentString
-    assert(rsp1 == "foo:bar:bas 1.0\n")
+    assert(rsp1 == "linkerd_foo:bar:bas 1.0\n")
     v = 2.0f
     val rsp2 = await(handler(Request(prometheusPath))).contentString
-    assert(rsp2 == "foo:bar:bas 2.0\n")
+    assert(rsp2 == "linkerd_foo:bar:bas 2.0\n")
   }
 
   test("stat") {
@@ -54,35 +55,35 @@ class PrometheusTelemeterTest extends FunSuite {
     metricsTreeStat.snapshot()
 
     val rsp1 = await(handler(Request(prometheusPath))).contentString
-    assert(rsp1 == """foo:bar:bas_count 1
-                     |foo:bar:bas_sum 1
-                     |foo:bar:bas_avg 1.0
-                     |foo:bar:bas{quantile="0"} 1
-                     |foo:bar:bas{quantile="0.5"} 1
-                     |foo:bar:bas{quantile="0.9"} 1
-                     |foo:bar:bas{quantile="0.95"} 1
-                     |foo:bar:bas{quantile="0.99"} 1
-                     |foo:bar:bas{quantile="0.999"} 1
-                     |foo:bar:bas{quantile="0.9999"} 1
-                     |foo:bar:bas{quantile="1"} 1
+    assert(rsp1 == """linkerd_foo:bar:bas_count 1
+                     |linkerd_foo:bar:bas_sum 1
+                     |linkerd_foo:bar:bas_avg 1.0
+                     |linkerd_foo:bar:bas{quantile="0"} 1
+                     |linkerd_foo:bar:bas{quantile="0.5"} 1
+                     |linkerd_foo:bar:bas{quantile="0.9"} 1
+                     |linkerd_foo:bar:bas{quantile="0.95"} 1
+                     |linkerd_foo:bar:bas{quantile="0.99"} 1
+                     |linkerd_foo:bar:bas{quantile="0.999"} 1
+                     |linkerd_foo:bar:bas{quantile="0.9999"} 1
+                     |linkerd_foo:bar:bas{quantile="1"} 1
                      |""".stripMargin)
 
     // second data point
     stat.add(2.0f)
     metricsTreeStat.snapshot()
 
     val rsp2 = await(handler(Request(prometheusPath))).contentString
-    assert(rsp2 == """foo:bar:bas_count 2
-                     |foo:bar:bas_sum 3
-                     |foo:bar:bas_avg 1.5
-                     |foo:bar:bas{quantile="0"} 1
-                     |foo:bar:bas{quantile="0.5"} 1
-                     |foo:bar:bas{quantile="0.9"} 2
-                     |foo:bar:bas{quantile="0.95"} 2
-                     |foo:bar:bas{quantile="0.99"} 2
-                     |foo:bar:bas{quantile="0.999"} 2
-                     |foo:bar:bas{quantile="0.9999"} 2
-                     |foo:bar:bas{quantile="1"} 2
+    assert(rsp2 == """linkerd_foo:bar:bas_count 2
+                     |linkerd_foo:bar:bas_sum 3
+                     |linkerd_foo:bar:bas_avg 1.5
+                     |linkerd_foo:bar:bas{quantile="0"} 1
+                     |linkerd_foo:bar:bas{quantile="0.5"} 1
+                     |linkerd_foo:bar:bas{quantile="0.9"} 2
+                     |linkerd_foo:bar:bas{quantile="0.95"} 2
+                     |linkerd_foo:bar:bas{quantile="0.99"} 2
+                     |linkerd_foo:bar:bas{quantile="0.999"} 2
+                     |linkerd_foo:bar:bas{quantile="0.9999"} 2
+                     |linkerd_foo:bar:bas{quantile="1"} 2
                      |""".stripMargin)
   }
 
@@ -91,7 +92,7 @@ class PrometheusTelemeterTest extends FunSuite {
     val counter = stats.scope("rt", "incoming", "service", """\x5b\x31\x32\x33\x2e\x31\x32\x33\x2e\x31\x32\x33\x2e\x31\x32\x33\x5dun"esc""").counter("requests")
     counter.incr()
     val rsp = await(handler(Request(prometheusPath))).contentString
-    assert(rsp == """rt:service:requests{rt="incoming", service="\\x5b\\x31\\x32\\x33\\x2e\\x31\\x32\\x33\\x2e\\x31\\x32\\x33\\x2e\\x31\\x32\\x33\\x5dun\\esc"} 1
+    assert(rsp == """linkerd_rt:service:requests{rt="incoming", service="\\x5b\\x31\\x32\\x33\\x2e\\x31\\x32\\x33\\x2e\\x31\\x32\\x33\\x2e\\x31\\x32\\x33\\x5dun\\esc"} 1
 """)
   }
 
@@ -100,30 +101,30 @@ class PrometheusTelemeterTest extends FunSuite {
     val counter = stats.scope("rt", "incoming", "service", "/svc/foo").counter("requests")
     counter.incr()
     val rsp = await(handler(Request(prometheusPath))).contentString
-    assert(rsp == "rt:service:requests{rt=\"incoming\", service=\"/svc/foo\"} 1\n")
+    assert(rsp == "linkerd_rt:service:requests{rt=\"incoming\", service=\"/svc/foo\"} 1\n")
   }
 
   test("bound stats are labelled") {
     val (stats, handler) = statsAndHandler
     stats.scope("rt", "incoming", "client", "/#/bar").counter("requests").incr()
     val rsp = await(handler(Request(prometheusPath))).contentString
     assert(rsp ==
-      "rt:client:requests{rt=\"incoming\", client=\"/#/bar\"} 1\n")
+      "linkerd_rt:client:requests{rt=\"incoming\", client=\"/#/bar\"} 1\n")
   }
 
   test("bound stats with path scope are labelled") {
     val (stats, handler) = statsAndHandler
     stats.scope("rt", "incoming", "client", "/#/bar", "service", "/svc/foo").counter("requests").incr()
     val rsp = await(handler(Request(prometheusPath))).contentString
     assert(rsp ==
-      "rt:client:service:requests{rt=\"incoming\", client=\"/#/bar\", service=\"/svc/foo\"} 1\n")
+      "linkerd_rt:client:service:requests{rt=\"incoming\", client=\"/#/bar\", service=\"/svc/foo\"} 1\n")
   }
 
   test("server stats are labelled") {
     val (stats, handler) = statsAndHandler
     val counter = stats.scope("rt", "incoming", "server", "127.0.0.1/4141").counter("requests")
     counter.incr()
     val rsp = await(handler(Request(prometheusPath))).contentString
-    assert(rsp == "rt:server:requests{rt=\"incoming\", server=\"127.0.0.1/4141\"} 1\n")
+    assert(rsp == "linkerd_rt:server:requests{rt=\"incoming\", server=\"127.0.0.1/4141\"} 1\n")
   }
 }