Template include & exclude

cmd/pint/tests/0037_disable_checks.txt

@@ -6,7 +6,7 @@ cmp stderr stderr.txt
 level=INFO msg="Loading configuration file" path=.pint.hcl
 level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/0001.yml rules=3
-level=INFO msg="Configured new Prometheus server" name=prom uris=1
+level=INFO msg="Configured new Prometheus server" name=prom uris=1 tags=[] include=[] exclude=[]
 level=DEBUG msg="Generated all Prometheus servers" count=1
 level=DEBUG msg="Found alerting rule" path=rules/0001.yml alert=default-for lines=1-3
 level=DEBUG msg="Starting query workers" name=prom uri=http://127.0.0.1 workers=16

cmd/pint/tests/0039_prom_selected_path.txt

@@ -6,7 +6,7 @@ cmp stderr stderr.txt
 level=INFO msg="Loading configuration file" path=.pint.hcl
 level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/0001.yml rules=3
-level=INFO msg="Configured new Prometheus server" name=disabled uris=1
+level=INFO msg="Configured new Prometheus server" name=disabled uris=1 tags=[] include=["^invalid/.+$"] exclude=["^invalid/rules/.+$"]
 level=DEBUG msg="Generated all Prometheus servers" count=1
 level=DEBUG msg="Found alerting rule" path=rules/0001.yml alert=first lines=1-3
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp"] path=rules/0001.yml rule=first

cmd/pint/tests/0063_lint_offline.txt

@@ -5,7 +5,7 @@ cmp stderr stderr.txt
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
 level=INFO msg="Finding all rules to check" paths=["rules"]
-level=INFO msg="Configured new Prometheus server" name=disabled uris=1
+level=INFO msg="Configured new Prometheus server" name=disabled uris=1 tags=[] include=["^invalid/.+$"] exclude=[]
 -- rules/ok.yml --
 - record: sum:foo
   expr: sum(foo)

cmd/pint/tests/0080_lint_online.txt

@@ -12,7 +12,7 @@ cmp stderr stderr.txt
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
 level=INFO msg="Finding all rules to check" paths=["rules"]
-level=INFO msg="Configured new Prometheus server" name=prom1 uris=1
+level=INFO msg="Configured new Prometheus server" name=prom1 uris=1 tags=[] include=[] exclude=[]
 level=WARN msg="No results for Prometheus uptime metric, you might have set uptime config option to a missing metric, please check your config" name=prom1 metric=prometheus_ready
 level=WARN msg="Using dummy Prometheus uptime metric results with no gaps" name=prom1 metric=prometheus_ready
 rules/1.yml:2 Warning: http_errors_total[2d] selector is trying to query Prometheus for 2d worth of metrics, but prometheus "prom1" at http://127.0.0.1:7080 is configured to only keep 1d of metrics history (promql/range_query)

cmd/pint/tests/0103_file_disable.txt

@@ -6,7 +6,7 @@ cmp stderr stderr.txt
 level=INFO msg="Loading configuration file" path=.pint.hcl
 level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/0001.yml rules=1
-level=INFO msg="Configured new Prometheus server" name=prom uris=1
+level=INFO msg="Configured new Prometheus server" name=prom uris=1 tags=[] include=[] exclude=[]
 level=DEBUG msg="Generated all Prometheus servers" count=1
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=colo:test1 lines=9-10
 level=DEBUG msg="Starting query workers" name=prom uri=http://127.0.0.1:7103 workers=16

cmd/pint/tests/0108_rule_duplicate.txt

@@ -5,7 +5,7 @@ cmp stderr stderr.txt
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
 level=INFO msg="Finding all rules to check" paths=["rules"]
-level=INFO msg="Configured new Prometheus server" name=prom uris=1
+level=INFO msg="Configured new Prometheus server" name=prom uris=1 tags=[] include=[] exclude=[]
 level=ERROR msg="Query returned an error" err="failed to query Prometheus config: Get \"http://127.0.0.1:7108/api/v1/status/config\": dial tcp 127.0.0.1:7108: connect: connection refused" uri=http://127.0.0.1:7108 query=/api/v1/status/config
 level=ERROR msg="Query returned an error" err="failed to query Prometheus config: Get \"http://127.0.0.1:7108/api/v1/status/config\": dial tcp 127.0.0.1:7108: connect: connection refused" uri=http://127.0.0.1:7108 query=/api/v1/status/config
 level=ERROR msg="Query returned an error" err="failed to query Prometheus config: Get \"http://127.0.0.1:7108/api/v1/status/config\": dial tcp 127.0.0.1:7108: connect: connection refused" uri=http://127.0.0.1:7108 query=/api/v1/status/config

cmd/pint/tests/0109_rule_duplicate_multiple_proms_include.txt

@@ -5,8 +5,8 @@ cmp stderr stderr.txt
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
 level=INFO msg="Finding all rules to check" paths=["rules"]
-level=INFO msg="Configured new Prometheus server" name=prom1 uris=1
-level=INFO msg="Configured new Prometheus server" name=prom2 uris=1
+level=INFO msg="Configured new Prometheus server" name=prom1 uris=1 tags=[] include=["^rules/0001.yml$"] exclude=[]
+level=INFO msg="Configured new Prometheus server" name=prom2 uris=1 tags=[] include=["^rules/0002.yml$"] exclude=[]
 -- rules/0001.yml --
 - record: "colo:duplicate"
   expr: sum(foo) without(job)

cmd/pint/tests/0110_rule_duplicate_multiple_proms_exclude.txt

@@ -5,8 +5,8 @@ cmp stderr stderr.txt
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
 level=INFO msg="Finding all rules to check" paths=["rules"]
-level=INFO msg="Configured new Prometheus server" name=prom1 uris=1
-level=INFO msg="Configured new Prometheus server" name=prom2 uris=1
+level=INFO msg="Configured new Prometheus server" name=prom1 uris=1 tags=[] include=[] exclude=["^rules/0002.yml$"]
+level=INFO msg="Configured new Prometheus server" name=prom2 uris=1 tags=[] include=[] exclude=["^rules/0001.yml$"]
 -- rules/0001.yml --
 - record: "colo:duplicate"
   expr: sum(foo) without(job)

cmd/pint/tests/0115_file_disable_tag.txt

@@ -6,7 +6,7 @@ cmp stderr stderr.txt
 level=INFO msg="Loading configuration file" path=.pint.hcl
 level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/0001.yml rules=1
-level=INFO msg="Configured new Prometheus server" name=prom uris=1
+level=INFO msg="Configured new Prometheus server" name=prom uris=1 tags=["foo","bar"] include=[] exclude=[]
 level=DEBUG msg="Generated all Prometheus servers" count=1
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=colo:test1 lines="6 8"
 level=DEBUG msg="Starting query workers" name=prom uri=http://127.0.0.1:7103 workers=16

cmd/pint/tests/0144_discovery_filepath.txt

@@ -19,8 +19,8 @@ level=DEBUG msg="Rendered Prometheus server" name=prom2 uri=https://prom2.exampl
 level=DEBUG msg="Path discovery match" match=^(?P<name>\w+).ya?ml$ path=prom2.yml
 level=DEBUG msg="Extracted regexp variables" regexp=^(?P<name>\w+).ya?ml$ vars={"name":"prom2"}
 level=DEBUG msg="Rendered Prometheus server" name=prom2 uri=https://prom2.example.com headers=["X-Host"] timeout=5s concurrency=16 rateLimit=100 uptime=up tags=["name/prom2"] required=true
-level=INFO msg="Configured new Prometheus server" name=prom1 uris=2
-level=INFO msg="Configured new Prometheus server" name=prom2 uris=2
+level=INFO msg="Configured new Prometheus server" name=prom1 uris=2 tags=["name/prom1"] include=[] exclude=["^.*$"]
+level=INFO msg="Configured new Prometheus server" name=prom2 uris=2 tags=["name/prom2"] include=[] exclude=["^.*$"]
 level=DEBUG msg="Generated all Prometheus servers" count=2
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=sum:up lines=4-5
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp"] path=rules/0001.yml rule=sum:up

cmd/pint/tests/0149_discovery_prom.txt

@@ -18,7 +18,7 @@ level=DEBUG msg="Rendered Prometheus server" name=prom-ha uri=https://prom1.exam
 level=DEBUG msg="Rendered Prometheus server" name=prom-ha uri=https://prom2.example.com headers=["X-Host"] timeout=5s concurrency=16 rateLimit=100 uptime=up tags=[] required=false
 level=DEBUG msg="Stopping query workers" name=discovery uri=http://127.0.0.1:7149
 level=DEBUG msg="Added new failover URI" name=prom-ha uri=https://prom2.example.com
-level=INFO msg="Configured new Prometheus server" name=prom-ha uris=2
+level=INFO msg="Configured new Prometheus server" name=prom-ha uris=2 tags=[] include=[] exclude=["^.*$"]
 level=DEBUG msg="Generated all Prometheus servers" count=1
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=sum:up lines=4-5
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp"] path=rules/0001.yml rule=sum:up

cmd/pint/tests/0150_discovery_prom_dup_tags.txt

@@ -17,6 +17,7 @@ level=DEBUG msg="Parsed response" uri=http://127.0.0.1:7150 query=prometheus_rea
 level=DEBUG msg="Rendered Prometheus server" name=prom-ha uri=https://prom1.example.com headers=["X-Host"] timeout=5s concurrency=16 rateLimit=100 uptime=up tags=["prom1"] required=false
 level=DEBUG msg="Rendered Prometheus server" name=prom-ha uri=https://prom2.example.com headers=["X-Host"] timeout=5s concurrency=16 rateLimit=100 uptime=up tags=["prom2"] required=false
 level=DEBUG msg="Stopping query workers" name=discovery uri=http://127.0.0.1:7150
+level=WARN msg="Duplicated prometheus server with different tags" name=prom-ha a=["prom2"] b=["prom1"]
 level=ERROR msg="Fatal error" err="Duplicated name for Prometheus server definition: prom-ha"
 -- rules/0001.yml --
 groups:

cmd/pint/tests/0152_discovery_prom_dup_uptime.txt

@@ -26,7 +26,7 @@ level=DEBUG msg="Parsed response" uri=http://127.0.0.1:7152 query=prometheus_rea
 level=DEBUG msg="Rendered Prometheus server" name=prom-ha uri=https://prom1.example.com headers=[] timeout=5s concurrency=16 rateLimit=100 uptime=prom2 tags=[] required=false
 level=DEBUG msg="Rendered Prometheus server" name=prom-ha uri=https://prom2.example.com headers=[] timeout=5s concurrency=16 rateLimit=100 uptime=prom2 tags=[] required=false
 level=DEBUG msg="Stopping query workers" name=discovery uri=http://127.0.0.1:7152
-level=INFO msg="Configured new Prometheus server" name=prom-ha uris=2
+level=INFO msg="Configured new Prometheus server" name=prom-ha uris=2 tags=[] include=[] exclude=["^.*$"]
 level=DEBUG msg="Generated all Prometheus servers" count=1
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=sum:up lines=4-5
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp"] path=rules/0001.yml rule=sum:up

cmd/pint/tests/0155_discovery_prom_dup_include.txt

@@ -0,0 +1,39 @@
+http response prometheus /api/v1/query 200 {"status":"success","data":{"resultType":"vector","result":[{"metric":{"name":"prom1"},"value":[1698249632.491,"1"]},{"metric":{"name":"prom2"},"value":[1698249632.491,"1"]}]}}
+http start prometheus 127.0.0.1:7155
+
+pint.error -l debug --no-color lint rules
+! stdout .
+cmp stderr stderr.txt
+
+-- stderr.txt --
+level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
+level=DEBUG msg="File parsed" path=rules/0001.yml rules=1
+level=DEBUG msg="Starting query workers" name=discovery uri=http://127.0.0.1:7155 workers=1
+level=INFO msg="Finding Prometheus servers using Prometheus API query" uri=http://127.0.0.1:7155 query=prometheus_ready
+level=DEBUG msg="Scheduling prometheus query" uri=http://127.0.0.1:7155 query=prometheus_ready
+level=DEBUG msg="Running prometheus query" uri=http://127.0.0.1:7155 query=prometheus_ready
+level=DEBUG msg="Parsed response" uri=http://127.0.0.1:7155 query=prometheus_ready series=2
+level=DEBUG msg="Rendered Prometheus server" name=prom-ha uri=https://prom1.example.com headers=[] timeout=2m0s concurrency=16 rateLimit=100 uptime=up tags=[] required=false
+level=DEBUG msg="Rendered Prometheus server" name=prom-ha uri=https://prom2.example.com headers=[] timeout=2m0s concurrency=16 rateLimit=100 uptime=up tags=[] required=false
+level=DEBUG msg="Stopping query workers" name=discovery uri=http://127.0.0.1:7155
+level=WARN msg="Duplicated prometheus server with different include" name=prom-ha a=["^prom2$"] b=["^prom1$"]
+level=ERROR msg="Fatal error" err="Duplicated name for Prometheus server definition: prom-ha"
+-- rules/0001.yml --
+groups:
+- name: foo
+  rules:
+  - record: sum:up
+    expr: sum(up)
+-- .pint.hcl --
+discovery {
+  prometheusQuery {
+    uri = "http://127.0.0.1:7155"
+    query = "prometheus_ready"
+    template {
+      name     = "prom-ha"
+      uri      = "https://{{ $name }}.example.com"
+      include = [ "{{ $name }}"]
+    }
+  }
+}

cmd/pint/tests/0156_discovery_prom_dup_exclude.txt

@@ -0,0 +1,39 @@
+http response prometheus /api/v1/query 200 {"status":"success","data":{"resultType":"vector","result":[{"metric":{"name":"prom1"},"value":[1698249632.491,"1"]},{"metric":{"name":"prom2"},"value":[1698249632.491,"1"]}]}}
+http start prometheus 127.0.0.1:7156
+
+pint.error -l debug --no-color lint rules
+! stdout .
+cmp stderr stderr.txt
+
+-- stderr.txt --
+level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
+level=DEBUG msg="File parsed" path=rules/0001.yml rules=1
+level=DEBUG msg="Starting query workers" name=discovery uri=http://127.0.0.1:7156 workers=1
+level=INFO msg="Finding Prometheus servers using Prometheus API query" uri=http://127.0.0.1:7156 query=prometheus_ready
+level=DEBUG msg="Scheduling prometheus query" uri=http://127.0.0.1:7156 query=prometheus_ready
+level=DEBUG msg="Running prometheus query" uri=http://127.0.0.1:7156 query=prometheus_ready
+level=DEBUG msg="Parsed response" uri=http://127.0.0.1:7156 query=prometheus_ready series=2
+level=DEBUG msg="Rendered Prometheus server" name=prom-ha uri=https://prom1.example.com headers=[] timeout=2m0s concurrency=16 rateLimit=100 uptime=up tags=[] required=false
+level=DEBUG msg="Rendered Prometheus server" name=prom-ha uri=https://prom2.example.com headers=[] timeout=2m0s concurrency=16 rateLimit=100 uptime=up tags=[] required=false
+level=DEBUG msg="Stopping query workers" name=discovery uri=http://127.0.0.1:7156
+level=WARN msg="Duplicated prometheus server with different exclude" name=prom-ha a=["^prom2$"] b=["^prom1$"]
+level=ERROR msg="Fatal error" err="Duplicated name for Prometheus server definition: prom-ha"
+-- rules/0001.yml --
+groups:
+- name: foo
+  rules:
+  - record: sum:up
+    expr: sum(up)
+-- .pint.hcl --
+discovery {
+  prometheusQuery {
+    uri = "http://127.0.0.1:7156"
+    query = "prometheus_ready"
+    template {
+      name     = "prom-ha"
+      uri      = "https://{{ $name }}.example.com"
+      exclude  = [ "{{ $name }}"]
+    }
+  }
+}

docs/changelog.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## v0.48.1
+
+- Prometheus `template` under `discovery` block can now template
+  `include` and `exclude` fields.
+
 ## v0.48.0
 
 ### Added

docs/configuration.md

@@ -404,6 +404,8 @@ Fields that are allowed to be templated are:
 - `failover`
 - `headers`
 - `tags`
+- `include`
+- `exclude`
 
 ```js
 template {
@@ -430,7 +432,7 @@ template {
 
 Generated Prometheus servers will be deduplicated. If there are multiple servers with the same
 name but different URIs then extra URIs will be added to `failover` list.
-Servers must have identical tags to be deduplicated.
+Servers must have identical `name`, `tags`, `include` and `exclude` fields to be deduplicated.
 
 ### Examples
 

internal/config/discovery.go

@@ -26,7 +26,31 @@ func isEqualFailoverGroup(a, b *promapi.FailoverGroup) bool {
 	if a.Name() != b.Name() {
 		return false
 	}
+	if !slices.Equal(a.Include(), b.Include()) {
+		slog.Warn(
+			"Duplicated prometheus server with different include",
+			slog.String("name", a.Name()),
+			slog.Any("a", a.Include()),
+			slog.Any("b", b.Include()),
+		)
+		return false
+	}
+	if !slices.Equal(a.Exclude(), b.Exclude()) {
+		slog.Warn(
+			"Duplicated prometheus server with different exclude",
+			slog.String("name", a.Name()),
+			slog.Any("a", a.Exclude()),
+			slog.Any("b", b.Exclude()),
+		)
+		return false
+	}
 	if !slices.Equal(a.Tags(), b.Tags()) {
+		slog.Warn(
+			"Duplicated prometheus server with different tags",
+			slog.String("name", a.Name()),
+			slog.Any("a", a.Tags()),
+			slog.Any("b", b.Tags()),
+		)
 		return false
 	}
 	return true
@@ -137,18 +161,18 @@ func (pt PrometheusTemplate) Render(data map[string]string) (*promapi.FailoverGr
 	var err error
 	var name, uri string
 	if name, err = renderTemplate(pt.Name, data); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("bad name template %q: %w", pt.Name, err)
 	}
 	if uri, err = renderTemplate(pt.URI, data); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("bad uri template %q: %w", pt.URI, err)
 	}
 
 	failover := make([]string, 0, len(pt.Failover))
 	var furi string
 	for _, f := range pt.Failover {
 		furi, err = renderTemplate(f, data)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("bad failover template %q: %w", f, err)
 		}
 		failover = append(failover, furi)
 	}
@@ -159,22 +183,42 @@ func (pt PrometheusTemplate) Render(data map[string]string) (*promapi.FailoverGr
 	for k, v := range pt.Headers {
 		key, err = renderTemplate(k, data)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("bad header key template %q: %w", k, err)
 		}
 		val, err = renderTemplate(v, data)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("bad header value template %q: %w", v, err)
 		}
 		headerNames = append(headerNames, key)
 		headers[key] = val
 	}
 
+	include := make([]string, 0, len(pt.Include))
+	var inc string
+	for _, t := range pt.Include {
+		inc, err = renderTemplate(t, data)
+		if err != nil {
+			return nil, fmt.Errorf("bad include template %q: %w", t, err)
+		}
+		include = append(include, inc)
+	}
+
+	exclude := make([]string, 0, len(pt.Exclude))
+	var exc string
+	for _, t := range pt.Exclude {
+		exc, err = renderTemplate(t, data)
+		if err != nil {
+			return nil, fmt.Errorf("bad exclude template %q: %w", t, err)
+		}
+		exclude = append(exclude, exc)
+	}
+
 	tags := make([]string, 0, len(pt.Tags))
 	var tag string
 	for _, t := range pt.Tags {
 		tag, err = renderTemplate(t, data)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("bad tag template %q: %w", t, err)
 		}
 		tags = append(tags, tag)
 	}
@@ -188,8 +232,8 @@ func (pt PrometheusTemplate) Render(data map[string]string) (*promapi.FailoverGr
 		Concurrency: pt.Concurrency,
 		RateLimit:   pt.RateLimit,
 		Uptime:      pt.Uptime,
-		Include:     pt.Include,
-		Exclude:     pt.Exclude,
+		Include:     include,
+		Exclude:     exclude,
 		Tags:        tags,
 		Required:    pt.Required,
 		TLS:         pt.TLS,