Renderer refactor

promgen/management/commands/rules.py

@@ -4,7 +4,7 @@
 import logging
 
 from django.core.management.base import BaseCommand
-from promgen import prometheus, tasks
+from promgen import renderers, tasks
 
 logger = logging.getLogger(__name__)
 
@@ -28,4 +28,4 @@ def handle(self, **kwargs):
             # Since we're already working with utf8 encoded data, we can skip
             # the newline ending here
             self.stdout.ending = None
-            self.stdout.buffer.write(prometheus.render_rules())
+            self.stdout.buffer.write(renderers.rules())

promgen/management/commands/targets.py

@@ -4,7 +4,7 @@
 import logging
 
 from django.core.management.base import BaseCommand
-from promgen import prometheus, tasks
+from promgen import renderers, tasks
 
 logger = logging.getLogger(__name__)
 
@@ -23,4 +23,4 @@ def handle(self, **kwargs):
         if kwargs['out']:
             tasks.write_config(kwargs['out'], kwargs['reload'], kwargs['mode'])
         else:
-            self.stdout.write(prometheus.render_config())
+            self.stdout.write(renderers.targets())

promgen/management/commands/urls.py

@@ -4,7 +4,7 @@
 import logging
 
 from django.core.management.base import BaseCommand
-from promgen import models, prometheus, tasks
+from promgen import models, prometheus, renderers, tasks
 
 logger = logging.getLogger(__name__)
 
@@ -23,4 +23,4 @@ def handle(self, **kwargs):
         if kwargs['out']:
             tasks.write_rules(kwargs['out'], kwargs['reload'])
         else:
-            self.stdout.write(prometheus.render_urls())
+            self.stdout.write(renderers.urls())

promgen/prometheus.py

@@ -2,7 +2,6 @@
 # These sources are released under the terms of the MIT license: see LICENSE
 import collections
 import datetime
-import json
 import logging
 import subprocess
 import tempfile
@@ -15,7 +14,7 @@
 from django.core.exceptions import ValidationError
 from django.utils import timezone
 
-from promgen import models, renderers, serializers, util
+from promgen import models, renderers, util
 
 logger = logging.getLogger(__name__)
 
@@ -34,7 +33,7 @@ def check_rules(rules):
         # Normally we wouldn't bother saving a copy to a variable here and would
         # leave it in the fp.write() call, but saving a copy in the variable
         # means we can see the rendered output in a Sentry stacktrace
-        rendered = render_rules(rules)
+        rendered = renderers.rules(rules)
         fp.write(rendered)
         fp.flush()
 
@@ -47,99 +46,6 @@ def check_rules(rules):
             raise ValidationError(rendered.decode('utf8') + e.output.decode('utf8'))
 
 
-def render_rules(rules=None):
-    '''
-    Render rules in a format that Prometheus understands
-
-    :param rules: List of rules
-    :type rules: list(Rule)
-    :param int version: Prometheus rule format (1 or 2)
-    :return: Returns rules in yaml or Prometheus v1 format
-    :rtype: bytes
-
-    This function can render in either v1 or v2 format
-    We call prefetch_related_objects within this function to populate the
-    other related objects that are mostly used for the sub lookups.
-    '''
-    if rules is None:
-        rules = models.Rule.objects.filter(enabled=True)
-
-    return renderers.RuleRenderer().render(
-        serializers.AlertRuleSerializer(rules, many=True).data
-    )
-
-
-def render_urls():
-    urls = collections.defaultdict(list)
-
-    for url in models.URL.objects.prefetch_related(
-        "project__service",
-        "project__shard",
-        "project",
-    ):
-        urls[
-            (
-                url.project.name,
-                url.project.service.name,
-                url.project.shard.name,
-                url.probe.module,
-            )
-        ].append(url.url)
-
-    data = [
-        {
-            "labels": {
-                "project": k[0],
-                "service": k[1],
-                "job": k[3],
-                "__shard": k[2],
-                "__param_module": k[3],
-            },
-            "targets": v,
-        }
-        for k, v in urls.items()
-    ]
-    return json.dumps(data, indent=2, sort_keys=True)
-
-
-def render_config(service=None, project=None):
-    data = []
-    for exporter in models.Exporter.objects.prefetch_related(
-        "project__farm__host_set",
-        "project__farm",
-        "project__service",
-        "project__shard",
-        "project",
-    ):
-        if not exporter.project.farm:
-            continue
-        if service and exporter.project.service.name != service.name:
-            continue
-        if project and exporter.project.name != project.name:
-            continue
-        if not exporter.enabled:
-            continue
-
-        labels = {
-            "__shard": exporter.project.shard.name,
-            "service": exporter.project.service.name,
-            "project": exporter.project.name,
-            "farm": exporter.project.farm.name,
-            "__farm_source": exporter.project.farm.source,
-            "job": exporter.job,
-            "__scheme__": exporter.scheme,
-        }
-        if exporter.path:
-            labels["__metrics_path__"] = exporter.path
-
-        hosts = []
-        for host in exporter.project.farm.host_set.all():
-            hosts.append("{}:{}".format(host.name, exporter.port))
-
-        data.append({"labels": labels, "targets": hosts})
-    return json.dumps(data, indent=2, sort_keys=True)
-
-
 def import_rules_v2(config, content_object=None):
     '''
     Loop through a dictionary and add rules to the database

promgen/renderers.py

@@ -4,8 +4,11 @@
 import yaml
 from rest_framework import renderers
 
+from promgen import models, serializers
+
 
 # https://www.django-rest-framework.org/api-guide/renderers/#custom-renderers
+# https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules
 class RuleRenderer(renderers.BaseRenderer):
     format = "yaml"
     media_type = "application/x-yaml"
@@ -18,3 +21,29 @@ def render(self, data, media_type=None, renderer_context=None):
             allow_unicode=True,
             encoding=self.charset,
         )
+
+
+# https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config
+class ScrapeRenderer(renderers.JSONRenderer):
+    pass
+    # TODO handle grouping
+
+
+def rules(rules=None):
+    if rules is None:
+        rules = models.Rule.objects
+    return RuleRenderer().render(
+        serializers.AlertRuleSerializer(rules, many=True).data
+    )
+
+
+def urls():
+    return ScrapeRenderer().render(
+        serializers.UrlSeralizer(models.URL.objects, many=True).data
+    )
+
+
+def targets():
+    return ScrapeRenderer().render(
+        serializers.TargetSeralizer(models.Exporters.objects, many=True).data
+    )

promgen/rest.py

@@ -5,30 +5,40 @@
 from rest_framework.decorators import action
 from rest_framework.response import Response
 
-from django.http import HttpResponse
-
-from promgen import filters, models, prometheus, renderers, serializers
+from promgen import filters, models, renderers, serializers
 
 
 class AllViewSet(viewsets.ViewSet):
     permission_classes = [permissions.AllowAny]
 
-    @action(detail=False, methods=["get"], renderer_classes=[renderers.RuleRenderer])
+    @action(detail=False, renderer_classes=[renderers.RuleRenderer])
     def rules(self, request):
-        rules = models.Rule.objects.filter(enabled=True)
+        rules = models.Rule.objects
         return Response(
             serializers.AlertRuleSerializer(rules, many=True).data,
             headers={"Content-Disposition": "attachment; filename=alert.rule.yml"},
         )
 
+    @action(detail=False, renderer_classes=[renderers.ScrapeRenderer])
+    def urls(self, request):
+        return Response(
+            serializers.UrlSeralizer(models.URL.objects.all(), many=True).data
+        )
+
+    @action(detail=False, renderer_classes=[renderers.ScrapeRenderer])
+    def targets(self, request):
+        return Response(
+            serializers.TargetSeralizer(models.Exporter.objects, many=True).data
+        )
+
 
 class ShardViewSet(viewsets.ModelViewSet):
     queryset = models.Shard.objects.all()
     filterset_class = filters.ShardFilter
     serializer_class = serializers.ShardSerializer
-    lookup_field = 'name'
+    lookup_field = "name"
 
-    @action(detail=True, methods=['get'])
+    @action(detail=True, methods=["get"])
     def services(self, request, name):
         shard = self.get_object()
         return Response(
@@ -60,35 +70,46 @@ class ServiceViewSet(NotifierMixin, RuleMixin, viewsets.ModelViewSet):
     queryset = models.Service.objects.all()
     filterset_class = filters.ServiceFilter
     serializer_class = serializers.ServiceSerializer
-    lookup_value_regex = '[^/]+'
-    lookup_field = 'name'
+    lookup_value_regex = "[^/]+"
+    lookup_field = "name"
 
-    @action(detail=True, methods=['get'])
+    @action(detail=True, methods=["get"])
     def projects(self, request, name):
         service = self.get_object()
         return Response(
             serializers.ProjectSerializer(service.project_set.all(), many=True).data
         )
 
-    @action(detail=True, methods=['get'])
+    @action(detail=True)
     def targets(self, request, name):
-        return HttpResponse(
-            prometheus.render_config(service=self.get_object()),
-            content_type='application/json',
+        return Response(
+            serializers.TargetSeralizer(
+                models.Exporter.objects.filter(project__service__name=name), many=True
+            ).data
         )
 
 
 class ProjectViewSet(NotifierMixin, RuleMixin, viewsets.ModelViewSet):
     queryset = models.Project.objects.prefetch_related("service", "shard", "farm")
     filterset_class = filters.ProjectFilter
     serializer_class = serializers.ProjectSerializer
-    lookup_value_regex = '[^/]+'
-    lookup_field = 'name'
+    lookup_value_regex = "[^/]+"
+    lookup_field = "name"
 
-    @action(detail=True, methods=['get'])
+    @action(detail=True)
     def targets(self, request, name):
-        return HttpResponse(
-            prometheus.render_config(project=self.get_object()),
-            content_type='application/json',
+        return Response(
+            serializers.TargetSeralizer(
+                models.Exporter.objects.filter(project__name=name), many=True
+            ).data
+        )
+
+    @action(detail=True, renderer_classes=[renderers.ScrapeRenderer])
+    def urls(self, request, name):
+        return Response(
+            serializers.UrlSeralizer(self.get_object().url_set.all(), many=True).data
         )
 
+    @urls.mapping.post
+    def post_url(self, request, name):
+        raise NotImplementedError("TODO")