diff --git a/promgen/management/commands/rules.py b/promgen/management/commands/rules.py index fb1c8f2f4..fb0c15397 100644 --- a/promgen/management/commands/rules.py +++ b/promgen/management/commands/rules.py @@ -4,7 +4,7 @@ import logging from django.core.management.base import BaseCommand -from promgen import prometheus, tasks +from promgen import renderers, tasks logger = logging.getLogger(__name__) @@ -28,4 +28,4 @@ def handle(self, **kwargs): # Since we're already working with utf8 encoded data, we can skip # the newline ending here self.stdout.ending = None - self.stdout.buffer.write(prometheus.render_rules()) + self.stdout.buffer.write(renderers.rules()) diff --git a/promgen/management/commands/targets.py b/promgen/management/commands/targets.py index c485df65e..7b1dccb5d 100644 --- a/promgen/management/commands/targets.py +++ b/promgen/management/commands/targets.py @@ -4,7 +4,7 @@ import logging from django.core.management.base import BaseCommand -from promgen import prometheus, tasks +from promgen import renderers, tasks logger = logging.getLogger(__name__) @@ -23,4 +23,4 @@ def handle(self, **kwargs): if kwargs['out']: tasks.write_config(kwargs['out'], kwargs['reload'], kwargs['mode']) else: - self.stdout.write(prometheus.render_config()) + self.stdout.write(renderers.targets()) diff --git a/promgen/management/commands/urls.py b/promgen/management/commands/urls.py index 84d7a41a5..908704e2b 100644 --- a/promgen/management/commands/urls.py +++ b/promgen/management/commands/urls.py @@ -4,7 +4,7 @@ import logging from django.core.management.base import BaseCommand -from promgen import models, prometheus, tasks +from promgen import models, prometheus, renderers, tasks logger = logging.getLogger(__name__) @@ -23,4 +23,4 @@ def handle(self, **kwargs): if kwargs['out']: tasks.write_rules(kwargs['out'], kwargs['reload']) else: - self.stdout.write(prometheus.render_urls()) + self.stdout.write(renderers.urls()) diff --git a/promgen/prometheus.py b/promgen/prometheus.py index 41e4bc34a..3c5a477d8 100644 --- a/promgen/prometheus.py +++ b/promgen/prometheus.py @@ -2,7 +2,6 @@ # These sources are released under the terms of the MIT license: see LICENSE import collections import datetime -import json import logging import subprocess import tempfile @@ -15,7 +14,7 @@ from django.core.exceptions import ValidationError from django.utils import timezone -from promgen import models, renderers, serializers, util +from promgen import models, renderers, util logger = logging.getLogger(__name__) @@ -34,7 +33,7 @@ def check_rules(rules): # Normally we wouldn't bother saving a copy to a variable here and would # leave it in the fp.write() call, but saving a copy in the variable # means we can see the rendered output in a Sentry stacktrace - rendered = render_rules(rules) + rendered = renderers.rules(rules) fp.write(rendered) fp.flush() @@ -47,99 +46,6 @@ def check_rules(rules): raise ValidationError(rendered.decode('utf8') + e.output.decode('utf8')) -def render_rules(rules=None): - ''' - Render rules in a format that Prometheus understands - - :param rules: List of rules - :type rules: list(Rule) - :param int version: Prometheus rule format (1 or 2) - :return: Returns rules in yaml or Prometheus v1 format - :rtype: bytes - - This function can render in either v1 or v2 format - We call prefetch_related_objects within this function to populate the - other related objects that are mostly used for the sub lookups. - ''' - if rules is None: - rules = models.Rule.objects.filter(enabled=True) - - return renderers.RuleRenderer().render( - serializers.AlertRuleSerializer(rules, many=True).data - ) - - -def render_urls(): - urls = collections.defaultdict(list) - - for url in models.URL.objects.prefetch_related( - "project__service", - "project__shard", - "project", - ): - urls[ - ( - url.project.name, - url.project.service.name, - url.project.shard.name, - url.probe.module, - ) - ].append(url.url) - - data = [ - { - "labels": { - "project": k[0], - "service": k[1], - "job": k[3], - "__shard": k[2], - "__param_module": k[3], - }, - "targets": v, - } - for k, v in urls.items() - ] - return json.dumps(data, indent=2, sort_keys=True) - - -def render_config(service=None, project=None): - data = [] - for exporter in models.Exporter.objects.prefetch_related( - "project__farm__host_set", - "project__farm", - "project__service", - "project__shard", - "project", - ): - if not exporter.project.farm: - continue - if service and exporter.project.service.name != service.name: - continue - if project and exporter.project.name != project.name: - continue - if not exporter.enabled: - continue - - labels = { - "__shard": exporter.project.shard.name, - "service": exporter.project.service.name, - "project": exporter.project.name, - "farm": exporter.project.farm.name, - "__farm_source": exporter.project.farm.source, - "job": exporter.job, - "__scheme__": exporter.scheme, - } - if exporter.path: - labels["__metrics_path__"] = exporter.path - - hosts = [] - for host in exporter.project.farm.host_set.all(): - hosts.append("{}:{}".format(host.name, exporter.port)) - - data.append({"labels": labels, "targets": hosts}) - return json.dumps(data, indent=2, sort_keys=True) - - def import_rules_v2(config, content_object=None): ''' Loop through a dictionary and add rules to the database diff --git a/promgen/renderers.py b/promgen/renderers.py index bd423aae5..3cd0de815 100644 --- a/promgen/renderers.py +++ b/promgen/renderers.py @@ -4,8 +4,11 @@ import yaml from rest_framework import renderers +from promgen import models, serializers + # https://www.django-rest-framework.org/api-guide/renderers/#custom-renderers +# https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules class RuleRenderer(renderers.BaseRenderer): format = "yaml" media_type = "application/x-yaml" @@ -18,3 +21,29 @@ def render(self, data, media_type=None, renderer_context=None): allow_unicode=True, encoding=self.charset, ) + + +# https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config +class ScrapeRenderer(renderers.JSONRenderer): + pass + # TODO handle grouping + + +def rules(rules=None): + if rules is None: + rules = models.Rule.objects + return RuleRenderer().render( + serializers.AlertRuleSerializer(rules, many=True).data + ) + + +def urls(): + return ScrapeRenderer().render( + serializers.UrlSeralizer(models.URL.objects, many=True).data + ) + + +def targets(): + return ScrapeRenderer().render( + serializers.TargetSeralizer(models.Exporters.objects, many=True).data + ) diff --git a/promgen/rest.py b/promgen/rest.py index c773061c2..a84746b0e 100644 --- a/promgen/rest.py +++ b/promgen/rest.py @@ -5,30 +5,40 @@ from rest_framework.decorators import action from rest_framework.response import Response -from django.http import HttpResponse - -from promgen import filters, models, prometheus, renderers, serializers +from promgen import filters, models, renderers, serializers class AllViewSet(viewsets.ViewSet): permission_classes = [permissions.AllowAny] - @action(detail=False, methods=["get"], renderer_classes=[renderers.RuleRenderer]) + @action(detail=False, renderer_classes=[renderers.RuleRenderer]) def rules(self, request): - rules = models.Rule.objects.filter(enabled=True) + rules = models.Rule.objects return Response( serializers.AlertRuleSerializer(rules, many=True).data, headers={"Content-Disposition": "attachment; filename=alert.rule.yml"}, ) + @action(detail=False, renderer_classes=[renderers.ScrapeRenderer]) + def urls(self, request): + return Response( + serializers.UrlSeralizer(models.URL.objects.all(), many=True).data + ) + + @action(detail=False, renderer_classes=[renderers.ScrapeRenderer]) + def targets(self, request): + return Response( + serializers.TargetSeralizer(models.Exporter.objects, many=True).data + ) + class ShardViewSet(viewsets.ModelViewSet): queryset = models.Shard.objects.all() filterset_class = filters.ShardFilter serializer_class = serializers.ShardSerializer - lookup_field = 'name' + lookup_field = "name" - @action(detail=True, methods=['get']) + @action(detail=True, methods=["get"]) def services(self, request, name): shard = self.get_object() return Response( @@ -60,21 +70,22 @@ class ServiceViewSet(NotifierMixin, RuleMixin, viewsets.ModelViewSet): queryset = models.Service.objects.all() filterset_class = filters.ServiceFilter serializer_class = serializers.ServiceSerializer - lookup_value_regex = '[^/]+' - lookup_field = 'name' + lookup_value_regex = "[^/]+" + lookup_field = "name" - @action(detail=True, methods=['get']) + @action(detail=True, methods=["get"]) def projects(self, request, name): service = self.get_object() return Response( serializers.ProjectSerializer(service.project_set.all(), many=True).data ) - @action(detail=True, methods=['get']) + @action(detail=True) def targets(self, request, name): - return HttpResponse( - prometheus.render_config(service=self.get_object()), - content_type='application/json', + return Response( + serializers.TargetSeralizer( + models.Exporter.objects.filter(project__service__name=name), many=True + ).data ) @@ -82,13 +93,23 @@ class ProjectViewSet(NotifierMixin, RuleMixin, viewsets.ModelViewSet): queryset = models.Project.objects.prefetch_related("service", "shard", "farm") filterset_class = filters.ProjectFilter serializer_class = serializers.ProjectSerializer - lookup_value_regex = '[^/]+' - lookup_field = 'name' + lookup_value_regex = "[^/]+" + lookup_field = "name" - @action(detail=True, methods=['get']) + @action(detail=True) def targets(self, request, name): - return HttpResponse( - prometheus.render_config(project=self.get_object()), - content_type='application/json', + return Response( + serializers.TargetSeralizer( + models.Exporter.objects.filter(project__name=name), many=True + ).data + ) + + @action(detail=True, renderer_classes=[renderers.ScrapeRenderer]) + def urls(self, request, name): + return Response( + serializers.UrlSeralizer(self.get_object().url_set.all(), many=True).data ) + @urls.mapping.post + def post_url(self, request, name): + raise NotImplementedError("TODO") diff --git a/promgen/serializers.py b/promgen/serializers.py index 98a66d420..ff81dd914 100644 --- a/promgen/serializers.py +++ b/promgen/serializers.py @@ -82,6 +82,11 @@ def many_init(cls, queryset, *args, **kwargs): # do our prefetch operations in one go, before passing it off # to our custom list renderer to group things in a dictionary kwargs["child"] = cls() + + # TODO Cleanup later + if hasattr(queryset, 'filter'): + queryset = queryset.filter(enabled=True) + prefetch_related_objects( queryset, "content_object", @@ -101,3 +106,89 @@ def to_representation(self, obj): "labels": obj.labels, "annotations": obj.annotations, } + + +class URLList(serializers.ListSerializer): + def to_representation(self, data): + labels = {} + targets = collections.defaultdict(list) + for item in data: + data = self.child.to_representation(item) + fingerprint = str(data["labels"]) + labels[fingerprint] = data["labels"] + targets[fingerprint] += data["targets"] + return [{"labels": labels[f], "targets": targets[f]} for f in labels] + + +class UrlSeralizer(serializers.ModelSerializer): + class Meta: + model = models.URL + exclude = ("id",) + + @classmethod + def many_init(cls, queryset, *args, **kwargs): + # when rendering many items at once, we want to make sure we + # do our prefetch operations in one go, before passing it off + # to our custom list renderer to group things in a dictionary + kwargs["child"] = cls() + prefetch_related_objects( + queryset, "project__service", "project__shard", "project", "probe" + ) + return URLList(queryset, *args, **kwargs) + + def to_representation(self, obj): + return { + "labels": { + "project": obj.project.name, + "service": obj.project.service.name, + "job": obj.probe.module, + "__shard": obj.project.shard.name, + "__param_module": obj.probe.module, + }, + "targets": [obj.url], + } + + +class TargetSeralizer(serializers.ModelSerializer): + class Meta: + model = models.Exporter + exclude = ("id",) + + @classmethod + def many_init(cls, queryset, *args, **kwargs): + # when rendering many items at once, we want to make sure we + # do our prefetch operations in one go, before passing it off + # to our custom list renderer to group things in a dictionary + kwargs["child"] = cls() + queryset = queryset.filter(enabled=True).exclude(project__farm=None) + prefetch_related_objects( + queryset, + "project__farm__host_set", + "project__farm", + "project__service", + "project__shard", + "project", + ) + return serializers.ListSerializer(queryset, *args, **kwargs) + + def to_representation(self, exporter): + labels = { + "__shard": exporter.project.shard.name, + "service": exporter.project.service.name, + "project": exporter.project.name, + "farm": exporter.project.farm.name, + "__farm_source": exporter.project.farm.source, + "job": exporter.job, + "__scheme__": exporter.scheme, + } + + if exporter.path: + labels["__metrics_path__"] = exporter.path + + return { + "labels": labels, + "targets": [ + "{}:{}".format(host.name, exporter.port) + for host in exporter.project.farm.host_set.all() + ], + } diff --git a/promgen/tasks.py b/promgen/tasks.py index 3f09c2c82..b33c56834 100644 --- a/promgen/tasks.py +++ b/promgen/tasks.py @@ -8,7 +8,7 @@ from atomicwrites import atomic_write from celery import shared_task -from promgen import models, prometheus, util, notification +from promgen import models, notification, prometheus, renderers, util logger = logging.getLogger(__name__) @@ -103,7 +103,7 @@ def write_urls(path=None, reload=True, chmod=0o644): with atomic_write(path, overwrite=True) as fp: # Set mode on our temporary file before we write and move it os.chmod(fp.name, chmod) - fp.write(prometheus.render_urls()) + fp.write(renderers.urls()) if reload: reload_prometheus() @@ -115,7 +115,7 @@ def write_config(path=None, reload=True, chmod=0o644): with atomic_write(path, overwrite=True) as fp: # Set mode on our temporary file before we write and move it os.chmod(fp.name, chmod) - fp.write(prometheus.render_config()) + fp.write(renderers.targets()) if reload: reload_prometheus() @@ -127,6 +127,6 @@ def write_rules(path=None, reload=True, chmod=0o644): with atomic_write(path, mode="wb", overwrite=True) as fp: # Set mode on our temporary file before we write and move it os.chmod(fp.name, chmod) - fp.write(prometheus.render_rules()) + fp.write(renderers.rules()) if reload: reload_prometheus() diff --git a/promgen/templates/promgen/navbar.html b/promgen/templates/promgen/navbar.html index e3f460ccf..f15e56f63 100644 --- a/promgen/templates/promgen/navbar.html +++ b/promgen/templates/promgen/navbar.html @@ -34,9 +34,9 @@
  • API
    • -
  • Export Targets
    • +
  • Export Targets
  • Export Rules
    • -
  • Export URL
    • +
  • Export URL
    • {% if EXTERNAL_LINKS %} diff --git a/promgen/templates/promgen/project_detail_configuration.html b/promgen/templates/promgen/project_detail_configuration.html index 86fee57fe..2f3ca427c 100644 --- a/promgen/templates/promgen/project_detail_configuration.html +++ b/promgen/templates/promgen/project_detail_configuration.html @@ -31,6 +31,7 @@ diff --git a/promgen/tests/test_alert_rules.py b/promgen/tests/test_alert_rules.py index d957909d1..c49905d87 100644 --- a/promgen/tests/test_alert_rules.py +++ b/promgen/tests/test_alert_rules.py @@ -4,7 +4,7 @@ from unittest import mock import promgen.templatetags.promgen as macro -from promgen import models, prometheus, views +from promgen import models, prometheus, renderers, views from promgen.tests import PromgenTest from django.core.exceptions import ValidationError @@ -47,7 +47,7 @@ def setUp(self, mock_signal): @override_settings(PROMGEN_SCHEME='https') @mock.patch('django.dispatch.dispatcher.Signal.send') def test_write_new(self, mock_post): - result = prometheus.render_rules() + result = renderers.rules() self.assertEqual(result, _RULE_V2 % self.rule.id) @mock.patch('django.dispatch.dispatcher.Signal.send') diff --git a/promgen/views.py b/promgen/views.py index 582244fa7..14c0c2222 100644 --- a/promgen/views.py +++ b/promgen/views.py @@ -38,6 +38,7 @@ models, plugins, prometheus, + renderers, signals, tasks, util, @@ -953,9 +954,6 @@ def form_valid(self, form): class ApiConfig(View): - def get(self, request): - return HttpResponse(prometheus.render_config(), content_type='application/json') - def post(self, request, *args, **kwargs): try: body = json.loads(request.body.decode('utf-8')) @@ -983,7 +981,7 @@ def post(self, request): class _ExportRules(View): def format(self, rules=None, name='promgen'): - content = prometheus.render_rules(rules) + content = renderers.rules(rules) response = HttpResponse(content) response['Content-Type'] = 'application/x-yaml' response['Content-Disposition'] = 'attachment; filename=%s.rule.yml' % name @@ -1004,7 +1002,7 @@ def get(self, request, content_type, object_id): class URLConfig(View): def get(self, request): - return HttpResponse(prometheus.render_urls(), content_type='application/json') + return HttpResponse(renderers.urls(), content_type='application/json') def post(self, request): tasks.write_urls()