From deff405a12e1eee6a67748a0562c6a53e2f7788d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20=22decko=22=20de=20Brito?= Date: Mon, 2 Sep 2024 15:13:08 +0200 Subject: [PATCH] Re-enable Storage Metrics emmiter. Revert "Disable the Storage Metrics emmiter for now." This reverts commit 7029ee4de07af7a9a17bce9073102cdcf1ef1b91. Closes #5762 --- CHANGES/5762.feature | 2 ++ pulpcore/app/models/content.py | 11 ++++++++++- pulpcore/app/models/domain.py | 10 +--------- pulpcore/app/tasks/telemetry.py | 31 +++++++++++++++++++++++++++++++ pulpcore/app/util.py | 11 ++++++++--- pulpcore/app/wsgi.py | 6 ------ 6 files changed, 52 insertions(+), 19 deletions(-) create mode 100644 CHANGES/5762.feature create mode 100644 pulpcore/app/tasks/telemetry.py diff --git a/CHANGES/5762.feature b/CHANGES/5762.feature new file mode 100644 index 0000000000..bfa3291289 --- /dev/null +++ b/CHANGES/5762.feature @@ -0,0 +1,2 @@ +Re-enable the Domain Storage metric emmiter and adds a feature flag to it. +This is an experimental feature and can change without prior notice. diff --git a/pulpcore/app/models/content.py b/pulpcore/app/models/content.py index 5205715cab..8465b2e175 100644 --- a/pulpcore/app/models/content.py +++ b/pulpcore/app/models/content.py @@ -21,7 +21,7 @@ from django.forms.models import model_to_dict from django.utils.timezone import now from django_guid import get_guid -from django_lifecycle import BEFORE_UPDATE, BEFORE_SAVE, hook +from django_lifecycle import BEFORE_UPDATE, BEFORE_SAVE, AFTER_SAVE, AFTER_DELETE, hook from pulpcore.constants import ALL_KNOWN_CONTENT_CHECKSUMS from pulpcore.app import pulp_hashlib @@ -34,6 +34,7 @@ UnsupportedDigestValidationError, ) + # All available digest fields ordered by algorithm strength. _DIGEST_FIELDS = [] for alg in ("sha512", "sha384", "sha256", "sha224", "sha1", "md5"): @@ -390,6 +391,14 @@ def touch(self): """Update timestamp_of_interest.""" self.save(update_fields=["timestamp_of_interest"]) + @hook(AFTER_SAVE) + @hook(AFTER_DELETE) + def emmit_telemetry(self): + from pulpcore.plugin.tasking import dispatch + from pulpcore.app.tasks import telemetry + + dispatch(telemetry.emmit_disk_space_usage_telemetry, args=(self.pulp_domain.pk,)) + class PulpTemporaryFile(HandleTempFilesMixin, BaseModel): """ diff --git a/pulpcore/app/models/domain.py b/pulpcore/app/models/domain.py index e85eb0574d..6d2386fe15 100644 --- a/pulpcore/app/models/domain.py +++ b/pulpcore/app/models/domain.py @@ -1,6 +1,6 @@ from django.core.files.storage import default_storage from django.db import models -from django_lifecycle import hook, BEFORE_DELETE, BEFORE_UPDATE +from django_lifecycle import hook, BEFORE_DELETE, BEFORE_UPDATE, AFTER_CREATE from pulpcore.app.models import BaseModel, AutoAddObjPermsMixin from pulpcore.exceptions import DomainProtectedError @@ -72,14 +72,6 @@ def _cleanup_orphans_pre_delete(self): # Delete on by one to properly cleanup the storage. artifact.delete() - # Disabling Storage metrics until we find a solution to resource usage. - # https://github.com/pulp/pulpcore/issues/5468 - # @hook(AFTER_CREATE) - # def _report_domain_disk_usage(self): - # from pulpcore.app.util import DomainMetricsEmitterBuilder - # - # DomainMetricsEmitterBuilder.build(self) - class Meta: permissions = [ ("manage_roles_domain", "Can manage role assignments on domain"), diff --git a/pulpcore/app/tasks/telemetry.py b/pulpcore/app/tasks/telemetry.py new file mode 100644 index 0000000000..74bc9f1c3a --- /dev/null +++ b/pulpcore/app/tasks/telemetry.py @@ -0,0 +1,31 @@ +from logging import getLogger + +from opentelemetry import metrics + +from django.db.models import Sum + +_logger = getLogger(__name__) + + +def emmit_disk_space_usage_telemetry(domain_pk=None, *args, **kwargs): + from pulpcore.app.models import Artifact, Domain + + artifacts = Artifact.objects.values("pulp_domain__name").annotate( + total_size=Sum("size", default=0) + ) + + if domain_pk: + domain = Domain.objects.get(pk=domain_pk) + meter = metrics.get_meter(f"domain.{domain.name}.disk_usage.meter") + else: + _logger.info("Ready to calculate the space usage for all domains") + meter = metrics.get_meter("disk_usage.meter") + + gauge = meter.create_gauge( + name="disk_usage", + description="The total disk size per domain.", + unit="bytes", + ) + + for domain in artifacts: + gauge.set(domain["total_size"], {"domain_name": domain["pulp_domain__name"]}) diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py index 33edddcf36..0b6b40be82 100644 --- a/pulpcore/app/util.py +++ b/pulpcore/app/util.py @@ -584,10 +584,15 @@ def _disk_usage_callback(self): def init_domain_metrics_exporter(): - from pulpcore.app.models.domain import Domain + PULP_OTEL_SPACE_USAGE_TELEMETRY = ( + os.getenv("PULP_OTEL_SPACE_USAGE_TELEMETRY", "").lower() == "true" + ) - for domain in Domain.objects.all(): - DomainMetricsEmitter.build(domain) + if PULP_OTEL_SPACE_USAGE_TELEMETRY: + from pulpcore.app.models.domain import Domain + + for domain in Domain.objects.all(): + DomainMetricsEmitter.build(domain) class PGAdvisoryLock: diff --git a/pulpcore/app/wsgi.py b/pulpcore/app/wsgi.py index 5287224e8a..d06e92dfdb 100644 --- a/pulpcore/app/wsgi.py +++ b/pulpcore/app/wsgi.py @@ -17,9 +17,3 @@ application = get_wsgi_application() application = OpenTelemetryMiddleware(application) - -# Disabling Storage metrics until we find a solution to resource usage. -# https://github.com/pulp/pulpcore/issues/5468 -# from pulpcore.app.util import init_domain_metrics_exporter # noqa: E402 - -# init_domain_metrics_exporter()