Re-enable Storage Metrics emmiter.

Revert "Disable the Storage Metrics emmiter for now."
This reverts commit 7029ee4.

Closes pulp#5762

CHANGES/5762.feature

@@ -0,0 +1,2 @@
+Re-enable the Domain Storage metric emmiter and adds a feature flag to it.
+This is an experimental feature and can change without prior notice.

pulpcore/app/models/content.py

@@ -21,7 +21,7 @@
 from django.forms.models import model_to_dict
 from django.utils.timezone import now
 from django_guid import get_guid
-from django_lifecycle import BEFORE_UPDATE, BEFORE_SAVE, hook
+from django_lifecycle import BEFORE_UPDATE, BEFORE_SAVE, AFTER_SAVE, AFTER_DELETE, hook
 
 from pulpcore.constants import ALL_KNOWN_CONTENT_CHECKSUMS
 from pulpcore.app import pulp_hashlib
@@ -34,6 +34,7 @@
     UnsupportedDigestValidationError,
 )
 
+
 # All available digest fields ordered by algorithm strength.
 _DIGEST_FIELDS = []
 for alg in ("sha512", "sha384", "sha256", "sha224", "sha1", "md5"):
@@ -390,6 +391,14 @@ def touch(self):
         """Update timestamp_of_interest."""
         self.save(update_fields=["timestamp_of_interest"])
 
+    @hook(AFTER_SAVE)
+    @hook(AFTER_DELETE)
+    def emmit_telemetry(self):
+        from pulpcore.plugin.tasking import dispatch
+        from pulpcore.app.tasks import telemetry
+
+        dispatch(telemetry.emmit_disk_space_usage_telemetry, args=(self.pulp_domain.pk,))
+
 
 class PulpTemporaryFile(HandleTempFilesMixin, BaseModel):
     """

pulpcore/app/models/domain.py

@@ -2,7 +2,7 @@
 
 from django.core.files.storage import default_storage
 from django.db import models
-from django_lifecycle import hook, BEFORE_DELETE, BEFORE_UPDATE
+from django_lifecycle import hook, BEFORE_DELETE, BEFORE_UPDATE, AFTER_CREATE
 
 from pulpcore.app.models import BaseModel, AutoAddObjPermsMixin
 from pulpcore.exceptions import DomainProtectedError
@@ -74,14 +74,6 @@ def _cleanup_orphans_pre_delete(self):
             # Delete on by one to properly cleanup the storage.
             artifact.delete()
 
-    # Disabling Storage metrics until we find a solution to resource usage.
-    # https://github.com/pulp/pulpcore/issues/5468
-    # @hook(AFTER_CREATE)
-    # def _report_domain_disk_usage(self):
-    #     from pulpcore.app.util import DomainMetricsEmitterBuilder
-    #
-    #     DomainMetricsEmitterBuilder.build(self)
-
     class Meta:
         permissions = [
             ("manage_roles_domain", "Can manage role assignments on domain"),

pulpcore/app/tasks/telemetry.py

@@ -0,0 +1,31 @@
+from logging import getLogger
+
+from opentelemetry import metrics
+
+from django.db.models import Sum
+
+_logger = getLogger(__name__)
+
+
+def emmit_disk_space_usage_telemetry(domain_pk=None, *args, **kwargs):
+    from pulpcore.app.models import Artifact, Domain
+
+    artifacts = Artifact.objects.values("pulp_domain__name").annotate(
+        total_size=Sum("size", default=0)
+    )
+
+    if domain_pk:
+        domain = Domain.objects.get(pk=domain_pk)
+        meter = metrics.get_meter(f"domain.{domain.name}.disk_usage.meter")
+    else:
+        _logger.info("Ready to calculate the space usage for all domains")
+        meter = metrics.get_meter("disk_usage.meter")
+
+    gauge = meter.create_gauge(
+        name="disk_usage",
+        description="The total disk size per domain.",
+        unit="bytes",
+    )
+
+    for domain in artifacts:
+        gauge.set(domain["total_size"], {"domain_name": domain["pulp_domain__name"]})

pulpcore/app/util.py

@@ -580,10 +580,15 @@ def build(cls, domain):
 
 
 def init_domain_metrics_exporter():
-    from pulpcore.app.models.domain import Domain
+    PULP_OTEL_SPACE_USAGE_TELEMETRY = (
+        os.getenv("PULP_OTEL_SPACE_USAGE_TELEMETRY", "").lower() == "true"
+    )
 
-    for domain in Domain.objects.all():
-        DomainMetricsEmitterBuilder.build(domain)
+    if PULP_OTEL_SPACE_USAGE_TELEMETRY:
+        from pulpcore.app.models.domain import Domain
+
+        for domain in Domain.objects.all():
+            DomainMetricsEmitterBuilder.build(domain)
 
 
 class PGAdvisoryLock:

pulpcore/app/wsgi.py

@@ -17,9 +17,3 @@
 
 application = get_wsgi_application()
 application = OpenTelemetryMiddleware(application)
-
-# Disabling Storage metrics until we find a solution to resource usage.
-# https://github.com/pulp/pulpcore/issues/5468
-# from pulpcore.app.util import init_domain_metrics_exporter  # noqa: E402
-
-# init_domain_metrics_exporter()