feat: run tasks to calculate account-centric tasks (pypi#14143)

tests/unit/accounts/test_core.py

@@ -16,6 +16,7 @@
 import pretend
 import pytest
 
+from celery.schedules import crontab
 from pyramid.httpexceptions import HTTPUnauthorized
 
 from warehouse import accounts
@@ -34,6 +35,7 @@
     TokenServiceFactory,
     database_login_factory,
 )
+from warehouse.accounts.tasks import compute_user_metrics
 from warehouse.errors import BasicAuthBreachedPassword, BasicAuthFailedPassword
 from warehouse.events.tags import EventTag
 from warehouse.oidc.interfaces import SignedClaims
@@ -423,6 +425,7 @@ def test_includeme(monkeypatch):
         set_security_policy=pretend.call_recorder(lambda p: None),
         maybe_dotted=pretend.call_recorder(lambda path: path),
         add_route_predicate=pretend.call_recorder(lambda name, cls: None),
+        add_periodic_task=pretend.call_recorder(lambda *a, **kw: None),
     )
 
     accounts.includeme(config)
@@ -473,3 +476,7 @@ def test_includeme(monkeypatch):
             ]
         )
     ]
+    assert (
+        pretend.call(crontab(minute="*/20"), compute_user_metrics)
+        in config.add_periodic_task.calls
+    )

tests/unit/accounts/test_tasks.py

@@ -0,0 +1,58 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pretend
+
+from warehouse.accounts.tasks import compute_user_metrics
+
+from ...common.db.accounts import EmailFactory, UserFactory
+from ...common.db.packaging import ProjectFactory, ReleaseFactory
+
+
+def _create_email_project_with_release(user, verified):
+    EmailFactory.create(user=user, verified=verified)
+    result = ProjectFactory.create()
+    ReleaseFactory.create(project=result, uploader=user)
+    return result
+
+
+def test_compute_user_metrics(db_request, metrics):
+    # Create an active user with no email
+    UserFactory.create()
+    # Create an inactive user
+    UserFactory.create(is_active=False)
+    # Create a user with an unverified email
+    unverified_email_user = UserFactory.create()
+    EmailFactory.create(user=unverified_email_user, verified=False)
+    # Create a user with a verified email
+    verified_email_user = UserFactory.create()
+    EmailFactory.create(user=verified_email_user, verified=True)
+    # Create a user with a verified email and a release
+    verified_email_release_user = UserFactory.create()
+    _create_email_project_with_release(verified_email_release_user, verified=True)
+    # Create an active user with an unverified email and a release
+    unverified_email_release_user = UserFactory.create(is_active=True)
+    _create_email_project_with_release(unverified_email_release_user, verified=False)
+    compute_user_metrics(db_request)
+
+    assert metrics.gauge.calls == [
+        pretend.call("warehouse.users.count", 6),
+        pretend.call("warehouse.users.count", 5, tags={"active": "true"}),
+        pretend.call(
+            "warehouse.users.count", 3, tags={"active": "true", "verified": "false"}
+        ),
+        pretend.call(
+            "warehouse.users.count",
+            1,
+            tags={"active": "true", "verified": "false", "releases": "true"},
+        ),
+    ]

warehouse/accounts/__init__.py

@@ -10,6 +10,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from celery.schedules import crontab
+
 from warehouse.accounts.interfaces import (
     IEmailBreachedService,
     IPasswordBreachedService,
@@ -29,6 +31,7 @@
     TokenServiceFactory,
     database_login_factory,
 )
+from warehouse.accounts.tasks import compute_user_metrics
 from warehouse.admin.flags import AdminFlagValue
 from warehouse.macaroons.security_policy import MacaroonSecurityPolicy
 from warehouse.oidc.utils import OIDCContext
@@ -191,3 +194,6 @@ def includeme(config):
         IRateLimiter,
         name="accounts.search",
     )
+
+    # Add a periodic task to generate Account metrics
+    config.add_periodic_task(crontab(minute="*/20"), compute_user_metrics)

warehouse/accounts/tasks.py

@@ -0,0 +1,62 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sqlalchemy import func
+
+from warehouse import tasks
+from warehouse.accounts.models import Email, User
+from warehouse.metrics import IMetricsService
+from warehouse.packaging.models import Release
+
+
+@tasks.task(ignore_result=True, acks_late=True)
+def compute_user_metrics(request):
+    """
+    Report metrics about the users in the database.
+    """
+    metrics = request.find_service(IMetricsService, context=None)
+
+    # Total of users
+    metrics.gauge(
+        "warehouse.users.count",
+        request.db.query(func.count(User.id)).scalar(),
+    )
+
+    # Total of active users
+    metrics.gauge(
+        "warehouse.users.count",
+        request.db.query(func.count(User.id)).filter(User.is_active).scalar(),
+        tags={"active": "true"},
+    )
+
+    # Total active users with unverified emails
+    metrics.gauge(
+        "warehouse.users.count",
+        request.db.query(func.count(User.id))
+        .outerjoin(Email)
+        .filter(User.is_active)
+        .filter((Email.verified == None) | (Email.verified == False))  # noqa E711
+        .scalar(),
+        tags={"active": "true", "verified": "false"},
+    )
+
+    # Total active users with unverified emails, and have project releases
+    metrics.gauge(
+        "warehouse.users.count",
+        request.db.query(func.count(User.id))
+        .outerjoin(Email)
+        .join(Release, Release.uploader_id == User.id)
+        .filter(User.is_active)
+        .filter((Email.verified == None) | (Email.verified == False))  # noqa E711
+        .scalar(),
+        tags={"active": "true", "verified": "false", "releases": "true"},
+    )