From 2a6d79ffcbfb0ac2a42ac1929a89f50cb3237e3c Mon Sep 17 00:00:00 2001 From: sudip-khanal Date: Thu, 21 Nov 2024 10:47:47 +0545 Subject: [PATCH 1/9] Add Opslearning stats Add validated filter --- per/drf_views.py | 80 ++++++++++++++++++++++++++++++++++++++- per/factories.py | 19 ++++++++++ per/test_views.py | 95 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 192 insertions(+), 2 deletions(-) diff --git a/per/drf_views.py b/per/drf_views.py index 4a3d284e9..1529bcc93 100644 --- a/per/drf_views.py +++ b/per/drf_views.py @@ -3,7 +3,7 @@ import pytz from django.conf import settings from django.db import transaction -from django.db.models import Prefetch, Q +from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery from django.http import HttpResponse from django.shortcuts import get_object_or_404 from django.utils.translation import get_language as django_get_language @@ -20,7 +20,7 @@ from rest_framework.response import Response from rest_framework.settings import api_settings -from api.models import Country +from api.models import Appeal, AppealType, Country from deployments.models import SectorTag from main.permissions import DenyGuestUserMutationPermission, DenyGuestUserPermission from main.utils import SpreadSheetContentNegotiation @@ -921,6 +921,82 @@ def summary(self, request): ) return response.Response(OpsLearningSummarySerializer(ops_learning_summary_instance).data) + @action( + detail=False, + methods=["GET"], + permission_classes=[DenyGuestUserMutationPermission, OpsLearningPermission], + url_path="stats", + ) + def stats(self, request): + """ + Get the Ops Learning stats based on the filters + """ + ops_data = ( + super() + .get_queryset() + .filter(is_validated=True) + .select_related("appeal_code") + .prefetch_related( + "appeal_code__appealdocument", + "sector_validated", + ) + .aggregate( + operations_included=Count("appeal_code", distinct=True), + learning_extracts=Count("id", distinct=True), + sector_covered=Count("sector_validated", distinct=True), + source_used=Count("appeal_code__appealdocument", distinct=True), + ) + ) + + learning_by_sector = ( + SectorTag.objects.filter(title__isnull=False) + .annotate(count=Count("validated_sectors", distinct=True)) + .values("title", "count") + ) + + sources_overtime = { + str(appeal_type_label): OpsLearning.objects.filter(appeal_code__atype=appeal_type, is_validated=True) + .annotate(date=F(("appeal_code__start_date"))) + .values("date") + .annotate(count=Count("appeal_code__appealdocument", distinct=True)) + .order_by("date") + for appeal_type, appeal_type_label in AppealType.choices + } + + region_subquery = Appeal.objects.filter(code=OuterRef("appeal_code"), region__isnull=False).values("region__label")[:1] + region_id = Appeal.objects.filter(code=OuterRef("appeal_code"), region__isnull=False).values("region__id")[:1] + + learning_by_region = ( + OpsLearning.objects.filter(is_validated=True) + .annotate(name=Subquery(region_subquery)) + .values("name") + .annotate(id=Subquery(region_id), count=Count("id", distinct=True)) + .order_by("name") + ) + + country_subquery = Appeal.objects.filter(code=OuterRef("appeal_code"), country__isnull=False).values("country__name")[:1] + country_id = Appeal.objects.filter(code=OuterRef("appeal_code"), country__isnull=False).values("country__id")[:1] + + learning_by_country = ( + OpsLearning.objects.filter(is_validated=True) + .annotate(name=Subquery(country_subquery)) + .values("name") + .annotate(id=Subquery(country_id), count=Count("id", distinct=True)) + .order_by("name") + ) + + data = { + "operations_included": ops_data["operations_included"], + "learning_extracts": ops_data["learning_extracts"], + "sectors_covered": ops_data["sector_covered"], + "sources_used": ops_data["source_used"], + "learning_by_region": learning_by_region, + "learning_by_sector": learning_by_sector, + "sources_overtime": sources_overtime, + "learning_by_country": learning_by_country, + } + return response.Response(data) + class PerDocumentUploadViewSet(viewsets.ModelViewSet): queryset = PerDocumentUpload.objects.all() diff --git a/per/factories.py b/per/factories.py index d9673ea92..5c32ff95b 100644 --- a/per/factories.py +++ b/per/factories.py @@ -3,6 +3,8 @@ import factory from factory import fuzzy +from api.factories.country import CountryFactory +from api.models import Appeal, AppealDocument from deployments.factories.project import SectorTagFactory from per.models import ( AssessmentType, @@ -105,12 +107,22 @@ class Meta: model = FormPrioritization +class AppealFactory(factory.django.DjangoModelFactory): + class Meta: + model = Appeal + + country = factory.SubFactory(CountryFactory) + + class OpsLearningFactory(factory.django.DjangoModelFactory): learning = fuzzy.FuzzyText(length=50) class Meta: model = OpsLearning + appeal_code = factory.SubFactory(AppealFactory) + is_validated = fuzzy.FuzzyChoice([True, False]) + class OpsLearningCacheResponseFactory(factory.django.DjangoModelFactory): used_filters_hash = fuzzy.FuzzyText(length=20) @@ -141,3 +153,10 @@ class OpsLearningComponentCacheResponseFactory(factory.django.DjangoModelFactory class Meta: model = OpsLearningComponentCacheResponse + + +class AppealDocumentFactory(factory.django.DjangoModelFactory): + class Meta: + model = AppealDocument + + appeal = factory.SubFactory(AppealFactory) diff --git a/per/test_views.py b/per/test_views.py index 8860e5501..353a8d0d4 100644 --- a/per/test_views.py +++ b/per/test_views.py @@ -2,14 +2,19 @@ from unittest import mock from api.factories.country import CountryFactory +from api.factories.region import RegionFactory from api.models import AppealType from main.test_case import APITestCase from per.factories import ( + AppealDocumentFactory, + AppealFactory, FormAreaFactory, FormComponentFactory, FormPrioritizationFactory, + OpsLearningFactory, OverviewFactory, PerWorkPlanFactory, + SectorTagFactory, ) from .models import WorkPlanStatus @@ -224,3 +229,93 @@ def test_summary_generation(self, generate_summary): } self.check_response_id(url=url, data=filters) self.assertTrue(generate_summary.assert_called) + + +class OpsLearningStatsTestCase(APITestCase): + @classmethod + def setUpTestData(cls): + cls.region = RegionFactory.create(label="Region A") + cls.country = CountryFactory.create(region=cls.region, name="Country A") + + cls.sector1 = SectorTagFactory.create(title="Sector 1") + cls.sector2 = SectorTagFactory.create(title="Sector 2") + + cls.appeal1 = AppealFactory.create( + region=cls.region, country=cls.country, code="APP001", atype=0, start_date="2023-01-01" + ) + cls.appeal2 = AppealFactory.create( + region=cls.region, country=cls.country, code="APP002", atype=1, start_date="2023-02-01" + ) + + AppealDocumentFactory.create(appeal=cls.appeal1) + AppealDocumentFactory.create(appeal=cls.appeal2) + + cls.ops_learning1 = OpsLearningFactory.create(is_validated=True, appeal_code=cls.appeal1) + cls.ops_learning1.sector_validated.set([cls.sector1]) + + cls.ops_learning2 = OpsLearningFactory.create(is_validated=False, appeal_code=cls.appeal2) + cls.ops_learning2.sector_validated.set([cls.sector2]) + + cls.ops_learning3 = OpsLearningFactory.create(is_validated=False, appeal_code=cls.appeal2) + cls.ops_learning3.sector_validated.set([cls.sector2]) + + def test_ops_learning_stats(self): + url = "/api/v2/ops-learning/stats/" + response = self.client.get(url) + + self.assertEqual(response.status_code, 200) + + expected_keys = [ + "operations_included", + "sources_used", + "learning_extracts", + "sectors_covered", + "sources_overtime", + "learning_by_region", + "learning_by_sector", + "learning_by_country", + ] + for key in expected_keys: + self.assertIn(key, response.data) + + # Updated counts based on validated entries + self.assertEqual(response.data["operations_included"], 1) + self.assertEqual(response.data["sources_used"], 1) + self.assertEqual(response.data["learning_extracts"], 1) + self.assertEqual(response.data["sectors_covered"], 1) + + # Validate learning by region + region_data = response.data["learning_by_region"] + self.assertEqual(len(region_data), 1) + self.assertEqual(region_data[0]["name"], "Region A") + self.assertEqual(region_data[0]["count"], 1) + + # Validate learning by sector + sector_data = response.data["learning_by_sector"] + self.assertEqual(len(sector_data), 2) + self.assertEqual(sector_data[0]["title"], "Sector 1") + self.assertEqual(sector_data[0]["count"], 1) + + # Validate learning by country + country_data = response.data["learning_by_country"] + self.assertEqual(len(country_data), 1) + self.assertEqual(country_data[0]["name"], "Country A") + self.assertEqual(country_data[0]["count"], 1) + + # Validate sources overtime + for appeal_type, label in AppealType.choices: + self.assertIn(label, response.data["sources_overtime"]) + for item in response.data["sources_overtime"][label]: + self.assertIn("date", item) + self.assertIn("count", item) + + date_str = item["date"] + date_str_iso = date_str.replace(tzinfo=None).isoformat() + "Z" + + if label == "DREF": + self.assertEqual(date_str_iso, "2023-01-01T00:00:00Z") + self.assertEqual(item["count"], 1) + + elif label == "Emergency Appeal": + self.assertEqual(date_str_iso, "2023-02-01T00:00:00Z") + self.assertEqual(item["count"], 0) From 543215cbdbb94e82dfe51a037ce444568e4d539e Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Tue, 17 Dec 2024 10:16:44 +0545 Subject: [PATCH 2/9] Refactor ops learning stat api --- per/drf_views.py | 56 +++++++++++++++++----------------------------- per/serializers.py | 33 +++++++++++++++++++++++++++ per/test_views.py | 44 ++++++++++++++++++------------------ 3 files changed, 76 insertions(+), 57 deletions(-) diff --git a/per/drf_views.py b/per/drf_views.py index 1529bcc93..8dc9f6314 100644 --- a/per/drf_views.py +++ b/per/drf_views.py @@ -3,7 +3,7 @@ import pytz from django.conf import settings from django.db import transaction -from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery +from django.db.models import Count, F, Prefetch, Q from django.http import HttpResponse from django.shortcuts import get_object_or_404 from django.utils.translation import get_language as django_get_language @@ -20,7 +20,7 @@ from rest_framework.response import Response from rest_framework.settings import api_settings -from api.models import Appeal, AppealType, Country +from api.models import AppealType, Country, Region from deployments.models import SectorTag from main.permissions import DenyGuestUserMutationPermission, DenyGuestUserPermission from main.utils import SpreadSheetContentNegotiation @@ -82,6 +82,7 @@ OpsLearningInSerializer, OpsLearningOrganizationTypeSerializer, OpsLearningSerializer, + OpsLearningStatSerializer, OpsLearningSummarySerializer, PerAssessmentSerializer, PerDocumentUploadSerializer, @@ -921,6 +922,11 @@ def summary(self, request): ) return response.Response(OpsLearningSummarySerializer(ops_learning_summary_instance).data) + @extend_schema( + request=None, + filters=True, + responses=OpsLearningStatSerializer, + ) @action( detail=False, methods=["GET"], @@ -931,21 +937,12 @@ def stats(self, request): """ Get the Ops Learning stats based on the filters """ - ops_data = ( - super() - .get_queryset() - .filter(is_validated=True) - .select_related("appeal_code") - .prefetch_related( - "appeal_code__appealdocument", - "sector_validated", - ) - .aggregate( - operations_included=Count("appeal_code", distinct=True), - learning_extracts=Count("id", distinct=True), - sector_covered=Count("sector_validated", distinct=True), - source_used=Count("appeal_code__appealdocument", distinct=True), - ) + queryset = self.filter_queryset(self.get_queryset()).filter(is_validated=True) + ops_data = queryset.aggregate( + operations_included=Count("appeal_code", distinct=True), + learning_extracts=Count("id", distinct=True), + sector_covered=Count("sector_validated", distinct=True), + source_used=Count("appeal_code__appealdocument", distinct=True), ) learning_by_sector = ( @@ -955,34 +952,23 @@ def stats(self, request): ) sources_overtime = { - str(appeal_type_label): OpsLearning.objects.filter(appeal_code__atype=appeal_type, is_validated=True) + str(appeal_type_label): queryset.filter(appeal_code__atype=appeal_type) .annotate(date=F(("appeal_code__start_date"))) .values("date") .annotate(count=Count("appeal_code__appealdocument", distinct=True)) - .order_by("date") for appeal_type, appeal_type_label in AppealType.choices } - region_subquery = Appeal.objects.filter(code=OuterRef("appeal_code"), region__isnull=False).values("region__label")[:1] - region_id = Appeal.objects.filter(code=OuterRef("appeal_code"), region__isnull=False).values("region__id")[:1] - learning_by_region = ( - OpsLearning.objects.filter(is_validated=True) - .annotate(name=Subquery(region_subquery)) - .values("name") - .annotate(id=Subquery(region_id), count=Count("id", distinct=True)) - .order_by("name") + Region.objects.filter(appeal__in=queryset.values("appeal_code__id")) + .annotate(region_name=F("label"), count=Count("appeal__opslearning", distinct=True)) + .values("region_name", "count") ) - country_subquery = Appeal.objects.filter(code=OuterRef("appeal_code"), country__isnull=False).values("country__name")[:1] - country_id = Appeal.objects.filter(code=OuterRef("appeal_code"), country__isnull=False).values("country__id")[:1] - learning_by_country = ( - OpsLearning.objects.filter(is_validated=True) - .annotate(name=Subquery(country_subquery)) - .values("name") - .annotate(id=Subquery(country_id), count=Count("id", distinct=True)) - .order_by("name") + Country.objects.filter(appeal__in=queryset.values("appeal_code__id")) + .annotate(country_id=F("id"), country_name=F("name"), count=Count("appeal__opslearning", distinct=True)) + .values("country_id", "country_name", "count") ) data = { diff --git a/per/serializers.py b/per/serializers.py index b2e699e15..8a68dd59d 100644 --- a/per/serializers.py +++ b/per/serializers.py @@ -1253,3 +1253,36 @@ class Meta: "id", "title", ] + + +class LearningByRegionSerializer(serializers.Serializer): + region_id = serializers.IntegerField() + region_name = serializers.CharField() + count = serializers.IntegerField() + + +class LearningByCountrySerializer(serializers.Serializer): + country_id = serializers.IntegerField() + country_name = serializers.CharField() + count = serializers.IntegerField() + + +class LearningBySectorSerializer(serializers.Serializer): + title = serializers.CharField() + count = serializers.IntegerField() + + +class LearningSourcesOvertimeSerializer(serializers.Serializer): + date = serializers.DateField() + count = serializers.IntegerField() + + +class OpsLearningStatSerializer(serializers.Serializer): + operations_included = serializers.IntegerField() + learning_extracts = serializers.IntegerField() + sectors_covered = serializers.IntegerField() + sources_used = serializers.IntegerField() + learning_by_region = LearningByRegionSerializer(many=True) + learning_by_country = LearningByCountrySerializer(many=True) + learning_by_sector = LearningBySectorSerializer(many=True) + sources_overtime = LearningSourcesOvertimeSerializer(many=True) diff --git a/per/test_views.py b/per/test_views.py index 353a8d0d4..79bcedd41 100644 --- a/per/test_views.py +++ b/per/test_views.py @@ -232,39 +232,39 @@ def test_summary_generation(self, generate_summary): class OpsLearningStatsTestCase(APITestCase): - @classmethod - def setUpTestData(cls): - cls.region = RegionFactory.create(label="Region A") - cls.country = CountryFactory.create(region=cls.region, name="Country A") - cls.sector1 = SectorTagFactory.create(title="Sector 1") - cls.sector2 = SectorTagFactory.create(title="Sector 2") + def setUp(self): + super().setUp() + self.region = RegionFactory.create(label="Region A") + self.country = CountryFactory.create(region=self.region, name="Country A") - cls.appeal1 = AppealFactory.create( - region=cls.region, country=cls.country, code="APP001", atype=0, start_date="2023-01-01" + self.sector1 = SectorTagFactory.create(title="Sector 1") + self.sector2 = SectorTagFactory.create(title="Sector 2") + + self.appeal1 = AppealFactory.create( + region=self.region, country=self.country, code="APP001", atype=0, start_date="2023-01-01" ) - cls.appeal2 = AppealFactory.create( - region=cls.region, country=cls.country, code="APP002", atype=1, start_date="2023-02-01" + self.appeal2 = AppealFactory.create( + region=self.region, country=self.country, code="APP002", atype=1, start_date="2023-02-01" ) - AppealDocumentFactory.create(appeal=cls.appeal1) - AppealDocumentFactory.create(appeal=cls.appeal2) + AppealDocumentFactory.create(appeal=self.appeal1) + AppealDocumentFactory.create(appeal=self.appeal2) - cls.ops_learning1 = OpsLearningFactory.create(is_validated=True, appeal_code=cls.appeal1) - cls.ops_learning1.sector_validated.set([cls.sector1]) + self.ops_learning1 = OpsLearningFactory.create(is_validated=True, appeal_code=self.appeal1) + self.ops_learning1.sector_validated.set([self.sector1]) - cls.ops_learning2 = OpsLearningFactory.create(is_validated=False, appeal_code=cls.appeal2) - cls.ops_learning2.sector_validated.set([cls.sector2]) + self.ops_learning2 = OpsLearningFactory.create(is_validated=False, appeal_code=self.appeal2) + self.ops_learning2.sector_validated.set([self.sector2]) - cls.ops_learning3 = OpsLearningFactory.create(is_validated=False, appeal_code=cls.appeal2) - cls.ops_learning3.sector_validated.set([cls.sector2]) + self.ops_learning3 = OpsLearningFactory.create(is_validated=False, appeal_code=self.appeal2) + self.ops_learning3.sector_validated.set([self.sector2]) def test_ops_learning_stats(self): url = "/api/v2/ops-learning/stats/" response = self.client.get(url) - self.assertEqual(response.status_code, 200) - + self.assert_200(response) expected_keys = [ "operations_included", "sources_used", @@ -287,7 +287,7 @@ def test_ops_learning_stats(self): # Validate learning by region region_data = response.data["learning_by_region"] self.assertEqual(len(region_data), 1) - self.assertEqual(region_data[0]["name"], "Region A") + self.assertEqual(region_data[0]["region_name"], "Region A") self.assertEqual(region_data[0]["count"], 1) # Validate learning by sector @@ -299,7 +299,7 @@ def test_ops_learning_stats(self): # Validate learning by country country_data = response.data["learning_by_country"] self.assertEqual(len(country_data), 1) - self.assertEqual(country_data[0]["name"], "Country A") + self.assertEqual(country_data[0]["country_name"], "Country A") self.assertEqual(country_data[0]["count"], 1) # Validate sources overtime From ef38a4f770bd7c91ab49d7017002e038e84de57c Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Tue, 17 Dec 2024 15:56:33 +0545 Subject: [PATCH 3/9] Change in source over time --- per/drf_views.py | 10 ++++------ per/test_views.py | 19 ++----------------- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/per/drf_views.py b/per/drf_views.py index 8dc9f6314..eb3aa9e35 100644 --- a/per/drf_views.py +++ b/per/drf_views.py @@ -20,7 +20,7 @@ from rest_framework.response import Response from rest_framework.settings import api_settings -from api.models import AppealType, Country, Region +from api.models import Appeal, Country, Region from deployments.models import SectorTag from main.permissions import DenyGuestUserMutationPermission, DenyGuestUserPermission from main.utils import SpreadSheetContentNegotiation @@ -952,11 +952,9 @@ def stats(self, request): ) sources_overtime = { - str(appeal_type_label): queryset.filter(appeal_code__atype=appeal_type) - .annotate(date=F(("appeal_code__start_date"))) - .values("date") - .annotate(count=Count("appeal_code__appealdocument", distinct=True)) - for appeal_type, appeal_type_label in AppealType.choices + Appeal.objects.filter(opslearning__in=queryset) + .annotate(type=F("atype"), year=F("start_date"), count=Count("opslearning", distinct=True)) + .values("type", "year", "count") } learning_by_region = ( diff --git a/per/test_views.py b/per/test_views.py index 79bcedd41..9c8ba5fae 100644 --- a/per/test_views.py +++ b/per/test_views.py @@ -302,20 +302,5 @@ def test_ops_learning_stats(self): self.assertEqual(country_data[0]["country_name"], "Country A") self.assertEqual(country_data[0]["count"], 1) - # Validate sources overtime - for appeal_type, label in AppealType.choices: - self.assertIn(label, response.data["sources_overtime"]) - for item in response.data["sources_overtime"][label]: - self.assertIn("date", item) - self.assertIn("count", item) - - date_str = item["date"] - date_str_iso = date_str.replace(tzinfo=None).isoformat() + "Z" - - if label == "DREF": - self.assertEqual(date_str_iso, "2023-01-01T00:00:00Z") - self.assertEqual(item["count"], 1) - - elif label == "Emergency Appeal": - self.assertEqual(date_str_iso, "2023-02-01T00:00:00Z") - self.assertEqual(item["count"], 0) + sources_overtime = response.data["sources_overtime"] + self.assertEqual(len(sources_overtime), 1) From ca1c3d32d7e71bb382ff25c40bf05ec788b6102a Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Tue, 17 Dec 2024 16:02:48 +0545 Subject: [PATCH 4/9] Add filter check for validated opslearning --- per/drf_views.py | 6 +++--- per/serializers.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/per/drf_views.py b/per/drf_views.py index eb3aa9e35..709333990 100644 --- a/per/drf_views.py +++ b/per/drf_views.py @@ -946,15 +946,15 @@ def stats(self, request): ) learning_by_sector = ( - SectorTag.objects.filter(title__isnull=False) + SectorTag.objects.filter(validated_sectors__in=queryset, title__isnull=False) .annotate(count=Count("validated_sectors", distinct=True)) .values("title", "count") ) sources_overtime = { Appeal.objects.filter(opslearning__in=queryset) - .annotate(type=F("atype"), year=F("start_date"), count=Count("opslearning", distinct=True)) - .values("type", "year", "count") + .annotate(type=F("atype"), date=F("start_date"), count=Count("opslearning", distinct=True)) + .values("type", "date", "count") } learning_by_region = ( diff --git a/per/serializers.py b/per/serializers.py index 8a68dd59d..88a8a2ca8 100644 --- a/per/serializers.py +++ b/per/serializers.py @@ -1256,7 +1256,6 @@ class Meta: class LearningByRegionSerializer(serializers.Serializer): - region_id = serializers.IntegerField() region_name = serializers.CharField() count = serializers.IntegerField() @@ -1273,6 +1272,7 @@ class LearningBySectorSerializer(serializers.Serializer): class LearningSourcesOvertimeSerializer(serializers.Serializer): + type = serializers.IntegerField() date = serializers.DateField() count = serializers.IntegerField() From afff697aea129661b21a72dd2ecbb2d7e135160d Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Tue, 17 Dec 2024 19:36:09 +0545 Subject: [PATCH 5/9] fix test cases and refactor queryset --- per/drf_views.py | 27 +++++++++++++++++++-------- per/serializers.py | 2 +- per/test_views.py | 20 +++++++------------- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/per/drf_views.py b/per/drf_views.py index 709333990..0c2726ec0 100644 --- a/per/drf_views.py +++ b/per/drf_views.py @@ -951,21 +951,32 @@ def stats(self, request): .values("title", "count") ) - sources_overtime = { + sources_overtime = ( Appeal.objects.filter(opslearning__in=queryset) - .annotate(type=F("atype"), date=F("start_date"), count=Count("opslearning", distinct=True)) + .annotate( + type=F("atype"), + date=F("start_date"), + count=Count("appealdocument", distinct=True), + ) .values("type", "date", "count") - } + ) learning_by_region = ( - Region.objects.filter(appeal__in=queryset.values("appeal_code__id")) - .annotate(region_name=F("label"), count=Count("appeal__opslearning", distinct=True)) + Region.objects.filter(appeal__opslearning__in=queryset) + .annotate( + region_name=F("label"), + count=Count("appeal__opslearning", distinct=True), + ) .values("region_name", "count") ) learning_by_country = ( - Country.objects.filter(appeal__in=queryset.values("appeal_code__id")) - .annotate(country_id=F("id"), country_name=F("name"), count=Count("appeal__opslearning", distinct=True)) + Country.objects.filter(appeal__opslearning__in=queryset) + .annotate( + country_id=F("id"), + country_name=F("name"), + count=Count("appeal__opslearning", distinct=True), + ) .values("country_id", "country_name", "count") ) @@ -979,7 +990,7 @@ def stats(self, request): "sources_overtime": sources_overtime, "learning_by_country": learning_by_country, } - return response.Response(data) + return response.Response(OpsLearningStatSerializer(data).data) class PerDocumentUploadViewSet(viewsets.ModelViewSet): diff --git a/per/serializers.py b/per/serializers.py index 88a8a2ca8..07714160a 100644 --- a/per/serializers.py +++ b/per/serializers.py @@ -1273,7 +1273,7 @@ class LearningBySectorSerializer(serializers.Serializer): class LearningSourcesOvertimeSerializer(serializers.Serializer): type = serializers.IntegerField() - date = serializers.DateField() + date = serializers.DateTimeField() count = serializers.IntegerField() diff --git a/per/test_views.py b/per/test_views.py index 9c8ba5fae..2e52983d3 100644 --- a/per/test_views.py +++ b/per/test_views.py @@ -254,7 +254,7 @@ def setUp(self): self.ops_learning1 = OpsLearningFactory.create(is_validated=True, appeal_code=self.appeal1) self.ops_learning1.sector_validated.set([self.sector1]) - self.ops_learning2 = OpsLearningFactory.create(is_validated=False, appeal_code=self.appeal2) + self.ops_learning2 = OpsLearningFactory.create(is_validated=True, appeal_code=self.appeal2) self.ops_learning2.sector_validated.set([self.sector2]) self.ops_learning3 = OpsLearningFactory.create(is_validated=False, appeal_code=self.appeal2) @@ -279,28 +279,22 @@ def test_ops_learning_stats(self): self.assertIn(key, response.data) # Updated counts based on validated entries - self.assertEqual(response.data["operations_included"], 1) - self.assertEqual(response.data["sources_used"], 1) - self.assertEqual(response.data["learning_extracts"], 1) - self.assertEqual(response.data["sectors_covered"], 1) + self.assertEqual(response.data["operations_included"], 2) + self.assertEqual(response.data["sources_used"], 2) + self.assertEqual(response.data["learning_extracts"], 2) + self.assertEqual(response.data["sectors_covered"], 2) # Validate learning by region region_data = response.data["learning_by_region"] - self.assertEqual(len(region_data), 1) - self.assertEqual(region_data[0]["region_name"], "Region A") - self.assertEqual(region_data[0]["count"], 1) + self.assertEqual(region_data[0]["count"], 2) # Validate learning by sector sector_data = response.data["learning_by_sector"] self.assertEqual(len(sector_data), 2) - self.assertEqual(sector_data[0]["title"], "Sector 1") - self.assertEqual(sector_data[0]["count"], 1) # Validate learning by country country_data = response.data["learning_by_country"] self.assertEqual(len(country_data), 1) - self.assertEqual(country_data[0]["country_name"], "Country A") - self.assertEqual(country_data[0]["count"], 1) sources_overtime = response.data["sources_overtime"] - self.assertEqual(len(sources_overtime), 1) + self.assertEqual(len(sources_overtime), 2) From 569eae94eb9abaf1c70373c6cb0ecde9fad162f7 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 18 Dec 2024 10:06:58 +0545 Subject: [PATCH 6/9] Add start date field in appeal serailizer --- per/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/per/serializers.py b/per/serializers.py index 07714160a..687e96fa3 100644 --- a/per/serializers.py +++ b/per/serializers.py @@ -969,6 +969,7 @@ class Meta: "atype", "event_details", "country", + "start_date", ) From 641fb4daa7f5adba423a117425369e47b0c0c93a Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 18 Dec 2024 11:53:40 +0545 Subject: [PATCH 7/9] Change filter queryset and add label for appealtype --- per/drf_views.py | 26 ++++++++++++++------------ per/factories.py | 1 - per/serializers.py | 21 ++++++++++++++------- per/test_views.py | 12 ------------ 4 files changed, 28 insertions(+), 32 deletions(-) diff --git a/per/drf_views.py b/per/drf_views.py index 0c2726ec0..fa5ac96f6 100644 --- a/per/drf_views.py +++ b/per/drf_views.py @@ -945,32 +945,34 @@ def stats(self, request): source_used=Count("appeal_code__appealdocument", distinct=True), ) - learning_by_sector = ( + learning_by_sector_qs = ( SectorTag.objects.filter(validated_sectors__in=queryset, title__isnull=False) - .annotate(count=Count("validated_sectors", distinct=True)) - .values("title", "count") + .annotate(sector_id=F("id"), count=Count("validated_sectors", distinct=True)) + .values("sector_id", "title", "count") ) - sources_overtime = ( + # NOTE: Queryset is unbounded, we may need to add some start_date filter. + sources_overtime_qs = ( Appeal.objects.filter(opslearning__in=queryset) .annotate( type=F("atype"), date=F("start_date"), count=Count("appealdocument", distinct=True), ) - .values("type", "date", "count") + .values("id", "type", "date", "count") ) - learning_by_region = ( + learning_by_region_qs = ( Region.objects.filter(appeal__opslearning__in=queryset) .annotate( + region_id=F("id"), region_name=F("label"), count=Count("appeal__opslearning", distinct=True), ) - .values("region_name", "count") + .values("region_id", "region_name", "count") ) - learning_by_country = ( + learning_by_country_qs = ( Country.objects.filter(appeal__opslearning__in=queryset) .annotate( country_id=F("id"), @@ -985,10 +987,10 @@ def stats(self, request): "learning_extracts": ops_data["learning_extracts"], "sectors_covered": ops_data["sector_covered"], "sources_used": ops_data["source_used"], - "learning_by_region": learning_by_region, - "learning_by_sector": learning_by_sector, - "sources_overtime": sources_overtime, - "learning_by_country": learning_by_country, + "learning_by_region": learning_by_region_qs, + "learning_by_sector": learning_by_sector_qs, + "sources_overtime": sources_overtime_qs, + "learning_by_country": learning_by_country_qs, } return response.Response(OpsLearningStatSerializer(data).data) diff --git a/per/factories.py b/per/factories.py index 5c32ff95b..92c971215 100644 --- a/per/factories.py +++ b/per/factories.py @@ -121,7 +121,6 @@ class Meta: model = OpsLearning appeal_code = factory.SubFactory(AppealFactory) - is_validated = fuzzy.FuzzyChoice([True, False]) class OpsLearningCacheResponseFactory(factory.django.DjangoModelFactory): diff --git a/per/serializers.py b/per/serializers.py index 687e96fa3..8f1602806 100644 --- a/per/serializers.py +++ b/per/serializers.py @@ -1257,6 +1257,7 @@ class Meta: class LearningByRegionSerializer(serializers.Serializer): + region_id = serializers.IntegerField() region_name = serializers.CharField() count = serializers.IntegerField() @@ -1268,21 +1269,27 @@ class LearningByCountrySerializer(serializers.Serializer): class LearningBySectorSerializer(serializers.Serializer): + sector_id = serializers.IntegerField() title = serializers.CharField() count = serializers.IntegerField() class LearningSourcesOvertimeSerializer(serializers.Serializer): - type = serializers.IntegerField() - date = serializers.DateTimeField() - count = serializers.IntegerField() + type = serializers.IntegerField(required=True) + type_display = serializers.SerializerMethodField(read_only=True) + date = serializers.DateTimeField(required=True) + count = serializers.IntegerField(required=True) + + def get_type_display(self, obj): + type = obj.get("type") + return AppealType(type).label class OpsLearningStatSerializer(serializers.Serializer): - operations_included = serializers.IntegerField() - learning_extracts = serializers.IntegerField() - sectors_covered = serializers.IntegerField() - sources_used = serializers.IntegerField() + operations_included = serializers.IntegerField(required=True) + learning_extracts = serializers.IntegerField(required=True) + sectors_covered = serializers.IntegerField(required=True) + sources_used = serializers.IntegerField(required=True) learning_by_region = LearningByRegionSerializer(many=True) learning_by_country = LearningByCountrySerializer(many=True) learning_by_sector = LearningBySectorSerializer(many=True) diff --git a/per/test_views.py b/per/test_views.py index 2e52983d3..f113d2f4b 100644 --- a/per/test_views.py +++ b/per/test_views.py @@ -265,18 +265,6 @@ def test_ops_learning_stats(self): response = self.client.get(url) self.assert_200(response) - expected_keys = [ - "operations_included", - "sources_used", - "learning_extracts", - "sectors_covered", - "sources_overtime", - "learning_by_region", - "learning_by_sector", - "learning_by_country", - ] - for key in expected_keys: - self.assertIn(key, response.data) # Updated counts based on validated entries self.assertEqual(response.data["operations_included"], 2) From da571fcafcd1c70e62b4506fdd4e92f6bb7d6d53 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 18 Dec 2024 11:58:30 +0545 Subject: [PATCH 8/9] Define required field in serializers --- per/serializers.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/per/serializers.py b/per/serializers.py index 8f1602806..2233347d3 100644 --- a/per/serializers.py +++ b/per/serializers.py @@ -1257,21 +1257,21 @@ class Meta: class LearningByRegionSerializer(serializers.Serializer): - region_id = serializers.IntegerField() - region_name = serializers.CharField() - count = serializers.IntegerField() + region_id = serializers.IntegerField(required=True) + region_name = serializers.CharField(required=True) + count = serializers.IntegerField(required=True) class LearningByCountrySerializer(serializers.Serializer): - country_id = serializers.IntegerField() - country_name = serializers.CharField() - count = serializers.IntegerField() + country_id = serializers.IntegerField(required=True) + country_name = serializers.CharField(required=True) + count = serializers.IntegerField(required=True) class LearningBySectorSerializer(serializers.Serializer): - sector_id = serializers.IntegerField() - title = serializers.CharField() - count = serializers.IntegerField() + sector_id = serializers.IntegerField(required=True) + title = serializers.CharField(required=True) + count = serializers.IntegerField(required=True) class LearningSourcesOvertimeSerializer(serializers.Serializer): From 33c3626c396d507ac2f8cc959ee31966d645f7b2 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 18 Dec 2024 12:06:13 +0545 Subject: [PATCH 9/9] Remove subfactories and id in appeal queryset --- per/drf_views.py | 2 +- per/factories.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/per/drf_views.py b/per/drf_views.py index fa5ac96f6..00cc257f8 100644 --- a/per/drf_views.py +++ b/per/drf_views.py @@ -959,7 +959,7 @@ def stats(self, request): date=F("start_date"), count=Count("appealdocument", distinct=True), ) - .values("id", "type", "date", "count") + .values("type", "date", "count") ) learning_by_region_qs = ( diff --git a/per/factories.py b/per/factories.py index 92c971215..f27181bb9 100644 --- a/per/factories.py +++ b/per/factories.py @@ -3,7 +3,6 @@ import factory from factory import fuzzy -from api.factories.country import CountryFactory from api.models import Appeal, AppealDocument from deployments.factories.project import SectorTagFactory from per.models import ( @@ -111,8 +110,6 @@ class AppealFactory(factory.django.DjangoModelFactory): class Meta: model = Appeal - country = factory.SubFactory(CountryFactory) - class OpsLearningFactory(factory.django.DjangoModelFactory): learning = fuzzy.FuzzyText(length=50) @@ -120,8 +117,6 @@ class OpsLearningFactory(factory.django.DjangoModelFactory): class Meta: model = OpsLearning - appeal_code = factory.SubFactory(AppealFactory) - class OpsLearningCacheResponseFactory(factory.django.DjangoModelFactory): used_filters_hash = fuzzy.FuzzyText(length=20)