From 9d4444ba7b582fe1cd803bd29369dca0c62ea41e Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Mon, 11 Nov 2024 14:09:09 +0545 Subject: [PATCH 1/5] fix translation issue on primary summary Add Lazytranslation flag for translation --- per/drf_views.py | 11 ++++++++++- per/ops_learning_summary.py | 10 ++++++++++ per/task.py | 15 ++++----------- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/per/drf_views.py b/per/drf_views.py index e00908b0f..baddcccd9 100644 --- a/per/drf_views.py +++ b/per/drf_views.py @@ -6,6 +6,7 @@ from django.db.models import Prefetch, Q from django.http import HttpResponse from django.shortcuts import get_object_or_404 +from django.utils.translation import get_language as django_get_language from django_filters import rest_framework as filters from django_filters.widgets import CSVWidget from drf_spectacular.utils import extend_schema @@ -882,7 +883,15 @@ def summary(self, request): if ops_learning_summary_instance.status == OpsLearningCacheResponse.Status.SUCCESS: return response.Response(OpsLearningSummarySerializer(ops_learning_summary_instance).data) - transaction.on_commit(lambda: generate_summary.delay(ops_learning_summary_instance.id, filter_data)) + requested_lang = django_get_language() + translation_lazy = requested_lang != "en" + transaction.on_commit( + lambda: generate_summary.delay( + ops_learning_summary_id=ops_learning_summary_instance.id, + filter_data=filter_data, + translation_lazy=translation_lazy, + ) + ) return response.Response(OpsLearningSummarySerializer(ops_learning_summary_instance).data) diff --git a/per/ops_learning_summary.py b/per/ops_learning_summary.py index 2fd323d54..a3f4dce88 100644 --- a/per/ops_learning_summary.py +++ b/per/ops_learning_summary.py @@ -665,6 +665,7 @@ def process_learnings_sector(sector, df, max_length_per_section): + sector + "\n----------------\n" + "\n----------------\n".join(df_sliced["learning"]) + + "\n\n" ) return learnings_sector @@ -681,6 +682,7 @@ def process_learnings_component(component, df, max_length_per_section): + component + "\n----------------\n" + "\n----------------\n".join(df_sliced["learning"]) + + "\n\n" ) return learnings_component @@ -944,6 +946,14 @@ def get_or_create_primary_summary( primary_summary=primary_summary, ) + # Translating the primary summary + transaction.on_commit( + lambda: translate_model_fields.delay( + get_model_name(type(ops_learning_summary_instance)), + ops_learning_summary_instance.pk, + ) + ) + @classmethod def get_or_create_secondary_summary( cls, diff --git a/per/task.py b/per/task.py index 478531ee8..06d0d892b 100644 --- a/per/task.py +++ b/per/task.py @@ -1,9 +1,7 @@ from celery import shared_task -from django.db import transaction +from django.test import override_settings from api.logger import logger -from api.utils import get_model_name -from lang.tasks import translate_model_fields from main.lock import RedisLockKey, redis_lock from per.models import OpsLearningCacheResponse from per.ops_learning_summary import OpsLearningSummaryTask @@ -63,12 +61,6 @@ def generate_ops_learning_summary(ops_learning_summary_id: int, filter_data: dic instance=ops_learning_summary_instance, status=OpsLearningCacheResponse.Status.SUCCESS, ) - transaction.on_commit( - lambda: translate_model_fields.delay( - get_model_name(type(ops_learning_summary_instance)), - ops_learning_summary_instance.pk, - ) - ) return True except Exception: OpsLearningSummaryTask.change_ops_learning_status( @@ -87,9 +79,10 @@ def generate_ops_learning_summary(ops_learning_summary_id: int, filter_data: dic @shared_task -def generate_summary(ops_learning_summary_id: int, filter_data: dict): +def generate_summary(ops_learning_summary_id: int, filter_data: dict, translation_lazy: bool = True): with redis_lock(key=RedisLockKey.OPERATION_LEARNING_SUMMARY, id=ops_learning_summary_id) as acquired: if not acquired: logger.warning("Ops learning summary generation is already in progress") return False - return generate_ops_learning_summary(ops_learning_summary_id, filter_data) + with override_settings(CELERY_TASK_ALWAYS_EAGER=not translation_lazy): + return generate_ops_learning_summary(ops_learning_summary_id, filter_data) From 142f5a9998679b9e601f9ba1fb09b7161e9e6258 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Thu, 14 Nov 2024 17:55:07 +0545 Subject: [PATCH 2/5] Add regenerate summary action in admin --- per/admin.py | 19 +++++++++++++++++++ per/ops_learning_summary.py | 16 ++++++++++++++-- per/task.py | 17 ++++++++++++++--- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/per/admin.py b/per/admin.py index afd838972..ca3d3a826 100644 --- a/per/admin.py +++ b/per/admin.py @@ -4,12 +4,14 @@ from django.contrib import admin from django.http import HttpResponse +from django.utils.translation import get_language as django_get_language from reversion_compare.admin import CompareVersionAdmin import per.models as models from api.models import Appeal from lang.admin import TranslationAdmin, TranslationInlineModelAdmin from per.admin_classes import GotoNextModelAdmin, RegionRestrictedAdmin +from per.task import generate_summary class FormDataInline(admin.TabularInline, TranslationInlineModelAdmin): @@ -324,6 +326,23 @@ class OpsLearningCacheResponseAdmin(TranslationAdmin): "exported_file", "exported_at", ) + actions = ["regenerate_summary"] + + def regenerate_summary(self, request, queryset): + """ + Regenerate the summary of the selected OpsLearningCacheResponse objects. + """ + requested_lang = django_get_language() + for obj in queryset: + generate_summary.delay( + ops_learning_summary_id=obj.id, + filter_data=obj.used_filters, + translation_lazy=requested_lang != "en", + # NOTE: Regenerating the summary will overwrite the cache + overwrite_prompt_cache=True, + ) + + regenerate_summary.short_description = "Regenerate Summary for selected Ops Learning Cache Response" class OpsLearningPromptResponseCacheAdmin(admin.ModelAdmin): diff --git a/per/ops_learning_summary.py b/per/ops_learning_summary.py index a3f4dce88..7ad59a17b 100644 --- a/per/ops_learning_summary.py +++ b/per/ops_learning_summary.py @@ -831,13 +831,21 @@ def _modify_summary(summary: dict) -> dict: return processed_summary @classmethod - def _get_or_create_summary(cls, prompt: str, prompt_hash: str, type: OpsLearningPromptResponseCache.PromptType) -> dict: + def _get_or_create_summary( + cls, prompt: str, prompt_hash: str, type: OpsLearningPromptResponseCache.PromptType, overwrite_prompt_cache: bool = False + ) -> dict: instance, created = OpsLearningPromptResponseCache.objects.get_or_create( prompt_hash=prompt_hash, type=type, defaults={"prompt": prompt}, ) - if created or not bool(instance.response): + """ + NOTE: + 1. If the prompt response is not found in the cache, it regenerates the summary + 2. If overwrite_prompt_cache is True, it regenerates the summary + 3. If new obj is created, it generates the summary + """ + if overwrite_prompt_cache or created or not bool(instance.response): summary = cls.generate_summary(prompt, type) instance.response = summary instance.save(update_fields=["response"]) @@ -926,6 +934,7 @@ def get_or_create_primary_summary( cls, ops_learning_summary_instance: OpsLearningCacheResponse, primary_learning_prompt: str, + overwrite_prompt_cache: bool = False, ): """Retrieves or Generates the primary summary based on the provided prompt.""" logger.info("Retrieving or generating primary summary.") @@ -938,6 +947,7 @@ def get_or_create_primary_summary( prompt=primary_learning_prompt, prompt_hash=primary_prompt_hash, type=OpsLearningPromptResponseCache.PromptType.PRIMARY, + overwrite_prompt_cache=overwrite_prompt_cache, ) # Saving into the database @@ -959,6 +969,7 @@ def get_or_create_secondary_summary( cls, ops_learning_summary_instance: OpsLearningCacheResponse, secondary_learning_prompt: str, + overwrite_prompt_cache: bool = False, ): """Retrieves or Generates the summary based on the provided prompts.""" logger.info("Retrieving or generating secondary summary.") @@ -971,6 +982,7 @@ def get_or_create_secondary_summary( prompt=secondary_learning_prompt, prompt_hash=secondary_prompt_hash, type=OpsLearningPromptResponseCache.PromptType.SECONDARY, + overwrite_prompt_cache=overwrite_prompt_cache, ) # Saving into the database diff --git a/per/task.py b/per/task.py index 06d0d892b..b831cd642 100644 --- a/per/task.py +++ b/per/task.py @@ -7,7 +7,7 @@ from per.ops_learning_summary import OpsLearningSummaryTask -def generate_ops_learning_summary(ops_learning_summary_id: int, filter_data: dict): +def generate_ops_learning_summary(ops_learning_summary_id: int, filter_data: dict, overwrite_prompt_cache: bool = False): ops_learning_summary_instance = OpsLearningCacheResponse.objects.filter(id=ops_learning_summary_id).first() if not ops_learning_summary_instance: logger.error("Ops learning summary not found", exc_info=True) @@ -44,6 +44,7 @@ def generate_ops_learning_summary(ops_learning_summary_id: int, filter_data: dic OpsLearningSummaryTask.get_or_create_primary_summary( ops_learning_summary_instance=ops_learning_summary_instance, primary_learning_prompt=primary_learning_prompt, + overwrite_prompt_cache=overwrite_prompt_cache, ) # Prioritize excerpts for secondary insights @@ -54,6 +55,7 @@ def generate_ops_learning_summary(ops_learning_summary_id: int, filter_data: dic OpsLearningSummaryTask.get_or_create_secondary_summary( ops_learning_summary_instance=ops_learning_summary_instance, secondary_learning_prompt=secondary_learning_prompt, + overwrite_prompt_cache=overwrite_prompt_cache, ) # Change Ops Learning Summary Status to SUCCESS @@ -79,10 +81,19 @@ def generate_ops_learning_summary(ops_learning_summary_id: int, filter_data: dic @shared_task -def generate_summary(ops_learning_summary_id: int, filter_data: dict, translation_lazy: bool = True): +def generate_summary( + ops_learning_summary_id: int, + filter_data: dict, + translation_lazy: bool = True, + overwrite_prompt_cache: bool = False, +): with redis_lock(key=RedisLockKey.OPERATION_LEARNING_SUMMARY, id=ops_learning_summary_id) as acquired: if not acquired: logger.warning("Ops learning summary generation is already in progress") return False with override_settings(CELERY_TASK_ALWAYS_EAGER=not translation_lazy): - return generate_ops_learning_summary(ops_learning_summary_id, filter_data) + return generate_ops_learning_summary( + ops_learning_summary_id=ops_learning_summary_id, + filter_data=filter_data, + overwrite_prompt_cache=overwrite_prompt_cache, + ) From 33c0ca0aab5d18d9f518e557df80032d632f2a3f Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Fri, 15 Nov 2024 15:27:59 +0545 Subject: [PATCH 3/5] Add filters for operational learning and refine summary content Add learning_type filter, validated_organization --- per/admin.py | 5 +++-- per/drf_views.py | 5 +++-- per/ops_learning_summary.py | 11 ++++++++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/per/admin.py b/per/admin.py index ca3d3a826..a0c63d8b2 100644 --- a/per/admin.py +++ b/per/admin.py @@ -309,8 +309,8 @@ def break_to_rows(many2many, many2many_validated, is_validated, idx): class OpsLearningCacheResponseAdmin(TranslationAdmin): search_fields = ( - "response", "id", + "used_ops_learning__appeal_code__aid", ) list_display = ( "__str__", @@ -319,6 +319,7 @@ class OpsLearningCacheResponseAdmin(TranslationAdmin): "insights3_title", "status", ) + list_filter = ("status",) used_ops_learning_in = "used_ops_learning_in" autocomplete_fields = ("used_ops_learning",) exclude = ( @@ -337,7 +338,7 @@ def regenerate_summary(self, request, queryset): generate_summary.delay( ops_learning_summary_id=obj.id, filter_data=obj.used_filters, - translation_lazy=requested_lang != "en", + translation_lazy=requested_lang == "en", # NOTE: Regenerating the summary will overwrite the cache overwrite_prompt_cache=True, ) diff --git a/per/drf_views.py b/per/drf_views.py index baddcccd9..71973269b 100644 --- a/per/drf_views.py +++ b/per/drf_views.py @@ -707,7 +707,9 @@ class Meta: "is_validated": ("exact",), "learning": ("exact", "icontains"), "learning_validated": ("exact", "icontains"), + "type_validated": ("exact", "in"), "organization_validated": ("exact",), + "organization_validated__title": ("exact", "in"), "appeal_code": ("exact", "in"), "appeal_code__code": ("exact", "icontains", "in"), "appeal_code__num_beneficiaries": ("exact", "gt", "gte", "lt", "lte"), @@ -884,12 +886,11 @@ def summary(self, request): return response.Response(OpsLearningSummarySerializer(ops_learning_summary_instance).data) requested_lang = django_get_language() - translation_lazy = requested_lang != "en" transaction.on_commit( lambda: generate_summary.delay( ops_learning_summary_id=ops_learning_summary_instance.id, filter_data=filter_data, - translation_lazy=translation_lazy, + translation_lazy=requested_lang == "en", ) ) return response.Response(OpsLearningSummarySerializer(ops_learning_summary_instance).data) diff --git a/per/ops_learning_summary.py b/per/ops_learning_summary.py index 7ad59a17b..58c4d2fd6 100644 --- a/per/ops_learning_summary.py +++ b/per/ops_learning_summary.py @@ -819,7 +819,7 @@ def _modify_summary(summary: dict) -> dict: continue if confidence_level in value["content"].lower(): parts = re.split(rf"(?i)\b{confidence_level}\b", value["content"]) - value["content"] = parts[0] + value["content"] = parts[0].strip() + "." value["confidence level"] = parts[1][1:].strip() return summary @@ -834,7 +834,7 @@ def _modify_summary(summary: dict) -> dict: def _get_or_create_summary( cls, prompt: str, prompt_hash: str, type: OpsLearningPromptResponseCache.PromptType, overwrite_prompt_cache: bool = False ) -> dict: - instance, created = OpsLearningPromptResponseCache.objects.get_or_create( + instance, created = OpsLearningPromptResponseCache.objects.update_or_create( prompt_hash=prompt_hash, type=type, defaults={"prompt": prompt}, @@ -845,7 +845,7 @@ def _get_or_create_summary( 2. If overwrite_prompt_cache is True, it regenerates the summary 3. If new obj is created, it generates the summary """ - if overwrite_prompt_cache or created or not bool(instance.response): + if overwrite_prompt_cache or created or bool(instance.response) is False: summary = cls.generate_summary(prompt, type) instance.response = summary instance.save(update_fields=["response"]) @@ -984,6 +984,11 @@ def get_or_create_secondary_summary( type=OpsLearningPromptResponseCache.PromptType.SECONDARY, overwrite_prompt_cache=overwrite_prompt_cache, ) + if overwrite_prompt_cache: + logger.info("Clearing the cache for secondary summary.") + # NOTE: find a better way to update the cache + OpsLearningComponentCacheResponse.objects.filter(filter_response=ops_learning_summary_instance).delete() + OpsLearningSectorCacheResponse.objects.filter(filter_response=ops_learning_summary_instance).delete() # Saving into the database cls.secondary_response_save_to_db( From 8c668ca48b7962733f754fa6057f4396f8a00f32 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Mon, 18 Nov 2024 21:25:25 +0545 Subject: [PATCH 4/5] Add post processing to check excerpts in summary Regenerating summary if any excerpts id exists in summary --- api/serializers.py | 1 + per/ops_learning_summary.py | 115 ++++++++++++++++++++---------------- 2 files changed, 65 insertions(+), 51 deletions(-) diff --git a/api/serializers.py b/api/serializers.py index f616a5a4b..eea93d23e 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -1561,6 +1561,7 @@ class Meta: "id", "code", "event", + "start_date", ) diff --git a/per/ops_learning_summary.py b/per/ops_learning_summary.py index 58c4d2fd6..97ceabb99 100644 --- a/per/ops_learning_summary.py +++ b/per/ops_learning_summary.py @@ -57,25 +57,25 @@ class OpsLearningSummaryTask: MIN_DIF_EXCERPTS = 3 primary_prompt = ( - "Please aggregate and summarize the provided data into UP TO THREE structured paragraphs. " - "The output MUST strictly adhere to the format below: " - "- *Title*: Each finding should begin with the main finding TITLE in bold. " + "Please aggregate and summarize the provided data into UP TO THREE structured paragraphs.\n" + "The output MUST strictly adhere to the format below:\n" + "- *Title*: Each finding should begin with the main finding TITLE in bold.\n" "Should be a high level summary of the finding below. " - "The length of the title MUST be between 20 and 30 characters." - "- *Excerpts ID*: Identify the ids of the excerpts you took into account for creating the summary. " + "The length of the title MUST be between 20 and 30 characters.\n" + "- *Excerpts ID*: Identify the ids of the excerpts you took into account for creating the summary.\n" "- Content: Aggregate findings so that they are supported by evidence from more than one report. " "Always integrate evidence from multiple reports or items into the paragraph, and " - "include the year and country of the evidence." + "include the year and country of the evidence.\n" "- *Confidence Level*: Based on the number of excerpts connected to the finding, " "assign a score from 1 to 5 where 1 is the lowest and 5 is the highest, e.g. 4/5" - "At the end of the summary, please highlight any contradictory country reports. " - "Important:" - "-- DO NOT mention the excerpts id in the content of the summary." - "-- DO NOT mention the confidence level in the content of the summary." - "-- DO NOT use data from any source other than the one provided." - "Output Format:" - "Provide your answer in valid JSON form. Reply with only the answer in valid JSON form and include no other commentary. " - "Example: " + "At the end of the summary, please highlight any contradictory country reports.\n" + "Important:\n\n" + "-- DO NOT mention the excerpts id in the content of the summary.\n" + "-- DO NOT mention the confidence level in the content of the summary.\n" + "-- DO NOT use data from any source other than the one provided.\n\n" + "Output Format:\n" + "Provide your answer in valid JSON form. Reply with only the answer in valid JSON form and include no other commentary.\n" + "Example:\n" '{"0": {"title": "Flexible and Adaptive Response Planning", "excerpts id":"123, 45" ' '"content": "Responses in Honduras, Peru, Ecuador, and Panama highlight the importance of adaptable strategies. ' "The shift from youth-focused MHPSS to inclusive care in Peru in 2021, the pivot from sanitation infrastructure " @@ -88,22 +88,22 @@ class OpsLearningSummaryTask: ) secondary_prompt = ( - "Please aggregate and summarize this data into structured paragraphs (as few as possible, as many as necessary). " - "The output SHOULD ALWAYS follow the format below: " - "- *Type*: Whether the paragraph is related to a 'sector' or a 'component' " - "- *Subtype*: Provides the name of the sector or of the component to which the paragraph refers." - "- *Excerpts ID*: Identify the ids of the excerpts you took into account for creating the summary." + "Please aggregate and summarize this data into structured paragraphs (as few as possible, as many as necessary). \n " + "The output SHOULD ALWAYS follow the format below:\n" + "- *Type*: Whether the paragraph is related to a 'sector' or a 'component'\n" + "- *Subtype*: Provides the name of the sector or of the component to which the paragraph refers.\n" + "- *Excerpts ID*: Identify the ids of the excerpts you took into account for creating the summary.\n" "*Content*: A short summary aggregating findings related to the Subtype, " "so that they are supported by evidence coming from more than one report, " "and there is ONLY ONE entry per subtype. Always integrate in the paragraph evidence that supports " "it from the data available from multiples reports or items, include year and country of the evidence. " - "The length of each paragraph MUST be between 20 and 30 words." - " Important:" - "- ONLY create one summary per subtype" - "- DO NOT mention the ids of the excerpts in the content of the summary." - "- DO NOT use data from any source other than the one provided. " - "Output Format:" - "Provide your answer in valid JSON form. Reply with ONLY the answer in JSON form and include NO OTHER COMMENTARY." + "The length of each paragraph MUST be between 20 and 30 words.\n" + " Important:\n\n" + "- ONLY create one summary per subtype\n" + "- DO NOT mention the ids of the excerpts in the content of the summary.\n" + "- DO NOT use data from any source other than the one provided.\n\n" + "Output Format:\n" + "Provide your answer in valid JSON form. Reply with ONLY the answer in JSON form and include NO OTHER COMMENTARY.\n" '{"0": {"type": "sector", "subtype": "shelter", "excerpts id":"43, 1375, 14543", "content": "lorem ipsum"}, ' '"1": {"type": "component", "subtype": "Information Management", "excerpts id":"23, 235", "content": "lorem ipsum"}, ' '"2": {"type": "sector", "subtype": "WASH", "excerpts id":"30, 40", "content": "lorem ipsum"}}' @@ -120,21 +120,21 @@ class OpsLearningSummaryTask: ) primary_instruction_prompt = ( - "You should:" - "1. Describe, Summarize and Compare: Identify and detail the who, what, where and when" - "2. Explain and Connect: Analyze why events happened and how they are related" - "3. Identify gaps: Assess what data is available, what is missing and potential biases" - "4. Identify key messages: Determine important stories and signals hidden in the data" - "5. Select top three: Select up to three findings to report" + "You should:\n" + "1. Describe, Summarize and Compare: Identify and detail the who, what, where and when " + "2. Explain and Connect: Analyze why events happened and how they are related " + "3. Identify gaps: Assess what data is available, what is missing and potential biases " + "4. Identify key messages: Determine important stories and signals hidden in the data " + "5. Select top three: Select up to three findings to report " ) secondary_instruction_prompt = ( - "You should for each section in the data (TYPE & SUBTYPE combination):" - "1. Describe, Summarize and Compare: Identify and detail the who, what, where and when" - "2. Explain and Connect: Analyze why events happened and how they are related" - "3. Identify gaps: Assess what data is available, what is missing and potential biases" - "4. Identify key messages: Determine if there are important stories and signals hidden in the data" - "5. Conclude and make your case" + "You should for each section in the data (TYPE & SUBTYPE combination):\n" + "1. Describe, Summarize and Compare: Identify and detail the who, what, where and when " + "2. Explain and Connect: Analyze why events happened and how they are related " + "3. Identify gaps: Assess what data is available, what is missing and potential biases " + "4. Identify key messages: Determine if there are important stories and signals hidden in the data " + "5. Conclude and make your case " ) @staticmethod @@ -552,7 +552,7 @@ def _build_intro_section(cls): return ( "I will provide you with a set of instructions, data, and formatting requests in three sections." + " I will pass you the INSTRUCTIONS section, are you ready?" - + "\n\n\n\n" + + "\n\n" ) @classmethod @@ -585,9 +585,9 @@ def _build_instruction_section(cls, request_filter: dict, df: pd.DataFrame, inst component_str = '", "'.join(components) instructions.append(f'and "{component_str}" aspects') - instructions.append("in Emergency Response.") + instructions.append("in Emergency Response. ") instructions.append("\n\n" + instruction) - instructions.append("\n\nI will pass you the DATA section, are you ready?\n\n\n") + instructions.append("\n\nI will pass you the DATA section, are you ready?\n\n") return "\n".join(instructions) @classmethod @@ -814,13 +814,30 @@ def _modify_summary(summary: dict) -> dict: Checks if the "Confidence level" is present in the primary response and skipping for the secondary summary """ for key, value in summary.items(): - confidence_level = "confidence level" - if key == "contradictory reports" or confidence_level in value: + if key == "contradictory reports": continue - if confidence_level in value["content"].lower(): - parts = re.split(rf"(?i)\b{confidence_level}\b", value["content"]) + + content = value.get("content", "") + excerpt_ids = value.get("excerpts id", "") + excerpt_id_list = ( + list(set(excerpt_ids)) + if isinstance(excerpt_ids, list) + else list(set(int(id.strip()) for id in excerpt_ids.split(",") if excerpt_ids and excerpt_ids != "")) + ) + + # Check if any excerpt id is present in the content and regenerate the summary if found + if any(re.search(rf"\b{id}\b", content) for id in excerpt_id_list): + return cls.generate_summary(prompt, type) + + value["content"] = content + value["excerpts id"] = excerpt_id_list + + # Extract and remove if `confidence level` exists in the content + confidence_level = "confidence level" + if confidence_level not in value and confidence_level in content.lower(): + parts = re.split(rf"(?i)\b{confidence_level}\b", content, maxsplit=1) value["content"] = parts[0].strip() + "." - value["confidence level"] = parts[1][1:].strip() + value["confidence level"] = parts[1].strip() return summary @@ -906,11 +923,7 @@ def secondary_response_save_to_db( type = value["type"].strip() subtype = value["subtype"].strip() content = value["content"].strip() - excerpt_ids = value["excerpts id"] - if isinstance(excerpt_ids, list): - excerpt_id_list = list(set(excerpt_ids if excerpt_ids else [])) - else: - excerpt_id_list = list(set(int(id.strip()) for id in excerpt_ids.split(",") if excerpt_ids and excerpt_ids != "")) + excerpt_id_list = value["excerpts id"] if type == "component" and len(excerpt_id_list) > 0: cls.add_used_ops_learnings_component( From 4df0af8c36519f1188cc0b675c1097c5cfbbc578 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Fri, 22 Nov 2024 13:54:29 +0545 Subject: [PATCH 5/5] Add organization type api and enum for learning type --- per/admin.py | 3 ++- per/drf_views.py | 26 ++++++++++++++++++++++++++ per/enums.py | 1 + per/serializers.py | 10 ++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/per/admin.py b/per/admin.py index a0c63d8b2..c19612c73 100644 --- a/per/admin.py +++ b/per/admin.py @@ -188,7 +188,7 @@ class OpsLearningAdmin(GotoNextModelAdmin): ls = ("organization", "organization_validated", "sector", "sector_validated", "per_component", "per_component_validated") list_filter = ("is_validated", "appeal_code__atype") + ls autocomplete_fields = ("appeal_code",) + ls - search_fields = ("learning", "learning_validated") + search_fields = ("learning", "learning_validated", "appeal_code__aid", "appeal_code__code") list_display = ("learning", "appeal_code", "is_validated", "modified_at") change_form_template = "admin/opslearning_change_form.html" actions = ["export_selected_records"] @@ -311,6 +311,7 @@ class OpsLearningCacheResponseAdmin(TranslationAdmin): search_fields = ( "id", "used_ops_learning__appeal_code__aid", + "used_ops_learning__appeal_code__code", ) list_display = ( "__str__", diff --git a/per/drf_views.py b/per/drf_views.py index 71973269b..4a3d284e9 100644 --- a/per/drf_views.py +++ b/per/drf_views.py @@ -17,6 +17,7 @@ from rest_framework.authentication import TokenAuthentication from rest_framework.decorators import action from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response from rest_framework.settings import api_settings from api.models import Country @@ -79,6 +80,7 @@ NiceDocumentSerializer, OpsLearningCSVSerializer, OpsLearningInSerializer, + OpsLearningOrganizationTypeSerializer, OpsLearningSerializer, OpsLearningSummarySerializer, PerAssessmentSerializer, @@ -866,6 +868,30 @@ def get_renderer_context(self): return context + @extend_schema( + request=None, + filters=False, + responses=OpsLearningOrganizationTypeSerializer(many=True), + ) + @action( + detail=False, + methods=["GET"], + permission_classes=[DenyGuestUserMutationPermission, OpsLearningPermission], + serializer_class=OpsLearningOrganizationTypeSerializer, + url_path="organization-type", + ) + def organization(self, request): + """ + Get the Organization Types + """ + queryset = OrganizationTypes.objects.exclude(is_deprecated=True) + serializer = OpsLearningOrganizationTypeSerializer(queryset, many=True) + page = self.paginate_queryset(queryset) + if page is not None: + serializer = OpsLearningOrganizationTypeSerializer(page, many=True) + return self.get_paginated_response(serializer.data) + return Response(serializer.data) + @extend_schema( request=None, filters=True, diff --git a/per/enums.py b/per/enums.py index 92cece111..608d6930f 100644 --- a/per/enums.py +++ b/per/enums.py @@ -6,4 +6,5 @@ "overviewassessmentmethods": models.Overview.AssessmentMethod, "component_status": models.FormComponent.FormComponentStatus, "supported_by_organization_type": models.PerWorkPlanComponent.SupportedByOrganizationType, + "learning_type": models.LearningType, } diff --git a/per/serializers.py b/per/serializers.py index f5d27e907..b2e699e15 100644 --- a/per/serializers.py +++ b/per/serializers.py @@ -42,6 +42,7 @@ OpsLearningCacheResponse, OpsLearningComponentCacheResponse, OpsLearningSectorCacheResponse, + OrganizationTypes, Overview, PerAssessment, PerComponentRating, @@ -1243,3 +1244,12 @@ def get_latest_appeal_date(self, obj): return Appeal.objects.filter(id__in=obj.used_ops_learning.values("appeal_code__id")).aggregate( max_start_date=models.Max("start_date"), )["max_start_date"] + + +class OpsLearningOrganizationTypeSerializer(serializers.ModelSerializer): + class Meta: + model = OrganizationTypes + fields = [ + "id", + "title", + ]