From 4b32aec9341c567eec5b760a6e3b122442b9e804 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Thu, 12 Sep 2024 13:44:52 +0200 Subject: [PATCH 01/25] extend task tracker to handle messages --- corehq/apps/geospatial/utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/corehq/apps/geospatial/utils.py b/corehq/apps/geospatial/utils.py index e43d12921d8b..57e056fae4e4 100644 --- a/corehq/apps/geospatial/utils.py +++ b/corehq/apps/geospatial/utils.py @@ -221,8 +221,9 @@ class CeleryTaskTracker(object): Simple Helper class using redis to track if a celery task was requested and is not completed yet. """ - def __init__(self, task_key): + def __init__(self, task_key, message_key=None): self.task_key = task_key + self.message_key = message_key self._client = get_redis_client() def mark_requested(self, timeout=ONE_DAY): @@ -234,4 +235,14 @@ def is_active(self): return self._client.has_key(self.task_key) def mark_completed(self): + self.clear_message() return self._client.delete(self.task_key) + + def get_message(self): + return self._client.get(self.message_key) + + def set_message(self, message, timeout=ONE_DAY * 3): + return self._client.set(self.message_key, message, timeout=timeout) + + def clear_message(self): + return self._client.delete(self.message_key) From 6280c1785c813ebc957da63d8b6a9de0519a8b75 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Thu, 12 Sep 2024 13:55:28 +0200 Subject: [PATCH 02/25] refactor index command functions --- .../index_geolocation_case_properties.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/corehq/apps/geospatial/management/commands/index_geolocation_case_properties.py b/corehq/apps/geospatial/management/commands/index_geolocation_case_properties.py index 2419a88483ae..58e767bfeeea 100644 --- a/corehq/apps/geospatial/management/commands/index_geolocation_case_properties.py +++ b/corehq/apps/geospatial/management/commands/index_geolocation_case_properties.py @@ -41,15 +41,23 @@ def index_case_docs(domain, query_limit=DEFAULT_QUERY_LIMIT, chunk_size=DEFAULT_ query = _es_case_query(domain, geo_case_property, case_type) count = query.count() print(f'{count} case(s) to process') - batch_count = 1 - if query_limit: - batch_count = math.ceil(count / query_limit) + batch_count = get_batch_count(count, query_limit) print(f"Cases will be processed in {batch_count} batches") for i in range(batch_count): print(f'Processing {i+1}/{batch_count}') - query = _es_case_query(domain, geo_case_property, case_type, size=query_limit) - case_ids = query.get_ids() - _index_case_ids(domain, case_ids, chunk_size) + process_batch(domain, geo_case_property, case_type, query_limit, chunk_size) + + +def get_batch_count(doc_count, query_limit): + if not query_limit: + return 1 + return math.ceil(doc_count / query_limit) + + +def process_batch(domain, geo_case_property, case_type, query_limit, chunk_size): + query = _es_case_query(domain, geo_case_property, case_type, size=query_limit) + case_ids = query.get_ids() + _index_case_ids(domain, case_ids, chunk_size) def _index_case_ids(domain, case_ids, chunk_size): From 7f52f0e3b54eb35d91065fe81a3fc887baa5cac3 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Thu, 12 Sep 2024 13:57:03 +0200 Subject: [PATCH 03/25] helper func to get celery task tracker --- corehq/apps/geospatial/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/corehq/apps/geospatial/utils.py b/corehq/apps/geospatial/utils.py index 57e056fae4e4..7a592f58124a 100644 --- a/corehq/apps/geospatial/utils.py +++ b/corehq/apps/geospatial/utils.py @@ -33,6 +33,12 @@ def get_geo_user_property(domain): return config.user_location_property_name +def get_celery_task_tracker(domain, base_key): + task_key = f'{base_key}_{domain}' + message_key = f'{base_key}_message_{domain}' + return CeleryTaskTracker(task_key, message_key) + + def _format_coordinates(lat, lon): return f"{lat} {lon} 0.0 0.0" From 2184e768eaf59f69835d6d184fcf94239d5396ad Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Thu, 12 Sep 2024 13:58:30 +0200 Subject: [PATCH 04/25] start celery task to index docs on enabling feature flag --- corehq/apps/geospatial/const.py | 2 ++ corehq/apps/geospatial/tasks.py | 56 ++++++++++++++++++++++++++++++++- corehq/toggles/__init__.py | 11 ++++++- 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/corehq/apps/geospatial/const.py b/corehq/apps/geospatial/const.py index 14142a7815ae..d744adef18a3 100644 --- a/corehq/apps/geospatial/const.py +++ b/corehq/apps/geospatial/const.py @@ -133,3 +133,5 @@ } } } + +INDEX_ES_TASK_HELPER_BASE_KEY = 'geo_cases_index_cases' diff --git a/corehq/apps/geospatial/tasks.py b/corehq/apps/geospatial/tasks.py index 62724e88a382..541cc620e80e 100644 --- a/corehq/apps/geospatial/tasks.py +++ b/corehq/apps/geospatial/tasks.py @@ -1,5 +1,24 @@ +from django.utils.translation import gettext as _ + +from corehq.util.decorators import serial_task + from corehq.apps.celery import task -from corehq.apps.geospatial.utils import CeleryTaskTracker, update_cases_owner +from corehq.apps.geospatial.const import INDEX_ES_TASK_HELPER_BASE_KEY +from corehq.apps.geospatial.utils import ( + get_celery_task_tracker, + CeleryTaskTracker, + update_cases_owner, + get_geo_case_property, +) +from corehq.apps.geospatial.management.commands.index_geolocation_case_properties import ( + _es_case_query, + get_batch_count, + process_batch, + DEFAULT_QUERY_LIMIT, + DEFAULT_CHUNK_SIZE, +) + +from settings import MAX_GEOSPATIAL_INDEX_DOC_LIMIT @task(queue="background_queue", ignore_result=True) @@ -9,3 +28,38 @@ def geo_cases_reassignment_update_owners(domain, case_owner_updates_dict, task_k finally: celery_task_tracker = CeleryTaskTracker(task_key) celery_task_tracker.mark_completed() + + +@serial_task('async-index-es-docs', timeout=30 * 60, queue='background_queue', ignore_result=True) +def index_es_docs_with_location_props(domain): + celery_task_tracker = get_celery_task_tracker(domain, INDEX_ES_TASK_HELPER_BASE_KEY) + if celery_task_tracker.is_active(): + return + + geo_case_prop = get_geo_case_property(domain) + query = _es_case_query(domain, geo_case_prop) + doc_count = query.count() + if doc_count > MAX_GEOSPATIAL_INDEX_DOC_LIMIT: + celery_task_tracker.set_message( + _('This domain contains too many cases and so they will not be made available ' + 'for use by this feature. Please reach out to support.') + ) + return + + celery_task_tracker.mark_requested() + batch_count = get_batch_count(doc_count, DEFAULT_QUERY_LIMIT) + try: + for i in range(batch_count): + progress = (i / batch_count) * 100 + celery_task_tracker.set_message( + _(f'Cases are being made ready for use by this feature. Please be patient. ({progress}%)') + ) + process_batch( + domain, + geo_case_prop, + case_type=None, + query_limit=DEFAULT_QUERY_LIMIT, + chunk_size=DEFAULT_CHUNK_SIZE, + ) + finally: + celery_task_tracker.mark_completed() diff --git a/corehq/toggles/__init__.py b/corehq/toggles/__init__.py index 54d6e2a18e89..3122c31a06c8 100644 --- a/corehq/toggles/__init__.py +++ b/corehq/toggles/__init__.py @@ -2555,13 +2555,22 @@ def _handle_attendance_tracking_role(domain, is_enabled): save_fn=_handle_attendance_tracking_role, ) + +def _handle_geospatial_es_index(domain, is_enabled): + from corehq.apps.geospatial.es import index_es_docs_with_location_props + + if is_enabled: + index_es_docs_with_location_props.delay(domain) + + GEOSPATIAL = StaticToggle( 'geospatial', 'Allows access to GIS functionality', TAG_SOLUTIONS_LIMITED, namespaces=[NAMESPACE_DOMAIN], description='Additional views will be added allowing for visually viewing ' - 'and assigning cases on a map.' + 'and assigning cases on a map.', + save_fn=_handle_geospatial_es_index, ) From d7abcee1d3181e759994f8462aff641e8d55a56c Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Thu, 12 Sep 2024 13:59:01 +0200 Subject: [PATCH 05/25] send task message to front-end --- corehq/apps/geospatial/reports.py | 4 ++++ corehq/apps/geospatial/views.py | 21 ++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/corehq/apps/geospatial/reports.py b/corehq/apps/geospatial/reports.py index 362eeb7a1548..9bc7b3ef7ad8 100644 --- a/corehq/apps/geospatial/reports.py +++ b/corehq/apps/geospatial/reports.py @@ -25,6 +25,7 @@ from corehq.util.quickcache import quickcache from .dispatchers import CaseManagementMapDispatcher +from corehq.apps.geospatial.const import INDEX_ES_TASK_HELPER_BASE_KEY from .es import ( BUCKET_CASES_AGG, CASE_PROPERTIES_AGG, @@ -38,6 +39,7 @@ geojson_to_es_geoshape, get_geo_case_property, validate_geometry, + get_celery_task_tracker, ) @@ -59,12 +61,14 @@ class BaseCaseMapReport(ProjectReport, CaseListMixin, XpathCaseSearchFilterMixin def template_context(self): # Whatever is specified here can be accessed through initial_page_data context = super(BaseCaseMapReport, self).template_context + celery_task_tracker = get_celery_task_tracker(self.domain, base_key=INDEX_ES_TASK_HELPER_BASE_KEY) context.update({ 'mapbox_access_token': settings.MAPBOX_ACCESS_TOKEN, 'saved_polygons': [ {'id': p.id, 'name': p.name, 'geo_json': p.geo_json} for p in GeoPolygon.objects.filter(domain=self.domain).all() ], + 'es_indexing_message': celery_task_tracker.get_message() }) return context diff --git a/corehq/apps/geospatial/views.py b/corehq/apps/geospatial/views.py index b11789345787..076f8c689c8d 100644 --- a/corehq/apps/geospatial/views.py +++ b/corehq/apps/geospatial/views.py @@ -47,7 +47,11 @@ from corehq.form_processor.models import CommCareCase from corehq.util.timezones.utils import get_timezone -from .const import GPS_POINT_CASE_PROPERTY, POLYGON_COLLECTION_GEOJSON_SCHEMA +from .const import ( + GPS_POINT_CASE_PROPERTY, + POLYGON_COLLECTION_GEOJSON_SCHEMA, + INDEX_ES_TASK_HELPER_BASE_KEY, +) from .models import GeoConfig, GeoPolygon from .utils import ( CaseOwnerUpdate, @@ -59,6 +63,7 @@ set_case_gps_property, set_user_gps_property, update_cases_owner, + get_celery_task_tracker, ) @@ -66,6 +71,16 @@ def geospatial_default(request, *args, **kwargs): return HttpResponseRedirect(CaseManagementMap.get_url(*args, **kwargs)) +class BaseGeospatialView(BaseDomainView): + + @property + def main_context(self): + context = super().main_context + celery_task_tracker = get_celery_task_tracker(self.domain, base_key=INDEX_ES_TASK_HELPER_BASE_KEY) + context['es_indexing_message'] = celery_task_tracker.get_message() + return context + + class CaseDisbursementAlgorithm(BaseDomainView): urlname = "case_disbursement" @@ -151,7 +166,7 @@ def delete(self, request, *args, **kwargs): }) -class BaseConfigView(BaseDomainView): +class BaseConfigView(BaseGeospatialView): section_name = _("Data") @method_decorator(toggles.GEOSPATIAL.required_decorator()) @@ -229,7 +244,7 @@ def page_context(self): return context -class GPSCaptureView(BaseDomainView): +class GPSCaptureView(BaseGeospatialView): urlname = 'gps_capture' template_name = 'gps_capture_view.html' From 5409b020af384aa54ac7d498d69c870db9d26d16 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Thu, 12 Sep 2024 13:59:17 +0200 Subject: [PATCH 06/25] display task message --- corehq/apps/geospatial/templates/case_grouping_map.html | 1 + corehq/apps/geospatial/templates/case_management.html | 1 + .../templates/geospatial/case_management_base.html | 1 + .../templates/geospatial/partials/index_alert.html | 9 +++++++++ .../apps/geospatial/templates/geospatial/settings.html | 1 + corehq/apps/geospatial/templates/gps_capture_view.html | 2 ++ 6 files changed, 15 insertions(+) create mode 100644 corehq/apps/geospatial/templates/geospatial/partials/index_alert.html diff --git a/corehq/apps/geospatial/templates/case_grouping_map.html b/corehq/apps/geospatial/templates/case_grouping_map.html index 16739daeb771..18152231f7c8 100644 --- a/corehq/apps/geospatial/templates/case_grouping_map.html +++ b/corehq/apps/geospatial/templates/case_grouping_map.html @@ -4,6 +4,7 @@ {% load hq_shared_tags %} {% block reportcontent %} +{% include 'geospatial/partials/index_alert.html' %}
diff --git a/corehq/apps/geospatial/templates/case_management.html b/corehq/apps/geospatial/templates/case_management.html index dcb46694a9cd..7d9bf4ef5694 100644 --- a/corehq/apps/geospatial/templates/case_management.html +++ b/corehq/apps/geospatial/templates/case_management.html @@ -2,6 +2,7 @@ {% load i18n %} {% block reportcontent %} +{% include 'geospatial/partials/index_alert.html' %}
diff --git a/corehq/apps/geospatial/templates/geospatial/case_management_base.html b/corehq/apps/geospatial/templates/geospatial/case_management_base.html index 78e070943920..4c1037cac6ea 100644 --- a/corehq/apps/geospatial/templates/geospatial/case_management_base.html +++ b/corehq/apps/geospatial/templates/geospatial/case_management_base.html @@ -23,3 +23,4 @@ {% registerurl 'location_search' domain %} {% registerurl 'reassign_cases' domain %} {% endblock %} +{% include 'geospatial/partials/index_alert.html' %} diff --git a/corehq/apps/geospatial/templates/geospatial/partials/index_alert.html b/corehq/apps/geospatial/templates/geospatial/partials/index_alert.html new file mode 100644 index 000000000000..2d478281cd68 --- /dev/null +++ b/corehq/apps/geospatial/templates/geospatial/partials/index_alert.html @@ -0,0 +1,9 @@ +{% load i18n %} + +{% if es_indexing_message %} +
+

+ {{ es_indexing_message }} +

+
+{% endif %} \ No newline at end of file diff --git a/corehq/apps/geospatial/templates/geospatial/settings.html b/corehq/apps/geospatial/templates/geospatial/settings.html index 6d7b6a935eca..cc587f25e22a 100644 --- a/corehq/apps/geospatial/templates/geospatial/settings.html +++ b/corehq/apps/geospatial/templates/geospatial/settings.html @@ -13,6 +13,7 @@ {% initial_page_data 'road_network_algorithm_slug' road_network_algorithm_slug %}
+ {% include 'geospatial/partials/index_alert.html' %} {% crispy form %}
{% endblock %} diff --git a/corehq/apps/geospatial/templates/gps_capture_view.html b/corehq/apps/geospatial/templates/gps_capture_view.html index 819dd74bad7d..d27ca6628194 100644 --- a/corehq/apps/geospatial/templates/gps_capture_view.html +++ b/corehq/apps/geospatial/templates/gps_capture_view.html @@ -24,7 +24,9 @@ {% registerurl 'paginate_mobile_workers' domain %} {% initial_page_data 'case_types_with_gps' case_types_with_gps %} {% initial_page_data 'couch_user_username' couch_user_username %} +{% registerurl 'geo_polygons' domain %} +{% include 'geospatial/partials/index_alert.html' %}