From 518465296cafd19807062a73892a4ac3f29907ce Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Tue, 1 Oct 2024 11:26:16 +0200 Subject: [PATCH 01/13] implement script --- corehq/mobile_ucr_v2_update_script.py | 109 ++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 corehq/mobile_ucr_v2_update_script.py diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py new file mode 100644 index 000000000000..063af74a5d39 --- /dev/null +++ b/corehq/mobile_ucr_v2_update_script.py @@ -0,0 +1,109 @@ +# This script updates the latest versions of all apps across domains that are using v1 with no manual references +# Steps followed +# 1. Get All domains with mobile ucr flag enabled +# 2. Get all apps for domain with latest released versions and mobile ucr versions that are not v2 +# 3. For each app, if it contains no V1 UCR references, update the version to 2 + +# How to run +# Can be run in django shell. Paste the script and execute the function process() +# File is stored in home directory of cchq user. + +# V1 Examples +# https://staging.commcarehq.org/a/test-commcare-superset/apps/view/f940fcc83bae44b8a0adaf719673fd1e/form/a0f3c5b483c645e78b6f89ee0b3b3c03/source/#form/table_child_count + + +import json +import re +import traceback + +from corehq.apps.app_manager.dbaccessors import ( + get_latest_app_ids_and_versions, + get_apps_by_id, +) +from corehq.apps.app_manager.const import MOBILE_UCR_VERSION_2 +from corehq.toggles import MOBILE_UCR +from corehq.toggles.shortcuts import find_domains_with_toggle_enabled +from corehq.util.log import with_progress_bar + + +def save_in_log(file, data): + print(data) + file.write(data + '\n') + + +def save_as_ndjson(path, data): + with open(path, 'a') as file: + print(json.dumps(data, separators=(',', ':')), file=file) + + +def read_ndjson_file(path): + with open(path, 'r') as file: + return [json.loads(line) for line in file.readlines()] + + +def has_non_v2_form(domain, app, log_file): + for form in app.get_forms(): + save_in_log(log_file, f"Processing Form: {domain}: {form.name}") + # The second condition should always be False if the first one is + # but just as a precaution we check for it + if V1_FIXTURE_IDENTIFIER in form.source or re.findall(V1_ALL_REFERENCES, form.source): + save_in_log(log_file, f"App Contains V1 Refs: {domain}: {app.name}") + return True + return False + + +def update_app(domain, app, log_file): + save_in_log(log_file, f"Updating App: {domain}: {app.name}: {app.id}") + app.mobile_ucr_restore_version = MOBILE_UCR_VERSION_2 + app.save() + + +PROCESSED_DOMAINS_PATH = '/home/zandre/cchq/updated_domains.ndjson' +LOG_FILE = '/home/zandre/cchq/update_to_v2_ucr_script.log' + +V1_FIXTURE_IDENTIFIER = 'src="jr://fixture/commcare:reports' +V1_FIXTURE_PATTERN = r'<.*src="jr://fixture/commcare:reports.*>' +V1_REFERENCES_PATTERN = r"<.*instance\('reports'\)/reports/.*>" +V1_ALL_REFERENCES = f"{V1_FIXTURE_PATTERN}|{V1_REFERENCES_PATTERN}" + + +skip_domains = [] + + +def process(): + try: + processed_domains = read_ndjson_file(PROCESSED_DOMAINS_PATH) + except FileNotFoundError: + processed_domains = [] + + mobile_ucr_domains = find_domains_with_toggle_enabled(MOBILE_UCR) + + log_file = open(LOG_FILE, 'a') + + save_in_log(log_file, f"Number of domains with mobile ucr flag enabled: {len(mobile_ucr_domains)} ") + + for domain in with_progress_bar(mobile_ucr_domains): + if domain in processed_domains: + save_in_log(log_file, f"Already processed domain: {domain}") + continue + if domain in skip_domains: + save_in_log(log_file, f"Skipped domain: {domain}") + continue + + save_in_log(log_file, f"Processing domain: {domain} ...") + app_ids = list(get_latest_app_ids_and_versions(domain)) + apps = get_apps_by_id(domain, app_ids) + for app in apps: + try: + # Don't look at app.is_released since the latest version might not be released yet + if app.mobile_ucr_restore_version != '2.0': + save_in_log(log_file, f"Processing App: {domain}: {app.name}: {app.id}") + if not has_non_v2_form(domain, app, log_file): + update_app(domain, app, log_file) + except Exception as e: + save_in_log(log_file, f"Error occurred for {domain}: {str(e)}") + save_in_log(log_file, traceback.format_exc()) + continue + save_as_ndjson(PROCESSED_DOMAINS_PATH, domain) + + log_file.close() From 30a6a13bfcaeb27f831adc4df3227b690895d9dc Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Tue, 1 Oct 2024 11:34:09 +0200 Subject: [PATCH 02/13] logging for apps containing v1 refs --- corehq/mobile_ucr_v2_update_script.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index 063af74a5d39..d949b8a7cb93 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -100,6 +100,11 @@ def process(): save_in_log(log_file, f"Processing App: {domain}: {app.name}: {app.id}") if not has_non_v2_form(domain, app, log_file): update_app(domain, app, log_file) + else: + save_in_log( + log_file, + f"App contains V1 references and couldn't updated: {domain}: {app.name}: {app.id}", + ) except Exception as e: save_in_log(log_file, f"Error occurred for {domain}: {str(e)}") save_in_log(log_file, traceback.format_exc()) From 0392d0f11fb111b9ed45e36d86082903b37fdd52 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Tue, 1 Oct 2024 12:15:48 +0200 Subject: [PATCH 03/13] compile re expression --- corehq/mobile_ucr_v2_update_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index d949b8a7cb93..405f6feb3623 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -46,7 +46,7 @@ def has_non_v2_form(domain, app, log_file): save_in_log(log_file, f"Processing Form: {domain}: {form.name}") # The second condition should always be False if the first one is # but just as a precaution we check for it - if V1_FIXTURE_IDENTIFIER in form.source or re.findall(V1_ALL_REFERENCES, form.source): + if V1_FIXTURE_IDENTIFIER in form.source or RE_V1_ALL_REFERENCES.search(form.source): save_in_log(log_file, f"App Contains V1 Refs: {domain}: {app.name}") return True return False @@ -64,7 +64,7 @@ def update_app(domain, app, log_file): V1_FIXTURE_IDENTIFIER = 'src="jr://fixture/commcare:reports' V1_FIXTURE_PATTERN = r'<.*src="jr://fixture/commcare:reports.*>' V1_REFERENCES_PATTERN = r"<.*instance\('reports'\)/reports/.*>" -V1_ALL_REFERENCES = f"{V1_FIXTURE_PATTERN}|{V1_REFERENCES_PATTERN}" +RE_V1_ALL_REFERENCES = re.compile(f"{V1_FIXTURE_PATTERN}|{V1_REFERENCES_PATTERN}") skip_domains = [] From b3e42528e1db8f52a77b9298b8592f6493bd9426 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Tue, 1 Oct 2024 12:16:22 +0200 Subject: [PATCH 04/13] use sets instead of lists --- corehq/mobile_ucr_v2_update_script.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index 405f6feb3623..b2d1b33bbb07 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -38,7 +38,7 @@ def save_as_ndjson(path, data): def read_ndjson_file(path): with open(path, 'r') as file: - return [json.loads(line) for line in file.readlines()] + return set(json.loads(line) for line in file.readlines()) def has_non_v2_form(domain, app, log_file): @@ -67,14 +67,14 @@ def update_app(domain, app, log_file): RE_V1_ALL_REFERENCES = re.compile(f"{V1_FIXTURE_PATTERN}|{V1_REFERENCES_PATTERN}") -skip_domains = [] +skip_domains = set() def process(): try: processed_domains = read_ndjson_file(PROCESSED_DOMAINS_PATH) except FileNotFoundError: - processed_domains = [] + processed_domains = set() mobile_ucr_domains = find_domains_with_toggle_enabled(MOBILE_UCR) From 27c1c2501790b14dfbbe049e3dff4d4c68a88656 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Thu, 3 Oct 2024 15:14:42 +0200 Subject: [PATCH 05/13] refactor logging --- corehq/mobile_ucr_v2_update_script.py | 56 +++++++++++++-------------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index b2d1b33bbb07..7e6c08aa92c7 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -26,9 +26,19 @@ from corehq.util.log import with_progress_bar -def save_in_log(file, data): +PROCESSED_DOMAINS_PATH = '/home/zandre/cchq/updated_domains.ndjson' +LOG_FILE = '/home/zandre/cchq/update_to_v2_ucr_script.log' + +V1_FIXTURE_IDENTIFIER = 'src="jr://fixture/commcare:reports' +V1_FIXTURE_PATTERN = r'<.*src="jr://fixture/commcare:reports.*>' +V1_REFERENCES_PATTERN = r"<.*instance\('reports'\)/reports/.*>" +RE_V1_ALL_REFERENCES = re.compile(f"{V1_FIXTURE_PATTERN}|{V1_REFERENCES_PATTERN}") + + +def save_in_log(data): print(data) - file.write(data + '\n') + with open(LOG_FILE, 'a') as file: + file.write(data + '\n') def save_as_ndjson(path, data): @@ -41,32 +51,23 @@ def read_ndjson_file(path): return set(json.loads(line) for line in file.readlines()) -def has_non_v2_form(domain, app, log_file): +def has_non_v2_form(domain, app): for form in app.get_forms(): - save_in_log(log_file, f"Processing Form: {domain}: {form.name}") + save_in_log(f"Processing Form: {domain}: {form.name}") # The second condition should always be False if the first one is # but just as a precaution we check for it if V1_FIXTURE_IDENTIFIER in form.source or RE_V1_ALL_REFERENCES.search(form.source): - save_in_log(log_file, f"App Contains V1 Refs: {domain}: {app.name}") + save_in_log(f"App Contains V1 Refs: {domain}: {app.name}") return True return False -def update_app(domain, app, log_file): - save_in_log(log_file, f"Updating App: {domain}: {app.name}: {app.id}") +def update_app(domain, app): + save_in_log(f"Updating App: {domain}: {app.name}: {app.id}") app.mobile_ucr_restore_version = MOBILE_UCR_VERSION_2 app.save() -PROCESSED_DOMAINS_PATH = '/home/zandre/cchq/updated_domains.ndjson' -LOG_FILE = '/home/zandre/cchq/update_to_v2_ucr_script.log' - -V1_FIXTURE_IDENTIFIER = 'src="jr://fixture/commcare:reports' -V1_FIXTURE_PATTERN = r'<.*src="jr://fixture/commcare:reports.*>' -V1_REFERENCES_PATTERN = r"<.*instance\('reports'\)/reports/.*>" -RE_V1_ALL_REFERENCES = re.compile(f"{V1_FIXTURE_PATTERN}|{V1_REFERENCES_PATTERN}") - - skip_domains = set() @@ -78,37 +79,32 @@ def process(): mobile_ucr_domains = find_domains_with_toggle_enabled(MOBILE_UCR) - log_file = open(LOG_FILE, 'a') - - save_in_log(log_file, f"Number of domains with mobile ucr flag enabled: {len(mobile_ucr_domains)} ") + save_in_log(f"Number of domains with mobile ucr flag enabled: {len(mobile_ucr_domains)} ") for domain in with_progress_bar(mobile_ucr_domains): if domain in processed_domains: - save_in_log(log_file, f"Already processed domain: {domain}") + save_in_log(f"Already processed domain: {domain}") continue if domain in skip_domains: - save_in_log(log_file, f"Skipped domain: {domain}") + save_in_log(f"Skipped domain: {domain}") continue - save_in_log(log_file, f"Processing domain: {domain} ...") + save_in_log(f"Processing domain: {domain} ...") app_ids = list(get_latest_app_ids_and_versions(domain)) apps = get_apps_by_id(domain, app_ids) for app in apps: try: # Don't look at app.is_released since the latest version might not be released yet if app.mobile_ucr_restore_version != '2.0': - save_in_log(log_file, f"Processing App: {domain}: {app.name}: {app.id}") - if not has_non_v2_form(domain, app, log_file): - update_app(domain, app, log_file) + save_in_log(f"Processing App: {domain}: {app.name}: {app.id}") + if not has_non_v2_form(domain, app): + update_app(domain, app) else: save_in_log( - log_file, f"App contains V1 references and couldn't updated: {domain}: {app.name}: {app.id}", ) except Exception as e: - save_in_log(log_file, f"Error occurred for {domain}: {str(e)}") - save_in_log(log_file, traceback.format_exc()) + save_in_log(f"Error occurred for {domain}: {str(e)}") + save_in_log(traceback.format_exc()) continue save_as_ndjson(PROCESSED_DOMAINS_PATH, domain) - - log_file.close() From eac2bcffdbf4c6e1c5088f96ef6a8452933ff5b8 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Thu, 3 Oct 2024 15:17:37 +0200 Subject: [PATCH 06/13] reorganize script flow --- corehq/mobile_ucr_v2_update_script.py | 67 +++++++++++++-------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index 7e6c08aa92c7..5ca9dfb21e46 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -34,40 +34,6 @@ V1_REFERENCES_PATTERN = r"<.*instance\('reports'\)/reports/.*>" RE_V1_ALL_REFERENCES = re.compile(f"{V1_FIXTURE_PATTERN}|{V1_REFERENCES_PATTERN}") - -def save_in_log(data): - print(data) - with open(LOG_FILE, 'a') as file: - file.write(data + '\n') - - -def save_as_ndjson(path, data): - with open(path, 'a') as file: - print(json.dumps(data, separators=(',', ':')), file=file) - - -def read_ndjson_file(path): - with open(path, 'r') as file: - return set(json.loads(line) for line in file.readlines()) - - -def has_non_v2_form(domain, app): - for form in app.get_forms(): - save_in_log(f"Processing Form: {domain}: {form.name}") - # The second condition should always be False if the first one is - # but just as a precaution we check for it - if V1_FIXTURE_IDENTIFIER in form.source or RE_V1_ALL_REFERENCES.search(form.source): - save_in_log(f"App Contains V1 Refs: {domain}: {app.name}") - return True - return False - - -def update_app(domain, app): - save_in_log(f"Updating App: {domain}: {app.name}: {app.id}") - app.mobile_ucr_restore_version = MOBILE_UCR_VERSION_2 - app.save() - - skip_domains = set() @@ -108,3 +74,36 @@ def process(): save_in_log(traceback.format_exc()) continue save_as_ndjson(PROCESSED_DOMAINS_PATH, domain) + + +def save_in_log(data): + print(data) + with open(LOG_FILE, 'a') as file: + file.write(data + '\n') + + +def save_as_ndjson(path, data): + with open(path, 'a') as file: + print(json.dumps(data, separators=(',', ':')), file=file) + + +def read_ndjson_file(path): + with open(path, 'r') as file: + return set(json.loads(line) for line in file.readlines()) + + +def has_non_v2_form(domain, app): + for form in app.get_forms(): + save_in_log(f"Processing Form: {domain}: {form.name}") + # The second condition should always be False if the first one is + # but just as a precaution we check for it + if V1_FIXTURE_IDENTIFIER in form.source or RE_V1_ALL_REFERENCES.search(form.source): + save_in_log(f"App Contains V1 Refs: {domain}: {app.name}") + return True + return False + + +def update_app(domain, app): + save_in_log(f"Updating App: {domain}: {app.name}: {app.id}") + app.mobile_ucr_restore_version = MOBILE_UCR_VERSION_2 + app.save() From 20d94010be2eaec504de8e804ffd266d1abd35bb Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Thu, 3 Oct 2024 15:28:39 +0200 Subject: [PATCH 07/13] add option for dry run --- corehq/mobile_ucr_v2_update_script.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index 5ca9dfb21e46..09521fb28b65 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -37,7 +37,7 @@ skip_domains = set() -def process(): +def process(dry_run=False): try: processed_domains = read_ndjson_file(PROCESSED_DOMAINS_PATH) except FileNotFoundError: @@ -64,7 +64,7 @@ def process(): if app.mobile_ucr_restore_version != '2.0': save_in_log(f"Processing App: {domain}: {app.name}: {app.id}") if not has_non_v2_form(domain, app): - update_app(domain, app) + update_app(domain, app, dry_run) else: save_in_log( f"App contains V1 references and couldn't updated: {domain}: {app.name}: {app.id}", @@ -103,7 +103,8 @@ def has_non_v2_form(domain, app): return False -def update_app(domain, app): +def update_app(domain, app, dry_run): save_in_log(f"Updating App: {domain}: {app.name}: {app.id}") - app.mobile_ucr_restore_version = MOBILE_UCR_VERSION_2 - app.save() + if not dry_run: + app.mobile_ucr_restore_version = MOBILE_UCR_VERSION_2 + app.save() From 25a0298903c43d9c7dc0f9e8fe6d95ab7e4e678c Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Wed, 16 Oct 2024 10:03:55 +0200 Subject: [PATCH 08/13] default to doing dry run --- corehq/mobile_ucr_v2_update_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index 09521fb28b65..c61649c73c60 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -37,7 +37,7 @@ skip_domains = set() -def process(dry_run=False): +def process(dry_run=True): try: processed_domains = read_ndjson_file(PROCESSED_DOMAINS_PATH) except FileNotFoundError: From 241fd96ec4c76cbf30babb80687c6c452e6e3034 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Wed, 16 Oct 2024 10:05:59 +0200 Subject: [PATCH 09/13] optimizations for low memory env --- corehq/mobile_ucr_v2_update_script.py | 44 ++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index c61649c73c60..69dd5cada7ce 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -14,13 +14,12 @@ import json import re +import resource import traceback -from corehq.apps.app_manager.dbaccessors import ( - get_latest_app_ids_and_versions, - get_apps_by_id, -) +from corehq.apps.app_manager.dbaccessors import get_apps_by_id from corehq.apps.app_manager.const import MOBILE_UCR_VERSION_2 +from corehq.apps.app_manager.models import Application from corehq.toggles import MOBILE_UCR from corehq.toggles.shortcuts import find_domains_with_toggle_enabled from corehq.util.log import with_progress_bar @@ -37,7 +36,10 @@ skip_domains = set() -def process(dry_run=True): +def process(dry_run=True, max_memory_size=None): + if max_memory_size: + set_max_memory(max_memory_size) + try: processed_domains = read_ndjson_file(PROCESSED_DOMAINS_PATH) except FileNotFoundError: @@ -108,3 +110,35 @@ def update_app(domain, app, dry_run): if not dry_run: app.mobile_ucr_restore_version = MOBILE_UCR_VERSION_2 app.save() + + +def set_max_memory(size): + """ + Can be used to set max memory for the process (used for low memory machines) + Example: 800 MB set_max_memory(1024 * 1000 * 800) + + - size: in bytes + """ + soft, hard = resource.getrlimit(resource.RLIMIT_AS) + resource.setrlimit(resource.RLIMIT_AS, (size, hard)) + + +def get_latest_app_ids_and_versions(domain): + key = [domain] + results = Application.get_db().view( + 'app_manager/applications_brief', + startkey=key + [{}], + endkey=key, + descending=True, + reduce=False, + include_docs=False, + ) + latest_ids_and_versions = {} + for result in results: + app_id = result['value']['_id'] + version = result['value']['version'] + + # Since we have sorted, we know the first instance is the latest version + if app_id not in latest_ids_and_versions: + latest_ids_and_versions[app_id] = version + return latest_ids_and_versions From 6dcc7195eb8f0401f068c92531e9c24a3f9ba031 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Wed, 16 Oct 2024 10:07:54 +0200 Subject: [PATCH 10/13] check for v1 refs in case list/detail section --- corehq/mobile_ucr_v2_update_script.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index 69dd5cada7ce..a0904de065a1 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -33,6 +33,12 @@ V1_REFERENCES_PATTERN = r"<.*instance\('reports'\)/reports/.*>" RE_V1_ALL_REFERENCES = re.compile(f"{V1_FIXTURE_PATTERN}|{V1_REFERENCES_PATTERN}") +# Pattern for references in case list/detail. +# The below instance needs to be referred in a fixture setup and +# is then later used in Case List/Detail filters and columns. +V1_CASE_LIST_REFERENCES_PATTERN = r"instance\('commcare:reports'\)/reports/report\[@id='.*'\]/rows/row" +RE_V1_CASE_LIST_REFERENCES_PATTERN = re.compile(V1_CASE_LIST_REFERENCES_PATTERN) + skip_domains = set() @@ -65,7 +71,7 @@ def process(dry_run=True, max_memory_size=None): # Don't look at app.is_released since the latest version might not be released yet if app.mobile_ucr_restore_version != '2.0': save_in_log(f"Processing App: {domain}: {app.name}: {app.id}") - if not has_non_v2_form(domain, app): + if not has_non_v2_reference(domain, app): update_app(domain, app, dry_run) else: save_in_log( @@ -94,7 +100,12 @@ def read_ndjson_file(path): return set(json.loads(line) for line in file.readlines()) -def has_non_v2_form(domain, app): +def has_non_v2_reference(domain, app): + suite = app.create_suite() + if re.search(RE_V1_CASE_LIST_REFERENCES_PATTERN, suite.decode()): + save_in_log(f"App Case List Contains V1 Refs: {domain}: {app.name}") + return True + for form in app.get_forms(): save_in_log(f"Processing Form: {domain}: {form.name}") # The second condition should always be False if the first one is From f86eeef663e85dfa01cd5574a555940403a6438f Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Wed, 16 Oct 2024 10:08:08 +0200 Subject: [PATCH 11/13] skip app if no reporting modules --- corehq/mobile_ucr_v2_update_script.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index a0904de065a1..ac0a3db4089c 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -68,6 +68,9 @@ def process(dry_run=True, max_memory_size=None): apps = get_apps_by_id(domain, app_ids) for app in apps: try: + if not list(app.get_report_modules()): + continue + # Don't look at app.is_released since the latest version might not be released yet if app.mobile_ucr_restore_version != '2.0': save_in_log(f"Processing App: {domain}: {app.name}: {app.id}") From 0833b6f5df6b69a8402aa6cdced6d78704b26c76 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Thu, 17 Oct 2024 10:08:34 +0200 Subject: [PATCH 12/13] support single app retrieval --- corehq/mobile_ucr_v2_update_script.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index ac0a3db4089c..3937a6ffb82a 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -42,7 +42,11 @@ skip_domains = set() -def process(dry_run=True, max_memory_size=None): +def process(dry_run=True, max_memory_size=None, single_app_retrieve=False): + """ + - single_app_retrieve: Only retrieve and process one app at a time from DB. Results + in more DB hits, but reduces memory usage + """ if max_memory_size: set_max_memory(max_memory_size) @@ -65,9 +69,17 @@ def process(dry_run=True, max_memory_size=None): save_in_log(f"Processing domain: {domain} ...") app_ids = list(get_latest_app_ids_and_versions(domain)) - apps = get_apps_by_id(domain, app_ids) - for app in apps: + if single_app_retrieve: + apps_or_ids = app_ids + else: + apps_or_ids = get_apps_by_id(domain, app_ids) + + for app_or_id in apps_or_ids: try: + if single_app_retrieve: + app = get_apps_by_id(domain, [app_or_id]) + else: + app = app_or_id if not list(app.get_report_modules()): continue From 9de34f22c168101c3c0d00760ae519b88f49dfbd Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Thu, 17 Oct 2024 10:11:35 +0200 Subject: [PATCH 13/13] small refactor --- corehq/mobile_ucr_v2_update_script.py | 30 ++++++++++++++++----------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py index 3937a6ffb82a..8df538e93624 100644 --- a/corehq/mobile_ucr_v2_update_script.py +++ b/corehq/mobile_ucr_v2_update_script.py @@ -80,18 +80,7 @@ def process(dry_run=True, max_memory_size=None, single_app_retrieve=False): app = get_apps_by_id(domain, [app_or_id]) else: app = app_or_id - if not list(app.get_report_modules()): - continue - - # Don't look at app.is_released since the latest version might not be released yet - if app.mobile_ucr_restore_version != '2.0': - save_in_log(f"Processing App: {domain}: {app.name}: {app.id}") - if not has_non_v2_reference(domain, app): - update_app(domain, app, dry_run) - else: - save_in_log( - f"App contains V1 references and couldn't updated: {domain}: {app.name}: {app.id}", - ) + process_app(domain, app, dry_run) except Exception as e: save_in_log(f"Error occurred for {domain}: {str(e)}") save_in_log(traceback.format_exc()) @@ -131,6 +120,23 @@ def has_non_v2_reference(domain, app): return False +def process_app(domain, app, dry_run): + if not list(app.get_report_modules()): + return + + # Don't look at app.is_released since the latest version might not be released yet + if app.mobile_ucr_restore_version == '2.0': + return + + save_in_log(f"Processing App: {domain}: {app.name}: {app.id}") + if not has_non_v2_reference(domain, app): + update_app(domain, app, dry_run) + else: + save_in_log( + f"App contains V1 references and couldn't updated: {domain}: {app.name}: {app.id}", + ) + + def update_app(domain, app, dry_run): save_in_log(f"Updating App: {domain}: {app.name}: {app.id}") if not dry_run: