From f95c092bf20813504a21e389ef2a657afca7204f Mon Sep 17 00:00:00 2001 From: Sid Date: Tue, 20 Aug 2024 10:06:43 -0400 Subject: [PATCH 01/36] Fix error llm DS-2747 --- swirl/connectors/gen_ai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swirl/connectors/gen_ai.py b/swirl/connectors/gen_ai.py index 897ffac1..97754112 100644 --- a/swirl/connectors/gen_ai.py +++ b/swirl/connectors/gen_ai.py @@ -90,9 +90,9 @@ def execute_search(self, session=None): { 'title': self.query_string_to_provider, 'body': f'{msg}', - 'author': f'{llm.get_provider().model}', + 'author': f'GenAI', 'date_published': str(datetime.now()), - 'model': f'{llm.get_provider().model}' + 'model': f'Unknown Model' } ] From a344d9c33fb7c8b59c719ea0827babb75b9124b4 Mon Sep 17 00:00:00 2001 From: Sid Date: Sun, 15 Sep 2024 23:52:15 -0400 Subject: [PATCH 02/36] Initial PII scanner for query and result pipelines using MS presidio DS-2710 --- requirements.txt | 2 + swirl/models.py | 6 +- swirl/processors/__init__.py | 2 +- swirl/processors/remove_pii.py | 118 +++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 swirl/processors/remove_pii.py diff --git a/requirements.txt b/requirements.txt index efc4512f..0c784e9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,3 +40,5 @@ pinecone-client pandas drf-spectacular qdrant-client==1.10.0 +presidio-analyzer +presidio-anonymizer \ No newline at end of file diff --git a/swirl/models.py b/swirl/models.py index 03e940ac..2240f68c 100644 --- a/swirl/models.py +++ b/swirl/models.py @@ -99,7 +99,8 @@ class SearchProvider(models.Model): ('GenAIQueryProcessor', 'GenAIQueryProcessor'), ('AdaptiveQueryProcessor', 'AdaptiveQueryProcessor'), ('NoModQueryProcessor', 'NoModQueryProcessor'), - ('SpellcheckQueryProcessor', 'SpellcheckQueryProcessor') + ('SpellcheckQueryProcessor', 'SpellcheckQueryProcessor'), + ('RemovePIIQueryProcessor', 'RemovePIIQueryProcessor'), ] query_processors = models.JSONField(default=getSearchProviderQueryProcessorsDefault, blank=True) query_mappings = models.CharField(max_length=2048, default=str, blank=True) @@ -114,7 +115,8 @@ class SearchProvider(models.Model): ('CleanTextResultProcessor','CleanTextResultProcessor'), ('RequireQueryStringInTitleResultProcessor','RequireQueryStringInTitleResultProcessor'), ('AutomaticPayloadMapperResultProcessor', 'AutomaticPayloadMapperResultProcessor'), - ('CosineRelevancyResultProcessor','CosineRelevancyResultProcessor') + ('CosineRelevancyResultProcessor','CosineRelevancyResultProcessor'), + ('RemovePIIResultProcessor', 'RemovePIIResultProcessor'), ] response_mappings = models.CharField(max_length=2048, default=str, blank=True) diff --git a/swirl/processors/__init__.py b/swirl/processors/__init__.py index d4282054..202db90c 100644 --- a/swirl/processors/__init__.py +++ b/swirl/processors/__init__.py @@ -13,9 +13,9 @@ from swirl.processors.gen_ai_query import * from swirl.processors.transform_query_processor import * from swirl.processors.date_finder import * +from swirl.processors.remove_pii import * from swirl.models import Search, SearchProvider - def alloc_processor(processor): if not processor: logger.error("blank processor") diff --git a/swirl/processors/remove_pii.py b/swirl/processors/remove_pii.py new file mode 100644 index 00000000..3150c9d5 --- /dev/null +++ b/swirl/processors/remove_pii.py @@ -0,0 +1,118 @@ +''' +@author: Sid Probstein +@contact: sid@swirl.today +''' + +from django.conf import settings + +from celery.utils.log import get_task_logger +logger = get_task_logger(__name__) + +from swirl.processors.generic import QueryProcessor, ResultProcessor + +from presidio_analyzer import AnalyzerEngine +from presidio_anonymizer import AnonymizerEngine, OperatorConfig + +# Instantiate Presidio Analyzer and Anonymizer +analyzer = AnalyzerEngine() +anonymizer = AnonymizerEngine() + +############################################# + +def remove_pii(text: str) -> str: + """ + Removes PII from the given text string using Presidio. + + :param text: The input string (either query or result) to clean. + :return: The text with PII removed. + """ + # Analyze the input text for PII entities + pii_entities = analyzer.analyze(text=text, language='en') + + if not pii_entities: + return text + + # operators = {"DEFAULT": OperatorConfig("replace", {"new_value": "<>"})} + operators = {"DEFAULT": OperatorConfig("redact")} + + # Remove PII by anonymizing the identified entities + anonymized_result = anonymizer.anonymize( + text=text, + analyzer_results=pii_entities, + operators=operators + ) + + return anonymized_result.text + +############################################# + +class RemovePIIQueryProcessor(QueryProcessor): + """ + A SWIRL metasearch query processor that removes PII from search queries. + Inherits from SWIRL's base QueryProcessor class. + """ + + type = 'RemovePIIQueryProcessor' + + def process(self) -> str: + """ + Overrides the base QueryProcessor's process method. + It processes the query to remove PII before the query is further handled. + + :return: The processed query with PII removed. + """ + + # Remove PII from the query + cleaned_query = remove_pii(self.query_string) + self.warning(f"Removed PII from {self.query_string} -> {cleaned_query}!") + + return cleaned_query + +############################################# + +class RemovePIIResultProcessor(ResultProcessor): + """ + A SWIRL result processor that removes PII from the search results. + Inherits from SWIRL's base ResultProcessor class. + """ + + type = "RemovePIIResultProcessor" + + def __init__(self, results, provider, query_string, request_id='', **kwargs): + super().__init__(results, provider, query_string, request_id=request_id, **kwargs) + self.modified = 0 + + def process(self) -> int: + """ + Overrides the base ResultProcessor's process method. + It processes each result to remove PII and tracks the number of modified results. + + :return: The number of modified results. + """ + logger.debug(f"Processing {len(self.results)} results for PII removal.") + + modified = 0 + for result in self.results: + pii_modified = False + + # Remove PII from 'title' and 'body' fields of each result + if 'title' in result: + cleaned_title = remove_pii(result['title']) + if cleaned_title != result['title']: + result['title'] = cleaned_title + pii_modified = True + + if 'body' in result: + cleaned_body = remove_pii(result['body']) + if cleaned_body != result['body']: + result['body'] = cleaned_body + pii_modified = True + + if pii_modified: + modified += 1 + + self.processed_results = self.results + self.modified = modified + logger.debug(f"PII removal complete. {self.modified} results modified.") + + return self.modified From a2befd100ab80281682313ead44e5e973b8614ae Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Tue, 17 Sep 2024 13:47:12 -0400 Subject: [PATCH 03/36] update banner.py version on develop --- swirl/banner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swirl/banner.py b/swirl/banner.py index 72a4a100..511358e9 100644 --- a/swirl/banner.py +++ b/swirl/banner.py @@ -10,9 +10,9 @@ class bcolors: ENDC = '\033[0m' BOLD = '\033[1m' -SWIRL_VERSION = '3.7-DEV' +SWIRL_VERSION = '3.9-DEV' -SWIRL_BANNER_TEXT = "__S_W_I_R_L__3_._7-DEV__________________________________________________________" +SWIRL_BANNER_TEXT = "__S_W_I_R_L__3_._9-DEV__________________________________________________________" SWIRL_BANNER = f'{bcolors.BOLD}{SWIRL_BANNER_TEXT}{bcolors.ENDC}' ############################################# From c96bec1fa6e7f1e339edd8ed10dfcf1fa7453790 Mon Sep 17 00:00:00 2001 From: Sid Date: Wed, 18 Sep 2024 13:57:53 -0400 Subject: [PATCH 04/36] Modified result processor to redact entities, but highlighting is conflicting --- swirl/processors/remove_pii.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/swirl/processors/remove_pii.py b/swirl/processors/remove_pii.py index 3150c9d5..2efe02c7 100644 --- a/swirl/processors/remove_pii.py +++ b/swirl/processors/remove_pii.py @@ -19,9 +19,26 @@ ############################################# -def remove_pii(text: str) -> str: +def redact_pii(text: str) -> str: + """ + Redacts PII from the given text string using Presidio. + + :param text: The input string (either query or result) to clean. + :return: The text with PII redacted. + """ + + return remove_pii(text, redact=True) + +def remove_pii(text: str, redact=False) -> str: """ Removes PII from the given text string using Presidio. + + This is confusing: + + By default, Presidio redacts entities, replacing it with . + The Presidio "redact" option removes the PII entirely. + + In SWIRL, remove means "remove the PII" and "redact" means "replace it with ". :param text: The input string (either query or result) to clean. :return: The text with PII removed. @@ -32,10 +49,11 @@ def remove_pii(text: str) -> str: if not pii_entities: return text - # operators = {"DEFAULT": OperatorConfig("replace", {"new_value": "<>"})} operators = {"DEFAULT": OperatorConfig("redact")} + if redact: + # if specified + operators = {"DEFAULT": OperatorConfig("replace")} - # Remove PII by anonymizing the identified entities anonymized_result = anonymizer.anonymize( text=text, analyzer_results=pii_entities, @@ -90,6 +108,7 @@ def process(self) -> int: :return: The number of modified results. """ logger.debug(f"Processing {len(self.results)} results for PII removal.") + self.warning("FOO Remove PII from results...") modified = 0 for result in self.results: @@ -97,14 +116,18 @@ def process(self) -> int: # Remove PII from 'title' and 'body' fields of each result if 'title' in result: - cleaned_title = remove_pii(result['title']) + self.warning("FOO Redacting title...") + cleaned_title = redact_pii(result['title']) if cleaned_title != result['title']: + self.warning("FOO Redacted title...: " + cleaned_title) result['title'] = cleaned_title pii_modified = True if 'body' in result: - cleaned_body = remove_pii(result['body']) + self.warning("FOO Redacting body...") + cleaned_body = redact_pii(result['body']) if cleaned_body != result['body']: + self.warning("FOO Redacted body...: " + cleaned_body) result['body'] = cleaned_body pii_modified = True @@ -113,6 +136,7 @@ def process(self) -> int: self.processed_results = self.results self.modified = modified + self.warning("FOO Modified: " + str(self.modified)) logger.debug(f"PII removal complete. {self.modified} results modified.") return self.modified From 6791ab2fbfb6b1111e6f79e0835efe25936de8a6 Mon Sep 17 00:00:00 2001 From: Sid Date: Thu, 19 Sep 2024 19:35:47 -0400 Subject: [PATCH 05/36] Redaction and re-highlighting of results working --- swirl/processors/remove_pii.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/swirl/processors/remove_pii.py b/swirl/processors/remove_pii.py index 2efe02c7..c5943bde 100644 --- a/swirl/processors/remove_pii.py +++ b/swirl/processors/remove_pii.py @@ -19,7 +19,7 @@ ############################################# -def redact_pii(text: str) -> str: +def redact_pii(text: str, query_string=None) -> str: """ Redacts PII from the given text string using Presidio. @@ -27,9 +27,11 @@ def redact_pii(text: str) -> str: :return: The text with PII redacted. """ - return remove_pii(text, redact=True) + return remove_pii(text, query_string, redact=True) -def remove_pii(text: str, redact=False) -> str: +from swirl.processors.utils import remove_tags, highlight_list + +def remove_pii(text: str, query_string=None, redact=False) -> str: """ Removes PII from the given text string using Presidio. @@ -43,6 +45,10 @@ def remove_pii(text: str, redact=False) -> str: :param text: The input string (either query or result) to clean. :return: The text with PII removed. """ + + # remove tags from the text + text = remove_tags(text) + # Analyze the input text for PII entities pii_entities = analyzer.analyze(text=text, language='en') @@ -60,7 +66,16 @@ def remove_pii(text: str, redact=False) -> str: operators=operators ) - return anonymized_result.text + anonymized_text = anonymized_result.text + + if redact: + anonymized_text = anonymized_text.replace('<', '[').replace('>', ']') + + if query_string: + highlighted_anonymized_text = highlight_list(anonymized_text, query_string.split()) + return highlighted_anonymized_text + + return anonymized_text ############################################# @@ -117,7 +132,7 @@ def process(self) -> int: # Remove PII from 'title' and 'body' fields of each result if 'title' in result: self.warning("FOO Redacting title...") - cleaned_title = redact_pii(result['title']) + cleaned_title = redact_pii(result['title'], self.query_string) if cleaned_title != result['title']: self.warning("FOO Redacted title...: " + cleaned_title) result['title'] = cleaned_title @@ -125,7 +140,7 @@ def process(self) -> int: if 'body' in result: self.warning("FOO Redacting body...") - cleaned_body = redact_pii(result['body']) + cleaned_body = redact_pii(result['body'], self.query_string) if cleaned_body != result['body']: self.warning("FOO Redacted body...: " + cleaned_body) result['body'] = cleaned_body From 30d5075ff042ed95d29f81c65083d7e8dc5e3b72 Mon Sep 17 00:00:00 2001 From: Sid Date: Thu, 19 Sep 2024 20:00:48 -0400 Subject: [PATCH 06/36] Rename ResultProcessor, cover payload str only --- swirl/models.py | 2 +- swirl/processors/remove_pii.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/swirl/models.py b/swirl/models.py index 2240f68c..b6cc719f 100644 --- a/swirl/models.py +++ b/swirl/models.py @@ -116,7 +116,7 @@ class SearchProvider(models.Model): ('RequireQueryStringInTitleResultProcessor','RequireQueryStringInTitleResultProcessor'), ('AutomaticPayloadMapperResultProcessor', 'AutomaticPayloadMapperResultProcessor'), ('CosineRelevancyResultProcessor','CosineRelevancyResultProcessor'), - ('RemovePIIResultProcessor', 'RemovePIIResultProcessor'), + ('RedactPIIResultProcessor', 'RedactPIIResultProcessor'), ] response_mappings = models.CharField(max_length=2048, default=str, blank=True) diff --git a/swirl/processors/remove_pii.py b/swirl/processors/remove_pii.py index c5943bde..ea358ef2 100644 --- a/swirl/processors/remove_pii.py +++ b/swirl/processors/remove_pii.py @@ -103,10 +103,11 @@ def process(self) -> str: ############################################# -class RemovePIIResultProcessor(ResultProcessor): +class RedactPIIResultProcessor(ResultProcessor): """ A SWIRL result processor that removes PII from the search results. Inherits from SWIRL's base ResultProcessor class. + Meant to be run after CosineResultProcessor. """ type = "RemovePIIResultProcessor" @@ -145,6 +146,17 @@ def process(self) -> int: self.warning("FOO Redacted body...: " + cleaned_body) result['body'] = cleaned_body pii_modified = True + + if 'payload' in result: + for key in result['payload']: + if type(result['payload'][key]) is not str: + continue + self.warning(f"FOO Redacting payload {key}...") + cleaned_payload = redact_pii(result['payload'][key], self.query_string) + if cleaned_payload != result['payload'][key]: + self.warning("FOO Redacted payload...: " + cleaned_payload) + result['payload'][key] = cleaned_payload + pii_modified = True if pii_modified: modified += 1 From 1485255d55e777ace9e73c2d500dc4d41692323c Mon Sep 17 00:00:00 2001 From: Sid Date: Thu, 19 Sep 2024 20:32:00 -0400 Subject: [PATCH 07/36] Added PostResultProcessor, removed debug etc --- swirl/models.py | 3 +- swirl/processors/remove_pii.py | 105 +++++++++++++++++++-------------- 2 files changed, 63 insertions(+), 45 deletions(-) diff --git a/swirl/models.py b/swirl/models.py index b6cc719f..133f5688 100644 --- a/swirl/models.py +++ b/swirl/models.py @@ -147,7 +147,7 @@ def getSearchPreQueryProcessorsDefault(): return [] def getSearchPostResultProcessorsDefault(): - return ["DedupeByFieldPostResultProcessor","CosineRelevancyPostResultProcessor"] + return ["DedupeByFieldPostResultProcessor","CosineRelevancyPostResultProcessor", "RedactPIIPostResultProcessor"] class Search(models.Model): id = models.BigAutoField(primary_key=True) @@ -179,6 +179,7 @@ class Search(models.Model): ('DropIrrelevantPostResultProcessor','DropIrrelevantPostResultProcessor'), ('DedupeByFieldPostResultProcessor', 'DedupeByFieldPostResultProcessor'), ('DedupeBySimilarityPostResultProcessor', 'DedupeBySimilarityPostResultProcessor'), + ('RedactPIIPostResultProcessor', 'RedactPIIPostResultProcessor'), ] post_result_processors = models.JSONField(default=getSearchPostResultProcessorsDefault, blank=True) result_url = models.CharField(max_length=2048, default='/swirl/results?search_id=%d&result_mixer=%s', blank=True) diff --git a/swirl/processors/remove_pii.py b/swirl/processors/remove_pii.py index ea358ef2..d12c3ce1 100644 --- a/swirl/processors/remove_pii.py +++ b/swirl/processors/remove_pii.py @@ -8,7 +8,7 @@ from celery.utils.log import get_task_logger logger = get_task_logger(__name__) -from swirl.processors.generic import QueryProcessor, ResultProcessor +from swirl.processors.generic import QueryProcessor, ResultProcessor, PostResultProcessor from presidio_analyzer import AnalyzerEngine from presidio_anonymizer import AnonymizerEngine, OperatorConfig @@ -35,23 +35,17 @@ def remove_pii(text: str, query_string=None, redact=False) -> str: """ Removes PII from the given text string using Presidio. - This is confusing: - By default, Presidio redacts entities, replacing it with . The Presidio "redact" option removes the PII entirely. - In SWIRL, remove means "remove the PII" and "redact" means "replace it with ". :param text: The input string (either query or result) to clean. :return: The text with PII removed. """ - # remove tags from the text text = remove_tags(text) + pii_entities = analyzer.analyze(text=text, language='en') - # Analyze the input text for PII entities - pii_entities = analyzer.analyze(text=text, language='en') - if not pii_entities: return text @@ -82,22 +76,17 @@ def remove_pii(text: str, query_string=None, redact=False) -> str: class RemovePIIQueryProcessor(QueryProcessor): """ A SWIRL metasearch query processor that removes PII from search queries. - Inherits from SWIRL's base QueryProcessor class. """ type = 'RemovePIIQueryProcessor' def process(self) -> str: """ - Overrides the base QueryProcessor's process method. - It processes the query to remove PII before the query is further handled. - :return: The processed query with PII removed. """ # Remove PII from the query cleaned_query = remove_pii(self.query_string) - self.warning(f"Removed PII from {self.query_string} -> {cleaned_query}!") return cleaned_query @@ -106,56 +95,41 @@ def process(self) -> str: class RedactPIIResultProcessor(ResultProcessor): """ A SWIRL result processor that removes PII from the search results. - Inherits from SWIRL's base ResultProcessor class. Meant to be run after CosineResultProcessor. """ type = "RemovePIIResultProcessor" - def __init__(self, results, provider, query_string, request_id='', **kwargs): - super().__init__(results, provider, query_string, request_id=request_id, **kwargs) - self.modified = 0 - def process(self) -> int: """ - Overrides the base ResultProcessor's process method. - It processes each result to remove PII and tracks the number of modified results. - :return: The number of modified results. """ logger.debug(f"Processing {len(self.results)} results for PII removal.") - self.warning("FOO Remove PII from results...") modified = 0 - for result in self.results: + for item in self.results: pii_modified = False # Remove PII from 'title' and 'body' fields of each result - if 'title' in result: - self.warning("FOO Redacting title...") - cleaned_title = redact_pii(result['title'], self.query_string) - if cleaned_title != result['title']: - self.warning("FOO Redacted title...: " + cleaned_title) - result['title'] = cleaned_title + if 'title' in item: + cleaned_title = redact_pii(item['title'], self.query_string) + if cleaned_title != item['title']: + item['title'] = cleaned_title pii_modified = True - if 'body' in result: - self.warning("FOO Redacting body...") - cleaned_body = redact_pii(result['body'], self.query_string) - if cleaned_body != result['body']: - self.warning("FOO Redacted body...: " + cleaned_body) - result['body'] = cleaned_body + if 'body' in item: + cleaned_body = redact_pii(item['body'], self.query_string) + if cleaned_body != item['body']: + item['body'] = cleaned_body pii_modified = True - if 'payload' in result: - for key in result['payload']: - if type(result['payload'][key]) is not str: + if 'payload' in item: + for key in item['payload']: + if type(item['payload'][key]) is not str: continue - self.warning(f"FOO Redacting payload {key}...") - cleaned_payload = redact_pii(result['payload'][key], self.query_string) - if cleaned_payload != result['payload'][key]: - self.warning("FOO Redacted payload...: " + cleaned_payload) - result['payload'][key] = cleaned_payload + cleaned_payload = redact_pii(item['payload'][key], self.query_string) + if cleaned_payload != item['payload'][key]: + item['payload'][key] = cleaned_payload pii_modified = True if pii_modified: @@ -163,7 +137,50 @@ def process(self) -> int: self.processed_results = self.results self.modified = modified - self.warning("FOO Modified: " + str(self.modified)) logger.debug(f"PII removal complete. {self.modified} results modified.") return self.modified + +############################################# + +class RedactPIIPostResultProcessor(PostResultProcessor): + """ + A SWIRL result processor that removes PII from all results. + """ + + type = "RemovePIIPostResultProcessor" + + def process(self) -> int: + """ + :return: The number of modified results. + """ + + modified = 0 + + for result in self.results: + for item in result.json_results: + pii_modified = False + if 'title' in item: + cleaned_title = redact_pii(item['title'], self.search.query_string_processed) + if cleaned_title != item['title']: + item['title'] = cleaned_title + pii_modified = True + if 'body' in item: + cleaned_body = redact_pii(item['body'], self.search.query_string_processed) + if cleaned_body != item['body']: + item['body'] = cleaned_body + pii_modified = True + if 'payload' in item: + for key in item['payload']: + if type(item['payload'][key]) is not str: + continue + cleaned_payload = redact_pii(item['payload'][key], self.search.query_string_processed) + if cleaned_payload != item['payload'][key]: + item['payload'][key] = cleaned_payload + pii_modified = True + if pii_modified: + modified += 1 + result.save() + + self.results_updated = modified + return self.results_updated From 187db33403b419852c7d3802df461e6785c70e0b Mon Sep 17 00:00:00 2001 From: dnicodemus-la Date: Mon, 23 Sep 2024 16:35:04 -0400 Subject: [PATCH 08/36] seperate untagged from anonized text and return the right for the right case. --- swirl/processors/remove_pii.py | 36 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/swirl/processors/remove_pii.py b/swirl/processors/remove_pii.py index d12c3ce1..6f7c9b65 100644 --- a/swirl/processors/remove_pii.py +++ b/swirl/processors/remove_pii.py @@ -38,13 +38,13 @@ def remove_pii(text: str, query_string=None, redact=False) -> str: By default, Presidio redacts entities, replacing it with . The Presidio "redact" option removes the PII entirely. In SWIRL, remove means "remove the PII" and "redact" means "replace it with ". - + :param text: The input string (either query or result) to clean. :return: The text with PII removed. """ - text = remove_tags(text) - pii_entities = analyzer.analyze(text=text, language='en') + untagged_text = remove_tags(text) + pii_entities = analyzer.analyze(text=untagged_text, language='en') if not pii_entities: return text @@ -55,11 +55,11 @@ def remove_pii(text: str, query_string=None, redact=False) -> str: operators = {"DEFAULT": OperatorConfig("replace")} anonymized_result = anonymizer.anonymize( - text=text, - analyzer_results=pii_entities, + text=untagged_text, + analyzer_results=pii_entities, operators=operators ) - + anonymized_text = anonymized_result.text if redact: @@ -77,17 +77,17 @@ class RemovePIIQueryProcessor(QueryProcessor): """ A SWIRL metasearch query processor that removes PII from search queries. """ - + type = 'RemovePIIQueryProcessor' def process(self) -> str: """ :return: The processed query with PII removed. """ - + # Remove PII from the query cleaned_query = remove_pii(self.query_string) - + return cleaned_query ############################################# @@ -97,7 +97,7 @@ class RedactPIIResultProcessor(ResultProcessor): A SWIRL result processor that removes PII from the search results. Meant to be run after CosineResultProcessor. """ - + type = "RemovePIIResultProcessor" def process(self) -> int: @@ -105,11 +105,11 @@ def process(self) -> int: :return: The number of modified results. """ logger.debug(f"Processing {len(self.results)} results for PII removal.") - + modified = 0 for item in self.results: pii_modified = False - + # Remove PII from 'title' and 'body' fields of each result if 'title' in item: cleaned_title = redact_pii(item['title'], self.query_string) @@ -131,14 +131,14 @@ def process(self) -> int: if cleaned_payload != item['payload'][key]: item['payload'][key] = cleaned_payload pii_modified = True - + if pii_modified: modified += 1 self.processed_results = self.results self.modified = modified logger.debug(f"PII removal complete. {self.modified} results modified.") - + return self.modified ############################################# @@ -147,16 +147,16 @@ class RedactPIIPostResultProcessor(PostResultProcessor): """ A SWIRL result processor that removes PII from all results. """ - + type = "RemovePIIPostResultProcessor" def process(self) -> int: """ :return: The number of modified results. """ - + modified = 0 - + for result in self.results: for item in result.json_results: pii_modified = False @@ -182,5 +182,5 @@ def process(self) -> int: modified += 1 result.save() - self.results_updated = modified + self.results_updated = modified return self.results_updated From 77c11fde2118d3a97865c0c2462bafc4bb308e98 Mon Sep 17 00:00:00 2001 From: dnicodemus-la Date: Mon, 23 Sep 2024 16:35:53 -0400 Subject: [PATCH 09/36] PII should not be a default --- swirl/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swirl/models.py b/swirl/models.py index 133f5688..5cf16137 100644 --- a/swirl/models.py +++ b/swirl/models.py @@ -147,7 +147,7 @@ def getSearchPreQueryProcessorsDefault(): return [] def getSearchPostResultProcessorsDefault(): - return ["DedupeByFieldPostResultProcessor","CosineRelevancyPostResultProcessor", "RedactPIIPostResultProcessor"] + return ["DedupeByFieldPostResultProcessor","CosineRelevancyPostResultProcessor"] class Search(models.Model): id = models.BigAutoField(primary_key=True) From 7a9a9f2fc45d7b3df0baa3474ca7adc23853eb1b Mon Sep 17 00:00:00 2001 From: Peter Kahn Date: Tue, 24 Sep 2024 10:39:09 -0400 Subject: [PATCH 10/36] DS-2940 setup unit tests with output processing, pr decoration (#1436) * setup unit tests with output processing, pr decoration --------- Co-authored-by: Peter Kahn --- .github/workflows/unit-tests.yml | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 48c852f2..5dd32025 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,12 +1,28 @@ name: Unit Tests on: - # Manual trigger only + pull_request: + # Run for all PRs to develop - means PR cannot merge until unit tests pass + branches: + - develop + # Skip non-code changes + paths-ignore: + - '.github/**' + - 'integrations/**' + - 'swirl-infra/**' + - 'docs/**' + - 'README.md' + - 'db.sqlite3.dist' workflow_dispatch: -jobs: +permissions: + contents: read + actions: read + checks: write + pull-requests: write - build: +jobs: + unit-tests: runs-on: ubuntu-latest steps: @@ -22,7 +38,8 @@ jobs: - name: Install the Unit Tests run: ./install-test.sh - name: Run the Unit Tests - run: pytest + # generate a JUnit XML report for the test results + run: pytest --junitxml=reports/junit.xml --tb=short --ignore=integrations - name: Upload Log Files if: always() uses: actions/upload-artifact@v4 @@ -31,3 +48,9 @@ jobs: path: | logs/ /var/log/syslog* + + - name: Publish Unit Test Results + if: always() + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + files: reports/junit.xml \ No newline at end of file From e95210b8c66ba12514487ec5a074a6d8adff66bd Mon Sep 17 00:00:00 2001 From: Peter Kahn Date: Tue, 24 Sep 2024 10:54:24 -0400 Subject: [PATCH 11/36] spell checker report --- .github/workflows/spell-checker.yml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/spell-checker.yml b/.github/workflows/spell-checker.yml index 8390041b..77d8a3c7 100644 --- a/.github/workflows/spell-checker.yml +++ b/.github/workflows/spell-checker.yml @@ -2,8 +2,25 @@ name: Check Spelling # Only allow manual run of this workflow from the Actions tab on: + pull_request: + # Run for all PRs to develop - means PR cannot merge until unit tests pass + branches: + - develop + - main + # Skip non-code changes + paths-ignore: + - '.github/**' + - 'integrations/**' + - 'swirl-infra/**' + - 'db.sqlite3.dist' workflow_dispatch: +permissions: + contents: read + actions: read + checks: write + pull-requests: write + jobs: build: runs-on: ubuntu-latest @@ -15,4 +32,10 @@ jobs: uses: crate-ci/typos@master # From here: https://github.com/crate-ci/typos with: config: ./.github/workflows/typos.toml - write_changes: true # Writes changes on the Action's local checkout \ No newline at end of file + write_changes: true # Writes changes on the Action's local checkout + + - name: Capture Git Diff + run: | + git diff > diff_output.txt + echo "## Git Diff Output" >> $GITHUB_STEP_SUMMARY + cat diff_output.txt >> $GITHUB_STEP_SUMMARY \ No newline at end of file From a67be126c19e01416a9c60e61bcd556315955003 Mon Sep 17 00:00:00 2001 From: Peter Kahn Date: Tue, 24 Sep 2024 10:56:37 -0400 Subject: [PATCH 12/36] add report --- .github/workflows/spell-checker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/spell-checker.yml b/.github/workflows/spell-checker.yml index 77d8a3c7..47b76678 100644 --- a/.github/workflows/spell-checker.yml +++ b/.github/workflows/spell-checker.yml @@ -35,6 +35,7 @@ jobs: write_changes: true # Writes changes on the Action's local checkout - name: Capture Git Diff + if: always() run: | git diff > diff_output.txt echo "## Git Diff Output" >> $GITHUB_STEP_SUMMARY From 1b6975c23625e13d5c10a648ada276904d0111f8 Mon Sep 17 00:00:00 2001 From: Peter Kahn Date: Tue, 24 Sep 2024 10:58:11 -0400 Subject: [PATCH 13/36] current reporting is fine --- .github/workflows/spell-checker.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/spell-checker.yml b/.github/workflows/spell-checker.yml index 47b76678..7e42aded 100644 --- a/.github/workflows/spell-checker.yml +++ b/.github/workflows/spell-checker.yml @@ -32,11 +32,4 @@ jobs: uses: crate-ci/typos@master # From here: https://github.com/crate-ci/typos with: config: ./.github/workflows/typos.toml - write_changes: true # Writes changes on the Action's local checkout - - - name: Capture Git Diff - if: always() - run: | - git diff > diff_output.txt - echo "## Git Diff Output" >> $GITHUB_STEP_SUMMARY - cat diff_output.txt >> $GITHUB_STEP_SUMMARY \ No newline at end of file + write_changes: true # Writes changes on the Action's local checkout \ No newline at end of file From dac7cda6494b7a24770bf79ff85c36219c06ef52 Mon Sep 17 00:00:00 2001 From: Peter Kahn Date: Tue, 24 Sep 2024 11:02:14 -0400 Subject: [PATCH 14/36] fix typo --- swirl/connectors/verify_ssl_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swirl/connectors/verify_ssl_common.py b/swirl/connectors/verify_ssl_common.py index 23942457..5e6c2c54 100644 --- a/swirl/connectors/verify_ssl_common.py +++ b/swirl/connectors/verify_ssl_common.py @@ -34,7 +34,7 @@ def get_creds(self, def_verify_certs=False): for cre in cred_list: if cre.startswith('bearer='): - # handle this speacial becauase tokens have '=' sign in them + # handle this special becauase tokens have '=' sign in them bearer = cre[len('bearer='):] if not bearer: self.log_invalid_credentials() From 9ed1a5e813c1f5abaf5eb304450d10d2ac3f87dc Mon Sep 17 00:00:00 2001 From: Igor Brito Date: Fri, 27 Sep 2024 10:26:40 -0300 Subject: [PATCH 15/36] feat: push migrations to the repository --- .gitignore | 1 - swirl/migrations/0001_initial.py | 143 +++++++++++++++++++++++++++++++ swirl/migrations/__init__.py | 0 3 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 swirl/migrations/0001_initial.py create mode 100644 swirl/migrations/__init__.py diff --git a/.gitignore b/.gitignore index 32042dec..a2de7d35 100644 --- a/.gitignore +++ b/.gitignore @@ -130,7 +130,6 @@ dmypy.json /static -swirl/migrations/ # emacs *~ diff --git a/swirl/migrations/0001_initial.py b/swirl/migrations/0001_initial.py new file mode 100644 index 00000000..7c2f8864 --- /dev/null +++ b/swirl/migrations/0001_initial.py @@ -0,0 +1,143 @@ +# Generated by Django 5.1.1 on 2024-09-27 13:26 + +import django.db.models.deletion +import swirl.models +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Authenticator', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ], + ), + migrations.CreateModel( + name='Search', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_updated', models.DateTimeField(auto_now=True)), + ('query_string', models.CharField(default=str, max_length=256)), + ('query_string_processed', models.CharField(blank=True, default=str, max_length=256)), + ('sort', models.CharField(blank=True, default='relevancy', max_length=50)), + ('results_requested', models.IntegerField(default=10)), + ('searchprovider_list', models.JSONField(blank=True, default=list)), + ('subscribe', models.BooleanField(default=False)), + ('status', models.CharField(default='NEW_SEARCH', max_length=50)), + ('time', models.FloatField(default=0.0)), + ('pre_query_processors', models.JSONField(blank=True, default=swirl.models.getSearchPreQueryProcessorsDefault)), + ('post_result_processors', models.JSONField(blank=True, default=swirl.models.getSearchPostResultProcessorsDefault)), + ('result_url', models.CharField(blank=True, default='/swirl/results?search_id=%d&result_mixer=%s', max_length=2048)), + ('new_result_url', models.CharField(blank=True, default='/swirl/results?search_id=%d&result_mixer=RelevancyNewItemsMixer', max_length=2048)), + ('messages', models.JSONField(blank=True, default=list)), + ('result_mixer', models.CharField(choices=[('DateMixer', 'DateMixer'), ('DateNewItemsMixer', 'DateNewItemsMixer'), ('RelevancyMixer', 'RelevancyMixer'), ('RelevancyNewItemsMixer', 'RelevancyNewItemsMixer'), ('RoundRobinMixer', 'RoundRobinMixer'), ('Stack1Mixer', 'Stack1Mixer'), ('Stack2Mixer', 'Stack2Mixer'), ('Stack3Mixer', 'Stack3Mixer'), ('StackNMixer', 'StackNMixer')], default='RelevancyMixer', max_length=200)), + ('retention', models.IntegerField(choices=[(0, 'Never expire'), (1, 'Expire after 1 hour'), (2, 'Expire after 1 day'), (3, 'Expire after 1 month')], default=0)), + ('tags', models.JSONField(default=list)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-date_updated'], + }, + ), + migrations.CreateModel( + name='Result', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_updated', models.DateTimeField(auto_now=True)), + ('provider_id', models.IntegerField(default=0)), + ('searchprovider', models.CharField(default=str, max_length=50)), + ('query_string_to_provider', models.CharField(default=str, max_length=256)), + ('result_processor_json_feedback', models.JSONField(default=list)), + ('query_to_provider', models.CharField(default=str, max_length=2048)), + ('query_processors', models.JSONField(blank=True, default=list)), + ('result_processors', models.JSONField(blank=True, default=list)), + ('messages', models.JSONField(blank=True, default=list)), + ('status', models.CharField(default=str, max_length=20)), + ('retrieved', models.IntegerField(default=0)), + ('found', models.IntegerField(default=0)), + ('time', models.FloatField(default=0.0)), + ('json_results', models.JSONField(default=list)), + ('tags', models.JSONField(default=list)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('search_id', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='swirl.search')), + ], + options={ + 'ordering': ['-date_updated'], + }, + ), + migrations.CreateModel( + name='SearchProvider', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('name', models.CharField(max_length=200)), + ('shared', models.BooleanField(default=False)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_updated', models.DateTimeField(auto_now=True)), + ('active', models.BooleanField(default=True)), + ('default', models.BooleanField(default=True)), + ('authenticator', models.CharField(blank=True, choices=[('Microsoft', 'Microsoft Authentication')], default='', max_length=200)), + ('connector', models.CharField(choices=[('ChatGPT', 'ChatGPT Query String'), ('GenAI', 'Generative AI'), ('RequestsGet', 'HTTP/GET returning JSON'), ('RequestsPost', 'HTTP/POST returning JSON'), ('Elastic', 'Elasticsearch Query String'), ('OpenSearch', 'OpenSearch Query String'), ('QdrantDB', 'QdrantDB'), ('BigQuery', 'Google BigQuery'), ('Sqlite3', 'Sqlite3'), ('M365OutlookMessages', 'M365 Outlook Messages'), ('M365OneDrive', 'M365 One Drive'), ('M365OutlookCalendar', 'M365 Outlook Calendar'), ('M365SharePointSites', 'M365 SharePoint Sites'), ('MicrosoftTeams', 'Microsoft Teams'), ('MongoDB', 'MongoDB'), ('Oracle', 'Oracle'), ('Snowflake', 'Snowflake'), ('PineconeDB', 'PineconeDB')], default='RequestsGet', max_length=200)), + ('url', models.CharField(blank=True, default=str, max_length=2048)), + ('query_template', models.CharField(blank=True, default='{url}?q={query_string}', max_length=2048)), + ('query_template_json', models.JSONField(blank=True, default={})), + ('post_query_template', models.JSONField(blank=True, default={})), + ('query_processors', models.JSONField(blank=True, default=swirl.models.getSearchProviderQueryProcessorsDefault)), + ('query_mappings', models.CharField(blank=True, default=str, max_length=2048)), + ('response_mappings', models.CharField(blank=True, default=str, max_length=2048)), + ('result_grouping_field', models.CharField(blank=True, default=str, max_length=1024)), + ('result_processors', models.JSONField(blank=True, default=swirl.models.getSearchProviderResultProcessorsDefault)), + ('result_mappings', models.CharField(blank=True, default=str, max_length=2048)), + ('results_per_query', models.IntegerField(default=10)), + ('eval_credentials', models.CharField(blank=True, default=str, max_length=100)), + ('credentials', models.CharField(blank=True, default=str, max_length=512)), + ('tags', models.JSONField(default=list)), + ('http_request_headers', models.JSONField(blank=True, default={})), + ('page_fetch_config_json', models.JSONField(blank=True, default={})), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['id'], + }, + ), + migrations.CreateModel( + name='OauthToken', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('idp', models.CharField(default='Microsoft', max_length=32)), + ('token', models.CharField(max_length=4096)), + ('refresh_token', models.CharField(blank=True, max_length=4096, null=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('owner', 'idp')}, + }, + ), + migrations.CreateModel( + name='QueryTransform', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('name', models.CharField(max_length=255)), + ('shared', models.BooleanField(default=False)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_updated', models.DateTimeField(auto_now=True)), + ('qrx_type', models.CharField(choices=[('rewrite', 'Rewrite'), ('synonym', 'Synonym'), ('bag', 'Synonym Bag')], default='rewrite', max_length=64)), + ('config_content', models.TextField()), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('name', 'qrx_type')}, + }, + ), + ] diff --git a/swirl/migrations/__init__.py b/swirl/migrations/__init__.py new file mode 100644 index 00000000..e69de29b From 4f2074375dd981477f97d063823a88f2b3b25cac Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Tue, 8 Oct 2024 16:56:46 -0400 Subject: [PATCH 16/36] update testing workflows with new env vars --- .github/workflows/qa-suite.yml | 2 ++ .github/workflows/test-build-pipeline.yml | 2 ++ .github/workflows/testing-wip.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.github/workflows/qa-suite.yml b/.github/workflows/qa-suite.yml index bf7a4253..5cdb41df 100644 --- a/.github/workflows/qa-suite.yml +++ b/.github/workflows/qa-suite.yml @@ -53,6 +53,8 @@ jobs: echo "MSAL_CB_PORT=8000" >> .env.qa echo "QA_ADMIN_PW=${{ secrets.QA_ADMIN_PW }}" >> .env.qa echo "QA_OPENAI_KEY=${{ secrets.QA_OPENAI_KEY }}" >> .env.qa + echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa + echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa echo "========" cat .env.qa echo "========" diff --git a/.github/workflows/test-build-pipeline.yml b/.github/workflows/test-build-pipeline.yml index 001589d9..e7084d2a 100644 --- a/.github/workflows/test-build-pipeline.yml +++ b/.github/workflows/test-build-pipeline.yml @@ -78,6 +78,8 @@ jobs: echo "MSAL_CB_PORT=8000" >> .env.qa echo "QA_ADMIN_PW=${{ secrets.QA_ADMIN_PW }}" >> .env.qa echo "QA_OPENAI_KEY=${{ secrets.QA_OPENAI_KEY }}" >> .env.qa + echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa + echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa echo "========" cat .env.qa echo "========" diff --git a/.github/workflows/testing-wip.yml b/.github/workflows/testing-wip.yml index 706e2609..7326098d 100644 --- a/.github/workflows/testing-wip.yml +++ b/.github/workflows/testing-wip.yml @@ -66,6 +66,8 @@ jobs: echo "MSAL_CB_PORT=8000" >> .env.qa echo "QA_ADMIN_PW=${{ secrets.QA_ADMIN_PW }}" >> .env.qa echo "QA_OPENAI_KEY=${{ secrets.QA_OPENAI_KEY }}" >> .env.qa + echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa + echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa echo "========" cat .env.qa echo "========" From baafb6b5e2221e7108c645d4bfdfaa8b1b836bab Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Wed, 9 Oct 2024 10:27:02 -0400 Subject: [PATCH 17/36] update testing workflows with new env vars --- .github/workflows/qa-suite.yml | 3 +++ .github/workflows/test-build-pipeline.yml | 3 +++ .github/workflows/testing-wip.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.github/workflows/qa-suite.yml b/.github/workflows/qa-suite.yml index 5cdb41df..30f280db 100644 --- a/.github/workflows/qa-suite.yml +++ b/.github/workflows/qa-suite.yml @@ -53,8 +53,11 @@ jobs: echo "MSAL_CB_PORT=8000" >> .env.qa echo "QA_ADMIN_PW=${{ secrets.QA_ADMIN_PW }}" >> .env.qa echo "QA_OPENAI_KEY=${{ secrets.QA_OPENAI_KEY }}" >> .env.qa + echo "QA_NLR_USERNAME=${{ secrets.QA_NLR_USERNAME }}" >> .env.qa + echo "QA_NLR_PASSWORD=${{ secrets.QA_NLR_PASSWORD }}" >> .env.qa echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa + echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa echo "========" cat .env.qa echo "========" diff --git a/.github/workflows/test-build-pipeline.yml b/.github/workflows/test-build-pipeline.yml index e7084d2a..7e808460 100644 --- a/.github/workflows/test-build-pipeline.yml +++ b/.github/workflows/test-build-pipeline.yml @@ -78,8 +78,11 @@ jobs: echo "MSAL_CB_PORT=8000" >> .env.qa echo "QA_ADMIN_PW=${{ secrets.QA_ADMIN_PW }}" >> .env.qa echo "QA_OPENAI_KEY=${{ secrets.QA_OPENAI_KEY }}" >> .env.qa + echo "QA_NLR_USERNAME=${{ secrets.QA_NLR_USERNAME }}" >> .env.qa + echo "QA_NLR_PASSWORD=${{ secrets.QA_NLR_PASSWORD }}" >> .env.qa echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa + echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa echo "========" cat .env.qa echo "========" diff --git a/.github/workflows/testing-wip.yml b/.github/workflows/testing-wip.yml index 7326098d..1751bb40 100644 --- a/.github/workflows/testing-wip.yml +++ b/.github/workflows/testing-wip.yml @@ -66,8 +66,11 @@ jobs: echo "MSAL_CB_PORT=8000" >> .env.qa echo "QA_ADMIN_PW=${{ secrets.QA_ADMIN_PW }}" >> .env.qa echo "QA_OPENAI_KEY=${{ secrets.QA_OPENAI_KEY }}" >> .env.qa + echo "QA_NLR_USERNAME=${{ secrets.QA_NLR_USERNAME }}" >> .env.qa + echo "QA_NLR_PASSWORD=${{ secrets.QA_NLR_PASSWORD }}" >> .env.qa echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa + echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa echo "========" cat .env.qa echo "========" From 22f9462563bd453792660e8969205f864117924d Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Thu, 10 Oct 2024 16:12:59 -0400 Subject: [PATCH 18/36] add GitHub env var to testing workflows --- .github/workflows/qa-suite.yml | 1 + .github/workflows/test-build-pipeline.yml | 1 + .github/workflows/testing-wip.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/qa-suite.yml b/.github/workflows/qa-suite.yml index 30f280db..d35855b8 100644 --- a/.github/workflows/qa-suite.yml +++ b/.github/workflows/qa-suite.yml @@ -58,6 +58,7 @@ jobs: echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa + echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa echo "========" cat .env.qa echo "========" diff --git a/.github/workflows/test-build-pipeline.yml b/.github/workflows/test-build-pipeline.yml index 7e808460..e1297790 100644 --- a/.github/workflows/test-build-pipeline.yml +++ b/.github/workflows/test-build-pipeline.yml @@ -83,6 +83,7 @@ jobs: echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa + echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa echo "========" cat .env.qa echo "========" diff --git a/.github/workflows/testing-wip.yml b/.github/workflows/testing-wip.yml index 1751bb40..6b0af40f 100644 --- a/.github/workflows/testing-wip.yml +++ b/.github/workflows/testing-wip.yml @@ -71,6 +71,7 @@ jobs: echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa + echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa echo "========" cat .env.qa echo "========" From 91b824b7dfba35d1045653243f362b9fc8804725 Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Fri, 11 Oct 2024 16:06:42 -0400 Subject: [PATCH 19/36] add new bigq env var handling to testing workflows --- .github/workflows/qa-suite.yml | 8 ++++++++ .github/workflows/test-build-pipeline.yml | 8 ++++++++ .github/workflows/testing-wip.yml | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/.github/workflows/qa-suite.yml b/.github/workflows/qa-suite.yml index d35855b8..5c0b0bac 100644 --- a/.github/workflows/qa-suite.yml +++ b/.github/workflows/qa-suite.yml @@ -46,6 +46,14 @@ jobs: python swirl.py start env: ALLOWED_HOSTS: localhost,host.docker.internal + - name: Decode BigQuery token and create JSON file + run: | + echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json" + env: + BIGQUERY_TOKEN_BASE64: ${{ secrets.QA_BIGQUERY_TOKEN_BASE64 }} + - name: Export BigQuery token file path as env var + run: | + echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> $GITHUB_ENV - name: Run the QA Suite run: | echo "SWIRL_TEST_HOST=localhost" > .env.qa diff --git a/.github/workflows/test-build-pipeline.yml b/.github/workflows/test-build-pipeline.yml index e1297790..fcdde785 100644 --- a/.github/workflows/test-build-pipeline.yml +++ b/.github/workflows/test-build-pipeline.yml @@ -71,6 +71,14 @@ jobs: python swirl.py start env: ALLOWED_HOSTS: localhost,host.docker.internal + - name: Decode BigQuery token and create JSON file + run: | + echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json" + env: + BIGQUERY_TOKEN_BASE64: ${{ secrets.QA_BIGQUERY_TOKEN_BASE64 }} + - name: Export BigQuery token file path as env var + run: | + echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> $GITHUB_ENV - name: Run the QA Suite run: | echo "SWIRL_TEST_HOST=localhost" > .env.qa diff --git a/.github/workflows/testing-wip.yml b/.github/workflows/testing-wip.yml index 6b0af40f..a73831d0 100644 --- a/.github/workflows/testing-wip.yml +++ b/.github/workflows/testing-wip.yml @@ -59,6 +59,14 @@ jobs: python swirl.py start env: ALLOWED_HOSTS: localhost,host.docker.internal + - name: Decode BigQuery token and create JSON file + run: | + echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json" + env: + BIGQUERY_TOKEN_BASE64: ${{ secrets.QA_BIGQUERY_TOKEN_BASE64 }} + - name: Export BigQuery token file path as env var + run: | + echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> $GITHUB_ENV - name: Run the QA Suite run: | echo "SWIRL_TEST_HOST=localhost" > .env.qa From af02a395643d3ce48431a8a86c2b40f87912c819 Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Fri, 11 Oct 2024 17:10:41 -0400 Subject: [PATCH 20/36] fix the bigquery env var setup in the testing workflows --- .github/workflows/qa-suite.yml | 4 +--- .github/workflows/test-build-pipeline.yml | 4 +--- .github/workflows/testing-wip.yml | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/qa-suite.yml b/.github/workflows/qa-suite.yml index 5c0b0bac..c27f2496 100644 --- a/.github/workflows/qa-suite.yml +++ b/.github/workflows/qa-suite.yml @@ -51,9 +51,6 @@ jobs: echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json" env: BIGQUERY_TOKEN_BASE64: ${{ secrets.QA_BIGQUERY_TOKEN_BASE64 }} - - name: Export BigQuery token file path as env var - run: | - echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> $GITHUB_ENV - name: Run the QA Suite run: | echo "SWIRL_TEST_HOST=localhost" > .env.qa @@ -67,6 +64,7 @@ jobs: echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa + echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> .env.qa echo "========" cat .env.qa echo "========" diff --git a/.github/workflows/test-build-pipeline.yml b/.github/workflows/test-build-pipeline.yml index fcdde785..15a69615 100644 --- a/.github/workflows/test-build-pipeline.yml +++ b/.github/workflows/test-build-pipeline.yml @@ -76,9 +76,6 @@ jobs: echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json" env: BIGQUERY_TOKEN_BASE64: ${{ secrets.QA_BIGQUERY_TOKEN_BASE64 }} - - name: Export BigQuery token file path as env var - run: | - echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> $GITHUB_ENV - name: Run the QA Suite run: | echo "SWIRL_TEST_HOST=localhost" > .env.qa @@ -92,6 +89,7 @@ jobs: echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa + echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> .env.qa echo "========" cat .env.qa echo "========" diff --git a/.github/workflows/testing-wip.yml b/.github/workflows/testing-wip.yml index a73831d0..f2feefc1 100644 --- a/.github/workflows/testing-wip.yml +++ b/.github/workflows/testing-wip.yml @@ -64,9 +64,6 @@ jobs: echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json" env: BIGQUERY_TOKEN_BASE64: ${{ secrets.QA_BIGQUERY_TOKEN_BASE64 }} - - name: Export BigQuery token file path as env var - run: | - echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> $GITHUB_ENV - name: Run the QA Suite run: | echo "SWIRL_TEST_HOST=localhost" > .env.qa @@ -80,6 +77,7 @@ jobs: echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa + echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> .env.qa echo "========" cat .env.qa echo "========" From 7087a88347f1e87f1e25e69723c2d17afa1ac837 Mon Sep 17 00:00:00 2001 From: dnicodemus-la Date: Wed, 16 Oct 2024 11:09:22 -0400 Subject: [PATCH 21/36] made change, added and tested migration --- ...esult_query_string_to_provider_and_more.py | 28 +++++++++++++++++++ swirl/models.py | 9 +++--- 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 swirl/migrations/0002_alter_result_query_string_to_provider_and_more.py diff --git a/swirl/migrations/0002_alter_result_query_string_to_provider_and_more.py b/swirl/migrations/0002_alter_result_query_string_to_provider_and_more.py new file mode 100644 index 00000000..c8cdcbf6 --- /dev/null +++ b/swirl/migrations/0002_alter_result_query_string_to_provider_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 5.1.1 on 2024-10-16 15:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('swirl', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='result', + name='query_string_to_provider', + field=models.CharField(default=str, max_length=2048), + ), + migrations.AlterField( + model_name='search', + name='query_string', + field=models.CharField(default=str, max_length=2048), + ), + migrations.AlterField( + model_name='search', + name='query_string_processed', + field=models.CharField(blank=True, default=str, max_length=2048), + ), + ] diff --git a/swirl/models.py b/swirl/models.py index 5cf16137..fde2cb45 100644 --- a/swirl/models.py +++ b/swirl/models.py @@ -12,6 +12,7 @@ def getSearchProviderQueryProcessorsDefault(): def getSearchProviderResultProcessorsDefault(): return ["MappingResultProcessor","DateFinderResultProcessor","CosineRelevancyResultProcessor"] +MAX_QUERY_STRING_LENGTH = 2048 class FlexibleChoiceField(models.CharField): """ Allow choices and free text so we can have a user named and shared query transform @@ -154,8 +155,8 @@ class Search(models.Model): owner = models.ForeignKey('auth.User', on_delete=models.CASCADE) date_created = models.DateTimeField(auto_now_add=True) date_updated = models.DateTimeField(auto_now=True) - query_string = models.CharField(max_length=256, default=str) - query_string_processed = models.CharField(max_length=256, default=str, blank=True) + query_string = models.CharField(max_length=MAX_QUERY_STRING_LENGTH, default=str) + query_string_processed = models.CharField(max_length=MAX_QUERY_STRING_LENGTH, default=str, blank=True) SORT_CHOICES = [ ('relevancy', 'relevancy'), ('date', 'date') @@ -225,9 +226,9 @@ class Result(models.Model): search_id = models.ForeignKey(Search, on_delete=models.CASCADE) provider_id = models.IntegerField(default=0) searchprovider = models.CharField(max_length=50, default=str) - query_string_to_provider = models.CharField(max_length=256, default=str) + query_string_to_provider = models.CharField(max_length=MAX_QUERY_STRING_LENGTH, default=str) result_processor_json_feedback = models.JSONField(default=list) - query_to_provider = models.CharField(max_length=2048, default=str) + query_to_provider = models.CharField(max_length=MAX_QUERY_STRING_LENGTH, default=str) query_processors = models.JSONField(default=list, blank=True) result_processors = models.JSONField(default=list, blank=True) messages = models.JSONField(default=list, blank=True) From 5b9ce11fdccc73cd7ba0967a8f302cef3c10f3b3 Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Wed, 16 Oct 2024 14:03:31 -0400 Subject: [PATCH 22/36] updates to our default BigQuery SearchProvider --- SearchProviders/company_data_bigquery.json | 31 ++++++++++++++++++++++ SearchProviders/funding_db_bigquery.json | 22 --------------- SearchProviders/preloaded.json | 18 ++++++++----- 3 files changed, 43 insertions(+), 28 deletions(-) create mode 100644 SearchProviders/company_data_bigquery.json delete mode 100644 SearchProviders/funding_db_bigquery.json diff --git a/SearchProviders/company_data_bigquery.json b/SearchProviders/company_data_bigquery.json new file mode 100644 index 00000000..9a8f5a96 --- /dev/null +++ b/SearchProviders/company_data_bigquery.json @@ -0,0 +1,31 @@ +{ + "name": "Company Data - BigQuery", + "active": false, + "default": false, + "connector": "BigQuery", + "url": "", + "query_template": "select {fields} from `{table}` where search({field1}, '{query_string}') or search({field2}, '{query_string}') or search({field3}, '{query_string}');", + "query_template_json": {}, + "post_query_template": {}, + "http_request_headers": {}, + "page_fetch_config_json": {}, + "query_processors": [ + "AdaptiveQueryProcessor" + ], + "query_mappings": "fields=*,sort_by_date=year_founded,table=company_dataset.company,field1=name,field2=domain,field3=locality", + "result_grouping_field": "", + "result_processors": [ + "MappingResultProcessor", + "CosineRelevancyResultProcessor" + ], + "response_mappings": "", + "result_mappings": "title=name,body='{name} was founded in {year_founded} and serves the {industry} industry. The company is located in {locality} and has approximately {current_employee_estimate} employees. The registered domain for this organization is: {domain}',url='https://www.{linkedin_url}',NO_PAYLOAD", + "results_per_query": 10, + "credentials": "/path/to/bigquery/token.json", + "eval_credentials": "", + "tags": [ + "Company", + "BigQuery", + "Internal" + ] +} diff --git a/SearchProviders/funding_db_bigquery.json b/SearchProviders/funding_db_bigquery.json deleted file mode 100644 index 76910705..00000000 --- a/SearchProviders/funding_db_bigquery.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "Company Funding Records - BigQuery", - "active": false, - "default": false, - "connector": "BigQuery", - "query_template": "select {fields} from `{table}` where search({field1}, '{query_string}') or search({field2}, '{query_string}');", - "query_processors": [ - "AdaptiveQueryProcessor" - ], - "query_mappings": "fields=*,sort_by_date=fundedDate,table=funding.funding,field1=company,field2=city", - "result_processors": [ - "MappingResultProcessor", - "CosineRelevancyResultProcessor" - ], - "result_mappings": "title='{company}',body='{company} raised ${raisedamt} series {round} on {fundeddate}. The company is located in {city} {state} and has {numemps} employees.',url=id,date_published=fundeddate,NO_PAYLOAD", - "credentials": "/path/to/bigquery/token.json", - "tags": [ - "Company", - "BigQuery", - "Internal" - ] -} diff --git a/SearchProviders/preloaded.json b/SearchProviders/preloaded.json index 694d9ef8..8521876f 100644 --- a/SearchProviders/preloaded.json +++ b/SearchProviders/preloaded.json @@ -326,28 +326,34 @@ ] }, { - "name": "Company Funding Records - BigQuery", + "name": "Company Data - BigQuery", + "description": "Searches info on 7 million companies worldwide ncluding Linkedin URL, company size, location, and number of employees. Search only with company name, domain or location. Supports many languages. Does not support NOT operator.", "active": false, "default": false, "connector": "BigQuery", "url": "", - "query_template": "select {fields} from `{table}` where search({field1}, '{query_string}') or search({field2}, '{query_string}');", + "query_template": "select {fields} from `{table}` where search({field1}, '{query_string}') or search({field2}, '{query_string}') or search({field3}, '{query_string}');", + "query_template_json": {}, + "post_query_template": {}, + "http_request_headers": {}, + "page_fetch_config_json": {}, "query_processors": [ "AdaptiveQueryProcessor" ], - "query_mappings": "fields=*,sort_by_date=fundedDate,table=funding.funding,field1=company,field2=city", + "query_mappings": "fields=*,sort_by_date=year_founded,table=company_dataset.company,field1=name,field2=domain,field3=locality", + "result_grouping_field": "", "result_processors": [ "MappingResultProcessor", "CosineRelevancyResultProcessor" ], "response_mappings": "", - "result_mappings": "title='{company}',body='{company} raised ${raisedamt} series {round} on {fundeddate}. The company is located in {city} {state} and has {numemps} employees.',url=id,date_published=fundeddate,NO_PAYLOAD", + "result_mappings": "title=name,body='{name} was founded in {year_founded} and serves the {industry} industry. The company is located in {locality} and has approximately {current_employee_estimate} employees. The registered domain for this organization is: {domain}',url='https://www.{linkedin_url}',NO_PAYLOAD", "results_per_query": 10, "credentials": "/path/to/bigquery/token.json", + "eval_credentials": "", "tags": [ "Company", - "BigQuery", - "Internal" + "BigQuery" ] }, { From db86f8e46c758b8a365554ab2c99e6f3c8aa327f Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Wed, 16 Oct 2024 14:07:40 -0400 Subject: [PATCH 23/36] updated db.dist including default SP changes --- db.sqlite3.dist | Bin 303104 -> 303104 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/db.sqlite3.dist b/db.sqlite3.dist index 422523e2ead117243c32cbe3774bdba1adf6c9c8..581623d70ab6700b38215c6af8449a0286e43e3a 100644 GIT binary patch delta 5320 zcmai2Yj7LY71nC6Bqzp}9pd;M%Z?q#V6XSyy?b}pu|w>H#?AvUn3$xY%Ens9BDUm6 zvJ+w4IN>!s3M5O}87PlJACwNvK&EU9Q^<6t|DeE>0xf;av_IMo!*kjnbfD>x)(+9` zw$+R@yZhbmp6{G{&bfEBbZ*Vkxit?oR%Zf%z)twCd~QG8EpeIW4?YmBjl;7|*TPS2 zKm61*R={r$pD-g;W07|LWB%uS;;dd39FpFWePn7$xDqu|%H6nJyG#wYF!SQ4&nM>-^z6?c_ zL?T@t3N!=5$Yzu2R2FRiH&V4@hcZ>6qnCvOc#k<)SF;6c%UWlROfsEFPUQ6Lk!(CY zn=8!cYnaMZBUg376gvu2m}l0;8r|+tDidYjCKz*Lam=6BUhW+uVn)U~Va#>KF#|Ol zJ!2$UW!m^g7_+Z9#;ob8aoZxQq7d_~n!#ERq5wABZoXAlZw{?%@G@nrDdy32TRcpx z!g>t16`6lp*Wks8qR`>iBHmTo;L$BIiPD`h(0v8yUT5ZN8%xJ1f~v`ihIh5Vn8D(h zbG1#S3SkTk!p!`%RxVXgKqL&lHG_f%bG)wJgDaScc(|#Ef48pQGqWtoR59PElRdg9 zp-kD^0J_GDy5#y!51U97tLD*quP%aBl@8Yx?Y>*@)kQ>E(uQk`cxyxbr6p?yeqj}x zdm6eu7O_lK&AhLHco7VdG=J03?nOjN63xm+?`)z5UYb`owz@frrpYQXZ*P>n2&06U zZ#4FJ5dz}O>L$^Hs2Wo=b8i#%A~KVy`FNA$MF?~d^RG?nT32ioMZuaWHTQZb3ZV39AjJUua!O#P4U1J5ehH64TcTyE(22*t2UK=RQA)dd&~BhZHoLW@@C|T z$ZTYLWMd>4{$u!+@FU?&cqH5&Iv;vF^rO(J(Ed<2`ZszTy@<}B6uJr#RKQ&d+tulWLXa8a`xaG+gp5Ur#!;c2a0?9xi zxq7hd2?*Zy@N-e6@-&PcwO`BgG+bT25U7T2`KynuXZg=K`_p;;{U8bi_;nDeHN5@V zQ9jLat8H#8|7A0LSf(6n)JK3sg=XFautythwRev1<2Dy2-plFM-{!!KT)1nr`sR7O`mO^9i)PL&SU+TZGdE=YlJUQ@ zk}#h&tisQeD&J-ansXbqiOLW2)^`+sqqU^+68vmze;@8ML99YztVtCyF4V&EJUpMXe}Z{#5QO>#lLutN7p=sma3}4-0r|7nbkv) zJzcNwGMa9X`ZC7&@S8lGZUbQVfGpnca zvF&mB2{99^6 zYps{skUW-$))Svh&!rMZqPv)a0~3yncmiZ(jhwK2-#wW$rW1G|rOz17H5o{xXY^#s zxswK_(-Zo1GMBfw+t6<7{9~w2ym

7#v$;StB!VWMjE0BesxCCFZiZOnx!8a%H`= z0X3gknaE*G>Dib*JDW)#PtNE$V>%yOn3&6Cj8rae%*;-w^M(;OvbiF-7+ZOi6=Gvx zBV!y+X5j#BBw~)(*kn2b)|1&-I&)Z0C2!UX$J1Cc+ZS7KMl7~#H=_64MlsPOg8dSufKY6|(urPnRf8mG5++RYvK&Ok z_S2W48V^g6;JD`_RT2sa!-dMNwfP?OMW|G!3|R~H?o+uA!iX(Mh`8!qhKOQPO;1spD1Y%1es|U(LY~6 z*GEd|;F&5avd@w(BeH$Vttee;6CftSZsbkZWkj~???9EPWY++QV(7`rZEeu|1RQA@sdkGhh7@L1n6LM8is3;nh|u%Uwj}#1 OEFes?_dkk$5d06wHvyyo delta 4894 zcmai%du$ZP9mj8P=I%_0IrA{K&o(|28^^(3&wFRV4%ne^204#HF?rB(4N!^XHcwYg{}Qi9|N5|DXN4_jni8#$McYELpcreRusW z>Z7h#ebhEqtIyT2#!A#|0|T(;Y-{=@5Jju4b4wfRknAQXjl~*D)=;Kn$|N)C4c6tQ zD;oU_6C#Y8J7W>N$?C7KwN5Nc2iqW%)7!6CZFZL1{9)PZKpP|^Gq!c8He1VW`f67N z+Ym{`@G8~j_HrAmwzJmf2bls(YQ0t4@9zaNnJ~;uoAp+GgEg=`6;u+5gmqx~jRB<; zNU+SxvhsJ!Q$da>6PeaB@2pGteItVrDK@8-?{&)ea%-}#v9b--O<_ucx3s7>{pB|2 z>zXPZx@{;%(z;y7D;+4pB_%l1tQ@4Q;rfOEX9gua(^Tfq)Hej)OiXSf>&>fyniLUfJ(eN?qFzae^{Z4{kQfXZ)}qGX+oTW%v2JN>4QLFBFtZ+PC_sd%=E2(D#DYX1X7L=V}bBdqN3E z+?hL&Mc18Uv$}kn7Dwjyp{>l%) zymQyfpda|9svt-v!UDQPP*=Ls>3jw3@kPbDsdg$#f>MvDv48#h;JB}cSeJxTtkSe% zsjD7(gb_ytJOY-v2bZ|^M&#P@YX%ZSaR$e=H`Ls`s}@L{ zU%O|jn9JujWGC}oliA%P`9AlHLUFJD>F=&Rh5Sek_uX9@9m^J{y+_h_cVTkc)*gTZ zcFQ_=&4!My-KE@gU&rjH&F0eM*}_CVmu{Q&{>Y9_&ZQ^vx~9fQ`U*Mw z<#lkCy|xXKg$w%2c4bJDKVL5i%|=k*4!*8+=S-6&Xu)EF@*2 zqv;aCHsy=`gZ7IL!xgr(3jW$3R2?eRei;^25d#~}>mBelU+NKEQWIktRC$Cl=eZkT zZ9pXqidpC~afyN4T(5ILH~ctOX_M*d+@-dzU_g&B>gW^_-{Mk7YHk=|Ha((jJ}%&W zKAY;8XEMZiFqTIIclu0tIG~cm5bC%TGJ7sz!uen={H<>_Qe6m4%+NVkgktP`)CYI? z`o(q37{fRi%OgZO^ZoFWFBa2H!^Av9GyM!=T&zz$}u-7LQIx!6@ zLMGB9(y$-h3Ok*a+hDJ+*Fs&!h&odTqk4oeM@jGUskpj27;cURyn2LkH{cd$*LL`! zZ*sUU5tU(!-y22mSKm{*rb;(YrqcqE{r zND>)RF1W4;Ma;SLi|{qy>NRv#D-a6Vb38)0y>E}2!|5D6ul>(yQEf^>-MQ2n4q5mf zkLfX%dLgGBN4}RGn>XuyR-p zoiTMs2pLw7P`jJLl>Napyw0hb4vDPp@T#}Lf#q16^{^N7rz0o_(e!^6lTy=-B+hk7Hv2MM-35p` yriN6-G9j2a7j5_(U&2UTox3; Date: Thu, 17 Oct 2024 16:01:13 -0400 Subject: [PATCH 24/36] update to Python 3.12.6 for Docker image and testing workflows --- .github/workflows/qa-suite.yml | 2 +- .github/workflows/test-build-pipeline.yml | 4 ++-- .github/workflows/testing-wip.yml | 2 +- .github/workflows/unit-tests.yml | 2 +- Dockerfile | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/qa-suite.yml b/.github/workflows/qa-suite.yml index c27f2496..b802324a 100644 --- a/.github/workflows/qa-suite.yml +++ b/.github/workflows/qa-suite.yml @@ -28,7 +28,7 @@ jobs: - name: Set Up Python uses: actions/setup-python@v5 with: - python-version: '3.12.5' + python-version: '3.12.6' cache: 'pip' - name: Install Swirl run: ./install.sh diff --git a/.github/workflows/test-build-pipeline.yml b/.github/workflows/test-build-pipeline.yml index 15a69615..13d0ae3f 100644 --- a/.github/workflows/test-build-pipeline.yml +++ b/.github/workflows/test-build-pipeline.yml @@ -20,7 +20,7 @@ jobs: - name: Set Up Python uses: actions/setup-python@v5 with: - python-version: '3.12.5' + python-version: '3.12.6' cache: 'pip' - name: Install Swirl run: ./install.sh @@ -53,7 +53,7 @@ jobs: - name: Set Up Python uses: actions/setup-python@v5 with: - python-version: '3.12.5' + python-version: '3.12.6' cache: 'pip' - name: Install Swirl run: ./install.sh diff --git a/.github/workflows/testing-wip.yml b/.github/workflows/testing-wip.yml index f2feefc1..55eeacbe 100644 --- a/.github/workflows/testing-wip.yml +++ b/.github/workflows/testing-wip.yml @@ -41,7 +41,7 @@ jobs: - name: Set Up Python uses: actions/setup-python@v5 with: - python-version: '3.12.5' + python-version: '3.12.6' cache: 'pip' - name: Install Swirl run: ./install.sh diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5dd32025..a4176c1e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -31,7 +31,7 @@ jobs: - name: Set Up Python uses: actions/setup-python@v5 with: - python-version: '3.12.5' + python-version: '3.12.6' cache: 'pip' - name: Install Swirl run: ./install.sh diff --git a/Dockerfile b/Dockerfile index 621ffe1a..bc2d6bd3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12.5-slim-bookworm +FROM python:3.12.6-slim-bookworm # Update, upgrade and install packages in a single RUN to reduce layers RUN apt-get update && apt-get install -y \ From 3a3cf02823a35ae965eea906913f66695d827369 Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Mon, 21 Oct 2024 13:49:57 -0400 Subject: [PATCH 25/36] remove the FILE_SYSTEM entry from OneDrive SearchProvider by default --- SearchProviders/microsoft.json | 2 +- SearchProviders/preloaded.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SearchProviders/microsoft.json b/SearchProviders/microsoft.json index 79dfe513..73dfc96e 100644 --- a/SearchProviders/microsoft.json +++ b/SearchProviders/microsoft.json @@ -75,7 +75,7 @@ "CosineRelevancyResultProcessor" ], "response_mappings": "", - "result_mappings": "title=resource.name,body='{resource.name} - {summary}',date_published=resource.createdDateTime,url=resource.webUrl,author=resource.createdBy.user.displayName,resource.lastModifiedBy.user.displayName,resource.lastModifiedDateTime,FILE_SYSTEM,NO_PAYLOAD", + "result_mappings": "title=resource.name,body='{resource.name} - {summary}',date_published=resource.createdDateTime,url=resource.webUrl,author=resource.createdBy.user.displayName,resource.lastModifiedBy.user.displayName,resource.lastModifiedDateTime,NO_PAYLOAD", "results_per_query": 10, "credentials": "", "eval_credentials": "", diff --git a/SearchProviders/preloaded.json b/SearchProviders/preloaded.json index 8521876f..6dd43257 100644 --- a/SearchProviders/preloaded.json +++ b/SearchProviders/preloaded.json @@ -643,7 +643,7 @@ "CosineRelevancyResultProcessor" ], "response_mappings": "", - "result_mappings": "title=resource.name,body='{resource.name} - {summary}',date_published=resource.createdDateTime,url=resource.webUrl,author=resource.createdBy.user.displayName,resource.lastModifiedBy.user.displayName,resource.lastModifiedDateTime,FILE_SYSTEM,NO_PAYLOAD", + "result_mappings": "title=resource.name,body='{resource.name} - {summary}',date_published=resource.createdDateTime,url=resource.webUrl,author=resource.createdBy.user.displayName,resource.lastModifiedBy.user.displayName,resource.lastModifiedDateTime,NO_PAYLOAD", "results_per_query": 10, "credentials": "", "eval_credentials": "", From c146f3fdd14a28f88818b1ec1bd87508247f3a67 Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Tue, 22 Oct 2024 10:52:34 -0400 Subject: [PATCH 26/36] fresh db.dist after SP defaults change --- db.sqlite3.dist | Bin 303104 -> 303104 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/db.sqlite3.dist b/db.sqlite3.dist index 581623d70ab6700b38215c6af8449a0286e43e3a..d29403dc7a882e93f97a9344e0f37442d38d0702 100644 GIT binary patch delta 3800 zcmZu!S!@+m7;bw{V>Q;4(&cu!T?C3PbUgb47`#Q{LQ`u6TdWEyMQm!NH5hG-1~-hT z5FKTbCSrIo#w8Io9z{eFAACaNgG-``TcS?{;}&^v`DdoFbLP(L|NY-N-@ne8{io~q zpRRx7)ur{6b`idYn>B)a+;C!)|H0BO*Q|un(m%I!Sw88_xRQj7^i@}oA-|Y!^w&4^ zq#PvE=9*q`*c3TD+mJp%NGxx-3>N-axBiR46abtsHq()#_TJC?AZ}A733Qn|)D#?sL+f+<(u%V)KX~g-qso;V{ z5~EAq5!l;Ya5WN4l-$w<8rOryCVyXZhvPu8WSk9lf1V{q=R+_e4YWbW`p;x_4lM5>{{>2$=1G4{P~^lMU}3^Ry@%%X&dV)o1B=$T5g zC{TEs8(}hHiV#W3rrmcHHwVWbM;(FfL0=~n3Qs~0>x}CnL}F}Dtwg7i3I(K)Ld%vb zj1kK0%Y7*CQYa_{$z^#EUWiZ;Ja{tKY;*nS-|B=q1*oEum-AvoW9u;_at;M^&1Hr< z<%Ni$_OL_`Cp%C={-6gl3LGQCt~2PdWIWQ7+$c=Sb;Ssz;OtuTU1dU-gsj09&d?Pj zLfP|Ipc|8+Xo#L;oYCSjLWSL5M873((Ga5`P@l3y$A~cY#SLgnG8P5VOT{y46C*+g zkmK&)!$EY3y=o&GN=j6a2rXGAIYc=BpImQ$*^Jh@WX`q58PlP>5MdFS+b4(6{Z)1I z6}5j{JaExe&8q4Xxyvf%{n@%lgG1M%b%`V!Pbyf{rab%@kuo^A74_QA5wt#;1LJYY ziOdKiMwGF@+v!p$j9k`e8yI>RM;%@@Xm1%se2elLPRCW z=kG_`FG!Fn2qs)*2#ygW?E{aX+2nl`#Imf#Ct`$f+j;=ix$&7WzVeL>X$cWx?C3#M z<5obyC}w;qfD$5FS*XCf7c4$H1oNnYuFB}E7~#S6^XSh+@C+JCNR!b}5uz}&hmWAH zWbaaVV!=OHMkEmeGi1LG&^KypwO6D5N*d_ IKrdDO2T?g%#sB~S delta 3875 zcmZu!S!`5Q7;ZZUaVr9yc7}GQiz3K&J^O+foXXOowN{Xd8>tpWO{EnVkeGl9#tjlr zbP^LZK4?sgMl|unEg><+Cl?;v;)BbB#s_y}VvNgw?;X=~?%aoY`M>`=|Mzd_+}VG8 z;r`iW05)P&o

7tAdiUl)6r34)4u7n2?O_;Ew-K#QY!L_l6yG6!KpiuVX?xh(s zW(?bTJ?DmIY#<*y+Os$?D`SMF_F#|Bm?fvwe%OOEW(rd5f(i@G+DK*W)`|+ss8 zK2=c}XNqBazN?%Unw26(TV5p@Gv|ccyQ(x}W`-&IW;LTv8qJmcyPDA_6%oP?_GXfl zj8SbL$(jXJO4v_&O&C&8siht0Bfb7CDsGG-xG<-sbariGa_XjA?K6GFkO^B&n0=zJ zFJmTxNc&!2*gwSt)#R1_herq3sdd|?#TDawZkW7&&)%JEY}a*{ZMuB%#Kt?ub`9RO z$dFKvSZ@zJ4SD&x9z@V_wF(K{;q-9pn^qM60XU; z(}$*84^6kdHeDE~t!$(tS5p?jCNQokrhFB_tFnmX^?|LZ=q8R8M%w}&V@g0D9uqLck+DCgt&Bgf)0yy27h|JQ^9;JSm!aDWXNJv%{Tuqi~?ri$^2HF;O`> z6NF-t@amv*%Kh_J;mD*%=8>Wd>zv{w2qDtFK7fApb@E71PE;->MhuSQmE4bu(cP_Z zK=SlJA9%zljB-wXF`}T*8+Y7V8?GOF0QJ`067)qNlTgDbX|4^AVuYtTx)V#$3%*ts z8AhmN&Ab%R#67NvNyuyq>>;KhVbbN(I!6^ z3E3NX{F-w~5tN~+RL19Fg>hz!x3=1hKya5LND zI<&aMJv@T;waq`=(*FJF^3?4o_sw~!P-!`AXXfvzKY1xyT2I?BeDV72o40M=vVNpKvmW)hZ?_|c zypv2siIRE4nji|Eg`GDb?)FZicl?x0gyHKlPeF=s>PtlPkaChYtJVgYrzc_`c zduk_|@*AZi2_IdAH&-u37)^OM-4b4XI)d98FVjGJ3~tnBnJFS!EVJy^+>R#04#0blMUG4ZiF2%3V-5Eq?9SK= z*JjW@PJ~oBwK0&rkR~O&kM^Mhb7y&wj>bylw6B2(?(Vq{&3K^zkQkn(<{@q%B6-j| z-MNR*U*7cuC?{}FH?MR9p$V_#rXE7Sgz?}F;jB3|;Khh&qz27+cCOb$0;w4;ny&(u zBC0<1IQqk@1%bpw{6U$$KY;)U+^p`%6R6)CWdJya|34;=5+gj(R&u}A(YxL&2RsU2 kCn9IQQbZ>6<^FgQX}=UC!thZjauvpiH15iy=()E40I(KqrvLx| From e58389f07c71c7fbc516f3947a612545c77578e5 Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Tue, 5 Nov 2024 10:49:15 -0500 Subject: [PATCH 27/36] update workflows with new secrets --- .github/workflows/docker-image.yml | 8 ++++---- .github/workflows/qa-suite.yml | 4 ++-- .github/workflows/test-build-pipeline.yml | 12 ++++++------ .github/workflows/testing-wip.yml | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 4592f4f0..dcf2bf72 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -14,8 +14,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.SBS_DOCKER_USER }} + password: ${{ secrets.SBS_DOCKER_PAT }} - name: Build and Push Swirl Docker Image run: | BRANCH_NAME=${GITHUB_REF#refs/heads/} @@ -25,8 +25,8 @@ jobs: - name: Update the Docker Repo Description uses: peter-evans/dockerhub-description@v4 with: - username: ${{ secrets.DOCKER_USERNAME_X }} - password: ${{ secrets.DOCKER_PASSWORD_X }} + username: ${{ secrets.SBS_DOCKER_USER }} + password: ${{ secrets.SBS_DOCKER_PAT }} repository: swirlai/swirl-search - name: Upload Log Files if: always() diff --git a/.github/workflows/qa-suite.yml b/.github/workflows/qa-suite.yml index b802324a..04e022fb 100644 --- a/.github/workflows/qa-suite.yml +++ b/.github/workflows/qa-suite.yml @@ -23,8 +23,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.SBS_DOCKER_USER }} + password: ${{ secrets.SBS_DOCKER_PAT }} - name: Set Up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/test-build-pipeline.yml b/.github/workflows/test-build-pipeline.yml index 13d0ae3f..4bafa0a4 100644 --- a/.github/workflows/test-build-pipeline.yml +++ b/.github/workflows/test-build-pipeline.yml @@ -48,8 +48,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.SBS_DOCKER_USER }} + password: ${{ secrets.SBS_DOCKER_PAT }} - name: Set Up Python uses: actions/setup-python@v5 with: @@ -104,8 +104,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.SBS_DOCKER_USER }} + password: ${{ secrets.SBS_DOCKER_PAT }} - name: Build and Push Swirl Docker Image run: | BRANCH_NAME=${GITHUB_REF#refs/heads/} @@ -115,8 +115,8 @@ jobs: - name: Update the Docker Repo Description uses: peter-evans/dockerhub-description@v4 with: - username: ${{ secrets.DOCKER_USERNAME_X }} - password: ${{ secrets.DOCKER_PASSWORD_X }} + username: ${{ secrets.SBS_DOCKER_USER }} + password: ${{ secrets.SBS_DOCKER_PAT }} repository: swirlai/swirl-search - name: Upload Log Files if: always() diff --git a/.github/workflows/testing-wip.yml b/.github/workflows/testing-wip.yml index 55eeacbe..1847a6b1 100644 --- a/.github/workflows/testing-wip.yml +++ b/.github/workflows/testing-wip.yml @@ -36,8 +36,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.SBS_DOCKER_USER }} + password: ${{ secrets.SBS_DOCKER_PAT }} - name: Set Up Python uses: actions/setup-python@v5 with: From c07446ce1eee16b8084f6c02c5664d360a3d0f83 Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Tue, 5 Nov 2024 18:41:11 -0500 Subject: [PATCH 28/36] add new Trello env var to testing workflows --- .github/workflows/qa-suite.yml | 1 + .github/workflows/test-build-pipeline.yml | 1 + .github/workflows/testing-wip.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/qa-suite.yml b/.github/workflows/qa-suite.yml index 04e022fb..a536ed93 100644 --- a/.github/workflows/qa-suite.yml +++ b/.github/workflows/qa-suite.yml @@ -65,6 +65,7 @@ jobs: echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> .env.qa + echo "QA_TRELLO_KEYS=${{ secrets.QA_TRELLO_KEYS }}" >> .env.qa echo "========" cat .env.qa echo "========" diff --git a/.github/workflows/test-build-pipeline.yml b/.github/workflows/test-build-pipeline.yml index 4bafa0a4..3499c2ec 100644 --- a/.github/workflows/test-build-pipeline.yml +++ b/.github/workflows/test-build-pipeline.yml @@ -90,6 +90,7 @@ jobs: echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> .env.qa + echo "QA_TRELLO_KEYS=${{ secrets.QA_TRELLO_KEYS }}" >> .env.qa echo "========" cat .env.qa echo "========" diff --git a/.github/workflows/testing-wip.yml b/.github/workflows/testing-wip.yml index 1847a6b1..0ae28419 100644 --- a/.github/workflows/testing-wip.yml +++ b/.github/workflows/testing-wip.yml @@ -78,6 +78,7 @@ jobs: echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> .env.qa + echo "QA_TRELLO_KEYS=${{ secrets.QA_TRELLO_KEYS }}" >> .env.qa echo "========" cat .env.qa echo "========" From 400a90c5dba51cd25515aa20d1d491b02819e825 Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Thu, 7 Nov 2024 10:54:37 -0500 Subject: [PATCH 29/36] update order of result_processors for Outlook Email SP --- SearchProviders/microsoft.json | 2 +- SearchProviders/preloaded.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SearchProviders/microsoft.json b/SearchProviders/microsoft.json index 73dfc96e..a9f5918f 100644 --- a/SearchProviders/microsoft.json +++ b/SearchProviders/microsoft.json @@ -13,8 +13,8 @@ "query_mappings": "NOT=true,NOT_CHAR=-", "result_grouping_field": "conversationId", "result_processors": [ - "MappingResultProcessor", "DedupeByFieldResultProcessor", + "MappingResultProcessor", "CosineRelevancyResultProcessor" ], "response_mappings": "", diff --git a/SearchProviders/preloaded.json b/SearchProviders/preloaded.json index 6dd43257..84d70ec9 100644 --- a/SearchProviders/preloaded.json +++ b/SearchProviders/preloaded.json @@ -578,8 +578,8 @@ "query_mappings": "NOT=true,NOT_CHAR=-", "result_grouping_field": "conversationId", "result_processors": [ - "MappingResultProcessor", "DedupeByFieldResultProcessor", + "MappingResultProcessor", "CosineRelevancyResultProcessor" ], "response_mappings": "", From 9b97484987be2e113d4b5f09211ba39e3381f94e Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Fri, 8 Nov 2024 16:59:38 -0500 Subject: [PATCH 30/36] update SP tags for dev-centric sources --- SearchProviders/atlassian.json | 4 ++-- SearchProviders/github.json | 8 ++++---- SearchProviders/google.json | 3 ++- SearchProviders/hacker_news.json | 4 ++-- SearchProviders/preloaded.json | 23 ++++++++++++----------- SearchProviders/youtrack.json | 4 ++-- 6 files changed, 24 insertions(+), 22 deletions(-) diff --git a/SearchProviders/atlassian.json b/SearchProviders/atlassian.json index 73684824..73e05054 100644 --- a/SearchProviders/atlassian.json +++ b/SearchProviders/atlassian.json @@ -22,7 +22,7 @@ "tags": [ "Jira", "Atlassian", - "Internal" + "Dev" ] }, { @@ -48,7 +48,7 @@ "tags": [ "Confluence", "Atlassian", - "Internal" + "Dev" ] }, { diff --git a/SearchProviders/github.json b/SearchProviders/github.json index 19317927..bf5f298b 100644 --- a/SearchProviders/github.json +++ b/SearchProviders/github.json @@ -28,7 +28,7 @@ "GitHub", "Code", "Internal", - "Development" + "Dev" ] }, { @@ -60,7 +60,7 @@ "GitHub", "Issues", "Internal", - "Development" + "Dev" ] }, { @@ -92,7 +92,7 @@ "GitHub", "PullRequests", "Internal", - "Development" + "Dev" ] }, { @@ -124,7 +124,7 @@ "GitHub", "Commits", "Internal", - "Development" + "Dev" ] } ] diff --git a/SearchProviders/google.json b/SearchProviders/google.json index fe768419..09972251 100644 --- a/SearchProviders/google.json +++ b/SearchProviders/google.json @@ -152,7 +152,8 @@ "credentials": "key=AIzaSyDvVeE-L6nCC9u-TTGuhggvSmzhtiTHJsA", "eval_credentials": "", "tags": [ - "Swirl" + "Swirl", + "Dev" ] } ] diff --git a/SearchProviders/hacker_news.json b/SearchProviders/hacker_news.json index 880ebefc..fb9c2564 100644 --- a/SearchProviders/hacker_news.json +++ b/SearchProviders/hacker_news.json @@ -26,7 +26,7 @@ "tags": [ "HackerNews", "Stories", - "Development" + "Dev" ] }, { @@ -56,7 +56,7 @@ "tags": [ "HackerNews", "Comments", - "Development" + "Dev" ] } diff --git a/SearchProviders/preloaded.json b/SearchProviders/preloaded.json index 84d70ec9..92abe197 100644 --- a/SearchProviders/preloaded.json +++ b/SearchProviders/preloaded.json @@ -155,7 +155,8 @@ "credentials": "key=AIzaSyDvVeE-L6nCC9u-TTGuhggvSmzhtiTHJsA", "eval_credentials": "", "tags": [ - "Swirl" + "Swirl", + "Dev" ] }, { @@ -480,7 +481,7 @@ "tags": [ "Articles", "YouTrack", - "Internal" + "Dev" ] }, { @@ -507,7 +508,7 @@ "tags": [ "Issues", "YouTrack", - "Internal" + "Dev" ] }, { @@ -533,7 +534,7 @@ "tags": [ "Confluence", "Atlassian", - "Internal" + "Dev" ] }, { @@ -559,7 +560,7 @@ "tags": [ "Jira", "Atlassian", - "Internal" + "Dev" ] }, { @@ -740,7 +741,7 @@ "GitHub", "Code", "Internal", - "Development" + "Dev" ] }, { @@ -772,7 +773,7 @@ "GitHub", "Issues", "Internal", - "Development" + "Dev" ] }, { @@ -804,7 +805,7 @@ "GitHub", "PullRequests", "Internal", - "Development" + "Dev" ] }, { @@ -836,7 +837,7 @@ "GitHub", "Commits", "Internal", - "Development" + "Dev" ] }, { @@ -967,7 +968,7 @@ "tags": [ "HackerNews", "Stories", - "Development" + "Dev" ] }, { @@ -996,7 +997,7 @@ "tags": [ "HackerNews", "Comments", - "Development" + "Dev" ] }, { diff --git a/SearchProviders/youtrack.json b/SearchProviders/youtrack.json index 6d3f6a9c..b7c81972 100644 --- a/SearchProviders/youtrack.json +++ b/SearchProviders/youtrack.json @@ -23,7 +23,7 @@ "tags": [ "Issues", "YouTrack", - "Internal" + "Dev" ] }, { @@ -50,7 +50,7 @@ "tags": [ "Articles", "YouTrack", - "Internal" + "Dev" ] } ] From 5181c1c79ab830ed85b87bca697523404774ca19 Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Mon, 11 Nov 2024 11:20:10 -0500 Subject: [PATCH 31/36] temporarily have the QA Suite workflow pull the Preview image for pre-release testing --- .github/workflows/qa-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qa-suite.yml b/.github/workflows/qa-suite.yml index cee546d9..1d1557ef 100644 --- a/.github/workflows/qa-suite.yml +++ b/.github/workflows/qa-suite.yml @@ -35,7 +35,7 @@ jobs: - name: Setup Swirl run: python swirl.py setup - name: Install the Latest Galaxy UI - run: ./install-ui.sh + run: ./install-ui.sh -p env: MSAL_CB_PORT: 8000 MSAL_HOST: localhost From 86cbbc773acdd154c4eff7adf50c3f195fd34b7d Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Mon, 11 Nov 2024 12:52:21 -0500 Subject: [PATCH 32/36] clean db.dist for the prerelease branch --- db.sqlite3.dist | Bin 303104 -> 303104 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/db.sqlite3.dist b/db.sqlite3.dist index 73efcb3c707047b89b00eb9782d4553fa2ff8e96..bd59bf664d09b01d7dba2fc9aaef5ad12c32d443 100644 GIT binary patch delta 5517 zcmai&du$xV8NhdM=WNIJ`Rv5@`E~Z%juV{P>)oB1eK;}6nS|I$OcIhfBq1CZ-`c*$ zzO(PnPL1o@2}uyIlCUi!(efx%Euw{%AhBvF5GwT_N>U+#XnCklwT%GrkdSBvYG>~f z;@e%RCwp~!zxf_B^F3yF>B7pT3oFkwR;3w+*+&1beome3f^7P^+s_0;QTp3;H`7aK z1HIHV2I#e$i<`xj2Yg)o`BM3iI|?_TF; zq#-j>S7UChZg-ARbzRZ%O`UYi*4&r}t6h3jT~kH*z;$%Y;oO+NR)52-2PsI~yOxgG zlN-}lv&KC}R5Y}I4IOhJH^!{#bk0WAkO*b-jhaZvF&kBr6%h}tHs7eLH~VWF+>)ZI zE9P|V^)5*jK~WiK%SryJw!y^}U6tTKYmRq@8eD!WP|=lbJoS4c^}E*0h8mq?6opzB zx3|zSk=&RIp(dw?3dW*>%}+zv>4Abp1hs)M^`ODrUsvzqP?uzFpee_HR99c^SQ|x_ z6v@0)r?}jbL`}uR4b-juIk#~ATBln`*Ca_Yr|VsAAzjg+K2Vo4`*yv{Eu>=%(LgB2 zTN~=#njzG5b7#Xkr^84WRn@$|0lGw}y9k+YHLP|Mg(iks(b(lC7)gq`v$54hRJtNL8jBu0>P)4vWgFw;N{?(L4WX-;Jv|zgPVf!U{A2Q;)CZ~ z`HK}kn!>>zsV!K;b(G{vax`Pa(j()+RbT&*ugvDAo>$oJbK<=l`%}}?Hy+)VI66Ff z?2hc<)LuD$WFRxUIw{?;^X41kqXVhz_~6{Yus*vZb7ap=`?C8+w3+FVku8!eZihoV zrUsFDG1XkKJ4g{^MWriuV(Ro1`_PmRu;rG2%J*@ZkJ)R+DtB3}nEN(s{bs>;{S)UN zXDdTRk1!=nf?*P+k&?%Y|5LoV=-HrJah8tVZM{Cnp`xm?d8Uf?pKBj0mf=2QtxspU zcYT0ixLVqfR&v&7(_D&WOD%R2_f?B;gZ~Ad^H(;4FPXB|(jSzr@SmVx$I9ySdWy`l zdb)D&0q!_AN|Ljj!a{O(hN~vOn&q_X?gmD2sH1~DyFD8_GGQ!V(dFNJ$&>53r%C4m zS83h1fy;2@zy@x$eIEdw$Ad@6@C?_>uD2BrvoZ3o8EOXGca5ZXmXk{FzV-H;nFr^| zPZciA_LJW#+&d&e^^pOMUY^jnHX`Wx+n^!?_Rmpdnz>;-qHF%#b>f+lr8?)B?&7hqnheDv=sSMFhE!h=t`OzTfkWDbO zl^wP^>%gdQ+1VjCKvqR0%U+^3v3}hEOx9^mR745uUNSb(G>e)$!M$*S1G}CCOO=&H z><*?pUD{tfRD?j9-9e*FV;4`im$fkKS41KuC4q3@1g@%z9FGM8$NeXFvVkq@tEyJq zkeZx|CFl6fv22X*=KB+4yJw8_oXap21qtdhr0L{Xfhz>ztFjqm!WhZ&^P>r}zYUC# z@3ny{8m~)jpqiX-19(@zi82k<8!0zKPUGY$B1JBkW1AgZ%eVP$w=|zrfQ&9iK7Mvqpx`jvM@ZA{n2_ zWYcpC{MB#dd^-p)UR9Kr7>{N6*wj=y^{vEYENe{6@$(}y>9mo|Mvcj-iPW57M2$=~ z2N(FOKV<}dFSU_2#u6ENyfor`-dTP$m8RAc89tRBizO3x#IlK0l22rM`T6{ag|@X_ zGwF%Gwd2|BRHnD5=h(4h!u&)cdDMs}l2M8l){^$s;CeE-8muo27G2Pxr05=jo+GNP zT5qog-(^ujis^zXBN4mzQ_d4b-fAQxgXFopKr^w{fIm2mVo8%^dMb8=0~NOk60Mil zf){;GDcUqVb3qqu0z0t88oun$vZOWDDVhMf~3+zp*uUT5LL1S2>#;ZXmubV1q>F7i-N8oq`Q}a zW(6hmylFk4fW^WIQKn+eehhIg-W4M0R-X=zyQDCdY5(_ zu$WOsk0PS(;X#hj2qcf)01lB4BY-D2cYsaS!OdVB8*W?X-3UB7x(P zyL2|NxP`pB#l4j(t zWH&JeL1*EtB>`(l)7;I7>ZL?qrgx?F$soAAOz-I`a(M^UyOsEdK)X{fR250}3_Y7@ z*!CA$_YQ$EU(467+K;!z%y}m8d3k^7(Guu?joZYWr;YT-zEd1IvkNq^hsn=&fu=%h zP*4?3)Z85m=7=Opztm@hb<&~lmS_= z2^OtGw*iAC&mX3H_>U;y%im)HH<_RJ{p9Ii)bR2xsQ9zL(X;7taYh1gLCs-uU zOaj?T9CfxJnj^ZZi1cC7C7LIiWKq#cu5gxEP#`^MdKkWiU^$X8T%aZODQca9WjVUxpq5f(4skWY?8?^3fbk zaF9CK9!5$9&=TBbZf z&>Fr6__?lvo`~t|JcN-)2;~U*^TVLldgOlagkxqZJ=%y^^(ZHKqDv(CeehY~a;O3n z={q$~A)8=iJ@60+J1U1Nh&rtg9xmpHCQH`41pLvph?+)U!g(&eO*Gm1+rwbMC8eme TF?bYSMX(8ytb>n$pZNX<^Q*A zjD%&c?fvd~f9IS#d-T$(qnB2lsaZMAFw9o?x$$-Hqiuwpe&NnD{=g1+@22hW64(GQ z6*WG1Z9}i<`K7}b8p0z;f2Ax`!D;P+80LlblDwYRB1F_Ujp(10g=_4HAWM=sxYot_(UQB_gW&bmc8} zNkLO(mGm}dCEu$II&q?c=7ucZ8VK6`7F0zh+}1Go-35ME>ghm@{S2pSL=^h#;Y>7p z=2D>6?jffVQPK2I1CreXPEi#>>QB|$!U1KfAgH|lL9ObLfs#v#E`}mb zL>3fTKNRY4A`%ot|7ED%i3p+~>tBR82O?{N!0Fv}&J@UyPfdTYPI9C`7C2RZyH0Tc zlB9`xxW3~^4gn!fi4axfke3NGM`L4?i6qBy{Jumokxs-W1Wpj!?GmCUaHPZIWhy`- zlRoh60upqo!g`tJ=BWNgeW=5(k!Y$Y`25c@g?=C7|ImNM-{ODO|A7B-|3?3q-*;i< zoqu29sVdVyitzfzj#t^X7r6(zx-+A5J)`ls(sNK7yLV@d=aX{>w#Qa;=~Sk7*KF!w zZ%^jCVsuna%^%u!_x721&-}VA!`$TH!TI4{zI#xMru5fSbvbF5l!!tkUQ~`votk1F zoAO|`(C|)qJ}LIFZ(f)#z04Y~EqZP_ckWE-V^hdm=51i|?)3i7+fcla_XLc5#&Z?M zeWBObeAsiCEjqn-M}DBWnLRs4*DatBE74~-R8B`0&~ItyJ*b3UJ%R!rNv59@6rztG zMy1BBhtX+N+UU2mP*nn9}Zw2H2J zj0A_oymM#5=bkQw@3T+ipW;jCwiRCH3R6;2G?2d+?Pjj%)1|M`*PlmQzd-!chBijuGnt<}R<8ouq3+OYpj=^DOyf7LqEc7+SMpjltmztIo-%{7HS=KjNPp2** z*{)uZB~Iep)SHMxO!Z~-Jp*l}S1+O_W9Lig9oBgFSE!MtdkxfW*C)x6N?hGF5n0IU zGeF<-9=j-!p?i2YS4>1yEYY`KM)$F8Uw27173K~WT}1(*9Kn^@5DW;R8I_qAkUaKv zrevn5JAWWg#4{{i`8>L)A1(b~{mI7SdS(!kSWw^#`IZtZS4NjQe7+^`@eu3VG(?~2 z#Gwt{iSfZqd}hw+tQs-bhdBmvrUQ6sA&}8uG~f)~(TJs+_h*u0@v)v*I^LF!-8B*K zG~Y-h$0N(X+x92o6JvbmXlimQmYlQx6*@-~={d@d;BH#qg^PMx+U`n?&2_dcEFW78 z&%_e5@v-pg1?!L4WO^|?8=pzUXTu9KDX_5^P9?(&miaL-zeuN>a7Zu}FNPOp)8<>@ zSaK|UAO<4I%w&9WYIZRUuM?@cczia}(w3Q-=uC{!m%8v8dh=>56t{&N`;zJSOfoh> zCsyMPx#d-W$Z(l($%}<3St19Fk5}VEte8_?m^-p6N{Tb37NV$I;>lvukmMCt~v#NGFJ}ZtN6LA|@hH z&TQE)+i*TAxybmA6(1>hBQNB+2(#yBp1Mb zfSYpz)J)28n&`BbMR?w7I{M}U&ZB?n!v%D40GAo#1V85~_^M<98TaMtg#`$D1<9$` zLPXAZM8-#KIa4Ew0(;YC1Xzfu8l4(ma!5(ATx3^UCZZAJqnq(NY;LC22vJ}|yO7F4 zWaG0=yuwrBV@f_K+~WN)TF3f~hK;z3&54w7P!~nyR7a%Ch#(n{MDev;DM2$S&5JI? zvJhFM=WoS(DH_2+`kyF1Y&_M2%h^yv+1K@5^7otdjrx|N_iL>0$aebXCTD#WS<{Hi zc+;RWG)=PAx6&BejQ`X_}zv^tjg%Irx6%Kg zdKX@`DxZ0n@qJh#7ws#k_nt?!%)_ue@AaH4rWbF+!EKY7m4&yaCMOf=*>HP!OCsHy zxhn#r(OFy`B9vs&**L^PWZ9eytw#SKeuXWFPo$VCb5aUw35+t{$=79sWJgiI@%a$`k()u!i6p(e69@YbLX3U0 zvl$rZ-xeaT$b#(TC|EQRRWu;P-@6$Lo#?C%F&)d%4Qe-{J2nK-jnmt9 zZ)kXE$EE>#bQ7+j@9)FmlFN*5bJ67jkM|n73>V;+k>^AMeRmWG*?si$QQV$uOn?9s z5gu(&^tma_Tx6YUIN{wIJ=n$KuSf>L<96b z_>H>!cN|CUI(V~VT@oPxnG`_~>2s4= zg~F*WR1{9Mx_ z)Io#iY^P}p(PU%sI8HiZ)l{CiagIPJJS5W0RfW-U3V-HEv)LKM#UP28h?Grpz?l3# z{@7tvQsG&_W%`nciCE*J%zz4=vxO-|cukgN7cyIjqR`|I@O67`H0V{q#Z?i=sfr>o z9(fFhYy(AvVyK+xay76JYexi(>lDA1dk-Wcv@(2z;6gzQS-uDU^KmTMQX@i3YLeU3 Owh%=&_B?@q;`u*#MK@mn From c8642f8cb16330678afdce4ca14e111167c4867e Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Mon, 11 Nov 2024 13:59:41 -0500 Subject: [PATCH 33/36] temporarily set the QA Suite workflow to pull the develop QA image for testing the prerelease branch --- .github/workflows/qa-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qa-suite.yml b/.github/workflows/qa-suite.yml index 1d1557ef..2953cefd 100644 --- a/.github/workflows/qa-suite.yml +++ b/.github/workflows/qa-suite.yml @@ -69,7 +69,7 @@ jobs: echo "========" cat .env.qa echo "========" - docker run --net=host --env-file .env.qa -t swirlai/swirl-search-qa:automated-tests-master sh -c "behave --tags=qa_suite,community" + docker run --net=host --env-file .env.qa -t swirlai/swirl-search-qa:automated-tests-develop sh -c "behave --tags=qa_suite,community" - name: Upload Log Files if: always() uses: actions/upload-artifact@v4 From 7b0df3a58d603af83d2a136dd8d369005010877b Mon Sep 17 00:00:00 2001 From: Erik Spears Date: Tue, 12 Nov 2024 12:36:11 -0500 Subject: [PATCH 34/36] cherry-pick ES size param fix from develop into prerelease branch --- swirl/connectors/elastic.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/swirl/connectors/elastic.py b/swirl/connectors/elastic.py index 4cd3134a..3f5612dc 100644 --- a/swirl/connectors/elastic.py +++ b/swirl/connectors/elastic.py @@ -67,9 +67,10 @@ def construct_query(self): self.error(f"elastic_query unexpectedly blank") self.query_to_provider = elastic_query + logger.debug(f"Constructed query_to_provider: {self.query_to_provider}") return - def execute_search(self, session=None): + def execute_search(self, size, session=None): logger.debug(f"{self}: execute_search()") @@ -129,10 +130,18 @@ def execute_search(self, session=None): else: self.status = "ERR_NO_QUERY_SPECIFIED" return + + # Extract size (int) - Optional + size_pattern = r"size=(\d+)" + match = re.search(size_pattern, self.query_to_provider) + if match: + size = int(match.group(1)) + else: + size = 10 # Default size if not specified response = None try: - response = es.search(index=index, query=query) + response = es.search(index=index, query=query, size=size) except ConnectionError as err: self.error(f"es.search reports: {err}") except NotFoundError: From 99feda98d7b016ecefdd5bfd54480ad5df9ae481 Mon Sep 17 00:00:00 2001 From: dnicodemus-la Date: Tue, 12 Nov 2024 15:40:06 -0500 Subject: [PATCH 35/36] fix for DS-3230 --- SearchProviders/elasticsearch.json | 2 +- SearchProviders/opensearch.json | 2 +- SearchProviders/preloaded.json | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/SearchProviders/elasticsearch.json b/SearchProviders/elasticsearch.json index 00e4a5a4..57075b9b 100644 --- a/SearchProviders/elasticsearch.json +++ b/SearchProviders/elasticsearch.json @@ -13,7 +13,7 @@ "MappingResultProcessor", "CosineRelevancyResultProcessor" ], - "result_mappings": "url=_source.url,date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD", + "result_mappings": "url='https:///email/_doc/{_id}',date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD", "credentials": "verify_certs=[True|False],ca_certs=/path/to/cert/file.crt,username:password", "tags": [ "Enron", diff --git a/SearchProviders/opensearch.json b/SearchProviders/opensearch.json index b20b6848..916397c4 100644 --- a/SearchProviders/opensearch.json +++ b/SearchProviders/opensearch.json @@ -13,7 +13,7 @@ "MappingResultProcessor", "CosineRelevancyResultProcessor" ], - "result_mappings": "url=_source.url,date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD", + "result_mappings": "url='https:///email/_doc/{_id}',date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD", "credentials": "verify_certs=[True|False],ca_certs=/path/to/cert/file.crt,username:password", "tags": [ "Enron", diff --git a/SearchProviders/preloaded.json b/SearchProviders/preloaded.json index 92abe197..07dc77cf 100644 --- a/SearchProviders/preloaded.json +++ b/SearchProviders/preloaded.json @@ -423,7 +423,7 @@ "CosineRelevancyResultProcessor" ], "response_mappings": "", - "result_mappings": "url=_source.url,date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD", + "result_mappings": "url='https:///email/_doc/{_id}',date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD", "results_per_query": 10, "credentials": "verify_certs=[True|False],ca_certs=/path/to/cert/file.crt,admin:admin", "tags": [ @@ -448,7 +448,7 @@ "CosineRelevancyResultProcessor" ], "response_mappings": "", - "result_mappings": "url=_source.url,date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD", + "result_mappings": "url='https:///email/_doc/{_id}',date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD", "results_per_query": 10, "credentials": "verify_certs=[True|False],ca_certs=/path/to/cert/file.crt,username:password", "tags": [ From 4bf9e8b746e0ebc3bdf07455038bfe147902b683 Mon Sep 17 00:00:00 2001 From: dnicodemus-la Date: Tue, 12 Nov 2024 16:08:31 -0500 Subject: [PATCH 36/36] reloaded w/ change to ES/OS --- db.sqlite3.dist | Bin 303104 -> 303104 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/db.sqlite3.dist b/db.sqlite3.dist index bd59bf664d09b01d7dba2fc9aaef5ad12c32d443..7982bdf13c5dde392b18be1c914c915c04780518 100644 GIT binary patch delta 3959 zcmai%Z)ly>9mjKXPqk&nTJz`L{7+_VO=?fh|3B4C6PZa(vzpbWT3kz-+$J%}?Y*Yb z+EUcYDwC<;!+c=ihA?C=RHhY=sDsLwz7e#@HpT|(-i*D_F`0tkn7`+FV(xQ#AMj0H z{66`f-|zhXo##G$@!E$kUi-Tjnzyk&Mt2jsn-XlfFhmExZ|?W3RC30He>V>lQvYkL zWSmV8ZEB!X!Dz8HxTR%>&zZLBM}xq}WBzo@WqL*=yXOYrKZ*H2T82HnF^18*b^zZW z^O4pepGzsl!R^5BiFwevD;>4bR#P5aYW-nzI%;j0VG{$vrDA(9*4E=Iq*fxRwvG7; zLJ67peysSYt;b_ZQ6VP!V>Vdm@#0g2F?L5E#CHqC*A|>8^rpE6UqVjy0w0a}#X?^? z8m$TZOYon9O?%Wz8pbBN!DB~ou-NS}DV3oUT`_yE*zKJr;4~JzU$kBfoD!vGIv|FF zaSXD3r>BD3amYx!9|Pq~tBGRl=P&Jk43rToCkip^@9<+F)(R2q>%g85qM4w;A?X9MU=j%?&pVJnQ_uw1;n^u@ zhXsyYW9~rMt57kw78-?atlh|9s*L3sWlo5&?kR=FbKxJ)qOI4JR%X{uEY0m&DzBZG zEzgyHFkD)lU0NAFGJkw-_{@=|`LjF1Pc-^0cX#-K0dEh6|1xN>=1s#~LH=ZY+7rT= z8?k6A@wWYVrBq%k&E7w^_+1WP52tQI52XEQ1zS4PPedejpWcK%NwvGi@J>pS(Mt)T z+I=;Gmh-h@SPYMmkQptM5bd5EMPH;`SWJyHFgbNSmk?{+OShrPRJ$2$4L2rZa${mi zGVmJ(Z*%)^M|)Bm!(eR)<(Ul=5t%x`LB~vcT6j62T=n4^91#;Qx#kV$n;mr9>DmFX zL}60k)3NS=bz)hxK?dFAG%~eI15{MOP-n>^!gM5?bYJa9O|?M-S@fdu`sKJzWVBp| z+#y(yp0nu7gx41lR1uN!B?sNxccV{hr36R_c&g9qI$^~9^$>ccrr;JsA~-~!QT&J) z;a;iyHa0SCsm-(1zn!N0=n2Fo^AbK>_w+zF>N8nk{kVeGDQU}s=SJtw^u}~Tw zoO)G41aoH}Mfa!taLl65yG%ty;wQGv4WCC}*Or{%7(Q5{H>WPTgor4*qFa3&eePw^ zK*6#r0EClx@|zu0;Q7?01QteVUB9(OLW~Z}zd&EsvLG-skcW&rCBzc<)HA3*)w=={ z`2A=z6+ob?quwaE&qMT1>OveF*p~2{BqI$Gfys)$yuSP;vYt%A@3pLM;ur{wsyp;7 HdcN@+mNJ() delta 3928 zcmZvfYiu0V7012an~;c_#Q3r6$NJGagqU!-??-|Q8w6R$F=-m&Ktzj@IBrrCd;KB` zr~(@`sHjk>S7oG1wBie@RQiA*E}~QoLTbN3qpGc{Rs#Bk4^^p(S|p@a?U@V1;zV+tF=h*0jA9m9c@am9rDN5@Dh`&&Yp zmPbdow9rXcEqA#)I(KKewpLkr?;hX>6aL-Kz+YRV347pX;9p7jUpsFL{e=?dzTLo& zC48)FG~`4XapET64I+n6jGk+Hk`5lm-HQiYu!Q4S~KpKwr08E)8b6QV@+y3<9qv{Ay_^ zt42vllyv_r*=$55iQuMIfr!2CM7a_$ZYeXh{)D|yt^}tss;F@9mTjPc5~Jl*FK9TC zXplX(WW9tnlv20Q6KD`t8E$LkB+egu0u91S$wjS{u;Jc7gRq<%>yGr|Fb2_@yJve< zU2VC0lYQryx~+EqO@ec}9^dttCE z5Xs;UnR{+9C{@9v(8T>_FuW`VYRoG`Tkk|OEjwnewAUZp@%U<~(AKg#*0R0S#; zw$)G1PS3E1md>4GQjNewfWMLXg@t=&51pT#nz{Js@_79|HhsFb^vK8@ee}qe@0y;e z%`cxDzf_yDj~re)eeB-j%g4`{m4!2B?xu`P@X5pVaWR&tUs|cx=NFgnT)KFE@nO6? zKYgn264Z~ll~VX$AA3S!rZnett$w*)xLj|2tzPV_j;5IIV+I>5A>ErCH;9aekD@Mr z^5x=}T5rjCDPx2pd0sKmI-*mkBYMvl4@G}|v-kyn{MF*&V)g(56O_jFp4FR#jfv#R z0Ui!`HHgbJ5sc-z#YCiW{k1oWYlUnyq6n2X&n+g#G~{}adu4hfL7ZU9E#r{vZHqS(Q+bJg+a<)_d<TRtaI76Oj_wIR1+Y9V}e`(9-`65ySahC~J702ZsSgnE!MP%@!LO zW|$F6RCC$$loihAyVfRl81@N;~VbTkQ77 z?m^>?K>$CKMi|pPe^O$z2Y4uhqk%8iit5}7iiwe7@bwaUW)giplO>L|GQ>2GBAyUR z;yN7l6GxHBJUGWvS}Sr&EhQ@Hxe5R2G1S)RAUKi>OJ%NGOmxyg(U0#(-Tov@(7=@l z3VX5n{y7E$O%9WKuYdb~^ta$j4HqmgzYGK6#QHy-M6U!M3N{A3yl$Q*P&F~J)W7y1 zx*T}G3yNSlwajo#B=KMSDtdcMqxuClP?!g`QJfDG?(2$pysx56+>v%-sr14cG8h(3F9cNRqmppM~kzcqopo zRz&6WLP`wv5N#n-dJ-!R%Q7cWO44~6J^u{q@be4k=}a(*;qKy9*rYKf(!?6e{`O^b z7Bw_V%wSLOoU2NS6dNr={EkcL$sia^g(l7#Z=90oO5!9c29=+K5f5Wkl ACjbBd