From b1bdfad8543dcb09525dc337b22ef11194baf270 Mon Sep 17 00:00:00 2001 From: Christian Gendreau Date: Tue, 13 Aug 2024 16:35:08 -0400 Subject: [PATCH 1/5] Prepare next development version --- pom.xml | 2 +- search-cli/pom.xml | 4 ++-- search-messaging/pom.xml | 2 +- search-ws/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index c93c1a6..d1cbfac 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.github.aafc-bicoe search-api-parent - 0.35 + 0.36-SNAPSHOT pom diff --git a/search-cli/pom.xml b/search-cli/pom.xml index d00de70..05269d2 100644 --- a/search-cli/pom.xml +++ b/search-cli/pom.xml @@ -5,7 +5,7 @@ search-api-parent io.github.aafc-bicoe - 0.35 + 0.36-SNAPSHOT search-cli @@ -27,7 +27,7 @@ io.github.aafc-bicoe search-messaging - 0.35 + 0.36-SNAPSHOT diff --git a/search-messaging/pom.xml b/search-messaging/pom.xml index 484afc3..0cad192 100644 --- a/search-messaging/pom.xml +++ b/search-messaging/pom.xml @@ -5,7 +5,7 @@ search-api-parent io.github.aafc-bicoe - 0.35 + 0.36-SNAPSHOT search-messaging diff --git a/search-ws/pom.xml b/search-ws/pom.xml index 8946440..477448f 100644 --- a/search-ws/pom.xml +++ b/search-ws/pom.xml @@ -5,7 +5,7 @@ search-api-parent io.github.aafc-bicoe - 0.35 + 0.36-SNAPSHOT search-ws From e236516bed110a9008d456877c1fac3086f5bb96 Mon Sep 17 00:00:00 2001 From: Brandon Andre Date: Fri, 13 Sep 2024 14:56:09 -0400 Subject: [PATCH 2/5] Support #34906 - Fields missing for export (#165) - Updated index settings to include constants for relationships. - Updated endpoints to include parentMaterialSample - Updated mappings to include dwcOtherCatalogNumbers, barcode, materialSampleRemarks, and preparationRemarks. - Update ci.yml - Fixed test --- .github/workflows/ci.yml | 4 +- .../dina_material_sample_index_settings.json | 24 + search-cli/src/main/resources/endpoints.yml | 2 + .../aafc/dina/search/cli/TestConstants.java | 3 + .../cli/messaging/DocumentProcessorIT.java | 3 +- .../dina_material_sample_index_settings.json | 415 ++++++++++++++++++ search-ws/src/main/resources/mappings.yml | 8 + 7 files changed, 456 insertions(+), 3 deletions(-) create mode 100644 search-cli/src/test/resources/es-mapping/dina_material_sample_index_settings.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7968d93..e2f6b1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: mvn help:evaluate -Dexpression=project.version -q -DforceStdout > version.txt - name: Store artifacts if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: build-artifacts path: | @@ -49,7 +49,7 @@ jobs: steps: - name: Download artifacts from build job - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: build-artifacts diff --git a/local/elastic-configurator-settings/collection-index/dina_material_sample_index_settings.json b/local/elastic-configurator-settings/collection-index/dina_material_sample_index_settings.json index b1e916b..3f77768 100644 --- a/local/elastic-configurator-settings/collection-index/dina_material_sample_index_settings.json +++ b/local/elastic-configurator-settings/collection-index/dina_material_sample_index_settings.json @@ -325,6 +325,30 @@ } } } + }, + "preparedBy": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "person" + } + } + } + } + }, + "parentMaterialSample": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "material-sample" + } + } + } + } } } } diff --git a/search-cli/src/main/resources/endpoints.yml b/search-cli/src/main/resources/endpoints.yml index f48e7b0..8501b6b 100644 --- a/search-cli/src/main/resources/endpoints.yml +++ b/search-cli/src/main/resources/endpoints.yml @@ -28,7 +28,9 @@ endpoints: - projects - hierarchy - storageUnitUsage + - parentMaterialSample relationshipsType: + - person - collecting-event metadata: indexName: dina_object_store_index diff --git a/search-cli/src/test/java/ca/gc/aafc/dina/search/cli/TestConstants.java b/search-cli/src/test/java/ca/gc/aafc/dina/search/cli/TestConstants.java index 368105a..609a034 100644 --- a/search-cli/src/test/java/ca/gc/aafc/dina/search/cli/TestConstants.java +++ b/search-cli/src/test/java/ca/gc/aafc/dina/search/cli/TestConstants.java @@ -8,9 +8,12 @@ public interface TestConstants { String AGENT_INDEX_MAPPING_FILE = "es-mapping/dina_agent_index_settings.json"; String OBJECT_STORE_INDEX_MAPPING_FILE = "es-mapping/dina_object_store_index_settings.json"; + String MATERIAL_SAMPLE_INDEX_MAPPING_FILE = "es-mapping/dina_material_sample_index_settings.json"; String AGENT_INDEX = "dina_agent_index"; String OBJECT_STORE_INDEX = "dina_object_store_index"; + String MATERIAL_SAMPLE_INDEX = "dina_material_sample_index"; + String PERSON_TYPE = "person"; String ORGANIZATION_TYPE = "organization"; diff --git a/search-cli/src/test/java/ca/gc/aafc/dina/search/cli/messaging/DocumentProcessorIT.java b/search-cli/src/test/java/ca/gc/aafc/dina/search/cli/messaging/DocumentProcessorIT.java index a6c6783..8dc0c1e 100644 --- a/search-cli/src/test/java/ca/gc/aafc/dina/search/cli/messaging/DocumentProcessorIT.java +++ b/search-cli/src/test/java/ca/gc/aafc/dina/search/cli/messaging/DocumentProcessorIT.java @@ -86,9 +86,10 @@ public void reIndexRelatedDocuments() { TestConstants.PERSON_DOCUMENT_TYPE, TestConstants.PERSON_DOCUMENT_ID, List.of(Pair.of("include", "organizations")), TestConstants.PERSON_RESPONSE_PATH); - // Create the agent index + // Create the indices ca.gc.aafc.dina.testsupport.elasticsearch.ElasticSearchTestUtils.createIndex(elasticSearchClient, TestConstants.AGENT_INDEX, TestConstants.AGENT_INDEX_MAPPING_FILE); ca.gc.aafc.dina.testsupport.elasticsearch.ElasticSearchTestUtils.createIndex(elasticSearchClient, TestConstants.OBJECT_STORE_INDEX, TestConstants.OBJECT_STORE_INDEX_MAPPING_FILE); + ca.gc.aafc.dina.testsupport.elasticsearch.ElasticSearchTestUtils.createIndex(elasticSearchClient, TestConstants.MATERIAL_SAMPLE_INDEX, TestConstants.MATERIAL_SAMPLE_INDEX_MAPPING_FILE); // index a metadata to trigger dynamic mapping ca.gc.aafc.dina.testsupport.elasticsearch.ElasticSearchTestUtils.indexDocument(elasticSearchClient, TestConstants.OBJECT_STORE_INDEX, "2", diff --git a/search-cli/src/test/resources/es-mapping/dina_material_sample_index_settings.json b/search-cli/src/test/resources/es-mapping/dina_material_sample_index_settings.json new file mode 100644 index 0000000..e1c9b4d --- /dev/null +++ b/search-cli/src/test/resources/es-mapping/dina_material_sample_index_settings.json @@ -0,0 +1,415 @@ +{ + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + }, + "mappings": { + "_meta": { + "version": { + "number": "1.0" + } + }, + "date_detection": true, + "numeric_detection": true, + "dynamic_templates": [ + { + "includes_as_nested": { + "match_mapping_type": "object", + "match": "included", + "mapping": { + "type": "nested" + } + } + }, + { + "id_as_keywords": { + "match_mapping_type": "string", + "match": "id", + "mapping": { + "type": "keyword" + } + } + }, + { + "type_as_keywords": { + "match_mapping_type": "string", + "match": "type", + "mapping": { + "type": "keyword" + } + } + }, + { + "verbatim_as_text": { + "match": "*verbatim*", + "mapping": { + "type": "text" + } + } + } + ], + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "material-sample" + }, + "attributes": { + "properties": { + "group": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "createdBy": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "createdOn": { + "type": "date" + }, + "dwcCatalogNumber": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "dwcOtherCatalogNumbers": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "materialSampleName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "preparationDate": { + "type": "date" + }, + "managedAttributes": { + "type": "object" + }, + "preparationRemarks": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "dwcDegreeOfEstablishment": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "host": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "barcode": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "publiclyReleasable": { + "type": "boolean" + }, + "notPubliclyReleasableReason": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tags": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "materialSampleState": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "materialSampleRemarks": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "startEventDateTime": { + "type": "date" + }, + "endEventDateTime": { + "type": "date" + }, + "hierarchy": { + "type": "nested", + "properties": { + "uuid": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "rank": { + "type": "integer" + } + } + } + } + }, + "relationships": { + "properties": { + "attachment": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "metadata" + } + } + } + } + }, + "collectingEvent": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "collecting-event" + } + } + } + } + }, + "organism": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "organism" + } + } + } + } + }, + "storageUnitUsage": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "storage-unit-usage" + } + } + } + } + }, + "collection": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "collection" + } + } + } + } + }, + "preparationType": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "preparation-type" + } + } + } + } + }, + "preparationMethod": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "preparation-method" + } + } + } + } + }, + "assemblages": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "assemblage" + } + } + } + } + }, + "projects": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "project" + } + } + } + } + }, + "preparedBy": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "person" + } + } + } + } + }, + "parentMaterialSample": { + "properties": { + "data": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "material-sample" + } + } + } + } + } + } + } + } + }, + "included" : { + "type": "nested", + "include_in_parent" : true, + "properties" : { + "attributes" : { + "properties" : { + "dwcRecordNumber": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "dwcRecordedBy": { + "type": "text", + "fields": { + "autocomplete": { + "type": "search_as_you_type", + "max_shingle_size": 3 + }, + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "determination" : { + "properties" : { + "verbatimDeterminer": { + "type": "text", + "fields": { + "autocomplete": { + "type": "search_as_you_type", + "max_shingle_size": 3 + }, + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + } + } + }, + "meta": { + "properties": { + "moduleVersion": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + } +} diff --git a/search-ws/src/main/resources/mappings.yml b/search-ws/src/main/resources/mappings.yml index 65d2d1a..47abf45 100644 --- a/search-ws/src/main/resources/mappings.yml +++ b/search-ws/src/main/resources/mappings.yml @@ -54,6 +54,14 @@ mappings: - name: preparationSubstrate type: text distinctTermAgg: true + - name: dwcOtherCatalogNumbers + type: text + - name: barcode + type: text + - name: materialSampleRemarks + type: text + - name: preparationRemarks + type: text collecting-event: - name: createdBy From 685adc0fcf5a5c3db0a126a81b467cd34c833398 Mon Sep 17 00:00:00 2001 From: Brandon Andre Date: Wed, 2 Oct 2024 10:08:26 -0400 Subject: [PATCH 3/5] updated mapping (#167) --- search-ws/src/main/resources/mappings.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/search-ws/src/main/resources/mappings.yml b/search-ws/src/main/resources/mappings.yml index 47abf45..a983afb 100644 --- a/search-ws/src/main/resources/mappings.yml +++ b/search-ws/src/main/resources/mappings.yml @@ -28,6 +28,8 @@ mappings: type: object - name: hostOrganism.name type: text + - name: effectiveScientificName + type: text - name: targetOrganismPrimaryScientificName type: text - name: extensionValues From a3c5f508bff30fb248337c838f450bc7cec05739 Mon Sep 17 00:00:00 2001 From: Christian Gendreau Date: Wed, 2 Oct 2024 15:31:15 -0400 Subject: [PATCH 4/5] Prepare 0.36 release --- pom.xml | 2 +- search-cli/pom.xml | 4 ++-- search-messaging/pom.xml | 2 +- search-ws/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index d1cbfac..42bd8b2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.github.aafc-bicoe search-api-parent - 0.36-SNAPSHOT + 0.36 pom diff --git a/search-cli/pom.xml b/search-cli/pom.xml index 05269d2..6ae0abc 100644 --- a/search-cli/pom.xml +++ b/search-cli/pom.xml @@ -5,7 +5,7 @@ search-api-parent io.github.aafc-bicoe - 0.36-SNAPSHOT + 0.36 search-cli @@ -27,7 +27,7 @@ io.github.aafc-bicoe search-messaging - 0.36-SNAPSHOT + 0.36 diff --git a/search-messaging/pom.xml b/search-messaging/pom.xml index 0cad192..ddb951f 100644 --- a/search-messaging/pom.xml +++ b/search-messaging/pom.xml @@ -5,7 +5,7 @@ search-api-parent io.github.aafc-bicoe - 0.36-SNAPSHOT + 0.36 search-messaging diff --git a/search-ws/pom.xml b/search-ws/pom.xml index 477448f..68c702f 100644 --- a/search-ws/pom.xml +++ b/search-ws/pom.xml @@ -5,7 +5,7 @@ search-api-parent io.github.aafc-bicoe - 0.36-SNAPSHOT + 0.36 search-ws From 2551707371dfaa4402603135ae6ac3937603cc34 Mon Sep 17 00:00:00 2001 From: Christian Gendreau Date: Wed, 2 Oct 2024 15:34:41 -0400 Subject: [PATCH 5/5] Updated OWASP config --- owasp-suppression.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owasp-suppression.xml b/owasp-suppression.xml index da29c01..a26e7c1 100644 --- a/owasp-suppression.xml +++ b/owasp-suppression.xml @@ -10,7 +10,7 @@ CVE about Keycloak Server - CVE-2021-3827 + CVE-2024-7341 no untrusted source